Pytorch tutorial LSTM - neural-network

Question

I was trying to implement the exercise about Sequence Models and Long-Short Term Memory Networks with Pytorch. The idea is to add an LSTM part-of-speech tagger character-level features but I can't seem to work it out. They gave as a hint that there should be two LSTMs involved, one that will output a character level representation and another one that will be in charge of predicting the Part-of-speech tag. I just can't figure out how to loop over the words level (in a sentence) and over the character (in each word of a sentence) and implement it in the forward function. Does anyone know how to do it ? Or encounter a similar situation ?
Here is my code:
class LSTMTaggerAug(nn.Module):
def __init__(self, embedding_dim_words, embedding_dim_chars, hidden_dim_words, hidden_dim_chars, vocab_size, tagset_size, charset_size):
super(LSTMTaggerAug, self).__init__()
self.hidden_dim_words = hidden_dim_words
self.hidden_dim_chars = hidden_dim_chars
self.word_embeddings = nn.Embedding(vocab_size, embedding_dim_words)
self.char_embeddings = nn.Embedding(charset_size, embedding_dim_chars)
self.lstm_char = nn.LSTM(embedding_dim_chars, hidden_dim_chars)
self.lstm_words = nn.LSTM(embedding_dim_words + hidden_dim_chars, hidden_dim_words)
self.hidden2tag = nn.Linear(hidden_dim_words, tagset_size)
self.hidden_char = self.init_hidden(c=False)
self.hidden_words = self.init_hidden(c=True)
def init_hidden(self, c=True):
if c:
return (autograd.Variable(torch.zeros(1, 1, self.hidden_dim_words)),
autograd.Variable(torch.zeros(1, 1, self.hidden_dim_words)))
else:
return (autograd.Variable(torch.zeros(1, 1, self.hidden_dim_chars)),
autograd.Variable(torch.zeros(1, 1, self.hidden_dim_chars)))
def forward(self, sentence, words):
# embeds = self.word_embeddings(sentence)
for ix, word in enumerate(sentence):
chars = words[ix]
char_embeds = self.char_embeddings(chars)
lstm_char_out, self.hidden_char = self.lstm_char(
char_embeds.view(len(chars), 1, -1), self.hidden_char)
char_rep = lstm_char_out[-1]
embeds = self.word_embeddings(word)
embeds_cat = torch.cat((embeds, char_rep), dim=1)
lstm_out, self.hidden_words = self.lstm_words(embeds_cat, self.hidden_words)
tag_space = self.hidden2tag(lstm_out.view(1, -1))
tag_score = F.log_softmax(tag_space, dim=1)
if ix==0:
tag_scores = tag_score
else:
tag_scores = torch.cat((tag_scores, tag_score), 0)
return tag_scores

Answer 1

The most naive way to do it according to your description would be to take a sentence s stripped of punctuation. Then split it into words:
words = s.split()
Take your first character level lstm, LSTMc and apply it to every word individually to encode the words (use the last output-state of the lstm to encode the word):
encoded_words = []
for word in words:
state = state_0
for char in word:
h, state = LSTMc(one_hot_encoding(char), state)
encoded_words.append(h)
After you've encoded the words, you pass word-level the part of speech tagger lstm LSTMw on the encoded words:
state = statew_0
parts_of_speech = []
for enc_word in encoded_words:
pos, state = LSTMw(enc_word, state)
parts_of_speech.append(pos)