open a branch to add a CNN layer on top of BiLSTM

config.py

@@ -15,7 +15,7 @@ class DefaultConfig():
     test_filename = "data/test_data"
     train_filename = "data/train_data"
     #word2vec_filename = "data/wikipedia-200-mincount-20-window-8-cbow.bin"
-    word2vec_filename = "data/wikipedia-100-mincount-20-window-5-cbow.bin"
+    word2vec_filename = "data/wikipedia-100-mincount-30-window-8-cbow.bin"
     tags_filename = "data/tags.txt"
     words_filename = "data/words.txt"
     chars_filename = "data/chars.txt"
@@ -24,15 +24,17 @@ class DefaultConfig():
     NONE = "O"
 
     # Hyper parameters
-    CHAR_EMB_DIM = 75
-    HIDDEN_SIZE = 300
+    CHAR_EMB_DIM = 120
+    HIDDEN_SIZE = 400
     FILTER_SIZE = [2, 3, 4]
-    N_FILTERS = 128
-    BATCH_SIZE = 64
-    MAX_LENGTH_WORD = 40
+    N_FILTERS = 64
+    BATCH_SIZE = 40
+    MAX_LENGTH_WORD = 50
     N_EPOCHS = 100
     LR = 0.001
     LR_DECAY = 0.95
     DROPOUT = 0.5
 
+    CRF = True
+
 

conll_reader.py

@@ -53,8 +53,8 @@ def get_vocabs(datasets):
         a set of all the words in the dataset
     """
     print("Building vocab...")
-    vocab_words = dict()
-    vocab_tags = dict()
+    vocab_words = set()
+    vocab_tags = set()
     for dataset in datasets:
         for words, tags in dataset:
             vocab_words.update(words)
@@ -70,7 +70,7 @@ def get_char_vocab(dataset):
     Returns:
         a set of all the characters in the dataset
     """
-    vocab_char = dict()
+    vocab_char = set()
     for words, _ in dataset:
         for word in words:
             vocab_char.update(word)

model.py

@@ -167,11 +167,16 @@ def add_loss_op(self):
         """
         Adds loss to self
         """
-        self.labels_pred = tf.cast(tf.argmax(self.logits, axis=-1), tf.int32)
-        losses = tf.nn.sparse_softmax_cross_entropy_with_logits(logits=self.logits, labels=self.labels)
-        mask = tf.sequence_mask(self.sentences_lengths)
-        losses = tf.boolean_mask(losses, mask)
-        self.loss = tf.reduce_mean(losses)
+        if self.cfg.CRF:
+            log_likelihood, self.transition_params = tf.contrib.crf.crf_log_likelihood(
+            self.logits, self.labels, self.sentences_lengths)
+            self.loss = tf.reduce_mean(-log_likelihood)
+        else:
+            self.labels_pred = tf.cast(tf.argmax(self.logits, axis=-1), tf.int32)
+            losses = tf.nn.sparse_softmax_cross_entropy_with_logits(logits=self.logits, labels=self.labels)
+            mask = tf.sequence_mask(self.sentences_lengths)
+            losses = tf.boolean_mask(losses, mask)
+            self.loss = tf.reduce_mean(losses)
 
 
     def add_train_op(self):
@@ -194,10 +199,23 @@ def predict_batch(self, sess, words, labels):
         """
         # get the feed dictionnary
         fd, sequence_lengths = self.get_feed_dict(words, labels, dropout=1.0)
-
-        labels_pred, loss = sess.run([self.labels_pred, self.loss], feed_dict=fd)
-
-        return labels_pred, sequence_lengths, loss
+        if self.cfg.CRF:
+            viterbi_sequences = []
+            logits, transition_params, loss = sess.run([self.logits, self.transition_params, self.loss],
+                    feed_dict=fd)
+            # iterate over the sentences
+            for logit, sequence_length in zip(logits, sequence_lengths):
+                # keep only the valid time steps
+                logit = logit[:sequence_length]
+                viterbi_sequence, viterbi_score = tf.contrib.crf.viterbi_decode(
+                                logit, transition_params)
+                viterbi_sequences += [viterbi_sequence]
+
+            return viterbi_sequences, sequence_lengths, loss
+
+        else:
+            labels_pred, loss = sess.run([self.labels_pred, self.loss], feed_dict=fd)
+            return labels_pred, sequence_lengths, loss
 
 
     def run_evaluate(self, sess, test, tags):

train.py

@@ -8,10 +8,10 @@
 from data_helper import batch_gen
 
 
-def train_model(cfg, train_set, dev_set, embeddings, tags, chars):
+def train_model(cfg, train_set, dev_set, embed, tags, chars):
 
     # Build Model
-    model = Model(cfg, embeddings, len(tags), len(chars))
+    model = Model(cfg, embed, len(tags), len(chars))
 
     with tf.Session() as sess:
         sess.run(tf.global_variables_initializer())