pos_with_features.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Features\n",
    "\n",
    "* So far we always worked purely with words and their embeddings\n",
    "* Other features might be useful\n",
    "* POS tagger as a running example: much can be guessed from the word's suffix: \"this is an *ooobviously* good idea\"\n",
    "  * no embedding for *ooobviously*\n",
    "  * but the suffix -ly and the context should tell us it's an adverb\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "## Read in the data\n",
    "import json\n",
    "import random\n",
    "import numpy\n",
    "\n",
    "def read_labeled_data(json_file):\n",
    "    with open(json_file) as f:\n",
    "        data=json.load(f)\n",
    "        texts=[one_example[\"text\"] for one_example in data]  #list of texts\n",
    "        labels=[one_example[\"tags\"] for one_example in data] # list of lists of output labels\n",
    "    return texts,labels\n",
    "\n",
    "texts_train,labels_train=read_labeled_data(\"data/pos_train_fi.json\")\n",
    "texts_devel,labels_devel=read_labeled_data(\"data/pos_devel_fi.json\")       "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "## Read in pre-trained embeddings\n",
    "from gensim.models import KeyedVectors\n",
    "\n",
    "# English model: wiki-news-300d-1M.vec\n",
    "# Finnish model: pb34_wf_200_v2_skgram.bin\n",
    "# these models are under /home/bio in the classroom machines\n",
    "#                        /home/ginter on the virtual server\n",
    "#                         ...don't make a copy of that file on the virtual server, just use it from that path\n",
    "#                         ...if you run things locally on your laptop, you can scp this model from the virtual machine\n",
    "vector_model=KeyedVectors.load_word2vec_format(\"data/pb34_wf_200_v2_skgram.bin\", binary=True, limit=100000)\n",
    "word_embeddings=vector_model.vectors # these are the vectors themselves\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "word_embeddings shape= (100000, 200)\n",
      "embeddings= [[ 2.0013428e-03  2.2097016e-03 -1.9151306e-03 ...  9.9411009e-05\n",
      "  -8.4304810e-04 -5.6327821e-04]\n",
      " [-1.0991982e-01 -1.7190212e-01  1.4615083e-01 ...  5.6789882e-02\n",
      "   5.0900381e-02 -8.7465588e-03]\n",
      " [-9.0047391e-03 -1.0183236e-01  1.5222897e-01 ...  5.6943305e-02\n",
      "   5.0679442e-02 -2.7512657e-03]\n",
      " ...\n",
      " [-1.7641033e-01 -3.0918688e-01  4.2229193e-01 ...  3.8911417e-01\n",
      "   1.8602428e-01 -2.6177013e-01]\n",
      " [ 2.3696978e-01 -3.1057227e-02 -9.4661742e-02 ...  2.1558458e-02\n",
      "   3.4130868e-01  2.6005360e-01]\n",
      " [ 2.0296814e-01  2.0556472e-01  6.9490981e-01 ...  2.3367092e-01\n",
      "  -1.2235161e-02  2.4763262e-01]]\n"
     ]
    }
   ],
   "source": [
    "# Just checking all is fine\n",
    "print(\"word_embeddings shape=\",word_embeddings.shape)\n",
    "print(\"embeddings=\",word_embeddings)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/ginter/venv-jupyter/lib/python3.5/site-packages/h5py/__init__.py:36: FutureWarning: Conversion of the second argument of issubdtype from `float` to `np.floating` is deprecated. In future, it will be treated as `np.float64 == np.dtype(float).type`.\n",
      "  from ._conv import register_converters as _register_converters\n",
      "Using TensorFlow backend.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "New embeddings shape= (100002, 200)\n",
      "[[-0.0140559   0.03390314 -0.08586288 ... -0.00762202 -0.01545846\n",
      "  -0.1188098 ]\n",
      " [ 0.05820623  0.01309959  0.02957606 ...  0.03504125 -0.0797373\n",
      "   0.08092583]\n",
      " [ 0.09913075  0.1094512  -0.09486048 ...  0.00492404 -0.04175796\n",
      "  -0.02790036]\n",
      " ...\n",
      " [-0.04595862 -0.08054972  0.11001598 ...  0.10137248  0.04846326\n",
      "  -0.06819666]\n",
      " [ 0.06736331 -0.00882863 -0.02690946 ...  0.00612841  0.09702369\n",
      "   0.07392534]\n",
      " [ 0.0625686   0.06336904  0.21421851 ...  0.07203328 -0.00377171\n",
      "   0.07633723]]\n"
     ]
    }
   ],
   "source": [
    "import keras.utils\n",
    "# The embeddings have one row for every word, and they are indexed from 0 upwards\n",
    "# For our tagger, we need words with index 0 and 1 to have a special meaning\n",
    "#       0 is the mask\n",
    "#       1 is OOV (out of vocabulary)\n",
    "# We need to make space for the two words:\n",
    "# 1) Add two rows into the word_embeddings matrix\n",
    "# 2) Renumber indices in the gensim model by 2, so that what was word 0 is now word 2, word 1 becomes word 3, etc...\n",
    "\n",
    "# ad 1:\n",
    "# Two rows with the right number of columns, and filled with random numbers\n",
    "two_random_rows=numpy.random.uniform(low=-0.01, high=0.01, size=(2,word_embeddings.shape[1]))\n",
    "# stack the two rows, and the embedding matrix on top of each other\n",
    "word_embeddings=numpy.vstack([two_random_rows,word_embeddings])\n",
    "\n",
    "# Normalize to unit length, works better this way\n",
    "word_embeddings=keras.utils.normalize(word_embeddings)\n",
    "\n",
    "# Alternative normalization code\n",
    "#norm=numpy.linalg.norm(word_embeddings,axis=1,keepdims=True) #magnitude of every row\n",
    "#word_embeddings/=norm #divide every row by magnitude, results in unit length vectors\n",
    "\n",
    "# Ad 2:\n",
    "# Now renumber all word indices, shifting them up by two\n",
    "for word_record in vector_model.vocab.values():\n",
    "    word_record.index+=2\n",
    "\n",
    "print(\"New embeddings shape=\",word_embeddings.shape)\n",
    "print(word_embeddings)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Adding features\n",
    "\n",
    "* So far, our examples were 2D matrices *sentence x word*\n",
    "  * `M[5,7]` was the vocabulary index of the 8th word in the 6th sentence (counting from zero)\n",
    "* We will add a dimension, so the matrices will look like sentence x word x feature like so:\n",
    "  * `M[5,7,0]` is the vocabulary index of the 8th word in the 6th sentence as before\n",
    "  * `M[5,7,1]` is the vocabulary index of the 1st feature of the 8th word in the 6th sentence\n",
    "  * ..."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "First 10 features [('^850', 7957), ('^Ža', 10432), ('meä$', 3397), ('317$', 8133), ('^5.2', 6755), ('^isi', 5081), ('^kit', 2586), ('^14-', 7365), ('hmo$', 3680), ('^pom', 4342)]\n",
      "Some text: [[1, 53, 15, 54, 1053, 160, 1054], [33026, 2, 21, 375, 274, 674, 686], [15478, 14, 66, 129, 80, 344, 82], [2199, 2, 49, 556, 50, 1055, 52], [4, 83, 84, 85, 85, 1, 1], [145, 26, 21, 119, 77, 121, 79], [80, 53, 36, 67, 158, 69, 711], [59588, 26, 72, 221, 227, 478, 1056], [1, 2, 278, 1057, 445, 1058, 1059], [3, 58, 59, 60, 60, 1, 1]]\n"
     ]
    }
   ],
   "source": [
    "def vectorize(texts,word_vocab,feature_vocab):\n",
    "    vectorized_texts=[] # List of sentences, each sentence is a list of words, and each word is a list of features\n",
    "    for one_text in texts:\n",
    "        vectorized_text=[] # One sentence, ie list of words, each being a list of features\n",
    "        for one_word in one_text:\n",
    "            # feature vector of this one word\n",
    "            # [ word_itself, last_character, last_two_characters, last_three_characters, \n",
    "            #                first character, first_two_characters, first_three_characters, ...]\n",
    "            one_word_feature_vector=[]\n",
    "            if one_word in word_vocab:\n",
    "                one_word_feature_vector.append(word_vocab[one_word].index) # the .index comes from gensim's vocab\n",
    "            else:\n",
    "                one_word_feature_vector.append(1) # OOV\n",
    "            #as a future-proof idea, let us mark the word with a beginning and end marker\n",
    "            marked=\"^\"+one_word+\"$\"\n",
    "            for affix_length in range(2,5): #2,3,4\n",
    "                suffix=marked[-affix_length:]  # g$  og$  dog$\n",
    "                prefix=marked[:affix_length]   # ^d  ^do  ^dog\n",
    "                if len(suffix)==affix_length: #if len(suffix) is less than the desired length, the word is too short\n",
    "                    one_word_feature_vector.append(feature_vocab.setdefault(suffix,len(feature_vocab)))\n",
    "                else:\n",
    "                    one_word_feature_vector.append(1) #No such suffix\n",
    "                if len(prefix)==affix_length: #if len(prefix) is less than the desired length, the word is too short\n",
    "                    one_word_feature_vector.append(feature_vocab.setdefault(prefix,len(feature_vocab)))\n",
    "                else:\n",
    "                    one_word_feature_vector.append(1) #No such prefix\n",
    "            \n",
    "            #Done with the word\n",
    "            vectorized_text.append(one_word_feature_vector)\n",
    "        #Done with the text\n",
    "        vectorized_texts.append(vectorized_text)\n",
    "    return numpy.array(vectorized_texts)\n",
    "\n",
    "feature_vocab={\"<SPECIAL>\":0,\"<NOSUCHSUFFIX>\":1} #these are just to reserve the indices 0 and 1\n",
    "vectorized_train=vectorize(texts_train,vector_model.vocab,feature_vocab)\n",
    "print(\"First 10 features\",list(feature_vocab.items())[:10]) #first 10 features\n",
    "print(\"Some text:\",vectorized_train[100])\n",
    "\n",
    "vectorized_devel=vectorize(texts_devel,vector_model.vocab,feature_vocab)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "### Only needed for me, not to block the whole GPU, you don't need this stuff\n",
    "from keras.backend.tensorflow_backend import set_session\n",
    "config = tf.ConfigProto()\n",
    "config.gpu_options.per_process_gpu_memory_fraction = 0.4\n",
    "set_session(tf.Session(config=config))\n",
    "### ---end of weird stuff\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Padded train shape (texts x words x features): (12217, 238, 7)\n",
      "Padded devel shape (texts x words x features): (1364, 238, 7)\n"
     ]
    }
   ],
   "source": [
    "from keras.preprocessing.sequence import pad_sequences\n",
    "\n",
    "# Now we pad the sequences to get everything into the right sizes\n",
    "padded_train_data=pad_sequences(vectorized_train,padding=\"post\")\n",
    "print(\"Padded train shape (texts x words x features):\",padded_train_data.shape)\n",
    "_,longest_train_sent,_=padded_train_data.shape\n",
    "padded_devel_data=pad_sequences(vectorized_devel,maxlen=longest_train_sent,padding=\"post\")\n",
    "print(\"Padded devel shape (texts x words x features):\",padded_devel_data.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "padded_train_labels shape= (12217, 238)\n",
      "padded_devel_labels shape= (1364, 238)\n"
     ]
    }
   ],
   "source": [
    "# Now the training data input part is done ... labels needed yet\n",
    "# Easiest way is to make our own vectorizer\n",
    "def vectorize_labels(labels,label_dictionary):\n",
    "    vectorized=[]\n",
    "    for one_text_labels in labels: #List like [\"NOUN\",\"VERB\",\"VERB\",\"PUNCT\"]\n",
    "        one_text_vectorized=[] #numerical indices of the labels\n",
    "        for one_label in one_text_labels:\n",
    "            one_text_vectorized.append(label_dictionary.setdefault(one_label,len(label_dictionary)))\n",
    "        vectorized.append(one_text_vectorized) #done with the sentence\n",
    "    return numpy.array(vectorized)\n",
    "\n",
    "label_dictionary={}\n",
    "vectorized_train_labels=vectorize_labels(labels_train,label_dictionary)\n",
    "padded_train_labels=pad_sequences(vectorized_train_labels,padding=\"post\")\n",
    "print(\"padded_train_labels shape=\",padded_train_labels.shape)\n",
    "vectorized_devel_labels=vectorize_labels(labels_devel,label_dictionary)\n",
    "padded_devel_labels=pad_sequences(vectorized_devel_labels,padding=\"post\",maxlen=longest_train_sent)\n",
    "print(\"padded_devel_labels shape=\",padded_devel_labels.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "  0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "  0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "  0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "  0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "  0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "  0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0]\n",
      " [1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "  0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "  0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "  0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "  0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "  0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "  0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0]\n",
      " [1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "  0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "  0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "  0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "  0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "  0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "  0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0]]\n"
     ]
    }
   ],
   "source": [
    "# Almost there ... we yet need the mask, telling which parts of each padded sequence are real words\n",
    "# and which are only the padding which should be ignored in the output\n",
    "\n",
    "#                           where(condition,value_if_true,value_if_false)\n",
    "# padded_train_data[:,:,0]  -> returns the first feature of every word, i.e. the index of this word in the vocabulary\n",
    "# here zero means padding\n",
    "sentence_mask_train = numpy.where(padded_train_data[:,:,0]>0,1,0)\n",
    "print(sentence_mask_train[:3])\n",
    "\n",
    "sentence_mask_devel = numpy.where(padded_devel_data[:,:,0]>0,1,0) \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "padded_train_data.shape (12217, 238, 7)\n",
      "padded_train_labels.shape (12217, 238)\n",
      "padded_devel_data.shape (1364, 238, 7)\n",
      "padded_devel_labels.shape (1364, 238)\n"
     ]
    }
   ],
   "source": [
    "# phew, finally everything in place:\n",
    "\n",
    "print(\"padded_train_data.shape\",padded_train_data.shape)\n",
    "print(\"padded_train_labels.shape\",padded_train_labels.shape)\n",
    "print(\"padded_devel_data.shape\",padded_devel_data.shape)\n",
    "print(\"padded_devel_labels.shape\",padded_devel_labels.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "from keras.models import Model\n",
    "from keras.layers import Input, Dense, Embedding, Activation, Conv1D, TimeDistributed\n",
    "from keras.layers import Bidirectional, Concatenate,Flatten,Reshape\n",
    "from keras.optimizers import SGD, Adam\n",
    "from keras.initializers import Constant\n",
    "from keras.layers import CuDNNLSTM as LSTM  #massive speedup on graphics cards\n",
    "#from keras.layers import LSTM\n",
    "from keras.callbacks import EarlyStopping\n",
    "\n",
    "\n",
    "\n",
    "example_count, sequence_len, feature_count = padded_train_data.shape\n",
    "_,word_embedding_dim=word_embeddings.shape\n",
    "feature_embedding_dim=100 #we need to decide on an embedding for the features\n",
    "\n",
    "word_input=Input(shape=(sequence_len,))\n",
    "feature_input=Input(shape=(sequence_len,feature_count-1)) #first feature is word, so feature_count-1\n",
    "word_embeddings_layer=Embedding(len(vector_model.vocab)+2,\\\n",
    "                     word_embedding_dim, mask_zero=False,\\\n",
    "                     trainable=False, weights=[word_embeddings])(word_input)\n",
    "feature_embeddings_layer=Embedding(len(feature_vocab),feature_embedding_dim,embeddings_initializer=Constant(value=0.1))(feature_input)\n",
    "feature_embeddings_layer_concat=Reshape((sequence_len,(feature_count-1)*feature_embedding_dim))(feature_embeddings_layer)\n",
    "word_and_f_emb_layer=Concatenate()([word_embeddings_layer,feature_embeddings_layer_concat])\n",
    "hidden_layer=TimeDistributed(Dense(100,activation=\"tanh\"))(word_and_f_emb_layer)  #Simple\n",
    "outp_layer=TimeDistributed(Dense(len(label_dictionary),activation=\"softmax\"))(hidden_layer)\n",
    "\n",
    "\n",
    "model=Model(inputs=[word_input,feature_input], outputs=[outp_layer])\n",
    "model.compile(optimizer=\"adam\",loss=\"sparse_categorical_crossentropy\", sample_weight_mode='temporal',weighted_metrics=[\"acc\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "input_8 (InputLayer)            (None, 238, 6)       0                                            \n",
      "__________________________________________________________________________________________________\n",
      "input_7 (InputLayer)            (None, 238)          0                                            \n",
      "__________________________________________________________________________________________________\n",
      "embedding_8 (Embedding)         (None, 238, 6, 100)  1137500     input_8[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "embedding_7 (Embedding)         (None, 238, 200)     20000400    input_7[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "reshape_4 (Reshape)             (None, 238, 600)     0           embedding_8[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_4 (Concatenate)     (None, 238, 800)     0           embedding_7[0][0]                \n",
      "                                                                 reshape_4[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "time_distributed_5 (TimeDistrib (None, 238, 100)     80100       concatenate_4[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "time_distributed_6 (TimeDistrib (None, 238, 15)      1515        time_distributed_5[0][0]         \n",
      "==================================================================================================\n",
      "Total params: 21,219,515\n",
      "Trainable params: 1,219,115\n",
      "Non-trainable params: 20,000,400\n",
      "__________________________________________________________________________________________________\n",
      "None\n",
      "word input shape (12217, 238)\n",
      "feature input shape (12217, 238, 6)\n",
      "output shape (12217, 238, 1)\n",
      "Train on 12217 samples, validate on 1364 samples\n",
      "Epoch 1/20\n",
      "12217/12217 [==============================] - 4s 291us/step - loss: 1.5818 - weighted_acc: 0.5346 - val_loss: 0.8572 - val_weighted_acc: 0.7756\n",
      "Epoch 2/20\n",
      "12217/12217 [==============================] - 3s 267us/step - loss: 0.5796 - weighted_acc: 0.8319 - val_loss: 0.4417 - val_weighted_acc: 0.8602\n",
      "Epoch 3/20\n",
      "12217/12217 [==============================] - 3s 268us/step - loss: 0.3771 - weighted_acc: 0.8804 - val_loss: 0.3584 - val_weighted_acc: 0.8840\n",
      "Epoch 4/20\n",
      "12217/12217 [==============================] - 3s 270us/step - loss: 0.3105 - weighted_acc: 0.9013 - val_loss: 0.3220 - val_weighted_acc: 0.8938\n",
      "Epoch 5/20\n",
      "12217/12217 [==============================] - 3s 263us/step - loss: 0.2739 - weighted_acc: 0.9114 - val_loss: 0.2981 - val_weighted_acc: 0.9019\n",
      "Epoch 6/20\n",
      "12217/12217 [==============================] - 3s 270us/step - loss: 0.2499 - weighted_acc: 0.9186 - val_loss: 0.2844 - val_weighted_acc: 0.9053\n",
      "Epoch 7/20\n",
      "12217/12217 [==============================] - 3s 264us/step - loss: 0.2331 - weighted_acc: 0.9234 - val_loss: 0.2749 - val_weighted_acc: 0.9085\n",
      "Epoch 8/20\n",
      "12217/12217 [==============================] - 3s 267us/step - loss: 0.2204 - weighted_acc: 0.9275 - val_loss: 0.2711 - val_weighted_acc: 0.9093\n",
      "Epoch 9/20\n",
      "12217/12217 [==============================] - 3s 262us/step - loss: 0.2112 - weighted_acc: 0.9297 - val_loss: 0.2691 - val_weighted_acc: 0.9118\n",
      "Epoch 10/20\n",
      "12217/12217 [==============================] - 3s 269us/step - loss: 0.2042 - weighted_acc: 0.9320 - val_loss: 0.2649 - val_weighted_acc: 0.9106\n",
      "Epoch 11/20\n",
      "12217/12217 [==============================] - 3s 265us/step - loss: 0.1982 - weighted_acc: 0.9333 - val_loss: 0.2597 - val_weighted_acc: 0.9155\n",
      "Epoch 12/20\n",
      "12217/12217 [==============================] - 3s 271us/step - loss: 0.1939 - weighted_acc: 0.9342 - val_loss: 0.2598 - val_weighted_acc: 0.9149\n",
      "Epoch 13/20\n",
      "12217/12217 [==============================] - 3s 263us/step - loss: 0.1899 - weighted_acc: 0.9352 - val_loss: 0.2590 - val_weighted_acc: 0.9141\n",
      "Epoch 00013: early stopping\n"
     ]
    }
   ],
   "source": [
    "print(model.summary())\n",
    "word_input_data_train=padded_train_data[:,:,0]\n",
    "feature_input_data_train=padded_train_data[:,:,1:]\n",
    "labels_output_train=numpy.expand_dims(padded_train_labels,-1)\n",
    "\n",
    "word_input_data_devel=padded_devel_data[:,:,0]\n",
    "feature_input_data_devel=padded_devel_data[:,:,1:]\n",
    "labels_output_devel=numpy.expand_dims(padded_devel_labels,-1)\n",
    "\n",
    "print(\"word input shape\",word_input_data_train.shape)\n",
    "print(\"feature input shape\",feature_input_data_train.shape)\n",
    "print(\"output shape\",labels_output_train.shape)\n",
    "# train\n",
    "# stop early\n",
    "es_callback=EarlyStopping(monitor='val_weighted_acc', min_delta=0, patience=2, verbose=1, mode='auto')\n",
    "hist=model.fit([word_input_data_train,feature_input_data_train],[labels_output_train],\\\n",
    "               validation_data=([word_input_data_devel,feature_input_data_devel],[labels_output_devel],sentence_mask_devel),\\\n",
    "               batch_size=200,sample_weight=sentence_mask_train,verbose=1,epochs=20,callbacks=[es_callback])"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "venv-jupyter",
   "language": "python",
   "name": "venv-jupyter"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}