cnn.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "from sklearn import model_selection, preprocessing, linear_model, naive_bayes, metrics, svm\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer, CountVectorizer\n",
    "from sklearn import decomposition, ensemble\n",
    "import pandas, xgboost, numpy, textblob, string\n",
    "from keras.preprocessing import text, sequence\n",
    "from keras import layers, models, optimizers\n",
    "from keras.utils.np_utils import to_categorical\n",
    "import _pickle as cPickle \n",
    "import pandas as pd\n",
    "\n",
    "from keras.models import load_model\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "body = pd.read_csv(\"data/train_bodies.csv\")\n",
    "stances = pd.read_csv(\"data/train_stances.csv\")\n",
    "data = pd.merge(body, stances, how='right', on='Body ID')\n",
    "targets = ['agree', 'disagree', 'discuss', 'unrelated']\n",
    "targets_dict = dict(zip(targets, range(len(targets))))\n",
    "# data['target'] = data['Stance']\n",
    "data['target'] = data['Stance'].replace(targets,targets_dict.values())\n",
    "data_y = data['target'].values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Body ID</th>\n",
       "      <th>articleBody</th>\n",
       "      <th>Headline</th>\n",
       "      <th>Stance</th>\n",
       "      <th>target</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>A small meteorite crashed into a wooded area i...</td>\n",
       "      <td>Soldier shot, Parliament locked down after gun...</td>\n",
       "      <td>unrelated</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>A small meteorite crashed into a wooded area i...</td>\n",
       "      <td>Tourist dubbed ‘Spider Man’ after spider burro...</td>\n",
       "      <td>unrelated</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "      <td>A small meteorite crashed into a wooded area i...</td>\n",
       "      <td>Luke Somers 'killed in failed rescue attempt i...</td>\n",
       "      <td>unrelated</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0</td>\n",
       "      <td>A small meteorite crashed into a wooded area i...</td>\n",
       "      <td>BREAKING: Soldier shot at War Memorial in Ottawa</td>\n",
       "      <td>unrelated</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>A small meteorite crashed into a wooded area i...</td>\n",
       "      <td>Giant 8ft 9in catfish weighing 19 stone caught...</td>\n",
       "      <td>unrelated</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Body ID                                        articleBody  \\\n",
       "0        0  A small meteorite crashed into a wooded area i...   \n",
       "1        0  A small meteorite crashed into a wooded area i...   \n",
       "2        0  A small meteorite crashed into a wooded area i...   \n",
       "3        0  A small meteorite crashed into a wooded area i...   \n",
       "4        0  A small meteorite crashed into a wooded area i...   \n",
       "\n",
       "                                            Headline     Stance  target  \n",
       "0  Soldier shot, Parliament locked down after gun...  unrelated       3  \n",
       "1  Tourist dubbed ‘Spider Man’ after spider burro...  unrelated       3  \n",
       "2  Luke Somers 'killed in failed rescue attempt i...  unrelated       3  \n",
       "3   BREAKING: Soldier shot at War Memorial in Ottawa  unrelated       3  \n",
       "4  Giant 8ft 9in catfish weighing 19 stone caught...  unrelated       3  "
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Body ID</th>\n",
       "      <th>articleBody</th>\n",
       "      <th>Headline</th>\n",
       "      <th>Stance</th>\n",
       "      <th>target</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>A small meteorite crashed into a wooded area i...</td>\n",
       "      <td>Soldier shot, Parliament locked down after gun...</td>\n",
       "      <td>unrelated</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>A small meteorite crashed into a wooded area i...</td>\n",
       "      <td>Tourist dubbed ‘Spider Man’ after spider burro...</td>\n",
       "      <td>unrelated</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "      <td>A small meteorite crashed into a wooded area i...</td>\n",
       "      <td>Luke Somers 'killed in failed rescue attempt i...</td>\n",
       "      <td>unrelated</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0</td>\n",
       "      <td>A small meteorite crashed into a wooded area i...</td>\n",
       "      <td>BREAKING: Soldier shot at War Memorial in Ottawa</td>\n",
       "      <td>unrelated</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>A small meteorite crashed into a wooded area i...</td>\n",
       "      <td>Giant 8ft 9in catfish weighing 19 stone caught...</td>\n",
       "      <td>unrelated</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Body ID                                        articleBody  \\\n",
       "0        0  A small meteorite crashed into a wooded area i...   \n",
       "1        0  A small meteorite crashed into a wooded area i...   \n",
       "2        0  A small meteorite crashed into a wooded area i...   \n",
       "3        0  A small meteorite crashed into a wooded area i...   \n",
       "4        0  A small meteorite crashed into a wooded area i...   \n",
       "\n",
       "                                            Headline     Stance  target  \n",
       "0  Soldier shot, Parliament locked down after gun...  unrelated       3  \n",
       "1  Tourist dubbed ‘Spider Man’ after spider burro...  unrelated       3  \n",
       "2  Luke Somers 'killed in failed rescue attempt i...  unrelated       3  \n",
       "3   BREAKING: Soldier shot at War Memorial in Ottawa  unrelated       3  \n",
       "4  Giant 8ft 9in catfish weighing 19 stone caught...  unrelated       3  "
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train = data\n",
    "train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "from gensim.scripts.glove2word2vec import glove2word2vec\n",
    "from gensim.models.keyedvectors import KeyedVectors\n",
    "# glove2word2vec(glove_input_file=\"glove.6B.50d.txt\", word2vec_output_file=\"gensim_glove_vectors.txt\")\n",
    "glove_model = KeyedVectors.load_word2vec_format(\"gensim_glove_vectors.txt\", binary=False)\n",
    "from nltk.corpus import stopwords\n",
    "from nltk.tokenize import word_tokenize\n",
    "stop_words = set(stopwords.words('english'))\n",
    "def process(x):\n",
    "    word_tokens = word_tokenize(x)\n",
    "    embedding = [glove_model.index2word.index(word.lower()) for word in word_tokens if word not in stop_words and word.lower() in glove_model.index2word] \n",
    "    return embedding"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Body ID</th>\n",
       "      <th>articleBody</th>\n",
       "      <th>Headline</th>\n",
       "      <th>Stance</th>\n",
       "      <th>target</th>\n",
       "      <th>w2v</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>A small meteorite crashed into a wooded area i...</td>\n",
       "      <td>Soldier shot, Parliament locked down after gun...</td>\n",
       "      <td>unrelated</td>\n",
       "      <td>3</td>\n",
       "      <td>[2481, 635, 1, 668, 5304, 6990, 27543, 136, 25...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>A small meteorite crashed into a wooded area i...</td>\n",
       "      <td>Tourist dubbed ‘Spider Man’ after spider burro...</td>\n",
       "      <td>unrelated</td>\n",
       "      <td>3</td>\n",
       "      <td>[3439, 5929, 4548, 11539, 300, 3071, 11539, 24...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "      <td>A small meteorite crashed into a wooded area i...</td>\n",
       "      <td>Luke Somers 'killed in failed rescue attempt i...</td>\n",
       "      <td>unrelated</td>\n",
       "      <td>3</td>\n",
       "      <td>[7298, 34765, 848, 1894, 1266, 5402, 57, 2, 7,...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0</td>\n",
       "      <td>A small meteorite crashed into a wooded area i...</td>\n",
       "      <td>BREAKING: Soldier shot at War Memorial in Ottawa</td>\n",
       "      <td>unrelated</td>\n",
       "      <td>3</td>\n",
       "      <td>[2923, 45, 2481, 635, 136, 2520, 6270, 2, 7, 3...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>A small meteorite crashed into a wooded area i...</td>\n",
       "      <td>Giant 8ft 9in catfish weighing 19 stone caught...</td>\n",
       "      <td>unrelated</td>\n",
       "      <td>3</td>\n",
       "      <td>[1752, 273631, 21900, 8867, 904, 2155, 1921, 9...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Body ID                                        articleBody  \\\n",
       "0        0  A small meteorite crashed into a wooded area i...   \n",
       "1        0  A small meteorite crashed into a wooded area i...   \n",
       "2        0  A small meteorite crashed into a wooded area i...   \n",
       "3        0  A small meteorite crashed into a wooded area i...   \n",
       "4        0  A small meteorite crashed into a wooded area i...   \n",
       "\n",
       "                                            Headline     Stance  target  \\\n",
       "0  Soldier shot, Parliament locked down after gun...  unrelated       3   \n",
       "1  Tourist dubbed ‘Spider Man’ after spider burro...  unrelated       3   \n",
       "2  Luke Somers 'killed in failed rescue attempt i...  unrelated       3   \n",
       "3   BREAKING: Soldier shot at War Memorial in Ottawa  unrelated       3   \n",
       "4  Giant 8ft 9in catfish weighing 19 stone caught...  unrelated       3   \n",
       "\n",
       "                                                 w2v  \n",
       "0  [2481, 635, 1, 668, 5304, 6990, 27543, 136, 25...  \n",
       "1  [3439, 5929, 4548, 11539, 300, 3071, 11539, 24...  \n",
       "2  [7298, 34765, 848, 1894, 1266, 5402, 57, 2, 7,...  \n",
       "3  [2923, 45, 2481, 635, 136, 2520, 6270, 2, 7, 3...  \n",
       "4  [1752, 273631, 21900, 8867, 904, 2155, 1921, 9...  "
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train['w2v'] = (train['Headline'] + '. ' + train['articleBody']).map(process)\n",
    "train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "X = train['w2v']\n",
    "y = train['target']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "from keras.preprocessing.sequence import pad_sequences\n",
    "X_pad = pad_sequences(X, padding='post', maxlen=300)\n",
    "y_pad = keras.utils.to_categorical(y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "vocab_size = 400000\n",
    "embedding_dim = 50"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "X_train, X_test, y_train, y_test = train_test_split(X_pad, y_pad, test_size=0.2, random_state=42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/gjx/.local/lib/python3.6/site-packages/ipykernel_launcher.py:3: DeprecationWarning: Call to deprecated `wv` (Attribute will be removed in 4.0.0, use self instead).\n",
      "  This is separate from the ipykernel package so we can avoid doing imports until\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "embedding_6 (Embedding)      (None, 300, 50)           20000000  \n",
      "_________________________________________________________________\n",
      "conv1d_3 (Conv1D)            (None, 296, 128)          32128     \n",
      "_________________________________________________________________\n",
      "global_max_pooling1d_6 (Glob (None, 128)               0         \n",
      "_________________________________________________________________\n",
      "dense_11 (Dense)             (None, 10)                1290      \n",
      "_________________________________________________________________\n",
      "dense_12 (Dense)             (None, 4)                 44        \n",
      "=================================================================\n",
      "Total params: 20,033,462\n",
      "Trainable params: 33,462\n",
      "Non-trainable params: 20,000,000\n",
      "_________________________________________________________________\n",
      "Train on 26784 samples, validate on 13193 samples\n",
      "Epoch 1/100\n",
      "26784/26784 [==============================] - 40s 1ms/step - loss: 0.6855 - acc: 0.7405 - val_loss: 0.6261 - val_acc: 0.7543\n",
      "Epoch 2/100\n",
      "26784/26784 [==============================] - 42s 2ms/step - loss: 0.5710 - acc: 0.7808 - val_loss: 0.5844 - val_acc: 0.7872\n",
      "Epoch 3/100\n",
      "26784/26784 [==============================] - 42s 2ms/step - loss: 0.5200 - acc: 0.8030 - val_loss: 0.5284 - val_acc: 0.8043\n",
      "Epoch 4/100\n",
      "26784/26784 [==============================] - 42s 2ms/step - loss: 0.4892 - acc: 0.8172 - val_loss: 0.5074 - val_acc: 0.8100\n",
      "Epoch 5/100\n",
      "26784/26784 [==============================] - 39s 1ms/step - loss: 0.4640 - acc: 0.8261 - val_loss: 0.4971 - val_acc: 0.8167\n",
      "Epoch 6/100\n",
      "26784/26784 [==============================] - 45s 2ms/step - loss: 0.4477 - acc: 0.8312 - val_loss: 0.4842 - val_acc: 0.8239\n",
      "Epoch 7/100\n",
      "26784/26784 [==============================] - 43s 2ms/step - loss: 0.4298 - acc: 0.8398 - val_loss: 0.4890 - val_acc: 0.8251\n",
      "Epoch 8/100\n",
      "26784/26784 [==============================] - 43s 2ms/step - loss: 0.4221 - acc: 0.8410 - val_loss: 0.4952 - val_acc: 0.8053\n",
      "Epoch 9/100\n",
      "26784/26784 [==============================] - 41s 2ms/step - loss: 0.4085 - acc: 0.8491 - val_loss: 0.4664 - val_acc: 0.8339\n",
      "Epoch 10/100\n",
      "26784/26784 [==============================] - 39s 1ms/step - loss: 0.3970 - acc: 0.8543 - val_loss: 0.4558 - val_acc: 0.8307\n",
      "Epoch 11/100\n",
      "26784/26784 [==============================] - 38s 1ms/step - loss: 0.3890 - acc: 0.8546 - val_loss: 0.4495 - val_acc: 0.8310\n",
      "Epoch 12/100\n",
      "26784/26784 [==============================] - 39s 1ms/step - loss: 0.3861 - acc: 0.8557 - val_loss: 0.4421 - val_acc: 0.8401\n",
      "Epoch 13/100\n",
      "26784/26784 [==============================] - 42s 2ms/step - loss: 0.3779 - acc: 0.8601 - val_loss: 0.4519 - val_acc: 0.8355\n",
      "Epoch 14/100\n",
      "26784/26784 [==============================] - 42s 2ms/step - loss: 0.3694 - acc: 0.8626 - val_loss: 0.4459 - val_acc: 0.8346\n",
      "Epoch 15/100\n",
      "26784/26784 [==============================] - 41s 2ms/step - loss: 0.3642 - acc: 0.8653 - val_loss: 0.4622 - val_acc: 0.8373\n",
      "Epoch 16/100\n",
      "26784/26784 [==============================] - 42s 2ms/step - loss: 0.3600 - acc: 0.8671 - val_loss: 0.4296 - val_acc: 0.8450\n",
      "Epoch 17/100\n",
      "26784/26784 [==============================] - 42s 2ms/step - loss: 0.3530 - acc: 0.8707 - val_loss: 0.4756 - val_acc: 0.8414\n",
      "Epoch 18/100\n",
      "26784/26784 [==============================] - 42s 2ms/step - loss: 0.3520 - acc: 0.8721 - val_loss: 0.4249 - val_acc: 0.8449\n",
      "Epoch 19/100\n",
      "26784/26784 [==============================] - 45s 2ms/step - loss: 0.3466 - acc: 0.8730 - val_loss: 0.4454 - val_acc: 0.8438\n",
      "Epoch 20/100\n",
      "26784/26784 [==============================] - 64s 2ms/step - loss: 0.3455 - acc: 0.8730 - val_loss: 0.4604 - val_acc: 0.8312\n",
      "Epoch 21/100\n",
      "26784/26784 [==============================] - 44s 2ms/step - loss: 0.3396 - acc: 0.8753 - val_loss: 0.4526 - val_acc: 0.8353\n",
      "Epoch 22/100\n",
      "26784/26784 [==============================] - 42s 2ms/step - loss: 0.3363 - acc: 0.8774 - val_loss: 0.4368 - val_acc: 0.8477\n",
      "Epoch 23/100\n",
      "26784/26784 [==============================] - 43s 2ms/step - loss: 0.3347 - acc: 0.8785 - val_loss: 0.4293 - val_acc: 0.8495\n",
      "Epoch 24/100\n",
      "26784/26784 [==============================] - 43s 2ms/step - loss: 0.3322 - acc: 0.8799 - val_loss: 0.4798 - val_acc: 0.8429\n",
      "Epoch 25/100\n",
      "26784/26784 [==============================] - 42s 2ms/step - loss: 0.3293 - acc: 0.8794 - val_loss: 0.4312 - val_acc: 0.8403\n",
      "Epoch 26/100\n",
      "26784/26784 [==============================] - 42s 2ms/step - loss: 0.3306 - acc: 0.8782 - val_loss: 0.4364 - val_acc: 0.8447\n",
      "Epoch 27/100\n",
      "26784/26784 [==============================] - 42s 2ms/step - loss: 0.3219 - acc: 0.8835 - val_loss: 0.4261 - val_acc: 0.8489\n",
      "Epoch 28/100\n",
      "26784/26784 [==============================] - 43s 2ms/step - loss: 0.3204 - acc: 0.8843 - val_loss: 0.4313 - val_acc: 0.8473\n",
      "Epoch 29/100\n",
      "26784/26784 [==============================] - 41s 2ms/step - loss: 0.3165 - acc: 0.8850 - val_loss: 0.4268 - val_acc: 0.8457\n",
      "Epoch 30/100\n",
      "26784/26784 [==============================] - 40s 2ms/step - loss: 0.3132 - acc: 0.8877 - val_loss: 0.4440 - val_acc: 0.8426\n",
      "Epoch 31/100\n",
      "26784/26784 [==============================] - 40s 2ms/step - loss: 0.3122 - acc: 0.8872 - val_loss: 0.4294 - val_acc: 0.8476\n",
      "Epoch 32/100\n",
      "26784/26784 [==============================] - 40s 2ms/step - loss: 0.3109 - acc: 0.8881 - val_loss: 0.4178 - val_acc: 0.8536\n",
      "Epoch 33/100\n",
      "26784/26784 [==============================] - 43s 2ms/step - loss: 0.3094 - acc: 0.8891 - val_loss: 0.4168 - val_acc: 0.8537\n",
      "Epoch 34/100\n",
      "26784/26784 [==============================] - 41s 2ms/step - loss: 0.3085 - acc: 0.8886 - val_loss: 0.4270 - val_acc: 0.8549\n",
      "Epoch 35/100\n",
      "26784/26784 [==============================] - 42s 2ms/step - loss: 0.3046 - acc: 0.8917 - val_loss: 0.4620 - val_acc: 0.8430\n",
      "Epoch 36/100\n",
      "26784/26784 [==============================] - 42s 2ms/step - loss: 0.3034 - acc: 0.8908 - val_loss: 0.4974 - val_acc: 0.8407\n",
      "Epoch 37/100\n",
      "26784/26784 [==============================] - 42s 2ms/step - loss: 0.3037 - acc: 0.8900 - val_loss: 0.4164 - val_acc: 0.8464\n",
      "Epoch 38/100\n",
      "26784/26784 [==============================] - 45s 2ms/step - loss: 0.3002 - acc: 0.8908 - val_loss: 0.4193 - val_acc: 0.8490\n",
      "Epoch 39/100\n",
      "26784/26784 [==============================] - 63s 2ms/step - loss: 0.3019 - acc: 0.8928 - val_loss: 0.4229 - val_acc: 0.8561\n",
      "Epoch 40/100\n",
      "26784/26784 [==============================] - 61s 2ms/step - loss: 0.3023 - acc: 0.8912 - val_loss: 0.4161 - val_acc: 0.8556\n",
      "Epoch 41/100\n",
      "26784/26784 [==============================] - 42s 2ms/step - loss: 0.3015 - acc: 0.8928 - val_loss: 0.4309 - val_acc: 0.8416\n",
      "Epoch 42/100\n",
      "26784/26784 [==============================] - 61s 2ms/step - loss: 0.2984 - acc: 0.8939 - val_loss: 0.4263 - val_acc: 0.8561\n",
      "Epoch 43/100\n",
      "26784/26784 [==============================] - 59s 2ms/step - loss: 0.2957 - acc: 0.8951 - val_loss: 0.4276 - val_acc: 0.8482\n",
      "Epoch 44/100\n",
      "26784/26784 [==============================] - 63s 2ms/step - loss: 0.2936 - acc: 0.8934 - val_loss: 0.4141 - val_acc: 0.8517\n",
      "Epoch 45/100\n",
      "26784/26784 [==============================] - 47s 2ms/step - loss: 0.2929 - acc: 0.8955 - val_loss: 0.4204 - val_acc: 0.8587\n",
      "Epoch 46/100\n",
      "26784/26784 [==============================] - 46s 2ms/step - loss: 0.2916 - acc: 0.8960 - val_loss: 0.4144 - val_acc: 0.8473\n",
      "Epoch 47/100\n",
      "26784/26784 [==============================] - 50s 2ms/step - loss: 0.2890 - acc: 0.8949 - val_loss: 0.4166 - val_acc: 0.8542\n",
      "Epoch 48/100\n",
      "26784/26784 [==============================] - 59s 2ms/step - loss: 0.2911 - acc: 0.8957 - val_loss: 0.4097 - val_acc: 0.8543\n",
      "Epoch 49/100\n",
      "26784/26784 [==============================] - 58s 2ms/step - loss: 0.2893 - acc: 0.8965 - val_loss: 0.4251 - val_acc: 0.8595\n",
      "Epoch 50/100\n",
      "26784/26784 [==============================] - 53s 2ms/step - loss: 0.2892 - acc: 0.8967 - val_loss: 0.4305 - val_acc: 0.8529\n",
      "Epoch 51/100\n",
      "26784/26784 [==============================] - 43s 2ms/step - loss: 0.2856 - acc: 0.8989 - val_loss: 0.4194 - val_acc: 0.8580\n",
      "Epoch 52/100\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "26784/26784 [==============================] - 41s 2ms/step - loss: 0.2854 - acc: 0.8986 - val_loss: 0.4183 - val_acc: 0.8602\n",
      "Epoch 53/100\n",
      "26784/26784 [==============================] - 50s 2ms/step - loss: 0.2877 - acc: 0.8952 - val_loss: 0.4097 - val_acc: 0.8558\n",
      "Epoch 54/100\n",
      "26784/26784 [==============================] - 45s 2ms/step - loss: 0.2844 - acc: 0.8959 - val_loss: 0.4644 - val_acc: 0.8560\n",
      "Epoch 55/100\n",
      "26784/26784 [==============================] - 60s 2ms/step - loss: 0.2820 - acc: 0.8990 - val_loss: 0.4484 - val_acc: 0.8560\n",
      "Epoch 56/100\n",
      "26784/26784 [==============================] - 56s 2ms/step - loss: 0.2823 - acc: 0.8983 - val_loss: 0.4528 - val_acc: 0.8573\n",
      "Epoch 57/100\n",
      "26784/26784 [==============================] - 42s 2ms/step - loss: 0.2795 - acc: 0.8995 - val_loss: 0.4194 - val_acc: 0.8565\n",
      "Epoch 58/100\n",
      "26784/26784 [==============================] - 51s 2ms/step - loss: 0.2818 - acc: 0.8970 - val_loss: 0.4795 - val_acc: 0.8565\n",
      "Epoch 59/100\n",
      "26784/26784 [==============================] - 70s 3ms/step - loss: 0.2829 - acc: 0.8988 - val_loss: 0.4749 - val_acc: 0.8541\n",
      "Epoch 60/100\n",
      "26784/26784 [==============================] - 63s 2ms/step - loss: 0.2750 - acc: 0.8999 - val_loss: 0.4299 - val_acc: 0.8605\n",
      "Epoch 61/100\n",
      "26784/26784 [==============================] - 56s 2ms/step - loss: 0.2755 - acc: 0.9000 - val_loss: 0.4097 - val_acc: 0.8583\n",
      "Epoch 62/100\n",
      "26784/26784 [==============================] - 52s 2ms/step - loss: 0.2742 - acc: 0.8990 - val_loss: 0.4666 - val_acc: 0.8282\n",
      "Epoch 63/100\n",
      "26784/26784 [==============================] - 57s 2ms/step - loss: 0.2737 - acc: 0.8990 - val_loss: 0.4269 - val_acc: 0.8645\n",
      "Epoch 64/100\n",
      "26784/26784 [==============================] - 42s 2ms/step - loss: 0.2687 - acc: 0.9019 - val_loss: 0.4297 - val_acc: 0.8587\n",
      "Epoch 65/100\n",
      "26784/26784 [==============================] - 42s 2ms/step - loss: 0.2663 - acc: 0.9030 - val_loss: 0.4166 - val_acc: 0.8643\n",
      "Epoch 66/100\n",
      "26784/26784 [==============================] - 43s 2ms/step - loss: 0.2678 - acc: 0.9002 - val_loss: 0.4355 - val_acc: 0.8600\n",
      "Epoch 67/100\n",
      "26784/26784 [==============================] - 43s 2ms/step - loss: 0.2648 - acc: 0.9023 - val_loss: 0.4636 - val_acc: 0.8580\n",
      "Epoch 68/100\n",
      "26784/26784 [==============================] - 55s 2ms/step - loss: 0.2644 - acc: 0.9013 - val_loss: 0.4722 - val_acc: 0.8588\n",
      "Epoch 69/100\n",
      "26784/26784 [==============================] - 65s 2ms/step - loss: 0.2624 - acc: 0.9020 - val_loss: 0.4558 - val_acc: 0.8621\n",
      "Epoch 70/100\n",
      "26784/26784 [==============================] - 48s 2ms/step - loss: 0.2603 - acc: 0.9011 - val_loss: 0.4479 - val_acc: 0.8617\n",
      "Epoch 71/100\n",
      "26784/26784 [==============================] - 45s 2ms/step - loss: 0.2594 - acc: 0.9031 - val_loss: 0.4238 - val_acc: 0.8575\n",
      "Epoch 72/100\n",
      "26784/26784 [==============================] - 46s 2ms/step - loss: 0.2534 - acc: 0.9048 - val_loss: 0.4746 - val_acc: 0.8590\n",
      "Epoch 73/100\n",
      "26784/26784 [==============================] - 49s 2ms/step - loss: 0.2557 - acc: 0.9045 - val_loss: 0.4676 - val_acc: 0.8437\n",
      "Epoch 74/100\n",
      "26784/26784 [==============================] - 41s 2ms/step - loss: 0.2524 - acc: 0.9043 - val_loss: 0.4344 - val_acc: 0.8664\n",
      "Epoch 75/100\n",
      "26784/26784 [==============================] - 42s 2ms/step - loss: 0.2498 - acc: 0.9061 - val_loss: 0.4538 - val_acc: 0.8496\n",
      "Epoch 76/100\n",
      "26784/26784 [==============================] - 41s 2ms/step - loss: 0.2502 - acc: 0.9048 - val_loss: 0.4653 - val_acc: 0.8492\n",
      "Epoch 77/100\n",
      "26784/26784 [==============================] - 42s 2ms/step - loss: 0.2521 - acc: 0.9056 - val_loss: 0.4398 - val_acc: 0.8636\n",
      "Epoch 78/100\n",
      "26784/26784 [==============================] - 42s 2ms/step - loss: 0.2462 - acc: 0.9071 - val_loss: 0.4503 - val_acc: 0.8669\n",
      "Epoch 79/100\n",
      "26784/26784 [==============================] - 41s 2ms/step - loss: 0.2460 - acc: 0.9066 - val_loss: 0.4696 - val_acc: 0.8383\n",
      "Epoch 80/100\n",
      "26784/26784 [==============================] - 41s 2ms/step - loss: 0.2469 - acc: 0.9067 - val_loss: 0.4324 - val_acc: 0.8639\n",
      "Epoch 81/100\n",
      "26784/26784 [==============================] - 41s 2ms/step - loss: 0.2447 - acc: 0.9067 - val_loss: 0.4564 - val_acc: 0.8595\n",
      "Epoch 82/100\n",
      "26784/26784 [==============================] - 42s 2ms/step - loss: 0.2432 - acc: 0.9073 - val_loss: 0.4712 - val_acc: 0.8494\n",
      "Epoch 83/100\n",
      "26784/26784 [==============================] - 42s 2ms/step - loss: 0.2385 - acc: 0.9091 - val_loss: 0.4591 - val_acc: 0.8662\n",
      "Epoch 84/100\n",
      "26784/26784 [==============================] - 43s 2ms/step - loss: 0.2418 - acc: 0.9077 - val_loss: 0.4695 - val_acc: 0.8677\n",
      "Epoch 85/100\n",
      "26784/26784 [==============================] - 42s 2ms/step - loss: 0.2405 - acc: 0.9076 - val_loss: 0.4407 - val_acc: 0.8567\n",
      "Epoch 86/100\n",
      "26784/26784 [==============================] - 41s 2ms/step - loss: 0.2397 - acc: 0.9072 - val_loss: 0.4571 - val_acc: 0.8594\n",
      "Epoch 87/100\n",
      "26784/26784 [==============================] - 41s 2ms/step - loss: 0.2343 - acc: 0.9092 - val_loss: 0.4513 - val_acc: 0.8578\n",
      "Epoch 88/100\n",
      "26784/26784 [==============================] - 46s 2ms/step - loss: 0.2374 - acc: 0.9074 - val_loss: 0.4407 - val_acc: 0.8611\n",
      "Epoch 89/100\n",
      "26784/26784 [==============================] - 45s 2ms/step - loss: 0.2307 - acc: 0.9107 - val_loss: 0.5076 - val_acc: 0.8614\n",
      "Epoch 90/100\n",
      "26784/26784 [==============================] - 43s 2ms/step - loss: 0.2359 - acc: 0.9080 - val_loss: 0.4685 - val_acc: 0.8504\n",
      "Epoch 91/100\n",
      "26784/26784 [==============================] - 43s 2ms/step - loss: 0.2325 - acc: 0.9091 - val_loss: 0.4561 - val_acc: 0.8591\n",
      "Epoch 92/100\n",
      "26784/26784 [==============================] - 46s 2ms/step - loss: 0.2338 - acc: 0.9099 - val_loss: 0.4542 - val_acc: 0.8619\n",
      "Epoch 93/100\n",
      "26784/26784 [==============================] - 44s 2ms/step - loss: 0.2318 - acc: 0.9110 - val_loss: 0.4971 - val_acc: 0.8609\n",
      "Epoch 94/100\n",
      "26784/26784 [==============================] - 43s 2ms/step - loss: 0.2295 - acc: 0.9122 - val_loss: 0.4349 - val_acc: 0.8638\n",
      "Epoch 95/100\n",
      "26784/26784 [==============================] - 42s 2ms/step - loss: 0.2319 - acc: 0.9092 - val_loss: 0.4783 - val_acc: 0.8467\n",
      "Epoch 96/100\n",
      "26784/26784 [==============================] - 55s 2ms/step - loss: 0.2299 - acc: 0.9106 - val_loss: 0.5302 - val_acc: 0.8589\n",
      "Epoch 97/100\n",
      "26784/26784 [==============================] - 57s 2ms/step - loss: 0.2290 - acc: 0.9123 - val_loss: 0.4686 - val_acc: 0.8674\n",
      "Epoch 98/100\n",
      "26784/26784 [==============================] - 44s 2ms/step - loss: 0.2245 - acc: 0.9121 - val_loss: 0.5201 - val_acc: 0.8591\n",
      "Epoch 99/100\n",
      "26784/26784 [==============================] - 45s 2ms/step - loss: 0.2256 - acc: 0.9128 - val_loss: 0.4906 - val_acc: 0.8564\n",
      "Epoch 100/100\n",
      "26784/26784 [==============================] - 44s 2ms/step - loss: 0.2265 - acc: 0.9131 - val_loss: 0.4845 - val_acc: 0.8669\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0x7f9d77af9be0>"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model = Sequential()\n",
    "model.add(layers.Embedding(vocab_size, embedding_dim, \n",
    "                           weights=[glove_model.wv.vectors], \n",
    "                           input_length=300, \n",
    "                           trainable=False))\n",
    "model.add(layers.Conv1D(128, 5, activation='relu'))\n",
    "model.add(layers.GlobalMaxPool1D())\n",
    "model.add(layers.Dense(10, activation='relu'))\n",
    "model.add(layers.Dense(4, activation='softmax'))\n",
    "model.compile(optimizer='adam',\n",
    "              loss='categorical_crossentropy',\n",
    "              metrics=['accuracy'])\n",
    "model.summary()\n",
    "model.fit(X_train, y_train, validation_split=0.33, batch_size=16, epochs=100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "LABELS = ['agree', 'disagree', 'discuss', 'unrelated']\n",
    "LABELS_RELATED = ['unrelated','related']\n",
    "RELATED = LABELS[0:3]\n",
    "\n",
    "def score_submission(gold_labels, test_labels):\n",
    "    score = 0.0\n",
    "    cm = [[0, 0, 0, 0],\n",
    "          [0, 0, 0, 0],\n",
    "          [0, 0, 0, 0],\n",
    "          [0, 0, 0, 0]]\n",
    "\n",
    "    for i, (g, t) in enumerate(zip(gold_labels, test_labels)):\n",
    "        g_stance, t_stance = g, t\n",
    "        if g_stance == t_stance:\n",
    "            score += 0.25\n",
    "            if g_stance != 'unrelated':\n",
    "                score += 0.50\n",
    "        if g_stance in RELATED and t_stance in RELATED:\n",
    "            score += 0.25\n",
    "\n",
    "        cm[LABELS.index(g_stance)][LABELS.index(t_stance)] += 1\n",
    "\n",
    "    return score, cm\n",
    "\n",
    "\n",
    "def print_confusion_matrix(cm):\n",
    "    lines = []\n",
    "    header = \"|{:^11}|{:^11}|{:^11}|{:^11}|{:^11}|\".format('', *LABELS)\n",
    "    line_len = len(header)\n",
    "    lines.append(\"-\"*line_len)\n",
    "    lines.append(header)\n",
    "    lines.append(\"-\"*line_len)\n",
    "\n",
    "    hit = 0\n",
    "    total = 0\n",
    "    for i, row in enumerate(cm):\n",
    "        hit += row[i]\n",
    "        total += sum(row)\n",
    "        lines.append(\"|{:^11}|{:^11}|{:^11}|{:^11}|{:^11}|\".format(LABELS[i],\n",
    "                                                                   *row))\n",
    "        lines.append(\"-\"*line_len)\n",
    "    print('\\n'.join(lines))\n",
    "\n",
    "\n",
    "def report_score(actual,predicted):\n",
    "    score,cm = score_submission(actual,predicted)\n",
    "    best_score, _ = score_submission(actual,actual)\n",
    "\n",
    "    print_confusion_matrix(cm)\n",
    "    print(\"Score: \" +str(score) + \" out of \" + str(best_score) + \"\\t(\"+str(score*100/best_score) + \"%)\")\n",
    "    return score*100/best_score\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-------------------------------------------------------------\n",
      "|           |   agree   | disagree  |  discuss  | unrelated |\n",
      "-------------------------------------------------------------\n",
      "|   agree   |   2025    |    38     |    91     |    745    |\n",
      "-------------------------------------------------------------\n",
      "| disagree  |    365    |    135    |    19     |    179    |\n",
      "-------------------------------------------------------------\n",
      "|  discuss  |    377    |     4     |   5066    |   1646    |\n",
      "-------------------------------------------------------------\n",
      "| unrelated |    128    |     7     |    292    |   28860   |\n",
      "-------------------------------------------------------------\n",
      "Score: 14664.5 out of 18011.75\t(81.4162976945605%)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "81.4162976945605"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "actual = np.argmax(y_train, axis=1)\n",
    "predicted = np.argmax(model.predict(X_train), axis=1)\n",
    "report_score([LABELS[e] for e in actual],[LABELS[e] for e in predicted])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-------------------------------------------------------------\n",
      "|           |   agree   | disagree  |  discuss  | unrelated |\n",
      "-------------------------------------------------------------\n",
      "|   agree   |    432    |    22     |    49     |    276    |\n",
      "-------------------------------------------------------------\n",
      "| disagree  |    67     |    13     |     8     |    54     |\n",
      "-------------------------------------------------------------\n",
      "|  discuss  |    121    |     2     |   1163    |    530    |\n",
      "-------------------------------------------------------------\n",
      "| unrelated |    44     |     6     |    150    |   7058    |\n",
      "-------------------------------------------------------------\n",
      "Score: 3439.75 out of 4551.5\t(75.5739865978249%)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "75.5739865978249"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "actual = np.argmax(y_test, axis=1)\n",
    "predicted = np.argmax(model.predict(X_test), axis=1)\n",
    "report_score([LABELS[e] for e in actual],[LABELS[e] for e in predicted])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}