Siamese.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "from sklearn import model_selection, preprocessing, linear_model, naive_bayes, metrics, svm\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer, CountVectorizer\n",
    "from sklearn import decomposition, ensemble\n",
    "import pandas, xgboost, numpy, textblob, string\n",
    "from keras.preprocessing import text, sequence\n",
    "from keras import layers, models, optimizers\n",
    "from keras.utils.np_utils import to_categorical\n",
    "import _pickle as cPickle \n",
    "import pandas as pd\n",
    "\n",
    "from keras.models import load_model\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "bodies = pd.read_csv('data/train_bodies.csv')\n",
    "stances = pd.read_csv('data/train_stances.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Associate titles with bodies by ID\n",
    "bd = []\n",
    "for i in range(stances['Body ID'].shape[0]):\n",
    "    index = stances['Body ID'][i]\n",
    "    bd.append(bodies[bodies['Body ID']==index]['articleBody'].values[0])\n",
    "    \n",
    "se = pd.Series(bd)\n",
    "stances['articleBody'] = se.values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "88 \n",
      " 27579\n"
     ]
    }
   ],
   "source": [
    "#Find the longest article\n",
    "ind_max = 0\n",
    "for i in range(stances.shape[0]):\n",
    "    if len(stances['articleBody'][i])>len(stances['articleBody'][ind_max]):\n",
    "        ind_max=i\n",
    "print(ind_max,'\\n',len(stances['articleBody'][ind_max]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Headline</th>\n",
       "      <th>Body ID</th>\n",
       "      <th>Stance</th>\n",
       "      <th>articleBody</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Police find mass graves with at least '15 bodi...</td>\n",
       "      <td>712</td>\n",
       "      <td>unrelated</td>\n",
       "      <td>Danny Boyle is directing the untitled film\\n\\n...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Hundreds of Palestinians flee floods in Gaza a...</td>\n",
       "      <td>158</td>\n",
       "      <td>agree</td>\n",
       "      <td>Hundreds of Palestinians were evacuated from t...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Christian Bale passes on role of Steve Jobs, a...</td>\n",
       "      <td>137</td>\n",
       "      <td>unrelated</td>\n",
       "      <td>30-year-old Moscow resident was hospitalized w...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>HBO and Apple in Talks for $15/Month Apple TV ...</td>\n",
       "      <td>1034</td>\n",
       "      <td>unrelated</td>\n",
       "      <td>(Reuters) - A Canadian soldier was shot at the...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Spider burrowed through tourist's stomach and ...</td>\n",
       "      <td>1923</td>\n",
       "      <td>disagree</td>\n",
       "      <td>Fear not arachnophobes, the story of Bunbury's...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                            Headline  Body ID     Stance  \\\n",
       "0  Police find mass graves with at least '15 bodi...      712  unrelated   \n",
       "1  Hundreds of Palestinians flee floods in Gaza a...      158      agree   \n",
       "2  Christian Bale passes on role of Steve Jobs, a...      137  unrelated   \n",
       "3  HBO and Apple in Talks for $15/Month Apple TV ...     1034  unrelated   \n",
       "4  Spider burrowed through tourist's stomach and ...     1923   disagree   \n",
       "\n",
       "                                         articleBody  \n",
       "0  Danny Boyle is directing the untitled film\\n\\n...  \n",
       "1  Hundreds of Palestinians were evacuated from t...  \n",
       "2  30-year-old Moscow resident was hospitalized w...  \n",
       "3  (Reuters) - A Canadian soldier was shot at the...  \n",
       "4  Fear not arachnophobes, the story of Bunbury's...  "
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "stances.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "is_similar = list(stances['Stance'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'agree', 'disagree', 'discuss', 'unrelated'}"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "set(is_similar)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[0,\n",
       " 1,\n",
       " 0,\n",
       " 0,\n",
       " 3,\n",
       " 1,\n",
       " 0,\n",
       " 0,\n",
       " 1,\n",
       " 0,\n",
       " 2,\n",
       " 1,\n",
       " 0,\n",
       " 0,\n",
       " 2,\n",
       " 0,\n",
       " 2,\n",
       " 1,\n",
       " 0,\n",
       " 2,\n",
       " 0,\n",
       " 2,\n",
       " 0,\n",
       " 0,\n",
       " 1,\n",
       " 1,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 2,\n",
       " 0,\n",
       " 2,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 2,\n",
       " 1,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 1,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 3,\n",
       " 0,\n",
       " 1,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 3,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 2,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 1,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 2,\n",
       " 0,\n",
       " 0,\n",
       " 1,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 2,\n",
       " 1,\n",
       " 2,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 2,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 1,\n",
       " 1,\n",
       " 0,\n",
       " 1,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 2,\n",
       " 0,\n",
       " 2,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 1,\n",
       " 1,\n",
       " 2,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 2,\n",
       " 0,\n",
       " 2,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 2,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 2,\n",
       " 2,\n",
       " 2,\n",
       " 0,\n",
       " 0,\n",
       " 2,\n",
       " 0,\n",
       " 0,\n",
       " 2,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 1,\n",
       " 0,\n",
       " 0,\n",
       " 3,\n",
       " 0,\n",
       " 1,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 3,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 1,\n",
       " 2,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 2,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 2,\n",
       " 0,\n",
       " 0,\n",
       " 2,\n",
       " 0,\n",
       " 0,\n",
       " 1,\n",
       " 0,\n",
       " 2,\n",
       " 0,\n",
       " 2,\n",
       " 0,\n",
       " 2,\n",
       " 0,\n",
       " 0,\n",
       " 1,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 1,\n",
       " 2,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 2,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 2,\n",
       " 0,\n",
       " 2,\n",
       " 0,\n",
       " 0,\n",
       " 2,\n",
       " 0,\n",
       " 1,\n",
       " 0,\n",
       " 0,\n",
       " 2,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 2,\n",
       " 0,\n",
       " 1,\n",
       " 2,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 2,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 2,\n",
       " 0,\n",
       " 0,\n",
       " 2,\n",
       " 0,\n",
       " 2,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 3,\n",
       " 0,\n",
       " 0,\n",
       " 2,\n",
       " 3,\n",
       " 2,\n",
       " 2,\n",
       " 2,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 2,\n",
       " 0,\n",
       " 2,\n",
       " 0,\n",
       " 0,\n",
       " 3,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 1,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 2,\n",
       " 2,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 2,\n",
       " 2,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 1,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 1,\n",
       " 0,\n",
       " 0,\n",
       " 1,\n",
       " 2,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 1,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 2,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 3,\n",
       " 0,\n",
       " 0,\n",
       " 2,\n",
       " 1,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 2,\n",
       " 0,\n",
       " 0,\n",
       " 2,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 2,\n",
       " 0,\n",
       " 2,\n",
       " 0,\n",
       " 0,\n",
       " 3,\n",
       " 0,\n",
       " 3,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 2,\n",
       " 0,\n",
       " 2,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 1,\n",
       " 0,\n",
       " 2,\n",
       " 1,\n",
       " 1,\n",
       " 0,\n",
       " 2,\n",
       " 2,\n",
       " 0,\n",
       " 1,\n",
       " 0,\n",
       " 1,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 2,\n",
       " 1,\n",
       " 2,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 2,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 2,\n",
       " 0,\n",
       " 0,\n",
       " 2,\n",
       " 0,\n",
       " 3,\n",
       " 0,\n",
       " 2,\n",
       " 2,\n",
       " 2,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 2,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 2,\n",
       " 0,\n",
       " 3,\n",
       " 0,\n",
       " 1,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 2,\n",
       " 0,\n",
       " 2,\n",
       " 0,\n",
       " 2,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 2,\n",
       " 0,\n",
       " 0,\n",
       " 2,\n",
       " 0,\n",
       " 2,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 2,\n",
       " 2,\n",
       " 1,\n",
       " 0,\n",
       " 2,\n",
       " 0,\n",
       " 1,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 1,\n",
       " 2,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 1,\n",
       " 0,\n",
       " 3,\n",
       " 0,\n",
       " 0,\n",
       " 2,\n",
       " 3,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 2,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 2,\n",
       " 1,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 2,\n",
       " 0,\n",
       " 2,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 1,\n",
       " 0,\n",
       " 1,\n",
       " 0,\n",
       " 1,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 2,\n",
       " 2,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 2,\n",
       " 0,\n",
       " 1,\n",
       " 0,\n",
       " 0,\n",
       " 2,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 2,\n",
       " 0,\n",
       " 0,\n",
       " 2,\n",
       " 0,\n",
       " 0,\n",
       " 2,\n",
       " 2,\n",
       " 0,\n",
       " 2,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 2,\n",
       " 2,\n",
       " 2,\n",
       " 0,\n",
       " 1,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 1,\n",
       " 0,\n",
       " 2,\n",
       " 0,\n",
       " 0,\n",
       " 2,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 2,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 1,\n",
       " 0,\n",
       " 2,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 2,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 1,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 1,\n",
       " 0,\n",
       " 0,\n",
       " 1,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 2,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 1,\n",
       " 2,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 2,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 1,\n",
       " 0,\n",
       " 1,\n",
       " 1,\n",
       " 1,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 2,\n",
       " 2,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 1,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 2,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 1,\n",
       " 0,\n",
       " 2,\n",
       " 2,\n",
       " 0,\n",
       " 2,\n",
       " 0,\n",
       " 2,\n",
       " 0,\n",
       " 2,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 2,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 2,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 2,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 1,\n",
       " 0,\n",
       " 2,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 2,\n",
       " 1,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 1,\n",
       " 0,\n",
       " 2,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 2,\n",
       " 2,\n",
       " 0,\n",
       " 2,\n",
       " 0,\n",
       " 0,\n",
       " 2,\n",
       " 0,\n",
       " 1,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 2,\n",
       " 0,\n",
       " 2,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 2,\n",
       " 1,\n",
       " 0,\n",
       " 0,\n",
       " 1,\n",
       " 0,\n",
       " 1,\n",
       " 1,\n",
       " 1,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 2,\n",
       " 0,\n",
       " 0,\n",
       " 2,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 3,\n",
       " 1,\n",
       " 2,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 2,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 2,\n",
       " 2,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 2,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 2,\n",
       " 2,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 2,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 2,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 2,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 2,\n",
       " 0,\n",
       " 0,\n",
       " 2,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 2,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 2,\n",
       " 0,\n",
       " 3,\n",
       " 0,\n",
       " 2,\n",
       " 0,\n",
       " 1,\n",
       " 2,\n",
       " 2,\n",
       " 0,\n",
       " 0,\n",
       " 2,\n",
       " 0,\n",
       " 1,\n",
       " 0,\n",
       " 2,\n",
       " 0,\n",
       " 1,\n",
       " 0,\n",
       " 0,\n",
       " 1,\n",
       " 2,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 2,\n",
       " 0,\n",
       " 1,\n",
       " 2,\n",
       " 0,\n",
       " 2,\n",
       " 0,\n",
       " 1,\n",
       " 0,\n",
       " 1,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 2,\n",
       " 0,\n",
       " 0,\n",
       " 2,\n",
       " 0,\n",
       " 2,\n",
       " 0,\n",
       " 2,\n",
       " 2,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 2,\n",
       " 0,\n",
       " 1,\n",
       " 0,\n",
       " 0,\n",
       " 2,\n",
       " 2,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 2,\n",
       " 2,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 2,\n",
       " 2,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 2,\n",
       " 1,\n",
       " 2,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 1,\n",
       " 2,\n",
       " 0,\n",
       " 0,\n",
       " 1,\n",
       " 0,\n",
       " 0,\n",
       " 2,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 2,\n",
       " 1,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 2,\n",
       " 0,\n",
       " 0,\n",
       " 2,\n",
       " 1,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 2,\n",
       " 1,\n",
       " 0,\n",
       " 2,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 1,\n",
       " 0,\n",
       " 2,\n",
       " 0,\n",
       " 0,\n",
       " 1,\n",
       " 1,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 1,\n",
       " 0,\n",
       " 0,\n",
       " 2,\n",
       " 1,\n",
       " 0,\n",
       " 2,\n",
       " ...]"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "new_set = []\n",
    "for i in is_similar:\n",
    "    if i == 'unrelated':\n",
    "        new_set.append(0)\n",
    "    elif i == 'agree':\n",
    "        new_set.append(1)\n",
    "    elif i == 'discuss':\n",
    "        new_set.append(2)\n",
    "    else:\n",
    "        new_set.append(3)\n",
    "new_set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "is_similar = new_set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "49972"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(sentences1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [],
   "source": [
    "# keras imports\n",
    "from keras.layers import Dense, Input, LSTM, Dropout, Bidirectional\n",
    "from keras.callbacks import EarlyStopping, ModelCheckpoint\n",
    "from keras.layers.normalization import BatchNormalization\n",
    "from keras.layers.embeddings import Embedding\n",
    "from keras.layers.merge import concatenate\n",
    "from keras.callbacks import TensorBoard\n",
    "from keras.models import load_model\n",
    "from keras.models import Model\n",
    "\n",
    "# std imports\n",
    "import time\n",
    "import gc\n",
    "import os\n",
    "\n",
    "from inputHandler import create_train_dev_set\n",
    "\n",
    "\n",
    "\n",
    "class SiameseBiLSTM:\n",
    "    def __init__(self, embedding_dim, max_sequence_length, number_lstm, number_dense, rate_drop_lstm, \n",
    "                 rate_drop_dense, hidden_activation, validation_split_ratio):\n",
    "        self.embedding_dim = embedding_dim\n",
    "        self.max_sequence_length = max_sequence_length\n",
    "        self.number_lstm_units = number_lstm\n",
    "        self.rate_drop_lstm = rate_drop_lstm\n",
    "        self.number_dense_units = number_dense\n",
    "        self.activation_function = hidden_activation\n",
    "        self.rate_drop_dense = rate_drop_dense\n",
    "        self.validation_split_ratio = validation_split_ratio\n",
    "\n",
    "    def train_model(self, sentences_pair, is_similar, embedding_meta_data, model_save_directory='./'):\n",
    "        \"\"\"\n",
    "        Train Siamese network to find similarity between sentences in `sentences_pair`\n",
    "            Steps Involved:\n",
    "                1. Pass the each from sentences_pairs  to bidirectional LSTM encoder.\n",
    "                2. Merge the vectors from LSTM encodes and passed to dense layer.\n",
    "                3. Pass the  dense layer vectors to sigmoid output layer.\n",
    "                4. Use cross entropy loss to train weights\n",
    "        Args:\n",
    "            sentences_pair (list): list of tuple of sentence pairs\n",
    "            is_similar (list): target value 1 if same sentences pair are similar otherwise 0\n",
    "            embedding_meta_data (dict): dict containing tokenizer and word embedding matrix\n",
    "            model_save_directory (str): working directory for where to save models\n",
    "\n",
    "        Returns:\n",
    "            return (best_model_path):  path of best model\n",
    "        \"\"\"\n",
    "        tokenizer, embedding_matrix = embedding_meta_data['tokenizer'], embedding_meta_data['embedding_matrix']\n",
    "        print(\"##############################\")\n",
    "        train_data_x1, train_data_x2, train_labels, leaks_train, \\\n",
    "        val_data_x1, val_data_x2, val_labels, leaks_val = create_train_dev_set(tokenizer, sentences_pair,\n",
    "                                                                               is_similar, self.max_sequence_length,\n",
    "                                                                               self.validation_split_ratio)\n",
    "\n",
    "        if train_data_x1 is None:\n",
    "            print(\"++++ !! Failure: Unable to train model ++++\")\n",
    "            return None\n",
    "\n",
    "        nb_words = len(tokenizer.word_index) + 1\n",
    "\n",
    "        # Creating word embedding layer\n",
    "        embedding_layer = Embedding(nb_words, self.embedding_dim, weights=[embedding_matrix],\n",
    "                                    input_length=self.max_sequence_length, trainable=False)\n",
    "\n",
    "        # Creating LSTM Encoder\n",
    "        lstm_layer = Bidirectional(LSTM(self.number_lstm_units, dropout=self.rate_drop_lstm, recurrent_dropout=self.rate_drop_lstm))\n",
    "\n",
    "        # Creating LSTM Encoder layer for First Sentence\n",
    "        sequence_1_input = Input(shape=(self.max_sequence_length,), dtype='int32')\n",
    "        embedded_sequences_1 = embedding_layer(sequence_1_input)\n",
    "        x1 = lstm_layer(embedded_sequences_1)\n",
    "\n",
    "        # Creating LSTM Encoder layer for Second Sentence\n",
    "        sequence_2_input = Input(shape=(self.max_sequence_length,), dtype='int32')\n",
    "        embedded_sequences_2 = embedding_layer(sequence_2_input)\n",
    "        x2 = lstm_layer(embedded_sequences_2)\n",
    "\n",
    "        # Creating leaks input\n",
    "        leaks_input = Input(shape=(leaks_train.shape[1],))\n",
    "        leaks_dense = Dense(int(self.number_dense_units/2), activation=self.activation_function)(leaks_input)\n",
    "\n",
    "        # Merging two LSTM encodes vectors from sentences to\n",
    "        # pass it to dense layer applying dropout and batch normalisation\n",
    "        merged = concatenate([x1, x2, leaks_dense])\n",
    "        merged = BatchNormalization()(merged)\n",
    "        merged = Dropout(self.rate_drop_dense)(merged)\n",
    "        merged = Dense(self.number_dense_units, activation=self.activation_function)(merged)\n",
    "        merged = BatchNormalization()(merged)\n",
    "        merged = Dropout(self.rate_drop_dense)(merged)\n",
    "        preds = Dense(4, activation='softmax')(merged)\n",
    "\n",
    "        model = Model(inputs=[sequence_1_input, sequence_2_input, leaks_input], outputs=preds)\n",
    "        model.compile(loss='sparse_categorical_crossentropy', optimizer='adam', metrics=['acc'])\n",
    "        print(model.summary())\n",
    "        early_stopping = EarlyStopping(monitor='val_loss', patience=3)\n",
    "\n",
    "        STAMP = 'lstm_%d_%d_%.2f_%.2f' % (self.number_lstm_units, self.number_dense_units, self.rate_drop_lstm, self.rate_drop_dense)\n",
    "\n",
    "        checkpoint_dir = model_save_directory + 'checkpoints/' + str(int(time.time())) + '/'\n",
    "\n",
    "        if not os.path.exists(checkpoint_dir):\n",
    "            os.makedirs(checkpoint_dir)\n",
    "\n",
    "        bst_model_path = checkpoint_dir + STAMP + '.h5'\n",
    "\n",
    "        model_checkpoint = ModelCheckpoint(bst_model_path, save_best_only=True, save_weights_only=False)\n",
    "\n",
    "        tensorboard = TensorBoard(log_dir=checkpoint_dir + \"logs/{}\".format(time.time()))\n",
    "\n",
    "        model.fit([train_data_x1, train_data_x2, leaks_train], train_labels,\n",
    "                  validation_data=([val_data_x1, val_data_x2, leaks_val], val_labels),\n",
    "                  epochs=200, batch_size=64, shuffle=True,\n",
    "                  callbacks=[early_stopping, model_checkpoint, tensorboard])\n",
    "\n",
    "        return bst_model_path\n",
    "\n",
    "\n",
    "    def update_model(self, saved_model_path, new_sentences_pair, is_similar, embedding_meta_data):\n",
    "        \"\"\"\n",
    "        Update trained siamese model for given new sentences pairs \n",
    "            Steps Involved:\n",
    "                1. Pass the each from sentences from new_sentences_pair to bidirectional LSTM encoder.\n",
    "                2. Merge the vectors from LSTM encodes and passed to dense layer.\n",
    "                3. Pass the  dense layer vectors to sigmoid output layer.\n",
    "                4. Use cross entropy loss to train weights\n",
    "        Args:\n",
    "            model_path (str): model path of already trained siamese model\n",
    "            new_sentences_pair (list): list of tuple of new sentences pairs\n",
    "            is_similar (list): target value 1 if same sentences pair are similar otherwise 0\n",
    "            embedding_meta_data (dict): dict containing tokenizer and word embedding matrix\n",
    "\n",
    "        Returns:\n",
    "            return (best_model_path):  path of best model\n",
    "        \"\"\"\n",
    "        tokenizer = embedding_meta_data['tokenizer']\n",
    "        train_data_x1, train_data_x2, train_labels, leaks_train, \\\n",
    "        val_data_x1, val_data_x2, val_labels, leaks_val = create_train_dev_set(tokenizer, new_sentences_pair,\n",
    "                                                                               is_similar, self.max_sequence_length,\n",
    "                                                                               self.validation_split_ratio)\n",
    "        model = load_model(saved_model_path)\n",
    "        model_file_name = saved_model_path.split('/')[-1]\n",
    "        new_model_checkpoint_path  = saved_model_path.split('/')[:-2] + str(int(time.time())) + '/' \n",
    "\n",
    "        new_model_path = new_model_checkpoint_path + model_file_name\n",
    "        model_checkpoint = ModelCheckpoint(new_model_checkpoint_path + model_file_name,\n",
    "                                           save_best_only=True, save_weights_only=False)\n",
    "\n",
    "        early_stopping = EarlyStopping(monitor='val_loss', patience=100)\n",
    "\n",
    "        tensorboard = TensorBoard(log_dir=new_model_checkpoint_path + \"logs/{}\".format(time.time()))\n",
    "\n",
    "        model.fit([train_data_x1, train_data_x2, leaks_train], train_labels,\n",
    "                  validation_data=([val_data_x1, val_data_x2, leaks_val], val_labels),\n",
    "                  epochs=500, batch_size=3, shuffle=True)\n",
    "#                   callbacks=[early_stopping, model_checkpoint, tensorboard])\n",
    "\n",
    "        return new_model_path"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Embedding matrix shape: (160, 50)\n",
      "Null word embeddings: 1\n",
      "##############################\n",
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "input_46 (InputLayer)           (None, 10)           0                                            \n",
      "__________________________________________________________________________________________________\n",
      "input_47 (InputLayer)           (None, 10)           0                                            \n",
      "__________________________________________________________________________________________________\n",
      "embedding_16 (Embedding)        (None, 10, 50)       8000        input_46[0][0]                   \n",
      "                                                                 input_47[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "input_48 (InputLayer)           (None, 3)            0                                            \n",
      "__________________________________________________________________________________________________\n",
      "bidirectional_16 (Bidirectional (None, 100)          40400       embedding_16[0][0]               \n",
      "                                                                 embedding_16[1][0]               \n",
      "__________________________________________________________________________________________________\n",
      "dense_46 (Dense)                (None, 25)           100         input_48[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_16 (Concatenate)    (None, 225)          0           bidirectional_16[0][0]           \n",
      "                                                                 bidirectional_16[1][0]           \n",
      "                                                                 dense_46[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_31 (BatchNo (None, 225)          900         concatenate_16[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "dropout_31 (Dropout)            (None, 225)          0           batch_normalization_31[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "dense_47 (Dense)                (None, 50)           11300       dropout_31[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_32 (BatchNo (None, 50)           200         dense_47[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "dropout_32 (Dropout)            (None, 50)           0           batch_normalization_32[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "dense_48 (Dense)                (None, 4)            204         dropout_32[0][0]                 \n",
      "==================================================================================================\n",
      "Total params: 61,104\n",
      "Trainable params: 52,554\n",
      "Non-trainable params: 8,550\n",
      "__________________________________________________________________________________________________\n",
      "None\n",
      "Train on 9000 samples, validate on 1000 samples\n",
      "Epoch 1/200\n",
      " 768/9000 [=>............................] - ETA: 6:04 - loss: 1.8980 - acc: 0.2669"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/gjx/.local/lib/python3.6/site-packages/keras/callbacks.py:120: UserWarning: Method on_batch_end() is slow compared to the batch update (1.618231). Check your callbacks.\n",
      "  % delta_t_median)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "9000/9000 [==============================] - 40s 4ms/step - loss: 1.3885 - acc: 0.4493 - val_loss: 0.9111 - val_acc: 0.7370\n",
      "Epoch 2/200\n",
      "9000/9000 [==============================] - 3s 365us/step - loss: 0.9542 - acc: 0.6731 - val_loss: 0.7661 - val_acc: 0.7530\n",
      "Epoch 3/200\n",
      "9000/9000 [==============================] - 3s 342us/step - loss: 0.8576 - acc: 0.7140 - val_loss: 0.7452 - val_acc: 0.7560\n",
      "Epoch 4/200\n",
      "9000/9000 [==============================] - 2s 270us/step - loss: 0.8203 - acc: 0.7227 - val_loss: 0.7499 - val_acc: 0.7550\n",
      "Epoch 5/200\n",
      "9000/9000 [==============================] - 3s 285us/step - loss: 0.8107 - acc: 0.7268 - val_loss: 0.7413 - val_acc: 0.7540\n",
      "Epoch 6/200\n",
      "9000/9000 [==============================] - 3s 357us/step - loss: 0.7941 - acc: 0.7287 - val_loss: 0.7461 - val_acc: 0.7560\n",
      "Epoch 7/200\n",
      "1024/9000 [==>...........................] - ETA: 2s - loss: 0.8037 - acc: 0.7305"
     ]
    }
   ],
   "source": [
    "# from model import SiameseBiLSTM\n",
    "from inputHandler import word_embed_meta_data, create_test_data\n",
    "from config import siamese_config\n",
    "import pandas as pd\n",
    "\n",
    "############ Data Preperation ##########\n",
    "\n",
    "# df = pd.read_csv('sample_data.csv')\n",
    "\n",
    "sentences1 = list(stances['Headline'])[:10000]\n",
    "sentences2 = list(stances['articleBody'])[:10000]\n",
    "# is_similar = list(stances['Stance'])\n",
    "# del df\n",
    "is_similar = np.array(new_set[:10000]).reshape(10000,1)\n",
    "######## Word Embedding ############\n",
    "\n",
    "tokenizer, embedding_matrix = word_embed_meta_data(sentences1 + sentences2,  siamese_config['EMBEDDING_DIM'])\n",
    "\n",
    "embedding_meta_data = {\n",
    "\t'tokenizer': tokenizer,\n",
    "\t'embedding_matrix': embedding_matrix\n",
    "}\n",
    "\n",
    "## creating sentence pairs\n",
    "sentences_pair = [(x1, x2) for x1, x2 in zip(sentences1, sentences2)]\n",
    "del sentences1\n",
    "del sentences2\n",
    "\n",
    "######## Training ########\n",
    "\n",
    "class Configuration(object):\n",
    "    \"\"\"Dump stuff here\"\"\"\n",
    "\n",
    "CONFIG = Configuration()\n",
    "\n",
    "CONFIG.embedding_dim = siamese_config['EMBEDDING_DIM']\n",
    "CONFIG.max_sequence_length = siamese_config['MAX_SEQUENCE_LENGTH']\n",
    "CONFIG.number_lstm_units = siamese_config['NUMBER_LSTM']\n",
    "CONFIG.rate_drop_lstm = siamese_config['RATE_DROP_LSTM']\n",
    "CONFIG.number_dense_units = siamese_config['NUMBER_DENSE_UNITS']\n",
    "CONFIG.activation_function = siamese_config['ACTIVATION_FUNCTION']\n",
    "CONFIG.rate_drop_dense = siamese_config['RATE_DROP_DENSE']\n",
    "CONFIG.validation_split_ratio = siamese_config['VALIDATION_SPLIT']\n",
    "\n",
    "siamese = SiameseBiLSTM(CONFIG.embedding_dim , CONFIG.max_sequence_length, CONFIG.number_lstm_units , CONFIG.number_dense_units, CONFIG.rate_drop_lstm, CONFIG.rate_drop_dense, CONFIG.activation_function, CONFIG.validation_split_ratio)\n",
    "\n",
    "best_model_path = siamese.train_model(sentences_pair, is_similar, embedding_meta_data, model_save_directory='./')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from operator import itemgetter\n",
    "from keras.models import load_model\n",
    "\n",
    "model = load_model(best_model_path)\n",
    "\n",
    "test_sentence_pairs = [('What can make Physics easy to learn?','How can you make physics easy to learn?'),('How many times a day do a clocks hands overlap?','What does it mean that every time I look at the clock the numbers are the same?')]\n",
    "\n",
    "test_data_x1, test_data_x2, leaks_test = create_test_data(tokenizer,test_sentence_pairs,  siamese_config['MAX_SEQUENCE_LENGTH'])\n",
    "\n",
    "preds = list(model.predict([test_data_x1, test_data_x2, leaks_test], verbose=1).ravel())\n",
    "results = [(x, y, z) for (x, y), z in zip(test_sentence_pairs, preds)]\n",
    "results.sort(key=itemgetter(2), reverse=True)\n",
    "print results"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}