Conv2D - MaxPooling(Q-QA).ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Conv2D - MaxPooling (Q+QA)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "\n",
    "import os\n",
    "import sys\n",
    "import certifi\n",
    "import unicodedata\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import re\n",
    "import csv\n",
    "import math\n",
    "import requests\n",
    "import time\n",
    "import random\n",
    "import scipy.stats as stats\n",
    "import matplotlib.pyplot as plt\n",
    "import gensim\n",
    "import keras\n",
    "import h5py\n",
    "import pydot\n",
    "import os\n",
    "import sklearn\n",
    "import operator\n",
    "\n",
    "from keras.layers.merge import Add\n",
    "from keras.preprocessing.text import Tokenizer\n",
    "from keras.preprocessing.sequence import pad_sequences\n",
    "from keras.utils import to_categorical, plot_model\n",
    "from keras.layers import Conv2D, MaxPooling2D, Embedding, Merge, LSTM, Bidirectional, GlobalMaxPooling1D, Dot, Lambda, Dropout\n",
    "from keras.layers import Average, dot,Permute, Lambda,Layer, add, Concatenate, Dense, LSTM, Input, concatenate, merge, Add, Reshape, Flatten\n",
    "from keras.models import Model, load_model\n",
    "from keras.models import Sequential\n",
    "from numpy import newaxis\n",
    "from keras.models import model_from_json\n",
    "from keras import callbacks\n",
    "from keras.callbacks import ModelCheckpoint, EarlyStopping,CSVLogger\n",
    "from collections import defaultdict\n",
    "from scipy.stats.stats import pearsonr\n",
    "from keras import backend as K\n",
    "from sklearn.model_selection import StratifiedKFold\n",
    "from keras import metrics\n",
    "\n",
    "K.os.environ[\"CUDA_DEVICE_ORDER\"]=\"PCI_BUS_ID\"   # see issue #152\n",
    "K.os.environ[\"CUDA_VISIBLE_DEVICES\"]=\"0\"\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "# LOAD WORD2VEC MODEL"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#LOAD GENSIM MODEL WITH WORD2VEC \n",
    "model_W2C = gensim.models.Word2Vec.load('model/vector_50/mymodelFull')\n",
    "\n",
    "# load the whole embedding into memory\n",
    "embeddings_index = dict()\n",
    "f = open('wiki-w2v/it-vectors.50.5.50.w2v', encoding = \"utf-8\")\n",
    "for line in f:\n",
    "    values = line.split()\n",
    "    word = values[0]\n",
    "    coefs = np.asarray(values[1:], dtype='float32')\n",
    "    embeddings_index[word] = coefs\n",
    "f.close()\n",
    "print('Loaded %s word vectors.' % len(embeddings_index))\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# create a weight matrix for words in training docs\n",
    "vocab_size = len(model_W2C.wv.vocab)\n",
    "embedding_matrix = np.zeros((vocab_size, 50))\n",
    "for i in range(len(model_W2C.wv.vocab)):\n",
    "    embedding_vector = embeddings_index.get(model_W2C.wv.index2word[i])\n",
    "    if embedding_vector is not None:\n",
    "        embedding_matrix[i] = embedding_vector"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# DATA TRAINING MANIPULATION"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#LOAD TOKENIZER FILES\n",
    "def loadSentences(filename):\n",
    "    array = []\n",
    "    with open(filename) as f:\n",
    "        ar = []\n",
    "        for line in f:\n",
    "            if (str(line) != \"[\\n\") & (str(line) != \"]\\n\"):\n",
    "                ar.append(re.sub('\\n$', '', line))\n",
    "                #print (line)\n",
    "            if (str(line) == \"]\\n\"):\n",
    "                array.append(ar)\n",
    "                ar = []\n",
    "            \n",
    "    f.close()\n",
    "    return array\n",
    "\n",
    "#ANSWER TRAINING\n",
    "original_QA_train = loadSentences(\"tokenizer/tokenizerQuestionAnswer.txt\")\n",
    "QA_train = loadSentences(\"tokenizer/tokenizerQuestionAnswer.txt\")\n",
    "\n",
    "#QUESTION  TRAINING\n",
    "original_Q_train = loadSentences(\"tokenizer/tokenizerQuestion.txt\")\n",
    "Q_train = loadSentences(\"tokenizer/tokenizerQuestion.txt\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "ANSWER"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(QA_train[3])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "QUESTION"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(Q_train[3])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#DATASET MANIPULATION FOR TRAINING\n",
    "random_idx = [random.randrange(0, len(original_Q_train)) for i in range(0, len(original_Q_train))]\n",
    "\n",
    "new = []\n",
    "for i in random_idx:#len(A_train)):\n",
    "    new.append(original_Q_train[i])\n",
    "Q_train.extend(new)\n",
    "\n",
    "new = []\n",
    "for i in random_idx:#len(A_train)):\n",
    "    new_random = random.randrange(0, len(random_idx))\n",
    "    while (new_random == i):\n",
    "        new_random = random.randrange(0, len(random_idx))\n",
    "    new.append(original_A_train[new_random])\n",
    "A_train.extend(new)\n",
    "\n",
    "print(len(A_train))\n",
    "print(len(Q_train))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Y_train LABEL CLASS FOR EVALUATION TRAINING\n",
    "Y_train = []\n",
    "for i in range(0, (int(len(QA_train)/2))):\n",
    "    a=[]\n",
    "    a.append(1)\n",
    "    Y_train.append(a)\n",
    "for i in range(int(len(QA_train)/2+1), len(QA_train) + 1):\n",
    "    a=[]\n",
    "    a.append(0)\n",
    "    Y_train.append(a)\n",
    "Y_train = np.array(Y_train)\n",
    "print(len(Q_train),len(QA_train),len(Y_train))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#SUBSTITUTE WORD WITH INDEX\n",
    "def replace_matched_items(array, dictionary):\n",
    "       for lst in array:\n",
    "              for ind, item in enumerate(lst):\n",
    "                          lst[ind] = dictionary.get(item, item)       "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#CLEAN ZEROS\n",
    "def cleaningZero(array):\n",
    "       for elem in array:\n",
    "            while 0 in elem: \n",
    "                elem.remove(0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "#vocab - uso lo stesso vocab\n",
    "vocab = dict([(k, v.index) for k, v in model_W2C.wv.vocab.items()])\n",
    "\n",
    "#REPLACE  WITH VOCAB INDEX\n",
    "replace_matched_items(Q_train,vocab)\n",
    "cleaningZero(Q_train)\n",
    "\n",
    "replace_matched_items(QA_train,vocab)\n",
    "cleaningZero(QA_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#PAD DOCUMENTS FOR QUESTION AND ANSWER\n",
    "max_len = max(len(max(QA_train,key=len)), len(max(Q_train,key=len)))\n",
    "Q_train_padded = keras.preprocessing.sequence.pad_sequences(Q_train, padding='post', maxlen = max_len)\n",
    "QA_train_padded = keras.preprocessing.sequence.pad_sequences(QA_train, padding='post', maxlen = max_len)\n",
    "#print(Q_train_padded.shape, QA_train_padded.shape,Y_train.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# KERAS MODEL (INPUTS: Q - A)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#FUNCTON FOR MATRIX MULTIPLICATION\n",
    "class OuterProduct(Layer):\n",
    "    def __init__(self,name=\"OuterProduct\",**kwargs):\n",
    "        super().__init__()\n",
    "\n",
    "    def build(self,input_shape):\n",
    "        self.length = input_shape[0][1]\n",
    "        super().build(input_shape)\n",
    "\n",
    "    def call(self,x):\n",
    "        a, b = x\n",
    "#        print(a.shape,b.shape,a[:, newaxis, :].shape,b[:, :, newaxis].shape)\n",
    "        outerProduct = a[:, newaxis, :] * b[:, :, newaxis]\n",
    " #       print(outerProduct.shape)\n",
    "        outerProduct = K.sum(outerProduct,axis=-1)\n",
    "  #      print(outerProduct.shape)\n",
    "\n",
    "        return outerProduct\n",
    "\n",
    "    def compute_output_shape(self, input_shape):\n",
    "        return (input_shape[0][0], input_shape[0][1], input_shape[0][1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Save the model after every epoch. https://keras.io/callbacks/#modelcheckpoint\n",
    "checkpoint = ModelCheckpoint(\"keras_models/Q_QA_Conv2D_MaxPool.h5\", monitor='val_acc', verbose=1, save_best_only=True, mode='max')\n",
    "#Stop training when a monitored quantity has stopped improving. https://keras.io/callbacks/#earlystopping\n",
    "#stop = EarlyStopping(monitor='val_loss', min_delta=0, patience=10, verbose=1, mode='auto')\n",
    "csv_logger = CSVLogger('history/CONV_Q-QA/log.csv', append=True, separator=';')\n",
    "\n",
    "callbacks_list = [checkpoint,csv_logger]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "#LAYER EMBEDDING-BiLSTM FOR ANSWER\n",
    "QA_input = Input(batch_shape=(None,QA_train_padded.shape[1],), name='A_input')\n",
    "QA = Embedding(len(vocab),weights=[embedding_matrix],output_dim=50,name=\"Embedding_A\",trainable=True)(QA_input)\n",
    "QA_LSTM = Bidirectional(LSTM(50, dropout=0.5, recurrent_dropout=0.5,return_sequences=True),name=\"LSTM_A\")(QA)\n",
    "\n",
    "\n",
    "#LAYER EMBEDDING-BiLSTM FOR QUESTIONS\n",
    "Q_input = Input(batch_shape=(None,Q_train_padded.shape[1],),  name='Q_input')\n",
    "Q = Embedding(len(vocab),output_dim=50,name=\"Embedding_Q\",trainable=True)(Q_input)\n",
    "Q_LSTM = Bidirectional(LSTM(50, dropout=0.5, recurrent_dropout=0.5,return_sequences=True),name=\"LSTM_QA\")(Q)\n",
    "\n",
    "#PRODOTTO DELLE MATRICI EMB\n",
    "M1 = OuterProduct()([QA, Q])\n",
    "M1 = Reshape((-1,QA_train_padded.shape[1],Q_train_padded.shape[1]),name=\"M1_Embedding\")(M1)\n",
    "\n",
    "M2 = OuterProduct()([QA_LSTM, Q_LSTM])\n",
    "M2 = Reshape((-1,QA_train_padded.shape[1],Q_train_padded.shape[1]),name=\"M2_LSTM\")(M2)\n",
    "\n",
    "M12 = Concatenate(axis=1,name=\"M1xM2\")([M1, M2])\n",
    "\n",
    "\n",
    "#CONVUTIONAL\n",
    "Convo = Conv2D(8, 3, activation='relu', data_format='channels_first',name=\"Convolutional\")(M12)\n",
    "Convo = Dropout(.5)(Convo)\n",
    "\n",
    "#MAXPOOLING\n",
    "Pool = MaxPooling2D(pool_size=(2, 2), data_format='channels_first',name=\"Max_Pooling\")(Convo)\n",
    "Pool = Dropout(.5)(Pool)\n",
    "Pool = Flatten()(Pool)\n",
    "\n",
    "#MLP\n",
    "dense = Dense(200, activation='relu')(Reshape((-1,))(Pool))\n",
    "dense = Dropout(.5)(dense)\n",
    "dense= Dense(100, activation='relu')(dense)\n",
    "dense = Dropout(.5)(dense)\n",
    "\n",
    "output = Dense(1,activation='sigmoid')(dense)\n",
    "\n",
    "model = Model(inputs=[Q_input,QA_input], outputs=output)\n",
    "model.compile(loss='binary_crossentropy',optimizer='adam',metrics=['accuracy',metrics.mean_squared_error])\n",
    "\n",
    "#model.summary(line_length=200)\n",
    "print(\"Model Ready...\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# PLOT PNG KERAS MODEL"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#PRINT MODEL LAYERS ON FILE.PNG\n",
    "plot_model(model, to_file='image/Q_QA_Conv2D_MaxPool.png', show_shapes=True, show_layer_names=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# TRAIN MODEL Q - QA\n",
    "Strified K-Fold with 10 splits"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "skf = StratifiedKFold(n_splits=10, shuffle=True)\n",
    "def train_and_evaluate__model(m, q1, a2, y3, q4, a5, y6):\n",
    "    history = model.fit([q1,a2],y3, batch_size=12, epochs=50, validation_data=([q4,a5],y6),callbacks = callbacks_list)\n",
    "    accuracy_history = history.history['acc']\n",
    "    val_accuracy_history = history.history['val_acc']\n",
    "    print (\"Last training accuracy: \" + str(accuracy_history[-1]) + \", last validation accuracy: \" + str(val_accuracy_history[-1]))"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {
    "scrolled": true
   },
   "source": [
    "c, r = Y_train.shape\n",
    "Y_train_p = Y_train.reshape(c,)\n",
    "\n",
    "for i, (train, test) in enumerate(skf.split(QA_train_padded, Y_train_p)):\n",
    "            print (\"Running Fold\", i+1, \"/\", 10)\n",
    "            train_and_evaluate__model(model, Q_train_padded[train],QA_train_padded[train],Y_train[train], Q_train_padded[test],QA_train_padded[test],Y_train[test])\n",
    "            "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "history = model.fit([Q_train_padded,A_train_padded],Y_train, batch_size=25, epochs=80, validation_split=0.1,callbacks = callbacks_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# summarize history for accuracy\n",
    "plt.plot(history.history['acc'])\n",
    "plt.plot(history.history['val_acc'])\n",
    "plt.title('model accuracy')\n",
    "plt.ylabel('accuracy')\n",
    "plt.xlabel('epoch')\n",
    "plt.legend(['train', 'test'], loc='upper left')\n",
    "plt.show()\n",
    "plt.savefig(fname)\n",
    "\n",
    "# summarize history for loss\n",
    "plt.plot(history.history['loss'])\n",
    "plt.plot(history.history['val_loss'])\n",
    "plt.title('model loss')\n",
    "plt.ylabel('loss')\n",
    "plt.xlabel('epoch')\n",
    "plt.legend(['train', 'test'], loc='upper left')\n",
    "plt.show()\n",
    "\n",
    "# summarize history for accuracy\n",
    "plt.plot(history.history['mean_squared_error'])\n",
    "plt.plot(history.history['val_mean_squared_error'])\n",
    "plt.title('model mean_squared_error')\n",
    "plt.ylabel('mean_squared_errory')\n",
    "plt.xlabel('epoch')\n",
    "plt.legend(['train', 'test'], loc='upper left')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "TextProject",
   "language": "python",
   "name": "textproject"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}