QuestionGenerator - Organised.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "!export LC_ALL=en_US.UTF-8\n",
    "!CUDA_VISIBLE_DEVICES=\"2\"\n",
    "import sys\n",
    "import tensorflow as tf\n",
    "import json\n",
    "from pprint import pprint\n",
    "import re\n",
    "import numpy as np\n",
    "from embedding import *\n",
    "import nltk\n",
    "import itertools\n",
    "import random\n",
    "np.random.seed(0)\n",
    "orig_stdout = sys.stdout"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "#SpaCy\n",
    "import spacy\n",
    "nlp = spacy.load('en_core_web_lg')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['/gpu:0']"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from tensorflow.python.client import device_lib\n",
    "\n",
    "def get_available_gpus():\n",
    "    local_device_protos = device_lib.list_local_devices()\n",
    "    return [x.name for x in local_device_protos if x.device_type == 'GPU']\n",
    "get_available_gpus()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow.contrib.seq2seq as seq2seq\n",
    "from tensorflow.python.layers.core import Dense"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = json.load(open('../train-v1.1.json'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "from nltk.corpus import stopwords\n",
    "#nltk.download('stopwords')\n",
    "nltkStopWords = stopwords.words('english')\n",
    "import nltk"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "def extractor(data):\n",
    "    contexts = []\n",
    "    qas = []\n",
    "    for i in range(len(data[\"data\"])):\n",
    "        for j in range(len(data[\"data\"][i][\"paragraphs\"])):\n",
    "            contexts.append(data[\"data\"][i][\"paragraphs\"][j][\"context\"])\n",
    "            qas.append(data[\"data\"][i][\"paragraphs\"][j][\"qas\"])\n",
    "    return (contexts,qas)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "CapPassage = False\n",
    "\n",
    "from nltk.tokenize import word_tokenize\n",
    "contexts,qas = extractor(data)\n",
    "\n",
    "def find_sub_list(sl,l):\n",
    "    sll=len(sl)\n",
    "    for ind in (i for i,e in enumerate(l) if e==sl[0]):\n",
    "        if l[ind:ind+sll]==sl:\n",
    "            return ind,ind+sll\n",
    "    return (-1,-1)\n",
    "\n",
    "def capPassage(passage,answer,cap_length = 30):\n",
    "    y = np.zeros(cap_length)\n",
    "    left,right = find_sub_list(answer,passage)\n",
    "    if(left==-1):\n",
    "        return passage[0:cap_length]\n",
    "    left = left - int((cap_length - len(answer))/2)\n",
    "    right = right + int((cap_length + len(answer))/2)\n",
    "    if(left < 0):\n",
    "        left = 0\n",
    "    if(right > len(passage)):\n",
    "        right = len(passage)\n",
    "    return passage[left:right]\n",
    "    \n",
    "def findAnsVec(answer,passage):\n",
    "    ans = np.zeros((len(passage)))\n",
    "    start,end = find_sub_list(answer,passage)\n",
    "    if(start==-1):\n",
    "        start = passage.index(answer[0])\n",
    "        end = start + len(answer)\n",
    "    ans[start:end] = 1\n",
    "    return ans"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train_comp_all = []\n",
    "X_train_comp_ans_all = []\n",
    "X_train_ans_all = []\n",
    "Y_train_ques_all = []\n",
    "invalid = 0\n",
    "X_train_ans_label_all = []\n",
    "for i,context in enumerate(contexts):\n",
    "    passage = word_tokenize(context.lower())\n",
    "    \n",
    "    a_lab = np.zeros(len(passage))\n",
    "    for j,_ in enumerate(qas[i]):\n",
    "        answer = word_tokenize(qas[i][j][\"answers\"][0]['text'].lower())\n",
    "        start,end = find_sub_list(answer,passage)\n",
    "        if start == -1:\n",
    "            continue\n",
    "        a_lab[start:end+1] = 1\n",
    "            \n",
    "            \n",
    "    for j,_ in enumerate(qas[i]):\n",
    "        try:\n",
    "            question = word_tokenize(qas[i][j]['question'].lower())\n",
    "            answer = word_tokenize(qas[i][j][\"answers\"][0]['text'].lower())\n",
    "            \n",
    "            if CapPassage:\n",
    "                cappedPassage = capPassage(passage,answer)\n",
    "            else:\n",
    "                cappedPassage = passage\n",
    "            \n",
    "            X_train_comp_ans_all.append(findAnsVec(answer,passage))\n",
    "            X_train_ans_label_all.append(a_lab)\n",
    "            X_train_comp_all.append(cappedPassage)\n",
    "            X_train_ans_all.append(answer)\n",
    "            Y_train_ques_all.append(question)\n",
    "        except Exception as e:\n",
    "            invalid = invalid+1\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "from collections import Counter\n",
    "import operator\n",
    "def findKMostFrequentWords(k):\n",
    "    ctr = Counter([item for sublist in X_train_comp_all for item in sublist] + [item for sublist in Y_train_ques_all for item in sublist])\n",
    "    sorted_ctr = sorted(ctr.items(), key=operator.itemgetter(1), reverse=True)\n",
    "    return [item[0] for item in sorted_ctr[0:k]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "wordsToTake = 80000\n",
    "words = findKMostFrequentWords(400000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "102979"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(words)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(80000, 100)"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "_word_to_idx_reduced = {}\n",
    "_idx_to_word_reduced = []\n",
    "\n",
    "\n",
    "def _add_word_reduced(word):\n",
    "    idx = len(_idx_to_word_reduced)\n",
    "    _word_to_idx_reduced[word] = idx\n",
    "    _idx_to_word_reduced.append(word)\n",
    "    return idx\n",
    "\n",
    "\n",
    "PAD_TOKEN = _add_word_reduced(PAD_WORD)\n",
    "UNKNOWN_TOKEN = _add_word_reduced(UNKNOWN_WORD)\n",
    "START_TOKEN = _add_word_reduced(START_WORD)\n",
    "END_TOKEN = _add_word_reduced(END_WORD)\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "dimensions = glove.shape[1]\n",
    "reduced_glove = []\n",
    "reduced_glove.append(np.zeros(dimensions))\n",
    "reduced_glove.append(-np.ones(dimensions))\n",
    "reduced_glove.append(np.ones(dimensions))\n",
    "\n",
    "for word in words:\n",
    "    l = look_up_word(word)\n",
    "    if(l != UNKNOWN_TOKEN):\n",
    "        idx = _add_word_reduced(word)\n",
    "        reduced_glove.append(glove[l])\n",
    "    if(len(reduced_glove) == wordsToTake):\n",
    "        break\n",
    "        \n",
    "def look_up_word_reduced(word):\n",
    "    return _word_to_idx_reduced.get(word, UNKNOWN_TOKEN)\n",
    "\n",
    "\n",
    "def look_up_token_reduced(token):\n",
    "    return _idx_to_word_reduced[token]\n",
    "\n",
    "reduced_glove = np.array(reduced_glove)\n",
    "reduced_glove.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "cuda2.cims.nyu.edu\n",
      "1493\n",
      "rev . john j. cavanaugh , c.s.c . served more than half , lobund institute for animal studies and medieval institute . hall of liberal arts ( "
     ]
    }
   ],
   "source": [
    "!hostname\n",
    "print(invalid)\n",
    "for i in np.where(X_train_ans_label_all[110] == 1)[0]:\n",
    "    print(X_train_comp_all[110][i], end = ' ', sep = ' ')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['architecturally', ',', 'the', 'school', 'has', 'a', 'catholic', 'character', '.', 'atop', 'the', 'main', 'building', \"'s\", 'gold', 'dome', 'is', 'a', 'golden', 'statue', 'of', 'the', 'virgin', 'mary', '.', 'immediately', 'in', 'front', 'of', 'the', 'main', 'building', 'and', 'facing', 'it', ',', 'is', 'a', 'copper', 'statue', 'of', 'christ', 'with', 'arms', 'upraised', 'with', 'the', 'legend', '``', 'ad', 'me', 'omnes', \"''\", '.', 'next', 'to', 'the', 'main', 'building', 'is', 'the', 'basilica', 'of', 'the', 'sacred', 'heart', '.', 'immediately', 'behind', 'the', 'basilica', 'is', 'the', 'grotto', ',', 'a', 'marian', 'place', 'of', 'prayer', 'and', 'reflection', '.', 'it', 'is', 'a', 'replica', 'of', 'the', 'grotto', 'at', 'lourdes', ',', 'france', 'where', 'the', 'virgin', 'mary', 'reputedly', 'appeared', 'to', 'saint', 'bernadette', 'soubirous', 'in', '1858.', 'at', 'the', 'end', 'of', 'the', 'main', 'drive', '(', 'and', 'in', 'a', 'direct', 'line', 'that', 'connects', 'through', '3', 'statues', 'and', 'the', 'gold', 'dome', ')', ',', 'is', 'a', 'simple', ',', 'modern', 'stone', 'statue', 'of', 'mary', '.']\n",
      "['saint', 'bernadette', 'soubirous']\n"
     ]
    }
   ],
   "source": [
    "print(X_train_comp_all[0])\n",
    "print(X_train_ans_all[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(101, 104)"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "find_sub_list(X_train_ans_all[0] , X_train_comp_all[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1493\n",
      "['the', 'success', 'of', 'its', 'football', 'team', 'made', 'notre', 'dame', 'a', 'household', 'name', '.', 'the', 'success', 'of', 'note', 'dame', 'reflected', 'rising', 'status', 'of', 'irish', 'americans', 'and', 'catholics', 'in', 'the', '1920s', '.', 'catholics', 'rallied', 'up', 'around', 'the', 'team', 'and', 'listen', 'to', 'the', 'games', 'on', 'the', 'radio', ',', 'especially', 'when', 'it', 'knocked', 'off', 'the', 'schools', 'that', 'symbolized', 'the', 'protestant', 'establishment', 'in', 'america', '—', 'harvard', ',', 'yale', ',', 'princeton', ',', 'and', 'army', '.', 'yet', 'this', 'role', 'as', 'high-profile', 'flagship', 'institution', 'of', 'catholicism', 'made', 'it', 'an', 'easy', 'target', 'of', 'anti-catholicism', '.', 'the', 'most', 'remarkable', 'episode', 'of', 'violence', 'was', 'the', 'clash', 'between', 'notre', 'dame', 'students', 'and', 'the', 'ku', 'klux', 'klan', 'in', '1924.', 'nativism', 'and', 'anti-catholicism', ',', 'especially', 'when', 'directed', 'towards', 'immigrants', ',', 'were', 'cornerstones', 'of', 'the', 'kkk', \"'s\", 'rhetoric', ',', 'and', 'notre', 'dame', 'was', 'seen', 'as', 'a', 'symbol', 'of', 'the', 'threat', 'posed', 'by', 'the', 'catholic', 'church', '.', 'the', 'klan', 'decided', 'to', 'have', 'a', 'week-long', 'klavern', 'in', 'south', 'bend', '.', 'clashes', 'with', 'the', 'student', 'body', 'started', 'on', 'march', '17', ',', 'when', 'students', ',', 'aware', 'of', 'the', 'anti-catholic', 'animosity', ',', 'blocked', 'the', 'klansmen', 'from', 'descending', 'from', 'their', 'trains', 'in', 'the', 'south', 'bend', 'station', 'and', 'ripped', 'the', 'kkk', 'clothes', 'and', 'regalia', '.', 'on', 'may', '19', 'thousands', 'of', 'students', 'massed', 'downtown', 'protesting', 'the', 'klavern', ',', 'and', 'only', 'the', 'arrival', 'of', 'college', 'president', 'fr', '.', 'matthew', 'walsh', 'prevented', 'any', 'further', 'clashes', '.', 'the', 'next', 'day', ',', 'football', 'coach', 'knute', 'rockne', 'spoke', 'at', 'a', 'campus', 'rally', 'and', 'implored', 'the', 'students', 'to', 'obey', 'the', 'college', 'president', 'and', 'refrain', 'from', 'further', 'violence', '.', 'a', 'few', 'days', 'later', 'the', 'klavern', 'broke', 'up', ',', 'but', 'the', 'hostility', 'shown', 'by', 'the', 'students', 'was', 'an', 'omen', 'and', 'a', 'contribution', 'to', 'the', 'downfall', 'of', 'the', 'kkk', 'in', 'indiana', '.']\n",
      "['the', 'ku', 'klux', 'klan']\n",
      "['notre', 'dame', 'students', 'had', 'a', 'showdown', 'in', '1924', 'with', 'which', 'anti-catholic', 'group', '?']\n",
      "['human', 'interference', 'has', 'nearly', 'exterminated', 'the', 'trees', 'in', 'many', 'areas', ',', 'and', ',', 'except', 'for', 'the', 'beech', 'forests', 'of', 'the', 'austrian', 'alps', ',', 'forests', 'of', 'deciduous', 'trees', 'are', 'rarely', 'found', 'after', 'the', 'extreme', 'deforestation', 'between', 'the', '17th', 'and', '19th', 'centuries', '.', 'the', 'vegetation', 'has', 'changed', 'since', 'the', 'second', 'half', 'of', 'the', '20th', 'century', ',', 'as', 'the', 'high', 'alpine', 'meadows', 'cease', 'to', 'be', 'harvested', 'for', 'hay', 'or', 'used', 'for', 'grazing', 'which', 'eventually', 'might', 'result', 'in', 'a', 'regrowth', 'of', 'forest', '.', 'in', 'some', 'areas', 'the', 'modern', 'practice', 'of', 'building', 'ski', 'runs', 'by', 'mechanical', 'means', 'has', 'destroyed', 'the', 'underlying', 'tundra', 'from', 'which', 'the', 'plant', 'life', 'can', 'not', 'recover', 'during', 'the', 'non-skiing', 'months', ',', 'whereas', 'areas', 'that', 'still', 'practice', 'a', 'natural', 'piste', 'type', 'of', 'ski', 'slope', 'building', 'preserve', 'the', 'fragile', 'underlayers', '.']\n",
      "[0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.\n",
      " 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 1. 1. 0. 0. 0. 0. 0.\n",
      " 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.\n",
      " 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.\n",
      " 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.\n",
      " 0. 0. 0. 0. 0. 0. 0. 0.]\n",
      "['the', 'vegetation']\n",
      "[1. 1. 1. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 1.\n",
      " 1. 1. 1. 1. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 1. 1. 1. 0. 0. 0. 0.\n",
      " 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.\n",
      " 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 1. 1.\n",
      " 1. 1. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.\n",
      " 0. 0. 0. 0. 0. 0. 0. 0.]\n",
      "['what', 'has', 'changed', 'since', 'the', 'second', 'half', 'of', 'the', '20th', 'century', '?']\n"
     ]
    }
   ],
   "source": [
    "print(invalid)\n",
    "print(X_train_comp_all[101])\n",
    "print(X_train_ans_all[101])\n",
    "print(Y_train_ques_all[101])\n",
    "\n",
    "c = list(zip(X_train_comp_all,X_train_comp_ans_all, X_train_ans_all, X_train_ans_label_all,Y_train_ques_all))\n",
    "np.random.shuffle(c)\n",
    "X_train_comp_all_shuffled,X_train_comp_ans_all_shuffled, X_train_ans_shuffled, X_train_ans_label_shuffled,Y_train_ques_all_shuffled = zip(*c)\n",
    "\n",
    "print(X_train_comp_all_shuffled[101])\n",
    "print(X_train_comp_ans_all_shuffled[101])\n",
    "print(X_train_ans_shuffled[101])\n",
    "print(X_train_ans_label_shuffled[101])\n",
    "print(Y_train_ques_all_shuffled[101])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "86106"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(X_train_comp_all)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "examples_to_take_train = 86080\n",
    "\n",
    "X_train_comp = X_train_comp_all_shuffled[0:examples_to_take_train]\n",
    "X_train_comp_ans = X_train_comp_ans_all_shuffled[0:examples_to_take_train]\n",
    "X_train_ans = X_train_ans_shuffled[0:examples_to_take_train]\n",
    "X_train_ans_label = X_train_ans_label_shuffled[0:examples_to_take_train]\n",
    "Y_train_ques = Y_train_ques_all_shuffled[0:examples_to_take_train]\n",
    "answer_indices = [np.where(x==1)[0].tolist() for x in X_train_comp_ans]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "max_document_len = len(max(X_train_comp,key=len))\n",
    "max_answer_len = len(max(X_train_ans,key=len))\n",
    "max_question_len = len(max(Y_train_ques,key=len)) + 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "document_tokens = np.zeros((examples_to_take_train, max_document_len), dtype=np.int32)\n",
    "document_lengths = np.zeros(examples_to_take_train, dtype=np.int32)\n",
    "answer_labels = np.zeros((examples_to_take_train, max_document_len), dtype=np.int32)\n",
    "answer_masks = np.zeros((examples_to_take_train, max_answer_len, max_document_len), dtype=np.int32)\n",
    "answer_lengths = np.zeros(examples_to_take_train, dtype=np.int32)\n",
    "question_input_tokens = np.zeros((examples_to_take_train, max_question_len), dtype=np.int32)\n",
    "question_output_tokens = np.zeros((examples_to_take_train, max_question_len), dtype=np.int32)\n",
    "question_lengths = np.zeros(examples_to_take_train, dtype=np.int32)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(86080, 766)\n"
     ]
    }
   ],
   "source": [
    "print(answer_labels.shape)\n",
    "for i in range(examples_to_take_train):\n",
    "    answer_labels[i,0:len(X_train_ans_label[i])] = X_train_ans_label[i]\n",
    "    for j, word in enumerate(X_train_comp[i]):\n",
    "        document_tokens[i, j] = look_up_word_reduced(word)\n",
    "    document_lengths[i] = len(X_train_comp[i])\n",
    "\n",
    "    for j, index in enumerate(answer_indices[i]):\n",
    "        answer_masks[i, j, index] = 1\n",
    "    answer_lengths[i] = len(answer_indices[i])\n",
    "    \n",
    "    #print(Y_train_ques[i])\n",
    "    question_input_words = ([START_WORD] + Y_train_ques[i])\n",
    "    question_output_words = (Y_train_ques[i] + [END_WORD])\n",
    "\n",
    "    for j, word in enumerate(question_input_words):\n",
    "            question_input_tokens[i, j] = look_up_word_reduced(word)\n",
    "    for j, word in enumerate(question_output_words):\n",
    "        question_output_tokens[i, j] = look_up_word_reduced(word)\n",
    "    question_lengths[i] = len(question_input_words)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(80000, 100)\n"
     ]
    }
   ],
   "source": [
    "print(reduced_glove.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "def sentences_to_indices_glove(X,max_len):\n",
    "    \n",
    "    m = len(X)                                 \n",
    "    \n",
    "    X_indices = np.full([m,max_len],look_up_word_reduced(PAD_WORD))\n",
    "    \n",
    "    for i in range(m):\n",
    "        j = 0\n",
    "        for w in X[i]:\n",
    "            if(j>=max_len):\n",
    "                break;\n",
    "            \n",
    "            X_indices[i, j] = look_up_word_reduced(w)\n",
    "            j = j+1\n",
    "        if(j<max_len):\n",
    "            X_indices[i,j] = look_up_word_reduced(END_WORD)\n",
    "    return X_indices"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "document_tokens = sentences_to_indices_glove(X_train_comp, max_document_len)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ 2039,    39,    11,   116,     6,    91, 15677,  1753,  6427,\n",
       "         100,     4, 73713, 73714,     8, 37471, 21714, 73715,  3204,\n",
       "         649,   571,     8, 17202,  2039,     9,    43,  3706,     5,\n",
       "        1266,     7,  2039, 21017,    67,    35,  1750,    14,   120,\n",
       "           9,    11, 33021,  9781,    21,  2039,     7,    63,  2039,\n",
       "       21017,    28, 13701,  3204,     9,    40, 11991,     5,     8,\n",
       "          28,  1021,    10,    38,   416,  2114,  2039,   415,    72,\n",
       "         120,  1487,     7,     4,    48,     8,   180,   526,    82,\n",
       "        2006,  1299,    14,   535,    21,  2039,    22, 17202, 37471,\n",
       "       21714,   429,    12,   405, 73716,     7,  2039,    41,    39,\n",
       "          11,  2863, 35780,     5,    16, 73717,    15,   472, 73718,\n",
       "           7,     4,  2006,    39,    60,    11, 35780,     6,  2039,\n",
       "          18,    92, 21062,    94,     8,    31,   169,  1031,     9,\n",
       "          20, 41874,    78,    18,   191, 21062,    94,     7,     3,\n",
       "           0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
       "           0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
       "           0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
       "           0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
       "           0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
       "           0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
       "           0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
       "           0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
       "           0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
       "           0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
       "           0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
       "           0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
       "           0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
       "           0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
       "           0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
       "           0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
       "           0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
       "           0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
       "           0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
       "           0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
       "           0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
       "           0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
       "           0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
       "           0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
       "           0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
       "           0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
       "           0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
       "           0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
       "           0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
       "           0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
       "           0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
       "           0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
       "           0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
       "           0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
       "           0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
       "           0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
       "           0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
       "           0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
       "           0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
       "           0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
       "           0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
       "           0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
       "           0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
       "           0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
       "           0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
       "           0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
       "           0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
       "           0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
       "           0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
       "           0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
       "           0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
       "           0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
       "           0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
       "           0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
       "           0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
       "           0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
       "           0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
       "           0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
       "           0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
       "           0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
       "           0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
       "           0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
       "           0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
       "           0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
       "           0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
       "           0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
       "           0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
       "           0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
       "           0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
       "           0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
       "           0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
       "           0])"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "document_tokens[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(86080, 766)"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "answer_labels.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "import math\n",
    "\n",
    "def createBatch(inputs,batch_size,shuffle=False):\n",
    "    outputs = []\n",
    "    start = 0\n",
    "    while start < len(inputs[0]):\n",
    "        end = min(len(inputs[0]), start + batch_size)\n",
    "        output = {'document_tokens':[],\n",
    "                    'document_lengths':[],\n",
    "                    'answer_labels':[],\n",
    "                    'answer_mask': [],\n",
    "                    'answer_lengths': [],\n",
    "                    'question_input_tokens':[],\n",
    "                    'question_output_tokens':[],\n",
    "                    'question_lengths':[],\n",
    "                 }\n",
    "        \n",
    "        for index,inp in enumerate(inputs):\n",
    "            maxD = max(inputs[1][start:start+batch_size])\n",
    "            maxA = max(inputs[4][start:start+batch_size])\n",
    "            maxQ = max(inputs[7][start:start+batch_size])\n",
    "            \n",
    "            if index == 0:\n",
    "                output['document_tokens'].append(inp[start:end,0:maxD])\n",
    "            elif index==1:\n",
    "                output['document_lengths'].append(inp[start:end])\n",
    "            elif index==2:\n",
    "                output['answer_labels'].append(inp[start:end,0:maxD])\n",
    "            elif index==3:\n",
    "                output['answer_mask'].append(inp[start:end,0:maxA,0:maxD])\n",
    "            elif index==4:\n",
    "                output['answer_lengths'].append(inp[start:end])\n",
    "            elif index==5:\n",
    "                output['question_input_tokens'].append(inp[start:end, 0:maxQ])\n",
    "            elif index==6:\n",
    "                output['question_output_tokens'].append(inp[start:end, 0:maxQ])\n",
    "            elif index==7:\n",
    "                output['question_lengths'].append(inp[start:end])\n",
    "            \n",
    "        output[\"document_tokens\"] = np.array(output[\"document_tokens\"])\n",
    "        output[\"document_lengths\"] = np.array(output[\"document_lengths\"])\n",
    "        output[\"answer_labels\"] = np.array(output[\"answer_labels\"])\n",
    "        output[\"answer_mask\"] = np.array(output[\"answer_mask\"])\n",
    "        output[\"answer_lengths\"] = np.array(output[\"answer_lengths\"])\n",
    "        output[\"question_input_tokens\"] = np.array(output[\"question_input_tokens\"])\n",
    "        output[\"question_output_tokens\"] = np.array(output[\"question_output_tokens\"])\n",
    "        output[\"question_lengths\"] = np.array(output[\"question_lengths\"])\n",
    "        outputs.append(output)\n",
    "        start = start + batch_size\n",
    "            \n",
    "    return outputs\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "batch_size = 32\n",
    "batch_input = createBatch([document_tokens,document_lengths,answer_labels,answer_masks,answer_lengths,question_input_tokens,question_output_tokens,question_lengths]\n",
    "                    ,batch_size)\n",
    "\n",
    "for b in batch_input:\n",
    "    for k, v in b.items():\n",
    "        b[k] = v.squeeze(0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "No of features: 8\n",
      "No of batches: 2690\n"
     ]
    }
   ],
   "source": [
    "print(\"No of features:\",len(batch_input[0]))\n",
    "print(\"No of batches:\",len(batch_input))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of train batches =  2686\n",
      "Number of test batches =  2\n"
     ]
    }
   ],
   "source": [
    "split_ratio = 0.999\n",
    "split = int(len(batch_input) * split_ratio)\n",
    "batch_input_train = batch_input[0:split]\n",
    "batch_input_test = batch_input[split:]\n",
    "\n",
    "if len(batch_input_train[-1]['document_tokens'] < batch_size):\n",
    "    batch_input_train = batch_input_train[:-1]\n",
    "\n",
    "if len(batch_input_test) > 0 and len(batch_input_test[-1]['document_tokens'] < batch_size):\n",
    "    batch_input_test = batch_input_test[:-1]\n",
    "    \n",
    "print(\"Number of train batches = \", len(batch_input_train))\n",
    "print(\"Number of test batches = \", len(batch_input_test))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "class HelperFunctions:\n",
    "    def __init__(self):\n",
    "        None\n",
    "    def getDoc(self, batch, batch_num, example_num):\n",
    "        document = itertools.takewhile(lambda t: t != END_TOKEN, batch[batch_num]['document_tokens'][example_num])\n",
    "        doc = \" \".join(look_up_token_reduced(token) for token in document)\n",
    "        return doc\n",
    "\n",
    "    def getQues(self, batch, batch_num, example_num):\n",
    "        question = itertools.takewhile(lambda t: t != END_TOKEN, batch[batch_num]['question_output_tokens'][example_num])\n",
    "        ques = \" \".join(look_up_token_reduced(token) for token in question)\n",
    "        return ques\n",
    "    \n",
    "    def getAns(self, batch, batch_num, example_num):\n",
    "        ans = ''\n",
    "        for i in range(batch[batch_num]['answer_lengths'][example_num]):\n",
    "            ans = ans + look_up_token_reduced(batch[batch_num]['document_tokens'][example_num][np.where(batch[batch_num]['answer_mask'][example_num][i] == 1)[0][0]]) + ' '\n",
    "        return ans\n",
    "    \n",
    "    def safe_exp(self, value):\n",
    "        try:\n",
    "            ans = math.exp(value)\n",
    "        except OverflowError:\n",
    "            ans = float(\"inf\")\n",
    "        return ans"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "class FileUtils:\n",
    "    def __init__(self, filepath):\n",
    "        self.filepath = filepath\n",
    "    def clearFile(self):\n",
    "        open(self.filepath, 'w').close()\n",
    "    def appendString(self, string):\n",
    "        with open(self.filepath, \"a\") as myfile:\n",
    "            myfile.write(string)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "class SimilarityScoreUtils:\n",
    "    \n",
    "    def __init__(self):\n",
    "        None\n",
    "    \n",
    "    def bleu_score(self, ground_truth_question, generated_question):\n",
    "        return nltk.translate.bleu_score.sentence_bleu(ground_truth_question,generated_question)\n",
    "    \n",
    "    def spacy_similarity_score(self, ground_truth_question, generated_question):\n",
    "        return nlp(ground_truth_question).similarity(nlp(generated_question))\n",
    "    \n",
    "    def aggregate_score(self, inference, key):\n",
    "        score = 0\n",
    "        for s in inference[key]:\n",
    "            score += s\n",
    "        score /= len(inference[key])\n",
    "        return score\n",
    "    \n",
    "    def find_similarity_score_from_inference(self, inferences, batch_size):\n",
    "        bleu_score = 0\n",
    "        spacy_score = 0\n",
    "        for inference in inferences:\n",
    "            bleu_score += self.aggregate_score(inference, 'bleu_score')\n",
    "            spacy_score += self.aggregate_score(inference, 'spacy_score')\n",
    "        bleu_score /= batch_size\n",
    "        spacy_score /= batch_size\n",
    "        return (bleu_score, spacy_score)\n",
    "\n",
    "    def find_similarity_on_batch(self, batch, batch_size, qgen, beam = False, beam_num = 5):\n",
    "\n",
    "        if beam :\n",
    "            inferences = qgen.generateQuestionsFromBatchWithBeam([batch], 0, batch_size, beam_num, shuffle = False, generateGroundTruthQuestion = True)\n",
    "        else:\n",
    "            inferences = qgen.generateQuestionsFromBatch([batch], 0, batch_size, shuffle = False, generateGroundTruthQuestion = True)\n",
    "            \n",
    "        \n",
    "        return self.find_similarity_score_from_inference(inferences, batch_size)\n",
    "            \n",
    "    def find_similarity_on_set(self, batch_input, batch_size, qgen, beam = False, beam_num = 5):\n",
    "        bleu_scores = 0\n",
    "        spacy_scores = 0\n",
    "        num_batches = len(batch_input)\n",
    "        \n",
    "        for batch in batch_input:\n",
    "            bleu_score, spacy_score = self.find_similarity_on_batch(batch, batch_size, qgen, beam, beam_num)\n",
    "            bleu_scores += bleu_score\n",
    "            spacy_scores += spacy_score\n",
    "        \n",
    "        bleu_scores /= num_batches\n",
    "        spacy_scores /= num_batches\n",
    "\n",
    "        return (bleu_scores, spacy_scores)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "metadata": {},
   "outputs": [],
   "source": [
    "class InferenceUtils:\n",
    "    def __init__(self, batch_size, qgen):\n",
    "        self.batch_size = batch_size\n",
    "        self.qgen = qgen\n",
    "        \n",
    "    def printInferences(self, inferences, printGroundTruthQuestion = True, printSimilarityScores = True):\n",
    "        for inference in inferences:\n",
    "            print('----------------------------------------------------------------------------------------------------')\n",
    "            print('Comprehension : ')\n",
    "            print(inference['passage'])\n",
    "\n",
    "            if printGroundTruthQuestion:\n",
    "                print('*****************************************************************************************************')\n",
    "                print('Ground Truth Question : ')\n",
    "                print(inference['ground_truth_question'])\n",
    "            print('*****************************************************************************************************')\n",
    "            print('Ground Truth Answer: ')\n",
    "            print(inference['ground_truth_answer'])\n",
    "            print('*****************************************************************************************************')\n",
    "            for i,gen_ques in enumerate(inference['generated_questions']):\n",
    "                print('Generated Question Number :  ', i+1)\n",
    "                print(gen_ques)\n",
    "                if printSimilarityScores:\n",
    "                    print(\"BLEU Score : \", inference['bleu_score'][i])\n",
    "                    print(\"SpaCy Similarity Score : \", inference['spacy_score'][i])\n",
    "                print('*****************************************************************************************************')\n",
    "            print('----------------------------------------------------------------------------------------------------')\n",
    "        \n",
    "    def createTokens(self, passage):\n",
    "        passage = word_tokenize(passage.lower())\n",
    "        passage_len = len(passage)\n",
    "        p = np.array(sentences_to_indices_glove([passage], passage_len))\n",
    "        p_len = np.array(passage_len).reshape((1,))\n",
    "        p_batch = np.repeat(p,repeats=self.batch_size, axis = 0)\n",
    "        p_len_batch = np.repeat(p_len,repeats=self.batch_size, axis = 0)\n",
    "        return p_batch, p_len_batch\n",
    "            \n",
    "    \n",
    "    def makeInferenceOnText(self, passage, answer = None, use_beam = False, num_questions = 5):\n",
    "        \n",
    "        \n",
    "        if answer is None:\n",
    "            answers = self.qgen.generateAnswers(passage)\n",
    "            print(\"Answers = \", answers)\n",
    "            answers = np.nonzero(answers)[0]\n",
    "            outputs = []\n",
    "            for i in np.split(answers, np.where(np.diff(answers) != 1)[0]+1):\n",
    "                print(i)\n",
    "                left,right = i[0],i[-1]+1\n",
    "                answer = \" \".join(word_tokenize(passage.lower())[j] for j in range(left,right))\n",
    "                self.makeInferenceOnText(passage, answer, use_beam, num_questions)\n",
    "            return\n",
    "        \n",
    "        passage = word_tokenize(passage.lower())\n",
    "        passage_len = len(passage)\n",
    "\n",
    "        answer = word_tokenize(answer.lower())\n",
    "        answer_len = len(answer)    \n",
    "\n",
    "        print(\"Passage Length = \", passage_len, \", Answer Length = \",answer_len)\n",
    "\n",
    "        left,right = find_sub_list(answer,passage)\n",
    "        if(left==-1):\n",
    "            print(\"Couldn't find answer in the passage !!\")\n",
    "            return\n",
    "        p = np.array(sentences_to_indices_glove([passage], passage_len))\n",
    "\n",
    "        p_len = np.array(passage_len).reshape((1,))\n",
    "        a_len = np.array(answer_len).reshape((1,))\n",
    "\n",
    "        ans_labels = np.zeros((1,passage_len))\n",
    "        ans_labels[0][left:right] = 1\n",
    "\n",
    "        enc_mask = np.zeros((1,answer_len, passage_len))\n",
    "        for i in range(left,right):\n",
    "            enc_mask[0,i-left,i] = 1\n",
    "\n",
    "        p_batch = np.repeat(p,repeats=self.batch_size, axis = 0)\n",
    "        p_len_batch = np.repeat(p_len,repeats=self.batch_size, axis = 0)\n",
    "        a_labels_batch = np.repeat(ans_labels,repeats=self.batch_size, axis = 0)\n",
    "        enc_mask_batch = np.repeat(enc_mask,repeats=self.batch_size, axis = 0)\n",
    "        a_len_batch = np.repeat(a_len,repeats=self.batch_size, axis = 0)\n",
    "        \n",
    "        output = {'document_tokens':p_batch,\n",
    "                    'document_lengths':p_len_batch,\n",
    "                    'answer_labels':a_labels_batch,\n",
    "                    'answer_mask': enc_mask_batch,\n",
    "                    'answer_lengths': a_len_batch\n",
    "                 }\n",
    "        if use_beam:\n",
    "            inferences = self.qgen.generateQuestionsFromBatchWithBeam(\n",
    "                batch = [output], \n",
    "                batchNum = 0, \n",
    "                numExamples = 1, \n",
    "                numQuestions = num_questions, \n",
    "                shuffle = False,\n",
    "                generateGroundTruthQuestion = False,\n",
    "                generateScores = False)\n",
    "        else: \n",
    "            inferences = self.qgen.generateQuestionsFromBatch([output], 0, 1, shuffle = False, generateGroundTruthQuestion = False, generateScores = False)\n",
    "            \n",
    "        self.printInferences(inferences, printGroundTruthQuestion = False, printSimilarityScores = False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {},
   "outputs": [],
   "source": [
    "class QGen:\n",
    "    def __init__(\n",
    "        self, learning_rate, cell_size , n_layers, reduced_glove, batch_size, \n",
    "            attention_type = 'Luong', cell_type = 'GRU', sess=tf.Session(), \n",
    "            helperFunctions = HelperFunctions(), fileUtils = FileUtils('/data/ra2630/tfLog64k'), similarityScoreUtils = SimilarityScoreUtils() ,\n",
    "            grad_clip=1.0, beam_width=10, force_teaching_ratio=0.5, dropout_probability = [0.4, 0.3, 0.3]):\n",
    "        self.learning_rate = learning_rate\n",
    "        self.cell_size = cell_size\n",
    "        self.cell_type = cell_type\n",
    "        self.attention_type = attention_type\n",
    "        self.n_layers = n_layers\n",
    "        self.grad_clip = grad_clip\n",
    "        self.glove_embedding = reduced_glove\n",
    "        self.batch_size = batch_size\n",
    "        self.beam_width = beam_width\n",
    "        self.force_teaching_ratio = force_teaching_ratio\n",
    "        self.dropout_probability_document_encoder = dropout_probability[0]\n",
    "        self.dropout_probability_answer_encoder = dropout_probability[1]\n",
    "        self.dropout_probability_question_decoder = dropout_probability[2]\n",
    "        self.sess = sess\n",
    "        self.helperFunctions = helperFunctions\n",
    "        self.fileUtils = fileUtils\n",
    "        self.similarityScoreUtils = similarityScoreUtils\n",
    "        self.inferenceUtils = InferenceUtils(self.batch_size, self)\n",
    "        self.build_graph()\n",
    "        \n",
    "    def saveSession(self, filepath):\n",
    "        print(\"Saving Session to file : \", filepath)\n",
    "        saver = tf.train.Saver()\n",
    "        saver.save(self.sess, filepath)\n",
    "        print(\"Session Saved !\")\n",
    "        \n",
    "    def loadFromSession(self, filepath):\n",
    "        print(\"Loading Session from file \", filepath)\n",
    "        saver = tf.train.Saver()\n",
    "        saver.restore(self.sess, filepath)\n",
    "        self.sess.run(tf.tables_initializer())\n",
    "        print(\"Session restored !\")\n",
    "\n",
    "\n",
    "    def build_graph(self):\n",
    "        self.add_embedding_layer()\n",
    "        self.add_document_encoder_layer()\n",
    "        self.add_answer_encoder_layer()\n",
    "        with tf.variable_scope('decode'):\n",
    "            self.add_decoder_for_training()\n",
    "        with tf.variable_scope('decode', reuse=True):\n",
    "            self.add_beam_decoder_for_inference()\n",
    "        self.add_decoder_for_inference()\n",
    "            \n",
    "        \n",
    "        self.add_backward_path()\n",
    "\n",
    "\n",
    "    def add_embedding_layer(self): \n",
    "        self.embedding = tf.get_variable(\"embedding\", initializer=self.glove_embedding)\n",
    "        self. embedding = tf.cast(self.embedding, dtype=tf.float32)\n",
    "        self.embedding_dimensions = reduced_glove.shape[1]\n",
    "        self.vocabulary_size = reduced_glove.shape[0]\n",
    "        \n",
    "    def add_document_encoder_layer(self):\n",
    "        self.d_tokens = tf.placeholder(tf.int32, shape=[None, None])\n",
    "        self.d_lengths = tf.placeholder(tf.int32, shape=[None])\n",
    "        \n",
    "        document_emb = tf.nn.embedding_lookup(self.embedding, self.d_tokens)\n",
    "        document_emb = tf.cast(document_emb, dtype=tf.float64)\n",
    "        \n",
    "        if self.cell_type == 'LSTM':\n",
    "            self.document_forward_cell = tf.contrib.rnn.LSTMCell(self.cell_size)\n",
    "            self.document_backward_cell = tf.contrib.rnn.LSTMCell(self.cell_size)\n",
    "        elif self.cell_type == 'GRU':\n",
    "            self.document_forward_cell = tf.contrib.rnn.GRUCell(self.cell_size)\n",
    "            self.document_backward_cell = tf.contrib.rnn.GRUCell(self.cell_size)\n",
    "        \n",
    "        self.document_forward_cell = tf.nn.rnn_cell.DropoutWrapper(\n",
    "            self.document_forward_cell, \n",
    "            output_keep_prob = self.dropout_probability_document_encoder,\n",
    "            state_keep_prob = self.dropout_probability_document_encoder\n",
    "        )\n",
    "        \n",
    "        self.document_backward_cell = tf.nn.rnn_cell.DropoutWrapper(\n",
    "            self.document_backward_cell, \n",
    "            output_keep_prob=self.dropout_probability_document_encoder,\n",
    "            state_keep_prob = self.dropout_probability_document_encoder\n",
    "        )\n",
    "        \n",
    "        self.document_encoder_outputs, self.document_encoder_states = tf.nn.bidirectional_dynamic_rnn(\n",
    "            self.document_forward_cell, \n",
    "            self.document_backward_cell, \n",
    "            document_emb, \n",
    "            self.d_lengths, \n",
    "            dtype=tf.float64\n",
    "        )\n",
    "        \n",
    "        self.document_encoder_outputs = tf.concat(self.document_encoder_outputs, 2)\n",
    "        self.document_encoder_outputs = tf.cast(self.document_encoder_outputs,tf.float32)\n",
    "        self.answer_tags = tf.layers.dense(inputs = self.document_encoder_outputs, units=2)\n",
    "\n",
    "\n",
    "        self.a_labels = tf.placeholder(tf.int32, shape=[None, None])\n",
    "        self.answer_mask = tf.sequence_mask(self.d_lengths, dtype=tf.float32)\n",
    "        self.answer_loss = seq2seq.sequence_loss(\n",
    "            logits=self.answer_tags, targets=self.a_labels, weights=self.answer_mask)\n",
    "        \n",
    "        self.answer_loss = tf.Print(self.answer_loss, [self.answer_loss], message=\"This is answer_loss: \")\n",
    "        \n",
    "\n",
    "\n",
    "    def add_answer_encoder_layer(self):\n",
    "        self.answer_encoder_input_mask = tf.placeholder(\n",
    "        tf.float32, shape=[None, None, None])\n",
    "        \n",
    "        self.answer_encoder_inputs = tf.matmul(self.answer_encoder_input_mask, self.document_encoder_outputs)\n",
    "        self.answer_encoder_lengths = tf.placeholder(tf.int32, shape=[None])\n",
    "\n",
    "        if self.cell_type == 'GRU':\n",
    "            self.answer_encoder_cell = tf.contrib.rnn.GRUCell(self.document_forward_cell.state_size + self.document_backward_cell.state_size)\n",
    "            self.question_decoder_cell = tf.contrib.rnn.GRUCell(self.answer_encoder_cell.state_size)\n",
    "        \n",
    "        elif self.cell_type == 'LSTM':\n",
    "            self.answer_encoder_cell = tf.contrib.rnn.LSTMCell(self.document_forward_cell.state_size[0] + self.document_backward_cell.state_size[0])\n",
    "            self.question_decoder_cell = tf.contrib.rnn.LSTMCell(self.answer_encoder_cell.state_size[0])\n",
    "            \n",
    "        self.answer_encoder_cell = tf.nn.rnn_cell.DropoutWrapper(\n",
    "            self.answer_encoder_cell,\n",
    "            state_keep_prob=self.dropout_probability_answer_encoder\n",
    "        )\n",
    "        \n",
    "        self.question_decoder_cell = tf.nn.rnn_cell.DropoutWrapper(\n",
    "            self.question_decoder_cell, \n",
    "            state_keep_prob = self.dropout_probability_question_decoder,\n",
    "            output_keep_prob=self.dropout_probability_question_decoder\n",
    "        )\n",
    "            \n",
    "        _, self.answer_encoder_state = tf.nn.dynamic_rnn(\n",
    "            self.answer_encoder_cell, self.answer_encoder_inputs, self.answer_encoder_lengths, dtype=tf.float32)\n",
    "\n",
    "\n",
    "\n",
    "    def add_attention_for_training(self):\n",
    "        if self.attention_type == 'Luong':\n",
    "            self.attention_function = tf.contrib.seq2seq.LuongAttention\n",
    "        else:\n",
    "            self.attention_function = tf.contrib.seq2seq.BahdanauAttention\n",
    "            \n",
    "        if self.cell_type == 'GRU':\n",
    "            attention_mechanism = self.attention_function(\n",
    "                num_units = self.answer_encoder_cell.state_size, \n",
    "                memory = self.document_encoder_outputs,\n",
    "                memory_sequence_length = self.d_lengths)\n",
    "\n",
    "            self.question_decoder_cell_attention = tf.contrib.seq2seq.AttentionWrapper(\n",
    "                cell = self.question_decoder_cell, \n",
    "                attention_mechanism = attention_mechanism,\n",
    "                attention_layer_size = self.question_decoder_cell.state_size)\n",
    "\n",
    "        elif self.cell_type == 'LSTM':\n",
    "            attention_mechanism = self.attention_function(\n",
    "                num_units = self.answer_encoder_cell.state_size[0], \n",
    "                memory = self.document_encoder_outputs,\n",
    "                memory_sequence_length = self.d_lengths)\n",
    "\n",
    "            self.question_decoder_cell_attention = tf.contrib.seq2seq.AttentionWrapper(\n",
    "                cell = self.question_decoder_cell, \n",
    "                attention_mechanism = attention_mechanism,\n",
    "                attention_layer_size = self.question_decoder_cell.state_size[0])\n",
    "        \n",
    "        self.question_decoder_cell_attention_without_beam = self.question_decoder_cell_attention\n",
    "\n",
    "\n",
    "    def add_decoder_for_training(self):\n",
    "        self.add_attention_for_training()\n",
    "        self.question_decoder_inputs = tf.placeholder(tf.int32, shape=[None, None])\n",
    "        self.question_decoder_lengths = tf.placeholder(tf.int32, shape=[None])\n",
    "\n",
    "        question_decoder_embedding = tf.nn.embedding_lookup(self.embedding, self.question_decoder_inputs)\n",
    "        question_decoder_embedding = tf.cast(question_decoder_embedding,tf.float32)\n",
    "\n",
    "        #helper = seq2seq.TrainingHelper(question_decoder_embedding , self.question_decoder_lengths)\n",
    "        helper = seq2seq.ScheduledEmbeddingTrainingHelper(\n",
    "            inputs = question_decoder_embedding,\n",
    "            sequence_length = self.question_decoder_lengths,\n",
    "            embedding = self.embedding,\n",
    "            sampling_probability = 1 - self.force_teaching_ratio,\n",
    "            time_major = False)\n",
    "\n",
    "        self.projection = Dense(self.vocabulary_size, use_bias=False)\n",
    "        \n",
    "        decoder = seq2seq.BasicDecoder(\n",
    "            self.question_decoder_cell_attention, \n",
    "            helper, \n",
    "            self.question_decoder_cell_attention.zero_state(self.batch_size, dtype=tf.float32).clone(\n",
    "              cell_state=self.answer_encoder_state),\n",
    "            output_layer = self.projection\n",
    "        )\n",
    "        \n",
    "        decoder_outputs, _, _ = seq2seq.dynamic_decode(decoder)\n",
    "        self.training_question_decoder_outputs = decoder_outputs.rnn_output\n",
    "        \n",
    "    def add_decoder_for_inference(self):\n",
    "        helper = seq2seq.GreedyEmbeddingHelper(\n",
    "            self.embedding, \n",
    "            tf.fill([self.batch_size], START_TOKEN), \n",
    "            END_TOKEN\n",
    "        )\n",
    "        decoder = seq2seq.BasicDecoder(\n",
    "            self.question_decoder_cell_attention_without_beam, \n",
    "            helper, \n",
    "            self.question_decoder_cell_attention_without_beam.zero_state(self.batch_size, dtype=tf.float32).clone(\n",
    "                  cell_state=self.answer_encoder_state), output_layer=self.projection)\n",
    "        \n",
    "        decoder_outputs, _, _ = seq2seq.dynamic_decode(decoder, maximum_iterations=max_question_len)\n",
    "        self.decoder_outputs = decoder_outputs.rnn_output\n",
    "        \n",
    "        \n",
    "\n",
    "\n",
    "    def add_beam_attention_for_inference(self):\n",
    "        self.document_encoder_output_tiled = tf.contrib.seq2seq.tile_batch(self.document_encoder_outputs, self.beam_width)\n",
    "        self.answer_encoder_state_tiled = tf.contrib.seq2seq.tile_batch(self.answer_encoder_state, self.beam_width)\n",
    "        self.document_lengths_tiled = tf.contrib.seq2seq.tile_batch(self.d_lengths, self.beam_width)\n",
    "\n",
    "        if self.cell_type == 'GRU':\n",
    "            attention_mechanism = self.attention_function(\n",
    "                num_units = self.answer_encoder_cell.state_size, \n",
    "                memory = self.document_encoder_output_tiled,\n",
    "                memory_sequence_length = self.document_lengths_tiled)\n",
    "\n",
    "            self.question_decoder_cell_attention = tf.contrib.seq2seq.AttentionWrapper(\n",
    "                cell = self.question_decoder_cell, \n",
    "                attention_mechanism = attention_mechanism,\n",
    "                attention_layer_size = self.question_decoder_cell.state_size)\n",
    "            \n",
    "\n",
    "        elif self.cell_type == 'LSTM':\n",
    "            attention_mechanism = self.attention_function(\n",
    "                num_units = self.answer_encoder_cell.state_size[0], \n",
    "                memory = self.document_encoder_output_tiled,\n",
    "                memory_sequence_length = self.document_lengths_tiled)\n",
    "\n",
    "            self.question_decoder_cell_attention = tf.contrib.seq2seq.AttentionWrapper(\n",
    "                cell = self.question_decoder_cell, \n",
    "                attention_mechanism = attention_mechanism,\n",
    "                attention_layer_size = self.question_decoder_cell.state_size[0])\n",
    "\n",
    "\n",
    "\n",
    "    def add_beam_decoder_for_inference(self):\n",
    "        self.add_beam_attention_for_inference()\n",
    "        \n",
    "        decoder = tf.contrib.seq2seq.BeamSearchDecoder(\n",
    "            cell = self.question_decoder_cell_attention,\n",
    "            embedding = self.embedding,\n",
    "            start_tokens = tf.fill([self.batch_size], START_TOKEN),\n",
    "            end_token = END_TOKEN,\n",
    "            initial_state = self.question_decoder_cell_attention.zero_state(self.batch_size * self.beam_width,tf.float32).clone(\n",
    "                cell_state = self.answer_encoder_state_tiled),\n",
    "            beam_width = self.beam_width,\n",
    "            output_layer = self.projection,\n",
    "            length_penalty_weight = 0.0)\n",
    "        \n",
    "        \n",
    "        predicting_decoder_output, _, _ = tf.contrib.seq2seq.dynamic_decode(\n",
    "            decoder = decoder,\n",
    "            maximum_iterations = max_question_len\n",
    "        )\n",
    "        self.predicting_question_ids = predicting_decoder_output.predicted_ids[:, :, :]\n",
    "\n",
    "\n",
    "    def add_backward_path(self):\n",
    "        self.decoder_labels = tf.placeholder(tf.int64, shape=[None, None])\n",
    "        question_mask = tf.sequence_mask(self.question_decoder_lengths ,dtype=tf.float32)\n",
    "        self.question_loss = seq2seq.sequence_loss(\n",
    "            logits = self.training_question_decoder_outputs, \n",
    "            targets = self.decoder_labels,\n",
    "            weights = question_mask)\n",
    "        self.question_loss = tf.Print(self.question_loss, [self.question_loss], message=\"This is question_loss: \")\n",
    "        \n",
    "        self.net_loss = self.question_loss + self.answer_loss\n",
    "        \n",
    "        optimizer = tf.train.AdamOptimizer(learning_rate=self.learning_rate)\n",
    "        gvs = optimizer.compute_gradients(self.net_loss)\n",
    "        capped_gvs = [(tf.clip_by_value(grad, -self.grad_clip, self.grad_clip), var) for grad, var in gvs]\n",
    "        self.train_op = optimizer.apply_gradients(capped_gvs)\n",
    "\n",
    "\n",
    "    def fit(self, batch_input_train, batch_input_test, path_to_save_file,save_after_every_epoch = 5, n_epoch=60, generateTrain = False, generateTest = False, clearFile = False, resumeTraining = False):\n",
    "\n",
    "        if not resumeTraining:\n",
    "            self.sess.run(tf.global_variables_initializer())\n",
    "        if clearFile:\n",
    "            self.fileUtils.clearFile()\n",
    "        \n",
    "        for epoch in range(1, n_epoch + 1):\n",
    "            batch_loss = 0\n",
    "            perplexity_score = 0\n",
    "            for batchNum in range(len(batch_input_train)):\n",
    "                t = self.sess.run([self.train_op, self.net_loss, self.question_loss, self.answer_loss], {\n",
    "                    self.d_tokens: batch_input_train[batchNum]['document_tokens'],\n",
    "                    self.d_lengths: batch_input_train[batchNum]['document_lengths'],\n",
    "                    self.a_labels: batch_input_train[batchNum]['answer_labels'],\n",
    "                    self.answer_encoder_input_mask: batch_input_train[batchNum]['answer_mask'],\n",
    "                    self.answer_encoder_lengths: batch_input_train[batchNum]['answer_lengths'],\n",
    "                    self.question_decoder_inputs: batch_input_train[batchNum]['question_input_tokens'],\n",
    "                    self.decoder_labels: batch_input_train[batchNum]['question_output_tokens'],\n",
    "                    self.question_decoder_lengths: batch_input_train[batchNum]['question_lengths']\n",
    "                })\n",
    "                p_score = self.helperFunctions.safe_exp((t[1] / max_question_len) * self.batch_size)\n",
    "                print(\"Epoch : {0}, Batch: {1}/{2}, Loss: {3}, perplexity_score: {4}\".format(epoch, batchNum, len(batch_input_train), t[1], p_score))\n",
    "                batch_loss += t[1]\n",
    "                perplexity_score+= p_score\n",
    "                sys.stdout.flush()\n",
    "            batch_loss /= len(batch_input_train)\n",
    "            print(\"Epochs: {0}/{1} | Batch - Loss (Train): {2} | Perplexity (Train) : {3}\".format(epoch, n_epoch, batch_loss, perplexity_score))\n",
    "            sys.stdout.flush()\n",
    "            \n",
    "            \n",
    "            \n",
    "            batch_loss = 0\n",
    "            perplexity_score = 0\n",
    "            for batchNum in range(len(batch_input_test)):\n",
    "                t = self.sess.run([self.net_loss], {\n",
    "                    self.d_tokens: batch_input_test[batchNum]['document_tokens'],\n",
    "                    self.d_lengths: batch_input_test[batchNum]['document_lengths'],\n",
    "                    self.a_labels: batch_input_test[batchNum]['answer_labels'],\n",
    "                    self.answer_encoder_input_mask: batch_input_test[batchNum]['answer_mask'],\n",
    "                    self.answer_encoder_lengths: batch_input_test[batchNum]['answer_lengths'],\n",
    "                    self.question_decoder_inputs: batch_input_test[batchNum]['question_input_tokens'],\n",
    "                    self.decoder_labels: batch_input_test[batchNum]['question_output_tokens'],\n",
    "                    self.question_decoder_lengths: batch_input_test[batchNum]['question_lengths']\n",
    "                })\n",
    "\n",
    "                batch_loss += t[0]\n",
    "                p_score = self.helperFunctions.safe_exp((t[0] / max_question_len) * self.batch_size)\n",
    "                perplexity_score+= p_score\n",
    "\n",
    "            batch_loss /= len(batch_input_test)\n",
    "\n",
    "            print(\"Epochs: {0}/{1} | Batch - Loss (Test): {2} | Perplexity (Test) : {3}\".format(epoch, n_epoch, batch_loss, perplexity_score))\n",
    "            sys.stdout.flush()\n",
    "                    \n",
    "            \n",
    "            if generateTrain:\n",
    "                print(\"Train Samples Generated : \")\n",
    "                inferences = self.generateQuestionsFromBatch(batch_input_train, random.randint(0,len(batch_input_train)-1), 5)\n",
    "                self.inferenceUtils.printInferences(inferences)\n",
    "                bleu_score, spacy_score = self.similarityScoreUtils.find_similarity_on_batch(\n",
    "                    batch_input_train[random.randint(0, len(batch_input_train)-1)], self.batch_size, self, beam = False)\n",
    "                print(\"For Non-Beam Question Generation, Bleu Score = {0} and Spacy Score = {1}\".format(bleu_score, spacy_score))\n",
    "                \n",
    "                inferences = self.generateQuestionsFromBatchWithBeam(batch_input_train, random.randint(0,len(batch_input_train)-1), 5, 5)\n",
    "                self.inferenceUtils.printInferences(inferences)\n",
    "                bleu_score, spacy_score = self.similarityScoreUtils.find_similarity_on_batch(\n",
    "                    batch_input_train[random.randint(0, len(batch_input_train)-1)], self.batch_size, self, beam = True, beam_num = 5)\n",
    "                print(\"For Beam Question Generation, Bleu Score = {0} and Spacy Score = {1}\".format(bleu_score, spacy_score))\n",
    "                \n",
    "            if generateTest:\n",
    "                print(\"Test Samples Generated : \")\n",
    "                inferences = self.generateQuestionsFromBatch(batch_input_test, random.randint(0,len(batch_input_test)-1), 5)\n",
    "                self.inferenceUtils.printInferences(inferences)\n",
    "                bleu_score, spacy_score = self.similarityScoreUtils.find_similarity_on_batch(\n",
    "                    batch_input_test[random.randint(0, len(batch_input_test)-1)], self.batch_size, self, beam = False)\n",
    "                print(\"For Non-Beam Question Generation, Bleu Score = {0} and Spacy Score = {1}\".format(bleu_score, spacy_score))\n",
    "                \n",
    "                inferences = self.generateQuestionsFromBatchWithBeam(batch_input_test, random.randint(0,len(batch_input_test)-1), 5, 5)\n",
    "                self.inferenceUtils.printInferences(inferences)\n",
    "                bleu_score, spacy_score = self.similarityScoreUtils.find_similarity_on_batch(\n",
    "                    batch_input_test[random.randint(0, len(batch_input_test)-1)], self.batch_size, self, beam = True, beam_num = 5)\n",
    "                print(\"For Beam Question Generation, Bleu Score = {0} and Spacy Score = {1}\".format(bleu_score, spacy_score))\n",
    "                \n",
    "            if epoch % save_after_every_epoch == 0:\n",
    "                self.saveSession(path_to_save_file)\n",
    "    \n",
    "            \n",
    "    def generateAnswers(self, passage):\n",
    "        p_batch, p_len_batch = self.inferenceUtils.createTokens(passage)\n",
    "        answers = self.sess.run(qgen.answer_tags, {\n",
    "            qgen.d_tokens: p_batch,\n",
    "            qgen.d_lengths: p_len_batch,\n",
    "        })\n",
    "        answers = np.argmax(answers[0], 1)\n",
    "        return answers\n",
    "        \n",
    "        \n",
    "    def generateQuestionsFromBatch(self, batch, batchNum, numExamples, shuffle = True, generateGroundTruthQuestion = True, generateScores = True):\n",
    "        \n",
    "        inferences = []\n",
    "        \n",
    "        if shuffle:\n",
    "            p = random.sample(range(0, self.batch_size), self.batch_size)\n",
    "        else:\n",
    "            p = range(0,self.batch_size)\n",
    "        j = 0\n",
    "        questions = self.sess.run(self.decoder_outputs, {\n",
    "            self.d_tokens: batch[batchNum]['document_tokens'][p],\n",
    "            self.d_lengths: batch[batchNum]['document_lengths'][p],\n",
    "            self.a_labels: batch[batchNum]['answer_labels'][p],\n",
    "            self.answer_encoder_input_mask: batch[batchNum]['answer_mask'][p],\n",
    "            self.answer_encoder_lengths: batch[batchNum]['answer_lengths'][p],\n",
    "        })\n",
    "        p = p[:numExamples]\n",
    "\n",
    "        qs = np.argmax(questions, 2)\n",
    "        for i in p:\n",
    "            inference = {\n",
    "                'passage' : '',\n",
    "                'ground_truth_question' : '',\n",
    "                'ground_truth_answer' : '',\n",
    "                'generated_questions' : [],\n",
    "                'bleu_score' : [],\n",
    "                'spacy_score' : []\n",
    "            }\n",
    "            question = itertools.takewhile(lambda t: t != END_TOKEN, qs[j])\n",
    "            inference['passage'] = self.helperFunctions.getDoc(batch, batchNum , i)\n",
    "            if generateGroundTruthQuestion:\n",
    "                inference['ground_truth_question'] = self.helperFunctions.getQues(batch, batchNum, i)\n",
    "            inference['ground_truth_answer'] = self.helperFunctions.getAns(batch, batchNum, i)\n",
    "            generated_question = \" \".join(look_up_token_reduced(token) for token in question)\n",
    "            inference['generated_questions'] = [generated_question]\n",
    "            if generateScores :\n",
    "                inference['bleu_score'] = [self.similarityScoreUtils.bleu_score(inference['ground_truth_question'], generated_question)]\n",
    "                inference['spacy_score'] = [self.similarityScoreUtils.spacy_similarity_score(inference['ground_truth_question'], generated_question)]\n",
    "            j=j+1\n",
    "            inferences.append(inference)\n",
    "\n",
    "        return inferences\n",
    "        \n",
    "        \n",
    "    \n",
    "    def generateQuestionsFromBatchWithBeam(self, batch, batchNum, numExamples, numQuestions, shuffle = True, generateGroundTruthQuestion = True, generateScores = True):\n",
    "        \n",
    "        inferences = []\n",
    "        if shuffle:\n",
    "            p = random.sample(range(0, self.batch_size), self.batch_size)\n",
    "        else:\n",
    "            p = range(0,self.batch_size)\n",
    "        j = 0\n",
    "        questions = self.sess.run(self.predicting_question_ids, {\n",
    "            self.d_tokens: batch[batchNum]['document_tokens'][p],\n",
    "            self.d_lengths: batch[batchNum]['document_lengths'][p],\n",
    "            self.a_labels: batch[batchNum]['answer_labels'][p],\n",
    "            self.answer_encoder_input_mask: batch[batchNum]['answer_mask'][p],\n",
    "            self.answer_encoder_lengths: batch[batchNum]['answer_lengths'][p],\n",
    "        })\n",
    "        p = p[:numExamples]\n",
    "        for i in p:\n",
    "            inference = {\n",
    "                'passage' : '',\n",
    "                'ground_truth_question' : '',\n",
    "                'ground_truth_answer' : '',\n",
    "                'generated_questions' : [],\n",
    "                'bleu_score' : [],\n",
    "                'spacy_score' : []\n",
    "            }\n",
    "            inference['passage'] = self.helperFunctions.getDoc(batch, batchNum ,i)\n",
    "            if generateGroundTruthQuestion:\n",
    "                inference['ground_truth_question'] = self.helperFunctions.getQues(batch, batchNum, i)\n",
    "            inference['ground_truth_answer'] = self.helperFunctions.getAns(batch, batchNum, i)\n",
    "            generated_questions = []\n",
    "            bleu_scores = []\n",
    "            spacy_scores = []\n",
    "            for k in range(numQuestions):\n",
    "                question = itertools.takewhile(lambda t: t != END_TOKEN, questions[j,:,k])\n",
    "                generated_question = \" \".join(look_up_token_reduced(token) for token in question)\n",
    "                generated_questions.append(generated_question)\n",
    "                if generateScores :\n",
    "                    bleu_scores.append(self.similarityScoreUtils.bleu_score(inference['ground_truth_question'], generated_question))\n",
    "                    spacy_scores.append(self.similarityScoreUtils.spacy_similarity_score(inference['ground_truth_question'], generated_question))\n",
    "                \n",
    "            j=j+1\n",
    "            if generateScores :\n",
    "                inference['bleu_score'] = bleu_scores\n",
    "                inference['spacy_score'] = spacy_scores\n",
    "\n",
    "            inference['generated_questions'] = generated_questions\n",
    "            inferences.append(inference)\n",
    "        return inferences"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/data/ra2630/miniconda3/envs/qgen/lib/python3.6/site-packages/tensorflow/python/ops/gradients_impl.py:95: UserWarning: Converting sparse IndexedSlices to a dense Tensor of unknown shape. This may consume a large amount of memory.\n",
      "  \"Converting sparse IndexedSlices to a dense Tensor of unknown shape. \"\n"
     ]
    }
   ],
   "source": [
    "tf.reset_default_graph()\n",
    "config = tf.ConfigProto()\n",
    "config.gpu_options.allow_growth = True\n",
    "session = tf.Session(config=config)\n",
    "\n",
    "qgen = QGen(learning_rate = 3e-4,\n",
    "            cell_size = 500,\n",
    "            n_layers = 1,\n",
    "            reduced_glove = reduced_glove,\n",
    "            batch_size = batch_size,\n",
    "            attention_type = 'Bahdanau',\n",
    "            cell_type = 'LSTM',\n",
    "            sess=session,\n",
    "            grad_clip=1.,\n",
    "            beam_width=10,\n",
    "            force_teaching_ratio=0.5,\n",
    "            dropout_probability = [1.0, 1.0, 1.0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "sys.stdout = open('/data/ra2630/tfLog72k-4', 'a', 1)\n",
    "qgen.fit(\n",
    "    batch_input_train, \n",
    "    batch_input_test, \n",
    "    n_epoch=15,\n",
    "    path_to_save_file = '/data/ra2630/tf72k-4',\n",
    "    save_after_every_epoch = 3,\n",
    "    generateTrain = True, \n",
    "    generateTest = True,\n",
    "    clearFile=False, \n",
    "    resumeTraining = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loading Session from file  /data/ra2630/tf72k-2\n",
      "INFO:tensorflow:Restoring parameters from /data/ra2630/tf72k-2\n",
      "Session restored !\n"
     ]
    }
   ],
   "source": [
    "qgen.loadFromSession(\"/data/ra2630/tf72k-2\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 106,
   "metadata": {},
   "outputs": [],
   "source": [
    "inferenceUtils = InferenceUtils(batch_size=batch_size, qgen=qgen)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "inferenceUtils.printInferences(qgen.generateQuestionsFromBatch(\n",
    "    batch = batch_input_train, \n",
    "    batchNum = 2, \n",
    "    numExamples = 5, \n",
    "    shuffle = True\n",
    "))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "inferenceUtils.printInferences(qgen.generateQuestionsFromBatchWithBeam(\n",
    "            batch = batch_input_test, \n",
    "            batchNum = 0, \n",
    "            numExamples = 32, \n",
    "            numQuestions = 10, \n",
    "            shuffle = True,\n",
    "            generateGroundTruthQuestion = True,\n",
    "            generateScores = True\n",
    "))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Passage Length =  52 , Answer Length =  3\n",
      "NumQues =  10\n",
      "----------------------------------------------------------------------------------------------------\n",
      "Comprehension : \n",
      "india , officially the republic of india ( iast : <UNK> <UNK> ) , [ e ] is a country in south asia . it is the seventh-largest country by area , the second-most populous country ( with over 1.2 billion people ) , and the most populous democracy in the world\n",
      "*****************************************************************************************************\n",
      "Ground Truth Answer: \n",
      "1.2 billion people \n",
      "*****************************************************************************************************\n",
      "Generated Question Number :   1\n",
      "how many people live in india ?\n",
      "*****************************************************************************************************\n",
      "Generated Question Number :   2\n",
      "how many people does india have ?\n",
      "*****************************************************************************************************\n",
      "Generated Question Number :   3\n",
      "how many people does india have ?\n",
      "*****************************************************************************************************\n",
      "Generated Question Number :   4\n",
      "how many people live in india in india ?\n",
      "*****************************************************************************************************\n",
      "Generated Question Number :   5\n",
      "how many people are in india ?\n",
      "*****************************************************************************************************\n",
      "Generated Question Number :   6\n",
      "how many people in india in india ?\n",
      "*****************************************************************************************************\n",
      "Generated Question Number :   7\n",
      "how many people are in india ?\n",
      "*****************************************************************************************************\n",
      "Generated Question Number :   8\n",
      "how many people live in india ? india ?\n",
      "*****************************************************************************************************\n",
      "Generated Question Number :   9\n",
      "how many people are the country of india ?\n",
      "*****************************************************************************************************\n",
      "Generated Question Number :   10\n",
      "how many people are in the country ?\n",
      "*****************************************************************************************************\n",
      "----------------------------------------------------------------------------------------------------\n",
      "[{'passage': 'india , officially the republic of india ( iast : <UNK> <UNK> ) , [ e ] is a country in south asia . it is the seventh-largest country by area , the second-most populous country ( with over 1.2 billion people ) , and the most populous democracy in the world', 'ground_truth_question': '', 'ground_truth_answer': '1.2 billion people ', 'generated_questions': ['how many people live in india ?', 'how many people does india have ?', 'how many people does india have ?', 'how many people live in india in india ?', 'how many people are in india ?', 'how many people in india in india ?', 'how many people are in india ?', 'how many people live in india ? india ?', 'how many people are the country of india ?', 'how many people are in the country ?'], 'bleu_score': [], 'spacy_score': []}]\n"
     ]
    }
   ],
   "source": [
    "inferenceUtils.makeInferenceOnText(\n",
    "    passage = \"India, officially the Republic of India (IAST: Bhārat Gaṇarājya),[e] is a country in South Asia. It is the seventh-largest country by area, the second-most populous country (with over 1.2 billion people), and the most populous democracy in the world\",\n",
    "    answer = '1.2 billion people',\n",
    "    num_questions = 10,\n",
    "    use_beam = True)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 112,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Answers =  [0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 1 0 0 0 0 0 0 0 1 0 0 0 0 0 0 1\n",
      " 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1\n",
      " 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1]\n",
      "[20 21]\n",
      "Passage Length =  92 , Answer Length =  2\n",
      "----------------------------------------------------------------------------------------------------\n",
      "Comprehension : \n",
      "new york university ( nyu ) is a private nonprofit research university based in new york city . founded in 1831 , nyu 's main campus is centered in manhattan , located with its core in greenwich village , and campuses based throughout new york city . nyu is also a worldwide university , operating nyu abu dhabi and nyu shanghai , and centers in accra , berlin , buenos aires , florence , london , madrid , paris , prague , sydney , tel aviv , and washington , d.c .\n",
      "*****************************************************************************************************\n",
      "Ground Truth Answer: \n",
      "1831 , \n",
      "*****************************************************************************************************\n",
      "Generated Question Number :   1\n",
      "when is new york university founded ?\n",
      "*****************************************************************************************************\n",
      "Generated Question Number :   2\n",
      "when was new york university founded ?\n",
      "*****************************************************************************************************\n",
      "Generated Question Number :   3\n",
      "when is new york city founded ?\n",
      "*****************************************************************************************************\n",
      "Generated Question Number :   4\n",
      "when did new york university founded ?\n",
      "*****************************************************************************************************\n",
      "Generated Question Number :   5\n",
      "when was new york city founded ?\n",
      "*****************************************************************************************************\n",
      "Generated Question Number :   6\n",
      "when did new york city founded ?\n",
      "*****************************************************************************************************\n",
      "Generated Question Number :   7\n",
      "when was new york founded ?\n",
      "*****************************************************************************************************\n",
      "Generated Question Number :   8\n",
      "when is new york founded ?\n",
      "*****************************************************************************************************\n",
      "Generated Question Number :   9\n",
      "when is new york university founded founded ?\n",
      "*****************************************************************************************************\n",
      "Generated Question Number :   10\n",
      "when was new york university founded founded ?\n",
      "*****************************************************************************************************\n",
      "----------------------------------------------------------------------------------------------------\n",
      "[29]\n",
      "Passage Length =  92 , Answer Length =  1\n",
      "----------------------------------------------------------------------------------------------------\n",
      "Comprehension : \n",
      "new york university ( nyu ) is a private nonprofit research university based in new york city . founded in 1831 , nyu 's main campus is centered in manhattan , located with its core in greenwich village , and campuses based throughout new york city . nyu is also a worldwide university , operating nyu abu dhabi and nyu shanghai , and centers in accra , berlin , buenos aires , florence , london , madrid , paris , prague , sydney , tel aviv , and washington , d.c .\n",
      "*****************************************************************************************************\n",
      "Ground Truth Answer: \n",
      "manhattan \n",
      "*****************************************************************************************************\n",
      "Generated Question Number :   1\n",
      "in what borough is the university located ?\n",
      "*****************************************************************************************************\n",
      "Generated Question Number :   2\n",
      "in what borough is new york city located ?\n",
      "*****************************************************************************************************\n",
      "Generated Question Number :   3\n",
      "in what borough is the world university located ?\n",
      "*****************************************************************************************************\n",
      "Generated Question Number :   4\n",
      "in what city is the university located ?\n",
      "*****************************************************************************************************\n",
      "Generated Question Number :   5\n",
      "in what city is the university located ?\n",
      "*****************************************************************************************************\n",
      "Generated Question Number :   6\n",
      "in what borough is the university located located ?\n",
      "*****************************************************************************************************\n",
      "Generated Question Number :   7\n",
      "in what borough is the university located located ?\n",
      "*****************************************************************************************************\n",
      "Generated Question Number :   8\n",
      "in what borough is the city located ?\n",
      "*****************************************************************************************************\n",
      "Generated Question Number :   9\n",
      "in what borough is the city located ?\n",
      "*****************************************************************************************************\n",
      "Generated Question Number :   10\n",
      "in what borough is the world university located located ?\n",
      "*****************************************************************************************************\n",
      "----------------------------------------------------------------------------------------------------\n",
      "[36 37]\n",
      "Passage Length =  92 , Answer Length =  2\n",
      "----------------------------------------------------------------------------------------------------\n",
      "Comprehension : \n",
      "new york university ( nyu ) is a private nonprofit research university based in new york city . founded in 1831 , nyu 's main campus is centered in manhattan , located with its core in greenwich village , and campuses based throughout new york city . nyu is also a worldwide university , operating nyu abu dhabi and nyu shanghai , and centers in accra , berlin , buenos aires , florence , london , madrid , paris , prague , sydney , tel aviv , and washington , d.c .\n",
      "*****************************************************************************************************\n",
      "Ground Truth Answer: \n",
      "greenwich village \n",
      "*****************************************************************************************************\n",
      "Generated Question Number :   1\n",
      "in what city is new york university located ?\n",
      "*****************************************************************************************************\n",
      "Generated Question Number :   2\n",
      "in what city is new york city located ?\n",
      "*****************************************************************************************************\n",
      "Generated Question Number :   3\n",
      "in what city is the university located ?\n",
      "*****************************************************************************************************\n",
      "Generated Question Number :   4\n",
      "in what city is the university located ?\n",
      "*****************************************************************************************************\n",
      "Generated Question Number :   5\n",
      "in what village is new york university located ?\n",
      "*****************************************************************************************************\n",
      "Generated Question Number :   6\n",
      "in what village is new york city located ?\n",
      "*****************************************************************************************************\n",
      "Generated Question Number :   7\n",
      "in what city is new york located ?\n",
      "*****************************************************************************************************\n",
      "Generated Question Number :   8\n",
      "in what village is the new york university located ?\n",
      "*****************************************************************************************************\n",
      "Generated Question Number :   9\n",
      "in what city is new york located ?\n",
      "*****************************************************************************************************\n",
      "Generated Question Number :   10\n",
      "in what city is the new york city located ?\n",
      "*****************************************************************************************************\n",
      "----------------------------------------------------------------------------------------------------\n",
      "[52 53 54 55]\n",
      "Passage Length =  92 , Answer Length =  4\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "----------------------------------------------------------------------------------------------------\n",
      "Comprehension : \n",
      "new york university ( nyu ) is a private nonprofit research university based in new york city . founded in 1831 , nyu 's main campus is centered in manhattan , located with its core in greenwich village , and campuses based throughout new york city . nyu is also a worldwide university , operating nyu abu dhabi and nyu shanghai , and centers in accra , berlin , buenos aires , florence , london , madrid , paris , prague , sydney , tel aviv , and washington , d.c .\n",
      "*****************************************************************************************************\n",
      "Ground Truth Answer: \n",
      "university , operating nyu \n",
      "*****************************************************************************************************\n",
      "Generated Question Number :   1\n",
      "what is the name of the new york university ?\n",
      "*****************************************************************************************************\n",
      "Generated Question Number :   2\n",
      "what is the name of the new york university in new york city ?\n",
      "*****************************************************************************************************\n",
      "Generated Question Number :   3\n",
      "what is the name of the new york university ?\n",
      "*****************************************************************************************************\n",
      "Generated Question Number :   4\n",
      "what is the name of the city that is located in new york city ?\n",
      "*****************************************************************************************************\n",
      "Generated Question Number :   5\n",
      "what is the name of the private that of new york city ?\n",
      "*****************************************************************************************************\n",
      "Generated Question Number :   6\n",
      "what is the name of the new york university ?\n",
      "*****************************************************************************************************\n",
      "Generated Question Number :   7\n",
      "what is the name of the new york university ?\n",
      "*****************************************************************************************************\n",
      "Generated Question Number :   8\n",
      "what is the name of the new york university ?\n",
      "*****************************************************************************************************\n",
      "Generated Question Number :   9\n",
      "what is the name of the new york university in new york ?\n",
      "*****************************************************************************************************\n",
      "Generated Question Number :   10\n",
      "what is the name of the new york university in new york city\n",
      "*****************************************************************************************************\n",
      "----------------------------------------------------------------------------------------------------\n",
      "[65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88\n",
      " 89 90 91]\n",
      "Passage Length =  92 , Answer Length =  27\n",
      "----------------------------------------------------------------------------------------------------\n",
      "Comprehension : \n",
      "new york university ( nyu ) is a private nonprofit research university based in new york city . founded in 1831 , nyu 's main campus is centered in manhattan , located with its core in greenwich village , and campuses based throughout new york city . nyu is also a worldwide university , operating nyu abu dhabi and nyu shanghai , and centers in accra , berlin , buenos aires , florence , london , madrid , paris , prague , sydney , tel aviv , and washington , d.c .\n",
      "*****************************************************************************************************\n",
      "Ground Truth Answer: \n",
      "accra , berlin , buenos aires , florence , london , madrid , paris , prague , sydney , tel aviv , and washington , d.c . \n",
      "*****************************************************************************************************\n",
      "Generated Question Number :   1\n",
      "in which locations is new york city located ?\n",
      "*****************************************************************************************************\n",
      "Generated Question Number :   2\n",
      "in what locations is new york city located ?\n",
      "*****************************************************************************************************\n",
      "Generated Question Number :   3\n",
      "in which locations is new york university located ?\n",
      "*****************************************************************************************************\n",
      "Generated Question Number :   4\n",
      "in what locations is new york university located ?\n",
      "*****************************************************************************************************\n",
      "Generated Question Number :   5\n",
      "in which locations is new york city ?\n",
      "*****************************************************************************************************\n",
      "Generated Question Number :   6\n",
      "in what locations is new york city ?\n",
      "*****************************************************************************************************\n",
      "Generated Question Number :   7\n",
      "what are the locations of new york city ?\n",
      "*****************************************************************************************************\n",
      "Generated Question Number :   8\n",
      "in what locations is new york city ?\n",
      "*****************************************************************************************************\n",
      "Generated Question Number :   9\n",
      "in which locations is new york city located in\n",
      "*****************************************************************************************************\n",
      "Generated Question Number :   10\n",
      "in what locations of new york city is located in new york city ?\n",
      "*****************************************************************************************************\n",
      "----------------------------------------------------------------------------------------------------\n"
     ]
    }
   ],
   "source": [
    "inferenceUtils.makeInferenceOnText(\n",
    "    passage = \"New York University (NYU) is a private nonprofit research university based in New York City. Founded in 1831, NYU's main campus is centered in Manhattan, located with its core in Greenwich Village, and campuses based throughout New York City. NYU is also a worldwide university, operating NYU Abu Dhabi and NYU Shanghai, and centers in Accra, Berlin, Buenos Aires, Florence, London, Madrid, Paris, Prague, Sydney, Tel Aviv, and Washington, D.C.\" , \n",
    "    use_beam = True, \n",
    "    num_questions = 10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 113,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Passage Length =  92 , Answer Length =  4\n",
      "----------------------------------------------------------------------------------------------------\n",
      "Comprehension : \n",
      "new york university ( nyu ) is a private nonprofit research university based in new york city . founded in 1831 , nyu 's main campus is centered in manhattan , located with its core in greenwich village , and campuses based throughout new york city . nyu is also a worldwide university , operating nyu abu dhabi and nyu shanghai , and centers in accra , berlin , buenos aires , florence , london , madrid , paris , prague , sydney , tel aviv , and washington , d.c .\n",
      "*****************************************************************************************************\n",
      "Ground Truth Answer: \n",
      "private nonprofit research university \n",
      "*****************************************************************************************************\n",
      "Generated Question Number :   1\n",
      "what type of university is new york university ?\n",
      "*****************************************************************************************************\n",
      "Generated Question Number :   2\n",
      "what type of university is new york city ?\n",
      "*****************************************************************************************************\n",
      "Generated Question Number :   3\n",
      "what type of university does new york have ?\n",
      "*****************************************************************************************************\n",
      "Generated Question Number :   4\n",
      "what type of university is new york university located ?\n",
      "*****************************************************************************************************\n",
      "Generated Question Number :   5\n",
      "what type of university does new york have ?\n",
      "*****************************************************************************************************\n",
      "----------------------------------------------------------------------------------------------------\n"
     ]
    }
   ],
   "source": [
    "inferenceUtils.makeInferenceOnText(\n",
    "    passage = \"New York University (NYU) is a private nonprofit research university based in New York City. Founded in 1831, NYU's main campus is centered in Manhattan, located with its core in Greenwich Village, and campuses based throughout New York City. NYU is also a worldwide university, operating NYU Abu Dhabi and NYU Shanghai, and centers in Accra, Berlin, Buenos Aires, Florence, London, Madrid, Paris, Prague, Sydney, Tel Aviv, and Washington, D.C.\" , \n",
    "    answer = 'private nonprofit research university' ,\n",
    "    use_beam = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 114,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Answers =  [0 0 0 0 0 0 0 0 0 0 0 0 0 1 0 0 0 0 0 0 1 1 1 0 0 0 1 1 1 1 1 1 1 1 1 0 0\n",
      " 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0]\n",
      "[13]\n",
      "Passage Length =  61 , Answer Length =  1\n",
      "----------------------------------------------------------------------------------------------------\n",
      "Comprehension : \n",
      "the election of president and vice president of the united states is an indirect election in which citizens of the united states who are registered to vote in one of the 50 u.s. states or in washington , d.c. cast ballots not directly for those offices , but instead for members of the u.s. electoral college , known as electors .\n",
      "*****************************************************************************************************\n",
      "Ground Truth Answer: \n",
      "indirect \n",
      "*****************************************************************************************************\n",
      "Generated Question Number :   1\n",
      "what type of election is the united states ?\n",
      "*****************************************************************************************************\n",
      "Generated Question Number :   2\n",
      "what type of election is the us ?\n",
      "*****************************************************************************************************\n",
      "Generated Question Number :   3\n",
      "what type of election of the united states ?\n",
      "*****************************************************************************************************\n",
      "Generated Question Number :   4\n",
      "what type of election does the united states have ?\n",
      "*****************************************************************************************************\n",
      "Generated Question Number :   5\n",
      "what type of election of the united states ?\n",
      "*****************************************************************************************************\n",
      "Generated Question Number :   6\n",
      "what type of election does the united states ?\n",
      "*****************************************************************************************************\n",
      "Generated Question Number :   7\n",
      "what type of election is the us ?\n",
      "*****************************************************************************************************\n",
      "Generated Question Number :   8\n",
      "what type of election does the united states serve ?\n",
      "*****************************************************************************************************\n",
      "Generated Question Number :   9\n",
      "what type of election is the united states of the us states ?\n",
      "*****************************************************************************************************\n",
      "Generated Question Number :   10\n",
      "what type of election does the united states ?\n",
      "*****************************************************************************************************\n",
      "----------------------------------------------------------------------------------------------------\n",
      "[20 21 22]\n",
      "Passage Length =  61 , Answer Length =  3\n",
      "----------------------------------------------------------------------------------------------------\n",
      "Comprehension : \n",
      "the election of president and vice president of the united states is an indirect election in which citizens of the united states who are registered to vote in one of the 50 u.s. states or in washington , d.c. cast ballots not directly for those offices , but instead for members of the u.s. electoral college , known as electors .\n",
      "*****************************************************************************************************\n",
      "Ground Truth Answer: \n",
      "united states who \n",
      "*****************************************************************************************************\n",
      "Generated Question Number :   1\n",
      "who is the us electoral college ?\n",
      "*****************************************************************************************************\n",
      "Generated Question Number :   2\n",
      "who is the election of the united states ?\n",
      "*****************************************************************************************************\n",
      "Generated Question Number :   3\n",
      "who is the us electoral college ?\n",
      "*****************************************************************************************************\n",
      "Generated Question Number :   4\n",
      "who is the candidate of the united states ?\n",
      "*****************************************************************************************************\n",
      "Generated Question Number :   5\n",
      "who is the election of the united states ?\n",
      "*****************************************************************************************************\n",
      "Generated Question Number :   6\n",
      "who is the candidate of the united states ?\n",
      "*****************************************************************************************************\n",
      "Generated Question Number :   7\n",
      "who is the president of the united states ?\n",
      "*****************************************************************************************************\n",
      "Generated Question Number :   8\n",
      "who is responsible for the election of the us states ?\n",
      "*****************************************************************************************************\n",
      "Generated Question Number :   9\n",
      "who is the the election of the united states ? ?\n",
      "*****************************************************************************************************\n",
      "Generated Question Number :   10\n",
      "who is the us electoral college ?\n",
      "*****************************************************************************************************\n",
      "----------------------------------------------------------------------------------------------------\n",
      "[26 27 28 29 30 31 32 33 34]\n",
      "Passage Length =  61 , Answer Length =  9\n",
      "----------------------------------------------------------------------------------------------------\n",
      "Comprehension : \n",
      "the election of president and vice president of the united states is an indirect election in which citizens of the united states who are registered to vote in one of the 50 u.s. states or in washington , d.c. cast ballots not directly for those offices , but instead for members of the u.s. electoral college , known as electors .\n",
      "*****************************************************************************************************\n",
      "Ground Truth Answer: \n",
      "vote in one of the 50 u.s. states or \n",
      "*****************************************************************************************************\n",
      "Generated Question Number :   1\n",
      "what is the presidential election of the united states ?\n",
      "*****************************************************************************************************\n",
      "Generated Question Number :   2\n",
      "what is the state of the united states ?\n",
      "*****************************************************************************************************\n",
      "Generated Question Number :   3\n",
      "what is the state of the us electoral college ?\n",
      "*****************************************************************************************************\n",
      "Generated Question Number :   4\n",
      "what is the presidential election of the us states ?\n",
      "*****************************************************************************************************\n",
      "Generated Question Number :   5\n",
      "what is the presidential election of the us ?\n",
      "*****************************************************************************************************\n",
      "Generated Question Number :   6\n",
      "what is the state of the united states ?\n",
      "*****************************************************************************************************\n",
      "Generated Question Number :   7\n",
      "what is the electoral president of the united states ?\n",
      "*****************************************************************************************************\n",
      "Generated Question Number :   8\n",
      "what is the presidential election of the us ?\n",
      "*****************************************************************************************************\n",
      "Generated Question Number :   9\n",
      "what is the presidential election of the us states ? ?\n",
      "*****************************************************************************************************\n",
      "Generated Question Number :   10\n",
      "what is the presidential election of the united states in the us ?\n",
      "*****************************************************************************************************\n",
      "----------------------------------------------------------------------------------------------------\n"
     ]
    }
   ],
   "source": [
    "inferenceUtils.makeInferenceOnText(\n",
    "    passage = 'The election of President and Vice President of the United States is an indirect election in which citizens of the United States who are registered to vote in one of the 50 U.S. states or in Washington, D.C. cast ballots not directly for those offices, but instead for members of the U.S. Electoral College, known as electors.' , \n",
    "    num_questions = 10 , \n",
    "    use_beam = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 119,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Passage Length =  61 , Answer Length =  1\n",
      "----------------------------------------------------------------------------------------------------\n",
      "Comprehension : \n",
      "the election of president and vice president of the united states is an indirect election in which citizens of the united states who are registered to vote in one of the 50 u.s. states or in washington , d.c. cast ballots not directly for those offices , but instead for members of the u.s. electoral college , known as electors .\n",
      "*****************************************************************************************************\n",
      "Ground Truth Answer: \n",
      "electors \n",
      "*****************************************************************************************************\n",
      "Generated Question Number :   1\n",
      "what is the term of the united states ?\n",
      "*****************************************************************************************************\n",
      "Generated Question Number :   2\n",
      "what is the term for the united states ?\n",
      "*****************************************************************************************************\n",
      "Generated Question Number :   3\n",
      "what is the name of the united states ?\n",
      "*****************************************************************************************************\n",
      "Generated Question Number :   4\n",
      "what is the term for the us electoral college ?\n",
      "*****************************************************************************************************\n",
      "Generated Question Number :   5\n",
      "what is the term of the u.s. electoral college ?\n",
      "*****************************************************************************************************\n",
      "Generated Question Number :   6\n",
      "what is the term of the us electoral college ?\n",
      "*****************************************************************************************************\n",
      "Generated Question Number :   7\n",
      "what is the name of the united states ?\n",
      "*****************************************************************************************************\n",
      "Generated Question Number :   8\n",
      "what is the term of the united states in the us states ?\n",
      "*****************************************************************************************************\n",
      "Generated Question Number :   9\n",
      "what is the name of the united states in the us states ?\n",
      "*****************************************************************************************************\n",
      "Generated Question Number :   10\n",
      "what is the term for the united states in the us states ?\n",
      "*****************************************************************************************************\n",
      "----------------------------------------------------------------------------------------------------\n"
     ]
    }
   ],
   "source": [
    "inferenceUtils.makeInferenceOnText(\n",
    "    passage = 'The election of President and Vice President of the United States is an indirect election in which citizens of the United States who are registered to vote in one of the 50 U.S. states or in Washington, D.C. cast ballots not directly for those offices, but instead for members of the U.S. Electoral College, known as electors.' , \n",
    "    num_questions = 10 , \n",
    "    answer = 'electors' ,\n",
    "    use_beam = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 120,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Passage Length =  61 , Answer Length =  1\n",
      "----------------------------------------------------------------------------------------------------\n",
      "Comprehension : \n",
      "the election of president and vice president of the united states is an indirect election in which citizens of the united states who are registered to vote in one of the 50 u.s. states or in washington , d.c. cast ballots not directly for those offices , but instead for members of the u.s. electoral college , known as electors .\n",
      "*****************************************************************************************************\n",
      "Ground Truth Answer: \n",
      "50 \n",
      "*****************************************************************************************************\n",
      "Generated Question Number :   1\n",
      "how many us states are there ?\n",
      "*****************************************************************************************************\n",
      "Generated Question Number :   2\n",
      "how many us states are in the united states ?\n",
      "*****************************************************************************************************\n",
      "Generated Question Number :   3\n",
      "how many us states are in the us ?\n",
      "*****************************************************************************************************\n",
      "Generated Question Number :   4\n",
      "how many us states are there ?\n",
      "*****************************************************************************************************\n",
      "Generated Question Number :   5\n",
      "how many us states are there in the us ?\n",
      "*****************************************************************************************************\n",
      "Generated Question Number :   6\n",
      "how many us states are there in the us ?\n",
      "*****************************************************************************************************\n",
      "Generated Question Number :   7\n",
      "how many us states are in the us ?\n",
      "*****************************************************************************************************\n",
      "Generated Question Number :   8\n",
      "how many us states are the united states ?\n",
      "*****************************************************************************************************\n",
      "Generated Question Number :   9\n",
      "how many us states does the united states have ?\n",
      "*****************************************************************************************************\n",
      "Generated Question Number :   10\n",
      "how many us states are there ?\n",
      "*****************************************************************************************************\n",
      "----------------------------------------------------------------------------------------------------\n"
     ]
    }
   ],
   "source": [
    "inferenceUtils.makeInferenceOnText(\n",
    "    passage = 'The election of President and Vice President of the United States is an indirect election in which citizens of the United States who are registered to vote in one of the 50 U.S. states or in Washington, D.C. cast ballots not directly for those offices, but instead for members of the U.S. Electoral College, known as electors.' , \n",
    "    num_questions = 10 , \n",
    "    answer = '50' ,\n",
    "    use_beam = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 121,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Answers =  [0 0 0 0 0 0 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0]\n",
      "[6 7 8]\n",
      "Passage Length =  23 , Answer Length =  3\n",
      "----------------------------------------------------------------------------------------------------\n",
      "Comprehension : \n",
      "the largest of these is the <UNK> square shopping centre , one of the largest city centre shopping complexes in the uk .\n",
      "*****************************************************************************************************\n",
      "Ground Truth Answer: \n",
      "<UNK> square shopping \n",
      "*****************************************************************************************************\n",
      "Generated Question Number :   1\n",
      "what is the largest city centre shopping center ?\n",
      "*****************************************************************************************************\n",
      "Generated Question Number :   2\n",
      "what is the largest city centre shopping center ?\n",
      "*****************************************************************************************************\n",
      "Generated Question Number :   3\n",
      "what is the largest city centre shopping center ? the uk ?\n",
      "*****************************************************************************************************\n",
      "Generated Question Number :   4\n",
      "what is the largest city centre shopping centre ?\n",
      "*****************************************************************************************************\n",
      "Generated Question Number :   5\n",
      "what is the largest city centre shopping center center\n",
      "*****************************************************************************************************\n",
      "Generated Question Number :   6\n",
      "what is the largest city centre shopping center center\n",
      "*****************************************************************************************************\n",
      "Generated Question Number :   7\n",
      "which is the largest city centre shopping center ?\n",
      "*****************************************************************************************************\n",
      "Generated Question Number :   8\n",
      "which is the largest city centre shopping center in the uk ?\n",
      "*****************************************************************************************************\n",
      "Generated Question Number :   9\n",
      "which is the largest city centre shopping center ?\n",
      "*****************************************************************************************************\n",
      "Generated Question Number :   10\n",
      "which is the largest city centre shopping center ?\n",
      "*****************************************************************************************************\n",
      "----------------------------------------------------------------------------------------------------\n"
     ]
    }
   ],
   "source": [
    "inferenceUtils.makeInferenceOnText(\n",
    "    passage = 'the largest of these is the eldon square shopping centre , one of the largest city centre shopping complexes in the uk.', \n",
    "    num_questions = 10 , \n",
    "    use_beam = True)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 128,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Answers =  [1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      " 0 0 0 0 0 0 0 0 0 0 0]\n",
      "[0 1 2]\n",
      "Passage Length =  48 , Answer Length =  3\n",
      "----------------------------------------------------------------------------------------------------\n",
      "Comprehension : \n",
      "sam <UNK> is one of the leading researchers in the field of natural language processing ( <UNK> ) , and recently joined nyu as an assistant professor in computational linguistics , a joint position between nyu ’ s linguistics department , and the center for data science .\n",
      "*****************************************************************************************************\n",
      "Ground Truth Answer: \n",
      "sam <UNK> is \n",
      "*****************************************************************************************************\n",
      "Generated Question Number :   1\n",
      "who is one of the leading researchers in the field of natural language processing ?\n",
      "*****************************************************************************************************\n",
      "Generated Question Number :   2\n",
      "who is one of leading researchers in the field of natural language processing ?\n",
      "*****************************************************************************************************\n",
      "Generated Question Number :   3\n",
      "what is one of the leading researchers in the field of natural language processing ?\n",
      "*****************************************************************************************************\n",
      "Generated Question Number :   4\n",
      "what is one of leading researchers in the field of natural language processing ?\n",
      "*****************************************************************************************************\n",
      "Generated Question Number :   5\n",
      "who is one of leading researchers in the field of natural language processing\n",
      "*****************************************************************************************************\n",
      "Generated Question Number :   6\n",
      "who is one of the leading researchers in the field of natural ?\n",
      "*****************************************************************************************************\n",
      "Generated Question Number :   7\n",
      "who is one of the leading researchers in the field of natural ?\n",
      "*****************************************************************************************************\n",
      "Generated Question Number :   8\n",
      "who is one of the leading researchers in history ?\n",
      "*****************************************************************************************************\n",
      "Generated Question Number :   9\n",
      "what is one of leading researchers in the field of natural language processing\n",
      "*****************************************************************************************************\n",
      "Generated Question Number :   10\n",
      "who is one of the leading researchers in fields ?\n",
      "*****************************************************************************************************\n",
      "----------------------------------------------------------------------------------------------------\n",
      "[15 16 17]\n",
      "Passage Length =  48 , Answer Length =  3\n",
      "----------------------------------------------------------------------------------------------------\n",
      "Comprehension : \n",
      "sam <UNK> is one of the leading researchers in the field of natural language processing ( <UNK> ) , and recently joined nyu as an assistant professor in computational linguistics , a joint position between nyu ’ s linguistics department , and the center for data science .\n",
      "*****************************************************************************************************\n",
      "Ground Truth Answer: \n",
      "( <UNK> ) \n",
      "*****************************************************************************************************\n",
      "Generated Question Number :   1\n",
      "what is the name of the leading researchers in the field of natural language processing ?\n",
      "*****************************************************************************************************\n",
      "Generated Question Number :   2\n",
      "what is one of the leading researchers in the field of natural language processing ?\n",
      "*****************************************************************************************************\n",
      "Generated Question Number :   3\n",
      "what is the name of the leading researchers ?\n",
      "*****************************************************************************************************\n",
      "Generated Question Number :   4\n",
      "what is the name of the leading researchers ?\n",
      "*****************************************************************************************************\n",
      "Generated Question Number :   5\n",
      "what is the name of the leading researchers ?\n",
      "*****************************************************************************************************\n",
      "Generated Question Number :   6\n",
      "what is the name of the leading researchers in\n",
      "*****************************************************************************************************\n",
      "Generated Question Number :   7\n",
      "what is the name of the leading researchers in ?\n",
      "*****************************************************************************************************\n",
      "Generated Question Number :   8\n",
      "what is the name of the leading researchers in ?\n",
      "*****************************************************************************************************\n",
      "Generated Question Number :   9\n",
      "what is the name of the leading researchers ?\n",
      "*****************************************************************************************************\n",
      "Generated Question Number :   10\n",
      "what is the name of the leading researchers in\n",
      "*****************************************************************************************************\n",
      "----------------------------------------------------------------------------------------------------\n"
     ]
    }
   ],
   "source": [
    "inferenceUtils.makeInferenceOnText(\n",
    "    passage = 'Sam Bowman is one of the leading researchers in the field of natural language processing (NLP), and recently joined NYU as an Assistant Professor in Computational Linguistics, a joint position between NYU’s Linguistics department, and the Center for Data Science.', \n",
    "    num_questions = 10 , \n",
    "    use_beam = True)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 158,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Answers =  [1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      " 0 0 0 0 0 0 0 0 0 1 1 1]\n",
      "[0 1]\n",
      "Passage Length =  49 , Answer Length =  2\n",
      "----------------------------------------------------------------------------------------------------\n",
      "Comprehension : \n",
      "artificial neural networks are machine learning models : computational systems that are designed to learn how to perform some kind of behavior , given previous examples shown through data . this can include anything from labeling an image , to translating a text , to steering a robot .\n",
      "*****************************************************************************************************\n",
      "Ground Truth Answer: \n",
      "artificial neural \n",
      "*****************************************************************************************************\n",
      "Generated Question Number :   1\n",
      "what type of learning are machine learning ?\n",
      "*****************************************************************************************************\n",
      "Generated Question Number :   2\n",
      "what type of learning are machine learning ?\n",
      "*****************************************************************************************************\n",
      "Generated Question Number :   3\n",
      "what type of learning is machine learning ?\n",
      "*****************************************************************************************************\n",
      "Generated Question Number :   4\n",
      "what type of learning are machine learning ?\n",
      "*****************************************************************************************************\n",
      "Generated Question Number :   5\n",
      "what type of learning is machine learning ?\n",
      "*****************************************************************************************************\n",
      "----------------------------------------------------------------------------------------------------\n",
      "[46 47 48]\n",
      "Passage Length =  49 , Answer Length =  3\n",
      "----------------------------------------------------------------------------------------------------\n",
      "Comprehension : \n",
      "artificial neural networks are machine learning models : computational systems that are designed to learn how to perform some kind of behavior , given previous examples shown through data . this can include anything from labeling an image , to translating a text , to steering a robot .\n",
      "*****************************************************************************************************\n",
      "Ground Truth Answer: \n",
      "a robot . \n",
      "*****************************************************************************************************\n",
      "Generated Question Number :   1\n",
      "what can be used to organize a signal ?\n",
      "*****************************************************************************************************\n",
      "Generated Question Number :   2\n",
      "what can be used to signal a signal ?\n",
      "*****************************************************************************************************\n",
      "Generated Question Number :   3\n",
      "what can be used to organize a messages ?\n",
      "*****************************************************************************************************\n",
      "Generated Question Number :   4\n",
      "what can be used to organize a messages ?\n",
      "*****************************************************************************************************\n",
      "Generated Question Number :   5\n",
      "what can be used to organize a signal ? ?\n",
      "*****************************************************************************************************\n",
      "----------------------------------------------------------------------------------------------------\n"
     ]
    }
   ],
   "source": [
    "inferenceUtils.makeInferenceOnText(\n",
    "    passage = 'Artificial neural networks are machine learning models : computational systems that are designed to learn how to perform some kind of behavior, given previous examples shown through data.  This can include anything from labeling an image, to translating a text, to steering a robot.', \n",
    "    num_questions = 5 , \n",
    "    use_beam = True)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 204,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Passage Length =  92 , Answer Length =  1\n",
      "----------------------------------------------------------------------------------------------------\n",
      "Comprehension : \n",
      "chris , however , revealed he was told <UNK> would retain him , only to not hear from the franchise again . 'i was their biggest draw . it was disappointing from that end , because they had called me . they wanted me in the team and i was told that i will be retained , ' <UNK> told the times of india . <UNK> they never called back after that . so that gave me the impression that they did n't want me , and it 's fine . '\n",
      "*****************************************************************************************************\n",
      "Ground Truth Answer: \n",
      "chris \n",
      "*****************************************************************************************************\n",
      "Generated Question Number :   1\n",
      "who said that they would not be acted the times of india ?\n",
      "*****************************************************************************************************\n",
      "Generated Question Number :   2\n",
      "who said that they would not be acted ?\n",
      "*****************************************************************************************************\n",
      "Generated Question Number :   3\n",
      "who was the only person to be retained ?\n",
      "*****************************************************************************************************\n",
      "Generated Question Number :   4\n",
      "who was the only person to be retained ?\n",
      "*****************************************************************************************************\n",
      "Generated Question Number :   5\n",
      "who said that they would not be retained ?\n",
      "*****************************************************************************************************\n",
      "----------------------------------------------------------------------------------------------------\n"
     ]
    }
   ],
   "source": [
    "inferenceUtils.makeInferenceOnText(\n",
    "    passage = \"Chris, however, revealed he was told RCB would retain him, only to not hear from the franchise again. 'I was their biggest draw. It was disappointing from that end, because they had called me. They wanted me in the team and I was told that I will be retained,' Gayle told The Times of India. 'But they never called back after that. So that gave me the impression that they didn't want me, and it's fine.'\", \n",
    "    answer = 'Chris',\n",
    "    num_questions = 5 , \n",
    "    use_beam = True)\n",
    "\n",
    "\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 233,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Answers =  [1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      " 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 1 0 0]\n",
      "[0 1 2 3 4 5]\n",
      "Passage Length =  70 , Answer Length =  6\n",
      "----------------------------------------------------------------------------------------------------\n",
      "Comprehension : \n",
      "the american idol songwriter contest was also held this season . from ten of the most popular submissions , each of the final two contestants chose a song to perform , although neither of their selections was used as the `` coronation song '' . the winning song , `` the time of my life '' , was recorded by david cook and released on may 22 , 2008 .\n",
      "*****************************************************************************************************\n",
      "Ground Truth Answer: \n",
      "the american idol songwriter contest was \n",
      "*****************************************************************************************************\n",
      "Generated Question Number :   1\n",
      "did the idol of my life '' appear ?\n",
      "*****************************************************************************************************\n",
      "Generated Question Number :   2\n",
      "did the idol of my life '' cover ?\n",
      "*****************************************************************************************************\n",
      "Generated Question Number :   3\n",
      "why was the first of my american idol ?\n",
      "*****************************************************************************************************\n",
      "Generated Question Number :   4\n",
      "did the idol of my life '' ?\n",
      "*****************************************************************************************************\n",
      "Generated Question Number :   5\n",
      "did the idol of my life '' begin ?\n",
      "*****************************************************************************************************\n",
      "Generated Question Number :   6\n",
      "what was the first song of the american idol ?\n",
      "*****************************************************************************************************\n",
      "Generated Question Number :   7\n",
      "did the idol of my life '' appear ? ?\n",
      "*****************************************************************************************************\n",
      "Generated Question Number :   8\n",
      "did the idol of my life '' begin to perform ?\n",
      "*****************************************************************************************************\n",
      "Generated Question Number :   9\n",
      "what was the first song of the american idol ? ?\n",
      "*****************************************************************************************************\n",
      "Generated Question Number :   10\n",
      "did the idol of my life '' ?\n",
      "*****************************************************************************************************\n",
      "----------------------------------------------------------------------------------------------------\n",
      "[66 67]\n",
      "Passage Length =  70 , Answer Length =  2\n",
      "----------------------------------------------------------------------------------------------------\n",
      "Comprehension : \n",
      "the american idol songwriter contest was also held this season . from ten of the most popular submissions , each of the final two contestants chose a song to perform , although neither of their selections was used as the `` coronation song '' . the winning song , `` the time of my life '' , was recorded by david cook and released on may 22 , 2008 .\n",
      "*****************************************************************************************************\n",
      "Ground Truth Answer: \n",
      "22 , \n",
      "*****************************************************************************************************\n",
      "Generated Question Number :   1\n",
      "on what date was the idol of my life '' released ?\n",
      "*****************************************************************************************************\n",
      "Generated Question Number :   2\n",
      "how long was the time of my life '' released ?\n",
      "*****************************************************************************************************\n",
      "Generated Question Number :   3\n",
      "when was the first of my life '' released ?\n",
      "*****************************************************************************************************\n",
      "Generated Question Number :   4\n",
      "how long was the first of my life '' released ?\n",
      "*****************************************************************************************************\n",
      "Generated Question Number :   5\n",
      "how long did the idol of my life '' released ?\n",
      "*****************************************************************************************************\n",
      "Generated Question Number :   6\n",
      "when was the first of my life '' released ?\n",
      "*****************************************************************************************************\n",
      "Generated Question Number :   7\n",
      "how many did the idol of my life '' released ?\n",
      "*****************************************************************************************************\n",
      "Generated Question Number :   8\n",
      "when was the first of my my life '' released ?\n",
      "*****************************************************************************************************\n",
      "Generated Question Number :   9\n",
      "how many did the idol of my life '' appear ?\n",
      "*****************************************************************************************************\n",
      "Generated Question Number :   10\n",
      "on what date did the idol of my life '' appear ?\n",
      "*****************************************************************************************************\n",
      "----------------------------------------------------------------------------------------------------\n"
     ]
    }
   ],
   "source": [
    "inferenceUtils.makeInferenceOnText(\"the american idol songwriter contest was also held this season . from ten of the most popular submissions , each of the final two contestants chose a song to perform , although neither of their selections was used as the `` coronation song '' . the winning song , `` the time of my life '' , was recorded by david cook and released on may 22 , 2008 . \",\n",
    "    num_questions = 10 , \n",
    "    use_beam = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "sys.stdout = orig_stdout"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 232,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "10"
      ]
     },
     "execution_count": 232,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "10"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}