doc2vec-u.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Import all the Libraries needed"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import gensim\n",
    "import smart_open\n",
    "from gensim.models.doc2vec import Doc2Vec, TaggedDocument\n",
    "from pprint import pprint\n",
    "import multiprocessing\n",
    "from gensim.models.callbacks import CallbackAny2Vec\n",
    "\n",
    "import nltk\n",
    "from nltk.corpus import stopwords\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "doc_embedding_size = 400"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Clean the memory using garbage collector"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import gc\n",
    "gc.collect()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Get the total number of cores and see if its fast version of Gensim"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "cores = multiprocessing.cpu_count()\n",
    "assert gensim.models.doc2vec.FAST_VERSION > -1, \"This will be painfully slow otherwise\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Read the whole dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "stop_words = nltk.corpus.stopwords.words('english')\n",
    "def pre_process(line):\n",
    "    word_list = nltk.tokenize.word_tokenize(line)\n",
    "    word_list = [w.strip().lower() for w in word_list]\n",
    "    word_list = [w for w in word_list if w not in stop_words]\n",
    "    return word_list\n",
    "\n",
    "def read_corpus(fname, pre_processed=True):\n",
    "    with smart_open.smart_open(fname, encoding=\"iso-8859-1\") as f:\n",
    "        loop = 0\n",
    "        for i, line in enumerate(f):\n",
    "            if not pre_processed:\n",
    "                yield gensim.models.doc2vec.TaggedDocument(pre_process(line), [i])\n",
    "            else:\n",
    "                yield gensim.models.doc2vec.TaggedDocument(line, [i])\n",
    "\n",
    "            loop += 1\n",
    "            if loop % 10000 == 0:\n",
    "                print(loop)\n",
    "\n",
    "            if loop == 1000000:\n",
    "                break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "10000\n",
      "20000\n",
      "30000\n",
      "40000\n",
      "50000\n",
      "60000\n",
      "70000\n",
      "80000\n",
      "90000\n",
      "100000\n",
      "110000\n",
      "120000\n",
      "130000\n",
      "140000\n",
      "150000\n",
      "160000\n",
      "170000\n",
      "180000\n",
      "190000\n",
      "200000\n",
      "210000\n",
      "220000\n",
      "230000\n",
      "240000\n",
      "250000\n",
      "260000\n",
      "270000\n",
      "280000\n",
      "290000\n",
      "300000\n",
      "310000\n",
      "320000\n",
      "330000\n",
      "340000\n",
      "350000\n",
      "360000\n",
      "370000\n",
      "380000\n",
      "390000\n",
      "400000\n",
      "410000\n",
      "420000\n",
      "430000\n",
      "440000\n",
      "450000\n",
      "460000\n",
      "470000\n",
      "480000\n",
      "490000\n",
      "500000\n",
      "510000\n",
      "520000\n",
      "530000\n",
      "540000\n",
      "550000\n",
      "560000\n",
      "570000\n",
      "580000\n",
      "590000\n",
      "600000\n",
      "610000\n",
      "620000\n",
      "630000\n",
      "640000\n",
      "650000\n",
      "660000\n",
      "670000\n",
      "680000\n",
      "690000\n",
      "700000\n",
      "710000\n",
      "720000\n",
      "730000\n",
      "740000\n",
      "750000\n",
      "760000\n",
      "770000\n",
      "780000\n",
      "790000\n",
      "800000\n",
      "810000\n",
      "820000\n",
      "830000\n",
      "840000\n",
      "850000\n",
      "860000\n",
      "870000\n",
      "880000\n",
      "890000\n",
      "900000\n",
      "910000\n",
      "920000\n",
      "930000\n",
      "940000\n",
      "950000\n",
      "960000\n",
      "970000\n",
      "980000\n",
      "990000\n",
      "1000000\n"
     ]
    }
   ],
   "source": [
    "documents = list(read_corpus('./data/dataPost1.txt'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1000000"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(documents)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Epoch Logger for verbose print"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "class EpochLogger(CallbackAny2Vec):\n",
    "    '''Callback to log information about training'''\n",
    "\n",
    "    def __init__(self):\n",
    "        self.epoch = 0\n",
    "        self.batch = 1\n",
    "\n",
    "    def on_epoch_begin(self, model):\n",
    "        print(\"Epoch #{} start\".format(self.epoch))\n",
    "\n",
    "    def on_epoch_end(self, model):\n",
    "        model.save('./data/model-epoch' + str(self.epoch) + \".model\")\n",
    "        print(\"Epoch #{} end and saved\".format(self.epoch))    \n",
    "        self.epoch += 1\n",
    "\n",
    "    def on_train_begin(self, model):\n",
    "        print(\"train started\")\n",
    "        self.batch = 1\n",
    "\n",
    "    def on_train_end(self, model):\n",
    "        print(\"train ended\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Model for Doc2Vec, Used PV-DBOW method"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "models = [\n",
    "    # PV-DBOW \n",
    "    Doc2Vec(dm=0, dbow_words=1, vector_size=doc_embedding_size, window=8, min_count=2, epochs=10, workers=cores),\n",
    "    # PV-DM w/average\n",
    "#     Doc2Vec(dm=1, dm_mean=1, vector_size=200, window=8, min_count=2, epochs =10, workers=cores),\n",
    "]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Build vocabulary for model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Doc2Vec(dbow+w,d400,n5,w8,mc2,s0.001,t16)\n"
     ]
    }
   ],
   "source": [
    "models[0].build_vocab(documents)\n",
    "print(str(models[0]))\n",
    "# models[1].reset_from(models[0])\n",
    "# print(str(models[1]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Train model with the data set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "for index, model in enumerate(models):\n",
    "    %time model.train(documents, total_examples=model.corpus_count, epochs=10, callbacks=[EpochLogger()])\n",
    "    model.save('./data/DbowModel' + str(index) + \".model\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Load the pre-trained model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "for index, model in enumerate(models):\n",
    "    models[index] = Doc2Vec.load('./data/DbowModel' + str(index) + \".model\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Functions for testing purpose"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from pprint import pprint\n",
    "def get(model, st):\n",
    "    tokens = pre_process(st)\n",
    "    print(tokens)\n",
    "    new_vector = model.infer_vector(tokens)\n",
    "    sims = model.docvecs.most_similar([new_vector], topn=20)\n",
    "    for index, val in sims:\n",
    "        print(str(documents[index].words.rstrip()) + \" - \" + str(val))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Doc2Vec(dbow+w,d400,n5,w8,mc2,s0.001,t16)\n",
      "\n",
      "\n",
      "Original Penguin (PENH8) Men's Smith Solid Tie\n",
      "['original', 'penguin', '(', 'penh8', ')', 'men', \"'s\", 'smith', 'solid', 'tie']\n",
      "original penguin men 's smith solid tie - 0.8894153833389282\n",
      "original penguin men 's smith solid tie - 0.8893281817436218\n",
      "original penguin men 's aliso solid tie - 0.8378052711486816\n",
      "original penguin men 's zion solid tie - 0.8307269811630249\n",
      "original penguin men 's zion solid tie - 0.8230248689651489\n",
      "original penguin men 's pique solid tie - 0.8162338733673096\n",
      "original penguin men 's pique solid tie - 0.8046824932098389\n",
      "original penguin men 's bimini floral tie - 0.7894967198371887\n",
      "original penguin men 's pique solid to-be-tied bowtie - 0.7888813614845276\n",
      "original penguin men 's clemens plaid tie - 0.7862534523010254\n",
      "original penguin men 's clemens plaid tie - 0.7828032374382019\n",
      "original penguin men 's trevini stripe tie - 0.7779017686843872\n",
      "original penguin men 's park check tie - 0.7688165903091431\n",
      "original penguin men 's kobuk dot to-be-tied bowtie - 0.7687689065933228\n",
      "original penguin men 's bimini floral tie - 0.7634896636009216\n",
      "original penguin men 's kilwin grid tie - 0.7576247453689575\n",
      "original penguin men 's bimini floral tie - 0.7562023401260376\n",
      "original penguin men 's master plaid tie - 0.7538378238677979\n",
      "original penguin men 's park check tie - 0.7482122182846069\n",
      "original penguin men 's avenue grid tie - 0.7364397048950195\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/iamdipta/.local/lib/python3.5/site-packages/gensim/matutils.py:737: FutureWarning: Conversion of the second argument of issubdtype from `int` to `np.signedinteger` is deprecated. In future, it will be treated as `np.int64 == np.dtype(int).type`.\n",
      "  if np.issubdtype(vec.dtype, np.int):\n"
     ]
    }
   ],
   "source": [
    "guess_it = \"Original Penguin (PENH8) Men's Smith Solid Tie\" \n",
    "for model in models:\n",
    "    print(str(model))\n",
    "    print(\"\\n\")\n",
    "    print(guess_it)\n",
    "    get(model, guess_it)\n",
    "    \n",
    "# Have to use dbow"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Save the doc2vec embedding"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "!rm -r ./data/dbow.txt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "models[0].docvecs.save_word2vec_format('./data/dbow.txt', binary=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Load the Embeddings with Dimensions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1000001 ./data/dbow.txt\r\n"
     ]
    }
   ],
   "source": [
    "EMBEDDING_FILE = \"./data/dbow.txt\"\n",
    "DIMENSION_FILE = \"./data/dataDimesions.txt\"\n",
    "\n",
    "!wc -l ./data/dbow.txt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0\n",
      "10000\n",
      "20000\n",
      "30000\n",
      "40000\n",
      "50000\n",
      "60000\n",
      "70000\n",
      "80000\n",
      "90000\n",
      "100000\n",
      "110000\n",
      "120000\n",
      "130000\n",
      "140000\n",
      "150000\n",
      "160000\n",
      "170000\n",
      "180000\n",
      "190000\n",
      "200000\n",
      "210000\n",
      "220000\n",
      "230000\n",
      "240000\n",
      "250000\n",
      "260000\n",
      "270000\n",
      "280000\n",
      "290000\n",
      "300000\n",
      "310000\n",
      "320000\n",
      "330000\n",
      "340000\n",
      "350000\n",
      "360000\n",
      "370000\n",
      "380000\n",
      "390000\n",
      "400000\n",
      "410000\n",
      "420000\n",
      "430000\n",
      "440000\n",
      "450000\n",
      "460000\n",
      "470000\n",
      "480000\n",
      "490000\n",
      "500000\n",
      "510000\n",
      "520000\n",
      "530000\n",
      "540000\n",
      "550000\n",
      "560000\n",
      "570000\n",
      "580000\n",
      "590000\n",
      "600000\n",
      "610000\n",
      "620000\n",
      "630000\n",
      "640000\n",
      "650000\n",
      "660000\n",
      "670000\n",
      "680000\n",
      "690000\n",
      "700000\n",
      "710000\n",
      "720000\n",
      "730000\n",
      "740000\n",
      "750000\n",
      "760000\n",
      "770000\n",
      "780000\n",
      "790000\n",
      "800000\n",
      "810000\n",
      "820000\n",
      "830000\n",
      "840000\n",
      "850000\n",
      "860000\n",
      "870000\n",
      "880000\n",
      "890000\n",
      "900000\n",
      "910000\n",
      "920000\n",
      "930000\n",
      "940000\n",
      "950000\n",
      "960000\n",
      "970000\n",
      "980000\n",
      "990000\n",
      "1000000\n"
     ]
    }
   ],
   "source": [
    "X = []\n",
    "with open(EMBEDDING_FILE, \"r\") as ip:\n",
    "    flg = 0\n",
    "    \n",
    "    for line in ip:\n",
    "        if flg:\n",
    "            data = line.split()\n",
    "            del data[0]\n",
    "            data = [float(i) for i in data]\n",
    "            X.append(data)\n",
    "        if flg%10000 == 0:\n",
    "            print(flg)\n",
    "        flg += 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "Y = []\n",
    "with open(DIMENSION_FILE, \"r\") as op:\n",
    "    for line in op:\n",
    "        data = line.split()\n",
    "        data = list(map(float, data))\n",
    "        Y.append(data)\n",
    "        if len(Y) == len(X):\n",
    "            break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1000000 1000000\n"
     ]
    }
   ],
   "source": [
    "print(len(X), len(Y))\n",
    "\n",
    "for index, doc in enumerate(X):\n",
    "    if len(doc) != doc_embedding_size:\n",
    "        del X[index]\n",
    "        del Y[index]\n",
    "#         print(index)\n",
    "#         print(doc)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1000000 1000000\n",
      "400 4\n"
     ]
    }
   ],
   "source": [
    "print(len(X), len(Y))\n",
    "print(len(X[0]), len(Y[0]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[]\n",
      "2\n",
      "<class 'numpy.ndarray'> <class 'numpy.ndarray'>\n"
     ]
    }
   ],
   "source": [
    "# print(X[:2])\n",
    "# print(len(X), len(X[0]))\n",
    "# print(len(Y), len(Y[0]))\n",
    "# docs = np.array([np.array(X[i]) for i in range(1000000)])\n",
    "# docs = np.array([x for x in X])\n",
    "# target = np.array(Y)\n",
    "\n",
    "print(list(filter(lambda l: len(l) == 1, X)))\n",
    "docs = np.array(X, ndmin=2, dtype=float)\n",
    "# docs = np.ndarray(shape=(1000000, 200), buffer=docs, dtype=float)\n",
    "target = np.array(Y)\n",
    "\n",
    "# ratio = int( docs.shape[0]/10 )\n",
    "ratio = int( docs.shape[0] / 10 )\n",
    "X_train = docs[ratio:]\n",
    "X_test = docs[:ratio]\n",
    "y_train = target[ratio:]\n",
    "y_test = target[:ratio]\n",
    "\n",
    "# print(docs[:2])\n",
    "print(docs.ndim)\n",
    "print(type(docs), type(docs[0]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(1000000, 400) (1000000, 4)\n"
     ]
    }
   ],
   "source": [
    "print(docs.shape, target.shape)\n",
    "# docs[0:2]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Neural Network\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "generations = 10000"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 810000 samples, validate on 90000 samples\n",
      "Epoch 1/10000\n",
      "810000/810000 [==============================] - 33s 41us/step - loss: 259.9963 - mean_absolute_error: 258.2704 - val_loss: 247.7401 - val_mean_absolute_error: 245.4633\n",
      "Epoch 2/10000\n",
      "810000/810000 [==============================] - 15s 19us/step - loss: 238.0880 - mean_absolute_error: 235.1061 - val_loss: 237.7431 - val_mean_absolute_error: 234.1344\n",
      "Epoch 3/10000\n",
      "810000/810000 [==============================] - 15s 19us/step - loss: 230.5702 - mean_absolute_error: 226.4636 - val_loss: 233.6584 - val_mean_absolute_error: 229.1527\n",
      "Epoch 4/10000\n",
      "810000/810000 [==============================] - 15s 18us/step - loss: 226.4189 - mean_absolute_error: 221.5583 - val_loss: 230.2521 - val_mean_absolute_error: 225.0996\n",
      "Epoch 5/10000\n",
      "810000/810000 [==============================] - 15s 19us/step - loss: 223.3366 - mean_absolute_error: 217.8419 - val_loss: 227.1312 - val_mean_absolute_error: 221.4027\n",
      "Epoch 6/10000\n",
      "810000/810000 [==============================] - 15s 19us/step - loss: 220.8289 - mean_absolute_error: 214.8242 - val_loss: 225.6836 - val_mean_absolute_error: 219.4815\n",
      "Epoch 7/10000\n",
      "810000/810000 [==============================] - 15s 19us/step - loss: 218.9244 - mean_absolute_error: 212.4624 - val_loss: 226.7207 - val_mean_absolute_error: 220.0853\n",
      "Epoch 8/10000\n",
      "810000/810000 [==============================] - 15s 19us/step - loss: 217.2044 - mean_absolute_error: 210.3474 - val_loss: 223.4056 - val_mean_absolute_error: 216.4034\n",
      "Epoch 9/10000\n",
      "810000/810000 [==============================] - 15s 18us/step - loss: 216.0987 - mean_absolute_error: 208.8954 - val_loss: 223.8730 - val_mean_absolute_error: 216.5326\n",
      "Epoch 10/10000\n",
      "810000/810000 [==============================] - 15s 19us/step - loss: 214.9975 - mean_absolute_error: 207.4910 - val_loss: 223.0593 - val_mean_absolute_error: 215.4403\n",
      "Epoch 11/10000\n",
      "810000/810000 [==============================] - 15s 19us/step - loss: 214.0792 - mean_absolute_error: 206.3004 - val_loss: 223.0324 - val_mean_absolute_error: 215.1718\n",
      "Epoch 12/10000\n",
      "810000/810000 [==============================] - 15s 19us/step - loss: 213.3145 - mean_absolute_error: 205.2962 - val_loss: 221.5529 - val_mean_absolute_error: 213.4509\n",
      "Epoch 13/10000\n",
      "810000/810000 [==============================] - 15s 19us/step - loss: 212.6104 - mean_absolute_error: 204.3545 - val_loss: 221.0997 - val_mean_absolute_error: 212.7777\n",
      "Epoch 14/10000\n",
      "810000/810000 [==============================] - 15s 19us/step - loss: 211.9718 - mean_absolute_error: 203.5130 - val_loss: 221.3354 - val_mean_absolute_error: 212.8064\n",
      "Epoch 15/10000\n",
      "810000/810000 [==============================] - 15s 18us/step - loss: 211.5571 - mean_absolute_error: 202.8968 - val_loss: 220.6041 - val_mean_absolute_error: 211.8660\n",
      "Epoch 16/10000\n",
      "810000/810000 [==============================] - 15s 18us/step - loss: 210.8819 - mean_absolute_error: 202.0358 - val_loss: 221.9867 - val_mean_absolute_error: 213.1018\n",
      "Epoch 17/10000\n",
      "810000/810000 [==============================] - 15s 18us/step - loss: 210.4542 - mean_absolute_error: 201.4656 - val_loss: 221.2448 - val_mean_absolute_error: 212.2157\n",
      "Epoch 18/10000\n",
      "810000/810000 [==============================] - 15s 19us/step - loss: 210.2227 - mean_absolute_error: 201.1030 - val_loss: 222.2073 - val_mean_absolute_error: 213.0353\n",
      "Epoch 19/10000\n",
      "810000/810000 [==============================] - 15s 19us/step - loss: 209.6272 - mean_absolute_error: 200.3583 - val_loss: 218.9548 - val_mean_absolute_error: 209.6302\n",
      "Epoch 20/10000\n",
      "810000/810000 [==============================] - 15s 19us/step - loss: 209.3852 - mean_absolute_error: 199.9914 - val_loss: 219.8950 - val_mean_absolute_error: 210.4370\n",
      "Epoch 21/10000\n",
      "810000/810000 [==============================] - 15s 19us/step - loss: 208.7372 - mean_absolute_error: 199.2265 - val_loss: 220.1034 - val_mean_absolute_error: 210.5619\n",
      "Epoch 22/10000\n",
      "810000/810000 [==============================] - 15s 18us/step - loss: 208.5803 - mean_absolute_error: 198.9372 - val_loss: 219.4281 - val_mean_absolute_error: 209.7432\n",
      "Epoch 23/10000\n",
      "810000/810000 [==============================] - 15s 18us/step - loss: 208.3250 - mean_absolute_error: 198.5584 - val_loss: 220.3377 - val_mean_absolute_error: 210.5517\n",
      "Epoch 24/10000\n",
      "810000/810000 [==============================] - 15s 19us/step - loss: 208.0410 - mean_absolute_error: 198.1843 - val_loss: 219.7442 - val_mean_absolute_error: 209.8363\n",
      "Epoch 25/10000\n",
      "810000/810000 [==============================] - 15s 18us/step - loss: 207.7810 - mean_absolute_error: 197.8133 - val_loss: 219.1539 - val_mean_absolute_error: 209.1698\n",
      "Epoch 26/10000\n",
      "810000/810000 [==============================] - 15s 18us/step - loss: 207.4556 - mean_absolute_error: 197.4281 - val_loss: 218.8236 - val_mean_absolute_error: 208.7596\n",
      "Epoch 27/10000\n",
      "810000/810000 [==============================] - 15s 19us/step - loss: 207.3412 - mean_absolute_error: 197.2156 - val_loss: 219.4297 - val_mean_absolute_error: 209.2808\n",
      "Epoch 28/10000\n",
      "810000/810000 [==============================] - 15s 18us/step - loss: 207.0073 - mean_absolute_error: 196.8132 - val_loss: 218.1036 - val_mean_absolute_error: 207.9140\n",
      "Epoch 29/10000\n",
      "810000/810000 [==============================] - 15s 18us/step - loss: 206.6157 - mean_absolute_error: 196.3559 - val_loss: 219.7564 - val_mean_absolute_error: 209.4611\n",
      "Epoch 30/10000\n",
      "810000/810000 [==============================] - 15s 18us/step - loss: 206.4293 - mean_absolute_error: 196.0986 - val_loss: 218.8683 - val_mean_absolute_error: 208.5379\n",
      "Epoch 31/10000\n",
      "810000/810000 [==============================] - 15s 19us/step - loss: 206.2840 - mean_absolute_error: 195.8895 - val_loss: 219.1496 - val_mean_absolute_error: 208.7402\n",
      "Epoch 32/10000\n",
      "810000/810000 [==============================] - 16s 19us/step - loss: 206.1735 - mean_absolute_error: 195.7182 - val_loss: 219.1045 - val_mean_absolute_error: 208.6270\n",
      "Epoch 33/10000\n",
      "810000/810000 [==============================] - 15s 18us/step - loss: 205.8861 - mean_absolute_error: 195.3614 - val_loss: 218.8074 - val_mean_absolute_error: 208.2732\n",
      "Epoch 34/10000\n",
      "810000/810000 [==============================] - 15s 19us/step - loss: 205.6657 - mean_absolute_error: 195.0972 - val_loss: 217.3672 - val_mean_absolute_error: 206.8010\n",
      "Epoch 35/10000\n",
      "810000/810000 [==============================] - 15s 19us/step - loss: 205.5256 - mean_absolute_error: 194.9112 - val_loss: 218.4243 - val_mean_absolute_error: 207.7903\n",
      "Epoch 36/10000\n",
      "810000/810000 [==============================] - 15s 19us/step - loss: 205.5105 - mean_absolute_error: 194.8448 - val_loss: 219.7509 - val_mean_absolute_error: 209.0757\n",
      "Epoch 37/10000\n",
      "810000/810000 [==============================] - 15s 19us/step - loss: 205.3195 - mean_absolute_error: 194.5755 - val_loss: 219.0414 - val_mean_absolute_error: 208.3231\n",
      "Epoch 38/10000\n",
      "810000/810000 [==============================] - 15s 19us/step - loss: 205.1142 - mean_absolute_error: 194.3526 - val_loss: 218.5090 - val_mean_absolute_error: 207.7123\n",
      "Epoch 39/10000\n",
      "810000/810000 [==============================] - 15s 19us/step - loss: 204.8647 - mean_absolute_error: 194.0611 - val_loss: 218.6209 - val_mean_absolute_error: 207.8320\n",
      "Epoch 40/10000\n",
      "810000/810000 [==============================] - 15s 19us/step - loss: 204.6654 - mean_absolute_error: 193.8255 - val_loss: 218.3627 - val_mean_absolute_error: 207.5216\n",
      "Epoch 41/10000\n",
      "810000/810000 [==============================] - 15s 19us/step - loss: 204.4567 - mean_absolute_error: 193.5609 - val_loss: 220.5168 - val_mean_absolute_error: 209.6253\n",
      "Epoch 42/10000\n",
      "810000/810000 [==============================] - 15s 19us/step - loss: 204.5460 - mean_absolute_error: 193.5929 - val_loss: 218.6827 - val_mean_absolute_error: 207.7181\n",
      "Epoch 43/10000\n",
      "810000/810000 [==============================] - 15s 19us/step - loss: 204.2940 - mean_absolute_error: 193.2780 - val_loss: 217.8979 - val_mean_absolute_error: 206.8905\n",
      "Epoch 44/10000\n",
      "810000/810000 [==============================] - 15s 19us/step - loss: 204.1835 - mean_absolute_error: 193.1443 - val_loss: 218.0293 - val_mean_absolute_error: 206.9608\n",
      "Epoch 45/10000\n",
      "810000/810000 [==============================] - 15s 18us/step - loss: 204.0620 - mean_absolute_error: 192.9817 - val_loss: 217.3295 - val_mean_absolute_error: 206.2462\n",
      "Epoch 46/10000\n",
      "810000/810000 [==============================] - 15s 19us/step - loss: 203.9496 - mean_absolute_error: 192.8381 - val_loss: 217.2441 - val_mean_absolute_error: 206.1411\n",
      "Epoch 47/10000\n",
      "810000/810000 [==============================] - 15s 18us/step - loss: 203.6056 - mean_absolute_error: 192.4510 - val_loss: 216.5341 - val_mean_absolute_error: 205.4088\n",
      "Epoch 48/10000\n",
      "810000/810000 [==============================] - 15s 19us/step - loss: 203.5035 - mean_absolute_error: 192.3528 - val_loss: 218.3619 - val_mean_absolute_error: 207.2091\n",
      "Epoch 49/10000\n",
      "810000/810000 [==============================] - 15s 19us/step - loss: 203.4788 - mean_absolute_error: 192.2810 - val_loss: 217.8622 - val_mean_absolute_error: 206.6568\n",
      "Epoch 50/10000\n",
      "810000/810000 [==============================] - 15s 19us/step - loss: 203.4163 - mean_absolute_error: 192.1908 - val_loss: 217.7465 - val_mean_absolute_error: 206.5310\n",
      "Epoch 51/10000\n",
      "810000/810000 [==============================] - 15s 18us/step - loss: 203.1022 - mean_absolute_error: 191.8442 - val_loss: 217.5137 - val_mean_absolute_error: 206.2708\n",
      "Epoch 52/10000\n",
      "810000/810000 [==============================] - 15s 18us/step - loss: 203.1396 - mean_absolute_error: 191.8730 - val_loss: 217.1624 - val_mean_absolute_error: 205.9071\n",
      "Epoch 53/10000\n",
      "810000/810000 [==============================] - 16s 19us/step - loss: 202.9705 - mean_absolute_error: 191.6976 - val_loss: 218.6011 - val_mean_absolute_error: 207.2726\n",
      "Epoch 54/10000\n",
      "810000/810000 [==============================] - 15s 19us/step - loss: 202.9606 - mean_absolute_error: 191.6310 - val_loss: 216.6291 - val_mean_absolute_error: 205.2936\n",
      "Epoch 55/10000\n",
      "810000/810000 [==============================] - 15s 19us/step - loss: 202.8502 - mean_absolute_error: 191.4849 - val_loss: 216.8236 - val_mean_absolute_error: 205.4404\n",
      "Epoch 56/10000\n",
      "810000/810000 [==============================] - 15s 19us/step - loss: 202.6672 - mean_absolute_error: 191.2747 - val_loss: 219.4784 - val_mean_absolute_error: 208.1007\n",
      "Epoch 57/10000\n",
      "810000/810000 [==============================] - 15s 19us/step - loss: 202.7083 - mean_absolute_error: 191.2822 - val_loss: 219.0965 - val_mean_absolute_error: 207.6741\n",
      "Epoch 58/10000\n",
      "810000/810000 [==============================] - 15s 19us/step - loss: 202.4399 - mean_absolute_error: 191.0142 - val_loss: 216.7853 - val_mean_absolute_error: 205.3385\n",
      "Epoch 59/10000\n",
      "810000/810000 [==============================] - 15s 19us/step - loss: 202.5906 - mean_absolute_error: 191.1455 - val_loss: 215.9156 - val_mean_absolute_error: 204.4996\n",
      "Epoch 60/10000\n",
      "810000/810000 [==============================] - 15s 18us/step - loss: 202.1230 - mean_absolute_error: 190.6429 - val_loss: 216.8609 - val_mean_absolute_error: 205.3628\n",
      "Epoch 61/10000\n",
      "810000/810000 [==============================] - 15s 19us/step - loss: 202.0483 - mean_absolute_error: 190.5439 - val_loss: 217.1540 - val_mean_absolute_error: 205.6783\n",
      "Epoch 62/10000\n",
      "810000/810000 [==============================] - 15s 19us/step - loss: 202.2219 - mean_absolute_error: 190.6395 - val_loss: 217.5122 - val_mean_absolute_error: 205.9360\n",
      "Epoch 63/10000\n",
      "810000/810000 [==============================] - 15s 19us/step - loss: 201.7864 - mean_absolute_error: 190.2130 - val_loss: 217.9806 - val_mean_absolute_error: 206.4281\n",
      "Epoch 64/10000\n",
      "810000/810000 [==============================] - 15s 19us/step - loss: 201.7037 - mean_absolute_error: 190.1050 - val_loss: 216.8742 - val_mean_absolute_error: 205.2589\n",
      "Epoch 65/10000\n",
      "810000/810000 [==============================] - 15s 19us/step - loss: 201.8395 - mean_absolute_error: 190.2463 - val_loss: 216.0498 - val_mean_absolute_error: 204.4363\n",
      "Epoch 66/10000\n",
      "810000/810000 [==============================] - 15s 19us/step - loss: 201.6599 - mean_absolute_error: 190.0431 - val_loss: 216.6651 - val_mean_absolute_error: 205.0415\n",
      "Epoch 67/10000\n",
      "810000/810000 [==============================] - 15s 19us/step - loss: 201.7063 - mean_absolute_error: 190.0805 - val_loss: 215.5653 - val_mean_absolute_error: 203.9076\n",
      "Epoch 68/10000\n",
      "810000/810000 [==============================] - 15s 19us/step - loss: 201.4868 - mean_absolute_error: 189.8145 - val_loss: 217.0043 - val_mean_absolute_error: 205.3605\n",
      "Epoch 69/10000\n",
      "810000/810000 [==============================] - 15s 19us/step - loss: 201.4910 - mean_absolute_error: 189.8046 - val_loss: 218.1723 - val_mean_absolute_error: 206.4846\n",
      "Epoch 70/10000\n",
      "810000/810000 [==============================] - 15s 19us/step - loss: 201.4636 - mean_absolute_error: 189.7691 - val_loss: 216.4007 - val_mean_absolute_error: 204.6817\n",
      "Epoch 71/10000\n",
      "810000/810000 [==============================] - 15s 19us/step - loss: 201.2811 - mean_absolute_error: 189.5360 - val_loss: 216.7161 - val_mean_absolute_error: 205.0041\n",
      "Epoch 72/10000\n",
      "810000/810000 [==============================] - 15s 19us/step - loss: 201.1283 - mean_absolute_error: 189.4087 - val_loss: 216.4252 - val_mean_absolute_error: 204.7332\n",
      "Epoch 73/10000\n",
      "810000/810000 [==============================] - 15s 19us/step - loss: 200.9373 - mean_absolute_error: 189.2179 - val_loss: 215.3177 - val_mean_absolute_error: 203.6055\n",
      "Epoch 74/10000\n",
      "810000/810000 [==============================] - 15s 19us/step - loss: 200.8674 - mean_absolute_error: 189.1380 - val_loss: 217.5089 - val_mean_absolute_error: 205.7521\n",
      "Epoch 75/10000\n",
      "810000/810000 [==============================] - 15s 18us/step - loss: 201.0195 - mean_absolute_error: 189.2429 - val_loss: 214.9669 - val_mean_absolute_error: 203.1636\n",
      "Epoch 76/10000\n",
      "810000/810000 [==============================] - 15s 19us/step - loss: 200.7340 - mean_absolute_error: 188.9341 - val_loss: 218.1389 - val_mean_absolute_error: 206.3683\n",
      "Epoch 77/10000\n",
      "810000/810000 [==============================] - 15s 19us/step - loss: 200.7045 - mean_absolute_error: 188.9000 - val_loss: 215.6028 - val_mean_absolute_error: 203.8197\n",
      "Epoch 78/10000\n",
      "810000/810000 [==============================] - 15s 19us/step - loss: 200.7073 - mean_absolute_error: 188.9077 - val_loss: 215.8307 - val_mean_absolute_error: 204.0279\n",
      "Epoch 79/10000\n",
      "810000/810000 [==============================] - 15s 19us/step - loss: 200.5177 - mean_absolute_error: 188.6884 - val_loss: 216.2549 - val_mean_absolute_error: 204.3862\n",
      "Epoch 80/10000\n",
      "810000/810000 [==============================] - 15s 19us/step - loss: 200.6418 - mean_absolute_error: 188.7742 - val_loss: 217.0283 - val_mean_absolute_error: 205.1444\n",
      "Epoch 81/10000\n",
      "810000/810000 [==============================] - 15s 19us/step - loss: 200.5046 - mean_absolute_error: 188.5903 - val_loss: 215.5539 - val_mean_absolute_error: 203.6248\n",
      "Epoch 82/10000\n",
      "810000/810000 [==============================] - 15s 19us/step - loss: 200.5840 - mean_absolute_error: 188.6494 - val_loss: 216.5474 - val_mean_absolute_error: 204.6503\n",
      "Epoch 83/10000\n",
      "810000/810000 [==============================] - 15s 19us/step - loss: 200.1512 - mean_absolute_error: 188.2343 - val_loss: 216.7244 - val_mean_absolute_error: 204.8357\n",
      "Epoch 84/10000\n",
      "810000/810000 [==============================] - 15s 19us/step - loss: 200.2204 - mean_absolute_error: 188.3089 - val_loss: 217.4816 - val_mean_absolute_error: 205.5554\n",
      "Epoch 85/10000\n",
      "810000/810000 [==============================] - 15s 19us/step - loss: 200.2903 - mean_absolute_error: 188.3218 - val_loss: 215.8316 - val_mean_absolute_error: 203.8782\n",
      "Epoch 86/10000\n",
      "810000/810000 [==============================] - 15s 19us/step - loss: 200.0652 - mean_absolute_error: 188.1036 - val_loss: 218.6672 - val_mean_absolute_error: 206.6609\n",
      "Epoch 87/10000\n",
      "810000/810000 [==============================] - 15s 19us/step - loss: 200.0198 - mean_absolute_error: 188.0444 - val_loss: 216.7579 - val_mean_absolute_error: 204.7708\n",
      "Epoch 88/10000\n",
      "810000/810000 [==============================] - 15s 19us/step - loss: 199.7701 - mean_absolute_error: 187.7854 - val_loss: 214.9150 - val_mean_absolute_error: 202.9530\n",
      "Epoch 89/10000\n",
      "810000/810000 [==============================] - 15s 19us/step - loss: 199.9481 - mean_absolute_error: 187.9569 - val_loss: 215.6186 - val_mean_absolute_error: 203.6476\n",
      "Epoch 90/10000\n",
      "810000/810000 [==============================] - 15s 19us/step - loss: 199.8301 - mean_absolute_error: 187.8205 - val_loss: 219.2575 - val_mean_absolute_error: 207.2590\n",
      "Epoch 91/10000\n",
      "810000/810000 [==============================] - 15s 19us/step - loss: 200.0802 - mean_absolute_error: 188.0338 - val_loss: 214.9926 - val_mean_absolute_error: 202.9533\n",
      "Epoch 92/10000\n",
      "810000/810000 [==============================] - 15s 19us/step - loss: 199.8141 - mean_absolute_error: 187.7573 - val_loss: 216.7650 - val_mean_absolute_error: 204.7252\n",
      "Epoch 93/10000\n",
      "810000/810000 [==============================] - 15s 19us/step - loss: 199.8627 - mean_absolute_error: 187.8230 - val_loss: 216.4792 - val_mean_absolute_error: 204.4366\n",
      "Epoch 94/10000\n",
      "810000/810000 [==============================] - 15s 19us/step - loss: 199.8396 - mean_absolute_error: 187.7671 - val_loss: 214.2045 - val_mean_absolute_error: 202.1671\n",
      "Epoch 95/10000\n",
      "810000/810000 [==============================] - 15s 19us/step - loss: 199.5422 - mean_absolute_error: 187.4860 - val_loss: 216.7791 - val_mean_absolute_error: 204.6909\n",
      "Epoch 96/10000\n",
      "810000/810000 [==============================] - 15s 19us/step - loss: 199.4901 - mean_absolute_error: 187.4061 - val_loss: 215.4176 - val_mean_absolute_error: 203.3237\n",
      "Epoch 97/10000\n",
      "810000/810000 [==============================] - 15s 19us/step - loss: 199.5777 - mean_absolute_error: 187.4806 - val_loss: 217.4748 - val_mean_absolute_error: 205.3602\n",
      "Epoch 98/10000\n",
      "810000/810000 [==============================] - 15s 19us/step - loss: 199.4353 - mean_absolute_error: 187.3417 - val_loss: 214.8467 - val_mean_absolute_error: 202.7580\n",
      "Epoch 99/10000\n",
      "810000/810000 [==============================] - 15s 19us/step - loss: 199.2591 - mean_absolute_error: 187.1742 - val_loss: 218.4992 - val_mean_absolute_error: 206.3952\n",
      "Epoch 100/10000\n",
      "810000/810000 [==============================] - 15s 19us/step - loss: 198.9211 - mean_absolute_error: 186.8089 - val_loss: 215.9747 - val_mean_absolute_error: 203.8323\n",
      "Epoch 101/10000\n",
      "810000/810000 [==============================] - 15s 19us/step - loss: 199.1573 - mean_absolute_error: 187.0233 - val_loss: 216.8824 - val_mean_absolute_error: 204.7414\n",
      "Epoch 102/10000\n",
      "810000/810000 [==============================] - 15s 19us/step - loss: 198.9755 - mean_absolute_error: 186.8205 - val_loss: 215.0491 - val_mean_absolute_error: 202.9169\n",
      "Epoch 103/10000\n",
      "810000/810000 [==============================] - 15s 19us/step - loss: 199.0315 - mean_absolute_error: 186.8614 - val_loss: 216.0836 - val_mean_absolute_error: 203.9093\n",
      "Epoch 104/10000\n",
      "810000/810000 [==============================] - 15s 19us/step - loss: 198.8735 - mean_absolute_error: 186.7189 - val_loss: 215.6075 - val_mean_absolute_error: 203.4419\n",
      "Epoch 105/10000\n",
      "810000/810000 [==============================] - 15s 19us/step - loss: 199.0149 - mean_absolute_error: 186.8355 - val_loss: 214.8481 - val_mean_absolute_error: 202.6621\n",
      "Epoch 106/10000\n",
      "810000/810000 [==============================] - 15s 19us/step - loss: 198.9145 - mean_absolute_error: 186.6843 - val_loss: 216.3695 - val_mean_absolute_error: 204.1340\n",
      "Epoch 107/10000\n",
      "810000/810000 [==============================] - 15s 19us/step - loss: 198.8266 - mean_absolute_error: 186.5874 - val_loss: 215.3845 - val_mean_absolute_error: 203.1449\n",
      "Epoch 108/10000\n",
      "810000/810000 [==============================] - 15s 19us/step - loss: 198.7800 - mean_absolute_error: 186.5437 - val_loss: 218.5229 - val_mean_absolute_error: 206.2885\n",
      "Epoch 109/10000\n",
      "810000/810000 [==============================] - 15s 19us/step - loss: 198.9923 - mean_absolute_error: 186.7598 - val_loss: 214.8731 - val_mean_absolute_error: 202.6187\n",
      "Epoch 110/10000\n",
      "810000/810000 [==============================] - 15s 19us/step - loss: 198.8590 - mean_absolute_error: 186.5578 - val_loss: 215.3116 - val_mean_absolute_error: 203.0236\n",
      "100000/100000 [==============================] - 1s 8us/step\n",
      "Current one:  [201.1434777294922, 188.85543999023437]\n"
     ]
    }
   ],
   "source": [
    "from tensorflow import keras\n",
    "\n",
    "model = keras.Sequential()\n",
    "activation_function = keras.layers.PReLU()\n",
    "\n",
    "# Input Layer\n",
    "model.add(keras.layers.Dense(405, kernel_initializer=keras.initializers.random_uniform, input_dim=doc_embedding_size,\n",
    "                             kernel_regularizer=keras.regularizers.l2(0.01),\n",
    "                             bias_regularizer=keras.regularizers.l2(0.01), name=\"input\"))\n",
    "model.add(activation_function)\n",
    "\n",
    "# Hidden Layer\n",
    "model.add(keras.layers.Dense(405, kernel_initializer='normal', name=\"hidden1\"))\n",
    "model.add(activation_function)\n",
    "model.add(keras.layers.Dropout(0.2))\n",
    "\n",
    "model.add(keras.layers.Dense(405, kernel_initializer='normal', name=\"hidden2\"))\n",
    "model.add(activation_function)\n",
    "model.add(keras.layers.Dropout(0.2))\n",
    "\n",
    "# Output Layer\n",
    "model.add(keras.layers.Dense(4, kernel_initializer='normal', name=\"output\"))\n",
    "\n",
    "model.compile(loss='mean_absolute_error', optimizer=\"Adam\", metrics=['mean_absolute_error'])\n",
    "\n",
    "checkpoint = keras.callbacks.ModelCheckpoint(\"./temp/best.h5\", monitor='val_loss', verbose=1, save_best_only=True,\n",
    "                                                 mode='auto')\n",
    "\n",
    "earlyStopping = keras.callbacks.EarlyStopping(patience=16, mode=\"auto\")\n",
    "callbacks_list = [earlyStopping]\n",
    "\n",
    "# model = create_neural_model()\n",
    "model.fit(X_train, y_train, epochs=generations, batch_size=256, validation_split=0.1, callbacks=callbacks_list)\n",
    "print(\"Current one: \", model.evaluate(X_test, y_test, batch_size=256))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}