O_Word2Vec_TF_NCE_O.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "from Utils.FS import file\n",
    "from nltk.tokenize.punkt import PunktSentenceTokenizer\n",
    "from nltk.tokenize import word_tokenize\n",
    "from nltk.corpus import brown\n",
    "from scipy.sparse import coo_matrix, dok_matrix\n",
    "from scipy.sparse.linalg import svds\n",
    "from sklearn.preprocessing import normalize\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "from sklearn.manifold import TSNE\n",
    "from wordcloud import WordCloud\n",
    "from nltk.corpus import stopwords\n",
    "import tensorflow as tf\n",
    "import math\n",
    "import TextPreprocess.words2dict as words2dict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "np.random.seed(1234)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "sents = brown.sents()\n",
    "sents = [[token.lower() for token in sent] for sent in sents]\n",
    "words = brown.words()\n",
    "words = [word.lower() for word in words]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of tokens: 1161192\n",
      "Number of sentences: 57340\n",
      "Longest sentences length: 180\n"
     ]
    }
   ],
   "source": [
    "print(\"Number of tokens: {}\".format(len(words)))\n",
    "print(\"Number of sentences: {}\".format(len(sents)))\n",
    "print(\"Longest sentences length: {}\".format(max([len(sent) for sent in sents])))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of unique tokens: 49815\n"
     ]
    }
   ],
   "source": [
    "words_dict, inv_words_dict = words2dict.convert(words)\n",
    "words_size = len(words_dict)\n",
    "print(\"Number of unique tokens: {}\".format(words_size))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "def singleSideWindow(sents, words_dict, window_size, reverse = False):\n",
    "    window = []\n",
    "    row = []\n",
    "    col = []\n",
    "    data = []\n",
    "        \n",
    "    for sent in reversed(sents) if reverse else sents:\n",
    "        for word in reversed(sent) if reverse else sent:\n",
    "            for w in window:\n",
    "                if w == word:\n",
    "                    continue\n",
    "                row.append(words_dict[word])\n",
    "                col.append(words_dict[w])\n",
    "                data.append(1)\n",
    "            if len(window) == window_size:\n",
    "                window.pop(0)\n",
    "            window.append(word)\n",
    "    return coo_matrix((data, (row, col)), shape=(len(words_dict), len(words_dict)), dtype='float64')\n",
    "    \n",
    "\n",
    "def sents2wordContextMatrix(sents, words_dict, window_size = 5):\n",
    "    m = coo_matrix((words_size, words_size), 'float64')\n",
    "    \n",
    "    print('Doing forward pass...')\n",
    "    m += singleSideWindow(sents, words_dict, window_size)\n",
    "    \n",
    "    print('Doing backward pass...')\n",
    "    m += singleSideWindow(sents, words_dict, window_size, True)\n",
    "    \n",
    "    return m\n",
    "\n",
    "def sents2wordCoocurrenceMatrix(sents, words_dict, window_size = 10):\n",
    "    #don't really care edge cases....\n",
    "\n",
    "    window = []\n",
    "    row = []\n",
    "    col = []\n",
    "    data = []\n",
    "    for sent in sents:\n",
    "        for word in sent:\n",
    "            for i in range(len(window)- 1):\n",
    "                for j in range(i+1, len(window)):\n",
    "                    row += [words_dict[window[i]], words_dict[window[j]]]\n",
    "                    col += [words_dict[window[j]], words_dict[window[i]]]\n",
    "                    data += [1, 1]\n",
    "            if len(window) == window_size:\n",
    "                window.pop(0)\n",
    "            window.append(word)\n",
    "    print('Preparing sparse matrix...')\n",
    "    print('Length of data: {}'.format(len(data)))\n",
    "    return coo_matrix((data, (row,col)), shape=(words_size, words_size), dtype='float64').multiply(1/len(data))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "def singleSideWindowGenerator(c, w, sents, words_dict, window_size = 5, batch_size = 32, reverse = False):\n",
    "    window = ['--' for i in range(window_size)]\n",
    "    for sent in reversed(sents) if reverse else sents:\n",
    "        for word in reversed(sent) if reverse else sent:\n",
    "            for context in window:\n",
    "                c.append(words_dict[context])\n",
    "                w.append([words_dict[word]])\n",
    "                if(len(c) == batch_size):\n",
    "                    yield c, w\n",
    "                    c = []\n",
    "                    w = []\n",
    "            window.pop(0)\n",
    "            window.append(word)\n",
    "\n",
    "def sents2batchGenerator(sents, words_dict, window_size = 5, batch_size=32):\n",
    "    c = []\n",
    "    w = []\n",
    "\n",
    "    while(True):\n",
    "        window = ['--' for i in range(window_size)]\n",
    "        for sent in sents:\n",
    "            for word in sent:\n",
    "                for context in window:\n",
    "                    c.append([words_dict[context]])\n",
    "                    w.append(words_dict[word])\n",
    "                    if(len(c) == batch_size):\n",
    "                        yield w, c\n",
    "                        c = []\n",
    "                        w = []\n",
    "                window.pop(0)\n",
    "                window.append(word)\n",
    "\n",
    "        window = ['--' for i in range(window_size)]\n",
    "        for sent in reversed(sents):\n",
    "            for word in reversed(sent):\n",
    "                for context in window:\n",
    "                    c.append([words_dict[context]])\n",
    "                    w.append(words_dict[word])\n",
    "                    if(len(c) == batch_size):\n",
    "                        yield w, c\n",
    "                        c = []\n",
    "                        w = []\n",
    "                window.pop(0)\n",
    "                window.append(word)\n",
    "\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "def sents2freq(sents):\n",
    "    freq = {}\n",
    "    for sent in sents:\n",
    "        for word in sent:\n",
    "            if word in freq.keys():\n",
    "                freq[word] += 1\n",
    "            else:\n",
    "                freq[word] = 1\n",
    "    return freq\n",
    "\n",
    "words_freq = sents2freq(sents)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "WINDOW_SIZE = 2\n",
    "BATCH_SIZE = 32\n",
    "generator = sents2batchGenerator(sents, words_dict, window_size = WINDOW_SIZE, batch_size=BATCH_SIZE)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "DIMENSION = 50\n",
    "VOCABULAY_SIZE = len(words_dict)\n",
    "NEGATIVE_SAMPLE = 64\n",
    "\n",
    "graph = tf.Graph()\n",
    "\n",
    "with graph.as_default():\n",
    "    \n",
    "    inputs = tf.placeholder(tf.int32, shape=[BATCH_SIZE])\n",
    "    labels = tf.placeholder(tf.int32, shape=[BATCH_SIZE, 1])\n",
    "\n",
    "    with tf.device('/cpu:0'):\n",
    "    \n",
    "        embeddings = tf.Variable(\n",
    "            tf.random_uniform([VOCABULAY_SIZE, DIMENSION], -1.0, 1.0))\n",
    "        embed = tf.nn.embedding_lookup(embeddings, inputs, max_norm=1)\n",
    "\n",
    "        nce_weights = tf.Variable(\n",
    "            tf.truncated_normal([VOCABULAY_SIZE, DIMENSION],\n",
    "                                stddev=1.0 / math.sqrt(DIMENSION)))\n",
    "\n",
    "        nce_biases = tf.Variable(tf.zeros([VOCABULAY_SIZE]))\n",
    "\n",
    "        loss = tf.reduce_mean(\n",
    "          tf.nn.nce_loss(weights=nce_weights,\n",
    "                     biases=nce_biases,\n",
    "                     labels=labels,\n",
    "                     inputs=embed,\n",
    "                     num_sampled=NEGATIVE_SAMPLE,\n",
    "                     num_classes=VOCABULAY_SIZE))\n",
    "\n",
    "        #optimizer = tf.train.GradientDescentOptimizer(1.0).minimize(loss)\n",
    "        optimizer = tf.train.MomentumOptimizer(1.0, 0.5).minimize(loss)\n",
    "        #optimizer = tf.train.AdamOptimizer().minimize(loss) # super slow\n",
    "        \n",
    "        init = tf.global_variables_initializer()\n",
    "        word2VecSaver = tf.train.Saver({'Words2Vec': embeddings, 'NCE_Weights': nce_weights, 'NCE_Biases': nce_biases})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "def cloestWord(word, words_vec, count = 10, method=None):\n",
    "    if method == 'cos':\n",
    "        dist = np.array([ sum(words_vec[word] * words_vec[key]) for key in words_vec.keys()])\n",
    "        top_ten = dist.argsort()[::-1][:10]\n",
    "    else:\n",
    "        dist = np.array([ sum(np.square(np.array(words_vec[word]) - np.array(words_vec[key]))) for key in words_vec.keys()])\n",
    "        top_ten = dist.argsort()[:10]\n",
    "    return [list(words_vec.keys())[i] for i in top_ten]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Average loss at step  2000 :  145.534481876\n",
      "['two', 'on', 'will', ',', 'not', \"''\", 'in', 'who', 'a', 'been']\n",
      "Average loss at step  4000 :  90.5510077658\n",
      "['two', ',', 'from', '.', 'with', 'and', 'of', 'have', 'are', 'her']\n",
      "Average loss at step  6000 :  66.2646737881\n",
      "['two', 'day', 'last', 'would', 'both', 'new', 'also', 'or', 'president', 'the']\n",
      "Average loss at step  8000 :  51.167251024\n",
      "['two', ',', 'little', 'to', 'other', 'and', 'the', 'his', \"''\", 'three']\n",
      "Average loss at step  10000 :  40.4493307657\n",
      "['two', 'three', 'in', 'national', 'the', 'through', 'service', ',', 'last', 'those']\n",
      "Average loss at step  12000 :  33.0869439597\n",
      "['two', '``', 'three', 'four', 'national', 'john', 'that', 'most', 'new', 'those']\n",
      "Average loss at step  14000 :  26.6413221374\n",
      "['two', 'time', 'business', 'show', 'three', 'last', 'new', 'government', 'four', 'washington']\n",
      "Average loss at step  16000 :  22.351596437\n",
      "['two', 'business', '.', 'children', 'work', 'that', 'three', 'last', ',', 'berlin']\n",
      "Average loss at step  18000 :  19.0118421838\n",
      "['two', 'life', 'tax', 'things', 'people', 'right', 'late', 'high', '``', 'washington']\n",
      "Average loss at step  20000 :  15.985894666\n",
      "['two', 'things', 'chicago', 'three', 'young', 'school', 'use', 'business', 'left', '--']\n",
      "Average loss at step  22000 :  13.4307703619\n",
      "['two', 'action', 'tax', 'things', 'social', 'left', 'death', 'washington', 'foreign', 'leader']\n",
      "Average loss at step  24000 :  12.0125480549\n",
      "['two', 'again', 'around', 'under', 'began', 'less', 'rather', 'held', 'along', 'later']\n",
      "Average loss at step  26000 :  10.8412814343\n",
      "['two', 'until', 'within', 'under', '4', 'every', 'several', 'began', 'six', 'along']\n",
      "Average loss at step  28000 :  9.64319984579\n",
      "['two', 'students', 'police', 'education', 'reason', 'congress', 'five', 'things', 'music', 'president']\n",
      "Average loss at step  30000 :  8.76215399623\n",
      "['two', 'late', 'labor', 'company', 'reason', 'central', 'people', 'side', 'students', 'berlin']\n",
      "Average loss at step  32000 :  7.96407820511\n",
      "['two', 'late', 'president', 'name', 'secretary', 'reason', 'police', 'bill', 'session', 'problems']\n",
      "Average loss at step  34000 :  7.32995491076\n",
      "['two', 'late', 'company', 'following', 'street', 'five', 'individual', 'students', 'police', 'spring']\n",
      "Average loss at step  36000 :  6.94989142251\n",
      "['two', 'five', 'berlin', 'individual', 'leaders', 'meeting', 'months', 'problems', 'tax', 'north']\n",
      "Average loss at step  38000 :  6.63472750187\n",
      "['two', 'three', 'berlin', 'water', 'five', 'color', '1960', 'earlier', 'coming', 'nuclear']\n",
      "Average loss at step  40000 :  6.07103734195\n",
      "['two', 'three', 'four', 'seven', 'production', 'making', '7', 'eight', 'things', 'miss']\n",
      "Average loss at step  42000 :  5.76795178187\n",
      "['two', 'three', 'students', 'following', 'next', 'first', 'most', 'general', 'second', 'modern']\n",
      "Average loss at step  44000 :  5.89538223505\n",
      "['two', 'six', 'most', 'production', 'three', 'students', 'seven', 'mother', 'five', 'berlin']\n",
      "Average loss at step  46000 :  5.66049512208\n",
      "['two', 'six', 'three', 'fall', 'seven', 'most', 'american', 'mother', 'election', 'questions']\n",
      "Average loss at step  48000 :  5.43780510366\n",
      "['two', 'light', 'primary', 'fall', 'election', 'face', 'several', 'american', 'west', 'congress']\n",
      "Average loss at step  50000 :  5.26113907361\n",
      "['two', 'fall', 'several', 'light', 'four', 'primary', 'effect', 'six', 'election', 'three']\n",
      "Average loss at step  52000 :  5.57945463836\n",
      "['two', 'several', 'four', 'three', 'five', 'most', 'fall', 'love', 'effect', 'making']\n",
      "Average loss at step  54000 :  5.27039329743\n",
      "['two', 'four', 'five', 'several', 'fall', 'three', 'love', 'most', 'light', 'earlier']\n",
      "Average loss at step  56000 :  5.40678031874\n",
      "['two', 'four', 'several', 'five', 'three', 'fall', 'most', 'last', 'love', 'earlier']\n",
      "Average loss at step  58000 :  5.24756932151\n",
      "['two', 'three', 'four', 'five', 'several', 'six', 'most', 'first', 'congress', 'fact']\n",
      "Average loss at step  60000 :  5.02466796267\n",
      "['two', 'three', 'most', 'four', 'first', 'five', 'past', 'several', 'american', 'six']\n",
      "Average loss at step  62000 :  5.03648509395\n",
      "['two', 'five', 'six', 'four', 'past', 'u.s.', 'three', 'most', 'first', 'international']\n",
      "Average loss at step  64000 :  4.94821839559\n",
      "['two', 'most', 'five', 'three', 'six', 'past', 'four', 'u.s.', 'american', 'earlier']\n",
      "Average loss at step  66000 :  5.02909833348\n",
      "['two', 'most', 'three', 'five', 'six', 'past', 'earlier', 'several', 'four', 'women']\n",
      "Average loss at step  68000 :  4.95940374672\n",
      "['two', 'most', 'three', 'five', 'six', 'several', 'past', 'four', 'earlier', 'women']\n",
      "Average loss at step  70000 :  4.80027772462\n",
      "['two', 'most', 'three', 'five', 'several', 'spring', 'six', 'hopes', 'earlier', 'past']\n",
      "Average loss at step  72000 :  4.87928738761\n",
      "['two', 'three', 'five', 'six', 'spring', 'earlier', 'most', 'several', 'cars', 'four']\n",
      "Average loss at step  74000 :  5.11500684798\n",
      "['two', 'three', 'six', 'five', 'several', 'earlier', 'cars', 'weeks', 'spring', 'points']\n",
      "Average loss at step  76000 :  4.74482366204\n",
      "['two', 'three', 'five', 'several', 'six', 'weeks', 'cars', 'first', 'points', 'earlier']\n",
      "Average loss at step  78000 :  4.75884514987\n",
      "['two', 'three', 'five', 'weeks', 'six', 'several', 'first', 'cars', 'points', 'other']\n",
      "Average loss at step  80000 :  4.78380792725\n",
      "['two', 'three', 'weeks', 'six', 'five', 'several', 'months', 'cars', 'years', 'points']\n",
      "Average loss at step  82000 :  4.73765541339\n",
      "['two', 'three', 'weeks', 'five', 'years', 'six', 'other', 'months', 'several', 'first']\n",
      "Average loss at step  84000 :  4.72280604839\n",
      "['two', 'three', 'weeks', 'five', 'years', 'six', 'other', 'several', 'months', 'points']\n",
      "Average loss at step  86000 :  4.65973889804\n",
      "['two', 'three', 'weeks', 'other', 'five', 'years', 'six', 'several', 'months', 'types']\n",
      "Average loss at step  88000 :  4.58595388997\n",
      "['two', 'three', 'other', 'five', 'weeks', 'years', 'six', 'some', 'four', 'several']\n",
      "Average loss at step  90000 :  4.76102268898\n",
      "['two', 'three', 'other', 'some', 'four', 'five', 'weeks', 'six', 'several', 'years']\n",
      "Average loss at step  92000 :  4.78561499023\n",
      "['two', 'three', 'other', 'five', 'some', 'four', 'weeks', 'years', 'types', 'six']\n",
      "Average loss at step  94000 :  4.99304813087\n",
      "['two', 'three', 'other', 'five', 'four', 'some', 'types', 'weeks', 'passion', 'six']\n",
      "Average loss at step  96000 :  4.91177560735\n",
      "['two', 'three', 'other', 'five', 'four', 'some', 'types', 'several', 'weeks', 'six']\n",
      "Average loss at step  98000 :  4.84725561881\n",
      "['two', 'three', 'five', 'four', 'other', 'several', 'types', 'many', 'six', 'weeks']\n",
      "Average loss at step  100000 :  4.73758619845\n",
      "['two', 'three', 'five', 'several', 'four', 'other', 'many', 'six', 'types', 'some']\n",
      "Average loss at step  102000 :  4.72823374689\n",
      "['two', 'three', 'several', 'five', 'four', 'other', 'types', 'many', 'six', 'ten']\n",
      "Average loss at step  104000 :  4.73695300293\n",
      "['two', 'three', 'several', 'five', 'four', 'other', 'types', 'six', 'many', 'ten']\n",
      "Average loss at step  106000 :  4.65004274011\n",
      "['two', 'three', 'several', 'five', 'four', 'six', 'other', 'many', 'types', 'ten']\n",
      "Average loss at step  108000 :  4.77491356218\n",
      "['two', 'three', 'several', 'five', 'four', 'six', 'many', 'ten', 'other', 'fiscal']\n",
      "Average loss at step  110000 :  4.92067105412\n",
      "['two', 'three', 'several', 'five', 'four', 'six', 'ten', 'many', 'other', 'fiscal']\n",
      "Average loss at step  112000 :  4.85361227632\n",
      "['two', 'three', 'five', 'several', 'four', 'six', 'ten', 'many', 'other', 'fiscal']\n",
      "Average loss at step  114000 :  4.86017460358\n",
      "['two', 'three', 'several', 'five', 'four', 'six', 'ten', 'many', 'other', 'last']\n",
      "Average loss at step  116000 :  4.8059397217\n",
      "['two', 'three', 'several', 'five', 'four', 'six', 'many', 'ten', 'other', 'last']\n",
      "Average loss at step  118000 :  4.82752692139\n",
      "['two', 'three', 'several', 'five', 'four', 'six', 'ten', 'many', 'other', 'last']\n",
      "Average loss at step  120000 :  4.82690660095\n",
      "['two', 'three', 'several', 'four', 'five', 'ten', 'six', 'many', 'few', 'other']\n",
      "Average loss at step  122000 :  4.86623352015\n",
      "['two', 'three', 'several', 'four', 'five', 'ten', 'six', 'many', 'few', 'other']\n",
      "Average loss at step  124000 :  4.84596658349\n",
      "['two', 'three', 'several', 'five', 'four', 'six', 'ten', 'many', 'few', 'seven']\n",
      "Average loss at step  126000 :  4.86489758968\n",
      "['two', 'three', 'five', 'several', 'four', 'six', 'ten', 'many', 'few', 'last']\n",
      "Average loss at step  128000 :  4.81295422959\n",
      "['two', 'three', 'several', 'five', 'four', 'six', 'many', 'few', 'ten', 'last']\n",
      "Average loss at step  130000 :  4.87895786297\n",
      "['two', 'three', 'several', 'four', 'five', 'six', 'many', 'few', 'ten', 'last']\n",
      "Average loss at step  132000 :  4.76150890565\n",
      "['two', 'three', 'four', 'several', 'five', 'six', 'ten', 'many', 'few', 'hundred']\n",
      "Average loss at step  134000 :  4.89292191744\n",
      "['two', 'three', 'several', 'four', 'five', 'six', 'few', 'many', 'ten', 'hundred']\n",
      "Average loss at step  136000 :  4.88672031975\n",
      "['two', 'three', 'several', 'four', 'five', 'six', 'many', 'few', 'ten', 'hundred']\n",
      "Average loss at step  138000 :  4.84502191973\n",
      "['two', 'three', 'several', 'four', 'five', 'six', 'many', 'few', 'ten', 'hundred']\n",
      "Average loss at step  140000 :  4.86193640363\n",
      "['two', 'three', 'several', 'four', 'five', 'many', 'few', 'six', 'ten', 'some']\n",
      "Average loss at step  142000 :  4.91462424159\n",
      "['two', 'three', 'several', 'four', 'many', 'few', 'five', 'six', 'ten', 'eight']\n",
      "Average loss at step  144000 :  4.91030984557\n",
      "['two', 'three', 'several', 'four', 'five', 'few', 'six', 'many', 'ten', 'eight']\n",
      "Average loss at step  146000 :  4.79642269635\n",
      "['two', 'three', 'several', 'four', 'five', 'few', 'six', 'many', 'eight', 'ten']\n",
      "Average loss at step  148000 :  4.88085806072\n",
      "['two', 'three', 'several', 'five', 'four', 'six', 'eight', 'few', 'ten', 'many']\n",
      "Average loss at step  150000 :  4.86272187448\n",
      "['two', 'three', 'several', 'five', 'eight', 'four', 'six', 'seven', 'weeks', 'few']\n",
      "Average loss at step  152000 :  4.83039597917\n",
      "['two', 'three', 'several', 'four', 'eight', 'five', 'six', 'weeks', 'seven', 'many']\n",
      "Average loss at step  154000 :  4.81285980296\n",
      "['two', 'three', 'four', 'eight', 'several', 'five', 'six', 'weeks', 'seven', 'ten']\n",
      "Average loss at step  156000 :  4.84281280565\n",
      "['two', 'three', 'four', 'eight', 'several', 'six', 'five', 'weeks', 'persons', 'seven']\n",
      "Average loss at step  158000 :  4.83311832201\n",
      "['two', 'three', 'four', 'eight', 'several', 'six', 'five', 'weeks', 'seven', 'fifteen']\n",
      "Average loss at step  160000 :  4.72155948985\n",
      "['two', 'three', 'four', 'eight', 'several', 'six', 'five', 'weeks', 'persons', 'seven']\n",
      "Average loss at step  162000 :  4.78122199547\n",
      "['two', 'three', 'four', 'eight', 'several', 'six', 'persons', 'five', 'numbers', 'weeks']\n",
      "Average loss at step  164000 :  4.77687848532\n",
      "['two', 'three', 'four', 'eight', 'persons', 'several', 'six', 'numbers', 'men', 'five']\n",
      "Average loss at step  166000 :  4.7662567091\n",
      "['two', 'three', 'four', 'eight', 'persons', 'numbers', 'six', 'five', 'several', 'men']\n",
      "Average loss at step  168000 :  4.78634997857\n",
      "['two', 'three', 'four', 'eight', 'persons', 'six', 'five', 'numbers', 'several', 'companies']\n",
      "Average loss at step  170000 :  4.78300127792\n",
      "['two', 'three', 'four', 'eight', 'persons', 'six', 'several', 'five', 'numbers', 'companies']\n",
      "Average loss at step  172000 :  4.72786782742\n",
      "['two', 'three', 'four', 'eight', 'six', 'persons', 'five', 'several', 'numbers', 'companies']\n",
      "Average loss at step  174000 :  4.7474600594\n",
      "['two', 'three', 'eight', 'four', 'six', 'five', 'persons', 'numbers', 'several', 'companies']\n",
      "Average loss at step  176000 :  4.72240144277\n",
      "['two', 'three', 'eight', 'four', 'five', 'six', 'persons', 'numbers', 'several', 'companies']\n",
      "Average loss at step  178000 :  4.74791362894\n",
      "['two', 'three', 'eight', 'four', 'five', 'six', 'persons', 'numbers', 'several', 'seven']\n",
      "Average loss at step  180000 :  4.73102366292\n",
      "['two', 'three', 'eight', 'five', 'four', 'six', 'numbers', 'persons', 'several', 'seven']\n",
      "Average loss at step  182000 :  4.70323960078\n",
      "['two', 'three', 'eight', 'five', 'six', 'four', 'persons', 'numbers', 'several', 'seven']\n",
      "Average loss at step  184000 :  4.67723438215\n",
      "['two', 'three', 'eight', 'five', 'six', 'four', 'several', 'persons', 'numbers', 'seven']\n",
      "Average loss at step  186000 :  4.53549463391\n",
      "['two', 'three', 'eight', 'five', 'four', 'six', 'several', 'persons', 'numbers', 'seven']\n",
      "Average loss at step  188000 :  4.54519453967\n",
      "['two', 'three', 'eight', 'four', 'five', 'six', 'several', 'persons', 'companies', 'numbers']\n",
      "Average loss at step  190000 :  4.63702347648\n",
      "['two', 'three', 'six', 'eight', 'four', 'five', 'several', 'persons', 'results', 'fifteen']\n",
      "Average loss at step  192000 :  4.57538958228\n",
      "['two', 'three', 'four', 'eight', 'six', 'results', 'five', 'differences', 'companies', 'several']\n",
      "Average loss at step  194000 :  4.5856169672\n",
      "['two', 'three', 'results', 'four', 'differences', 'eight', 'companies', 'five', 'six', 'several']\n",
      "Average loss at step  196000 :  4.67113605177\n",
      "['two', 'three', 'results', 'differences', 'four', 'five', 'eight', 'six', 'companies', 'several']\n",
      "Average loss at step  198000 :  4.68841110003\n",
      "['two', 'three', 'four', 'differences', 'five', 'results', 'eight', 'several', 'companies', 'fall']\n"
     ]
    }
   ],
   "source": [
    "num_steps = 200000\n",
    "MODEL = './model/brown-Words2Vec-{}.ckpt'.format(DIMENSION)\n",
    "\n",
    "with tf.Session(graph=graph) as session:\n",
    "    init.run()\n",
    "      \n",
    "    average_loss = 0\n",
    "    for step in range(num_steps):\n",
    "        batch_inputs, batch_labels = next(generator)\n",
    "        feed_dict = {inputs: batch_inputs, labels: batch_labels}\n",
    "\n",
    "        _, loss_val = session.run([optimizer, loss], feed_dict=feed_dict)\n",
    "        average_loss += loss_val\n",
    "\n",
    "        if step % 2000 == 0:\n",
    "            if step > 0:\n",
    "                average_loss /= 2000\n",
    "                print('Average loss at step ', step, ': ', average_loss)\n",
    "                average_loss = 0\n",
    "                \n",
    "                emb = embeddings.eval()\n",
    "                normalize(emb, norm='l2', axis=1, copy=False)\n",
    "                words_vec = {}\n",
    "                for i in range(emb.shape[0]):\n",
    "                    words_vec[inv_words_dict[i]] = emb[i]\n",
    "                print(cloestWord('two', words_vec, method='cos'))\n",
    "                \n",
    "                \n",
    "    save_path = word2VecSaver.save(session, MODEL)\n",
    "    final_embeddings = embeddings.eval()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(49815, 50)\n"
     ]
    }
   ],
   "source": [
    "from sklearn.preprocessing import normalize\n",
    "\n",
    "normalize(final_embeddings, norm='l2', axis=1, copy=False)\n",
    "print(final_embeddings.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1.0000000533964339"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sum((final_embeddings[2] ** 2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "words_vec = {}\n",
    "for i in range(final_embeddings.shape[0]):\n",
    "    words_vec[inv_words_dict[i]] = final_embeddings[i]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "def plotData(vocabs, X, Y):\n",
    "    plt.clf()\n",
    "    plt.figure(figsize=(36, 36))\n",
    "    plt.scatter(X, Y)\n",
    "    plt.axis([min(X), max(X), min(Y), max(Y)])\n",
    "    for label, x, y in zip(vocabs, X, Y):\n",
    "        plt.annotate(label, xy=(x, y), xytext=(0, 0), textcoords='offset points')\n",
    "    plt.show()    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "def plot(vocabs, words_vec):\n",
    "    X = [words_vec[vocab][0] for vocab in vocabs]\n",
    "    Y = [words_vec[vocab][1] for vocab in vocabs]\n",
    "    plotData(vocabs, X, Y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "def plotTSNE(vocabs, vectors):\n",
    "    tsne = TSNE(perplexity=30, n_components=2, n_iter=5000, random_state = 7890, method='exact')\n",
    "    #np.set_printoptions(suppress=True)\n",
    "    data = np.array([vectors[vocab] for vocab in vocabs])    \n",
    "    DATA = tsne.fit_transform(data)\n",
    "    X = DATA[:, 0]\n",
    "    Y = DATA[:, 1]\n",
    "    \n",
    "    plotData(vocabs, X, Y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Start: Loading Glove Model\n",
      "End: Loaded 400000 rows.\n"
     ]
    }
   ],
   "source": [
    "from DataLoader import GloVe\n",
    "\n",
    "glove = GloVe.load2('./data/GloVe/glove.6B.50d.txt')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1611\n"
     ]
    }
   ],
   "source": [
    "vocabs = ['man', 'woman', 'king', 'queen', 'male', 'female', 'boy', 'girl']\n",
    "np.random.seed(1234)\n",
    "\n",
    "random_vocabs = []\n",
    "for i in np.random.randint(0, len(words_dict), 2000):\n",
    "    if inv_words_dict[i] in glove.keys():\n",
    "        random_vocabs.append(inv_words_dict[i])\n",
    "        \n",
    "print(len(random_vocabs))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "#plotTSNE(random_vocabs, words_vec)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "#plotTSNE(random_vocabs, glove)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['man', 'woman', 'person', 'child', 'student', 'girl', 'lady', 'boy', 'officer', 'party']\n",
      "['man', 'woman', 'another', 'boy', 'one', 'old', 'turned', 'whose', 'himself', 'who']\n"
     ]
    }
   ],
   "source": [
    "print(cloestWord('man', words_vec, method='cos'))\n",
    "print(cloestWord('man', glove))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['woman', 'boy', 'man', 'child', 'girl', 'person', 'student', 'writer', 'emory', 'teacher']\n",
      "['woman', 'girl', 'man', 'mother', 'boy', 'her', 'she', 'herself', 'victim', 'child']\n"
     ]
    }
   ],
   "source": [
    "print(cloestWord('woman', words_vec, method='cos'))\n",
    "print(cloestWord('woman', glove))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['however', 'but', 'although', 'indeed', 'since', 'nevertheless', 'perhaps', 'then', 'unfortunately', 'neither']\n",
      "['however', 'although', 'though', 'as', 'both', 'latter', 'also', '.', 'same', 'fact']\n"
     ]
    }
   ],
   "source": [
    "print(cloestWord('however', words_vec, method='cos'))\n",
    "print(cloestWord('however', glove))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['his', 'her', 'the', 'its', 'their', 'our', 'my', 'your', 'a', 'this']\n",
      "['his', 'he', 'himself', 'him', 'took', 'came', 'her', 'when', 'having', 'she']\n"
     ]
    }
   ],
   "source": [
    "print(cloestWord('his', words_vec, method='cos'))\n",
    "print(cloestWord('his', glove))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['zero', 'screaming', 'shelters', 'scenic', 'debts', 'correct', 'discouraged', 'games', 'unity', 'cranston']\n",
      "['zero', 'mean', 'assuming', 'hence', 'actual', 'approaching', 'comparable', 'measured', 'limit', 'comparison']\n"
     ]
    }
   ],
   "source": [
    "print(cloestWord('zero', words_vec, method='cos'))\n",
    "print(cloestWord('zero', glove))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['one', 'none', 'schwarzkopf', 'matter', 'speaking', 'question', 'apparent', 'some', 'plenty', 'mantle']\n",
      "['one', 'another', 'only', 'same', '.', 'as', 'well', 'but', 'with', 'making']\n"
     ]
    }
   ],
   "source": [
    "print(cloestWord('one', words_vec, method='cos'))\n",
    "print(cloestWord('one', glove))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['two', 'three', 'four', 'differences', 'five', 'eight', 'several', 'six', 'companies', 'helping']\n",
      "['two', 'three', 'four', 'five', 'six', 'seven', 'eight', 'nine', 'with', 'several']\n"
     ]
    }
   ],
   "source": [
    "print(cloestWord('two', words_vec, method='cos'))\n",
    "print(cloestWord('two', glove))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}