Skip-gram-with-NS.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using Theano backend.\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "np.random.seed(13)\n",
    "\n",
    "from keras.models import Sequential, Model\n",
    "from keras.layers import Embedding, Reshape, Activation, Input\n",
    "from keras.layers.merge import Dot\n",
    "from keras.utils import np_utils\n",
    "from keras.utils.data_utils import get_file\n",
    "from keras.preprocessing.text import Tokenizer\n",
    "from keras.preprocessing.sequence import skipgrams\n",
    "\n",
    "import gensim"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "3388"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "path = get_file('alice.txt', origin='http://www.gutenberg.org/files/11/11-0.txt')\n",
    "corpus = open(path).readlines()\n",
    "\n",
    "corpus = [sentence for sentence in corpus if sentence.count(' ') >= 2]\n",
    "tokenizer = Tokenizer()\n",
    "tokenizer.fit_on_texts(corpus)\n",
    "V = len(tokenizer.word_index) + 1\n",
    "V"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "____________________________________________________________________________________________________\n",
      "Layer (type)                     Output Shape          Param #     Connected to                     \n",
      "====================================================================================================\n",
      "input_1 (InputLayer)             (None, 1)             0                                            \n",
      "____________________________________________________________________________________________________\n",
      "input_2 (InputLayer)             (None, 1)             0                                            \n",
      "____________________________________________________________________________________________________\n",
      "embedding_1 (Embedding)          (None, 1, 128)        433664      input_1[0][0]                    \n",
      "____________________________________________________________________________________________________\n",
      "embedding_2 (Embedding)          (None, 1, 128)        433664      input_2[0][0]                    \n",
      "____________________________________________________________________________________________________\n",
      "dot_1 (Dot)                      (None, 1, 1)          0           embedding_1[0][0]                \n",
      "                                                                   embedding_2[0][0]                \n",
      "____________________________________________________________________________________________________\n",
      "reshape_1 (Reshape)              (None, 1)             0           dot_1[0][0]                      \n",
      "____________________________________________________________________________________________________\n",
      "activation_1 (Activation)        (None, 1)             0           reshape_1[0][0]                  \n",
      "====================================================================================================\n",
      "Total params: 867,328\n",
      "Trainable params: 867,328\n",
      "Non-trainable params: 0\n",
      "____________________________________________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "dim_embedddings = 128\n",
    "\n",
    "# inputs\n",
    "w_inputs = Input(shape=(1, ), dtype='int32')\n",
    "w = Embedding(V, dim_embedddings)(w_inputs)\n",
    "\n",
    "# context\n",
    "c_inputs = Input(shape=(1, ), dtype='int32')\n",
    "c  = Embedding(V, dim_embedddings)(c_inputs)\n",
    "o = Dot(axes=2)([w, c])\n",
    "o = Reshape((1,), input_shape=(1, 1))(o)\n",
    "o = Activation('sigmoid')(o)\n",
    "\n",
    "SkipGram = Model(inputs=[w_inputs, c_inputs], outputs=o)\n",
    "SkipGram.summary()\n",
    "SkipGram.compile(loss='binary_crossentropy', optimizer='adam')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1078.05439098\n",
      "741.469174184\n",
      "689.020133168\n",
      "661.459642701\n",
      "638.09191246\n"
     ]
    }
   ],
   "source": [
    "for _ in range(5):\n",
    "    loss = 0.\n",
    "    for i, doc in enumerate(tokenizer.texts_to_sequences(corpus)):\n",
    "        data, labels = skipgrams(sequence=doc, vocabulary_size=V, window_size=5, negative_samples=5.)\n",
    "        x = [np.array(x) for x in zip(*data)]\n",
    "        y = np.array(labels, dtype=np.int32)\n",
    "        if x:\n",
    "            loss += SkipGram.train_on_batch(x, y)\n",
    "\n",
    "    print(loss)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "f = open('vectors.txt' ,'w')\n",
    "f.write('{} {}\\n'.format(V-1, dim_embedddings))\n",
    "vectors = SkipGram.get_weights()[0]\n",
    "for word, i in tokenizer.word_index.items():\n",
    "    f.write('{} {}\\n'.format(word, ' '.join(map(str, list(vectors[i, :])))))\n",
    "f.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "w2v = gensim.models.KeyedVectors.load_word2vec_format('./vectors.txt', binary=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('hearts', 0.7544310092926025),\n",
       " ('king', 0.733845055103302),\n",
       " ('wildly', 0.6959143280982971),\n",
       " ('verse', 0.6711217164993286),\n",
       " ('turning', 0.6667185425758362),\n",
       " ('taken', 0.6665735244750977),\n",
       " ('cook', 0.660643458366394),\n",
       " ('trumpet', 0.62113356590271),\n",
       " ('repeated', 0.6186539530754089),\n",
       " ('pointing', 0.6166180968284607)]"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "w2v.most_similar(positive=['queen'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('thought', 0.6293938755989075),\n",
       " ('poor', 0.5651217103004456),\n",
       " ('glad', 0.5522480607032776),\n",
       " ('‘i’m', 0.5497287511825562),\n",
       " ('rather', 0.5482580065727234),\n",
       " ('feeling', 0.5272138118743896),\n",
       " ('‘yes', 0.5177067518234253),\n",
       " ('indeed', 0.516873836517334),\n",
       " ('pleaded', 0.515944242477417),\n",
       " ('hasn’t', 0.5104849338531494)]"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "w2v.most_similar(positive=['alice'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('queen', 0.5757458806037903),\n",
       " ('king', 0.5742576718330383),\n",
       " ('hearts', 0.571501612663269),\n",
       " ('outside', 0.5713008642196655),\n",
       " ('verse', 0.5598728060722351),\n",
       " ('status', 0.5433791279792786),\n",
       " ('check', 0.5398269891738892),\n",
       " ('‘call', 0.5262266993522644),\n",
       " ('u', 0.5262154340744019),\n",
       " ('mock', 0.5253645181655884)]"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "w2v.most_similar(positive=['the'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}