bow_classifier_features.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# What is the network learning?\n",
    "\n",
    "* We can gain some intuition in what the network is learning\n",
    "* Especially fruitful will turn out to look at the weights of the hidden layer\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<keras.engine.training.Model object at 0x7f348132e1d0>\n"
     ]
    }
   ],
   "source": [
    "import json\n",
    "import pickle\n",
    "from keras.models import Model, model_from_json\n",
    "\n",
    "\n",
    "def load_model(model_name):\n",
    "\n",
    "    with open(model_name+\".model.json\", \"rt\") as f:\n",
    "        model=model_from_json(f.read())\n",
    "    model.load_weights(model_name+\".weights.h5\")\n",
    "    \n",
    "    with open(model_name+\".encoders.pickle\",\"rb\") as f:\n",
    "        label_encoder,vectorizer=pickle.load(f)\n",
    "    \n",
    "    return model,label_encoder,vectorizer\n",
    "\n",
    "model,label_encoder,vectorizer=load_model(\"models/imdb_bow\")\n",
    "print(model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/svg+xml": [
       "<svg height=\"200pt\" viewBox=\"0.00 0.00 236.00 221.00\" width=\"213pt\" xmlns=\"http://www.w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\">\n",
       "<g class=\"graph\" id=\"graph0\" transform=\"scale(.9028 .9028) rotate(0) translate(4 217)\">\n",
       "<title>G</title>\n",
       "<polygon fill=\"#ffffff\" points=\"-4,4 -4,-217 232,-217 232,4 -4,4\" stroke=\"transparent\"/>\n",
       "<!-- 139863482828816 -->\n",
       "<g class=\"node\" id=\"node1\">\n",
       "<title>139863482828816</title>\n",
       "<polygon fill=\"none\" points=\"0,-166.5 0,-212.5 228,-212.5 228,-166.5 0,-166.5\" stroke=\"#000000\"/>\n",
       "<text fill=\"#000000\" font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"38.5\" y=\"-185.8\">InputLayer</text>\n",
       "<polyline fill=\"none\" points=\"77,-166.5 77,-212.5 \" stroke=\"#000000\"/>\n",
       "<text fill=\"#000000\" font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"104.5\" y=\"-197.3\">input:</text>\n",
       "<polyline fill=\"none\" points=\"77,-189.5 132,-189.5 \" stroke=\"#000000\"/>\n",
       "<text fill=\"#000000\" font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"104.5\" y=\"-174.3\">output:</text>\n",
       "<polyline fill=\"none\" points=\"132,-166.5 132,-212.5 \" stroke=\"#000000\"/>\n",
       "<text fill=\"#000000\" font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"180\" y=\"-197.3\">(None, 74849)</text>\n",
       "<polyline fill=\"none\" points=\"132,-189.5 228,-189.5 \" stroke=\"#000000\"/>\n",
       "<text fill=\"#000000\" font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"180\" y=\"-174.3\">(None, 74849)</text>\n",
       "</g>\n",
       "<!-- 139863482828240 -->\n",
       "<g class=\"node\" id=\"node2\">\n",
       "<title>139863482828240</title>\n",
       "<polygon fill=\"none\" points=\"13.5,-83.5 13.5,-129.5 214.5,-129.5 214.5,-83.5 13.5,-83.5\" stroke=\"#000000\"/>\n",
       "<text fill=\"#000000\" font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"38.5\" y=\"-102.8\">Dense</text>\n",
       "<polyline fill=\"none\" points=\"63.5,-83.5 63.5,-129.5 \" stroke=\"#000000\"/>\n",
       "<text fill=\"#000000\" font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"91\" y=\"-114.3\">input:</text>\n",
       "<polyline fill=\"none\" points=\"63.5,-106.5 118.5,-106.5 \" stroke=\"#000000\"/>\n",
       "<text fill=\"#000000\" font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"91\" y=\"-91.3\">output:</text>\n",
       "<polyline fill=\"none\" points=\"118.5,-83.5 118.5,-129.5 \" stroke=\"#000000\"/>\n",
       "<text fill=\"#000000\" font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"166.5\" y=\"-114.3\">(None, 74849)</text>\n",
       "<polyline fill=\"none\" points=\"118.5,-106.5 214.5,-106.5 \" stroke=\"#000000\"/>\n",
       "<text fill=\"#000000\" font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"166.5\" y=\"-91.3\">(None, 200)</text>\n",
       "</g>\n",
       "<!-- 139863482828816&#45;&gt;139863482828240 -->\n",
       "<g class=\"edge\" id=\"edge1\">\n",
       "<title>139863482828816-&gt;139863482828240</title>\n",
       "<path d=\"M114,-166.3799C114,-158.1745 114,-148.7679 114,-139.8786\" fill=\"none\" stroke=\"#000000\"/>\n",
       "<polygon fill=\"#000000\" points=\"117.5001,-139.784 114,-129.784 110.5001,-139.784 117.5001,-139.784\" stroke=\"#000000\"/>\n",
       "</g>\n",
       "<!-- 139863484386832 -->\n",
       "<g class=\"node\" id=\"node3\">\n",
       "<title>139863484386832</title>\n",
       "<polygon fill=\"none\" points=\"20,-.5 20,-46.5 208,-46.5 208,-.5 20,-.5\" stroke=\"#000000\"/>\n",
       "<text fill=\"#000000\" font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"45\" y=\"-19.8\">Dense</text>\n",
       "<polyline fill=\"none\" points=\"70,-.5 70,-46.5 \" stroke=\"#000000\"/>\n",
       "<text fill=\"#000000\" font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"97.5\" y=\"-31.3\">input:</text>\n",
       "<polyline fill=\"none\" points=\"70,-23.5 125,-23.5 \" stroke=\"#000000\"/>\n",
       "<text fill=\"#000000\" font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"97.5\" y=\"-8.3\">output:</text>\n",
       "<polyline fill=\"none\" points=\"125,-.5 125,-46.5 \" stroke=\"#000000\"/>\n",
       "<text fill=\"#000000\" font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"166.5\" y=\"-31.3\">(None, 200)</text>\n",
       "<polyline fill=\"none\" points=\"125,-23.5 208,-23.5 \" stroke=\"#000000\"/>\n",
       "<text fill=\"#000000\" font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"166.5\" y=\"-8.3\">(None, 2)</text>\n",
       "</g>\n",
       "<!-- 139863482828240&#45;&gt;139863484386832 -->\n",
       "<g class=\"edge\" id=\"edge2\">\n",
       "<title>139863482828240-&gt;139863484386832</title>\n",
       "<path d=\"M114,-83.3799C114,-75.1745 114,-65.7679 114,-56.8786\" fill=\"none\" stroke=\"#000000\"/>\n",
       "<polygon fill=\"#000000\" points=\"117.5001,-56.784 114,-46.784 110.5001,-56.784 117.5001,-56.784\" stroke=\"#000000\"/>\n",
       "</g>\n",
       "</g>\n",
       "</svg>"
      ],
      "text/plain": [
       "<IPython.core.display.SVG object>"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from IPython.display import SVG\n",
    "from keras.utils.vis_utils import model_to_dot\n",
    "\n",
    "SVG(model_to_dot(model,show_shapes=True,show_layer_names=False,dpi=65).create(prog='dot', format='svg'))\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "* This is our model (watch out, Keras plots models from top to bottom)\n",
    "* Hidden layer has N nodes with 74849 inputs each, one input for one word in the vocabulary\n",
    "* We can also look at it the other way around: each word is assigned one weight for each hidden layer node\n",
    "* So each word is seen by the network as a N-dimensional vector\n",
    "* But what are these vectors? What properties should they have? Let us find out!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(74849, 200)"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "learned_weights=model.layers[1].get_weights()[0]\n",
    "learned_weights.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "* A good and easy way to explore the vectors is through their neighborhood\n",
    "* This is easy to implement so let's try"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "bad, mst3k, stinker, fest, baldwin, embarrassing, lousy, mushrooms, knockoff, unconvincing, save, degenerate, unlikeable, horrid, avoid, nisha, kothari, poor, recycling, lifeless, mess, dreadful, disjointed, squandered, signal, cohesion, monotonous, wooden, horrible, incoherent, \n",
      "terrible, tubes, fest, ridiculous, overacts, qualified, alright, fails, hackers, signal, miscast, unwatchable, disjointed, squandered, mess, massacres, appalling, mst3k, boredom, horrible, paycheck, stinker, cardboard, tiresome, kareena, embarrassing, programming, knockoff, horrid, mushrooms, \n",
      "great, preferring, awe, mcintire, wonderful, loved, appreciated, excellently, fantastic, amazing, boothe, perfect, surprisingly, superb, antwone, glee, judson, intricately, natures, sumpter, quintessential, heavenly, infernal, superbly, moodiness, riff, admiration, enjoyable, flawless, mccartney, \n",
      "enjoyable, loved, flawless, awe, superb, great, quintessential, perfect, natures, amazing, fantastic, today, mcintire, antwone, infernal, glee, excellently, surprisingly, admiration, underrated, preferring, sumpter, judson, intricately, ralph, superbly, complain, appreciated, wonderful, moodiness, \n"
     ]
    }
   ],
   "source": [
    "import numpy\n",
    "\n",
    "def nearest(word,learned_weights,vectorizer):\n",
    "    inverse_vocab=dict((v,k) for k,v in vectorizer.vocabulary_.items())\n",
    "    word_idx=vectorizer.vocabulary_[word]\n",
    "    word_vector=learned_weights[word_idx]\n",
    "    x=numpy.linalg.norm(word_vector-learned_weights,axis=-1)\n",
    "    nearest=numpy.argsort(x)\n",
    "    for idx in nearest[:30]:\n",
    "        print(inverse_vocab[idx], end=\", \")\n",
    "    print()\n",
    "    \n",
    "nearest(\"bad\",learned_weights,vectorizer)\n",
    "nearest(\"terrible\",learned_weights,vectorizer)\n",
    "\n",
    "nearest(\"great\",learned_weights,vectorizer)\n",
    "nearest(\"enjoyable\",learned_weights,vectorizer)\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}