q3-generate.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import getpass\n",
    "import sys\n",
    "import time\n",
    "\n",
    "import numpy as np\n",
    "from copy import deepcopy\n",
    "\n",
    "from utils import calculate_perplexity, get_ptb_dataset, Vocab\n",
    "from utils import ptb_iterator, sample\n",
    "\n",
    "import tensorflow as tf\n",
    "from tensorflow.python.ops.seq2seq import sequence_loss\n",
    "from model import LanguageModel"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from q3_RNNLM import RNNLM_Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "class Config(object):\n",
    "  \"\"\"Holds model hyperparams and data information.\n",
    "\n",
    "  The config class is used to store various hyperparameters and dataset\n",
    "  information parameters. Model objects are passed a Config() object at\n",
    "  instantiation.\n",
    "  \"\"\"\n",
    "  batch_size = 64\n",
    "  embed_size = 50\n",
    "  hidden_size = 100\n",
    "  num_steps = 10\n",
    "  max_epochs = 16\n",
    "  early_stopping = 2\n",
    "  dropout = 0.9\n",
    "  lr = 0.001\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "class Self:\n",
    "    vocab = Vocab()\n",
    "    config = Config()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "self = Self()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "929589.0 total words with 10000 uniques\n"
     ]
    }
   ],
   "source": [
    "self.vocab.construct(get_ptb_dataset('train'))\n",
    "self.encoded_train = np.array(\n",
    "    [self.vocab.encode(word) for word in get_ptb_dataset('train')],\n",
    "    dtype=np.int32)\n",
    "self.encoded_valid = np.array(\n",
    "    [self.vocab.encode(word) for word in get_ptb_dataset('valid')],\n",
    "    dtype=np.int32)\n",
    "self.encoded_test = np.array(\n",
    "    [self.vocab.encode(word) for word in get_ptb_dataset('test')],\n",
    "    dtype=np.int32)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Restore trained model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "929589.0 total words with 10000 uniques\n",
      "929589.0 total words with 10000 uniques\n"
     ]
    }
   ],
   "source": [
    "config = Config()\n",
    "gen_config = deepcopy(config)\n",
    "gen_config.batch_size = gen_config.num_steps = 1\n",
    "\n",
    "# We create the training model and generative model\n",
    "with tf.variable_scope('RNNLM') as scope:\n",
    "    model = RNNLM_Model(config)\n",
    "    # This instructs gen_model to reuse the same variables as the model above\n",
    "    scope.reuse_variables()\n",
    "    gen_model = RNNLM_Model(gen_config)\n",
    "\n",
    "init = tf.global_variables_initializer()\n",
    "saver = tf.train.Saver()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "session = tf.InteractiveSession()\n",
    "session.run(init)\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "saver.restore(session, './weights-rnnlm/ptb_rnnlm.weights')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "=-==-==-==-==-=\n",
      "Test perplexity: 158.574432373\n",
      "=-==-==-==-==-=\n"
     ]
    }
   ],
   "source": [
    "test_pp = model.run_epoch(session, model.encoded_test)\n",
    "print '=-=' * 5\n",
    "print 'Test perplexity: {}'.format(test_pp)\n",
    "print '=-=' * 5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": []
    }
   ],
   "source": [
    "valid_pp = model.run_epoch(session, model.encoded_valid)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "174.01003"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "valid_pp"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Generator"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "state = gen_model.initial_state.eval()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,\n",
       "         0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,\n",
       "         0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,\n",
       "         0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,\n",
       "         0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,\n",
       "         0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,\n",
       "         0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,\n",
       "         0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.]], dtype=float32)"
      ]
     },
     "execution_count": 69,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "state"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1, 100)"
      ]
     },
     "execution_count": 70,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "state.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "starting_text = 'in palo alto'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "tokens = [gen_model.vocab.encode(word) for word in starting_text.split()]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[108, 7639, 7640]"
      ]
     },
     "execution_count": 73,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tokens"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "inputs = [tokens[-1:]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[[7640]]"
      ]
     },
     "execution_count": 75,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "inputs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "\n",
    "state, y_pred = session.run([gen_model.final_state, gen_model.predictions[-1]], feed_dict={\n",
    "    gen_model.input_placeholder: inputs,\n",
    "    gen_model.dropout_placeholder: 1,\n",
    "    gen_model.initial_state: state,\n",
    "})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[  2.69278278e-06,   2.39400225e-04,   8.66865935e-06, ...,\n",
       "          9.53101021e-04,   8.45851581e-07,   2.09635556e-05]])"
      ]
     },
     "execution_count": 77,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_pred"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python [conda env:stanford]",
   "language": "python",
   "name": "conda-env-stanford-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}