Lab_12_Hello_RNN.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "tf.set_random_seed(777)\n",
    "idx2char = ['h', 'i', 'e', 'l', 'o']\n",
    "# Teach hello: hihell -> ihello\n",
    "x_data = [[0, 1, 0, 2, 3, 3]]   # hihell\n",
    "x_one_hot = [[[1, 0, 0, 0, 0],   # h 0\n",
    "              [0, 1, 0, 0, 0],   # i 1\n",
    "              [1, 0, 0, 0, 0],   # h 0\n",
    "              [0, 0, 1, 0, 0],   # e 2\n",
    "              [0, 0, 0, 1, 0],   # l 3\n",
    "              [0, 0, 0, 1, 0]]]  # l 3\n",
    "\n",
    "y_data = [[1, 0, 2, 3, 3, 4]]    # ihello"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "num_classes = 5\n",
    "input_dim = 5\n",
    "hidden_size = 5\n",
    "batch_size = 1\n",
    "sequence_length = 6\n",
    "learning_rate = 0.1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "X =  tf.placeholder(tf.float32,[None,sequence_length, input_dim]) #X one-hot\n",
    "Y = tf.placeholder(tf.int32,[None, sequence_length]) #Y Label"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "cell = tf.contrib.rnn.BasicLSTMCell(num_units = hidden_size, state_is_tuple =True)\n",
    "initial_state = cell.zero_state(batch_size, tf.float32)\n",
    "outputs, _states = tf.nn.dynamic_rnn(cell, X, initial_state =initial_state, dtype = tf.float32)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "#FCL\n",
    "X_for_fc = tf.reshape(outputs, [-1, hidden_size])\n",
    "outputs = tf.contrib.layers.fully_connected(inputs= X_for_fc, \n",
    "                                            num_outputs= num_classes, activation_fn = None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "#reshape out for squence loss\n",
    "outputs = tf.reshape(outputs, [batch_size, sequence_length, num_classes])\n",
    "weights = tf.ones([batch_size, sequence_length])\n",
    "sequence_loss = tf.contrib.seq2seq.sequence_loss(\n",
    "    logits = outputs, targets = Y, weights= weights)\n",
    "loss = tf.reduce_mean(sequence_loss)\n",
    "train = tf.train.AdamOptimizer(learning_rate = learning_rate).minimize(loss)\n",
    "prediction = tf.argmax(outputs, axis=2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 loss:  1.55194 prediction:  [[3 3 3 3 3 3]] true Y:  [[1, 0, 2, 3, 3, 4]]\n",
      "\tPrediction str:  llllll\n",
      "1 loss:  1.43735 prediction:  [[3 3 3 3 3 3]] true Y:  [[1, 0, 2, 3, 3, 4]]\n",
      "\tPrediction str:  llllll\n",
      "2 loss:  1.36917 prediction:  [[3 3 3 3 3 3]] true Y:  [[1, 0, 2, 3, 3, 4]]\n",
      "\tPrediction str:  llllll\n",
      "3 loss:  1.23897 prediction:  [[3 0 3 3 3 3]] true Y:  [[1, 0, 2, 3, 3, 4]]\n",
      "\tPrediction str:  lhllll\n",
      "4 loss:  1.0793 prediction:  [[0 0 0 3 3 4]] true Y:  [[1, 0, 2, 3, 3, 4]]\n",
      "\tPrediction str:  hhhllo\n",
      "5 loss:  0.925557 prediction:  [[0 0 0 3 3 4]] true Y:  [[1, 0, 2, 3, 3, 4]]\n",
      "\tPrediction str:  hhhllo\n",
      "6 loss:  0.772792 prediction:  [[0 0 2 3 3 4]] true Y:  [[1, 0, 2, 3, 3, 4]]\n",
      "\tPrediction str:  hhello\n",
      "7 loss:  0.631739 prediction:  [[0 0 2 3 3 4]] true Y:  [[1, 0, 2, 3, 3, 4]]\n",
      "\tPrediction str:  hhello\n",
      "8 loss:  0.521488 prediction:  [[0 0 2 3 3 4]] true Y:  [[1, 0, 2, 3, 3, 4]]\n",
      "\tPrediction str:  hhello\n",
      "9 loss:  0.441557 prediction:  [[1 0 2 3 3 4]] true Y:  [[1, 0, 2, 3, 3, 4]]\n",
      "\tPrediction str:  ihello\n",
      "10 loss:  0.378233 prediction:  [[1 0 2 3 3 4]] true Y:  [[1, 0, 2, 3, 3, 4]]\n",
      "\tPrediction str:  ihello\n",
      "11 loss:  0.320094 prediction:  [[1 0 2 3 3 4]] true Y:  [[1, 0, 2, 3, 3, 4]]\n",
      "\tPrediction str:  ihello\n",
      "12 loss:  0.264802 prediction:  [[1 0 2 3 3 4]] true Y:  [[1, 0, 2, 3, 3, 4]]\n",
      "\tPrediction str:  ihello\n",
      "13 loss:  0.217596 prediction:  [[1 0 2 3 3 4]] true Y:  [[1, 0, 2, 3, 3, 4]]\n",
      "\tPrediction str:  ihello\n",
      "14 loss:  0.180873 prediction:  [[1 0 2 3 3 4]] true Y:  [[1, 0, 2, 3, 3, 4]]\n",
      "\tPrediction str:  ihello\n",
      "15 loss:  0.152059 prediction:  [[1 0 2 3 3 4]] true Y:  [[1, 0, 2, 3, 3, 4]]\n",
      "\tPrediction str:  ihello\n",
      "16 loss:  0.127219 prediction:  [[1 0 2 3 3 4]] true Y:  [[1, 0, 2, 3, 3, 4]]\n",
      "\tPrediction str:  ihello\n",
      "17 loss:  0.104071 prediction:  [[1 0 2 3 3 4]] true Y:  [[1, 0, 2, 3, 3, 4]]\n",
      "\tPrediction str:  ihello\n",
      "18 loss:  0.0826445 prediction:  [[1 0 2 3 3 4]] true Y:  [[1, 0, 2, 3, 3, 4]]\n",
      "\tPrediction str:  ihello\n",
      "19 loss:  0.0640462 prediction:  [[1 0 2 3 3 4]] true Y:  [[1, 0, 2, 3, 3, 4]]\n",
      "\tPrediction str:  ihello\n",
      "20 loss:  0.0490863 prediction:  [[1 0 2 3 3 4]] true Y:  [[1, 0, 2, 3, 3, 4]]\n",
      "\tPrediction str:  ihello\n",
      "21 loss:  0.0377641 prediction:  [[1 0 2 3 3 4]] true Y:  [[1, 0, 2, 3, 3, 4]]\n",
      "\tPrediction str:  ihello\n",
      "22 loss:  0.0294893 prediction:  [[1 0 2 3 3 4]] true Y:  [[1, 0, 2, 3, 3, 4]]\n",
      "\tPrediction str:  ihello\n",
      "23 loss:  0.0234952 prediction:  [[1 0 2 3 3 4]] true Y:  [[1, 0, 2, 3, 3, 4]]\n",
      "\tPrediction str:  ihello\n",
      "24 loss:  0.019105 prediction:  [[1 0 2 3 3 4]] true Y:  [[1, 0, 2, 3, 3, 4]]\n",
      "\tPrediction str:  ihello\n",
      "25 loss:  0.0158157 prediction:  [[1 0 2 3 3 4]] true Y:  [[1, 0, 2, 3, 3, 4]]\n",
      "\tPrediction str:  ihello\n",
      "26 loss:  0.0132834 prediction:  [[1 0 2 3 3 4]] true Y:  [[1, 0, 2, 3, 3, 4]]\n",
      "\tPrediction str:  ihello\n",
      "27 loss:  0.0112821 prediction:  [[1 0 2 3 3 4]] true Y:  [[1, 0, 2, 3, 3, 4]]\n",
      "\tPrediction str:  ihello\n",
      "28 loss:  0.00966547 prediction:  [[1 0 2 3 3 4]] true Y:  [[1, 0, 2, 3, 3, 4]]\n",
      "\tPrediction str:  ihello\n",
      "29 loss:  0.00833894 prediction:  [[1 0 2 3 3 4]] true Y:  [[1, 0, 2, 3, 3, 4]]\n",
      "\tPrediction str:  ihello\n",
      "30 loss:  0.00724114 prediction:  [[1 0 2 3 3 4]] true Y:  [[1, 0, 2, 3, 3, 4]]\n",
      "\tPrediction str:  ihello\n",
      "31 loss:  0.00633073 prediction:  [[1 0 2 3 3 4]] true Y:  [[1, 0, 2, 3, 3, 4]]\n",
      "\tPrediction str:  ihello\n",
      "32 loss:  0.00557797 prediction:  [[1 0 2 3 3 4]] true Y:  [[1, 0, 2, 3, 3, 4]]\n",
      "\tPrediction str:  ihello\n",
      "33 loss:  0.00495955 prediction:  [[1 0 2 3 3 4]] true Y:  [[1, 0, 2, 3, 3, 4]]\n",
      "\tPrediction str:  ihello\n",
      "34 loss:  0.00445539 prediction:  [[1 0 2 3 3 4]] true Y:  [[1, 0, 2, 3, 3, 4]]\n",
      "\tPrediction str:  ihello\n",
      "35 loss:  0.0040472 prediction:  [[1 0 2 3 3 4]] true Y:  [[1, 0, 2, 3, 3, 4]]\n",
      "\tPrediction str:  ihello\n",
      "36 loss:  0.00371787 prediction:  [[1 0 2 3 3 4]] true Y:  [[1, 0, 2, 3, 3, 4]]\n",
      "\tPrediction str:  ihello\n",
      "37 loss:  0.00345157 prediction:  [[1 0 2 3 3 4]] true Y:  [[1, 0, 2, 3, 3, 4]]\n",
      "\tPrediction str:  ihello\n",
      "38 loss:  0.00323401 prediction:  [[1 0 2 3 3 4]] true Y:  [[1, 0, 2, 3, 3, 4]]\n",
      "\tPrediction str:  ihello\n",
      "39 loss:  0.00305262 prediction:  [[1 0 2 3 3 4]] true Y:  [[1, 0, 2, 3, 3, 4]]\n",
      "\tPrediction str:  ihello\n",
      "40 loss:  0.00289698 prediction:  [[1 0 2 3 3 4]] true Y:  [[1, 0, 2, 3, 3, 4]]\n",
      "\tPrediction str:  ihello\n",
      "41 loss:  0.00275899 prediction:  [[1 0 2 3 3 4]] true Y:  [[1, 0, 2, 3, 3, 4]]\n",
      "\tPrediction str:  ihello\n",
      "42 loss:  0.00263276 prediction:  [[1 0 2 3 3 4]] true Y:  [[1, 0, 2, 3, 3, 4]]\n",
      "\tPrediction str:  ihello\n",
      "43 loss:  0.0025147 prediction:  [[1 0 2 3 3 4]] true Y:  [[1, 0, 2, 3, 3, 4]]\n",
      "\tPrediction str:  ihello\n",
      "44 loss:  0.00240271 prediction:  [[1 0 2 3 3 4]] true Y:  [[1, 0, 2, 3, 3, 4]]\n",
      "\tPrediction str:  ihello\n",
      "45 loss:  0.00229617 prediction:  [[1 0 2 3 3 4]] true Y:  [[1, 0, 2, 3, 3, 4]]\n",
      "\tPrediction str:  ihello\n",
      "46 loss:  0.00219532 prediction:  [[1 0 2 3 3 4]] true Y:  [[1, 0, 2, 3, 3, 4]]\n",
      "\tPrediction str:  ihello\n",
      "47 loss:  0.00210076 prediction:  [[1 0 2 3 3 4]] true Y:  [[1, 0, 2, 3, 3, 4]]\n",
      "\tPrediction str:  ihello\n",
      "48 loss:  0.00201297 prediction:  [[1 0 2 3 3 4]] true Y:  [[1, 0, 2, 3, 3, 4]]\n",
      "\tPrediction str:  ihello\n",
      "49 loss:  0.0019327 prediction:  [[1 0 2 3 3 4]] true Y:  [[1, 0, 2, 3, 3, 4]]\n",
      "\tPrediction str:  ihello\n"
     ]
    }
   ],
   "source": [
    "with tf.Session() as sess:\n",
    "    sess.run(tf.global_variables_initializer())\n",
    "    for i in range(50):\n",
    "        l, _ = sess.run([loss, train], feed_dict={X: x_one_hot, Y: y_data})\n",
    "        result = sess.run(prediction, feed_dict = { X: x_one_hot})\n",
    "        print( i, \"loss: \", l, \"prediction: \", result, \"true Y: \", y_data)\n",
    "        \n",
    "        # print char using dic\n",
    "        result_str = [idx2char[c] for c in np.squeeze(result)]\n",
    "        print(\"\\tPrediction str: \", ''.join(result_str))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}