Neural_Network_Part2_Solutions.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "# Neural Network Part2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "from __future__ import print_function\n",
    "import numpy as np\n",
    "import tensorflow as tf\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "datetime.date(2017, 3, 22)"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from datetime import date\n",
    "date.today()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "author = \"kyubyong. https://github.com/Kyubyong/tensorflow-exercises\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'1.0.0'"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tf.__version__"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'1.12.0'"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.__version__"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "# Normalization"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "Q1. Apply `l2_normalize` to `x`."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[ 0.05096472  0.10192944  0.15289417  0.20385888  0.2548236   0.30578834\n",
      "  0.35675305  0.40771776  0.45868248  0.50964719]\n"
     ]
    }
   ],
   "source": [
    "_x = np.arange(1, 11)\n",
    "epsilon = 1e-12\n",
    "x = tf.convert_to_tensor(_x, tf.float32)\n",
    "\n",
    "output = tf.nn.l2_normalize(x, dim=0, epsilon=epsilon)\n",
    "with tf.Session() as sess:\n",
    "    _output = sess.run(output)\n",
    "\n",
    "    assert np.allclose(_output, _x / np.sqrt(np.maximum(np.sum(_x**2), epsilon)))\n",
    "print(_output)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "Q2. Calculate the mean and variance of `x` based on the sufficient statistics."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "5.5 8.25\n"
     ]
    }
   ],
   "source": [
    "_x = np.arange(1, 11)\n",
    "x = tf.convert_to_tensor(_x, tf.float32)\n",
    "\n",
    "counts_, sum_, sum_of_squares_, _ = tf.nn.sufficient_statistics(x, [0])\n",
    "mean, variance = tf.nn.normalize_moments(counts_, sum_, sum_of_squares_, shift=None)\n",
    "with tf.Session() as sess:\n",
    "    _mean, _variance = sess.run([mean, variance])\n",
    "print(_mean, _variance)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "Q3. Calculate the mean and variance of `x`."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "5.5 8.25\n"
     ]
    }
   ],
   "source": [
    "tf.reset_default_graph()\n",
    "_x = np.arange(1, 11)\n",
    "x = tf.convert_to_tensor(_x, tf.float32)\n",
    "\n",
    "output = tf.nn.moments(x, [0])\n",
    "with tf.Session() as sess:\n",
    "    _mean, _variance = sess.run(output)\n",
    "print(_mean, _variance)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "Q4. Calculate the mean and variance of `x` using `unique_x` and `counts`."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[1.8333334, 0.47222227]\n",
      "[1.8333334, 0.47222221]\n"
     ]
    }
   ],
   "source": [
    "tf.reset_default_graph()\n",
    "x = tf.constant([1, 1, 2, 2, 2, 3], tf.float32)\n",
    "\n",
    "# From `x`\n",
    "mean, variance = tf.nn.moments(x, [0])\n",
    "with tf.Session() as sess:\n",
    "    print(sess.run([mean, variance]))\n",
    "\n",
    "# From unique elements and their counts\n",
    "unique_x, _, counts = tf.unique_with_counts(x)\n",
    "mean, variance = tf.nn.weighted_moments(unique_x, [0], counts)\n",
    "with tf.Session() as sess:\n",
    "    print(sess.run([mean, variance]))\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "Q5. The code below is to implement the mnist classification task. Complete it by adding batch normalization."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Extracting MNIST_data/train-images-idx3-ubyte.gz\n",
      "Extracting MNIST_data/train-labels-idx1-ubyte.gz\n",
      "Extracting MNIST_data/t10k-images-idx3-ubyte.gz\n",
      "Extracting MNIST_data/t10k-labels-idx1-ubyte.gz\n",
      "training steps= 100 Acc. = 0.7739\n",
      "training steps= 200 Acc. = 0.8215\n",
      "training steps= 300 Acc. = 0.8529\n",
      "training steps= 400 Acc. = 0.8649\n",
      "training steps= 500 Acc. = 0.879\n",
      "training steps= 600 Acc. = 0.8804\n",
      "training steps= 700 Acc. = 0.8874\n",
      "training steps= 800 Acc. = 0.8981\n",
      "training steps= 900 Acc. = 0.8952\n",
      "training steps= 1000 Acc. = 0.8998\n",
      "training steps= 1100 Acc. = 0.9033\n",
      "training steps= 1200 Acc. = 0.9087\n",
      "training steps= 1300 Acc. = 0.9101\n",
      "training steps= 1400 Acc. = 0.9084\n",
      "training steps= 1500 Acc. = 0.9161\n",
      "training steps= 1600 Acc. = 0.9165\n",
      "training steps= 1700 Acc. = 0.9168\n",
      "training steps= 1800 Acc. = 0.9222\n",
      "training steps= 1900 Acc. = 0.9212\n",
      "training steps= 2000 Acc. = 0.9248\n",
      "training steps= 2100 Acc. = 0.9272\n",
      "training steps= 2200 Acc. = 0.9258\n",
      "training steps= 2300 Acc. = 0.9274\n",
      "training steps= 2400 Acc. = 0.9286\n",
      "training steps= 2500 Acc. = 0.9315\n",
      "training steps= 2600 Acc. = 0.931\n",
      "training steps= 2700 Acc. = 0.9275\n",
      "training steps= 2800 Acc. = 0.9344\n",
      "training steps= 2900 Acc. = 0.9304\n",
      "training steps= 3000 Acc. = 0.9306\n",
      "training steps= 3100 Acc. = 0.9316\n",
      "training steps= 3200 Acc. = 0.934\n",
      "training steps= 3300 Acc. = 0.9311\n",
      "training steps= 3400 Acc. = 0.9404\n",
      "training steps= 3500 Acc. = 0.935\n",
      "training steps= 3600 Acc. = 0.9396\n",
      "training steps= 3700 Acc. = 0.9389\n",
      "training steps= 3800 Acc. = 0.9377\n",
      "training steps= 3900 Acc. = 0.9357\n",
      "training steps= 4000 Acc. = 0.9432\n",
      "training steps= 4100 Acc. = 0.9407\n",
      "training steps= 4200 Acc. = 0.9412\n",
      "training steps= 4300 Acc. = 0.9409\n",
      "training steps= 4400 Acc. = 0.9399\n",
      "training steps= 4500 Acc. = 0.9446\n",
      "training steps= 4600 Acc. = 0.9456\n",
      "training steps= 4700 Acc. = 0.9424\n",
      "training steps= 4800 Acc. = 0.9408\n",
      "training steps= 4900 Acc. = 0.943\n",
      "training steps= 5000 Acc. = 0.9458\n",
      "training steps= 5100 Acc. = 0.9443\n",
      "training steps= 5200 Acc. = 0.9425\n",
      "training steps= 5300 Acc. = 0.9454\n",
      "training steps= 5400 Acc. = 0.946\n",
      "training steps= 5500 Acc. = 0.9456\n",
      "training steps= 5600 Acc. = 0.943\n",
      "training steps= 5700 Acc. = 0.9452\n",
      "training steps= 5800 Acc. = 0.9494\n",
      "training steps= 5900 Acc. = 0.9472\n",
      "training steps= 6000 Acc. = 0.9448\n",
      "training steps= 6100 Acc. = 0.9481\n",
      "training steps= 6200 Acc. = 0.9468\n",
      "training steps= 6300 Acc. = 0.949\n",
      "training steps= 6400 Acc. = 0.9492\n",
      "training steps= 6500 Acc. = 0.9503\n",
      "training steps= 6600 Acc. = 0.9494\n",
      "training steps= 6700 Acc. = 0.9509\n",
      "training steps= 6800 Acc. = 0.9502\n",
      "training steps= 6900 Acc. = 0.9531\n",
      "training steps= 7000 Acc. = 0.9488\n",
      "training steps= 7100 Acc. = 0.9528\n",
      "training steps= 7200 Acc. = 0.9526\n",
      "training steps= 7300 Acc. = 0.9511\n",
      "training steps= 7400 Acc. = 0.9549\n",
      "training steps= 7500 Acc. = 0.9529\n",
      "training steps= 7600 Acc. = 0.9524\n",
      "training steps= 7700 Acc. = 0.9501\n",
      "training steps= 7800 Acc. = 0.9546\n",
      "training steps= 7900 Acc. = 0.9536\n",
      "training steps= 8000 Acc. = 0.9559\n",
      "training steps= 8100 Acc. = 0.9534\n",
      "training steps= 8200 Acc. = 0.954\n",
      "training steps= 8300 Acc. = 0.9543\n",
      "training steps= 8400 Acc. = 0.9538\n",
      "training steps= 8500 Acc. = 0.9555\n",
      "training steps= 8600 Acc. = 0.9504\n",
      "training steps= 8700 Acc. = 0.9545\n",
      "training steps= 8800 Acc. = 0.9537\n",
      "training steps= 8900 Acc. = 0.957\n",
      "training steps= 9000 Acc. = 0.9555\n",
      "training steps= 9100 Acc. = 0.9559\n",
      "training steps= 9200 Acc. = 0.9538\n",
      "training steps= 9300 Acc. = 0.9557\n",
      "training steps= 9400 Acc. = 0.9527\n",
      "training steps= 9500 Acc. = 0.956\n",
      "training steps= 9600 Acc. = 0.9563\n",
      "training steps= 9700 Acc. = 0.9575\n",
      "training steps= 9800 Acc. = 0.9574\n",
      "training steps= 9900 Acc. = 0.9572\n",
      "training steps= 10000 Acc. = 0.9571\n",
      "99.0\n"
     ]
    }
   ],
   "source": [
    "# Load data \n",
    "from tensorflow.examples.tutorials.mnist import input_data\n",
    "mnist = input_data.read_data_sets('MNIST_data', one_hot=False)\n",
    "\n",
    "# build graph\n",
    "class Graph:\n",
    "    def __init__(self, is_training=False):\n",
    "        # Inputs and labels\n",
    "        self.x = tf.placeholder(tf.float32, shape=[None, 784])\n",
    "        self.y = tf.placeholder(tf.int32, shape=[None])\n",
    "\n",
    "        # Layer 1\n",
    "        w1 = tf.get_variable(\"w1\", shape=[784, 100], initializer=tf.truncated_normal_initializer())\n",
    "        output1 = tf.matmul(self.x, w1)\n",
    "        output1 = tf.contrib.layers.batch_norm(output1, center=True, scale=True, is_training=is_training, \n",
    "                                           updates_collections=None, activation_fn=tf.nn.relu)\n",
    "\n",
    "        #Layer 2\n",
    "        w2 = tf.get_variable(\"w2\", shape=[100, 10], initializer=tf.truncated_normal_initializer())\n",
    "        logits = tf.matmul(output1, w2)\n",
    "        preds = tf.to_int32(tf.arg_max(logits, dimension=1))\n",
    "\n",
    "        # training\n",
    "        loss = tf.nn.sparse_softmax_cross_entropy_with_logits(labels=self.y, logits=logits)\n",
    "        self.train_op = tf.train.GradientDescentOptimizer(0.01).minimize(loss)\n",
    "        self.acc = tf.reduce_mean(tf.to_float(tf.equal(self.y, preds)))\n",
    "\n",
    "\n",
    "# Training\n",
    "tf.reset_default_graph()\n",
    "g = Graph(is_training=True)\n",
    "init_op = tf.global_variables_initializer()\n",
    "with tf.Session() as sess:\n",
    "    sess.run(init_op)\n",
    "    saver = tf.train.Saver()\n",
    "    for i in range(1, 10000+1):\n",
    "        batch = mnist.train.next_batch(60)\n",
    "        sess.run(g.train_op, {g.x: batch[0], g.y: batch[1]})\n",
    "        # Evaluation\n",
    "        if i % 100 == 0:\n",
    "            print(\"training steps=\", i, \"Acc. =\", sess.run(g.acc, {g.x: mnist.test.images, g.y: mnist.test.labels}))\n",
    "    save_path = saver.save(sess, './my-model')\n",
    "        \n",
    "# Inference\n",
    "tf.reset_default_graph()\n",
    "g2 = Graph(is_training=False)\n",
    "with tf.Session() as sess:\n",
    "    saver = tf.train.Saver()\n",
    "    saver.restore(sess, save_path)\n",
    "    hits = 0\n",
    "    for i in range(100):\n",
    "        hits += sess.run(g2.acc, {g2.x: [mnist.test.images[i]], g2.y: [mnist.test.labels[i]]})\n",
    "    print(hits)\n",
    "\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "# Losses"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "source": [
    "Q06. Compute half the L2 norm of `x` without the sqrt.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "11.5\n",
      "11.5\n"
     ]
    }
   ],
   "source": [
    "tf.reset_default_graph()\n",
    "x = tf.constant([1, 1, 2, 2, 2, 3], tf.float32)\n",
    "\n",
    "output = tf.nn.l2_loss(x)\n",
    "with tf.Session() as sess:\n",
    "    print(sess.run(output))\n",
    "    print(sess.run(tf.reduce_sum(x**2)/2))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "# Classification"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "Q7. Compute softmax cross entropy between logits and labels. Note that the rank of them is not the same.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 108,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[ 3.20745516  2.13898396  3.07336521  2.74353743  3.27086067]\n",
      " [ 3.30273581  2.07290959  3.74236178  2.16736674  2.34212399]]\n"
     ]
    }
   ],
   "source": [
    "tf.reset_default_graph()\n",
    "logits = tf.random_normal(shape=[2, 5, 10])\n",
    "labels = tf.convert_to_tensor(np.random.randint(0, 10, size=[2, 5]), tf.int32)\n",
    "output = tf.nn.sparse_softmax_cross_entropy_with_logits(labels=labels, logits=logits)\n",
    "with tf.Session() as sess:\n",
    "    print(sess.run(output))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "Q8. Compute softmax cross entropy between logits and labels."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 110,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[ 4.10439205  3.70805836  3.14382291  2.34843588  3.06786823]\n",
      " [ 2.46797109  1.83231926  1.75755191  4.17633867  4.40706158]]\n"
     ]
    }
   ],
   "source": [
    "logits = tf.random_normal(shape=[2, 5, 10])\n",
    "labels = tf.convert_to_tensor(np.random.randint(0, 10, size=[2, 5]), tf.int32)\n",
    "labels = tf.one_hot(labels, depth=10)\n",
    "\n",
    "output = tf.nn.softmax_cross_entropy_with_logits(labels=labels, logits=logits)\n",
    "with tf.Session() as sess:\n",
    "    print(sess.run(output))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "# Embeddings"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "Q9. Map tensor `x` to the embedding."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 113,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[ 0.          0.2         0.1         0.30000001  0.40000001]\n"
     ]
    }
   ],
   "source": [
    "tf.reset_default_graph()\n",
    "x = tf.constant([0, 2, 1, 3, 4], tf.int32)\n",
    "embedding = tf.constant([0, 0.1, 0.2, 0.3, 0.4], tf.float32)\n",
    "output = tf.nn.embedding_lookup(embedding, x)\n",
    "with tf.Session() as sess:\n",
    "    print(sess.run(output))"
   ]
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}