GAN_MNIST.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/anaconda3/lib/python3.6/site-packages/h5py/__init__.py:36: FutureWarning: Conversion of the second argument of issubdtype from `float` to `np.floating` is deprecated. In future, it will be treated as `np.float64 == np.dtype(float).type`.\n",
      "  from ._conv import register_converters as _register_converters\n"
     ]
    }
   ],
   "source": [
    "%matplotlib inline\n",
    "\n",
    "import pickle as pkl\n",
    "import numpy as np\n",
    "import tensorflow as tf\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /anaconda3/lib/python3.6/site-packages/tensorflow/contrib/learn/python/learn/datasets/base.py:198: retry (from tensorflow.contrib.learn.python.learn.datasets.base) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use the retry module or similar alternatives.\n",
      "WARNING:tensorflow:From <ipython-input-3-a37a59f11b0d>:2: read_data_sets (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use alternatives such as official/mnist/dataset.py from tensorflow/models.\n",
      "WARNING:tensorflow:From /anaconda3/lib/python3.6/site-packages/tensorflow/contrib/learn/python/learn/datasets/mnist.py:260: maybe_download (from tensorflow.contrib.learn.python.learn.datasets.base) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please write your own downloading logic.\n",
      "WARNING:tensorflow:From /anaconda3/lib/python3.6/site-packages/tensorflow/contrib/learn/python/learn/datasets/mnist.py:262: extract_images (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use tf.data to implement this functionality.\n",
      "Extracting MNIST_data/train-images-idx3-ubyte.gz\n",
      "WARNING:tensorflow:From /anaconda3/lib/python3.6/site-packages/tensorflow/contrib/learn/python/learn/datasets/mnist.py:267: extract_labels (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use tf.data to implement this functionality.\n",
      "Extracting MNIST_data/train-labels-idx1-ubyte.gz\n",
      "Extracting MNIST_data/t10k-images-idx3-ubyte.gz\n",
      "Extracting MNIST_data/t10k-labels-idx1-ubyte.gz\n",
      "WARNING:tensorflow:From /anaconda3/lib/python3.6/site-packages/tensorflow/contrib/learn/python/learn/datasets/mnist.py:290: DataSet.__init__ (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use alternatives such as official/mnist/dataset.py from tensorflow/models.\n"
     ]
    }
   ],
   "source": [
    "from tensorflow.examples.tutorials.mnist import input_data\n",
    "mnist = input_data.read_data_sets('MNIST_data')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def model_inputs(real_dim, z_dim):\n",
    "    inputs_real = tf.placeholder(tf.float32, (None, real_dim), name='input_real')\n",
    "    inputs_z = tf.placeholder(tf.float32, (None, z_dim), name='input_z')\n",
    "    return inputs_real, inputs_z"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def generator(z, out_dim, n_units=128, reuse=False, alpha=0.01):\n",
    "    \"\"\"Generate image using whatever image input\n",
    "    'Generate image using whatever image input'\n",
    "    \n",
    "    Args:\n",
    "        z (int): Input dimension\n",
    "        out_dim (int): output dimension\n",
    "        n_units (int):  number of units\n",
    "        reuse (boolean): reuse data or not\n",
    "        alpha (float): Leaky ReLu coefficient\n",
    "    \n",
    "    Returns:\n",
    "        tuple: output tensor as tuple of (out_dim) dimensions\n",
    "    \"\"\"\n",
    "    with tf.variable_scope('generator', reuse=reuse):\n",
    "        # first hidden layer using leaky Relu not to make the output of x < 0  to zero.\n",
    "        h1 = tf.layers.dense(z, n_units, activation=None)\n",
    "        h1 = tf.maximum(alpha*h1, h1)\n",
    "        \n",
    "        # second hidden layer\n",
    "        # use tanh to transform output between -1 to 1 using Hyperbolic Tangent\n",
    "        logits = tf.layers.dense(h1, out_dim, activation=None)\n",
    "        out = tf.tanh(logits)\n",
    "        \n",
    "        return out\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def discriminator(x, n_units=128, reuse=False, alpha=0.01):\n",
    "    # Check\n",
    "    with tf.variable_scope('discriminator', reuse=reuse):\n",
    "        h1 = tf.layers.dense(x, n_units, activation=None)\n",
    "        h1 = tf.maximum(alpha*h1, h1)\n",
    "        logits = tf.layers.dense(h1, 1, activation=None)\n",
    "        out = tf.sigmoid(logits)\n",
    "        \n",
    "        return out, logits"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/100 D Loss: 0.3538 G Loss: 3.8255\n",
      "Epoch 2/100 D Loss: 0.3797 G Loss: 3.8904\n",
      "Epoch 3/100 D Loss: 0.3960 G Loss: 3.4977\n",
      "Epoch 4/100 D Loss: 0.4094 G Loss: 3.2013\n",
      "Epoch 5/100 D Loss: 0.5097 G Loss: 3.8279\n",
      "Epoch 6/100 D Loss: 0.8427 G Loss: 5.2526\n",
      "Epoch 7/100 D Loss: 0.6457 G Loss: 3.0421\n",
      "Epoch 8/100 D Loss: 0.8117 G Loss: 3.0135\n",
      "Epoch 9/100 D Loss: 0.5951 G Loss: 4.8739\n",
      "Epoch 10/100 D Loss: 0.9043 G Loss: 3.8397\n",
      "Epoch 11/100 D Loss: 0.9462 G Loss: 2.7342\n",
      "Epoch 12/100 D Loss: 0.8308 G Loss: 2.0498\n",
      "Epoch 13/100 D Loss: 0.7628 G Loss: 2.4464\n",
      "Epoch 14/100 D Loss: 0.8139 G Loss: 2.8229\n",
      "Epoch 15/100 D Loss: 0.9172 G Loss: 2.8867\n",
      "Epoch 16/100 D Loss: 1.4084 G Loss: 1.5346\n",
      "Epoch 17/100 D Loss: 0.6655 G Loss: 2.6643\n",
      "Epoch 18/100 D Loss: 0.6929 G Loss: 2.6844\n",
      "Epoch 19/100 D Loss: 0.9922 G Loss: 1.4684\n",
      "Epoch 20/100 D Loss: 1.4402 G Loss: 1.8080\n",
      "Epoch 21/100 D Loss: 1.2064 G Loss: 1.4936\n",
      "Epoch 22/100 D Loss: 0.8374 G Loss: 2.0069\n",
      "Epoch 23/100 D Loss: 0.7094 G Loss: 2.2571\n",
      "Epoch 24/100 D Loss: 1.4248 G Loss: 1.1176\n",
      "Epoch 25/100 D Loss: 0.9921 G Loss: 1.9969\n",
      "Epoch 26/100 D Loss: 1.4496 G Loss: 1.4896\n",
      "Epoch 27/100 D Loss: 0.9326 G Loss: 1.9766\n",
      "Epoch 28/100 D Loss: 0.9951 G Loss: 1.5131\n",
      "Epoch 29/100 D Loss: 0.8604 G Loss: 2.1149\n",
      "Epoch 30/100 D Loss: 0.8376 G Loss: 1.9272\n",
      "Epoch 31/100 D Loss: 0.9065 G Loss: 2.1281\n",
      "Epoch 32/100 D Loss: 0.8084 G Loss: 2.6373\n",
      "Epoch 33/100 D Loss: 1.0638 G Loss: 1.9476\n",
      "Epoch 34/100 D Loss: 0.8953 G Loss: 1.9676\n",
      "Epoch 35/100 D Loss: 0.8508 G Loss: 2.1190\n",
      "Epoch 36/100 D Loss: 0.8567 G Loss: 2.9622\n",
      "Epoch 37/100 D Loss: 1.3211 G Loss: 1.3233\n",
      "Epoch 38/100 D Loss: 0.8536 G Loss: 2.8726\n",
      "Epoch 39/100 D Loss: 0.9068 G Loss: 2.1466\n",
      "Epoch 40/100 D Loss: 1.1718 G Loss: 1.4512\n",
      "Epoch 41/100 D Loss: 0.7510 G Loss: 2.2537\n",
      "Epoch 42/100 D Loss: 0.9683 G Loss: 1.9507\n",
      "Epoch 43/100 D Loss: 0.8437 G Loss: 2.3296\n",
      "Epoch 44/100 D Loss: 0.8080 G Loss: 2.7869\n",
      "Epoch 45/100 D Loss: 0.9550 G Loss: 1.8907\n",
      "Epoch 46/100 D Loss: 0.8941 G Loss: 2.0336\n",
      "Epoch 47/100 D Loss: 0.9164 G Loss: 1.6738\n",
      "Epoch 48/100 D Loss: 0.9614 G Loss: 2.1416\n",
      "Epoch 49/100 D Loss: 0.9879 G Loss: 2.3713\n",
      "Epoch 50/100 D Loss: 1.0382 G Loss: 1.9977\n",
      "Epoch 51/100 D Loss: 0.7896 G Loss: 2.4393\n",
      "Epoch 52/100 D Loss: 0.8928 G Loss: 2.2581\n",
      "Epoch 53/100 D Loss: 0.8732 G Loss: 2.1672\n",
      "Epoch 54/100 D Loss: 0.8237 G Loss: 2.1016\n",
      "Epoch 55/100 D Loss: 1.0199 G Loss: 1.5568\n",
      "Epoch 56/100 D Loss: 0.8819 G Loss: 1.9499\n",
      "Epoch 57/100 D Loss: 0.7772 G Loss: 2.1323\n",
      "Epoch 58/100 D Loss: 1.0114 G Loss: 2.0863\n",
      "Epoch 59/100 D Loss: 0.6985 G Loss: 2.8048\n",
      "Epoch 60/100 D Loss: 0.7883 G Loss: 2.0957\n",
      "Epoch 61/100 D Loss: 0.8368 G Loss: 2.3726\n",
      "Epoch 62/100 D Loss: 0.8773 G Loss: 2.1428\n",
      "Epoch 63/100 D Loss: 0.9493 G Loss: 1.8016\n",
      "Epoch 64/100 D Loss: 0.9509 G Loss: 2.0305\n",
      "Epoch 65/100 D Loss: 0.8197 G Loss: 2.2839\n",
      "Epoch 66/100 D Loss: 0.8304 G Loss: 1.9124\n",
      "Epoch 67/100 D Loss: 0.8322 G Loss: 2.3196\n",
      "Epoch 68/100 D Loss: 0.8503 G Loss: 2.1043\n",
      "Epoch 69/100 D Loss: 1.0417 G Loss: 1.8868\n",
      "Epoch 70/100 D Loss: 0.8068 G Loss: 2.2113\n",
      "Epoch 71/100 D Loss: 0.8783 G Loss: 1.8527\n",
      "Epoch 72/100 D Loss: 0.9862 G Loss: 2.1715\n",
      "Epoch 73/100 D Loss: 0.9512 G Loss: 1.8683\n",
      "Epoch 74/100 D Loss: 0.7626 G Loss: 2.6311\n",
      "Epoch 75/100 D Loss: 0.7763 G Loss: 2.4928\n",
      "Epoch 76/100 D Loss: 0.9950 G Loss: 1.8113\n",
      "Epoch 77/100 D Loss: 0.7704 G Loss: 2.5556\n",
      "Epoch 78/100 D Loss: 1.1127 G Loss: 2.4767\n",
      "Epoch 79/100 D Loss: 0.9136 G Loss: 1.8969\n",
      "Epoch 80/100 D Loss: 0.8366 G Loss: 2.3201\n",
      "Epoch 81/100 D Loss: 1.0232 G Loss: 1.9624\n",
      "Epoch 82/100 D Loss: 0.9280 G Loss: 1.7129\n",
      "Epoch 83/100 D Loss: 0.9061 G Loss: 2.2870\n",
      "Epoch 84/100 D Loss: 0.7937 G Loss: 2.3846\n",
      "Epoch 85/100 D Loss: 0.9139 G Loss: 2.5847\n",
      "Epoch 86/100 D Loss: 0.9868 G Loss: 2.0181\n",
      "Epoch 87/100 D Loss: 0.9337 G Loss: 2.1088\n",
      "Epoch 88/100 D Loss: 0.8791 G Loss: 2.2413\n",
      "Epoch 89/100 D Loss: 0.8302 G Loss: 2.2585\n",
      "Epoch 90/100 D Loss: 0.9077 G Loss: 2.2193\n",
      "Epoch 91/100 D Loss: 1.1689 G Loss: 1.5150\n",
      "Epoch 92/100 D Loss: 0.9460 G Loss: 2.0315\n",
      "Epoch 93/100 D Loss: 0.9242 G Loss: 2.5222\n",
      "Epoch 94/100 D Loss: 0.9764 G Loss: 1.9778\n",
      "Epoch 95/100 D Loss: 0.8412 G Loss: 2.3747\n",
      "Epoch 96/100 D Loss: 0.8697 G Loss: 2.0731\n",
      "Epoch 97/100 D Loss: 0.8461 G Loss: 2.2020\n",
      "Epoch 98/100 D Loss: 1.0412 G Loss: 1.8680\n",
      "Epoch 99/100 D Loss: 1.0109 G Loss: 1.8277\n",
      "Epoch 100/100 D Loss: 1.0070 G Loss: 1.5217\n"
     ]
    }
   ],
   "source": [
    "if __name__ == '__main__':\n",
    "    # initialize hyper parameters (global variables)\n",
    "    input_size = 784 # 28 * 28\n",
    "    z_size = 100\n",
    "    generator_hidden_size = 128\n",
    "    discriminator_hidden_size = 128\n",
    "    alpha = 0.01\n",
    "    smooth = 0.1\n",
    "    \n",
    "    # define graph\n",
    "    tf.reset_default_graph()\n",
    "    input_real, input_z = model_inputs(input_size, z_size)\n",
    "    \n",
    "    generator_model = generator(input_z, input_size, n_units=generator_hidden_size, alpha=alpha)\n",
    "    discriminator_model_real, discriminator_logits_real = discriminator(input_real, discriminator_hidden_size, False, alpha=alpha)\n",
    "    discriminator_model_fake, discriminator_logits_fake = discriminator(generator_model, discriminator_hidden_size, True, alpha)\n",
    "    \n",
    "    # define loss functions\n",
    "    discriminator_loss_real = tf.reduce_mean(tf.nn.sigmoid_cross_entropy_with_logits(\n",
    "        logits=discriminator_logits_real,\n",
    "        labels=tf.ones_like(discriminator_logits_real)*(1-smooth)\n",
    "    ))\n",
    "    discriminator_loss_fake = tf.reduce_mean(tf.nn.sigmoid_cross_entropy_with_logits(\n",
    "        logits=discriminator_logits_fake,\n",
    "        labels=tf.zeros_like(discriminator_logits_real)\n",
    "    ))\n",
    "    discriminator_loss = discriminator_loss_real + discriminator_loss_fake\n",
    "    generator_loss = tf.reduce_mean(tf.nn.sigmoid_cross_entropy_with_logits(\n",
    "        logits=discriminator_logits_fake,\n",
    "        labels=tf.ones_like(discriminator_logits_fake)\n",
    "    ))\n",
    "    \n",
    "    # define optiomization\n",
    "    learning_rate = 0.002\n",
    "    trainable_vars = tf.trainable_variables()\n",
    "    generator_vars = [var for var in trainable_vars if var.name.startswith('generator')]\n",
    "    discriminator_vars = [var for var in trainable_vars if var.name.startswith('discriminator')]\n",
    "    \n",
    "    generator_train_optimize = tf.train.AdamOptimizer(learning_rate).minimize(generator_loss, var_list=generator_vars)\n",
    "    discriminator_train_optimize = tf.train.AdamOptimizer(learning_rate).minimize(discriminator_loss, var_list=discriminator_vars)\n",
    "    \n",
    "    # training\n",
    "    batch_size = 100\n",
    "    batch = mnist.train.next_batch(batch_size)\n",
    "    epochs = 100\n",
    "    samples = []\n",
    "    losses = []\n",
    "    saver = tf.train.Saver(var_list=generator_vars)\n",
    "    with tf.Session() as sess:\n",
    "        sess.run(tf.global_variables_initializer())\n",
    "        for e in range(epochs):\n",
    "            for i in range(mnist.train.num_examples//batch_size):\n",
    "                batch = mnist.train.next_batch(batch_size)\n",
    "                batch_images = batch[0].reshape((batch_size, input_size))\n",
    "                batch_images = batch_images * 2 -1\n",
    "                \n",
    "                batch_z = np.random.uniform(-1, 1, size=(batch_size, z_size))\n",
    "                _ = sess.run(discriminator_train_optimize, feed_dict={input_real: batch_images, input_z: batch_z})\n",
    "                _ = sess.run(generator_train_optimize, feed_dict={input_z: batch_z})\n",
    "            \n",
    "            train_loss_discriminator = sess.run(discriminator_loss, {input_real: batch_images, input_z: batch_z})\n",
    "            train_loss_generator = generator_loss.eval({input_z: batch_z})\n",
    "            \n",
    "            print('Epoch {}/{}'.format(e+1, epochs),\n",
    "                'D Loss: {:.4f}'.format(train_loss_discriminator),\n",
    "                'G Loss: {:.4f}'.format(train_loss_generator))\n",
    "            \n",
    "            losses.append({train_loss_discriminator, train_loss_generator})\n",
    "            \n",
    "            sample_z = np.random.uniform(-1, 1, size=(16, z_size))\n",
    "            gen_samples = sess.run(\n",
    "                generator(input_z, input_size, n_units=generator_hidden_size, reuse=True, alpha=alpha),\n",
    "                feed_dict={input_z: sample_z}\n",
    "            )\n",
    "            samples.append(gen_samples)\n",
    "            saver.save(sess, './checkpoints/generator.ckpt')\n",
    "    \n",
    "    with open('training_sample.pkl', 'wb') as f:\n",
    "        pkl.dump(samples, f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}