Ex10.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Exercise 10: Generative Networks"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "import torchvision"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Use built-in Pytorch function to download MNIST data in usable format \n",
    "# the images are automatically normalised to a mean of zero and sigma of 1 \n",
    "\n",
    "batch_size = 100\n",
    "\n",
    "train_loader = torch.utils.data.DataLoader(\n",
    "  torchvision.datasets.MNIST('.', train=True, download=True,\n",
    "                             transform=torchvision.transforms.Compose([\n",
    "                               torchvision.transforms.ToTensor(),\n",
    "                               torchvision.transforms.Normalize(\n",
    "                                 (0.1307,), (0.3081,)) # initial mean and width\n",
    "                             ])),\n",
    "  batch_size=batch_size, shuffle=True)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([100, 1, 28, 28])\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAO4AAADuCAYAAAA+7jsiAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAABctJREFUeJzt3bFrFHkYx+EZiYgEQRALbdJrYQQLQVDEKo0giGChjZWFFlpbCuJ/YGlho3aWYiNoY2wkEBAEUwnJoaWFmLnu4Ip9J9lx3Xx3n6d9d8Yf3n1uJK9z23Zd1wBZ9k37AMDuCRcCCRcCCRcCCRcCCRcCCRcCCRcCCRcCLezmw23b+mtWMGFd17V9n/HEhUDChUDChUDChUDChUDChUDChUDChUDChUDChUDChUDChUDChUDChUDChUDChUDChUDChUDChUDChUDChUDChUDChUDChUDChUDChUDChUDChUDChUDChUDChUDChUDChUDChUDChUDChUDChUAL0z7A33L27NmRs3fv3pXXnj59upx/+vRprDMl2L9//8jZw4cPy2vv379fzq9fv17Onz9/Xs7nmScuBBIuBBIuBBIuBBIuBBIuBBIuBJqbPe7Vq1dHzrquK6/t2zeur6+X81+/fpXzvazaxd67d6+8tu/3dWlpaawz4YkLkYQLgYQLgYQLgYQLgYQLgdq+H9n/78Ntu/MP7zGnTp0aOfv48eOge799+7acP3r0aND9J6nv1bsTJ06MnB07dmzQr338+PFyvrm5Oej+qbqua/s+44kLgYQLgYQLgYQLgYQLgYQLgYQLgebmtb4fP36MnL1586a89tKlS+X8/Pnz5fzChQvlfDe79D+tbeuV4ZCzPX78uJx///597HvPO09cCCRcCCRcCCRcCCRcCCRcCCRcCDQ37+NWDhw4UM7v3r1bzh88eFDOFxcXy/ms7nGXl5fL+dra2tj3nmXex4UZJVwIJFwIJFwIJFwIJFwIJFwIZI/7F/TtcW/fvj1y9vLly/Lar1+/jnOk//T989/e3h45+/z5c3lt33vKW1tb5Xxe2ePCjBIuBBIuBBIuBBIuBBIuBBIuBLLHnXO/f/8u59W/H69fvy6vXVlZGetM884eF2aUcCGQcCGQcCGQcCGQcCGQcCGQcCGQcCGQcCGQcCGQcCGQcCGQcCGQcCGQcCGQcCGQcCGQcCGQcCGQcCGQcCGQcCGQcCGQcCGQcCGQcCGQcCGQcCGQcCHQwrQPwGQtLi6W83376v92b29vj5y1be+3QTIhnrgQSLgQSLgQSLgQSLgQSLgQSLgQyB63aZpDhw6V82vXrpXz1dXVP3mcP+rGjRvlvNrTNk3TdF03cnby5Mny2qdPn5bzPtWeuDrXTrx//76cP3nyZND9J80TFwIJFwIJFwIJFwIJFwIJFwK1u/mxetu2w34Gv0f1/ej/1q1bg+7f9/rb0NXGEKlnG3quK1eulPNXr14Nuv8QXdf1vi/piQuBhAuBhAuBhAuBhAuBhAuBhAuB7HGbprl48WI5f/HiRTk/fPhwOU/dlTbNdM/27NmzkbPNzc3y2r5XCr98+VLOf/78Wc4nyR4XZpRwIZBwIZBwIZBwIZBwIZBwIZA97g4cOXKknN+5c6ecHzx4sJwfPXp05OzmzZvltUMN2eNubGyU116+fLmcb21tlfO+Xe2ssseFGSVcCCRcCCRcCCRcCCRcCCRcCGSPO+POnDlTzj98+FDOq6/h7Nvjnjt3rpx/+/atnM8re1yYUcKFQMKFQMKFQMKFQMKFQNZBc67v6yRXVlbGvvfy8nI5X1tbG/ves8w6CGaUcCGQcCGQcCGQcCGQcCGQcCHQwrQPwHStr6+X8yF7XCbHExcCCRcCCRcCCRcCCRcCCRcCCRcCCRcCCRcCCRcCCRcCCRcCCRcCCRcCCRcCeR93zq2urk77CIzBExcCCRcCCRcCCRcCCRcCCRcCCRcC+X5c2GN8Py7MKOFCIOFCIOFCIOFCIOFCIOFCIOFCIOFCIOFCIOFCIOFCIOFCIOFCIOFCIOFCIOFCIOFCIOFCIOFCIOFCIOFCoN1+zeY/TdNsTOIgQNM0TbO0kw/t6v+rDOwN/qgMgYQLgYQLgYQLgYQLgYQLgYQLgYQLgYQLgf4FoMkW6kKDJ1UAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAO4AAADuCAYAAAA+7jsiAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAABmNJREFUeJzt3U+Izf8ex/Hvuf2kianRSLH5WY8dGxFZaMKCtbIhG5GdRmqUUiRS1MjKWFgKG7G1k5SNDRuymPIvakoh567uXdzueX9n5gy/eZ15PLav8z1zmnr2VR/fM51ut9sAWf71T38AYP6EC4GEC4GEC4GEC4GEC4GEC4GEC4GEC4H+ms+LO52O/2YFv1m32+20vcYdFwIJFwIJFwIJFwIJFwIJFwIJFwIJFwIJFwIJFwIJFwIJFwIJFwIJFwLN67E+mI+dO3eW+7lz58p9bGys53bx4sXy2qtXr5Z7OndcCCRcCCRcCCRcCCRcCCRcCOQ4iAWbmJgo91OnTpX7yMhIuc/MzPTcXr16VV476NxxIZBwIZBwIZBwIZBwIZBwIZBwIZBz3GVu5cqV5X7hwoWe26FDh8pr285pDx48WO7Pnj3rub1586a8dtC540Ig4UIg4UIg4UIg4UIg4UIg4UIg57jL3LZt28r95MmTC37vS5culfu9e/fK/efPnwv+2YPOHRcCCRcCCRcCCRcCCRcCCRcCCRcCOccdcMeOHSv3trPWyuXLl8v97Nmz5e6cduHccSGQcCGQcCGQcCGQcCGQcCFQp9vtzv3Fnc7cX8wfMTQ0VO5tX2O6bt26cn/9+nXPbXx8vK+fzf/X7XY7ba9xx4VAwoVAwoVAwoVAwoVAwoVAwoVAHutb4oaHh8v99u3b5T46OlruX758KffqrNY57T/HHRcCCRcCCRcCCRcCCRcCCRcCCRcCOcddAqqz2lu3bpXX7t+/v9zfvn1b7lNTU+XurHZpcseFQMKFQMKFQMKFQMKFQMKFQMKFQL5X+Q9o++7jO3fu9NzazmnbHD16tNynp6f7en8Wn+9VhgElXAgkXAgkXAgkXAgkXAgkXAjkedw/YGJiotz7OavdvXt3uT958mTB783S5Y4LgYQLgYQLgYQLgYQLgYQLgTzWtwhWrVpV7rOzs+X+/fv3ntuZM2fKa69cuVLu5PFYHwwo4UIg4UIg4UIg4UIg4UIg4UIgj/XNwZo1a8r9/v375f7r169yf/nyZc8t+Zy27ffWdr7948ePxfw4A8UdFwIJFwIJFwIJFwIJFwIJFwIJFwI5x52D48ePl/v27dvLvXretmma5vr16z230dHR8totW7b0te/atavc5/O89v9av359uT969Kjc277Wdjlzx4VAwoVAwoVAwoVAwoVAwoVAwoVAznGbptm6dWu593ueePPmzXIfGRnpuT19+rS8duPGjQv5SP/V6dRf4dvPOW6bTZs2Lfhnnz59erE/ThR3XAgkXAgkXAgkXAgkXAgkXAgkXAjk7+M2TTM+Pl7uDx8+7Ov9v337Vu5DQ0N9vX/lxo0b5f7p06dyn5qa6rm1nSFPTk6W+969e8v9/fv3PbcNGzaU1ybz93FhQAkXAgkXAgkXAgkXAgkXAnmsbw7aHn1rs3r16nKv/gznixcvymv37NlT7h8+fCj3fnz9+rXcHz9+XO779u0r935/74PMHRcCCRcCCRcCCRcCCRcCCRcCCRcCOcedg36/orQ6p217/+fPn5fXzs7OLugz/ceKFSvKfWxsrOd24sSJ8trDhw+Xe9vvdXp6utyXM3dcCCRcCCRcCCRcCCRcCCRcCCRcCOQcd4k7cuRIud+9e7fcN2/eXO5tX027Y8eOcq/MzMyUe9s58IMHDxb8swedOy4EEi4EEi4EEi4EEi4EEi4EEi4E8mc2m6YZHh4u97bnQg8cOFDubd8P3M/zvh8/fiz3tWvXlnvbZ/v8+XPPre2Mue07od+9e1fuy5U/swkDSrgQSLgQSLgQSLgQSLgQyHHQHAwNDZX7xMREuU9OTpZ7v1//2o/z58+X+7Vr13pu1VERC+c4CAaUcCGQcCGQcCGQcCGQcCGQcCGQc1xYYpzjwoASLgQSLgQSLgQSLgQSLgQSLgQSLgQSLgQSLgQSLgQSLgQSLgQSLgQSLgQSLgQSLgQSLgQSLgQSLgQSLgQSLgQSLgQSLgQSLgQSLgQSLgQSLgQSLgQSLgQSLgQSLgQSLgQSLgQSLgQSLgQSLgQSLgQSLgQSLgQSLgQSLgQSLgQSLgQSLgQSLgT6a56v/9g0zdvf8UGApmma5u+5vKjT7XZ/9wcBFpl/KkMg4UIg4UIg4UIg4UIg4UIg4UIg4UIg4UKgfwOEAiVuodw12wAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAO4AAADuCAYAAAA+7jsiAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAABU9JREFUeJzt3TFPVFkcxuF7BRMrI1BgtxQkmvkOFiRUFFBR0KqdLY1fQQoLKzuDIYZQUdpIoo0FDfaYLEi3IRaEAk3utrvZnTPDDMPcF56n/Z975xT+PCbHgbppmgrIcmfcGwAuT7gQSLgQSLgQSLgQSLgQSLgQSLgQSLgQaPIyi+u69t+sYMSapql7rXHiQiDhQiDhQiDhQiDhQiDhQiDhQiDhQiDhQiDhQiDhQiDhQiDhQiDhQiDhQiDhQiDhQiDhQiDhQiDhQiDhQiDhQiDhQiDhQiDhQiDhQiDhQiDhQiDhQiDhQiDhQiDhQiDhQiDhQiDhQiDhQiDhQiDhQiDhQiDhQiDhQiDhQiDhQiDhQiDhQqDJcW/gJlhYWCjOl5aWivP19fWus6Zpis+enJwU51tbW8X5KH38+LE439vbu6ad3DxOXAgkXAgkXAgkXAgkXAgkXAhU97pu+Nfiuu5/8S3y9u3b4vz58+fXtJN2OT4+Ls7n5uauZyNhmqape61x4kIg4UIg4UIg4UIg4UIg4UIg4UIgX+u7Aq9evSrOO51Ocf758+eus/39/YH2dB22t7fHvYVby4kLgYQLgYQLgYQLgYQLgYQLgYQLgdzjXoHDw8Pi/MmTJ9e0k6vX60fPlnz48OEKd8I/OXEhkHAhkHAhkHAhkHAhkHAhkHAhkHvcW255ebk439nZ6TqbmJgoPnt0dDTQnujNiQuBhAuBhAuBhAuBhAuBhAuBhAuB3OPeclNTU8X55GT3PyLn5+fFZ9+/fz/QnujNiQuBhAuBhAuBhAuBhAuBhAuBXAfdcnfuDP5398bGRnF+dnY28Lspc+JCIOFCIOFCIOFCIOFCIOFCIOFCoLppmv4X13X/i2mFe/fuFecHBwfF+fz8/MDv/vXrV3HO/2uapu61xokLgYQLgYQLgYQLgYQLgYQLgYQLgXwf94ZbW1srzkv3tFVVVaenp11nl/k/AFwtJy4EEi4EEi4EEi4EEi4EEi4EEi4Eco97wz19+nSo5zc3N7vOfv/+PdS7GZwTFwIJFwIJFwIJFwIJFwIJFwIJFwK5xw03MzNTnE9PTw/1/q2traGeZzScuBBIuBBIuBBIuBBIuBBIuBDIdVC41dXV4vzx48fF+e7ubnH+7du3S++J0XPiQiDhQiDhQiDhQiDhQiDhQiDhQiD3uC3X62t7L168GOr9P3/+LM79CNZ2cuJCIOFCIOFCIOFCIOFCIOFCIOFCIPe4LTc7O1ucdzqdod7/5cuXoZ5nPJy4EEi4EEi4EEi4EEi4EEi4EEi4EMg9bsvdv39/qOfPz8+Lc/e4mZy4EEi4EEi4EEi4EEi4EEi4EMh1UAvcvXu36+zly5dDvXt/f784//Hjx1DvZzycuBBIuBBIuBBIuBBIuBBIuBBIuBCobpqm/8V13f9i+vbw4cOus5OTk5F+9rNnz4rzd+/ejfTz+a+maepea5y4EEi4EEi4EEi4EEi4EEi4EEi4EMj3cVvg0aNHY/vsr1+/ju2zGZwTFwIJFwIJFwIJFwIJFwIJFwIJFwK5x22BlZWVkb379evXxfn3799H9tmMjhMXAgkXAgkXAgkXAgkXAgkXAgkXArnHbYFPnz51nS0uLhafffDgQXH+5s2b4vzi4qI4p52cuBBIuBBIuBBIuBBIuBBIuBDIr9mElvFrNuGGEi4EEi4EEi4EEi4EEi4EEi4EEi4EEi4EEi4EEi4EEi4EEi4EEi4EEi4EuuyPZ/2rqqo/R7ERoKqqqvqjn0WX+iI90A7+qQyBhAuBhAuBhAuBhAuBhAuBhAuBhAuBhAuB/gbN4aRoxTpCAQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAO4AAADuCAYAAAA+7jsiAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAABk5JREFUeJzt3c2LzX8fx/HvmSZNWcjNVlOUlUJSSrJioZQQsbCwmIWN+yxESlmQm+wwYqEMs3WzkT/BQhazsbgapbiKhZnCjHNtr815H+Y3d6/zezy2r/PhaHr6qq/vOa12u90AWfoW+g0Af0+4EEi4EEi4EEi4EEi4EEi4EEi4EEi4EKj/b17carX8NyuYY+12u9XtNa64EEi4EEi4EEi4EEi4EEi4EEi4EEi4EEi4EEi4EEi4EEi4EEi4EEi4EEi4EEi4EEi4EEi4EEi4EEi4EEi4EEi4EEi4EEi4EEi4EEi4EEi4EEi4EEi4EEi4EEi4EEi4EEi4EEi4EEi4EEi4EEi4EEi4EEi4EEi4EEi4EEi4EEi4EEi4EEi4EKh/od8Ai9v69es7bps3by7PPn78uNynp6dn9J4Wu6VLl5b7xMTEP/49XHEhkHAhkHAhkHAhkHAhkHAhkNtBPW7ZsmXlfu/evXLft29fx62vr/57f8WKFeV+69atcl9IAwMD5T40NNRxO3PmTHl2cHBwRu/p/7niQiDhQiDhQiDhQiDhQiDhQiDhQqBWu93+8xe3Wn/+YmZFt3ulmzZtKveXL1+We7ef/8jISMdt69at5dluj+1t27at3OfShg0byv327dsz/rUPHjxY7l++fCn3drvd6vZ7uOJCIOFCIOFCIOFCIOFCIOFCIOFCIM/jLgLVvdrr16+XZ0+ePFnuHz58KPcTJ06U+6tXrzpuK1euLM92u585ly5fvlzup0+fLvdHjx6V+9mzZztuP3/+LM/OBldcCCRcCCRcCCRcCCRcCCRcCCRcCOR53HnQ7WsXHz582HHbv39/efbFixflfuTIkXL//v17uS+kjRs3dtyuXr1anu32rG+3+7gPHjwo97nkeVzoUcKFQMKFQMKFQMKFQMKFQMKFQJ7HnQfHjx8v9wMHDnTcRkdHy7OHDx8u99+/f5f7Qur2zOz58+c7buPj4+XZ7du3l/u7d+/KfbFzxYVAwoVAwoVAwoVAwoVAwoVAHuubB1+/fi33ycnJjtu6devKsxMTEzN6T7Nhx44d5T48PFzua9euLfdnz5513KqPR22apvn48WO5L2Ye64MeJVwIJFwIJFwIJFwIJFwIJFwI5LG+RaD6usqdO3eWZ58/f17uU1NT5b5q1apyv3TpUsdtaGioPPv+/ftyP3ToULl3e6Tx38wVFwIJFwIJFwIJFwIJFwIJFwIJFwJ5HnceXLhwodyvXLnScev283n79m25j42NlfuePXvKfWBgoON29+7d8my3P/di/orPheR5XOhRwoVAwoVAwoVAwoVAwoVAwoVA7uPOg/7++rHn3bt3d9yOHTtWnu12H7abVqu+Zfj06dOOW7ev+GRm3MeFHiVcCCRcCCRcCCRcCCRcCCRcCORzlReB6l7q6tWry7Pfvn0r926fu3z06NFy37JlS7mzMFxxIZBwIZBwIZBwIZBwIZBwIZDbQbNgcHCw3Lt9jOmuXbs6bq9fvy7P3rlzp9xv3rxZ7tPT0+V+8eLFcmdhuOJCIOFCIOFCIOFCIOFCIOFCIOFCIPdx/8DevXvLfXh4uNyXL19e7lNTUx23ycnJ8uz9+/fL/cePH+V+7ty5cn/y5Em5szBccSGQcCGQcCGQcCGQcCGQcCGQcCGQr9lsmubUqVPlfu3atXLv6/tnf/9VH8/6+fPn8uzIyEi537hxo9zHx8fLnfnnazahRwkXAgkXAgkXAgkXAgkXAgkXArmP2zTNr1+/yr26z9o0TfPp06dy7/ZVl2/evOm4jY6OlmfpPe7jQo8SLgQSLgQSLgQSLgQSLgRyO6hpmjVr1pT7kiVLyn1sbGw23w7/cm4HQY8SLgQSLgQSLgQSLgQSLgQSLgRyHxcWGfdxoUcJFwIJFwIJFwIJFwIJFwIJFwIJFwIJFwIJFwIJFwIJFwIJFwIJFwIJFwIJFwIJFwIJFwIJFwIJFwIJFwIJFwIJFwIJFwIJFwIJFwIJFwIJFwIJFwIJFwIJFwIJFwIJFwIJFwIJFwIJFwIJFwIJFwIJFwIJFwIJFwIJFwIJFwIJFwIJFwIJFwL1/+Xr/9s0zX/m4o0ATdM0zeCfvKjVbrfn+o0As8w/lSGQcCGQcCGQcCGQcCGQcCGQcCGQcCGQcCHQ/wATTBsqQDaBqwAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAO4AAADuCAYAAAA+7jsiAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAABjRJREFUeJzt3U+ITX0cx/F7/ClMrlLUrEyzsreytrGUBQsLY6MokbHGgpJSNkqZwsJONrIiNhQ1a1E2T7FQQzIKmZxn8yjS+d2Ze8/cmc99Xq/t95yZ7+Y9R/2ce6u6rjtAljUrvQCwdMKFQMKFQMKFQMKFQMKFQMKFQMKFQMKFQOuWcnFVVf6bFSyzuq6rXtd44kIg4UIg4UIg4UIg4UIg4UIg4UIg4UIg4UIg4UIg4UIg4UIg4UIg4UIg4UIg4UIg4UIg4UIg4UIg4UIg4UIg4UIg4UIg4UIg4UIg4UIg4UIg4UIg4UIg4UIg4UIg4UIg4UIg4UIg4UIg4UIg4UKgdSu9AINZv359cT41NVWc79mzpzg/ePDgUldaFfbt21ecP3jwoDhfWFhoc53WeeJCIOFCIOFCIOFCIOFCIOFCIOFCoKqu68VfXFWLv5hFm5iYaJwdO3aseO+BAweK8x07dvSz0si7cOFCcX727NkhbfK3uq6rXtd44kIg4UIg4UIg4UIg4UIg4UIgr/UNwfj4eHH+6NGjxtnk5GTb6zACPHEhkHAhkHAhkHAhkHAhkHAhkHAhkHPcIfj+/XtxPj8/P6RN2vXt27fivNdHx65du7bNdZbky5cvK/a72+CJC4GEC4GEC4GEC4GEC4GEC4GEC4F8POsqcPPmzcbZ4cOHB/rZvc5a379/X5zPzMw0zsbGxor3Hj9+vDjvdrvF+SA+f/5cnG/btq04//HjR5vrLImPZ4URJVwIJFwIJFwIJFwIJFwIJFwI5H3cVeDMmTONs9evXxfvPXLkSHF+6dKl4rx0htzplL+m8+HDh8V7l/Octpe9e/cW5yt5TtsGT1wIJFwIJFwIJFwIJFwIJFwIJFwI5H3c/7nNmzcX5y9evGic7dy5s+11/vDx48fG2fT0dPHeO3fuFOcLCwt97TQM3seFESVcCCRcCCRcCCRcCCRcCOS1vlVg06ZNjbNBj1wmJyeL89OnTxfny3nk8+HDh+L8ypUrjbPbt2+3vU4UT1wIJFwIJFwIJFwIJFwIJFwIJFwI5Bx3FSidlc7Ozg5xk3bNzc0V5/v37y/Onz592uY6I8UTFwIJFwIJFwIJFwIJFwIJFwIJFwI5x2XZHDp0qDh3Tts/T1wIJFwIJFwIJFwIJFwIJFwIJFwI5ByXvj1+/Lg4d067fDxxIZBwIZBwIZBwIZBwIZBwIZBwIZBzXIpmZmYaZ5cvXy7e+/Xr17bX4T+euBBIuBBIuBBIuBBIuBBIuBDIcRBFT548aZy9efNmiJvwO09cCCRcCCRcCCRcCCRcCCRcCCRcCFTVdb34i6tq8Re37MSJE8X5xMREcT49Pd3iNu0q7f7s2bPivePj4y1v86f5+fnG2a5du4r3OuftT13XVa9rPHEhkHAhkHAhkHAhkHAhkHAhkHAhUMz7uNu3by/OT5482ffPXukz3t27dzfOut3uEDf526dPnxpnPn515XjiQiDhQiDhQiDhQiDhQiDhQiDhQqCYc9xe1qwp/w06evRo4+zGjRvFe1+9etXXTr9cu3atOJ+ammqcbdy4caDfPajS12y+e/duiJvwO09cCCRcCCRcCCRcCCRcCCRcCBRzHDQ7OzvQ/WNjY42z58+fF+8d9PW1LVu2FOcbNmwY6OeXvH37tji/f/9+cX7r1q0Wt6EtnrgQSLgQSLgQSLgQSLgQSLgQSLgQKOZrNquq/M2DV69eLc57fU1nql6v1l28eLE4v379epvr0AJfswkjSrgQSLgQSLgQSLgQSLgQSLgQKOYct5etW7cW53Nzc0PapF13794tzs+fP1+cv3z5ssVtGAbnuDCihAuBhAuBhAuBhAuBhAuBhAuBRuYct9f7ut1ut3F27ty54r2nTp3qa6df7t27V5yXzmJ7ncP+/Pmzn5VYxZzjwogSLgQSLgQSLgQSLgQSLgQSLgQamXNcGBXOcWFECRcCCRcCCRcCCRcCCRcCCRcCCRcCCRcCCRcCCRcCCRcCCRcCCRcCCRcCCRcCCRcCCRcCCRcCCRcCCRcCCRcCrVvi9XOdTuef5VgE6HQ6nc6OxVy0pM9VBlYH/1SGQMKFQMKFQMKFQMKFQMKFQMKFQMKFQMKFQP8C2VYEh1yZIIoAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAO4AAADuCAYAAAA+7jsiAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAABiVJREFUeJzt3b1rFGsYxuEZTSUIYuP3RyHpY2NhYyURbVIIYqFFEOwVBImWgo3WYmejhWKlgmkVFPQf0CogCnJqBZHsaU5zOGeeMZvsZu/NdbXPzs4Q/fkKb2amHQwGDZBl22ZfALB2woVAwoVAwoVAwoVAwoVAwoVAwoVAwoVAM2v5cNu2fs0KRmwwGLR9n7HiQiDhQiDhQiDhQiDhQiDhQiDhQiDhQiDhQiDhQiDhQiDhQiDhQiDhQiDhQiDhQiDhQiDhQiDhQiDhQiDhQiDhQiDhQiDhQiDhQiDhQiDhQiDhQiDhQqA1va2PrefDhw+ds7m5ufLY+fn5cr68vDzUNWHFhUjChUDChUDChUDChUDChUDChUD2cbe4K1eulPNqr3YwGJTHLi4ulnP7uMOz4kIg4UIg4UIg4UIg4UIg4UIg4UIg+7hT7urVq+X83r17Y7oSNpIVFwIJFwIJFwIJFwIJFwIJFwK1fbdm/evDbfvnH2Ys9u/fX87fvHlTzg8fPlzO27btnPX93fn582c5n52dLeffvn0r59NqMBh0/9D/YcWFQMKFQMKFQMKFQMKFQMKFQMKFQG7rm3AzM/Uf0YMHD8r5kSNH1nX+1dXVztnr16/LY/tes3nt2rVyfv369XK+lVlxIZBwIZBwIZBwIZBwIZBwIZBwIZB93Am3tLRUzs+cOVPO13K/9f/5+PFj5+zGjRvlsSdPnizn6722rcyKC4GEC4GEC4GEC4GEC4GEC4GEC4Hs406Ac+fOdc5u3bpVHjvqvdCFhYXOWd9zj2/fvl3ODx06NNQ1YcWFSMKFQMKFQMKFQMKFQMKFQMKFQN6POwbVPm3TNM2zZ886Z33PVR71Pm7f+Svnz58v5w8fPiznu3btGvrcybwfF6aUcCGQcCGQcCGQcCGQcCGQ7aANcPny5XJ+586dcr53797OWdvWOwOj3g7avn37yL77/fv35fzEiRMjO/cksx0EU0q4EEi4EEi4EEi4EEi4EEi4EMg+7h84ffp0Oe+7Pe3AgQNDn3vU+7h9j1j1CNXxs48LU0q4EEi4EEi4EEi4EEi4EEi4EMhrNpummZ2dLeevXr0q5+vdS338+HHn7OLFi+v67j5PnjwZ6fczGlZcCCRcCCRcCCRcCCRcCCRcCCRcCLRl7set9mqXl5fLYw8ePFjO+36Gnz59KufV/b4rKyvrOnef+/fvl/OvX7+u6/tH5cWLF+W872c+ydyPC1NKuBBIuBBIuBBIuBBIuBBIuBBoavZxjx49Ws7fvn3bOaveT9s0TbNtW/3v2+rqajn//PlzOd+3b1/nbOfOneWxo34/bvVc50k+982bN8v53bt3h7qmcbCPC1NKuBBIuBBIuBBIuBBIuBBoah7Pevbs2XK+Z8+ezlnf1kLfdk/f8ceOHSvn6/nuUW/JpJ57bm5uTFeyOay4EEi4EEi4EEi4EEi4EEi4EEi4EGhq9nEfPXpUzk+dOtU5W1hY2OCrmRxfvnwp5+/evRv6u/sekdp3O2Of+fn5ztnS0lJ5bPLjWf+EFRcCCRcCCRcCCRcCCRcCCRcCCRcCTc3jWfvs2LGjc7Z79+4xXsl/HT9+vHP2/Pnz8ti+P7+nT5+W8wsXLpTzzTQz0/1rBn2P1P3+/Xs5//Xr11DXNA4ezwpTSrgQSLgQSLgQSLgQSLgQSLgQaGrux+3z48ePoWbjUN0zW71qctr9/v27c9Z3n/G0s+JCIOFCIOFCIOFCIOFCIOFCoC2zHZRqkl+zyeax4kIg4UIg4UIg4UIg4UIg4UIg4UIg4UIg4UIg4UIg4UIg4UIg4UIg4UIg4UIg9+NOuEuXLpXzxcXFcv7y5cuNvBwmhBUXAgkXAgkXAgkXAgkXAgkXAgkXArVreS5v27Ye4gsjNhgMet+tasWFQMKFQMKFQMKFQMKFQMKFQMKFQMKFQMKFQMKFQMKFQMKFQMKFQMKFQMKFQMKFQMKFQMKFQMKFQMKFQMKFQMKFQMKFQMKFQMKFQMKFQMKFQMKFQMKFQMKFQDNr/PxfTdOsjOJCgKZpmubIn3xoTe/HBSaD/ypDIOFCIOFCIOFCIOFCIOFCIOFCIOFCIOFCoL8BoUwofOf2AjYAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Helper function to draw MNIST image\n",
    "# expect a 2D tensor as input\n",
    "def draw_image(im):\n",
    "    plt.imshow(im, cmap='gray', interpolation='none')\n",
    "    plt.xticks([])\n",
    "    plt.yticks([])\n",
    "    plt.show()\n",
    "\n",
    "\n",
    "examples = enumerate(train_loader)\n",
    "for batch_idx, (example_data, example_targets) in examples:\n",
    "\n",
    "    # First print the shape of the example_data tensor\n",
    "    print(example_data.shape) # 100 examples, 1 color, 28x28 pixel images\n",
    "    \n",
    "    # Show the first 6 images\n",
    "    for i in range(6):\n",
    "        draw_image(example_data[i][0])\n",
    "        \n",
    "    break # Abort the loop over the input data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# A few useful functions\n",
    "\n",
    "# Produce random numbers as input for generator\n",
    "# torch.randn produces random numbers following \n",
    "# a Gaussian with mean=0 and sigma=1\n",
    "def noise(size):\n",
    "    return torch.autograd.Variable(torch.randn(size, 100))\n",
    "\n",
    "# Translate images to flat vectors (784 = 28*28)\n",
    "def images_to_vectors(images):\n",
    "    return images.view(images.size(0), 784)\n",
    "\n",
    "# Translate flat vectors to images in the format (color, vertical, horizontal)\n",
    "def vectors_to_images(vectors):\n",
    "    return vectors.view(vectors.size(0), 1, 28, 28)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Homework\n",
    "\n",
    "\n",
    "Implement two networks, named `discriminator` and `generator` (the exact names are important so the subsequent code picks them up correctly). The Discriminator network should receive an input list with 784 features and have one output node. As it does binary discrimination between real and fake images, using a sigmoid for the last layer is recommended. The generator should receive 100 input features (random numbers) and produce a final output of length 784 (28 times 28). Using `tanh` as activiation function for the final layer might be useful. Using the code below, see what image generation results you can acvieve\n",
    "\n",
    "Produce ten example image after training has completed."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Homework: Implement the networks\n",
    "discriminator = ...\n",
    "generator = ...\n",
    "\n",
    "# We need two optimisers, one for the generator, one for the discriminator network\n",
    "d_optimizer = torch.optim.Adam(discriminator.parameters(), lr=0.0002)\n",
    "g_optimizer = torch.optim.Adam(generator.parameters(), lr=0.0002)\n",
    "\n",
    "loss = nn.BCELoss() # Binary cross entropy loss function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# The code below handles the training of the GAN\n",
    "\n",
    "# Total number of epochs to train\n",
    "n_epochs = 20 # start with one epoch (you will want to increase this number)\n",
    "\n",
    "# Loop over epochs\n",
    "for epoch in range(n_epochs):\n",
    "    \n",
    "    # Loop over training examples\n",
    "    examples = enumerate(train_loader)\n",
    "    for i_batch, (real_batch, _) in examples:\n",
    "\n",
    "        # Batch size\n",
    "        N = real_batch.size(0)\n",
    "        \n",
    "        # First Train Discriminator\n",
    "        real_data = torch.autograd.Variable(images_to_vectors(real_batch))\n",
    "\n",
    "        # Generate fake data\n",
    "        # (detach avoid calculating gradients for generator when training discriminator)\n",
    "        fake_data = generator(noise(N)).detach()\n",
    "\n",
    "        # Reset gradient\n",
    "        d_optimizer.zero_grad()\n",
    "    \n",
    "        # Train discriminator on real data\n",
    "        prediction_real = discriminator(real_data)\n",
    "\n",
    "        # Calculate error and backpropagate\n",
    "        # We want the discriminator to report \"1\" for real images\n",
    "        ones = torch.autograd.Variable(torch.ones(N, 1))\n",
    "        error_real = loss(prediction_real, ones)\n",
    "        error_real.backward()\n",
    "\n",
    "        # Train discriminator on fake data\n",
    "        prediction_fake = discriminator(fake_data)\n",
    "        \n",
    "        # Calculate error and backpropagate\n",
    "        # We want the discriminator to report \"0\" for fake images\n",
    "        zeros =  torch.autograd.Variable(torch.zeros(N, 1))\n",
    "        error_fake = loss(prediction_fake, zeros)\n",
    "        error_fake.backward()\n",
    "\n",
    "        # Update weights with gradients\n",
    "        d_optimizer.step()\n",
    "\n",
    "        # Total discriminator error\n",
    "        d_error =  error_real + error_fake\n",
    "\n",
    "        # Train Generator\n",
    "        # Generate fake data\n",
    "        fake_data = generator(noise(N))\n",
    "\n",
    "        # Reset gradients\n",
    "        g_optimizer.zero_grad()\n",
    "\n",
    "        # Sample noise and generate fake data\n",
    "        prediction = discriminator(fake_data)\n",
    "\n",
    "        # Calculate error and backpropagate\n",
    "        # instead of maximising the loss for zeros we minimise the loss for ones\n",
    "        ones = torch.autograd.Variable(torch.ones(N, 1))\n",
    "        g_error = loss(prediction, ones)\n",
    "        g_error.backward()\n",
    "\n",
    "        # Update weights with gradients\n",
    "        g_optimizer.step()\n",
    "\n",
    "        # Show status every few batches\n",
    "        if (i_batch) % 50 == 0: \n",
    "                        \n",
    "            test_images = vectors_to_images(generator(noise(1)))\n",
    "            test_images = test_images.data\n",
    "            print(\"Epoch: {0}, Batch: {1}, Error D: {2}, Error G: {3}\".format(epoch,i_batch,d_error,g_error))\n",
    "            draw_image(test_images[0][0])\n",
    "                \n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}