06. Making Deep Learing Learn Faster/01. Batch Normalization - MXNet.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Batch Normalization - MXNet (From Scratch).ipynb",
      "version": "0.3.2",
      "provenance": [],
      "collapsed_sections": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "code",
      "metadata": {
        "id": "PRwW2a0itwE-",
        "colab_type": "code",
        "outputId": "fa46cc45-78cd-43c9-a84b-8486013bab4a",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 151
        }
      },
      "source": [
        "!pip install mxnet-cu100"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Requirement already satisfied: mxnet-cu100 in /usr/local/lib/python3.6/dist-packages (1.4.1)\n",
            "Requirement already satisfied: requests>=2.20.0 in /usr/local/lib/python3.6/dist-packages (from mxnet-cu100) (2.21.0)\n",
            "Requirement already satisfied: numpy<1.15.0,>=1.8.2 in /usr/local/lib/python3.6/dist-packages (from mxnet-cu100) (1.14.6)\n",
            "Requirement already satisfied: graphviz<0.9.0,>=0.8.1 in /usr/local/lib/python3.6/dist-packages (from mxnet-cu100) (0.8.4)\n",
            "Requirement already satisfied: idna<2.9,>=2.5 in /usr/local/lib/python3.6/dist-packages (from requests>=2.20.0->mxnet-cu100) (2.8)\n",
            "Requirement already satisfied: chardet<3.1.0,>=3.0.2 in /usr/local/lib/python3.6/dist-packages (from requests>=2.20.0->mxnet-cu100) (3.0.4)\n",
            "Requirement already satisfied: urllib3<1.25,>=1.21.1 in /usr/local/lib/python3.6/dist-packages (from requests>=2.20.0->mxnet-cu100) (1.24.3)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.6/dist-packages (from requests>=2.20.0->mxnet-cu100) (2019.3.9)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "yD7crq8C2Gof",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import mxnet as mx\n",
        "from mxnet import nd, autograd, gluon\n",
        "import matplotlib.pyplot as plt\n",
        "\n",
        "ctx = mx.gpu()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "e9V_k_6s2xhX",
        "colab_type": "text"
      },
      "source": [
        "#Getting MNIST dataset"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ZLDyQYZo2wxw",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def transform(data, label):\n",
        "    return nd.moveaxis(data.astype('float32'), 2, 0)/255.0, label.astype('float32')\n",
        "\n",
        "train_mnist = gluon.data.vision.datasets.MNIST(train = True, transform = transform)\n",
        "test_mnist = gluon.data.vision.datasets.MNIST(train = False, transform = transform)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "H81sRUEQuNTw",
        "colab_type": "code",
        "outputId": "4b1466da-0bec-4e39-f469-9665e2d48246",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "image, label = train_mnist[5]\n",
        "image.shape, label.shape"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "((1, 28, 28), ())"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 9
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "dWwYBdfFuerm",
        "colab_type": "code",
        "outputId": "e9b3129d-1ec9-4117-a11c-bb08710fbdf1",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 281
        }
      },
      "source": [
        "plt.imshow(image[0].asnumpy(), cmap = 'gray')\n",
        "plt.title(label)\n",
        "plt.show()"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAP8AAAEICAYAAACQ6CLfAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAD6RJREFUeJzt3X+sVGV+x/HPR8ANAlpZ6O2Na8Vd\n3CiFqIRoMaa1Ma6IWVEbdZEYStVrzRqqrUZim4g2xsV0bYwmJmxEceNqTUA0G+3iryxWzQZUVFR2\nQXMtInJj1Yo/Agt8+8c9172Ld565zJyZM/i8X8nkzpzvOXO+Gf1wzswzcx5HhADk56CqGwBQDcIP\nZIrwA5ki/ECmCD+QKcIPZIrwA5ki/Bmz/S3b99h+1/YO2+ttn5VY/xrbH9j+1PYy299qZ78oF+HP\n20hJWyT9taTDJP2rpIdtT9p3RdtnSlok6XRJR0n6rqSb2tUoyme+4YfBbL8m6aaIWLHP8l9I6o2I\nG4rHp0t6ICL+rII2UQKO/PiK7S5J35f0xhDlv5D06qDHr0rqsv3tdvSG8hF+SJJsj5L0gKTlEbFx\niFXGSvq/QY8H7o9rdW9oDcIP2T5I0s8l7ZJ0VY3VPpN06KDHA/d3tLA1tBDhz5xtS7pHUpekv42I\n39dY9Q1Jxw96fLyk7RHxvy1uES1C+HG3pOMk/TAivkysd7+kS21Psf0n6h8ZuK8N/aFF+LQ/Y7aP\nktQraaek3YNKV0h6TtKbkqZExP8U6/+TpOsljZa0QtI/RMTOdvaM8hB+IFOc9gOZIvxApgg/kCnC\nD2RqZDt3ZptPF4EWiwgPZ72mjvy2Z9n+re3Nthc181wA2qvhoT7bIyT9TtIZkt6TtFbS3Ih4M7EN\nR36gxdpx5D9J0uaIeCcidkl6SNKcJp4PQBs1E/4j1H8hiAHvFcv+iO0e2+tsr2tiXwBK1vIP/CJi\nqaSlEqf9QCdp5si/VdKRgx5/p1gG4ADQTPjXSjrG9tG2D5b0I0mPldMWgFZr+LQ/InbbvkrSrySN\nkLQsIoa6/BOADtTWX/Xxnh9ovbZ8yQfAgYvwA5ki/ECmCD+QKcIPZIrwA5ki/ECmCD+QKcIPZIrw\nA5ki/ECmCD+QKcIPZIrwA5ki/ECmCD+QKcIPZIrwA5ki/ECmCD+QKcIPZIrwA5ki/ECmCD+QKcIP\nZIrwA5ki/ECmCD+QKcIPZKrhKboBSRo3blyyPnbs2Jq1s88+O7ntxIkTk/Xbb789Wd+5c2eynrum\nwm+7V9IOSXsk7Y6IGWU0BaD1yjjy/01EfFjC8wBoI97zA5lqNvwhabXtl2z3DLWC7R7b62yva3Jf\nAErU7Gn/qRGx1fafSnrS9saIWDN4hYhYKmmpJNmOJvcHoCRNHfkjYmvxt0/SI5JOKqMpAK3XcPht\nj7E9buC+pB9I2lBWYwBaq5nT/i5Jj9geeJ5fRMR/ldIV2mbSpEnJ+vXXX5+sz5w5M1mfOnXq/rY0\nbN3d3cn6woULW7bvb4KGwx8R70g6vsReALQRQ31Apgg/kCnCD2SK8AOZIvxAphzRvi/d8Q2/1jj2\n2GNr1q6++urktvPmzUvWR48enawXQ701bdmypWZtx44dyW2PO+64ZP3DD9O/JzvttNNq1jZu3Jjc\n9kAWEen/KAWO/ECmCD+QKcIPZIrwA5ki/ECmCD+QKcIPZIpLd3eAww47LFlfsmRJsn7RRRfVrNW7\ntHazNm3alKyfeeaZNWujRo1KbltvLH7ChAlN1XPHkR/IFOEHMkX4gUwRfiBThB/IFOEHMkX4gUwx\nzt8BzjvvvGT9sssua1MnX/f2228n62eccUaynvo9/+TJkxvqCeXgyA9kivADmSL8QKYIP5Apwg9k\nivADmSL8QKYY5+8AF1xwQcueu7e3N1lfu3Ztsl5viu7UOH499a7Lj9aqe+S3vcx2n+0Ng5aNt/2k\n7U3F38Nb2yaAsg3ntP8+SbP2WbZI0tMRcYykp4vHAA4gdcMfEWskfbTP4jmSlhf3l0s6t+S+ALRY\no+/5uyJiW3H/A0ldtVa03SOpp8H9AGiRpj/wi4hITcAZEUslLZWYqBPoJI0O9W233S1Jxd++8loC\n0A6Nhv8xSfOL+/MlPVpOOwDape5pv+0HJZ0maYLt9yTdKOknkh62famkdyVd2Momv+kuv/zyZL2n\nJ/2RyerVq2vWNm/enNy2r6+6k7aurpofFaEN6oY/IubWKJ1eci8A2oiv9wKZIvxApgg/kCnCD2SK\n8AOZ4ie9HeD9999P1hcvXtyeRtps5syZVbeQNY78QKYIP5Apwg9kivADmSL8QKYIP5Apwg9kinH+\nzC1cuDBZHzNmTMv2PW3atKa2f+GFF5L1F198sann/6bjyA9kivADmSL8QKYIP5Apwg9kivADmSL8\nQKYY5z8AHHLIIcn6lClTatZuvPHG5LazZ89uqKcBBx2UPn7s3bu34eeud52DBQsWJOt79uxpeN85\n4MgPZIrwA5ki/ECmCD+QKcIPZIrwA5ki/ECmGOdvg1GjRiXrJ554YrK+YsWKZL27u7tm7csvv0xu\nW28svd5v4mfNmpWs1/uOQsrIken/Pc8///xk/Y477qhZ27VrV0M9fZPUPfLbXma7z/aGQcsW295q\ne31xa+6bIgDabjin/fdJGuqf9/+IiBOK2+PltgWg1eqGPyLWSPqoDb0AaKNmPvC7yvZrxduCw2ut\nZLvH9jrb65rYF4CSNRr+uyV9T9IJkrZJ+mmtFSNiaUTMiIgZDe4LQAs0FP6I2B4ReyJir6SfSTqp\n3LYAtFpD4bc9eGzpPEkbaq0LoDM5ItIr2A9KOk3SBEnbJd1YPD5BUkjqlXRFRGyruzM7vbMD1MEH\nH5ys1xsLX7lyZVP7v+mmm2rWnnnmmeS2zz//fLI+fvz4ZL3e80+dOjVZb6V58+bVrK1atSq57c6d\nO8tup20iwsNZr+6XfCJi7hCL79nvjgB0FL7eC2SK8AOZIvxApgg/kCnCD2Sq7lBfqTs7gIf6Uj/L\nvfnmm5PbXnfddU3t+4knnkjWL7nkkpq1Tz75JLntxIkTk/XHH0//Zmv69OnJeuqns7fddlty23rD\nhHPmzEnWU5566qlkfcmSJcn6xx9/3PC+JWn9+vVNbZ8y3KE+jvxApgg/kCnCD2SK8AOZIvxApgg/\nkCnCD2SKcf7CiBEjkvVbbrmlZu3aa69Nbvv5558n64sWLUrWH3rooWQ9NeY8Y0b6Akp33XVXsl5v\n+82bNyfrV155Zc3as88+m9z20EMPTdZPOeWUZD31k95zzjknue2YMWOS9Xq2bNmSrB999NFNPX8K\n4/wAkgg/kCnCD2SK8AOZIvxApgg/kCnCD2SKcf5Cajxaku68886atS+++CK5bU9PT7K+evXqZP3k\nk09O1hcsWFCzdtZZZyW3HT16dLJe71oF9957b7Jeb7y7KnPnDnVR6j+4+OKLm3r+a665Jlmv9/2I\nZjDODyCJ8AOZIvxApgg/kCnCD2SK8AOZIvxApoYzRfeRku6X1KX+KbmXRsQdtsdL+k9Jk9Q/TfeF\nEZG8mHknj/Nv25aeYTx1fft60zlv3LgxWa/32/HJkycn681YvHhxsn7rrbcm63v27CmxG5ShzHH+\n3ZL+OSKmSPpLST+2PUXSIklPR8Qxkp4uHgM4QNQNf0Rsi4iXi/s7JL0l6QhJcyQtL1ZbLuncVjUJ\noHz79Z7f9iRJJ0r6jaSuiBg4V/5A/W8LABwgRg53RdtjJa2QdHVEfGr/4W1FRESt9/O2eySlv9wO\noO2GdeS3PUr9wX8gIlYWi7fb7i7q3ZL6hto2IpZGxIyISF8JEkBb1Q2/+w/x90h6KyJuH1R6TNL8\n4v58SY+W3x6AVhnOUN+pkp6T9LqkvcXiG9T/vv9hSX8u6V31D/V9VOe5Onao75VXXknWp02b1qZO\nvq7eNNlr1qypWVu1alVy297e3mR99+7dyTo6z3CH+uq+54+I/5ZU68lO35+mAHQOvuEHZIrwA5ki\n/ECmCD+QKcIPZIrwA5ni0t2FcePGJevnnlv7d0vTp09PbtvXN+SXH7+ybNmyZD01Bbck7dq1K1lH\nXrh0N4Akwg9kivADmSL8QKYIP5Apwg9kivADmWKcH/iGYZwfQBLhBzJF+IFMEX4gU4QfyBThBzJF\n+IFMEX4gU4QfyBThBzJF+IFMEX4gU4QfyBThBzJF+IFM1Q2/7SNtP2v7Tdtv2P7HYvli21ttry9u\ns1vfLoCy1L2Yh+1uSd0R8bLtcZJeknSupAslfRYR/z7snXExD6Dlhnsxj5HDeKJtkrYV93fYfkvS\nEc21B6Bq+/We3/YkSSdK+k2x6Crbr9leZvvwGtv02F5ne11TnQIo1bCv4Wd7rKRfS7olIlba7pL0\noaSQ9G/qf2vw93Weg9N+oMWGe9o/rPDbHiXpl5J+FRG3D1GfJOmXETG1zvMQfqDFSruAp21LukfS\nW4ODX3wQOOA8SRv2t0kA1RnOp/2nSnpO0uuS9haLb5A0V9IJ6j/t75V0RfHhYOq5OPIDLVbqaX9Z\nCD/Qely3H0AS4QcyRfiBTBF+IFOEH8gU4QcyRfiBTBF+IFOEH8gU4QcyRfiBTBF+IFOEH8gU4Qcy\nVfcCniX7UNK7gx5PKJZ1ok7trVP7kuitUWX2dtRwV2zr7/m/tnN7XUTMqKyBhE7trVP7kuitUVX1\nxmk/kCnCD2Sq6vAvrXj/KZ3aW6f2JdFboyrprdL3/ACqU/WRH0BFCD+QqUrCb3uW7d/a3mx7URU9\n1GK71/brxbTjlc4vWMyB2Gd7w6Bl420/aXtT8XfIORIr6q0jpm1PTCtf6WvXadPdt/09v+0Rkn4n\n6QxJ70laK2luRLzZ1kZqsN0raUZEVP6FENt/JekzSfcPTIVm+zZJH0XET4p/OA+PiOs7pLfF2s9p\n21vUW61p5f9OFb52ZU53X4YqjvwnSdocEe9ExC5JD0maU0EfHS8i1kj6aJ/FcyQtL+4vV///PG1X\no7eOEBHbIuLl4v4OSQPTylf62iX6qkQV4T9C0pZBj99ThS/AEELSatsv2e6pupkhdA2aFu0DSV1V\nNjOEutO2t9M+08p3zGvXyHT3ZeMDv687NSKmSzpL0o+L09uOFP3v2TpprPZuSd9T/xyO2yT9tMpm\nimnlV0i6OiI+HVyr8rUboq9KXrcqwr9V0pGDHn+nWNYRImJr8bdP0iPqf5vSSbYPzJBc/O2ruJ+v\nRMT2iNgTEXsl/UwVvnbFtPIrJD0QESuLxZW/dkP1VdXrVkX410o6xvbRtg+W9CNJj1XQx9fYHlN8\nECPbYyT9QJ039fhjkuYX9+dLerTCXv5Ip0zbXmtaeVX82nXcdPcR0fabpNnq/8T/bUn/UkUPNfr6\nrqRXi9sbVfcm6UH1nwb+Xv2fjVwq6duSnpa0SdJTksZ3UG8/V/9U7q+pP2jdFfV2qvpP6V+TtL64\nza76tUv0Vcnrxtd7gUzxgR+QKcIPZIrwA5ki/ECmCD+QKcIPZIrwA5n6f1r83bQR3u6bAAAAAElF\nTkSuQmCC\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "x_eoQ8Cx2w0x",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "batch_size = 64\n",
        "train_data = gluon.data.DataLoader(train_mnist, batch_size, True)\n",
        "test_data = gluon.data.DataLoader(test_mnist, batch_size, False)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "uc2JVOAX4zsR",
        "colab_type": "text"
      },
      "source": [
        "##Defining Batch Norm Layer"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "60aSuiux4DR9",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def batch_norm(x, gamma, beta, momentum = 0.8, eps = 1e-5, name = '', training = True):\n",
        "    global moving_means, moving_vars\n",
        "\n",
        "    if len(x.shape) not in (2, 4):\n",
        "        raise ValueError('Only dense and cnn layers are suppported.')\n",
        "\n",
        "    if len(x.shape) == 2:\n",
        "        mean = nd.mean(x, axis=0)\n",
        "        variance = nd.mean((x - mean) ** 2, axis=0)\n",
        "        if training:\n",
        "            x_hat = (x - mean) * 1.0 / nd.sqrt(variance + eps)\n",
        "        else:\n",
        "            x_hat = (x - moving_means[name]) * 1.0 / nd.sqrt(moving_vars[name] + eps)\n",
        "        out = gamma * x_hat + beta\n",
        "\n",
        "    elif len(x.shape) == 4:\n",
        "        N, C, H, W = x.shape\n",
        "        mean = nd.mean(x, axis=(0,2,3))\n",
        "        variance = nd.mean((x - mean.reshape((1, C, 1, 1))) ** 2, axis=(0, 2, 3))\n",
        "        x_hat = (x - mean.reshape((1, C, 1, 1))) * 1.0 / nd.sqrt(variance.reshape((1, C, 1, 1)) + eps)\n",
        "        if training:\n",
        "            x_hat = (x - mean.reshape((1, C, 1, 1))) * 1.0 / nd.sqrt(variance.reshape((1, C, 1, 1)) + eps)\n",
        "        else:\n",
        "            x_hat = (x - moving_means[name].reshape((1, C, 1, 1))) * 1.0 \\\n",
        "                / nd.sqrt(moving_vars[name].reshape((1, C, 1, 1)) + eps)\n",
        "        out = gamma.reshape((1, C, 1, 1)) * x_hat + beta.reshape((1, C, 1, 1))\n",
        "\n",
        "    try:\n",
        "        moving_means, moving_vars\n",
        "    except:\n",
        "        moving_means, moving_vars = {}, {}\n",
        "\n",
        "    if name not in moving_means:\n",
        "        moving_means[name] = mean\n",
        "    else:\n",
        "        moving_means[name] = moving_means[name] * momentum + mean * (1.0 - momentum)\n",
        "    if name not in moving_vars:\n",
        "        moving_vars[name] = variance\n",
        "    else:\n",
        "        moving_vars[name] = moving_vars[name] * momentum + variance * (1.0 - momentum)\n",
        "    return out"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "FqqqOgfG0P7f",
        "colab_type": "text"
      },
      "source": [
        "##Initializing model parameters and batch normal parameters"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "M8bduuD24DOT",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "weight_scale = .01\n",
        "\n",
        "W1 = nd.random_normal(shape=(20, 1, 3,3), scale=weight_scale, ctx=ctx)\n",
        "b1 = nd.random_normal(shape=20, scale=weight_scale, ctx=ctx)\n",
        "\n",
        "gamma1 = nd.random_normal(shape=20, loc=1, scale=weight_scale, ctx=ctx)\n",
        "beta1 = nd.random_normal(shape=20, scale=weight_scale, ctx=ctx)\n",
        "\n",
        "W2 = nd.random_normal(shape=(50, 20, 5, 5), scale=weight_scale, ctx=ctx)\n",
        "b2 = nd.random_normal(shape=50, scale=weight_scale, ctx=ctx)\n",
        "\n",
        "gamma2 = nd.random_normal(shape=50, loc=1, scale=weight_scale, ctx=ctx)\n",
        "beta2 = nd.random_normal(shape=50, scale=weight_scale, ctx=ctx)\n",
        "\n",
        "W3 = nd.random_normal(shape=(800, 128), scale=weight_scale, ctx=ctx)\n",
        "b3 = nd.random_normal(shape=128, scale=weight_scale, ctx=ctx)\n",
        "\n",
        "gamma3 = nd.random_normal(shape=128, loc=1, scale=weight_scale, ctx=ctx)\n",
        "beta3 = nd.random_normal(shape=128, scale=weight_scale, ctx=ctx)\n",
        "\n",
        "W4 = nd.random_normal(shape=(128, 10), scale=weight_scale, ctx=ctx)\n",
        "b4 = nd.random_normal(shape=10, scale=weight_scale, ctx=ctx)\n",
        "\n",
        "params = [W1, b1, gamma1, beta1, W2, b2, gamma2, beta2, W3, b3, gamma3, beta3, W4, b4]\n",
        "\n",
        "for param in params:\n",
        "    param.attach_grad()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "uB9EJyuu4DM0",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def relu(x):\n",
        "    return nd.maximum(x, 0)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "GwO0Qwmy5fyn",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def softmax(linear):\n",
        "    exp = nd.exp(linear - nd.max(linear))\n",
        "    partition = nd.sum(exp).reshape((-1, 1))\n",
        "    return exp / partition"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "yvQc2ogE0WeL",
        "colab_type": "text"
      },
      "source": [
        "##Building the model"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "3FdYyCb65fvS",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def model(x, training = True):\n",
        "    h1_conv = nd.Convolution(data=x, weight=W1, bias=b1, kernel=(3,3), num_filter=20)\n",
        "    h1_normed = batch_norm(h1_conv, gamma1, beta1, name='bn1', training=training)\n",
        "    h1_activation = relu(h1_normed)\n",
        "    h1 = nd.Pooling(data=h1_activation, pool_type=\"avg\", kernel=(2,2), stride=(2,2))\n",
        "\n",
        "    h2_conv = nd.Convolution(data=h1, weight=W2, bias=b2, kernel=(5,5), num_filter=50)\n",
        "    h2_normed = batch_norm(h2_conv, gamma2, beta2, name='bn2', training=training)\n",
        "    h2_activation = relu(h2_normed)\n",
        "    h2 = nd.Pooling(data=h2_activation, pool_type=\"avg\", kernel=(2,2), stride=(2,2))\n",
        "\n",
        "    h2 = nd.flatten(h2)\n",
        "    \n",
        "    h3_linear = nd.dot(h2, W3) + b3\n",
        "    h3_normed = batch_norm(h3_linear, gamma3, beta3, name='bn3', training=training)\n",
        "    h3 = relu(h3_normed)\n",
        "    \n",
        "    yhat_linear = nd.dot(h3, W4) + b4\n",
        "    \n",
        "    return softmax(yhat_linear)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "UQtVGnWv8zcN",
        "colab_type": "code",
        "outputId": "a7f0d91a-7fd2-467d-d082-2476004e5cfd",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 50
        }
      },
      "source": [
        "for data, _ in train_data:\n",
        "    data = data.as_in_context(ctx)\n",
        "    break\n",
        "    \n",
        "output = model(data, training = True)\n",
        "\n",
        "print(data.shape)\n",
        "print(output.shape)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "(64, 1, 28, 28)\n",
            "(64, 10)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "HjbezA0S5fwQ",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def categoricalcrossentropy(pred, label):\n",
        "    return -nd.sum(label * nd.log(pred))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "AzdO_SXx5fuG",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def sgd(params, lr):\n",
        "    for param in params:\n",
        "        param[:] = param - lr * param.grad"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Ff2oB_yFWR9O",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def accuracy(predictions, labels):\n",
        "    return nd.mean(predictions.argmax(1) == labels.argmax(1))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "eGXnJ59P0bKz",
        "colab_type": "text"
      },
      "source": [
        "##Training the model"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "toHHcCNo6AMC",
        "colab_type": "code",
        "outputId": "de103c63-40c5-4901-c089-e9e587243a2a",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 185
        }
      },
      "source": [
        "epochs = 10\n",
        "num_batches = 60000/batch_size\n",
        "learning_rate = .001\n",
        "losses = []\n",
        "accs = []\n",
        "\n",
        "for epoch in range(epochs):\n",
        "    cumulative_loss = 0\n",
        "    cumulative_acc = 0\n",
        "    for features, labels in train_data:\n",
        "        features = features.as_in_context(ctx)\n",
        "        labels = labels.as_in_context(ctx).one_hot(10)\n",
        "        with autograd.record():\n",
        "            output = model(features, training = True)\n",
        "            loss = categoricalcrossentropy(output, labels)\n",
        "        loss.backward()\n",
        "        sgd(params, learning_rate)\n",
        "        cumulative_loss += loss\n",
        "        cumulative_acc += accuracy(output, labels)\n",
        "    print(f'Epoch: {epoch} | Loss: {cumulative_loss.asscalar()/num_batches:.5f} | Accuracy: {cumulative_acc.asscalar()/num_batches:.5f}')\n",
        "    losses.append(cumulative_loss.asscalar()/num_batches)\n",
        "    accs.append(cumulative_acc.asscalar()/num_batches)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Epoch: 0 | Loss: 285.27793 | Accuracy: 0.96815\n",
            "Epoch: 1 | Loss: 271.67513 | Accuracy: 0.98985\n",
            "Epoch: 2 | Loss: 269.75657 | Accuracy: 0.99202\n",
            "Epoch: 3 | Loss: 268.92355 | Accuracy: 0.99375\n",
            "Epoch: 4 | Loss: 268.42775 | Accuracy: 0.99487\n",
            "Epoch: 5 | Loss: 268.13422 | Accuracy: 0.99568\n",
            "Epoch: 6 | Loss: 267.89692 | Accuracy: 0.99640\n",
            "Epoch: 7 | Loss: 267.64055 | Accuracy: 0.99730\n",
            "Epoch: 8 | Loss: 267.53333 | Accuracy: 0.99738\n",
            "Epoch: 9 | Loss: 267.39447 | Accuracy: 0.99783\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "dJcq8cBl6AP_",
        "colab_type": "code",
        "outputId": "2d71b712-3363-415e-e08a-b60edb6ada35",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 545
        }
      },
      "source": [
        "plt.plot(losses, c = 'r')\n",
        "plt.title('Training Loss')\n",
        "plt.show()\n",
        "plt.plot(accs, c = 'g')\n",
        "plt.title('Training Accuracy')\n",
        "plt.show()"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYQAAAEICAYAAABfz4NwAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAIABJREFUeJzt3XuYXWV99vHvnWTCISHhkAFCDgQE\nhEAhgSFkbwQKorVoBa0VaEVaD1hftJDyYpW3emntwQKCWCsaQatXIyKQitVQRaGKQAKTMJKQRAIB\nTcgYBpAcOASG+b1/PGucncnsmT3Htffs+3Nd+9prP+tZa//WQOaetZ51UERgZmY2Ju8CzMysOjgQ\nzMwMcCCYmVnGgWBmZoADwczMMg4EMzMDHAhWpySNlbRd0syh7GtWy+TrEKwWSNpe8nFPYAfwWvb5\nQxGxaOSrGjxJ/whMj4i/zLsWs3F5F2BWiYiY2Dkt6UngAxHxk3L9JY2LiPaRqM1stPAhIxsVJP2j\npJsl3SRpG/AeSQVJSyU9L6lV0hclNWT9x0kKSbOyz/+Zzb9D0jZJ90s6pL99s/l/LOlRSVsk/Zuk\neyX95QC26WhJP8vqXynprSXz3iZpTfb9GyUtyNr3l7QkW+Y5ST8f6M/U6o8DwUaTdwDfBiYDNwPt\nwCXAFOBk4C3Ah3pZ/s+BTwL7Ar8BPtvfvpL2B74LXJ597xPAvP5uiKTxwA+AHwKNwALgZkmHZV2+\nAbw/IvYCjgV+lrVfDqzPljkQ+Pv+frfVLweCjSa/iIj/joiOiHgpIh6MiGUR0R4R64GFwGm9LH9r\nRDRHxKvAImDOAPq+DWiJiNuzedcCzwxgW04GxgNXRcSr2eGxO4DzsvmvArMl7RURz0XEipL2g4CZ\nEfFKRHgPwSrmQLDRZEPpB0lHSvqhpN9K2gr8A+mv9nJ+WzL9IjCxXMde+h5UWkekszY2VlB7dwcB\nv4mdz/r4NTAtm34H8HbgN5L+V9JJWfvnsn4/lfS4pMsH8N1WpxwINpp0P2Xuq8Aq4LCImAR8CtAw\n19AKTO/8IEl0/RLvj03AjGz5TjOBpwCyPZ+3A/uTDi19J2vfGhELImIWcA7wd5J62ysy+z0Hgo1m\newFbgBckHUXv4wdD5QfA8ZL+RNI40hhGYx/LjJW0e8lrN+A+0hjIZZIaJJ0BnEUaR9hD0p9LmpQd\nltoGdABk3/u6LEi2kE7N7RieTbXRxoFgo9llwIWkX5hfJQ00D6uI2AycC1wDPAu8DniIdN1EOe8B\nXip5/SoidgB/ApxNGoP4IvDnEbEuW+ZC4NfZobD3Z+sAeD1wF7AduBe4LiLuGbINtFHNF6aZDSNJ\nY0mHf97lX8xW7byHYDbEJL1F0t7ZoZ9Pks78eSDnssz65EAwG3pvIF0L0Ab8EfCO7BCQWVXzISMz\nMwO8h2BmZpmaurndlClTYtasWXmXYWZWU5YvX/5MRPR1+nNtBcKsWbNobm7Ouwwzs5oi6deV9PMh\nIzMzAxwIZmaWcSCYmRngQDAzs4wDwczMAAeCmZllHAhmZgbUSyDccQd87nN5V2FmVtX6DARJMyTd\nLWm1pEckXZK1z5G0VFKLpGZJ87L2P5S0JWtvkfSpMus9RNIySY9Jujl7qPjwuOsu+PSnYYfvL2Zm\nVk4lewjtwGURMRuYD1wsaTZwJfCZiJhDejThlSXL3BMRc7LXP5RZ778C10bEYcDvSA/5GB6FQgqD\nhx4atq8wM6t1fQZCRLRGxIpsehuwhvSM2AAmZd0mkx4CUpHs8X5nALdmTd8kPf91eBQK6f3++4ft\nK8zMal2/xhAkzQLmAsuAS4GrJG0ArgY+UdK1IOmXku6QdHQPq9oPeD4i2rPPGynzIHJJF2WHpJrb\n2tr6U26XqVNh1iy4776BLW9mVgcqDgRJE4HbgEsjYivwYWBBRMwAFgA3Zl1XAAdHxHHAvwHfG0yB\nEbEwIpoioqmxsc+b9ZVXLKZA8PMfzMx6VFEgSGoghcGiiFicNV8IdE7fAswDiIitEbE9m14CNEia\n0m2VzwJ7S+q82+p04KkBb0UlikXYtAk2bBjWrzEzq1WVnGUk0l//ayLimpJZm4DTsukzgHVZ/wOz\nZcjOPBpDCoDfi/SYtruBd2VNFwK3D3wzKtA5juDDRmZmPapkD+Fk4ALgjJJTSc8CPgh8XtIvgX8G\nLsr6vwtYlbV/ETgvCwAkLZF0UNbv74C/lfQYaUzhRobTscfCnnt6YNnMrIyaeqZyU1NTDOoBOaef\nDtu3w4MPDl1RZmZVTtLyiGjqq199XKncqViElhZ48cW8KzEzqzr1FQiFArS3gx/DaWa2i/oKhPnz\n07vHEczMdlFfgTBlCrz+9T7TyMysB/UVCJAOG/kCNTOzXdRfIBSL8Mwz8PjjeVdiZlZV6i8QfIGa\nmVmP6i8QZs+GSZM8sGxm1k39BcKYMV3jCGZm9nv1FwiQAmHVKti6Ne9KzMyqRn0GQrEIHR3wwAN5\nV2JmVjXqMxBOOgkkHzYyMytRn4EwaRIcc4wHls3MStRnIEAaR7j//nToyMzM6jgQikXYsgXWrs27\nEjOzqlDfgQAeRzAzy9RvIBx2WLrZnQPBzAyo50CQusYRzMysjgMBUiCsXQvPPZd3JWZmueszECTN\nkHS3pNWSHpF0SdY+R9JSSS2SmiXNy9r/QtLDklZKuk/ScWXW+x+SnsiWb5E0Z2g3rQKd4whLl474\nV5uZVZtK9hDagcsiYjYwH7hY0mzgSuAzETEH+FT2GeAJ4LSI+APgs8DCXtZ9eUTMyV4tA96KgWpq\ngrFjPY5gZgaM66tDRLQCrdn0NklrgGlAAJOybpOBTVmf0t+uS4HpQ1nwkJowAebM8TiCmRn9HEOQ\nNAuYCywDLgWukrQBuBr4RA+LvB+4o5dV/lN2eOlaSbuV+c6LskNSzW1tbf0ptzLFIixbBu3tQ79u\nM7MaUnEgSJoI3AZcGhFbgQ8DCyJiBrAAuLFb/9NJgfB3ZVb5CeBI4ERg33L9ImJhRDRFRFNjY2Ol\n5VauUIAXXoCVK4d+3WZmNaSiQJDUQAqDRRGxOGu+EOicvgWYV9L/WOAG4OyIeLandUZEayQ7gG+U\nLj+iOgeWfdjIzOpcJWcZifTX/5qIuKZk1ibgtGz6DGBd1n8mKSguiIhHe1nv1JL1nwOsGsgGDNrM\nmTB1qgeWzazu9TmoDJwMXACslNR5JtAVwAeB6ySNA14GLsrmfQrYD/hy+l1Pe0Q0AUhaAnwgIjYB\niyQ1AgJagL8emk3qJyntJXgPwczqXCVnGf2C9Eu7Jyf00P8DwAfKrOuskukzKqxx+BUKcNttsHkz\nHHBA3tWYmeWivq9U7uRxBDMzBwIAxx8P48d7HMHM6poDAWC33eCEExwIZlbXHAidikVoboZXXsm7\nEjOzXDgQOhUKsGMHPPRQ3pWYmeXCgdCpUEjvHlg2szrlQOh00EFw8MEeRzCzuuVAKFUsOhDMrG45\nEEoVi/DUU7BhQ96VmJmNOAdCqc5xBO8lmFkdciCUOvZY2HNPDyybWV1yIJRqaIATT/QegpnVJQdC\nd8ViuhbhpZfyrsTMbEQ5ELorFNLjNJub867EzGxEORC688CymdUpB0J3U6bAEUd4YNnM6o4DoSeF\nQtpDiMi7EjOzEeNA6EmxCG1tsH593pWYmY2YPgNB0gxJd0taLekRSZdk7XMkLZXUIqlZ0rysXZK+\nKOkxSQ9LOr7Mek+QtDLr90VlD2CuCh5HMLM6VMkeQjtwWUTMBuYDF0uaDVwJfCYi5gCfyj4D/DFw\nePa6CLi+zHqvBz5Y0vctA92IITd7Nkya5HEEM6srfQZCRLRGxIpsehuwBpgGBDAp6zYZ2JRNnw18\nK5KlwN6SppauM/s8KSKWRkQA3wLOGYoNGhJjx8L8+d5DMLO60q8xBEmzgLnAMuBS4CpJG4CrgU9k\n3aYBpXeH25i1lZqWtffWJ1+FAqxcCdu25V2JmdmIqDgQJE0EbgMujYitwIeBBRExA1gA3DgcBUq6\nKBujaG5raxuOr+hZsQgdHfDAAyP3nWZmOaooECQ1kMJgUUQszpovBDqnbwHmZdNPATNKFp+etZV6\nKmvvrQ8AEbEwIpoioqmxsbGScofGSSeB5MNGZlY3KjnLSKS//tdExDUlszYBp2XTZwDrsunvA+/N\nzjaaD2yJiNbSdWaft0qan63/vcDtg9uUITZ5Mhx9tAeWzaxujKugz8nABcBKSS1Z2xWkM4SukzQO\neJl0RhHAEuAs4DHgReCvOlckqSU7Kwng/wD/AewB3JG9qkuhALfckg4djfElG2Y2uvUZCBHxC6Dc\nNQIn9NA/gIvLrGtOyXQzcExlZeakWISvfQ3Wrk2nopqZjWL+s7c3xWJ692EjM6sDDoTeHH447Lef\nB5bNrC44EHojpXEE7yGYWR1wIPSlUIA1a+C55/KuxMxsWDkQ+tI5jrBsWb51mJkNMwdCX048Md3b\nyOMIZjbKORD6MmECHHecA8HMRj0HQiWKxXRPo/b2vCsxMxs2DoRKFAqwfTusWpV3JWZmw8aBUAlf\noGZmdcCBUImDD4YDD/Q4gpmNag6ESkhpL8F7CGY2ijkQKlUowOOPw+bNeVdiZjYsHAiV8jiCmY1y\nDoRKHX88jB/vQDCzUcuBUKndd0+h4IFlMxulHAj9USxCczO88krelZiZDTkHQn8UCvDyy9DS0ndf\nM7Ma40Doj86BZR82MrNRqM9AkDRD0t2SVkt6RNIlWfvNklqy15OSWrL2vyhpb5HUIWlOD+v9tKSn\nSvqdNfSbN8QOOghmzvTAspmNSuMq6NMOXBYRKyTtBSyXdGdEnNvZQdLngS0AEbEIWJS1/wHwvYgo\nd4zl2oi4elBbMNKKRfjFL/KuwsxsyPW5hxARrRGxIpveBqwBpnXOlyTg3cBNPSx+PvCdoSm1ShSL\nsHEjbNiQdyVmZkOqX2MIkmYBc4HSx4edAmyOiHU9LHIuPQdFp49IeljS1yXtU+Y7L5LULKm5ra2t\nP+UOj0IhvfuwkZmNMhUHgqSJwG3ApRGxtWTW+fTwS1/SScCLEVHuntHXA68D5gCtwOd76hQRCyOi\nKSKaGhsbKy13+Bx3HOyxhwPBzEadSsYQkNRACoNFEbG4pH0c8E7ghB4WO49e9g4i4vc3BZL0NeAH\nFdacr4aG9FhNn2lkZqNMJWcZCbgRWBMR13SbfSawNiI2dltmDGlcoez4gaSpJR/fAdTO02eKRVix\nAl56Ke9KzMyGTCWHjE4GLgDO6OEU0XJ7AacCGyJifWmjpBskNWUfr5S0UtLDwOnAgoFtQg4KhfQ4\nzeXL867EzGzI9HnIKCJ+AajMvL8s0/6/wPwe2j9QMn1BpUVWnc6B5fvugze8Id9azMyGiK9UHojG\nRjj8cA8sm9mo4kAYqEIh7SFE5F2JmdmQcCAMVLEITz8NTzyRdyVmZkPCgTBQpeMIZmajgANhoI4+\nGvbay4FgZqOGA2Ggxo6F+fM9sGxmo4YDYTAKBXj4Ydi2Le9KzMwGzYEwGMUidHTAgw/mXYmZ2aA5\nEAbjpJPSu8cRzGwUcCAMxt57p8FlB4KZjQIOhMEqFGDp0nToyMyshjkQBqtYhN/9Dn71q7wrMTMb\nFAfCYBWL6d2nn5pZjXMgDNYRR8C++3ocwcxqngNhsKQ0juA9BDOrcQ6EoVAowOrVaSzBzKxGORCG\nQuc4wtKl+dZhZjYIDoShcOKJMGaMDxuZWU1zIAyFiRPhuOM8sGxmNa3PQJA0Q9LdklZLekTSJVn7\nzZJasteTklqy9lmSXiqZ95Uy691X0p2S1mXv+wztpo2wYhGWLYPXXsu7EjOzAalkD6EduCwiZgPz\ngYslzY6IcyNiTkTMAW4DFpcs83jnvIj46zLr/Tjw04g4HPhp9rl2FQqwfTusWpV3JWZmA9JnIERE\na0SsyKa3AWuAaZ3zJQl4N3BTP7/7bOCb2fQ3gXP6uXx18QVqZlbj+jWGIGkWMBdYVtJ8CrA5ItaV\ntB0i6SFJP5N0SpnVHRARrdn0b4EDynznRZKaJTW3tbX1p9yRNWsWHHCAxxHMrGZVHAiSJpIODV0a\nEVtLZp3PznsHrcDMiJgL/C3wbUmTelt3RAQQZeYtjIimiGhqbGystNyRJ6W9BAeCmdWoigJBUgMp\nDBZFxOKS9nHAO4GbO9siYkdEPJtNLwceB47oYbWbJU3N1jMVeHqgG1E1CgV4/HF4uvY3xczqTyVn\nGQm4EVgTEdd0m30msDYiNpb0b5Q0Nps+FDgcWN/Dqr8PXJhNXwjc3v/yq4zHEcyshlWyh3AycAFw\nRsmppGdl885j18HkU4GHs9NQbwX+OiKeA5B0g6SmrN/ngDdJWkcKls8Nclvyd8IJ0NDgQDCzmqR0\n+L42NDU1RXNzc95l9G7+fBg/Hn7+87wrMTMDQNLyiGjqq5+vVB5qxSI8+CC8+mrelZiZ9YsDYagV\nCvDyy9DSknclZmb94kAYaoVCevfpp2ZWYxwIQ236dJgxwwPLZlZzHAjDwReomVkNciAMh2IRNmyA\njRv77mtmViUcCMOhcxzBh43MrIY4EIbDnDmwxx4+bGRmNcWBMBwaGqCpyXsIZlZTHAjDpViEFSvS\nNQlmZjXAgTBcCoV0tfLy5XlXYmZWEQfCcPEFamZWYxwIw2X//eGwwzyOYGY1w4EwnAqFtIdQQ3eU\nNbP65UAYTsUibN4MTzyRdyVmZn1yIAwnX6BmZjXEgTCcjjkGJk70wLKZ1QQHwnAaOzY9Qc17CGZW\nA/oMBEkzJN0tabWkRyRdkrXfXPKM5SezZygj6U2Slktamb2fUWa9n5b0VA/PaR5dCgX45S9h+/a8\nKzEz69W4Cvq0A5dFxApJewHLJd0ZEed2dpD0eWBL9vEZ4E8iYpOkY4AfAdPKrPvaiLh6EPVXv2IR\nOjrSYzVPPz3vaszMyupzDyEiWiNiRTa9DVhDyS94SQLeDdyU9XkoIjZlsx8B9pC021AXXjNOOim9\nexzBzKpcv8YQJM0C5gLLSppPATZHxLoeFvlTYEVE7Cizyo9IeljS1yXtU+Y7L5LULKm5ra2tP+VW\nh332gdmzHQhmVvUqDgRJE4HbgEsjYmvJrPPJ9g669T8a+FfgQ2VWeT3wOmAO0Ap8vqdOEbEwIpoi\noqmxsbHScqtLoQBLl6ZDR2ZmVaqiQJDUQAqDRRGxuKR9HPBO4OZu/acD/wW8NyIe72mdEbE5Il6L\niA7ga8C8gW1CDSgW4bnn4NFH867EzKysSs4yEnAjsCYiruk2+0xgbURsLOm/N/BD4OMRcW8v651a\n8vEdwKr+FF5TisX07tNPzayKVbKHcDJwAXBGD6eInseuh4s+AhwGfKqk//4Akm6Q1JT1uzI7NfVh\n4HRgwaC3plodcUQaS/A4gplVMUUN3Xitqakpmpub8y5jYN76VnjySXjkkbwrMbM6I2l5RDT11c9X\nKo+UQgFWr4bnn8+7EjOzHjkQRkrnOMLSpfnWYWZWhgNhpMybB2PGeGDZzKqWA2GkTJwIc+fC9dfD\nd7/rh+aYWdVxIIykb34TZs6Ec8+Ft70tDTKbmVUJB8JIOvpoWLYMvvAF+PnP0+err4b29rwrMzNz\nIIy4sWPhkkvSGUdnngmXXw5NTfDAA3lXZmZ1zoGQlxkz4Pbb4b/+C555Jj1I52/+BrZu7XtZM7Nh\n4EDI2znnpL2Fj34UvvQlOOooWLzYg85mNuIcCNVg0iS47ro0vrD//vCnf5qCYsOGvCszszriQKgm\nJ56Ynqx29dXwk5+kvYUvfMGDzmY2IhwI1WbcOLjssnTPo9NOgwUL0lPXli/PuzIzG+UcCNVq1iz4\nwQ/SRWybNqUrnRcsgO3b867MzEYpB0I1k+DP/gzWroUPfSiNM8yeDf/933lXZmajkAOhFkyeDF/+\nMtx7b5p++9vTwPNTT+VdmZmNIg6EWlIowIoV8C//AkuWpEHnL30JXnst78rMbBRwINSahgb4+MfT\noHOhkK5fKBbhl7/MuzIzq3EOhFp16KHwP/8D3/52ukneCSfAxz4GL7yQd2VmVqMcCLVMgvPPhzVr\n4H3vg6uuSjfMW7Ik78rMrAb1GQiSZki6W9JqSY9IuiRrv1lSS/Z6UlJLyTKfkPSYpF9J+qMy6z1E\n0rKs382Sxg/dZtWZffeFhQvTHVT33DM9v/ncc6G1Ne/KzKyGVLKH0A5cFhGzgfnAxZJmR8S5ETEn\nIuYAtwGLASTNBs4DjgbeAnxZ0tge1vuvwLURcRjwO+D9g9+cOnfKKdDSAp/9bLpx3lFHwVe+Ah0d\neVdmZjWgz0CIiNaIWJFNbwPWANM650sS8G7gpqzpbOA7EbEjIp4AHgPmla4zW+YM4Nas6ZvAOYPb\nFANg/Hj4+7+HlSvTuMKHPwxveAOsWpV3ZWZW5fo1hiBpFjAXWFbSfAqwOSLWZZ+nAaV3ZdtISYBk\n9gOej4j2Xvp0fudFkpolNbe1tfWn3Pp2+OHpfkjf+hasW5ce33nFFfDSS3lXZmZVquJAkDSRdGjo\n0ogovWn/+XTtHQy5iFgYEU0R0dTY2DhcXzM6SXDBBWnQ+T3vSdcvHHMM3Hln3pWZWRWqKBAkNZDC\nYFFELC5pHwe8E7i5pPtTwIySz9OztlLPAntny5frY0NlyhT4xjfg7rvTzfPe/OZ0w7xPfhLuuQde\nfTXvCs2sClRylpGAG4E1EXFNt9lnAmsjYmNJ2/eB8yTtJukQ4HBgp+dDRkQAdwPvypouBG4f2CZY\nxf7wD9MFbFdemR7l+c//DKeeCvvtB2efna56fvRRP5zHrE4p+vjHL+kNwD3ASqDzdJUrImKJpP8A\nlkbEV7ot8/+A95HOULo0Iu7I2pcAH4iITZIOBb4D7As8BLwnInb0VktTU1M0Nzf3cxOtrOefT3sN\nP/5xeq1fn9oPPhje9Ka0J/HGN6bTWs2sZklaHhFNffbrKxCqiQNhmD3+eBpf+PGP4a67YMuWNA7R\n1JTC4c1vTs9+Hu9LRsxqiQPBBqe9PT29rXPvYdmydBO9iRPToafOgDjiiBQaZla1HAg2tLZs6Tq8\ndOed8NhjqX3GjK5weOMb03iEmVUVB4INr/XrUzDceSf89KdpPEJKF8O9+c1pDKJY9OElsyrgQLCR\n094Ozc1d4w/3358OL02Y0HV46U1vgiOP9OElsxw4ECw/W7emw0udAbEuu4h9+vSucDjzzHR9hJkN\nOweCVY8nn+wKh5/8pOvw0ty5cPzx6TnRRx2V3mfM8F6E2RBzIFh1eu21rsNLd92VbrpXeo+qCRNS\nOHQGROf0oYemq6zNrN8cCFY7nnkm3W9p9eqd3zeWXAA/fnw6xbV0b+Koo1LbbrvlV7tZDag0EPwn\nl+VvypT0LIdTTtm5fetWWLt256BYvhxuuaXr9hpjxsDrXrdrUBx5ZLpmwswq5kCw6jVpEsybl16l\nXnop3XOp+x7FkiU736hv5sxdg+Koo3wrDrMyHAhWe/bYA447Lr1Kvfpquv1G98NPP/vZzs+BOOCA\nXfcmpk2DqVNh8mQPalvd8hiCjX4dHfDrX/c8TrFly859d98dDjwwvaZO7XovnT7wwBQqHuS2GuEx\nBLNOY8bAIYek11lndbVHQGtrOvzU2ppev/1t1/Sjj6a9i+ee23WdUhr76C00Ot/32mvkttVsEBwI\nVr8kOOig9OrNjh2wefPOYdE53fm+enWabm/fdfkJE8qHRen7lCnpORVmOXEgmPVlt93SAPXMmb33\n6+hIexPdw6J0+uGH4Uc/SmdQdTdmTBrwnjIl3SRwypSeX6XzJk9Oy5kNAQeC2VAZM6brF/Uxx/Te\n98UXdw2OzZvh2WfTdRnPPJNuIPjAA2m63GNOx47tCohKQ2TSJA+cW48cCGZ52HPPdPX1oYf23TcC\ntm3bOSxKX6Xtjz4K992Xpl97ref1jRvXc1CUvvbfHxobu16++K8uOBDMqp2U/qqfNCkNjFciIh2W\n6ilAuofI6tVdbR0dPa9v0qSukOjrvbERGhqGbvttxDgQzEYjKY0vTJ6cruSuREdHOg336afT/aXa\n2nqeXr8+PUGvra38Xsjee/ceGKVtU6b4FN4q0ed/BUkzgG8BBwABLIyI67J5HwUuBl4DfhgRH5P0\nF8DlJas4Fjg+Ilq6rffTwAeBzjubXRERSwa3OWY2YGPGwD77pNfrX993/46OdOfazqDo/t45vW4d\n3Htv2gsptwey7767BsWkSekixIG8dt/dZ2wNQCWx3A5cFhErJO0FLJd0JykgzgaOi4gdkvYHiIhF\nwCIASX8AfK97GJS4NiKuHvRWmNnI6zwrat9909Xefek8C6u38GhrS/evuueedMhrx46B19fQMPBA\nKX1NmJDui9X5Xjo9YcKoOjzWZyBERCvQmk1vk7QGmEb66/5zEbEjm/d0D4ufD3xn6Mo1s5pVehbW\nUUdVtkxHB7z8crr1SOmrp7b+vF54Ie2x9DSv3GGwcsaP3zUkeguQSttyePxsvw7cSZoFzAWWAVcB\np0j6J+Bl4P9GxIPdFjmXtBdRzkckvRdoJu2F/K6H77wIuAhgZl/ngZvZ6DJmTDoja889R+4729u7\nwuHFF9Nr+/YUItu3l5/u3tbaumt7f8KmoWHnkPjqV+HUU4dvu+lHIEiaCNwGXBoRWyWNA/YF5gMn\nAt+VdGhkN0eSdBLwYkSsKrPK64HPksYlPgt8Hnhf904RsRBYCOleRpXWa2Y2IOPGpduNDPUtRyLg\nlVcGFi7bt6cTBIZZRYEgqYEUBosiYnHWvBFYnAXAA5I6gCl0DRKfB9xUbp0Rsblk/V8DftD/8s3M\naoSUrufYbbd0/UcV6vOad0kCbgTWRMQ1JbO+B5ye9TkCGA88k30eA7ybXsYPJE0t+fgOoNyehJmZ\njYBK9hBOBi4AVkrqPFvoCuDrwNclrQJeAS6MrntpnwpsiIj1pSuSdAPwlYhoBq6UNId0yOhJ4EOD\n3RgzMxs4Pw/BzGyUq/R5CL5NopmZAQ4EMzPLOBDMzAxwIJiZWcaBYGZmQI2dZSSpDfj1ABefQnad\nhAH+eZTyz2Jn/nnsbDT8PA6OiMa+OtVUIAyGpOZKTruqF/55dPHPYmf+eeysnn4ePmRkZmaAA8HM\nzDL1FAgL8y6gyvjn0cU/i52ehzp+AAAChElEQVT557Gzuvl51M0YgpmZ9a6e9hDMzKwXDgQzMwPq\nJBAkvUXSryQ9JunjedeTF0kzJN0tabWkRyRdkndN1UDSWEkPSar7hzRJ2lvSrZLWSlojqZB3TXmR\ntCD7d7JK0k2Sds+7puE26gNB0ljg34E/BmYD50uanW9VuWknPbt6NunRpxfX8c+i1CXAmryLqBLX\nAf8TEUcCx1GnPxdJ04C/AZoi4hhgLOkpkKPaqA8EYB7wWESsj4hXSE9xOzvnmnIREa0RsSKb3kb6\nxz4t36ryJWk68FbghrxryZukyaSHW90IEBGvRMTz+VaVq3HAHtnz4/cENuVcz7Crh0CYBmwo+byR\nOv8lCCBpFjAXWJZvJbn7AvAxoCPvQqrAIaRnon8jO4R2g6QJeReVh4h4Crga+A3QCmyJiB/nW9Xw\nq4dAsG4kTQRuAy6NiK1515MXSW8Dno6I5XnXUiXGAccD10fEXOAFoC7H3CTtQzqScAhwEDBB0nvy\nrWr41UMgPAXMKPk8PWurS5IaSGGwKCIW511Pzk4G3i7pSdKhxDMk/We+JeVqI7AxIjr3Gm8lBUQ9\nOhN4IiLaIuJVYDFQzLmmYVcPgfAgcLikQySNJw0MfT/nmnIhSaTjw2si4pq868lbRHwiIqZHxCzS\n/xd3RcSo/yuwnIj4LbBB0uuzpjcCq3MsKU+/AeZL2jP7d/NG6mCAfVzeBQy3iGiX9BHgR6QzBb4e\nEY/kXFZeTgYuAFZKasnaroiIJTnWZNXlo8Ci7I+n9cBf5VxPLiJimaRbgRWks/Meog5uYeFbV5iZ\nGVAfh4zMzKwCDgQzMwMcCGZmlnEgmJkZ4EAwM7OMA8HMzAAHgpmZZf4/FHzzC8Ch6UAAAAAASUVO\nRK5CYII=\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYQAAAEICAYAAABfz4NwAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAIABJREFUeJzt3Xt0XXWd9/H3p0laeqdt2gBNSq/2\nXouEWijIxZEFgtwFFFBmqcw4gj4z6ijzrHF8GHnQWY4+ggxaEUdGscXqKM6AuFQYOb3Qi7S0pRTC\nLWl6SXqjpddcvs8fZzechrRJ2yQ7yfm81srinL1/e5/vPtr92fv322dvRQRmZma90i7AzMy6BgeC\nmZkBDgQzM0s4EMzMDHAgmJlZwoFgZmaAA8G6OUkFkt6SNKo925rlIweCdapkh3zor1HSvpz3Nx3r\n+iKiISIGRERle7Y9XpI+KSkkXdtRn2HWUeQfpllaJL0OfDIifn+UNoURUd95VZ0YSc8AU4BMRFzZ\nyZ9dEBENnfmZ1rP4DMG6FElfkzRf0s8k7QZulnS2pCWSdkraJOleSUVJ+8LkiHx08v4nyfwnJO2W\ntFjSmGNtm8y/VNJLkt6UdJ+khZJuPUrt44A5wG3ApZKGN5t/jaSVknZJqpB0cTJ9mKR/T7Zth6Rf\nJNM/KenpnOVbqv9+Sb+VtAc4T9IVOZ9RKekfm9XwvuS7fFNSlaRbku93o6ReOe2ul7TiGP6nsx7A\ngWBd0dXAI8BgYD5QD3wOKCa7w70E+KujLP9R4B+BoUAl8M/H2lbSCOBR4IvJ574GzGql7o8BSyLi\nF8ArybpJ1ncO8BDweeBk4ELgjWT2I0BvsmcWI4DvtPI5zev/P8BAYDHwFnBT8hkfAj4n6fKkhjHA\n48C3gGHAGcDqiFgM7Aben7PeW4CHj6EO6wEcCNYVZSLiNxHRGBH7ImJZRDwbEfUR8SowFzj/KMsv\niIjlEVEH/BSYeRxtLwdWRsSvk3nfBrYeaSWSRDYQHkkmPZK8P+QTwA8i4g/JdlVFxHpJZWR3xJ+O\niB0RURcRfzpKvc39Z0QsTtZ5ICL+GBFrk/ergHm8/V3dDDwREY8m3+XWiFiZzHs4mY+k4qSmnx1D\nHdYDOBCsK6rKfSNpkqT/lrRZ0i7gLrJH7UeyOef1XmDAcbQ9LbeOyA62bTjKet4HlJI9o4FsILxH\n0rTkfRnZs4bmyoCtEfHmUdZ9NM2/q7MlPS2pVtKbwCd5+7s6Ug0A/wFcKakvcCPwVETUHGdN1k05\nEKwran6lw/eBNcD4iBgEfAVQB9ewiewOHmg6Axh5lPYfJ/vvabWkzcBCstvx8WR+FTCuheWqgGJJ\ng1qYtwfol/P+lBbaNP+u5gG/AMoiYjDwIG9/V0eqgeTKqxXAVWS7i/6jpXbWszkQrDsYCLwJ7JE0\nmaOPH7SX/yJ7hP8hSYVkxzCGt9RQUj/gOrLdQjNz/v4WuElSAfBD4JOSLpTUS1KppIkRUQX8Hrhf\n0smSiiS9L1n1KmCGpOnJkfs/taHugcD2iNgvaTbZo/1DfgJcIunaZIC6WNK7c+Y/DNwJTAJ+3YbP\nsh7GgWDdwefJHmnvJnu2MP/ozU9cRGwBbiA7ALuN7JH1c8CBFppfk9T2k4jYfOgP+AHQF/hARCwC\nPgXcSzbcniLbhQNJ3z3wErAFuCOp4QXg/wJPA+uBtowtfBq4J7lC6x/IDowf2qbXyA40fwnYDvwZ\nmJ6z7C+AsWTHVfa14bOsh/HvEMzaIDnK3whcFxHPpF1PR0i6xV4Dbo2Ip1Mux1LgMwSzI5B0SdKN\n04fspal1wNKUy+pI15M9A/qftAuxdBSmXYBZF3Yu2auFCoG1wNUR0VKXUbcnKQNMAG4KdxvkLXcZ\nmZkZ4C4jMzNLdKsuo+Li4hg9enTaZZiZdSsrVqzYGhEtXjadq1sFwujRo1m+fHnaZZiZdSuS3mi9\nlbuMzMws4UAwMzPAgWBmZgkHgpmZAQ4EMzNLOBDMzAxwIJiZWaJb/Q7BzKwna4xGtu/bzpa3tlCz\np4Yte7aw5a0tbNmzhb+f8/ecfNLJHfr5DgQzsw5U11BH7d7aph17zZ6aptfN39fuqaUhGt6xjgIV\ncNP0mxwIZmZdzd66ve84im963ez99n3bW1zHSYUnUdK/hJIBJYwaPIry08qb3o/oP+Kw10P7DqWX\nOr6H34FgZl1aRNAQDdQ11FHfWH/YX11jC9Pasd3BhoNs3bv1sK6bmj01vHXwrRZrHdxnMCUDSijp\nX8KU4VO4cPSFLe7gS/qXMKD3ALLPJOo6HAhmlqo9B/ewtnYtq7es5vktz7O6ZjVra9fy5v43qW+s\nb7ELpbMUqIBh/YZR0j+7I59dOrvpdfMd/Ij+I+hT2Ce1WtuDA8HMOkVDYwOv7HjlsB3/6prVvLL9\nFYLsc1n6F/Vn2ohpXPGuKxjWbxiFvQop6lVEYa/Cw/6KClqY1s7tCnsVdko3TVfiQDCzdlezp+Yd\nO/61NWvZV78PgF7qxYShE5h5ykw+NuNjTC+ZzvQR0xkzZEze7YS7EgeCmR23fXX7eKH2BVbXvL3z\nf37L89TsqWlqU9K/hOkl0/nr8r9mRskMpo+YzpThU+hb1DfFyq0lDgQza1VjNPLajtcOO+JfvWU1\nL29/mcZoBLJXzUwbMY3LJlzG9BHTszv/kumM6D8i5eqtrRwIZnaYbXu3vX3EvyW7819Ts4Y9dXsA\nEGLskLHMKJnBDVNvaNrxjxsyjoJeBSlXbyfCgWCWh3bu30nF9op3/L28/eXDunuG9R3G9JLpfOKM\nTzC9JHvUP3X4VPr37p9i9dZRHAhmPdS2vdsO3+HvSHb6215m275th7UdOXAk44eO50Pv+hATh01s\nOuo/dcCpXe5aees4DgSzbioiqN1b2+KRfsX2Cnbs39HUVoiywWWMHzqeaydfy/ih45v+xg0dR7+i\nfiluiXUVDgSzLiwi2PzW5haP9Cu2V7DrwK6mtr3Ui9MHn874oeO5cdqNTTv8CUMnMGbIGE4qPCnF\nLbHuwIFglrKIYNNbm3hp20stHukfGsyF7C9nxwwZw/ih4zmn9Jy3d/rDJjD65NH0Luid4pZYd+dA\nMOskDY0NvL7zddZtXce62nXZ/yav3zzwZlO7ol5FjB0ylvFDx3PB6AuajvLHDx3PqMGjKCooSnEr\nrCdzIJi1swP1B3h5+8vv2Omv37ae/fX7m9qV9C9h8vDJfHT6R5lcPJlJxZOYMGwCZYPKfPmmpaJN\ngSDpEuA7QAHwYER8vdn804GHgOHAduDmiNiQzPsGcFnS9J8jYn4y/d+B84FDh0a3RsTKE9oas060\n+8BuXtz64juO+F/Z/sphN2QbffJoJhdP5v1j3s/k4ZOZMnwKk4onMbTv0BSrN3unVgNBUgFwP/AB\nYAOwTNJjEfFCTrNvAg9HxI8lXQTcA9wi6TLgPcBMoA/wtKQnIuLQSNgXI2JBO26PWbur3VPbYjdP\n1a6qpjaFvQqZMHQC00ZM48NTPszk4uyOf2LxRF/BY91GW84QZgEVEfEqgKR5wJVAbiBMAf4uef0U\n8Kuc6X+KiHqgXtLzwCXAo+1Qu1m7iQiqdlW9vdPP2flv3bu1qV2/on5MKp7E+05/H5OLJzcd8Y8b\nMs59+9bttSUQRgJVOe83AO9t1mYVcA3ZbqWrgYGShiXT/0nSvwL9gAs5PEjulvQV4A/AlyPiQPMP\nl3QbcBvAqFGj2rJNZkfVGI28vO1llmxYwuINi1mxaQUvbn3xsIeeDO07lMnFk7lq4lVMHj656Yi/\nbHCZ78ZpPVZ7DSp/AfiupFuBPwHVQENE/E7SWcAioBZYDBzqXL0T2Az0BuYCXwLuar7iiJibzKe8\nvDzaqV7LIzv372Rp9dKmAHh2w7NNP9oa1GcQ5aeV85cz//KwI/7h/Yb7F7qWd9oSCNVAWc770mRa\nk4jYSPYMAUkDgGsjYmcy727g7mTeI8BLyfRNyeIHJP2IbKiYnZCGxgbWbV3H4qrFLNmwhCXVS1hX\nu44gEGLqiKlcO/laZpfOZnbpbCYPn+wjfrNEWwJhGTBB0hiyQXAj8NHcBpKKge0R0Uj2yP+hZHoB\ncHJEbJM0A5gB/C6Zd2pEbFL2MOwqYE07bZPlka17t/LshmdZvCEbAEurl7L74G4g2+0zu3Q2H5n2\nEWaXzuas085i8EmDU67YrOtqNRAiol7S7cCTZC87fSgi1kq6C1geEY8BFwD3SAqyXUafSRYvAp5J\nTr13kb0ctT6Z91NJwwEBK4G/br/Nsp6orqGO1TWrs0f+SfdPxfYKIPsL3hklM7h5xs3MLp3N2aVn\nM37oeHf7mB0DRXSfbvny8vJYvnx52mVYJ9n81ubDun6WVS9regRjSf8Szi47m9kjs10/5aeV+5bM\nZkcgaUVElLfWzr9Uti7hQP0BVm5e2XTkv2TDEt548w0geyuHM049g9vOvK2p7//0waf76N+snTkQ\nLBW7DuziyYonWbxhMYs3LObPm/7MwYaDAJQNKmN26Ww+997PMbt0Nmeceobv1GnWCRwI1mn2HNzD\nf7/838xbM4/HX36cAw0HOKnwJMpPK+ezsz7bdPQ/ctDItEs1y0sOBOtQ++v389uK3zJvzTx+89Jv\n2Fu3l1MGnMJtZ97G9VOvZ9bIWb5ls1kX4UCwdnew4SC/f/X3zFszj1+v/zW7DuyiuF8xt8y4hRun\n3ch5o87z3TzNuiAHgrWL+sZ6nn79aeavmc8v1v2CHft3cPJJJ3Pt5Gu5cdqNXDj6Qt/rx6yLcyDY\ncWuMRjKVGeavmc+CdQuo2VPDgN4DuHLildww9QYuHncxfQr7pF2mmbWRA8GOSUTwbPWzzF8zn0df\neJSNuzfSt7Avl7/rcm6YegMfnPBB+hb1TbtMMzsODgRrVUTw3ObnmkLg9Z2v07ugN5eOv5Qbpt7A\nhyZ+iAG9B6RdppmdIAeCHdHamrXMWzOP+Wvn8/L2lynsVcgHxn6Ar57/Va6adJXvC2TWwzgQ7DAv\nbXuJ+WvmM3/tfNbWrqWXenHh6Av54jlf5JrJ1zCs37C0SzSzDuJAMF7f+XpTCDy3+TkAzht1Ht+9\n9LtcN+U6SgaUpFyhmXUGB0Keqt5Vzc9f+Dnz1szj2epnAZg1chbfuvhbfHjqhykdVJpyhWbW2RwI\neaSuoY5frvslDyx/gD+98SeCYOYpM/n6+7/O9VOvZ8yQMWmXaGYpciDkgZo9NcxdMZcHlj/Axt0b\nGTtkLF+94KvcMPUGJhZPTLs8M+siHAg92LLqZdy39D7mr53PwYaDXDzuYr5/+fe5dPylvnWEmb2D\nA6GHOdhwkAUvLOC+pfexZMMSBvQewKfe8ylun3U7k4onpV2emXVhDoQeYtPuTXx/xff5/orvs/mt\nzUwYOoHvXPIdbp15K4P6DEq7PDPrBhwI3dih20jct/Q+fr7259Q11nHp+Ev57Hs/y8XjLqaXeqVd\nopl1Iw6EbuhA/QEeXfso9y69l+UblzOozyD+5qy/4TNnfYYJwyakXZ6ZdVMOhG6kelc131v+Peb+\neS41e2qYVDyJ7176XT727o8xsM/AtMszs27OgdDFRQSLqhZx79J7+eW6X9LQ2MDl77qcO2bdwV+M\n/Qs/aN7M2o0DoYvaX7+fn63+GfctvY/nNj/H4D6D+eysz/KZWZ9h7JCxaZdnZj2QA6GLqXqzigeW\nP8DcFXPZtm8bU4dP5XuXfY+bZ9xM/9790y7PzHqwNgWCpEuA7wAFwIMR8fVm808HHgKGA9uBmyNi\nQzLvG8BlSdN/joj5yfQxwDxgGLACuCUiDp7wFnVDEcEzlc9w77P38qsXf0UQXDHxCu6YdQcXjr7Q\n3UJm1ilaDQRJBcD9wAeADcAySY9FxAs5zb4JPBwRP5Z0EXAPcIuky4D3ADOBPsDTkp6IiF3AN4Bv\nR8Q8Sd8DPgE80J4b19XtrdvLI6sf4b6l9/H8lucZctIQPn/25/n0WZ9m9Mmj0y7PzPJMW84QZgEV\nEfEqgKR5wJVAbiBMAf4uef0U8Kuc6X+KiHqgXtLzwCWSfg5cBHw0afdj4KvkSSC8sfMN/m3Zv/Hg\ncw+yfd92ZpTM4Acf+gEfnf5R+hX1S7s8M8tTbQmEkUBVzvsNwHubtVkFXEO2W+lqYKCkYcn0f5L0\nr0A/4EKyQTIM2JkExaF1jmzpwyXdBtwGMGrUqDaU23Wt2LiCrz3zNR5b/xhCXD35au6YdQfnjTrP\n3UJmlrr2GlT+AvBdSbcCfwKqgYaI+J2ks4BFQC2wGGg4lhVHxFxgLkB5eXm0U72drjEa+eAjH6Sh\nsYEvzfkSny7/NGWDy9Iuy8ysSVsCoRrI3XOVJtOaRMRGsmcISBoAXBsRO5N5dwN3J/MeAV4CtgEn\nSypMzhLesc6e5sWtL1Kzp4YfXfkjbp15a9rlmJm9Q1tudrMMmCBpjKTewI3AY7kNJBVLTTfOuZPs\nFUdIKki6jpA0A5gB/C4iguxYw3XJMh8Hfn2iG9OVZSozAJw76tyUKzEza1mrgZAcwd8OPAmsAx6N\niLWS7pJ0RdLsAmC9pJeAEpIzAqAIeEbSC2S7fW7OGTf4EvB3kirIjin8sJ22qUvKVGYo6V/CuCHj\n0i7FzKxFbRpDiIjHgcebTftKzusFwIIWlttP9kqjltb5KtkrmPJCpjLDuaPO9eCxmXVZvj9yJ6je\nVc1rO19zd5GZdWkOhE6wsGoh4PEDM+vaHAidIFOZoX9Rf2aeMjPtUszMjsiB0AkylRlml86msJfv\nJWhmXZcDoYPtPrCbVVtWubvIzLo8B0IHW7JhCY3R6EAwsy7PgdDBMpUZClTAe0c2v/2TmVnX4kDo\nYJmqDO8+5d1+5rGZdXkOhA5U11DHkg1LOLfM3UVm1vU5EDrQys0r2Vu31+MHZtYtOBA60KEb2s0Z\nNSflSszMWudA6ECZqgxjh4zltIGnpV2KmVmrHAgdJCKabmhnZtYdOBA6SMX2Cmr21HhA2cy6DQdC\nB/EDccysu3EgdJBMZYZhfYcxqXhS2qWYmbWJA6GDLKxayJxRc/xAHDPrNhwIHaB2Ty3rt633+IGZ\ndSsOhA5w6IE4/v2BmXUnDoQOkKnM0KegD2eeembapZiZtZkDoQNkKjPMGjmLPoV90i7FzKzNHAjt\nbG/dXlZsWuHLTc2s23EgtLOl1Uupb6x3IJhZt+NAaGeZygxCnF16dtqlmJkdEwdCO8tUZpg2YhpD\n+g5JuxQzs2PSpkCQdImk9ZIqJH25hfmnS/qDpOclPS2pNGfev0haK2mdpHuV/FIrabde0srkb0T7\nbVY6GhobWFS1yN1FZtYttRoIkgqA+4FLgSnARyRNadbsm8DDETEDuAu4J1n2HGAOMAOYBpwFnJ+z\n3E0RMTP5qznRjUnb6prV7D6424FgZt1SW84QZgEVEfFqRBwE5gFXNmszBfhj8vqpnPkBnAT0BvoA\nRcCWEy26q/IN7cysO2tLIIwEqnLeb0im5VoFXJO8vhoYKGlYRCwmGxCbkr8nI2JdznI/SrqL/lFH\nuOmPpNskLZe0vLa2tg3lpmdh1ULKBpUxavCotEsxMztm7TWo/AXgfEnPke0SqgYaJI0HJgOlZEPk\nIknnJcvcFBHTgfOSv1taWnFEzI2I8ogoHz58eDuV2/4igmfeeMa3qzCzbqstgVANlOW8L02mNYmI\njRFxTUScAfzvZNpOsmcLSyLirYh4C3gCODuZX538dzfwCNmuqW6r8s1KqndX+4Z2ZtZttSUQlgET\nJI2R1Bu4EXgst4GkYkmH1nUn8FDyupLsmUOhpCKyZw/rkvfFybJFwOXAmhPfnPR4/MDMurtWAyEi\n6oHbgSeBdcCjEbFW0l2SrkiaXQCsl/QSUALcnUxfALwCrCY7zrAqIn5DdoD5SUnPAyvJnnH8oN22\nKgWZygyD+gxi2ohpaZdiZnZcFBFp19Bm5eXlsXz58rTLaNH0B6ZTOqiUJ256Iu1SzMwOI2lFRJS3\n1s6/VG4HO/btYE3NGo8fmFm35kBoB4uqFgEePzCz7s2B0A4ylRmKehVx1siz0i7FzOy4ORDaQaYq\nw5mnnUm/on5pl2JmdtwcCCfoQP0BllUv8/iBmXV7DoQTtGLTCg40HPD4gZl1ew6EE3ToB2nnlJ2T\nciVmZifGgXCCMpUZJg6byPD+Xfc+S2ZmbeFAOAGN0cjCqoXuLjKzHsGBcAJe3Poi2/dtdyCYWY/g\nQDgBvqGdmfUkDoQTkKnMUNK/hHFDxqVdipnZCXMgnIBMZYZzR53LER72ZmbWrTgQjlP1rmpe2/ma\nu4vMrMdwIBynhVULAY8fmFnP4UA4TpnKDP2L+jPzlJlpl2Jm1i4cCMcpU5lhdulsCnsVpl2KmVm7\ncCAch90HdrNqyyp3F5lZj+JAOA5LNiyhMRqZUzYn7VLMzNqNA+E4ZCoz9FIvZpfOTrsUM7N240A4\nDpmqDDNPmcnAPgPTLsXMrN04EI5RXUMdSzYs8QNxzKzHcSAco5WbV7K3bq8HlM2sx3EgHKNDN7Sb\nM8oDymbWs7QpECRdImm9pApJX25h/umS/iDpeUlPSyrNmfcvktZKWifpXiU3/pF0pqTVyTqbpnd1\nmaoMY4eM5bSBp6VdiplZu2o1ECQVAPcDlwJTgI9ImtKs2TeBhyNiBnAXcE+y7DnAHGAGMA04Czg/\nWeYB4FPAhOTvkhPdmI4WEU03tDMz62nacoYwC6iIiFcj4iAwD7iyWZspwB+T10/lzA/gJKA30Aco\nArZIOhUYFBFLIiKAh4GrTmhLOkHF9gpq9tR4QNnMeqS2BMJIoCrn/YZkWq5VwDXJ66uBgZKGRcRi\nsgGxKfl7MiLWJctvaGWdAEi6TdJySctra2vbUG7H8QNxzKwna69B5S8A50t6jmyXUDXQIGk8MBko\nJbvDv0jSecey4oiYGxHlEVE+fHi6D7LPVGYY1ncYk4onpVqHmVlHaMud2aqBspz3pcm0JhGxkeQM\nQdIA4NqI2CnpU8CSiHgrmfcEcDbwH8l6jrjOrmhh1ULOKTvHD8Qxsx6pLWcIy4AJksZI6g3cCDyW\n20BSsaRD67oTeCh5XUn2zKFQUhHZs4d1EbEJ2CVpdnJ10ceAX7fD9nSY2j21rN+23t1FZtZjtRoI\nEVEP3A48CawDHo2ItZLuknRF0uwCYL2kl4AS4O5k+gLgFWA12XGGVRHxm2Te3wAPAhVJmyfaZYs6\niB+IY2Y9XZtu5h8RjwOPN5v2lZzXC8ju/Jsv1wD81RHWuZzspajdQqYyQ5+CPpx56plpl2Jm1iH8\nS+U2ylRmmDVyFn0K+6RdiplZh3AgtMHeur2s2LTC3UVm1qM5ENpgafVS6hvrHQhm1qM5ENogU5lB\niLNLz067FDOzDuNAaINMZYZpI6YxpO+QtEsxM+swDoRWNDQ2sKhqkbuLzKzHcyC0YnXNanYf3O1A\nMLMez4HQiqYH4pT5gThm1rM5EFqxsGohpYNKGTV4VNqlmJl1KAfCUUQEz7zxDOeOOtc3tDOzHs+B\ncBSVb1ZSvbvaD8Qxs7zgQDgKPxDHzPKJA+EoMpUZBvUZxLQR3eYefGZmx82BcBSZqgznlJ1DQa+C\ntEsxM+twDoQj2LFvB2tq1nj8wMzyhgPhCBZVLQI8fmBm+cOBcASZygxFvYo4a+RZaZdiZtYpHAhH\nkKnKcOZpZ9KvqF/apZiZdQoHQgv21+9nafVS367CzPKKA6EFKzau4GDDQY8fmFlecSC0YGHVQsA3\ntDOz/OJAaEGmMsPEYRMZ3n942qWYmXUaB0IzjdHIwqqF7i4ys7zjQGjmxa0vsn3fdgeCmeWdNgWC\npEskrZdUIenLLcw/XdIfJD0v6WlJpcn0CyWtzPnbL+mqZN6/S3otZ97M9t204+Mb2plZvipsrYGk\nAuB+4APABmCZpMci4oWcZt8EHo6IH0u6CLgHuCUingJmJusZClQAv8tZ7osRsaB9NqV9ZCozlPQv\nYdyQcWmXYmbWqdpyhjALqIiIVyPiIDAPuLJZmynAH5PXT7UwH+A64ImI2Hu8xXaGTGXGD8Qxs7zU\nlkAYCVTlvN+QTMu1CrgmeX01MFDSsGZtbgR+1mza3Uk307cl9WnpwyXdJmm5pOW1tbVtKPf4Ve+q\n5rWdr7m7yMzyUnsNKn8BOF/Sc8D5QDXQcGimpFOB6cCTOcvcCUwCzgKGAl9qacURMTciyiOifPjw\njr0M9NDvDxwIZpaPWh1DILtzL8t5X5pMaxIRG0nOECQNAK6NiJ05Ta4H/jMi6nKW2ZS8PCDpR2RD\nJVWZygz9ivrx7pJ3p12KmVmna8sZwjJggqQxknqT7fp5LLeBpGJJh9Z1J/BQs3V8hGbdRclZA8p2\n1l8FrDn28ttXpjLD7NLZFBUUpV2KmVmnazUQIqIeuJ1sd8864NGIWCvpLklXJM0uANZLegkoAe4+\ntLyk0WTPMP6n2ap/Kmk1sBooBr52QltygnYf2M2qLav8QBwzy1tt6TIiIh4HHm827Ss5rxcALV4+\nGhGv885BaCLiomMptKMt2bCExmj0+IGZ5S3/UjmRqczQS72YXTo77VLMzFLhQEhkqjLMPGUmA/sM\nTLsUM7NUOBCAuoY6lmxY4vEDM8trDgRg5eaV7K3b6/EDM8trDgTevqHdnFF+II6Z5S8HAtnxg7FD\nxnLawNPSLsXMLDV5HwgR0XRDOzOzfJb3gVCxvYKaPTV+frKZ5b28DwQ/EMfMLMuBUJlhaN+hTCqe\nlHYpZmapyvtAWFi1kDllc+ilvP8qzCzP5fVesHZPLeu3rXd3kZkZeR4IfiCOmdnb8joQMpUZ+hT0\n4cxTz0y7FDOz1OV9IMwaOYs+hS0+ztnMLK/kbSDsrdvLik0r3F1kZpbI20BYWr2U+sZ6B4KZWSJv\nAyFTmUGIs0vPTrsUM7MuIa8DYeqIqQzpOyTtUszMuoS8DISGxgYWVS3yA3HMzHLkZSCsrlnN7oO7\nPX5gZpYjLwPBN7QzM3unvAyEhVULKR1UyqjBo9Iuxcysy8i7QIgInnnjGc4ddS6S0i7HzKzLaFMg\nSLpE0npJFZK+3ML80yX9QdLzkp6WVJpMv1DSypy//ZKuSuaNkfRsss75knq376a1rPLNSqp3V3tA\n2cysmVYDQVIBcD9wKTAF+IgCMHRuAAAFhUlEQVSkKc2afRN4OCJmAHcB9wBExFMRMTMiZgIXAXuB\n3yXLfAP4dkSMB3YAn2iH7WmVxw/MzFrWljOEWUBFRLwaEQeBecCVzdpMAf6YvH6qhfkA1wFPRMRe\nZftqLgIWJPN+DFx1rMUfj0xlhkF9BjFtxLTO+Dgzs26jLYEwEqjKeb8hmZZrFXBN8vpqYKCkYc3a\n3Aj8LHk9DNgZEfVHWWeHyFRlOKfsHAp6FXTGx5mZdRvtNaj8BeB8Sc8B5wPVQMOhmZJOBaYDTx7r\niiXdJmm5pOW1tbUnVOSOfTtYU7PG4wdmZi1oSyBUA2U570uTaU0iYmNEXBMRZwD/O5m2M6fJ9cB/\nRkRd8n4bcLKkwiOtM2fdcyOiPCLKhw8f3oZyj2xR1SIA5oyac0LrMTPridoSCMuACclVQb3Jdv08\nlttAUrHU9FDiO4GHmq3jI7zdXUREBNmxhuuSSR8Hfn3s5R+bTGWGwl6FzBo5q6M/ysys22k1EJJ+\n/tvJdvesAx6NiLWS7pJ0RdLsAmC9pJeAEuDuQ8tLGk32DON/mq36S8DfSaogO6bwwxPakjbIVGU4\n89Qz6VfUr6M/ysys2ylsvQlExOPA482mfSXn9QLevmKo+bKv08KAcUS8SvYKpk6xv34/S6uXcses\nOzrrI83MupW8+aXyio0rONhw0L8/MDM7grwJhIVVCwGYU+YBZTOzluRNIGQqM0wcNpHh/U/sSiUz\ns54qLwKhMRpZWLXQ3UVmZkeRF4Hw4tYX2b5vuwPBzOwo8iIQfEM7M7PW5U0glPQvYdyQcWmXYmbW\nZbXpdwjd3eTiyYwcONIPxDEzO4q8CIQ7z7sz7RLMzLq8vOgyMjOz1jkQzMwMcCCYmVnCgWBmZoAD\nwczMEg4EMzMDHAhmZpZwIJiZGQDKPt64e5BUC7xxnIsXA1vbsZzuzt/H2/xdHM7fx+F6wvdxekS0\neu//bhUIJ0LS8ogoT7uOrsLfx9v8XRzO38fh8un7cJeRmZkBDgQzM0vkUyDMTbuALsbfx9v8XRzO\n38fh8ub7yJsxBDMzO7p8OkMwM7OjcCCYmRmQJ4Eg6RJJ6yVVSPpy2vWkRVKZpKckvSBpraTPpV1T\nVyCpQNJzkv4r7VrSJulkSQskvShpnaSz064pLZL+Nvl3skbSzySdlHZNHa3HB4KkAuB+4FJgCvAR\nSVPSrSo19cDnI2IKMBv4TB5/F7k+B6xLu4gu4jvAbyNiEvBu8vR7kTQS+CxQHhHTgALgxnSr6ng9\nPhCAWUBFRLwaEQeBecCVKdeUiojYFBF/Tl7vJvuPfWS6VaVLUilwGfBg2rWkTdJg4H3ADwEi4mBE\n7Ey3qlQVAn0lFQL9gI0p19Ph8iEQRgJVOe83kOc7QQBJo4EzgGfTrSR1/w/4e6Ax7UK6gDFALfCj\npAvtQUn90y4qDRFRDXwTqAQ2AW9GxO/Srarj5UMgWDOSBgC/AP5XROxKu560SLocqImIFWnX0kUU\nAu8BHoiIM4A9QF6OuUkaQrYnYQxwGtBf0s3pVtXx8iEQqoGynPelybS8JKmIbBj8NCJ+mXY9KZsD\nXCHpdbJdiRdJ+km6JaVqA7AhIg6dNS4gGxD56C+A1yKiNiLqgF8C56RcU4fLh0BYBkyQNEZSb7ID\nQ4+lXFMqJIls//C6iPhW2vWkLSLujIjSiBhN9v8Xf4yIHn8UeCQRsRmokjQxmfR+4IUUS0pTJTBb\nUr/k3837yYMB9sK0C+hoEVEv6XbgSbJXCjwUEWtTListc4BbgNWSVibT/iEiHk+xJuta7gB+mhw8\nvQr8Zcr1pCIinpW0APgz2avzniMPbmHhW1eYmRmQH11GZmbWBg4EMzMDHAhmZpZwIJiZGeBAMDOz\nhAPBzMwAB4KZmSX+PwGmJla3Tr9AAAAAAElFTkSuQmCC\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "X4WYxO2_0gbq",
        "colab_type": "text"
      },
      "source": [
        "##Testing the model"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4om_ZyHT6ALB",
        "colab_type": "code",
        "outputId": "79a9b5d5-29ac-452c-fced-0cd7fab0f9b2",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "acc = 0.0\n",
        "batches = 0\n",
        "for features, labels in test_data:\n",
        "    features = features.as_in_context(ctx)\n",
        "    labels = labels.as_in_context(ctx).one_hot(10)\n",
        "    predictions = model(features, training = False)\n",
        "    acc += accuracy(predictions, labels)\n",
        "    batches += 1\n",
        "print(f'Test Accuracy: {acc.asscalar()/batches}')"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Test Accuracy: 0.9939291401273885\n"
          ],
          "name": "stdout"
        }
      ]
    }
  ]
}