digit_recogniser.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Populating the interactive namespace from numpy and matplotlib\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "%pylab inline\n",
    "import os\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from scipy.misc import imread\n",
    "from sklearn.metrics import accuracy_score\n",
    "\n",
    "import tensorflow as tf\n",
    "import keras\n",
    "\n",
    "from keras.models import Sequential\n",
    "from keras.layers import Dense, Activation, Dropout, Convolution2D, Flatten, MaxPooling2D, Reshape, InputLayer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# To stop potential randomness\n",
    "seed = 128\n",
    "rng = np.random.RandomState(seed)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "root_dir = os.path.abspath('.')\n",
    "data_dir = os.path.join(root_dir, 'data')\n",
    "sub_dir = os.path.join(root_dir, 'sub')\n",
    "# check for existence\n",
    "os.path.exists(root_dir)\n",
    "os.path.exists(data_dir)\n",
    "os.path.exists(sub_dir)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>filename</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.png</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1.png</td>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2.png</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3.png</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4.png</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  filename  label\n",
       "0    0.png      4\n",
       "1    1.png      9\n",
       "2    2.png      1\n",
       "3    3.png      7\n",
       "4    4.png      3"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train = pd.read_csv(os.path.join(data_dir, 'Train', 'train.csv'))\n",
    "test = pd.read_csv(os.path.join(data_dir, 'Test.csv'))\n",
    "\n",
    "sample_submission = pd.read_csv(os.path.join(data_dir, 'Sample_Submission.csv'))\n",
    "\n",
    "train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAP4AAAD8CAYAAABXXhlaAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAH5BJREFUeJztnW134kgOhcW7MZD0zP7/X7izkwQw2Lzuhz235pYoE0gT\nIFv3OaeOTTpNyExfSyWppM7xeDQhRF50H/0BhBD3R8IXIkMkfCEyRMIXIkMkfCEyRMIXIkMkfCEy\npP/dP6DT6ahQQIgHcTweO6mvy+ILkSESvhAZIuELkSESvhAZIuELkSESvhAZIuELkSESvhAZIuEL\nkSESvhAZIuELkSESvhAZIuELkSESvhAZIuELkSESvhAZIuELkSESvhAZIuELkSESvhAZIuELkSES\nvhAZIuELkSESvhAZIuELkSESvhAZIuELkSESvhAZIuELkSESvhAZIuELkSESvhAZIuELkSESvhAZ\n0n/0B8idTqdz8vrc4u9J3V/C8Xi04/EY3ae+lrpPXf29eH4k/AeQEi5Wt9u1Xq8XrqnV7XZb12fi\nPx6Ptt/v7XA4tF4/W3gYpO7Fz0DCfwBe6Hzt9Xo2GAxaV7/ft16v13rtds/v3g6Hg223W9vtdmH5\n1/v93vb7fes9HgB8j/eW+H8GEv6dYUufstiDwcBGo5GNRiMbDofhnr+Wehjgvtfrnf35EPpms2ld\n+B48EPieHwDdbtd2u52ZydX/aUj4D4BFz249C388Hp+soiisKIrwQBgOhyf3lwi/aRqr69rquj65\nb5rm7EOBHwB4iMHVFz8HCf8BsJvf7XaDm97r9Ww4HFpRFFaWpZVlaZPJJFr8ABiNRuEer/v98/9L\nt9utrddrW61Wtl6vo/vVahU9DPAgqOv6JLbgRX84HKzT6cjy/xAk/AfgLT4WXHZY/Ol0arPZzGaz\nWbiH+MuyPPEIyrL8VPibzcaqqorWcrkM9xA/HgrYPqRiBxD9fr+/OKMgngMJ/86kIvgQfb/fjyz+\nZDKx2Wxmr6+vYU2n02D98T3sGQwGg7M/v2kaWywW0ZpMJrZYLILnsFqtbDgcBk+ErTsD0cMDkPh/\nDhL+N3Au385RewTqOGA3mUwiob+8vESvZ7NZtA3APV5/JvzhcBh9Hs4o4PPhIcRBQx9ErOs66fLj\nnq/+XjweCf/GtEXrfdQee3IsvC7LMrj37OLDzUdwr9/vhz31fr+37XZrdV2HKHsbiNDv93szsxBj\nGI1G4e/ia3goFUVh4/HYJpOJrVarKCawWq2s3++HrcB2uz2b6xfPgYT/DcCFh9XkPHtbxJ736RzM\n869Z+BAbcvOdTudT4XNK7ng8BpEPh0M7HA5J0SPI1zSNrVarEBeAF8Cfo9vtnhQC4bWCf8+DhH9j\nYPG9m4xVFMVJpN4LHRaW03i4R8oOgoPFN/ufO/3ZPhveAawvvBD8XRY95/txv1qtrCiKpOgR5PMF\nP/hs4nmQ8G+MF77Pt8OVZxee3fqyLE+Kd/gewTbeX3PJ7Wfg+/C9vV4viB7pxOFwGFXycXUfB/4g\n+uPxGL4H9xzfgGdxyecT90HCvzEQEIJ4cJexJpNJCNhx4A734/H4JJCGe7byvOC2X2JV/YEcWHx8\nXnbP+YGC5ff0HGPYbDbB3fdBPzwIZPmfAwn/G/AWH646rP3Ly4v9+vXLfv36ZX/88Ud0HY/H0WEc\nfzWz4EbvdrsgVHztswCaj+ZzRJ/FmgrOHY9Hq6oqcu9h6ZumsfV6fVLRhweDUn3PhYR/Y7yr7yvx\nptOpvb6+2q9fv+zPP/+0f/3rX/bnn3+G+/F4HN4ndfXWFZbYR+vb4MAjeyf+kE/quLCZWVmWZnZq\n6dfrtVVVZdvtNvw5Ph/eV+J/HiT8L3DuvDyi9ojGT6fTUHQD0XsXn13/oihaj8Iies9Rdr8uEX4q\nL4/XcOPZy/AeB6fzUOGHxa4+ix9ZB/EcSPhXkrKSvIqiOBE7L7j5XH6LSD3E4gNqfDBms9m01tM3\nTfNpOg97+XPFOecW3gO5f+T3m6YJ4kbMwHspEv7zIOFfCae8fAXeYDAINfZtC3t8VOD5vDz2zees\neupkHdY1Fj8lei4o4sM/ZhZO/vlCpMlkErYZeDD64qKmaSJPQDwWCf9KuOyWT8hBMJyuY7Hza3gC\n5yrxYNl9pdx6vT4ROw7VXFK55xt9+IcYnxMoyzJkDPD3zP4JXg6HQxuPx1FdAEfvEYBsmib8fvge\nM+X2H4mEfyXY73JeHnt63tdzrp6vcO+5OIddfVh8DpgtFgtbLpe2XC4j8aeuX3X12WOZzWYnZb2o\n7MNrdvW96M0ssvTs+rPFl/V/HBL+lXiL74N4Xuz+fjwenxTmsKsPKwnhL5dLm8/nNp/P7ePjw5bL\nZQik8fFZrM+En3L1eU0mE9tsNlFJL35XjtBji8NZCP4dOMWHBxtnDCT+xyLhXwn+gUO02NMjOu+r\n8fwqiiIqyOHFLjJEU1WVzedze3t7s7e3N1ssFifRdH6NdFobn/X0m06nUVUf0pFw5yFgWHyz2PXv\ndDqR6KuqCr8rW3wJ/rFI+FeSsvgcrf/sAVAURdRg03e1QVTfW/z393f7z3/+Yx8fH9HJOH9S7nf3\n+C8vL2ZmkaUvyzK4/lzlh+/jPgLdbjcSPTyalMXXQ+BxSPhXwg002PJzdZ7vkccLgkFV2263i86v\nw8L79f7+bu/v7zafz1tbZ13j6rel88ws+l04poDIPOoK4P2wF7Tf76M4BgKf2M70+/2TkuPU+X3x\nvUj4XyRVvONr1DmyjQo3rntPta+uqsr+/vvvsN7e3uz9/d0Wi4VVVRVEiBNzsMTXtLbmclxfTovP\niWwBPInlcmllWSZLgrl3oE8D+t6AXGacKlIS90HC/wKpYRjsspvFJass/E6n09rBtmkaq6rKPj4+\n7P393T4+PsI9hL9arcL3ohU2AnGXwKJPCR/vDXcdLntZlrZYLKJ9fuqaagKKNR6Po4cdL/5s4vuR\n8L/IubJds7j7LAvfzE4i8byQvvNrPp8Hi8/n5PlwzlfE77+G901Z/KIozMxOAoJc1LTb7c6Kn6sR\n1Z77cUj4v4m39hy59odocLCGO9siP8+v0eUGFp474qIs1w+4uEY4KaGlzgLww4gr+HCeILXHPxwO\nraIvisI2m81Jc04+uiuLfx8k/C9wztqz2+xd/V6vZ7vdLlj1j4+PkJ/HlffxXJyDe+zpU2Otrj2P\nz6OvIMaUxecAHUfi+/1+9AAYjUZ2PB6T+/yU8Pm/ker474uE/xu0nW03O93j48/RvgqR+re3txDE\ne3t7s+VyeXaSzbnBlte4+mxhUTHIOXhY/KIorKqqcB6BT+sNh8NQ5IPXZhaVMHuL3zTNyX8fbtEt\n7oOE/0Xaovoc3OOoPv5h13UdLD5y8//+97/tr7/+sr/++suqqooabfh7bmHNabBrXGR8PwsN971e\n78Tiox0XNw5Fey62/vAIzrn6KeHr5N79kfC/gE/TcQoM7jz+bDAYhHr1wWBg6/U6itYjeo/7qqqS\n02i/I+WV6nvvsxD4vVB6y+22U916j8djdGqR+w1isSfE3YYk/vsh4V8JN5aE9YYLbGZW13U0jMKX\n5TZNE1z7+Xweau8RoU/NofeFLvf4HVmc3F6LB2f6BxOsP3s/XCjEjTwR85DoH4OEfyV8eq6u61DC\nij8riuJkbj3fN00TLD4H81L790fktX2OPzVWm8dmpwKLXN3IMwEhfHhC3O1Hwr8vEv6V8Hn59Xod\nLD32qqPR6GQYJq/tdhsds4XFR5ru0dYev2PK4rP4fSqR4wYcAGTRIzOA9+I+fz4jIr4XCf9K+BCN\nn2TDZ885t8+vd7vdyWhqVOPxnvlRosfvmKpB8Ba/LZXozzN4i7/ZbMJ/J/5vI+6HhH8lbPHN/nH9\nMV4K7qtP8eF6OByS7bTY1ffR+ntb+1S5cWqP7/f5ZvFwUJ4vwMLn2Ics/mOQ8K8EQjf7x73ngJ7v\nU+8X/r5vqMnpulS0/Z7i5zgDN/lEjCIV4GNXP2XxOcLfJnxxPyT8K4Grz9Nh2KKncuN87yvn/JV/\nziNos/j9fj9y9duqBtv2+D6957dEEv59kfC/ANz9/1f8CT4/RssLPrUd8Wk9dv1TDUjk6t8XRVSE\nyBAJX4gMkfCFyBAJX4gMkfCFyBAJX4gMkfCFyBAJX5zgC3B8EQ7uVYDzc1EBj4jgc/Q8JmwymZxM\n+eVBGRiKKX4GsvgiAgdseCwWhI+hoJgWxCO+dbruZ6H/WyKCZwMOBoNg8TEjkEd9j0ajcOBGFv9n\nIeGLANfXX2Lx/YhvCf/nIOELM4vHgvmBoLD4s9nsxOJL+D8TCV+cHB++Zo8vV/9nIuFnjhcrXH3u\nk1eWZST8Nosvfg5K52UKu/Z89Q0yMQgD4ofoEdHnbrmy+D8HCT9DzrUG811xscfHPr8sy2QOX8L/\nWUj4GeLbhfGVW2TxKCwWPh4IvoWW+DlI+JmRGvLJbbBSFp+FPx6Po955svg/Ewk/U7gNNvf+9z3w\nIXq2+DweTBb/ZyLhZ8i5Rph8GKfN4sPK+waasvg/Bwk/Q9qE76fetO3xfZdcndD7eUj4mXGp6FPC\nx+JW2GqL/TOR8DPlswcALw7mDYfDs+/re/KjBz9PC/LDOB41GThnFJHJkM9E/7t7d57Cs9lsrGma\naEBoXdfWNE00fJOnCInvR8LPkEssve+uc6n4efwWpuw2TWN1XYcJwRA+Twj2MwPF9yLhZ4af93fO\n4n9lxBWEz0NBIXxY/Tbhi/sh4WeIn2jbZvG/2lMvZfEh+vV6bev1Ounqy+LfDwk/UzgNd6nFvwQE\n6Vj4dV0HV/+cxZfw74ei+pnR5uq3Wfxr03V+xDbv8dnaQ/gYuS3R3xcJP0NSrr63+G2jrC/hnKu/\nWq2C4GXxH4eEnyFs7X2lHnL13FmH9/gsztQ9LD0sOQSOBwD29pzPRy5f3A8JPzPY0vu6fFTmoUbf\nix9gH59aCNjx4gdBW/EO3lfcBwk/M1j4sPZ8ECfVXSdl9SFYFu/hcAhufJvoU8KXm39/JPwMYeGn\njt9yhx0+dtvpdCLrzuLFvRc8ix6pOy9+leveHwk/M85ZfCwWfaqDbqoWHyvl6vsHQMraS/j3RcLP\nEG6hndrjQ/zcSDPl6nPajtN3nKZLrTbRS/z3Q8LPDB/c4+O3n+3xzeLTd6mTd59Zewjfxwkk+vsi\n4WfGJcE9bqbZFtxLif+c6Fn8bRkBcT8k/EzxjTS4225qXYIXr2/kiQcOu/jI30v490XCzwyOyPt9\neV3XVhTFyT6dXfFUuS//OTwIHzBE2y4E9jiqv9/vzczk8t8RCT9DEFhj9xxVdb6GngUKeLuABwke\nCNy2y28hIHwf7DOzyPqL70fCz4zPLD53x/F19BB4t9u14/EYxG9mkfC9xWfh88OGawMOh0O4F9+P\nhJ8hnIprs/iw+qkDNNyT37v+3uL7nvy73c6apomyBHgIqWnn/ZDwM4MtPlxtf4jG5+Hb9vgc/Dsc\nDkH4qT0+xJ+y9Pgs4n5I+BnSZvEHg0HS1ec9Pgvfu/7H4/HTuXvb7TYpevXlvy8Sfma0WfzBYHDi\n6qf2+GYWhI7342vbFB62+L4hp0Zw3R8JP0Paovq9Xi/qlIPGGbzM7CS/73P+XBWIVN50OrWmaUJQ\nkPf4/BAAvpRXQb/bIuFnho/qo1EGxLharayqqqheny1yWZbRoE0/dHOz2YRA3WAwsKIobDqdhoo9\nPhGIjj/8IGiaJqrl54V8v/h9JPzM8HtruNqw3v1+P6rV5z/b7/c2mUxOpuXy+K3tdhtScxA+RN/t\ndoMngPfmzAAeDHzohwN/KvC5HRJ+hrCr793ufr9vy+UyOo7LvfJXq1U0TovbdY1Go/DenU4nPEQg\nenwPi5739sgMYPvR6/WC+49AooR/GyT8zPDBPX/UFn34WJQQ/WazsclkEgJ2nK4riiIIni3+8Xi0\nXq9nw+HQyrK00WgUNfPkn40tAlKLm80m/Hy5+bdFws8QFplZ/DAwsxP3nttkr1arkJrjohyk+1C7\nD4sPSz8ej+1wOFhRFNH5fn/Kz/983pYo3Xc7JPzMYKH516i99+49ov7r9dqm06lNp1Or6zqk/DjH\nPxqNwp59MBicFPuUZdlawMPxAXw25fm/Bwk/Q/goLIQHK8zNMbzox+NxGHqJ6D2LHpYa3XsR8ONV\n1/XJz+Y23PzZuMBIef7bIuFnBufG/bl4WFRvkfkBwPX7qfZZu90upAKLorB+vx/c/uFwaGYWFfTw\nwaCmaWy/35/8XKQbZfFvh4SfKVx7z5FyX0abOkHHX+MYQF3XNplMrCzLqFIP6bxOpxO2BlzogyIf\nWHy29k3TJBt+it9Dws8cnx7zwTQOsvnhF170qPZDHADeAUSPwB/ceeT1R6NRCBL6wqK6rkOQUMK/\nHRJ+pqTy4RCoPy3HlX5skb3oq6oKpbne0iOlZ/ZPjIEtPnffxdHduq6jvn8S/u2Q8DMEYsQ9f41T\nfZzm44q6lOixp1+v10lLD8vO6T7+OtcA8HRdCF/Bvdsi4WdMyur7Hngo6tntdqGSDm74cDgM4kQF\n33q9PhE99+0fDAYnFh/f2+/37Xg8Rg8TWfzvQcLPlFRwz3sCcPs5oNfv98Mx3lTN/nq9Tlr68Xhs\nm80mquzDn/MD4nA4RAeFeKiHhH87JPzMSVl9n5/nK5/KS626rpOin0wm4egvB/dg0Tl4WFVV6O8v\ni/89SPgi4rNxVvACfL98Tv2VZWmTycRWq1U4249cfb/fj94LDwmw2+1OBnf6wR6XflbRjoQvrsJP\nv/Etsf1kHe7nV9f1yfl9vyB2dPDxp/82m03yM6hZx3VI+OJL+A45eACkhM9Reg7o8d4dZ/pTYvdf\n8w06zHRW/1okfHE1/mw8l/siD++FD/FD9GYW3HYcBR4Oh7bdbiPB+7P/OCPAiz+XxH8ZEr74EudS\ngXy4xg/rGAwGZvaP2HHPwcDPLL6vKpTgr0fCF1+ChQYPAKk6PzKbLf5wOAyTejm6D4t/OBxa9/ej\n0ciapjkRPXf90QPgMiR8cTXs6vM9Iv6p4J6vxIPIzU6Ff87ao+wXnwOVhUr1XYeEL76Et/hmFpX8\ntu3xIWLU8ZvFwj8ej637e3yNfy4qCyX861ABtLgZPs3Gi+Ee/DxuGwsRfl6+UIjrBrjDj7gMCV/c\nHBTloJTXT9WB5UY5LoR8Trx+iIf4PSR8cVNggVn4fEiHK/JY+J8JPvV18XUkfHFz4Ibz1BweoHnO\n4oPPrH/qXlyOhC9uyiUWn61+SvhtFl6u/u2Q8MVN4YM3Xvhe9Jfs8S+1/OI6JHxxc7zF9+OyL3H1\nzU6FzR6BRP97KI8vbso5Vz9l8f00Xn6f1GtZ/9sg4Yur8fttfs3Vdm2LBY+/x6W3KAJKveav+xOC\n4nIkfHEV2L9z0Q3fv7y82Gw2s+l0GvXYh6VPtdLyAzP9AR9+je693PVX5/GvR8IXVwFX3o/JwtUL\nP5W75z09T805Ho+RwL3geUH8vt+/uAwJX1wF99PjOnpcWfiYpgvxQ/h+q8Dz+s5Ze7b4bPVl7a9H\nwhdXwak6Dtghap9y9WHx4eqDVC3/Jdaem3DI1f8aEr64Cm/xMfcO6/X11WazmU0mk+Dq+xQeW3gO\n6rGrf26v3yZ8cTkSvriKVOvssizDvLw2i8+uPrvoHLXnzj3n9vo+yi9rfz0SvrgKdvVh8SeTSRB9\n2x6fLT4H9cz+iepfYu232+1J6k+u/vVI+CKCA2+pBdceYp/NZjabzez19dVeXl6Cqw+Ln6rLR4NM\nHr7JIvdiRzAPAT2/TZDgr0fCz5hUNVxqUg43wkAADyJP3b+8vIT9PQJ6KNZha53qvb9er0PHns1m\nEwneiz3VBUhchoSfKanKOzOLAnfI1WMNh0MryzISuF/Y60+n05DGY+GbnVp6NOREJ16I/lzO/lyH\nH/E5En6GnCu55em2SNfxPQt/NptFooeLj4DfOYsP4WPcNgu/ruvI3ecIvkR/GyT8zOHjsD5iD/Hi\nOh6Pw76ehc9Xztvz/h7Veuyuc0DPW/ymaU6KdTgg6Ov09QC4Dgk/Q9oCd+iaMxgMQkEOgnS4IpiX\nWi8vLzYej6OtgT+Q4y1+m/B9pR5bfDMNzPxdJPxMSYneT7RB9J5d+bYFT6AoiiggyIHBtj0+B/b8\nHr9tfw8k/K8h4WeMFz2Ez62ykJ9/fX21X79+RXv51HU4HEbbB7+VaIvq82DNz8Qvfh8J/wnxYkn1\nm0udRefBFm1XL3S/Ujl6L3z8Odx/Ls/1tfj8WbGnZ7d+vV5bVVW2XC7DWq1WIa3HAT6J/nZI+E8C\nxMl59DZ3OVW1xtaQLa0fPOGXn1OPevtfv36FlarI42q81Pn61Gc7HA62Xq9tuVzaYrGw+XxuHx8f\nJ2s+n4cHQF3XQfhy62+HhP9gfKto35YaATLcdzqd6EiqX8fjMdkgg0Xur3w/Ho/t9fX1ZLHwfZts\n3xufa+/9Z12tVifCf39/D2uxWNhisQjC5+i+hH87JPwH0lY5xw0q/YLwOdWF++12a2YWeQo8goo9\nB/8AYOFzbp4r8pCu8220uGGmd+v5s223W6uqKoibLf77+7u9vb3ZYrGw1WoVFnL6svi3RcJ/EG37\ncFh8PvmGU25lWVq32w37Xl/XvtlszMyijjipTjle8Px6PB5He3t/Px6PIy8E7+876vhDN1jYx8/n\n88jiv7292dvbmy2Xy6iQR67+9yDhP5CU+GHxh8NhiKpzoK3b7YYCF7/6/b4dj8dom+C3C6kBlLyK\nooii9D5yXxRF61BL30oLVXmI0td1HQJ5sPjz+Ty4+RC+P5Uni397JPwHkBoB5YtoYPGRR8deu9vt\nRtZwvV5HDSzNLNnZFik6FNN415+Fz0U7vCaTiY1Go2Rs4JzFR46eA3spV//vv/+21WoVxQVwrz3+\nbZHwH4QX/2fC//Xrl/3xxx/W6/VsvV6HPXBq8GQqNoCYAcpnvfBxPxqNohQdFoSf6pmX6qHHwue0\nnbf43tVfrVYntfg6b397JPwHkhoCwcMoeBAFutx4sfryWLO08LkLzrljt6jY8wsPj37/f/9kfO0A\nUokI5HFBDiL5HNjDXr+qqvAQQ/GO+H4k/AeTEj9bfy9Q1NHzgRp+MJilXX2OwrcF9lC1l0rV8aSb\nVM88XJumCUKG0Nm9//j4sMViYVVVhb2/gnf3R8J/AlLjonyRDYJoyOVDpEVRRCmzTqdzUXDP79Fx\n5VRiWx98pOtSE26wl+dqPATxIHxYelTnSfj3R8J/ElL75TaLD0ufGjfV6XSSaTw/p65tpYqH2irz\nOPCGa13XJxaf9/N4AEj4j0XCfyJ8Wi9l8bkWPgXE21bEgyCgL+XlB01b3t8srrtHAI+LiM5ZfAgf\nQT4J/3FI+E+GP5TjLT5Ov6Xq7M9V5PF921l8Fr9P9fl0nc/VcwQ/ZfERvedDOOv1OjqEI+HfDwn/\nCfnM4qdm1vE2IGXJ2wSPn5c6m+8XDgh5i48o/maziSw+rD5bfFTmIeoPi69uufdFwn9Czu3x/aw6\nX6SDdJsXduradt92nBekLD6sPVt8H82H8NlD4Ok4Ev79kPCfjM9aUyGqjwi8WTzdhr+Wsuz+Z527\n5/52uCKAx00z+B7BO87VI4qPnL0/XOTbaonvR8J/IKl/6JgYC8s5n8+DC29mVtd1NKEG97C63Ngy\nlbbjfXqqMi7VyZa/xu48inP49WKxCLX3CORxUw3fJ19VeY9Bwn8Qx+Mx6kEH9vt9cJurqopEv9/v\nra7rqJTWt6hCdZ1fZhaOzabSgKnXqbXZbKJjs37x4Ruk7bihhj+fr6GXj0HCfwBe9LDCZrHFr6oq\nEj2s7XQ6PelNB0Gh/RXy8HhfiB4/yzfwSAkydd80TXDb/cK+HgvuPqft+OfI4j8OCf+BeNGbxcJP\niZ5r2tnSc8CtKIqoR12n0wlHdvEz+CANz6ZLFeXwta7rk727f+29AP95U56ExH9fJPwH4UXPlhjC\nN4tFD1HxwAmOiHNgEKLH/h7fwz+Hg2y8fIcf/r66rqPgna/Fh1vvg3++hZaGXj4WCf+BeNGb/WPx\nzf4RPZfcTiaTEMjjCTP8nl70KO/lh4PPHPh5dW2L9/FcjosrB/J44Wv8kEp1Chb3QcJ/AnxwDxY5\n1RG3qqrIveeHB9fTs+jxgOCf1zatNjWqml8j08AluHy/Xq9PvAR+nbLwEv39kfCfDLbIqeq6/X5/\n0jTTd8Bp68B7OByi1l08p47XuZ5+LHxeHMjjYGCqC7B4PBL+E3JJ1B/7flTr4Xu2220IAlZVZZPJ\nxD4+PkL6D806vXW/1NVH+6zU0VpYdEXtnx8J/4lh0ftGlrDYLHqk25BW85Nux+Nx1KUXy79uc9MR\nC+BDONwCW6m6n4OE/+R48UP4sPipbACPqPbXbrfbKm7Os7c1vOQeen6eva/IS1UBiudAwn9S2tx9\nFj6KcngmXVVVyQ48aKqBOEGbuD8r4uGzA37L4Ovula57XiT8JyZV1svC96Lnrru48sIBHl+e69tn\nnZvN53P//j6VrpO1fz4k/CeHBcOuPou+rXFG6or3bGuY2XZQh8XfFrX3dfd8lfCfi853/w/pdDr6\nP35DUs00OId/rsEGH8v9TJypwppU0c05yy6xP57j8Xjaxtlk8X8csp7iFnQ//xYhxP8bEr4QGSLh\nC5EhEr4QGSLhC5EhEr4QGSLhC5EhEr4QGSLhC5EhEr4QGSLhC5EhEr4QGSLhC5EhEr4QGSLhC5Eh\nEr4QGSLhC5EhEr4QGSLhC5Eh395sUwjxfMjiC5EhEr4QGSLhC5EhEr4QGSLhC5EhEr4QGSLhC5Eh\nEr4QGSLhC5EhEr4QGSLhC5EhEr4QGSLhC5EhEr4QGSLhC5EhEr4QGSLhC5EhEr4QGSLhC5EhEr4Q\nGfJfo4vCdQY/LsgAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x121f34da0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "img_name = rng.choice(train.filename)\n",
    "filepath = os.path.join(data_dir, 'Train', 'Images', 'train', img_name)\n",
    "\n",
    "img = imread(filepath, flatten=True)\n",
    "\n",
    "pylab.imshow(img, cmap='gray')\n",
    "pylab.axis('off')\n",
    "pylab.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# Store images as numpy arrays\n",
    "temp = []\n",
    "for img_name in train.filename:\n",
    "    image_path = os.path.join(data_dir, 'Train', 'Images', 'train', img_name)\n",
    "    img = imread(image_path, flatten=True)\n",
    "    img = img.astype('float32')\n",
    "    temp.append(img)\n",
    "    \n",
    "train_x = np.stack(temp)\n",
    "\n",
    "train_x /= 255.0\n",
    "train_x = train_x.reshape(-1, 784).astype('float32')\n",
    "\n",
    "temp = []\n",
    "for img_name in test.filename:\n",
    "    image_path = os.path.join(data_dir, 'Train', 'Images', 'test', img_name)\n",
    "    img = imread(image_path, flatten=True)\n",
    "    img = img.astype('float32')\n",
    "    temp.append(img)\n",
    "    \n",
    "test_x = np.stack(temp)\n",
    "\n",
    "test_x /= 255.0\n",
    "test_x = test_x.reshape(-1, 784).astype('float32')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# One hot encoding to convert values to categories\n",
    "train_y = keras.utils.np_utils.to_categorical(train.label.values)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Split training data into training and validation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# 70% split into training, 30% split into validation\n",
    "split_size = int(train_x.shape[0]*0.7)\n",
    "\n",
    "train_x, val_x = train_x[:split_size], train_x[split_size:]\n",
    "train_y, val_y = train_y[:split_size], train_y[split_size:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "34300    3\n",
       "34301    1\n",
       "34302    6\n",
       "34303    8\n",
       "34304    3\n",
       "34305    8\n",
       "34306    8\n",
       "34307    9\n",
       "34308    3\n",
       "34309    8\n",
       "34310    4\n",
       "34311    6\n",
       "34312    6\n",
       "34313    3\n",
       "34314    6\n",
       "34315    7\n",
       "34316    5\n",
       "34317    3\n",
       "34318    0\n",
       "34319    3\n",
       "34320    9\n",
       "34321    3\n",
       "34322    8\n",
       "34323    8\n",
       "34324    7\n",
       "34325    4\n",
       "34326    3\n",
       "34327    8\n",
       "34328    6\n",
       "34329    5\n",
       "        ..\n",
       "48970    7\n",
       "48971    5\n",
       "48972    0\n",
       "48973    1\n",
       "48974    4\n",
       "48975    1\n",
       "48976    7\n",
       "48977    5\n",
       "48978    6\n",
       "48979    5\n",
       "48980    6\n",
       "48981    3\n",
       "48982    5\n",
       "48983    5\n",
       "48984    9\n",
       "48985    2\n",
       "48986    9\n",
       "48987    0\n",
       "48988    0\n",
       "48989    7\n",
       "48990    0\n",
       "48991    1\n",
       "48992    1\n",
       "48993    6\n",
       "48994    9\n",
       "48995    2\n",
       "48996    4\n",
       "48997    9\n",
       "48998    3\n",
       "48999    0\n",
       "Name: label, dtype: int64"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train.label.ix[split_size:]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Set up training model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# 3 layers: Input, Hidden, Output\n",
    "# Input : 28 * 28 \n",
    "# Output : 10 * 1 classes\n",
    "# 50 hidden neurons\n",
    "# Adam as our optimization algorithms, which is an efficient variant of Gradient Descent algorithm\n",
    "\n",
    "# define vars\n",
    "input_num_units = 784\n",
    "hidden_num_units = 50\n",
    "output_num_units = 10\n",
    "\n",
    "epochs = 5\n",
    "batch_size = 128\n",
    "\n",
    "# create model\n",
    "model = Sequential([\n",
    "  Dense(units=hidden_num_units, input_dim=input_num_units, activation='relu'),\n",
    "  Dense(units=output_num_units, input_dim=hidden_num_units, activation='softmax'),\n",
    "])\n",
    "\n",
    "# compile the model with necessary attributes\n",
    "model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 34300 samples, validate on 14700 samples\n",
      "Epoch 1/5\n",
      "34300/34300 [==============================] - 1s - loss: 0.5661 - acc: 0.8473 - val_loss: 0.3135 - val_acc: 0.9115\n",
      "Epoch 2/5\n",
      "34300/34300 [==============================] - 1s - loss: 0.2592 - acc: 0.9263 - val_loss: 0.2512 - val_acc: 0.9301\n",
      "Epoch 3/5\n",
      "34300/34300 [==============================] - 1s - loss: 0.2108 - acc: 0.9403 - val_loss: 0.2188 - val_acc: 0.9390\n",
      "Epoch 4/5\n",
      "34300/34300 [==============================] - 1s - loss: 0.1807 - acc: 0.9481 - val_loss: 0.2014 - val_acc: 0.9428\n",
      "Epoch 5/5\n",
      "34300/34300 [==============================] - 1s - loss: 0.1596 - acc: 0.9545 - val_loss: 0.1862 - val_acc: 0.9470\n"
     ]
    }
   ],
   "source": [
    "trained_model = model.fit(train_x, train_y, epochs=epochs, batch_size=batch_size, validation_data=(val_x, val_y))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Evaluate model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "20672/21000 [============================>.] - ETA: 0sPrediction is:  9\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAP4AAAD8CAYAAABXXhlaAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAH/dJREFUeJztnety2soShRtzvznZOe//gqdOKjEGBALB+bFrTZZaM5Jw\nAGO0vqopyY6Nsfde6p6evvTO57MJIbrFy2e/ASHE/ZHwheggEr4QHUTCF6KDSPhCdBAJX4gOIuEL\n0UEGt/4BvV5PiQJCfBLn87kX+7wsvhAdRMIXooNI+EJ0EAlfiA4i4QvRQSR8ITqIhC9EB5Hwhegg\nEr4QHUTCF6KDSPhCdBAJX4gOIuEL0UEkfCE6iIQvRAeR8IXoIBK+EB1Ewheig0j4QnQQCV+IDiLh\nC9FBJHwhOoiEL0QHkfCF6CASvhAdRMIXooNI+EJ0EAlfiA4i4QvRQSR8ITqIhC9EB5HwheggEr4Q\nHUTCF6KDSPhCdBAJX4gOIuEL0UEkfCE6iIQvRAeR8IXoIBK+EB1Ewheigww++w2I69Lr9Sor9nl8\nLnZt4nw+J1fsa1OfS13F7ZHwvzBeqBB1v99PrpeXl7B6vV7l4ybxn89nK4oiuk6nk51Op9LX+ns8\nIPC1uOer/3ni+kj4X5SUte73+zYcDpNrMBjYy8tL8sHQJPzT6WSHw6GyjsejHQ4HK4rCzOJWnAXP\nDwvc4/X563u9nsR/AyT8L4h33/kewh+Px6U1Go3CdTAYJNfLS33YpygK2+/3pZXnebgviqLk9vv7\n0+lkx+PRjsejFUVhx+Ox8rDB90D0Ev/1kfC/KKk9e7/ft8FgYKPRyCaTiU2n09Iaj8c2HA5tNBpV\nvIHRaNQo/OPxaFmWhbXb7UofH4/Hyp6fPy6KouQl4L3joRDzOCT+6yPhf3H8A+Dl5SVY/Ol0avP5\nvLQmk0nFA+D7fr9f+/PyPLftdmubzcY2m024xzYiz/Pa4N/xeAzbijzPzeyP6OEtANxL9NdHwv+C\n1EXs2dWH8JfLpS2XS1ssFjafz208HocHAF8nk0mj8Pf7va3Xa3t/fw9Xjh3s9/uK2BG0Y+HDs2DR\nv7y8lPb4QKK/PhL+FyZm7SF8uPqz2cwWi4W9vr7at2/fbLFY2HQ6rWwD8PFgUP+/xG63s9VqFb5n\nOBwGIZ/PZxsOhyWh+4j94XCouPfY6+NkwSPhXx8J/0FJnbfz8RsEx9fpdBqEDrFjff/+PSl8FnId\nu92udBTID51er2e73a5i5Vn8h8MhbC/gKfBR4+FwCA8DnADw8vkCeih8DAn/AWFx+4XgHS8IaDAY\n2HQ6tW/fvgXh+8X7fLj3vNdvcvWHw6FNJpNwdIeAHN5Xk6t/OBxKwUC/8jwPUX9eCAbya8WCh6Id\nEv6DwS47W0MsuPGwmLjHms1mYU+PfT1/jMg+B/Rwhdtex/l8tvF4HEToTxIOh0NtcC/P86Tot9tt\nOB70C8FAPjVgTwLvTbRDwn8wIPyYRYfQvbXGPdz3xWIRAnl8XSwWwZ1PHek1Heedz2ebTCbhiM0f\nH7Iw8fW8YPG3221J8P6IkBfnF/R6veD2497MokFBkUbCfzC8xffWHeKezWalK+55zefz0nU2m9l4\nPK7dKjQJn4/W8IAajUY2nU5tv9/b8Xg0s2riDq6HwyEIna+pe7wn5ABwQBDo2O9yJPwHxEfnOQPP\nn83DomPxg8A/GKbTaUjS+WjKbuzYkFN3Y4JkMcLVh8C94DebjU0mE1uv16X3g4w/PAAYdvdFOyT8\nByNm8dmd53N5v/AQiG0B8Dm483xC4O/rwAMC780X6jRV6cHip9ZkMglBRhb94XCwPM+Tom9bWSj+\nRcJ/MPx5PKwqzuQhfD6q43u487Hg3Xg8Lgnqb8pyfXS9bVT9cDiEjL+U8AeDQVT0u90ubCXwb3yk\nKFe/PRL+J5E6p2ehw62H4GOi9+KfzWalmIC/csYcrt4lj5X7+vtU9L/p4XE4HEpbDR9rQPYeV/Hx\nkR5+diwRSLRHwv8E6s7px+NxJSrPK+beYx/vq+84MYYF6bPpYrXwsYYedR5CW2/BezTH49HG43H4\n2SjiQfUecgXw78gV2O12Je+lKAq5+xcg4d+ZpnP6yWQSFTgvH9ybzWZhb8wRehY9u8IcGWfLCqvJ\nmXixLD1+LX9/6e8/Go0qUXku2+WHE4KKWZYF0XNGoITfHgn/zjSd0+Mcnl16Tr/lyD1fOXjnHyjw\nJgAi435h/5zKGIyJn3+vtuD3Z0uOvwuE70WPr8EDA6LH+25zIiH+IOHfmaZzeljx5XJZyrHHFVH7\nWAIPR+296P1+2O+fsc7nc+l7cY/Pswfgf6+2vz8LmP8mEDTn5fO/4/2w6BH4k/AvQ8L/BOrO6Vn4\nr6+v9v37d/vnn3/sn3/+CcL3abq8kPDiV8rVR9AM63w+lzwGfD0LCwE4zqb7iKvPHw8Gg9CYg3v3\ncREQlhe93++LZiT8O9PmnJ7LaCH8Hz9+2I8fP2w2m5X28X674IXu3fKYxYeA0BijKIrwut7VBiz+\ntqLn7+W/BUfx+WHj+wxA3BA9gnyoMZDw2yPh35mmc3rUz/Me//v37/bjxw/7z3/+Y7PZrFKO6+/x\nc2JXs/Ien4W/3+/NzCqC5/ftxX9pZRy/DoucPQuIPlaRaGYl0W+3Wwn/A0j4d4bTXTn3ns/p/REe\np+XOZrNktL1N5h27+Sx4FMSgmcbxeLThcGhFUYTmGqfTqRI49BH5tu5+3ddx116O8BdFUSoj9icX\noj0S/g2JJcLEmmFyGu63b9/CkR2aY/D/5F7sqZ+VgiPh2B9zvvzpdEom/3CLbt5e8O8mvgYS/o1I\nudpohsmtsTgj7/X1NQif02/b7N/bwOfeEH6WZbZer22z2VhRFJWAoa/7x8eIvPNRnPgaSPg3gMXu\n731Qj/f0OL6Du88Wn4trPip6s7TF32w29v7+HhW+r8Abj8cV0bPbLx4fCf9GpFJduYbdu/oQPvb0\nnJHH+1l+fX/fBpS4Yn+fZVkQPlJoubgHffLQeSeWUKNc+a+FhH9j/AMg1gWXhY++eJx/z0d1ddH6\nNvAZOEfG1+u1rVarIPzxeGx5ngdrD9GjLJYfYtxZV3wNJPwbELP0HHn3Fp9d/dfX1+DiY/Ee/29S\nZc2aXX18Ps/z6Hw81L77QhsJ/2sh4d8Qb+194k6qxt5n88Vc/Y+SCu5B+IfDwSaTSUX4sPbn87lk\n6dGMQ67+10LCvxExa+/Fz+KB9edgnu+F13YvX2d5eWilFz/aW/saeO52y3t83uvjd0PabV11X5Nn\nwH8n/7fyfxdfjOQTlUQcCf9OxM7026zU96eINbjkK0+p9a48gn1+O7Df78MDCg8HnpaL5J8sy0IX\n31RKcZP4OT2X23vh6/M8Lw0AjfUgwO8bazYi/kXCvyExsfvrJeJvS6wdFu59RR6LHgE/zurzAp5O\npyXhcxvsLMtKguTlE3zwe3lB+viBr9KD8H1FIj9sYkM3Yj+ry0j4N+ZS8XuX/hri5447sQk1LH64\n87E6gH6/b9vttiJ27pLrW3zjFKDf79toNIr+fWI1AbF6fbTcgvB5AhD3IcBrauBGGgn/DtSJPyX6\nmKt/ibvPoucVG0vlXf1UDkKv17PRaFQrfNQWYAaemZU67cTcfP6ct/j4HFz/w+EQjjpZ/F74PGij\nTVyha0j4d6LuiO9W7r5vWulLcWOuvp9vj9fC8sLnDrmbzcayLCuJHpYe03fwu9cJERYfX9vv90PR\nUFuL74N8bQOLXUHCvzJtRHrpQ+BSvMX3oo+5+WzxU/34UJ3H465ms1kQ/Ww2C99vVhY9B+jwO8c+\nhsXH5/r9figcwjFkbI/PcQhsL3xdv/iDhH8D6v4nuyS677/mEmIW3/fXiwX4ENyLTazF3p+F70d3\noZkHRI9gIB8D8u/lxW/2p+dfzOtgV5+j+mzx+ev5ASj+IOHfAN6n+v53fkhl6jzaV+K1xUfufRLO\ner229XpdGlSJCbWcqOPbW/tIOX8uNr8+tdr+/fjK8NFd6u+GzkA8YUcWv4yEf2U4Kh1bsIzsrvp9\nKmfp+YBfEwjg4aiNj9z2+729vb3Zz58/7devX/b29mbr9TqIn1Ny8R6Gw2FJuMPhsBK554WOQag5\nwLw+no4D/IOgzYNBe/TrIOHfAN9aixtZcAEO9qjeXa0bhtEE3GHOwcd1s9nYarWyX79+2a9fv2y1\nWgXrD+FjTxzzPF5eXkInYD+FF/fL5TIIH6XFmNDrA27i85Dwr4y3+Jx3PxqNKhbfR6a5PbYXXhsQ\nANvv9yH/frVahSvW29ubvb29hUg8W/zUCO3hcFiqL+B2YDzgg6f8wOJzT7yY6PUguC8S/pXx59CI\nanMXXaScssXnPX+qNXYb4OrD4q9WK/v9+7f9/v27ZOV5weLjGA4WnzsAp8Z0x6b6+JHd7OpL9I+B\nhH9l2OKjgy4Ew0MwY3t8WFZO5rlU+HD12eL//v3b/ve//9nPnz9ttVqVeuxxgA8FNtjj432zl8Ii\nj13RPMQP/Uh1wZXoPwcJ/wbELD6LvmmP/zdn+d7iv7+/269fv+znz5/23//+11arVanABotTdSF8\nHMfVdf/1wz19daH/vRiJ/vOQ8K+Mt/hwl2E52eLzhFt29f0Z/iXwHp9dfRa+b1/NVzxo+ByeB3zw\n/t1P8V0sFjYejytHmDwMw0yCfwQk/BvAufdN9eSx8+i/wffN5xZbcOv9dFz8zJeXl8pDKjbA01t5\nvh+NRrU1CBL9YyDhPxn+oeOPFXkWfew6mUxC008e2AnRL5fLsF1ht55z5P825VjcHgn/CWHx+04/\nqI1PWWQIn0dz85UHffhc+UuTjcTnIeE/GTGL78XP2wq/ptNpZbgHXzHkwwclOdkoVnwkHgsJ/wnh\nBhoQPjfG5DiDv4fwOYjHH0+n0+horVj7778JUorbIuE/IbD4HFyEwE+nUylb0A/PwGSf1EL6bSw4\neY2+/+I+SPhPRt2JAlpfcXKN7+Hv0299Zt5oNKpUHsZSi2XtHxsJ/wmps/i9Xq+Sjecr7HwRDice\n4bgOP6cugi/BPy4S/g3w3W9i9fF+7jt/bZ21bBITR/N98tByubTj8RgVuV9IMOJ7tM6+JXWluug1\n4PsIxrrq+qsoI+FfGe5mywMruBEHBMV18miEAavsrWjb6LhPt0XjS7SjKooiCBli5o991WBs/35r\n6nr+cTcg/9CMPQj4+8UfJPwbwJlz6E0P4aCRRZZlJdFzB5xYkY5ZO9cZ1h4juH3jy6IogrBT19hR\n3T3d9lhbcFzrPKZUByBRRcK/Mr5/fZ7noXkkOtj4qjjf+gr95vB9l1hbX1nn+9qfTqcQ0W9asRFe\ntybVFty3B+f+gXVuv8QfR8K/Mj5XnkWPbrHz+TxYfO/qHw6HUsQcIM+9SXzYTsDVNyt3u8V7iNUO\n+DoCbmD5t8M6LyHVIRgWPyV+7ynE9vziXyT8K8MW34u+KAobDAbB4sdEj/ZXEP2lxS1s8c0s5Orj\nqA6vDUFzgZAvFuLP3XOPb2aVDsHcBDS1z28K9Ik/SPg3gC2+D/ZxX3oI37v6PHiC59G3AcI3K1t6\nCAVbiI+se7r6/LBkkdediKSsvqgi4V8ZFrp3+2FB2eLvdrtScO9wOJhZORHH95ivg4OC6JAbE0Hs\ntKDN/T3gPT678170KYtf1+Zb/IuEfwN4eAP+J355ebGiKEq18Rg9hd537+/v1uv1SgG2oihCxl0q\nuu/P/VMeQpstQ92/p9xn/z2xtN1LHhyxHAg/2JNnANRF9UUcCf/KcADOu61m/56j53luWZbZer22\nt7e3UjPK3W5X6VnHI6hSrbmaxMUivKR/fexat8ystDXw900/31t7zofApJ8m8esB0IyEfyNYKDzZ\nhYddbDabUs/58/lsu92ulFWH5BvszXmPzsIyS5/zxzyEtuKPLT9Bh+97vV7pVKLf75eClW1+vt/f\n83gvHw9h8deJXuIvI+HfAA7O4R6i8BafRX88Hi3LslAO60XPHXj9cV+qZVfdwyAlhpTQvTWOTeJF\ncJHn2/vOwW1+fpPwL3H3JfoqEv4NYWsDVxxJPVmWhcSY2NgrZNzhe5GGG+vNlxJRm7z+JvHH5uP5\nAZy88HBDvsClMYc6V99b/FiEX6Jvh4R/A3zgi/f8bPFxXIeGmDjmY0vvM/HYkgIIi3/WJZl+MfF5\ni990po6PEZyMvbfUwyn2831gj/f3sSGf3uLzfwc9AKpI+Dci9j+zmQWLz+49RI8BljFLj8IeHNFx\nQC91VNeWlOWPpcz6ozV/hcjxuphvzycdTT8fDxuuaPTBPW/x+T36v79EX0XCvyHe8vAen917Tpfd\n7XZRS59lWfgfHq8Fa8pW7qPn7THxxfb1sRJjFiLXF3A/gCbx8QPskj0+vCO2+KIZCf+OsCXje06P\nNbNQKecXcu9jPe+4kq6Opvp+eCE+QYY/VzeQA5V8EL3PpW/z3lj0LHif8YjpPxzVF+2Q8D8Jb90Q\n+OP9/nq9Ds0vIOjtdlsSu79vI/y6xaWv7E7zfV1knyP4eJiNRqOLjg+96HH0yUM+vfgl+suQ8D8B\nL3j+PI+/Qs96uM9FUdhms6ktpW1zph/LA8DCVsQny+AeZb54vx48fLB9YWvcdHaPK9c24CHIGY5I\necbpB36G9vLtkfDvTCxizv/GwT6eKw8xNAnfW/yYa+8TbLhpJqcVc9kw7pGMlCriKYqiNDC0jfB9\nbMFbfBb+ZrNJWnwJvz0S/icRS47p9XohkJVlWRAxH/lh3nysRXaTqw93PlaKi2tRFKVGIby2221w\n52Pz/xDE46k9Tfvv1Dk+W3xM/oW7v9lsKhZfwr8MCf8T8Nl83GQDFh8uuz/nx7y6VG/8pj0+qvZi\nooVrDqu62WxK9xC+//n8Mc7xx+NxJZ02dcTpPxez+H6Pz8JXYO9yJPxPIJbgA5ccIjf7I3pU863X\n61I/vNh9k/B5iKbvwgPhQ2CxdT6fKz35+YqJu4gN8P67zdm6t/hcyVjn6muPfxkS/ifBQvD78P1+\nH5Jk9vt9Sagscr9Q5VcHmnOkTgUOh4O9v7/barWKXs/nc6nf/nw+L+2z+/2+TSaTSuDNxzLq/i6x\nqD4H9zj2IFf/Y0j4DwD/D4t8d2/5eOS1t/R8bSN8FroX/vF4rIid7/n98nEdov1NCURN4uQjvVhZ\nbipVV6K/DAn/AYllsAFf5caZdUgGqgPCz/O84uazqw+X2gfR8Po4CeDGnjyMA7EInqQrHgcJ/wHx\nR32pbDsv+jaZeyjvTUXmOarPe2kE0fCz8TrwQNDME8JnD+Se7blFOyT8B8WL35eZ+nz2wWBQypVP\nwRl1sevpdCr1A/TC9xbfj+mqs/gS/uMg4T8YXvAowsG/8dex8DkZp45YAg9/fDqdoll7/tgM+QBN\nFl+u/mMi4T8gfM5vVh6mwYLnCri2LbBjabq8OKDor3xkxnt8L3wexYXthCz+YyHhPygx8SN1l0ty\nY/dN+K/3zTr9hBr/MRfhNAX3PmMSj2hGwn9AeB+fCuzx5z/SdSf1Gvi5dcssHdybz+fRoZuy+I+F\nhP/gtElzvSecdMTi92m8PHcvNW03lsnX9NBRM83rIOGLi/Hbi1ig0PfS562EFznf+6YfsWuqo65o\nj4QvPgSLn4OM/gGQijukrHhdB9/YqCyJ/2NI+OIi2HqnRB8btBmLIXDrbj6taLsk+I8j4YsP8RHx\nMyz+tj37Y5Zf4v8YOmMRF+OPAGMufuy40Oyynv1N+3t11f04Er64mJi1rwvweXc/JXpZ/Psh4YsP\n4cWfsvqpAF/M6v+N6CX+y5DwxcU0tej27n1K9LFBHW3dff8AEJch4Yur4sX+EfE3rdg+X+K/DAlf\nXI02Kbl1Lr4fwOmv3uLrSO/j6DhPXIVL8vD92X1bi49/l7X/eyR88Sm03ePHlj8RkNW/HAlf/DVN\nFYR8jfUTrLP4/nPcXFMR/Y8j4YtPwYs/tsdPPQBilXziMiR8cRHXqKmvO7+vs/ZY/rXE5Uj4Ikqq\n0UddBV6bM3xQl7obG8OtQN51kfBFhbq2Xr7Rhm+4kfo+X50Xg114cVskfFEi1YwTouaWWtyX36fp\n1lXlxZDY74uELyrEuupg1bXXaqrMSyHR3x8JX5RgNx2DNnix6P0knlTLLbxujFRPwdRVXAcJX1Rg\na+9HaXuLz1Y/VpFXZ/El5s9DufqiRMziQ+DcSdeLvq4ct81e3zfeFLdFwhcVIFwerunbZ/NAzCaL\nb1btuSc+F7n6ooS39hD8ZDIJQzMwG48n4qYCe3hNEEvZRWLO4XAII7t8+a0eFtdFwhclIPzUMMzX\n11dbLBY2n89LwzHrknk8PJ8vz3Pb7XaWZVkYzZ1lWRjaycM6Jf7rIeGLEsjOg3vPo7EWi0UQPg/H\n9IMxmwJ6sPQQ/n6/D6LfbrdhNHee58H6S/jXRXt8UQLCh5uPYZgQ/XK5DMKfTqel8/zYEV4sa89b\n/P1+b7vdLgg/yzLb7XYViy+uh4QvSrDFZ1d/sVjYcrksufps8f1gTC947rDLe3uIni2+XP3bI1df\nVEBEn139lMXnAF9sck5qj3+Jq88ttsR1kPBFiTqL//r6WtrjQ/gpi8+vCWKuPiz+ZrOJuvrY44vr\nIeGLEnV7fLj6iPKngnv8Wp5UcM/v8eXq3xYJv8PErLNP2JlMJiXx4xjPu/n9fj/aQttT5+ZvNhvb\nbDbB4u/3+2Dxi6K47R+jY0j4HSRVL9/r9UJiDhJ2IHwsfM5X56Vq7v3VR/Nxfr/ZbGy9Xtt6vS5Z\nfXb1ZfGvh4TfMVjwsbp7WPq6BwCn7nLWHvC98Pjj1DEeC58tfp7ncvVvgITfQVL19txoo87qx+rx\n/f7ep+bivs7NX6/XttlsQrBPFv92SPgdhHvnQbi4ssVPufpcphvb36d66bHFh7XnM3xYfFh67fFv\nh4TfMXzZLY7uUGXHok+Jnx8WvgFHUwPN4/FoeZ5HXX1YfRTrcNGOLP51kfA7SKzDjq+596Jn8XMP\nPq7KY2IttGOpuj64t9lsohN1JPzrIuF3DA7q+S47bQJ7k8mktpNu3VDMWOJOLLgX8xQk/Osi4XcM\nb+290LkEF+6+b7dVV4jjBe+HYUDsfsHt3+/30fFYEv11kfA7BgJ7yMPnvftsNrNv376FfHwO5tVV\n33lOp1Ow6rhirVarkJrrk3Q4+i9ui4TfESBQFOAgF386nYZa+/l8HoQ/n89tNpuVsvNSXXX8UExY\neg7gQeTv7+/RJB1fiCNrf1sk/A7gRcpFOJyHv1wuS8KHxfcddvxrMnD1eR8PkW+325LwUxYfr8OC\nl/ivi4T/5HjR+5JbWHyI3gu/bREO7/HZ4vvjutVqFaL3WZYlC3HUcfe2SPhPTModZ4ufEj631+J6\ne/+6/mPOzvPHde/v7/b+/l7Z46PuPiZ8fy+ug4T/pKRE/1GL7wtxmlx9b/HX67WtVqtg8b2rH8vH\nl+Bvh4T/5MQ64qT2+BA+Ant1HXbqiLn66/U6WHxV4H0+Ev4TEutrj5Wqt0dkH8d4HNhL1dvH4GIc\nPAA4U48bbPj++RL9/ZDwn5BY6S3uY5l5fJbPVr6u3l58bST8J4NTaX0uPVffpYpw8HnfNtvn4tch\ny/34SPhPSKrenkdi+Vx8Fn+q7PZSi68HwOMi4T8Z3tr7+fbcaCNl8f3ce99hR3x9JPwnJCZ+iLlN\n2a33FP5W+LL8j4eE/4TU1dtzcC9l8X0fvqZ5eIxE/jWQ8J+MVIcdHOF5i89Helixs39F9J8LCf8J\nSVl8CN8vDuYNh8PPfvviDihi82T4M3zfYcefz8c65IrnR8J/QmKi98L3Z/Ry57uFhP9kxPb4svjC\nI+E/IU17/JTwJf7uIOE/Ialz/CZXX3QHCf/JqBN9LBU31RdfPDf6r/2EtHX1/dw7Wf3uoHP8J4Tb\nV/mhFL5O3tfKp1po+975uPrOuNxKG4tbbPNYLLXU/jwk/CfDt77a7/elfnkozY25/P1+3/I8j+bp\n497MKmOx+B6ddNFpx9+r+85jIOE/GdzlFg0vIfrz+WwvLy8VV5/Ffzwew7/7bD68Dh4ssYVRWCnx\n+357sUab4vZI+E8IW3wW/el0Cu23WPhs1Y/HY2ls1ul0MrN/RX8+n63X65V66vmJOeimC5H7e7TV\n9hZfY7Dvi4T/ZLCrz6Or8Tkzq4iexV8UhU0mk2CFzf5M34FF5uGXPPNuv9+XRJ6y+PhaWfzPQ8J/\nMtjV9x/neW7n87ni3vNZPkQNEXI77tjrYTxWlmW22+1qBQ+L74N9Ev79kfCfDLbufpxVv9+30+lU\nETwLH99jZmFbMBqNwmw7M6sEDzEeK9ZGO9ZWm6P63GlXwr8fEv6TAeHCMvtZ9kVRVNx7ztyLWfrJ\nZFISpg8eQvixPb23/lmWhRMAPhXAw0bcBwn/CUEQD/dm5aYaGHKBCTm+tZY/m+ermYXIfWxhWg4E\nz+OyEAvAg8lfZfHvh4T/5CASjysH5na7XTi6YzefJ91C5KvVyn7//m29Xi+IGYs/fn9/D+JHBB/R\ne1j22Ahsif6+SPhPCoTO9zzlxif3cGwAkXqeeTefz20+n5uZhUAejuX4Y3wPT8Tl+XgxwUv090fC\n7wgxi4/kHh+pZwFzE87pdGpmVjqO4wV3Htbfn9enRmGL+yPhPzFs9fExj7GGF+CP57ghp19mFs2/\nx5XP9fl8H66+LP5jIOE/ORAVB/uOx2MlsQei5xx+34QTqbupdF1f+MP3PByT35es/+cg4XcEFhjS\nY1n0qSEa/nP4Pj6O88U6ELm/T7n5Ev396d36j97r9fRf9cHwpbZ8zp/6N1+Wm3LZecUi+BL5fTmf\nz9EmC7L4HUQCFOrAI0QHkfCF6CASvhAdRMIXooNI+EJ0EAlfiA4i4QvRQSR8ITqIhC9EB5Hwhegg\nEr4QHeTmRTpCiMdDFl+IDiLhC9FBJHwhOoiEL0QHkfCF6CASvhAdRMIXooNI+EJ0EAlfiA4i4QvR\nQSR8ITqIhC9EB5HwheggEr4QHUTCF6KDSPhCdBAJX4gOIuEL0UEkfCE6iIQvRAf5P0M0TourqjJa\nAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x121f411d0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "pred = model.predict_classes(test_x)\n",
    "\n",
    "img_name = rng.choice(test.filename)\n",
    "filepath = os.path.join(data_dir, 'Train', 'Images', 'test', img_name)\n",
    "\n",
    "img = imread(filepath, flatten=True)\n",
    "\n",
    "test_index = int(img_name.split('.')[0]) - train.shape[0]\n",
    "\n",
    "print(\"Prediction is: \", pred[test_index])\n",
    "\n",
    "pylab.imshow(img, cmap='gray')\n",
    "pylab.axis('off')\n",
    "pylab.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "'''\n",
    "Some important parameters to look out for while optimizing neural networks are:\n",
    "\n",
    "Type of architecture\n",
    "Number of Layers\n",
    "Number of Neurons in a layer\n",
    "Regularization parameters\n",
    "Learning Rate\n",
    "Type of optimization / backpropagation technique to use\n",
    "Dropout rate\n",
    "Weight sharing\n",
    "'''"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Train a wide model instead, more hidden neurons"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# 3 layers: Input, Hidden, Output\n",
    "# Input : 28 * 28 \n",
    "# Output : 10 * 1 classes\n",
    "# 500 hidden neurons\n",
    "# Adam as our optimization algorithms, which is an efficient variant of Gradient Descent algorithm\n",
    "\n",
    "# define vars\n",
    "input_num_units = 784\n",
    "hidden_num_units = 500\n",
    "output_num_units = 10\n",
    "\n",
    "epochs = 5\n",
    "batch_size = 128\n",
    "\n",
    "# create model\n",
    "model = Sequential([\n",
    "  Dense(units=hidden_num_units, input_dim=input_num_units, activation='relu'),\n",
    "  Dense(units=output_num_units, input_dim=hidden_num_units, activation='softmax'),\n",
    "])\n",
    "\n",
    "# compile the model with necessary attributes\n",
    "model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 34300 samples, validate on 14700 samples\n",
      "Epoch 1/5\n",
      "34300/34300 [==============================] - 5s - loss: 0.3375 - acc: 0.9063 - val_loss: 0.1938 - val_acc: 0.9462\n",
      "Epoch 2/5\n",
      "34300/34300 [==============================] - 4s - loss: 0.1470 - acc: 0.9576 - val_loss: 0.1403 - val_acc: 0.9598\n",
      "Epoch 3/5\n",
      "34300/34300 [==============================] - 4s - loss: 0.0968 - acc: 0.9729 - val_loss: 0.1175 - val_acc: 0.9646\n",
      "Epoch 4/5\n",
      "34300/34300 [==============================] - 4s - loss: 0.0698 - acc: 0.9797 - val_loss: 0.1010 - val_acc: 0.9692\n",
      "Epoch 5/5\n",
      "34300/34300 [==============================] - 4s - loss: 0.0507 - acc: 0.9853 - val_loss: 0.0942 - val_acc: 0.9713\n"
     ]
    }
   ],
   "source": [
    "trained_model = model.fit(train_x, train_y, epochs=epochs, batch_size=batch_size, validation_data=(val_x, val_y))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Train more hidden layers instead"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# 7 layers: Input, Hidden * 5, Output\n",
    "# Input : 28 * 28 \n",
    "# Output : 10 * 1 classes\n",
    "# 50 hidden neurons\n",
    "# Adam as our optimization algorithms, which is an efficient variant of Gradient Descent algorithm\n",
    "\n",
    "# define vars\n",
    "input_num_units = 784\n",
    "hidden1_num_units = 50\n",
    "hidden2_num_units = 50\n",
    "hidden3_num_units = 50\n",
    "hidden4_num_units = 50\n",
    "hidden5_num_units = 50\n",
    "output_num_units = 10\n",
    "\n",
    "epochs = 5\n",
    "batch_size = 128\n",
    "\n",
    "model = Sequential([\n",
    " Dense(units=hidden1_num_units, input_dim=input_num_units, activation='relu'),\n",
    " Dense(units=hidden2_num_units, input_dim=hidden1_num_units, activation='relu'),\n",
    " Dense(units=hidden3_num_units, input_dim=hidden2_num_units, activation='relu'),\n",
    " Dense(units=hidden4_num_units, input_dim=hidden3_num_units, activation='relu'),\n",
    " Dense(units=hidden5_num_units, input_dim=hidden4_num_units, activation='relu'),\n",
    " Dense(units=output_num_units, input_dim=hidden5_num_units, activation='softmax'),\n",
    " ])\n",
    "\n",
    "# compile the model with necessary attributes\n",
    "model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 34300 samples, validate on 14700 samples\n",
      "Epoch 1/5\n",
      "34300/34300 [==============================] - 2s - loss: 0.5938 - acc: 0.8188 - val_loss: 0.2619 - val_acc: 0.9223\n",
      "Epoch 2/5\n",
      "34300/34300 [==============================] - 1s - loss: 0.2120 - acc: 0.9377 - val_loss: 0.2020 - val_acc: 0.9402\n",
      "Epoch 3/5\n",
      "34300/34300 [==============================] - 1s - loss: 0.1592 - acc: 0.9527 - val_loss: 0.1746 - val_acc: 0.9466\n",
      "Epoch 4/5\n",
      "34300/34300 [==============================] - 1s - loss: 0.1271 - acc: 0.9614 - val_loss: 0.1903 - val_acc: 0.9436\n",
      "Epoch 5/5\n",
      "34300/34300 [==============================] - 1s - loss: 0.1062 - acc: 0.9674 - val_loss: 0.1486 - val_acc: 0.9541\n"
     ]
    }
   ],
   "source": [
    "# Overfitted model\n",
    "trained_model = model.fit(train_x, train_y, epochs=epochs, batch_size=batch_size, validation_data=(val_x, val_y))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Introduce Dropout layer to reduce overfitting"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# 7 layers: Input, Hidden * 5, Output\n",
    "# Input : 28 * 28 \n",
    "# Output : 10 * 1 classes\n",
    "# 50 hidden neurons\n",
    "# Adam as our optimization algorithms, which is an efficient variant of Gradient Descent algorithm\n",
    "\n",
    "# define vars\n",
    "input_num_units = 784\n",
    "hidden1_num_units = 50\n",
    "hidden2_num_units = 50\n",
    "hidden3_num_units = 50\n",
    "hidden4_num_units = 50\n",
    "hidden5_num_units = 50\n",
    "output_num_units = 10\n",
    "\n",
    "epochs = 5\n",
    "batch_size = 128\n",
    "\n",
    "dropout_ratio = 0.2\n",
    "\n",
    "model = Sequential([\n",
    " Dense(units=hidden1_num_units, input_dim=input_num_units, activation='relu'),\n",
    " Dropout(dropout_ratio),\n",
    " Dense(units=hidden2_num_units, input_dim=hidden1_num_units, activation='relu'),\n",
    " Dropout(dropout_ratio),\n",
    " Dense(units=hidden3_num_units, input_dim=hidden2_num_units, activation='relu'),\n",
    " Dropout(dropout_ratio),\n",
    " Dense(units=hidden4_num_units, input_dim=hidden3_num_units, activation='relu'),\n",
    " Dropout(dropout_ratio),\n",
    " Dense(units=hidden5_num_units, input_dim=hidden4_num_units, activation='relu'),\n",
    " Dropout(dropout_ratio),\n",
    " Dense(units=output_num_units, input_dim=hidden5_num_units, activation='softmax'),\n",
    " ])\n",
    "\n",
    "# compile the model with necessary attributes\n",
    "model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 34300 samples, validate on 14700 samples\n",
      "Epoch 1/5\n",
      "34300/34300 [==============================] - 3s - loss: 1.1866 - acc: 0.5913 - val_loss: 0.3904 - val_acc: 0.8950\n",
      "Epoch 2/5\n",
      "34300/34300 [==============================] - 2s - loss: 0.4963 - acc: 0.8579 - val_loss: 0.2807 - val_acc: 0.9252\n",
      "Epoch 3/5\n",
      "34300/34300 [==============================] - 2s - loss: 0.3802 - acc: 0.8980 - val_loss: 0.2452 - val_acc: 0.9339\n",
      "Epoch 4/5\n",
      "34300/34300 [==============================] - 2s - loss: 0.3249 - acc: 0.9164 - val_loss: 0.2102 - val_acc: 0.9428\n",
      "Epoch 5/5\n",
      "34300/34300 [==============================] - 2s - loss: 0.2930 - acc: 0.9224 - val_loss: 0.1998 - val_acc: 0.9459\n"
     ]
    }
   ],
   "source": [
    "trained_model = model.fit(train_x, train_y, epochs=epochs, batch_size=batch_size, validation_data=(val_x, val_y))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Increase number of epochs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# 7 layers: Input, Hidden * 5, Output\n",
    "# Input : 28 * 28 \n",
    "# Output : 10 * 1 classes\n",
    "# 50 hidden neurons\n",
    "# Adam as our optimization algorithms, which is an efficient variant of Gradient Descent algorithm\n",
    "\n",
    "# define vars\n",
    "input_num_units = 784\n",
    "hidden1_num_units = 50\n",
    "hidden2_num_units = 50\n",
    "hidden3_num_units = 50\n",
    "hidden4_num_units = 50\n",
    "hidden5_num_units = 50\n",
    "output_num_units = 10\n",
    "\n",
    "epochs = 50\n",
    "batch_size = 128\n",
    "\n",
    "dropout_ratio = 0.2\n",
    "\n",
    "model = Sequential([\n",
    " Dense(units=hidden1_num_units, input_dim=input_num_units, activation='relu'),\n",
    " Dropout(dropout_ratio),\n",
    " Dense(units=hidden2_num_units, input_dim=hidden1_num_units, activation='relu'),\n",
    " Dropout(dropout_ratio),\n",
    " Dense(units=hidden3_num_units, input_dim=hidden2_num_units, activation='relu'),\n",
    " Dropout(dropout_ratio),\n",
    " Dense(units=hidden4_num_units, input_dim=hidden3_num_units, activation='relu'),\n",
    " Dropout(dropout_ratio),\n",
    " Dense(units=hidden5_num_units, input_dim=hidden4_num_units, activation='relu'),\n",
    " Dropout(dropout_ratio),\n",
    " Dense(units=output_num_units, input_dim=hidden5_num_units, activation='softmax'),\n",
    " ])\n",
    "\n",
    "# compile the model with necessary attributes\n",
    "model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 34300 samples, validate on 14700 samples\n",
      "Epoch 1/50\n",
      "34300/34300 [==============================] - 3s - loss: 1.1448 - acc: 0.6056 - val_loss: 0.4032 - val_acc: 0.8867\n",
      "Epoch 2/50\n",
      "34300/34300 [==============================] - 2s - loss: 0.4844 - acc: 0.8636 - val_loss: 0.2850 - val_acc: 0.9222\n",
      "Epoch 3/50\n",
      "34300/34300 [==============================] - 2s - loss: 0.3703 - acc: 0.9000 - val_loss: 0.2349 - val_acc: 0.9368\n",
      "Epoch 4/50\n",
      "34300/34300 [==============================] - 2s - loss: 0.3191 - acc: 0.9173 - val_loss: 0.2222 - val_acc: 0.9401\n",
      "Epoch 5/50\n",
      "34300/34300 [==============================] - 2s - loss: 0.2862 - acc: 0.9248 - val_loss: 0.2062 - val_acc: 0.9450\n",
      "Epoch 6/50\n",
      "34300/34300 [==============================] - 2s - loss: 0.2679 - acc: 0.9317 - val_loss: 0.1979 - val_acc: 0.9488\n",
      "Epoch 7/50\n",
      "34300/34300 [==============================] - 2s - loss: 0.2505 - acc: 0.9345 - val_loss: 0.1837 - val_acc: 0.9518\n",
      "Epoch 8/50\n",
      "34300/34300 [==============================] - 2s - loss: 0.2304 - acc: 0.9399 - val_loss: 0.1690 - val_acc: 0.9563\n",
      "Epoch 9/50\n",
      "34300/34300 [==============================] - 2s - loss: 0.2183 - acc: 0.9448 - val_loss: 0.1796 - val_acc: 0.9539\n",
      "Epoch 10/50\n",
      "34300/34300 [==============================] - 2s - loss: 0.2063 - acc: 0.9457 - val_loss: 0.1711 - val_acc: 0.9560\n",
      "Epoch 11/50\n",
      "34300/34300 [==============================] - 2s - loss: 0.1923 - acc: 0.9499 - val_loss: 0.1751 - val_acc: 0.9548\n",
      "Epoch 12/50\n",
      "34300/34300 [==============================] - 2s - loss: 0.1873 - acc: 0.9513 - val_loss: 0.1645 - val_acc: 0.9574\n",
      "Epoch 13/50\n",
      "34300/34300 [==============================] - 2s - loss: 0.1850 - acc: 0.9508 - val_loss: 0.1615 - val_acc: 0.9595\n",
      "Epoch 14/50\n",
      "34300/34300 [==============================] - 2s - loss: 0.1745 - acc: 0.9552 - val_loss: 0.1674 - val_acc: 0.9573\n",
      "Epoch 15/50\n",
      "34300/34300 [==============================] - 2s - loss: 0.1745 - acc: 0.9541 - val_loss: 0.1591 - val_acc: 0.9571\n",
      "Epoch 16/50\n",
      "34300/34300 [==============================] - 2s - loss: 0.1691 - acc: 0.9559 - val_loss: 0.1518 - val_acc: 0.9590\n",
      "Epoch 17/50\n",
      "34300/34300 [==============================] - 2s - loss: 0.1585 - acc: 0.9583 - val_loss: 0.1624 - val_acc: 0.9587\n",
      "Epoch 18/50\n",
      "34300/34300 [==============================] - 2s - loss: 0.1601 - acc: 0.9572 - val_loss: 0.1618 - val_acc: 0.9596\n",
      "Epoch 19/50\n",
      "34300/34300 [==============================] - 2s - loss: 0.1547 - acc: 0.9596 - val_loss: 0.1564 - val_acc: 0.9594\n",
      "Epoch 20/50\n",
      "34300/34300 [==============================] - 2s - loss: 0.1477 - acc: 0.9610 - val_loss: 0.1565 - val_acc: 0.9608\n",
      "Epoch 21/50\n",
      "34300/34300 [==============================] - 2s - loss: 0.1526 - acc: 0.9596 - val_loss: 0.1529 - val_acc: 0.9614\n",
      "Epoch 22/50\n",
      "34300/34300 [==============================] - 2s - loss: 0.1430 - acc: 0.9618 - val_loss: 0.1526 - val_acc: 0.9612\n",
      "Epoch 23/50\n",
      "34300/34300 [==============================] - 2s - loss: 0.1433 - acc: 0.9623 - val_loss: 0.1465 - val_acc: 0.9632\n",
      "Epoch 24/50\n",
      "34300/34300 [==============================] - 2s - loss: 0.1465 - acc: 0.9610 - val_loss: 0.1520 - val_acc: 0.9606\n",
      "Epoch 25/50\n",
      "34300/34300 [==============================] - 2s - loss: 0.1360 - acc: 0.9638 - val_loss: 0.1525 - val_acc: 0.9623\n",
      "Epoch 26/50\n",
      "34300/34300 [==============================] - 2s - loss: 0.1314 - acc: 0.9650 - val_loss: 0.1600 - val_acc: 0.9599\n",
      "Epoch 27/50\n",
      "34300/34300 [==============================] - 2s - loss: 0.1288 - acc: 0.9657 - val_loss: 0.1483 - val_acc: 0.9614\n",
      "Epoch 28/50\n",
      "34300/34300 [==============================] - 2s - loss: 0.1234 - acc: 0.9674 - val_loss: 0.1599 - val_acc: 0.9601\n",
      "Epoch 29/50\n",
      "34300/34300 [==============================] - 2s - loss: 0.1258 - acc: 0.9667 - val_loss: 0.1576 - val_acc: 0.9595\n",
      "Epoch 30/50\n",
      "34300/34300 [==============================] - 2s - loss: 0.1244 - acc: 0.9664 - val_loss: 0.1489 - val_acc: 0.9615\n",
      "Epoch 31/50\n",
      "34300/34300 [==============================] - 2s - loss: 0.1209 - acc: 0.9673 - val_loss: 0.1558 - val_acc: 0.9602\n",
      "Epoch 32/50\n",
      "34300/34300 [==============================] - 2s - loss: 0.1185 - acc: 0.9678 - val_loss: 0.1532 - val_acc: 0.9632\n",
      "Epoch 33/50\n",
      "34300/34300 [==============================] - 2s - loss: 0.1190 - acc: 0.9684 - val_loss: 0.1524 - val_acc: 0.9618\n",
      "Epoch 34/50\n",
      "34300/34300 [==============================] - 2s - loss: 0.1265 - acc: 0.9655 - val_loss: 0.1538 - val_acc: 0.9622\n",
      "Epoch 35/50\n",
      "34300/34300 [==============================] - 2s - loss: 0.1192 - acc: 0.9684 - val_loss: 0.1539 - val_acc: 0.9620\n",
      "Epoch 36/50\n",
      "34300/34300 [==============================] - 2s - loss: 0.1199 - acc: 0.9675 - val_loss: 0.1598 - val_acc: 0.9612\n",
      "Epoch 37/50\n",
      "34300/34300 [==============================] - 2s - loss: 0.1155 - acc: 0.9690 - val_loss: 0.1587 - val_acc: 0.9618\n",
      "Epoch 38/50\n",
      "34300/34300 [==============================] - 2s - loss: 0.1115 - acc: 0.9691 - val_loss: 0.1591 - val_acc: 0.9620\n",
      "Epoch 39/50\n",
      "34300/34300 [==============================] - 2s - loss: 0.1118 - acc: 0.9699 - val_loss: 0.1536 - val_acc: 0.9619\n",
      "Epoch 40/50\n",
      "34300/34300 [==============================] - 2s - loss: 0.1078 - acc: 0.9704 - val_loss: 0.1574 - val_acc: 0.9640\n",
      "Epoch 41/50\n",
      "34300/34300 [==============================] - 2s - loss: 0.1143 - acc: 0.9694 - val_loss: 0.1577 - val_acc: 0.9624\n",
      "Epoch 42/50\n",
      "34300/34300 [==============================] - 2s - loss: 0.1055 - acc: 0.9721 - val_loss: 0.1532 - val_acc: 0.9620\n",
      "Epoch 43/50\n",
      "34300/34300 [==============================] - 2s - loss: 0.1087 - acc: 0.9711 - val_loss: 0.1548 - val_acc: 0.9618\n",
      "Epoch 44/50\n",
      "34300/34300 [==============================] - 2s - loss: 0.1061 - acc: 0.9718 - val_loss: 0.1520 - val_acc: 0.9622\n",
      "Epoch 45/50\n",
      "34300/34300 [==============================] - 2s - loss: 0.1072 - acc: 0.9701 - val_loss: 0.1554 - val_acc: 0.9636\n",
      "Epoch 46/50\n",
      "34300/34300 [==============================] - 2s - loss: 0.1025 - acc: 0.9722 - val_loss: 0.1623 - val_acc: 0.9616\n",
      "Epoch 47/50\n",
      "34300/34300 [==============================] - 2s - loss: 0.1091 - acc: 0.9703 - val_loss: 0.1574 - val_acc: 0.9627\n",
      "Epoch 48/50\n",
      "34300/34300 [==============================] - 2s - loss: 0.1024 - acc: 0.9718 - val_loss: 0.1512 - val_acc: 0.9640\n",
      "Epoch 49/50\n",
      "34300/34300 [==============================] - 2s - loss: 0.1010 - acc: 0.9725 - val_loss: 0.1626 - val_acc: 0.9620\n",
      "Epoch 50/50\n",
      "34300/34300 [==============================] - 2s - loss: 0.1055 - acc: 0.9711 - val_loss: 0.1594 - val_acc: 0.9631\n"
     ]
    }
   ],
   "source": [
    "trained_model = model.fit(train_x, train_y, epochs=epochs, batch_size=batch_size, validation_data=(val_x, val_y))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Wide and deep network"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# 7 layers: Input, Hidden * 5, Output\n",
    "# Input : 28 * 28 \n",
    "# Output : 10 * 1 classes\n",
    "# 500 hidden neurons\n",
    "# Adam as our optimization algorithms, which is an efficient variant of Gradient Descent algorithm\n",
    "\n",
    "# define vars\n",
    "input_num_units = 784\n",
    "hidden1_num_units = 500\n",
    "hidden2_num_units = 500\n",
    "hidden3_num_units = 500\n",
    "hidden4_num_units = 500\n",
    "hidden5_num_units = 500\n",
    "output_num_units = 10\n",
    "\n",
    "epochs = 25\n",
    "batch_size = 128\n",
    "\n",
    "dropout_ratio = 0.2\n",
    "\n",
    "model = Sequential([\n",
    " Dense(units=hidden1_num_units, input_dim=input_num_units, activation='relu'),\n",
    " Dropout(dropout_ratio),\n",
    " Dense(units=hidden2_num_units, input_dim=hidden1_num_units, activation='relu'),\n",
    " Dropout(dropout_ratio),\n",
    " Dense(units=hidden3_num_units, input_dim=hidden2_num_units, activation='relu'),\n",
    " Dropout(dropout_ratio),\n",
    " Dense(units=hidden4_num_units, input_dim=hidden3_num_units, activation='relu'),\n",
    " Dropout(dropout_ratio),\n",
    " Dense(units=hidden5_num_units, input_dim=hidden4_num_units, activation='relu'),\n",
    " Dropout(dropout_ratio),\n",
    " Dense(units=output_num_units, input_dim=hidden5_num_units, activation='softmax'),\n",
    " ])\n",
    "\n",
    "# compile the model with necessary attributes\n",
    "model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 34300 samples, validate on 14700 samples\n",
      "Epoch 1/25\n",
      "34300/34300 [==============================] - 18s - loss: 0.3876 - acc: 0.8789 - val_loss: 0.1568 - val_acc: 0.9535\n",
      "Epoch 2/25\n",
      "34300/34300 [==============================] - 16s - loss: 0.1545 - acc: 0.9552 - val_loss: 0.1390 - val_acc: 0.9571\n",
      "Epoch 3/25\n",
      "34300/34300 [==============================] - 15s - loss: 0.1155 - acc: 0.9650 - val_loss: 0.1071 - val_acc: 0.9683\n",
      "Epoch 4/25\n",
      "34300/34300 [==============================] - 16s - loss: 0.0934 - acc: 0.9734 - val_loss: 0.1081 - val_acc: 0.9700\n",
      "Epoch 5/25\n",
      "34300/34300 [==============================] - 16s - loss: 0.0796 - acc: 0.9768 - val_loss: 0.0941 - val_acc: 0.9744\n",
      "Epoch 6/25\n",
      "34300/34300 [==============================] - 16s - loss: 0.0654 - acc: 0.9803 - val_loss: 0.1067 - val_acc: 0.9737\n",
      "Epoch 7/25\n",
      "34300/34300 [==============================] - 17s - loss: 0.0607 - acc: 0.9818 - val_loss: 0.1102 - val_acc: 0.9709\n",
      "Epoch 8/25\n",
      "34300/34300 [==============================] - 16s - loss: 0.0534 - acc: 0.9852 - val_loss: 0.1098 - val_acc: 0.9731\n",
      "Epoch 9/25\n",
      "34300/34300 [==============================] - 18s - loss: 0.0514 - acc: 0.9852 - val_loss: 0.1021 - val_acc: 0.9746\n",
      "Epoch 10/25\n",
      "34300/34300 [==============================] - 17s - loss: 0.0456 - acc: 0.9866 - val_loss: 0.1270 - val_acc: 0.9706\n",
      "Epoch 11/25\n",
      "34300/34300 [==============================] - 15s - loss: 0.0463 - acc: 0.9864 - val_loss: 0.1201 - val_acc: 0.9704\n",
      "Epoch 12/25\n",
      "34300/34300 [==============================] - 15s - loss: 0.0390 - acc: 0.9890 - val_loss: 0.1154 - val_acc: 0.9737\n",
      "Epoch 13/25\n",
      "34300/34300 [==============================] - 16s - loss: 0.0399 - acc: 0.9882 - val_loss: 0.1100 - val_acc: 0.9762\n",
      "Epoch 14/25\n",
      "34300/34300 [==============================] - 15s - loss: 0.0337 - acc: 0.9903 - val_loss: 0.1234 - val_acc: 0.9743\n",
      "Epoch 15/25\n",
      "34300/34300 [==============================] - 15s - loss: 0.0357 - acc: 0.9893 - val_loss: 0.1164 - val_acc: 0.9730\n",
      "Epoch 16/25\n",
      "34300/34300 [==============================] - 15s - loss: 0.0338 - acc: 0.9895 - val_loss: 0.1351 - val_acc: 0.9739\n",
      "Epoch 17/25\n",
      "34300/34300 [==============================] - 15s - loss: 0.0298 - acc: 0.9912 - val_loss: 0.1281 - val_acc: 0.9742\n",
      "Epoch 18/25\n",
      "34300/34300 [==============================] - 17s - loss: 0.0306 - acc: 0.9916 - val_loss: 0.1074 - val_acc: 0.9768\n",
      "Epoch 19/25\n",
      "34300/34300 [==============================] - 18s - loss: 0.0293 - acc: 0.9917 - val_loss: 0.1135 - val_acc: 0.9769\n",
      "Epoch 20/25\n",
      "34300/34300 [==============================] - 16s - loss: 0.0278 - acc: 0.9932 - val_loss: 0.1268 - val_acc: 0.9764\n",
      "Epoch 21/25\n",
      "34300/34300 [==============================] - 14s - loss: 0.0280 - acc: 0.9917 - val_loss: 0.1186 - val_acc: 0.9763\n",
      "Epoch 22/25\n",
      "34300/34300 [==============================] - 16s - loss: 0.0275 - acc: 0.9920 - val_loss: 0.1304 - val_acc: 0.9750\n",
      "Epoch 23/25\n",
      "34300/34300 [==============================] - 15s - loss: 0.0269 - acc: 0.9926 - val_loss: 0.1096 - val_acc: 0.9768\n",
      "Epoch 24/25\n",
      "34300/34300 [==============================] - 16s - loss: 0.0225 - acc: 0.9946 - val_loss: 0.1279 - val_acc: 0.9755\n",
      "Epoch 25/25\n",
      "34300/34300 [==============================] - 16s - loss: 0.0300 - acc: 0.9923 - val_loss: 0.1100 - val_acc: 0.9774\n"
     ]
    }
   ],
   "source": [
    "trained_model = model.fit(train_x, train_y, epochs=epochs, batch_size=batch_size, validation_data=(val_x, val_y))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Use CNN instead of MLP"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# reshape data\n",
    "\n",
    "train_x_temp = train_x.reshape(-1, 28, 28, 1)\n",
    "val_x_temp = val_x.reshape(-1, 28, 28, 1)\n",
    "\n",
    "# define vars\n",
    "input_shape = (784,)\n",
    "input_reshape = (28, 28, 1)\n",
    "\n",
    "conv_num_filters = 5\n",
    "conv_filter_size = 5\n",
    "\n",
    "pool_size = (2, 2)\n",
    "\n",
    "hidden_num_units = 50\n",
    "output_num_units = 10\n",
    "\n",
    "epochs = 5\n",
    "batch_size = 128\n",
    "\n",
    "model = Sequential([\n",
    " InputLayer(input_shape=input_reshape),\n",
    "\n",
    " Convolution2D(25, (5, 5), activation='relu'),\n",
    " MaxPooling2D(pool_size=pool_size),\n",
    "\n",
    " Convolution2D(25, (5, 5), activation='relu'),\n",
    " MaxPooling2D(pool_size=pool_size),\n",
    "\n",
    " Convolution2D(25, (4, 4), activation='relu'),\n",
    " Flatten(),\n",
    "\n",
    " Dense(units=hidden_num_units, activation='relu'),\n",
    " Dense(units=output_num_units, input_dim=hidden_num_units, activation='softmax'),\n",
    "])\n",
    "\n",
    "model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 34300 samples, validate on 14700 samples\n",
      "Epoch 1/5\n",
      "34300/34300 [==============================] - 32s - loss: 0.2123 - acc: 0.9351 - val_loss: 0.1331 - val_acc: 0.9613\n",
      "Epoch 2/5\n",
      "34300/34300 [==============================] - 31s - loss: 0.1044 - acc: 0.9687 - val_loss: 0.1075 - val_acc: 0.9669\n",
      "Epoch 3/5\n",
      "34300/34300 [==============================] - 38s - loss: 0.0782 - acc: 0.9749 - val_loss: 0.0741 - val_acc: 0.9775\n",
      "Epoch 4/5\n",
      "34300/34300 [==============================] - 37s - loss: 0.0600 - acc: 0.9809 - val_loss: 0.0650 - val_acc: 0.9823\n",
      "Epoch 5/5\n",
      "34300/34300 [==============================] - 38s - loss: 0.0524 - acc: 0.9842 - val_loss: 0.0688 - val_acc: 0.9797\n"
     ]
    }
   ],
   "source": [
    "trained_model_conv = model.fit(train_x_temp, train_y, epochs=epochs, batch_size=batch_size, validation_data=(val_x_temp, val_y))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "20896/21000 [============================>.] - ETA: 0s"
     ]
    }
   ],
   "source": [
    "test_x_temp = test_x.reshape(-1, 28, 28, 1)\n",
    "pred = model.predict_classes(test_x_temp)\n",
    "sample_submission.filename = test.filename; sample_submission.label = pred\n",
    "sample_submission.to_csv(os.path.join(sub_dir, 'sub.csv'), index=False)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}