ex_4.2_keras_4_layes_and_norm.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "uniform model\n",
      "Train on 60000 samples, validate on 10000 samples\n",
      "Epoch 1/30\n",
      "60000/60000 [==============================] - 5s 80us/step - loss: 0.8477 - acc: 0.7652 - val_loss: 0.3957 - val_acc: 0.8887\n",
      "Epoch 2/30\n",
      "60000/60000 [==============================] - 4s 69us/step - loss: 0.3821 - acc: 0.8915 - val_loss: 0.3134 - val_acc: 0.9095\n",
      "Epoch 3/30\n",
      "60000/60000 [==============================] - 4s 70us/step - loss: 0.3141 - acc: 0.9101 - val_loss: 0.2761 - val_acc: 0.9205\n",
      "Epoch 4/30\n",
      "60000/60000 [==============================] - 4s 69us/step - loss: 0.2532 - acc: 0.9274 - val_loss: 0.2102 - val_acc: 0.9391\n",
      "Epoch 5/30\n",
      "60000/60000 [==============================] - 4s 70us/step - loss: 0.2083 - acc: 0.9391 - val_loss: 0.1943 - val_acc: 0.9422\n",
      "Epoch 6/30\n",
      "60000/60000 [==============================] - 4s 69us/step - loss: 0.1782 - acc: 0.9480 - val_loss: 0.1617 - val_acc: 0.9525\n",
      "Epoch 7/30\n",
      "60000/60000 [==============================] - 4s 71us/step - loss: 0.1567 - acc: 0.9545 - val_loss: 0.1449 - val_acc: 0.9565\n",
      "Epoch 8/30\n",
      "60000/60000 [==============================] - 4s 68us/step - loss: 0.1409 - acc: 0.9586 - val_loss: 0.1468 - val_acc: 0.9559\n",
      "Epoch 9/30\n",
      "60000/60000 [==============================] - 4s 68us/step - loss: 0.1291 - acc: 0.9616 - val_loss: 0.1290 - val_acc: 0.9612\n",
      "Epoch 10/30\n",
      "60000/60000 [==============================] - 4s 67us/step - loss: 0.1193 - acc: 0.9652 - val_loss: 0.1208 - val_acc: 0.9622\n",
      "Epoch 11/30\n",
      "60000/60000 [==============================] - 4s 68us/step - loss: 0.1106 - acc: 0.9672 - val_loss: 0.1190 - val_acc: 0.9633\n",
      "Epoch 12/30\n",
      "60000/60000 [==============================] - 4s 68us/step - loss: 0.1027 - acc: 0.9697 - val_loss: 0.1142 - val_acc: 0.9642\n",
      "Epoch 13/30\n",
      "60000/60000 [==============================] - 4s 68us/step - loss: 0.0948 - acc: 0.9719 - val_loss: 0.1123 - val_acc: 0.9662\n",
      "Epoch 14/30\n",
      "60000/60000 [==============================] - 4s 68us/step - loss: 0.0902 - acc: 0.9729 - val_loss: 0.1117 - val_acc: 0.9656\n",
      "Epoch 15/30\n",
      "60000/60000 [==============================] - 4s 70us/step - loss: 0.0826 - acc: 0.9747 - val_loss: 0.1004 - val_acc: 0.9685\n",
      "Epoch 16/30\n",
      "60000/60000 [==============================] - 4s 71us/step - loss: 0.0786 - acc: 0.9762 - val_loss: 0.1047 - val_acc: 0.9686\n",
      "Epoch 17/30\n",
      "60000/60000 [==============================] - 4s 72us/step - loss: 0.0743 - acc: 0.9771 - val_loss: 0.1098 - val_acc: 0.9657\n",
      "Epoch 18/30\n",
      "60000/60000 [==============================] - 4s 70us/step - loss: 0.0696 - acc: 0.9784 - val_loss: 0.0989 - val_acc: 0.9699\n",
      "Epoch 19/30\n",
      "60000/60000 [==============================] - 4s 67us/step - loss: 0.0666 - acc: 0.9796 - val_loss: 0.1137 - val_acc: 0.9658\n",
      "Epoch 20/30\n",
      "60000/60000 [==============================] - 4s 69us/step - loss: 0.0630 - acc: 0.9805 - val_loss: 0.0965 - val_acc: 0.9714\n",
      "Epoch 21/30\n",
      "60000/60000 [==============================] - 4s 68us/step - loss: 0.0588 - acc: 0.9819 - val_loss: 0.1040 - val_acc: 0.9696\n",
      "Epoch 22/30\n",
      "60000/60000 [==============================] - 4s 68us/step - loss: 0.0552 - acc: 0.9827 - val_loss: 0.0976 - val_acc: 0.9710\n",
      "Epoch 23/30\n",
      "60000/60000 [==============================] - 4s 75us/step - loss: 0.0535 - acc: 0.9833 - val_loss: 0.1180 - val_acc: 0.9644\n",
      "Epoch 24/30\n",
      "60000/60000 [==============================] - 4s 69us/step - loss: 0.0498 - acc: 0.9848 - val_loss: 0.1051 - val_acc: 0.9677\n",
      "Epoch 25/30\n",
      "60000/60000 [==============================] - 4s 68us/step - loss: 0.0483 - acc: 0.9850 - val_loss: 0.0948 - val_acc: 0.9714\n",
      "Epoch 26/30\n",
      "60000/60000 [==============================] - 4s 68us/step - loss: 0.0456 - acc: 0.9859 - val_loss: 0.0941 - val_acc: 0.9715\n",
      "Epoch 27/30\n",
      "60000/60000 [==============================] - 4s 71us/step - loss: 0.0437 - acc: 0.9865 - val_loss: 0.0938 - val_acc: 0.9720\n",
      "Epoch 28/30\n",
      "60000/60000 [==============================] - 4s 72us/step - loss: 0.0411 - acc: 0.9875 - val_loss: 0.0882 - val_acc: 0.9734\n",
      "Epoch 29/30\n",
      "60000/60000 [==============================] - 4s 68us/step - loss: 0.0387 - acc: 0.9882 - val_loss: 0.1055 - val_acc: 0.9688\n",
      "Epoch 30/30\n",
      "60000/60000 [==============================] - 4s 70us/step - loss: 0.0372 - acc: 0.9885 - val_loss: 0.0855 - val_acc: 0.9747\n",
      "glorot_normal model\n",
      "Train on 60000 samples, validate on 10000 samples\n",
      "Epoch 1/30\n",
      "60000/60000 [==============================] - 5s 79us/step - loss: 0.4436 - acc: 0.8718 - val_loss: 0.2997 - val_acc: 0.9129\n",
      "Epoch 2/30\n",
      "60000/60000 [==============================] - 4s 68us/step - loss: 0.2805 - acc: 0.9189 - val_loss: 0.2346 - val_acc: 0.9335\n",
      "Epoch 3/30\n",
      "60000/60000 [==============================] - 4s 71us/step - loss: 0.2205 - acc: 0.9352 - val_loss: 0.1884 - val_acc: 0.9449\n",
      "Epoch 4/30\n",
      "60000/60000 [==============================] - 4s 71us/step - loss: 0.1804 - acc: 0.9469 - val_loss: 0.1640 - val_acc: 0.9512\n",
      "Epoch 5/30\n",
      "60000/60000 [==============================] - 4s 69us/step - loss: 0.1524 - acc: 0.9546 - val_loss: 0.1383 - val_acc: 0.9579\n",
      "Epoch 6/30\n",
      "60000/60000 [==============================] - 4s 71us/step - loss: 0.1334 - acc: 0.9598 - val_loss: 0.1301 - val_acc: 0.9613\n",
      "Epoch 7/30\n",
      "60000/60000 [==============================] - 4s 69us/step - loss: 0.1174 - acc: 0.9655 - val_loss: 0.1218 - val_acc: 0.9628\n",
      "Epoch 8/30\n",
      "60000/60000 [==============================] - 4s 68us/step - loss: 0.1067 - acc: 0.9682 - val_loss: 0.1196 - val_acc: 0.9624\n",
      "Epoch 9/30\n",
      "60000/60000 [==============================] - 4s 70us/step - loss: 0.0959 - acc: 0.9711 - val_loss: 0.1028 - val_acc: 0.9678\n",
      "Epoch 10/30\n",
      "60000/60000 [==============================] - 4s 71us/step - loss: 0.0876 - acc: 0.9740 - val_loss: 0.1007 - val_acc: 0.9666\n",
      "Epoch 11/30\n",
      "60000/60000 [==============================] - 4s 71us/step - loss: 0.0802 - acc: 0.9759 - val_loss: 0.0938 - val_acc: 0.9701\n",
      "Epoch 12/30\n",
      "60000/60000 [==============================] - 4s 72us/step - loss: 0.0740 - acc: 0.9776 - val_loss: 0.0940 - val_acc: 0.9712\n",
      "Epoch 13/30\n",
      "60000/60000 [==============================] - 4s 72us/step - loss: 0.0693 - acc: 0.9789 - val_loss: 0.0941 - val_acc: 0.9717\n",
      "Epoch 14/30\n",
      "60000/60000 [==============================] - 4s 70us/step - loss: 0.0644 - acc: 0.9804 - val_loss: 0.0889 - val_acc: 0.9721\n",
      "Epoch 15/30\n",
      "60000/60000 [==============================] - 4s 70us/step - loss: 0.0606 - acc: 0.9815 - val_loss: 0.0889 - val_acc: 0.9725\n",
      "Epoch 16/30\n",
      "60000/60000 [==============================] - 4s 71us/step - loss: 0.0556 - acc: 0.9831 - val_loss: 0.0861 - val_acc: 0.9732\n",
      "Epoch 17/30\n",
      "60000/60000 [==============================] - 4s 72us/step - loss: 0.0520 - acc: 0.9848 - val_loss: 0.0948 - val_acc: 0.9698\n",
      "Epoch 18/30\n",
      "60000/60000 [==============================] - 4s 72us/step - loss: 0.0490 - acc: 0.9854 - val_loss: 0.0942 - val_acc: 0.9711\n",
      "Epoch 19/30\n",
      "60000/60000 [==============================] - 4s 71us/step - loss: 0.0460 - acc: 0.9859 - val_loss: 0.0857 - val_acc: 0.9749\n",
      "Epoch 20/30\n",
      "60000/60000 [==============================] - 4s 71us/step - loss: 0.0433 - acc: 0.9869 - val_loss: 0.0875 - val_acc: 0.9753\n",
      "Epoch 21/30\n",
      "60000/60000 [==============================] - 4s 67us/step - loss: 0.0408 - acc: 0.9874 - val_loss: 0.0823 - val_acc: 0.9752\n",
      "Epoch 22/30\n",
      "60000/60000 [==============================] - 4s 69us/step - loss: 0.0391 - acc: 0.9883 - val_loss: 0.0818 - val_acc: 0.9764\n",
      "Epoch 23/30\n",
      "60000/60000 [==============================] - 4s 68us/step - loss: 0.0362 - acc: 0.9892 - val_loss: 0.0816 - val_acc: 0.9765\n",
      "Epoch 24/30\n",
      "60000/60000 [==============================] - 4s 69us/step - loss: 0.0342 - acc: 0.9897 - val_loss: 0.0824 - val_acc: 0.9762\n",
      "Epoch 25/30\n",
      "60000/60000 [==============================] - 4s 68us/step - loss: 0.0326 - acc: 0.9906 - val_loss: 0.0919 - val_acc: 0.9725\n",
      "Epoch 26/30\n",
      "60000/60000 [==============================] - 4s 68us/step - loss: 0.0300 - acc: 0.9915 - val_loss: 0.0784 - val_acc: 0.9781\n",
      "Epoch 27/30\n",
      "60000/60000 [==============================] - 4s 70us/step - loss: 0.0296 - acc: 0.9913 - val_loss: 0.0820 - val_acc: 0.9758\n",
      "Epoch 28/30\n",
      "60000/60000 [==============================] - 4s 69us/step - loss: 0.0272 - acc: 0.9925 - val_loss: 0.0827 - val_acc: 0.9768\n",
      "Epoch 29/30\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "60000/60000 [==============================] - 4s 69us/step - loss: 0.0257 - acc: 0.9928 - val_loss: 0.0795 - val_acc: 0.9771\n",
      "Epoch 30/30\n",
      "60000/60000 [==============================] - 4s 70us/step - loss: 0.0247 - acc: 0.9931 - val_loss: 0.0825 - val_acc: 0.9776\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0x7f8a59551be0>"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from keras.models import Sequential\n",
    "from keras.layers import Dense\n",
    "from keras.layers.normalization import BatchNormalization\n",
    "from keras.datasets import mnist\n",
    "from keras.utils import np_utils\n",
    "\n",
    "def create_model(init):\n",
    "    model = Sequential()\n",
    "    model.add(Dense(100, input_shape=(784,), kernel_initializer=init, activation='tanh'))\n",
    "    model.add(Dense(100, kernel_initializer=init, activation='tanh'))\n",
    "    model.add(Dense(100, kernel_initializer=init, activation='tanh'))\n",
    "    model.add(Dense(100, kernel_initializer=init, activation='tanh'))\n",
    "    model.add(BatchNormalization())\n",
    "    model.add(Dense(10, kernel_initializer=init, activation='softmax'))\n",
    "    return model\n",
    "\n",
    "(x_train, y_train), (x_test, y_test) = mnist.load_data()\n",
    "\n",
    "\n",
    "y_train = np_utils.to_categorical(y_train, 10)\n",
    "y_test = np_utils.to_categorical(y_test, 10)\n",
    "\n",
    "x_train = x_train.reshape([-1, 784]) / 255\n",
    "x_test = x_test.reshape([-1, 784]) / 255\n",
    "\n",
    "print(\"uniform model\")\n",
    "uniform_model = create_model(\"uniform\")\n",
    "uniform_model.compile(loss=\"categorical_crossentropy\", optimizer=\"sgd\", metrics=[\"accuracy\"])\n",
    "uniform_model.fit(x_train, y_train, batch_size=64, epochs=30, verbose=1, validation_data=(x_test, y_test))\n",
    "\n",
    "print(\"glorot_normal model\")\n",
    "uniform_model = create_model(\"glorot_normal\")\n",
    "uniform_model.compile(loss=\"categorical_crossentropy\", optimizer=\"sgd\", metrics=[\"accuracy\"])\n",
    "uniform_model.fit(x_train, y_train, batch_size=64, epochs=30, verbose=1, validation_data=(x_test, y_test))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}