compressed-network.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import tensorflow as tf\n",
    "import keras\n",
    "import keras.backend as K"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "((X_train, Y_train), (X_test, Y_test)) = keras.datasets.mnist.load_data()\n",
    "X_train, X_test = X_train/255, X_test/255"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### a compressed matrix neural layer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "class CompressedDense(keras.layers.Layer):\n",
    "    \n",
    "    def __init__(self, units, freqs=128, **kwargs):\n",
    "        self.units = units\n",
    "        self.freqs = freqs\n",
    "        super(CompressedDense, self).__init__(**kwargs)\n",
    "    \n",
    "    def build(self, input_shape):\n",
    "        self.inputs = input_shape[1]\n",
    "        self.weight_padding = (self.inputs * self.units) - self.freqs\n",
    "        self.kernel = self.add_weight(\n",
    "            shape=(1,self.freqs,1),\n",
    "            initializer='glorot_uniform', name='kernel')\n",
    "        self.bias = self.add_weight(\n",
    "            shape=(self.units,),\n",
    "            initializer='zeros', name='bias')\n",
    "        super(CompressedDense, self).build(input_shape)\n",
    "    \n",
    "    def call(self, inputs):\n",
    "        weights = tf.spectral.idct(\n",
    "            K.permute_dimensions(\n",
    "                K.temporal_padding(\n",
    "                    self.kernel, (0,self.weight_padding)), (0,2,1)), norm='ortho')\n",
    "        weights = K.reshape(weights, (self.inputs, self.units))\n",
    "        return inputs @ weights + self.bias\n",
    "    \n",
    "    def compute_output_shape(self, input_shape):\n",
    "        return (input_shape[0], self.units)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### baseline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_20 (InputLayer)        (None, 28, 28)            0         \n",
      "_________________________________________________________________\n",
      "reshape_20 (Reshape)         (None, 784)               0         \n",
      "_________________________________________________________________\n",
      "dense_5 (Dense)              (None, 10)                7850      \n",
      "_________________________________________________________________\n",
      "batch_normalization_29 (Batc (None, 10)                40        \n",
      "_________________________________________________________________\n",
      "activation_16 (Activation)   (None, 10)                0         \n",
      "=================================================================\n",
      "Total params: 7,890\n",
      "Trainable params: 7,870\n",
      "Non-trainable params: 20\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "X = X_input = keras.layers.Input((28,28))\n",
    "X = keras.layers.Reshape((28*28,))(X)\n",
    "X = keras.layers.Dense(10)(X)\n",
    "X = keras.layers.BatchNormalization()(X)\n",
    "X = keras.layers.Activation('softmax')(X)\n",
    "M_baseline = keras.Model(X_input, X)\n",
    "M_baseline.compile('adam', 'sparse_categorical_crossentropy', ['acc'])\n",
    "M_baseline.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 60000 samples, validate on 10000 samples\n",
      "Epoch 1/200\n",
      "60000/60000 [==============================] - 2s 29us/step - loss: 1.0988 - acc: 0.7420 - val_loss: 0.7502 - val_acc: 0.8634\n",
      "Epoch 2/200\n",
      "60000/60000 [==============================] - 1s 14us/step - loss: 0.7362 - acc: 0.8691 - val_loss: 0.6469 - val_acc: 0.8840\n",
      "Epoch 3/200\n",
      "60000/60000 [==============================] - 1s 15us/step - loss: 0.6503 - acc: 0.8817 - val_loss: 0.5842 - val_acc: 0.8898\n",
      "Epoch 4/200\n",
      "60000/60000 [==============================] - 1s 14us/step - loss: 0.5935 - acc: 0.8871 - val_loss: 0.5366 - val_acc: 0.8972\n",
      "Epoch 5/200\n",
      "60000/60000 [==============================] - 1s 16us/step - loss: 0.5509 - acc: 0.8924 - val_loss: 0.5039 - val_acc: 0.8976\n",
      "Epoch 6/200\n",
      "60000/60000 [==============================] - 1s 16us/step - loss: 0.5169 - acc: 0.8951 - val_loss: 0.4785 - val_acc: 0.8988\n",
      "Epoch 7/200\n",
      "60000/60000 [==============================] - 1s 14us/step - loss: 0.4896 - acc: 0.8975 - val_loss: 0.4521 - val_acc: 0.9025\n",
      "Epoch 8/200\n",
      "60000/60000 [==============================] - 1s 14us/step - loss: 0.4671 - acc: 0.9003 - val_loss: 0.4351 - val_acc: 0.9018\n",
      "Epoch 9/200\n",
      "60000/60000 [==============================] - 1s 15us/step - loss: 0.4478 - acc: 0.9023 - val_loss: 0.4199 - val_acc: 0.9054\n",
      "Epoch 10/200\n",
      "60000/60000 [==============================] - 1s 15us/step - loss: 0.4314 - acc: 0.9034 - val_loss: 0.4039 - val_acc: 0.9089\n",
      "Epoch 11/200\n",
      "60000/60000 [==============================] - 1s 15us/step - loss: 0.4170 - acc: 0.9060 - val_loss: 0.3935 - val_acc: 0.9078\n",
      "Epoch 12/200\n",
      "60000/60000 [==============================] - 1s 17us/step - loss: 0.4043 - acc: 0.9069 - val_loss: 0.3870 - val_acc: 0.9073\n",
      "Epoch 13/200\n",
      "60000/60000 [==============================] - 1s 16us/step - loss: 0.3930 - acc: 0.9087 - val_loss: 0.3775 - val_acc: 0.9101\n",
      "Epoch 14/200\n",
      "60000/60000 [==============================] - 1s 14us/step - loss: 0.3819 - acc: 0.9092 - val_loss: 0.3650 - val_acc: 0.9119\n",
      "Epoch 15/200\n",
      "60000/60000 [==============================] - 1s 15us/step - loss: 0.3734 - acc: 0.9106 - val_loss: 0.3614 - val_acc: 0.9118\n",
      "Epoch 16/200\n",
      "60000/60000 [==============================] - 1s 15us/step - loss: 0.3658 - acc: 0.9113 - val_loss: 0.3570 - val_acc: 0.9098\n",
      "Epoch 17/200\n",
      "60000/60000 [==============================] - 1s 16us/step - loss: 0.3572 - acc: 0.9125 - val_loss: 0.3496 - val_acc: 0.9115\n",
      "Epoch 18/200\n",
      "60000/60000 [==============================] - 1s 14us/step - loss: 0.3514 - acc: 0.9142 - val_loss: 0.3430 - val_acc: 0.9130\n",
      "Epoch 19/200\n",
      "60000/60000 [==============================] - 1s 14us/step - loss: 0.3448 - acc: 0.9150 - val_loss: 0.3417 - val_acc: 0.9123\n",
      "Epoch 20/200\n",
      "60000/60000 [==============================] - 1s 15us/step - loss: 0.3391 - acc: 0.9159 - val_loss: 0.3314 - val_acc: 0.9154\n",
      "Epoch 21/200\n",
      "60000/60000 [==============================] - 1s 15us/step - loss: 0.3337 - acc: 0.9157 - val_loss: 0.3297 - val_acc: 0.9147\n",
      "Epoch 22/200\n",
      "60000/60000 [==============================] - 1s 13us/step - loss: 0.3289 - acc: 0.9168 - val_loss: 0.3238 - val_acc: 0.9160\n",
      "Epoch 23/200\n",
      "60000/60000 [==============================] - 1s 15us/step - loss: 0.3230 - acc: 0.9177 - val_loss: 0.3213 - val_acc: 0.9175\n",
      "Epoch 24/200\n",
      "60000/60000 [==============================] - 1s 14us/step - loss: 0.3196 - acc: 0.9189 - val_loss: 0.3216 - val_acc: 0.9164\n",
      "Epoch 25/200\n",
      "60000/60000 [==============================] - 1s 13us/step - loss: 0.3150 - acc: 0.9191 - val_loss: 0.3148 - val_acc: 0.9179\n",
      "Epoch 26/200\n",
      "60000/60000 [==============================] - 1s 14us/step - loss: 0.3125 - acc: 0.9192 - val_loss: 0.3121 - val_acc: 0.9194\n",
      "Epoch 27/200\n",
      "60000/60000 [==============================] - 1s 13us/step - loss: 0.3089 - acc: 0.9201 - val_loss: 0.3108 - val_acc: 0.9165\n",
      "Epoch 28/200\n",
      "60000/60000 [==============================] - 1s 13us/step - loss: 0.3060 - acc: 0.9201 - val_loss: 0.3110 - val_acc: 0.9168\n",
      "Epoch 29/200\n",
      "60000/60000 [==============================] - 1s 13us/step - loss: 0.3028 - acc: 0.9206 - val_loss: 0.3077 - val_acc: 0.9159\n",
      "Epoch 30/200\n",
      "60000/60000 [==============================] - 1s 13us/step - loss: 0.2989 - acc: 0.9216 - val_loss: 0.3066 - val_acc: 0.9175\n",
      "Epoch 31/200\n",
      "60000/60000 [==============================] - 1s 13us/step - loss: 0.2974 - acc: 0.9210 - val_loss: 0.3020 - val_acc: 0.9188\n",
      "Epoch 32/200\n",
      "60000/60000 [==============================] - 1s 12us/step - loss: 0.2937 - acc: 0.9217 - val_loss: 0.3012 - val_acc: 0.9170\n",
      "Epoch 33/200\n",
      "60000/60000 [==============================] - 1s 12us/step - loss: 0.2917 - acc: 0.9224 - val_loss: 0.3023 - val_acc: 0.9175\n",
      "Epoch 34/200\n",
      "60000/60000 [==============================] - 1s 18us/step - loss: 0.2887 - acc: 0.9230 - val_loss: 0.2986 - val_acc: 0.9192\n",
      "Epoch 35/200\n",
      "60000/60000 [==============================] - 1s 18us/step - loss: 0.2873 - acc: 0.9229 - val_loss: 0.2984 - val_acc: 0.9175\n",
      "Epoch 36/200\n",
      "60000/60000 [==============================] - 1s 15us/step - loss: 0.2849 - acc: 0.9244 - val_loss: 0.2954 - val_acc: 0.9179\n",
      "Epoch 37/200\n",
      "60000/60000 [==============================] - 1s 15us/step - loss: 0.2839 - acc: 0.9236 - val_loss: 0.2951 - val_acc: 0.9180\n",
      "Epoch 38/200\n",
      "60000/60000 [==============================] - 1s 14us/step - loss: 0.2818 - acc: 0.9252 - val_loss: 0.2949 - val_acc: 0.9187\n",
      "Epoch 39/200\n",
      "60000/60000 [==============================] - 1s 15us/step - loss: 0.2797 - acc: 0.9252 - val_loss: 0.2917 - val_acc: 0.9204\n",
      "Epoch 40/200\n",
      "60000/60000 [==============================] - 1s 15us/step - loss: 0.2777 - acc: 0.9251 - val_loss: 0.2912 - val_acc: 0.9198\n",
      "Epoch 41/200\n",
      "60000/60000 [==============================] - 1s 18us/step - loss: 0.2760 - acc: 0.9267 - val_loss: 0.2913 - val_acc: 0.9195\n",
      "Epoch 42/200\n",
      "60000/60000 [==============================] - 1s 16us/step - loss: 0.2745 - acc: 0.9261 - val_loss: 0.2905 - val_acc: 0.9200\n",
      "Epoch 43/200\n",
      "60000/60000 [==============================] - 1s 15us/step - loss: 0.2727 - acc: 0.9263 - val_loss: 0.2873 - val_acc: 0.9211\n",
      "Epoch 44/200\n",
      "60000/60000 [==============================] - 1s 12us/step - loss: 0.2715 - acc: 0.9257 - val_loss: 0.2890 - val_acc: 0.9193\n",
      "Epoch 45/200\n",
      "60000/60000 [==============================] - 1s 12us/step - loss: 0.2698 - acc: 0.9266 - val_loss: 0.2840 - val_acc: 0.9218\n",
      "Epoch 46/200\n",
      "60000/60000 [==============================] - 1s 14us/step - loss: 0.2691 - acc: 0.9267 - val_loss: 0.2867 - val_acc: 0.9204\n",
      "Epoch 47/200\n",
      "60000/60000 [==============================] - 1s 12us/step - loss: 0.2682 - acc: 0.9266 - val_loss: 0.2836 - val_acc: 0.9214\n",
      "Epoch 48/200\n",
      "60000/60000 [==============================] - 1s 13us/step - loss: 0.2659 - acc: 0.9281 - val_loss: 0.2868 - val_acc: 0.9219\n",
      "Epoch 49/200\n",
      "60000/60000 [==============================] - 1s 15us/step - loss: 0.2653 - acc: 0.9280 - val_loss: 0.2907 - val_acc: 0.9195\n",
      "Epoch 50/200\n",
      "60000/60000 [==============================] - 1s 13us/step - loss: 0.2649 - acc: 0.9277 - val_loss: 0.2887 - val_acc: 0.9201\n",
      "Epoch 51/200\n",
      "60000/60000 [==============================] - 1s 12us/step - loss: 0.2632 - acc: 0.9279 - val_loss: 0.2880 - val_acc: 0.9191\n",
      "Epoch 52/200\n",
      "60000/60000 [==============================] - 1s 15us/step - loss: 0.2624 - acc: 0.9277 - val_loss: 0.2836 - val_acc: 0.9214\n",
      "Epoch 53/200\n",
      "60000/60000 [==============================] - 1s 13us/step - loss: 0.2606 - acc: 0.9297 - val_loss: 0.2861 - val_acc: 0.9221\n",
      "Epoch 54/200\n",
      "60000/60000 [==============================] - 1s 18us/step - loss: 0.2599 - acc: 0.9294 - val_loss: 0.2840 - val_acc: 0.9217\n",
      "Epoch 55/200\n",
      "60000/60000 [==============================] - 1s 15us/step - loss: 0.2595 - acc: 0.9282 - val_loss: 0.2818 - val_acc: 0.9223\n",
      "Epoch 56/200\n",
      "60000/60000 [==============================] - 1s 18us/step - loss: 0.2586 - acc: 0.9287 - val_loss: 0.2834 - val_acc: 0.9207\n",
      "Epoch 57/200\n",
      "60000/60000 [==============================] - 1s 17us/step - loss: 0.2575 - acc: 0.9285 - val_loss: 0.2853 - val_acc: 0.9208\n",
      "Epoch 58/200\n",
      "60000/60000 [==============================] - 1s 16us/step - loss: 0.2569 - acc: 0.9296 - val_loss: 0.2832 - val_acc: 0.9216\n",
      "Epoch 59/200\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "60000/60000 [==============================] - 1s 14us/step - loss: 0.2557 - acc: 0.9301 - val_loss: 0.2822 - val_acc: 0.9223\n",
      "Epoch 60/200\n",
      "60000/60000 [==============================] - 1s 16us/step - loss: 0.2561 - acc: 0.9300 - val_loss: 0.2816 - val_acc: 0.9230\n",
      "Epoch 61/200\n",
      "60000/60000 [==============================] - 1s 14us/step - loss: 0.2555 - acc: 0.9295 - val_loss: 0.2811 - val_acc: 0.9222\n",
      "Epoch 62/200\n",
      "60000/60000 [==============================] - 1s 14us/step - loss: 0.2541 - acc: 0.9304 - val_loss: 0.2803 - val_acc: 0.9221\n",
      "Epoch 63/200\n",
      "60000/60000 [==============================] - 1s 15us/step - loss: 0.2530 - acc: 0.9305 - val_loss: 0.2808 - val_acc: 0.9244\n",
      "Epoch 64/200\n",
      "60000/60000 [==============================] - 1s 15us/step - loss: 0.2534 - acc: 0.9297 - val_loss: 0.2823 - val_acc: 0.9207\n",
      "Epoch 65/200\n",
      "60000/60000 [==============================] - 1s 13us/step - loss: 0.2530 - acc: 0.9305 - val_loss: 0.2858 - val_acc: 0.9206\n",
      "Epoch 66/200\n",
      "60000/60000 [==============================] - 1s 14us/step - loss: 0.2522 - acc: 0.9307 - val_loss: 0.2804 - val_acc: 0.9225\n",
      "Epoch 67/200\n",
      "60000/60000 [==============================] - 1s 13us/step - loss: 0.2510 - acc: 0.9309 - val_loss: 0.2809 - val_acc: 0.9246\n",
      "Epoch 68/200\n",
      "60000/60000 [==============================] - 1s 12us/step - loss: 0.2500 - acc: 0.9307 - val_loss: 0.2799 - val_acc: 0.9217\n",
      "Epoch 69/200\n",
      "60000/60000 [==============================] - 1s 13us/step - loss: 0.2498 - acc: 0.9305 - val_loss: 0.2821 - val_acc: 0.9231\n",
      "Epoch 70/200\n",
      "60000/60000 [==============================] - 1s 12us/step - loss: 0.2504 - acc: 0.9300 - val_loss: 0.2796 - val_acc: 0.9231\n",
      "Epoch 71/200\n",
      "60000/60000 [==============================] - 1s 12us/step - loss: 0.2491 - acc: 0.9312 - val_loss: 0.2868 - val_acc: 0.9209\n",
      "Epoch 72/200\n",
      "60000/60000 [==============================] - 1s 12us/step - loss: 0.2489 - acc: 0.9309 - val_loss: 0.2814 - val_acc: 0.9223\n",
      "Epoch 73/200\n",
      "60000/60000 [==============================] - 1s 16us/step - loss: 0.2481 - acc: 0.9314 - val_loss: 0.2812 - val_acc: 0.9237\n",
      "Epoch 74/200\n",
      "60000/60000 [==============================] - 1s 16us/step - loss: 0.2472 - acc: 0.9319 - val_loss: 0.2812 - val_acc: 0.9230\n",
      "Epoch 75/200\n",
      "60000/60000 [==============================] - 1s 16us/step - loss: 0.2477 - acc: 0.9320 - val_loss: 0.2792 - val_acc: 0.9231\n",
      "Epoch 76/200\n",
      "60000/60000 [==============================] - 1s 16us/step - loss: 0.2468 - acc: 0.9313 - val_loss: 0.2895 - val_acc: 0.9192\n",
      "Epoch 77/200\n",
      "60000/60000 [==============================] - 1s 15us/step - loss: 0.2459 - acc: 0.9327 - val_loss: 0.2830 - val_acc: 0.9220\n",
      "Epoch 78/200\n",
      "60000/60000 [==============================] - 1s 15us/step - loss: 0.2462 - acc: 0.9315 - val_loss: 0.2840 - val_acc: 0.9223\n",
      "Epoch 79/200\n",
      "60000/60000 [==============================] - 1s 15us/step - loss: 0.2461 - acc: 0.9318 - val_loss: 0.2822 - val_acc: 0.9239\n",
      "Epoch 80/200\n",
      "60000/60000 [==============================] - 1s 15us/step - loss: 0.2451 - acc: 0.9321 - val_loss: 0.2835 - val_acc: 0.9231\n",
      "Epoch 81/200\n",
      "60000/60000 [==============================] - 1s 16us/step - loss: 0.2442 - acc: 0.9319 - val_loss: 0.2796 - val_acc: 0.9238\n",
      "Epoch 82/200\n",
      "60000/60000 [==============================] - 1s 15us/step - loss: 0.2445 - acc: 0.9325 - val_loss: 0.2834 - val_acc: 0.9225\n",
      "Epoch 83/200\n",
      "60000/60000 [==============================] - 1s 15us/step - loss: 0.2444 - acc: 0.9323 - val_loss: 0.2824 - val_acc: 0.9220\n",
      "Epoch 84/200\n",
      "60000/60000 [==============================] - 1s 15us/step - loss: 0.2433 - acc: 0.9326 - val_loss: 0.2910 - val_acc: 0.9215\n",
      "Epoch 85/200\n",
      "60000/60000 [==============================] - 1s 15us/step - loss: 0.2440 - acc: 0.9321 - val_loss: 0.2849 - val_acc: 0.9213\n",
      "Epoch 86/200\n",
      "60000/60000 [==============================] - 1s 15us/step - loss: 0.2431 - acc: 0.9320 - val_loss: 0.2833 - val_acc: 0.9234\n",
      "Epoch 87/200\n",
      "60000/60000 [==============================] - 1s 15us/step - loss: 0.2433 - acc: 0.9317 - val_loss: 0.2798 - val_acc: 0.9250\n",
      "Epoch 88/200\n",
      "60000/60000 [==============================] - 1s 14us/step - loss: 0.2424 - acc: 0.9333 - val_loss: 0.2855 - val_acc: 0.9226\n",
      "Epoch 89/200\n",
      "60000/60000 [==============================] - 1s 15us/step - loss: 0.2416 - acc: 0.9326 - val_loss: 0.2817 - val_acc: 0.9238\n",
      "Epoch 90/200\n",
      "60000/60000 [==============================] - 1s 15us/step - loss: 0.2417 - acc: 0.9334 - val_loss: 0.2869 - val_acc: 0.9204\n",
      "Epoch 91/200\n",
      "60000/60000 [==============================] - 1s 15us/step - loss: 0.2414 - acc: 0.9332 - val_loss: 0.2814 - val_acc: 0.9216\n",
      "Epoch 92/200\n",
      "60000/60000 [==============================] - 1s 15us/step - loss: 0.2411 - acc: 0.9326 - val_loss: 0.2806 - val_acc: 0.9236\n",
      "Epoch 93/200\n",
      "60000/60000 [==============================] - 1s 15us/step - loss: 0.2418 - acc: 0.9327 - val_loss: 0.2815 - val_acc: 0.9225\n",
      "Epoch 94/200\n",
      "60000/60000 [==============================] - 1s 15us/step - loss: 0.2401 - acc: 0.9329 - val_loss: 0.2835 - val_acc: 0.9234\n",
      "Epoch 95/200\n",
      "60000/60000 [==============================] - 1s 15us/step - loss: 0.2402 - acc: 0.9334 - val_loss: 0.2795 - val_acc: 0.9239\n",
      "Epoch 96/200\n",
      "60000/60000 [==============================] - 1s 15us/step - loss: 0.2406 - acc: 0.9334 - val_loss: 0.2860 - val_acc: 0.9226\n",
      "Epoch 97/200\n",
      "60000/60000 [==============================] - 1s 15us/step - loss: 0.2400 - acc: 0.9332 - val_loss: 0.2841 - val_acc: 0.9229\n",
      "\n",
      "Epoch 00097: ReduceLROnPlateau reducing learning rate to 0.00010000000474974513.\n",
      "Epoch 98/200\n",
      "60000/60000 [==============================] - 1s 14us/step - loss: 0.2352 - acc: 0.9347 - val_loss: 0.2785 - val_acc: 0.9249\n",
      "Epoch 99/200\n",
      "60000/60000 [==============================] - 1s 15us/step - loss: 0.2332 - acc: 0.9356 - val_loss: 0.2783 - val_acc: 0.9247\n",
      "Epoch 100/200\n",
      "60000/60000 [==============================] - 1s 15us/step - loss: 0.2330 - acc: 0.9354 - val_loss: 0.2781 - val_acc: 0.9245\n",
      "Epoch 101/200\n",
      "60000/60000 [==============================] - 1s 17us/step - loss: 0.2331 - acc: 0.9355 - val_loss: 0.2783 - val_acc: 0.9247\n",
      "Epoch 102/200\n",
      "60000/60000 [==============================] - 1s 18us/step - loss: 0.2330 - acc: 0.9356 - val_loss: 0.2782 - val_acc: 0.9245\n",
      "Epoch 103/200\n",
      "60000/60000 [==============================] - 1s 15us/step - loss: 0.2328 - acc: 0.9361 - val_loss: 0.2782 - val_acc: 0.9244\n",
      "Epoch 104/200\n",
      "60000/60000 [==============================] - 1s 18us/step - loss: 0.2333 - acc: 0.9355 - val_loss: 0.2783 - val_acc: 0.9244\n",
      "Epoch 105/200\n",
      "60000/60000 [==============================] - 1s 17us/step - loss: 0.2328 - acc: 0.9357 - val_loss: 0.2787 - val_acc: 0.9247\n",
      "Epoch 106/200\n",
      "60000/60000 [==============================] - 1s 17us/step - loss: 0.2325 - acc: 0.9345 - val_loss: 0.2786 - val_acc: 0.9248\n",
      "Epoch 107/200\n",
      "60000/60000 [==============================] - 1s 17us/step - loss: 0.2316 - acc: 0.9356 - val_loss: 0.2790 - val_acc: 0.9247\n",
      "Epoch 108/200\n",
      "60000/60000 [==============================] - 1s 19us/step - loss: 0.2334 - acc: 0.9352 - val_loss: 0.2785 - val_acc: 0.9249\n",
      "Epoch 109/200\n",
      "60000/60000 [==============================] - 1s 16us/step - loss: 0.2320 - acc: 0.9358 - val_loss: 0.2791 - val_acc: 0.9248\n",
      "Epoch 110/200\n",
      "60000/60000 [==============================] - 1s 16us/step - loss: 0.2326 - acc: 0.9354 - val_loss: 0.2791 - val_acc: 0.9242\n",
      "\n",
      "Epoch 00110: ReduceLROnPlateau reducing learning rate to 1.0000000474974514e-05.\n",
      "Epoch 111/200\n",
      "60000/60000 [==============================] - 1s 15us/step - loss: 0.2322 - acc: 0.9364 - val_loss: 0.2791 - val_acc: 0.9242\n",
      "Epoch 112/200\n",
      "60000/60000 [==============================] - 1s 18us/step - loss: 0.2317 - acc: 0.9354 - val_loss: 0.2790 - val_acc: 0.9248\n",
      "Epoch 113/200\n",
      "60000/60000 [==============================] - 1s 17us/step - loss: 0.2317 - acc: 0.9362 - val_loss: 0.2791 - val_acc: 0.9247\n",
      "\n",
      "Epoch 00113: ReduceLROnPlateau reducing learning rate to 1.0000000656873453e-06.\n",
      "Epoch 114/200\n",
      "60000/60000 [==============================] - 1s 15us/step - loss: 0.2320 - acc: 0.9352 - val_loss: 0.2790 - val_acc: 0.9247\n",
      "Epoch 115/200\n",
      "60000/60000 [==============================] - 1s 13us/step - loss: 0.2311 - acc: 0.9356 - val_loss: 0.2791 - val_acc: 0.9246\n",
      "Epoch 116/200\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "60000/60000 [==============================] - 1s 17us/step - loss: 0.2315 - acc: 0.9367 - val_loss: 0.2790 - val_acc: 0.9243\n",
      "Epoch 117/200\n",
      "60000/60000 [==============================] - 1s 15us/step - loss: 0.2305 - acc: 0.9360 - val_loss: 0.2790 - val_acc: 0.9248\n",
      "Epoch 118/200\n",
      "60000/60000 [==============================] - 1s 18us/step - loss: 0.2306 - acc: 0.9362 - val_loss: 0.2790 - val_acc: 0.9247\n",
      "Epoch 119/200\n",
      "60000/60000 [==============================] - 1s 16us/step - loss: 0.2315 - acc: 0.9353 - val_loss: 0.2790 - val_acc: 0.9248\n",
      "Epoch 120/200\n",
      "60000/60000 [==============================] - 1s 15us/step - loss: 0.2318 - acc: 0.9366 - val_loss: 0.2790 - val_acc: 0.9244\n",
      "\n",
      "Epoch 00120: ReduceLROnPlateau reducing learning rate to 1.0000001111620805e-07.\n",
      "Epoch 121/200\n",
      "60000/60000 [==============================] - 1s 15us/step - loss: 0.2317 - acc: 0.9362 - val_loss: 0.2790 - val_acc: 0.9243\n",
      "Epoch 122/200\n",
      "60000/60000 [==============================] - 1s 15us/step - loss: 0.2329 - acc: 0.9360 - val_loss: 0.2790 - val_acc: 0.9246\n",
      "Epoch 123/200\n",
      "60000/60000 [==============================] - 1s 15us/step - loss: 0.2314 - acc: 0.9354 - val_loss: 0.2790 - val_acc: 0.9245\n",
      "\n",
      "Epoch 00123: ReduceLROnPlateau reducing learning rate to 1.000000082740371e-08.\n",
      "Epoch 124/200\n",
      "60000/60000 [==============================] - 1s 15us/step - loss: 0.2315 - acc: 0.9355 - val_loss: 0.2791 - val_acc: 0.9244\n",
      "Epoch 125/200\n",
      "60000/60000 [==============================] - 1s 15us/step - loss: 0.2313 - acc: 0.9357 - val_loss: 0.2791 - val_acc: 0.9243\n",
      "Epoch 126/200\n",
      "60000/60000 [==============================] - 1s 15us/step - loss: 0.2310 - acc: 0.9358 - val_loss: 0.2791 - val_acc: 0.9246\n",
      "\n",
      "Epoch 00126: ReduceLROnPlateau reducing learning rate to 1.000000082740371e-09.\n",
      "Epoch 127/200\n",
      "60000/60000 [==============================] - 1s 15us/step - loss: 0.2315 - acc: 0.9360 - val_loss: 0.2790 - val_acc: 0.9245\n",
      "Epoch 00127: early stopping\n",
      "CPU times: user 2min 12s, sys: 9.09 s, total: 2min 21s\n",
      "Wall time: 1min 55s\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0x7f0c1c1342e8>"
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%time\n",
    "M_baseline.fit(X_train, Y_train, validation_data=(X_test, Y_test), batch_size=512, epochs=200, callbacks=[\n",
    "    keras.callbacks.ReduceLROnPlateau('loss', patience=3, verbose=1),\n",
    "    keras.callbacks.EarlyStopping('loss', patience=10, verbose=1)\n",
    "])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### compressed test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_23 (InputLayer)        (None, 28, 28)            0         \n",
      "_________________________________________________________________\n",
      "reshape_23 (Reshape)         (None, 784)               0         \n",
      "_________________________________________________________________\n",
      "compressed_dense_22 (Compres (None, 10)                4106      \n",
      "_________________________________________________________________\n",
      "batch_normalization_32 (Batc (None, 10)                40        \n",
      "_________________________________________________________________\n",
      "activation_19 (Activation)   (None, 10)                0         \n",
      "=================================================================\n",
      "Total params: 4,146\n",
      "Trainable params: 4,126\n",
      "Non-trainable params: 20\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "X = X_input = keras.layers.Input((28,28))\n",
    "X = keras.layers.Reshape((28*28,))(X)\n",
    "X = CompressedDense(10, freqs=4096)(X)\n",
    "X = keras.layers.BatchNormalization()(X)\n",
    "X = keras.layers.Activation('softmax')(X)\n",
    "M = keras.Model(X_input, X)\n",
    "M.compile('rmsprop', 'sparse_categorical_crossentropy', ['acc'])\n",
    "M.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 60000 samples, validate on 10000 samples\n",
      "Epoch 1/200\n",
      "60000/60000 [==============================] - 2s 29us/step - loss: 1.4483 - acc: 0.6155 - val_loss: 1.1959 - val_acc: 0.6996\n",
      "Epoch 2/200\n",
      "60000/60000 [==============================] - 1s 16us/step - loss: 1.2084 - acc: 0.7087 - val_loss: 1.1052 - val_acc: 0.7155\n",
      "Epoch 3/200\n",
      "60000/60000 [==============================] - 1s 15us/step - loss: 1.1380 - acc: 0.7224 - val_loss: 1.0655 - val_acc: 0.7373\n",
      "Epoch 4/200\n",
      "60000/60000 [==============================] - 1s 16us/step - loss: 1.0789 - acc: 0.7384 - val_loss: 1.0193 - val_acc: 0.7330\n",
      "Epoch 5/200\n",
      "60000/60000 [==============================] - 1s 16us/step - loss: 1.0257 - acc: 0.7553 - val_loss: 0.9698 - val_acc: 0.7615\n",
      "Epoch 6/200\n",
      "60000/60000 [==============================] - 1s 14us/step - loss: 0.9770 - acc: 0.7686 - val_loss: 0.9235 - val_acc: 0.7814\n",
      "Epoch 7/200\n",
      "60000/60000 [==============================] - 1s 14us/step - loss: 0.9320 - acc: 0.7786 - val_loss: 0.8892 - val_acc: 0.7849\n",
      "Epoch 8/200\n",
      "60000/60000 [==============================] - 1s 14us/step - loss: 0.8924 - acc: 0.7850 - val_loss: 0.8578 - val_acc: 0.7902\n",
      "Epoch 9/200\n",
      "60000/60000 [==============================] - 1s 14us/step - loss: 0.8575 - acc: 0.7910 - val_loss: 0.8207 - val_acc: 0.7961\n",
      "Epoch 10/200\n",
      "60000/60000 [==============================] - 1s 14us/step - loss: 0.8258 - acc: 0.7947 - val_loss: 0.7939 - val_acc: 0.8008\n",
      "Epoch 11/200\n",
      "60000/60000 [==============================] - 1s 14us/step - loss: 0.7989 - acc: 0.7988 - val_loss: 0.7774 - val_acc: 0.7991\n",
      "Epoch 12/200\n",
      "60000/60000 [==============================] - 1s 14us/step - loss: 0.7751 - acc: 0.8011 - val_loss: 0.7522 - val_acc: 0.8030\n",
      "Epoch 13/200\n",
      "60000/60000 [==============================] - 1s 14us/step - loss: 0.7538 - acc: 0.8026 - val_loss: 0.7438 - val_acc: 0.7998\n",
      "Epoch 14/200\n",
      "60000/60000 [==============================] - 1s 14us/step - loss: 0.7352 - acc: 0.8051 - val_loss: 0.7140 - val_acc: 0.8058\n",
      "Epoch 15/200\n",
      "60000/60000 [==============================] - 1s 14us/step - loss: 0.7193 - acc: 0.8063 - val_loss: 0.7150 - val_acc: 0.8053\n",
      "Epoch 16/200\n",
      "60000/60000 [==============================] - 1s 14us/step - loss: 0.7044 - acc: 0.8090 - val_loss: 0.6898 - val_acc: 0.8096\n",
      "Epoch 17/200\n",
      "60000/60000 [==============================] - 1s 14us/step - loss: 0.6911 - acc: 0.8104 - val_loss: 0.6780 - val_acc: 0.8122\n",
      "Epoch 18/200\n",
      "60000/60000 [==============================] - 1s 14us/step - loss: 0.6796 - acc: 0.8129 - val_loss: 0.6699 - val_acc: 0.8129\n",
      "Epoch 19/200\n",
      "60000/60000 [==============================] - 1s 14us/step - loss: 0.6687 - acc: 0.8144 - val_loss: 0.6639 - val_acc: 0.8126\n",
      "Epoch 20/200\n",
      "60000/60000 [==============================] - 1s 14us/step - loss: 0.6606 - acc: 0.8162 - val_loss: 0.6494 - val_acc: 0.8150\n",
      "Epoch 21/200\n",
      "60000/60000 [==============================] - 1s 14us/step - loss: 0.6518 - acc: 0.8180 - val_loss: 0.6532 - val_acc: 0.8131\n",
      "Epoch 22/200\n",
      "60000/60000 [==============================] - 1s 14us/step - loss: 0.6437 - acc: 0.8189 - val_loss: 0.6378 - val_acc: 0.8160\n",
      "Epoch 23/200\n",
      "60000/60000 [==============================] - 1s 14us/step - loss: 0.6359 - acc: 0.8209 - val_loss: 0.6397 - val_acc: 0.8160\n",
      "Epoch 24/200\n",
      "60000/60000 [==============================] - 1s 14us/step - loss: 0.6307 - acc: 0.8215 - val_loss: 0.6253 - val_acc: 0.8200\n",
      "Epoch 25/200\n",
      "60000/60000 [==============================] - 1s 14us/step - loss: 0.6236 - acc: 0.8234 - val_loss: 0.6333 - val_acc: 0.8180\n",
      "Epoch 26/200\n",
      "60000/60000 [==============================] - 1s 14us/step - loss: 0.6189 - acc: 0.8250 - val_loss: 0.6164 - val_acc: 0.8216\n",
      "Epoch 27/200\n",
      "60000/60000 [==============================] - 1s 14us/step - loss: 0.6120 - acc: 0.8259 - val_loss: 0.6190 - val_acc: 0.8215\n",
      "Epoch 28/200\n",
      "60000/60000 [==============================] - 1s 14us/step - loss: 0.6086 - acc: 0.8266 - val_loss: 0.6149 - val_acc: 0.8184\n",
      "Epoch 29/200\n",
      "60000/60000 [==============================] - 1s 14us/step - loss: 0.6035 - acc: 0.8280 - val_loss: 0.6024 - val_acc: 0.8257\n",
      "Epoch 30/200\n",
      "60000/60000 [==============================] - 1s 14us/step - loss: 0.6002 - acc: 0.8280 - val_loss: 0.6034 - val_acc: 0.8271\n",
      "Epoch 31/200\n",
      "60000/60000 [==============================] - 1s 14us/step - loss: 0.5945 - acc: 0.8307 - val_loss: 0.5996 - val_acc: 0.8253\n",
      "Epoch 32/200\n",
      "60000/60000 [==============================] - 1s 14us/step - loss: 0.5905 - acc: 0.8331 - val_loss: 0.5958 - val_acc: 0.8276\n",
      "Epoch 33/200\n",
      "60000/60000 [==============================] - 1s 14us/step - loss: 0.5874 - acc: 0.8324 - val_loss: 0.6030 - val_acc: 0.8241\n",
      "Epoch 34/200\n",
      "60000/60000 [==============================] - 1s 14us/step - loss: 0.5837 - acc: 0.8337 - val_loss: 0.5891 - val_acc: 0.8303\n",
      "Epoch 35/200\n",
      "60000/60000 [==============================] - 1s 14us/step - loss: 0.5798 - acc: 0.8358 - val_loss: 0.5875 - val_acc: 0.8321\n",
      "Epoch 36/200\n",
      "60000/60000 [==============================] - 1s 14us/step - loss: 0.5767 - acc: 0.8347 - val_loss: 0.5839 - val_acc: 0.8304\n",
      "Epoch 37/200\n",
      "60000/60000 [==============================] - 1s 14us/step - loss: 0.5745 - acc: 0.8372 - val_loss: 0.5781 - val_acc: 0.8311\n",
      "Epoch 38/200\n",
      "60000/60000 [==============================] - 1s 14us/step - loss: 0.5711 - acc: 0.8375 - val_loss: 0.5766 - val_acc: 0.8323\n",
      "Epoch 39/200\n",
      "60000/60000 [==============================] - 1s 14us/step - loss: 0.5681 - acc: 0.8382 - val_loss: 0.5777 - val_acc: 0.8335\n",
      "Epoch 40/200\n",
      "60000/60000 [==============================] - 1s 14us/step - loss: 0.5658 - acc: 0.8385 - val_loss: 0.5788 - val_acc: 0.8333\n",
      "Epoch 41/200\n",
      "60000/60000 [==============================] - 1s 14us/step - loss: 0.5636 - acc: 0.8385 - val_loss: 0.5730 - val_acc: 0.8342\n",
      "Epoch 42/200\n",
      "60000/60000 [==============================] - 1s 14us/step - loss: 0.5612 - acc: 0.8404 - val_loss: 0.5714 - val_acc: 0.8344\n",
      "Epoch 43/200\n",
      "60000/60000 [==============================] - 1s 14us/step - loss: 0.5585 - acc: 0.8411 - val_loss: 0.5697 - val_acc: 0.8333\n",
      "Epoch 44/200\n",
      "60000/60000 [==============================] - 1s 14us/step - loss: 0.5579 - acc: 0.8401 - val_loss: 0.5655 - val_acc: 0.8379\n",
      "Epoch 45/200\n",
      "60000/60000 [==============================] - 1s 14us/step - loss: 0.5559 - acc: 0.8404 - val_loss: 0.5698 - val_acc: 0.8340\n",
      "Epoch 46/200\n",
      "60000/60000 [==============================] - 1s 14us/step - loss: 0.5540 - acc: 0.8412 - val_loss: 0.5675 - val_acc: 0.8364\n",
      "Epoch 47/200\n",
      "60000/60000 [==============================] - 1s 14us/step - loss: 0.5508 - acc: 0.8423 - val_loss: 0.5611 - val_acc: 0.8371\n",
      "Epoch 48/200\n",
      "60000/60000 [==============================] - 1s 15us/step - loss: 0.5497 - acc: 0.8427 - val_loss: 0.5768 - val_acc: 0.8351\n",
      "Epoch 49/200\n",
      "60000/60000 [==============================] - 1s 14us/step - loss: 0.5483 - acc: 0.8437 - val_loss: 0.5632 - val_acc: 0.8339\n",
      "Epoch 50/200\n",
      "60000/60000 [==============================] - 1s 14us/step - loss: 0.5453 - acc: 0.8440 - val_loss: 0.5640 - val_acc: 0.8357\n",
      "Epoch 51/200\n",
      "60000/60000 [==============================] - 1s 14us/step - loss: 0.5445 - acc: 0.8441 - val_loss: 0.5567 - val_acc: 0.8365\n",
      "Epoch 52/200\n",
      "60000/60000 [==============================] - 1s 14us/step - loss: 0.5443 - acc: 0.8436 - val_loss: 0.5611 - val_acc: 0.8403\n",
      "Epoch 53/200\n",
      "60000/60000 [==============================] - 1s 14us/step - loss: 0.5416 - acc: 0.8446 - val_loss: 0.5578 - val_acc: 0.8366\n",
      "Epoch 54/200\n",
      "60000/60000 [==============================] - 1s 14us/step - loss: 0.5419 - acc: 0.8449 - val_loss: 0.5564 - val_acc: 0.8373\n",
      "Epoch 55/200\n",
      "60000/60000 [==============================] - 1s 14us/step - loss: 0.5407 - acc: 0.8452 - val_loss: 0.5572 - val_acc: 0.8369\n",
      "Epoch 56/200\n",
      "60000/60000 [==============================] - 1s 14us/step - loss: 0.5399 - acc: 0.8445 - val_loss: 0.5588 - val_acc: 0.8368\n",
      "Epoch 57/200\n",
      "60000/60000 [==============================] - 1s 14us/step - loss: 0.5376 - acc: 0.8465 - val_loss: 0.5574 - val_acc: 0.8389\n",
      "Epoch 58/200\n",
      "60000/60000 [==============================] - 1s 14us/step - loss: 0.5372 - acc: 0.8467 - val_loss: 0.5562 - val_acc: 0.8335\n",
      "Epoch 59/200\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "60000/60000 [==============================] - 1s 14us/step - loss: 0.5350 - acc: 0.8457 - val_loss: 0.5641 - val_acc: 0.8363\n",
      "Epoch 60/200\n",
      "60000/60000 [==============================] - 1s 14us/step - loss: 0.5347 - acc: 0.8468 - val_loss: 0.5518 - val_acc: 0.8385\n",
      "Epoch 61/200\n",
      "60000/60000 [==============================] - 1s 14us/step - loss: 0.5345 - acc: 0.8473 - val_loss: 0.5541 - val_acc: 0.8385\n",
      "Epoch 62/200\n",
      "60000/60000 [==============================] - 1s 14us/step - loss: 0.5319 - acc: 0.8485 - val_loss: 0.5581 - val_acc: 0.8385\n",
      "Epoch 63/200\n",
      "60000/60000 [==============================] - 1s 14us/step - loss: 0.5322 - acc: 0.8476 - val_loss: 0.5505 - val_acc: 0.8394\n",
      "Epoch 64/200\n",
      "60000/60000 [==============================] - 1s 14us/step - loss: 0.5314 - acc: 0.8478 - val_loss: 0.5485 - val_acc: 0.8422\n",
      "Epoch 65/200\n",
      "60000/60000 [==============================] - 1s 14us/step - loss: 0.5303 - acc: 0.8478 - val_loss: 0.5563 - val_acc: 0.8357\n",
      "Epoch 66/200\n",
      "60000/60000 [==============================] - 1s 14us/step - loss: 0.5303 - acc: 0.8481 - val_loss: 0.5485 - val_acc: 0.8416\n",
      "Epoch 67/200\n",
      "60000/60000 [==============================] - 1s 14us/step - loss: 0.5290 - acc: 0.8483 - val_loss: 0.5521 - val_acc: 0.8407\n",
      "Epoch 68/200\n",
      "60000/60000 [==============================] - 1s 14us/step - loss: 0.5295 - acc: 0.8473 - val_loss: 0.5526 - val_acc: 0.8394\n",
      "Epoch 69/200\n",
      "60000/60000 [==============================] - 1s 14us/step - loss: 0.5277 - acc: 0.8488 - val_loss: 0.5532 - val_acc: 0.8411\n",
      "Epoch 70/200\n",
      "60000/60000 [==============================] - 1s 14us/step - loss: 0.5275 - acc: 0.8487 - val_loss: 0.5480 - val_acc: 0.8432\n",
      "Epoch 71/200\n",
      "60000/60000 [==============================] - 1s 14us/step - loss: 0.5265 - acc: 0.8485 - val_loss: 0.5464 - val_acc: 0.8422\n",
      "Epoch 72/200\n",
      "60000/60000 [==============================] - 1s 14us/step - loss: 0.5269 - acc: 0.8485 - val_loss: 0.5474 - val_acc: 0.8428\n",
      "Epoch 73/200\n",
      "60000/60000 [==============================] - 1s 14us/step - loss: 0.5262 - acc: 0.8482 - val_loss: 0.5489 - val_acc: 0.8436\n",
      "Epoch 74/200\n",
      "60000/60000 [==============================] - 1s 14us/step - loss: 0.5252 - acc: 0.8479 - val_loss: 0.5493 - val_acc: 0.8392\n",
      "Epoch 75/200\n",
      "60000/60000 [==============================] - 1s 14us/step - loss: 0.5237 - acc: 0.8493 - val_loss: 0.5531 - val_acc: 0.8399\n",
      "Epoch 76/200\n",
      "60000/60000 [==============================] - 1s 14us/step - loss: 0.5242 - acc: 0.8497 - val_loss: 0.5467 - val_acc: 0.8435\n",
      "Epoch 77/200\n",
      "60000/60000 [==============================] - 1s 14us/step - loss: 0.5232 - acc: 0.8495 - val_loss: 0.5475 - val_acc: 0.8424\n",
      "Epoch 78/200\n",
      "60000/60000 [==============================] - 1s 14us/step - loss: 0.5227 - acc: 0.8487 - val_loss: 0.5445 - val_acc: 0.8426\n",
      "Epoch 79/200\n",
      "60000/60000 [==============================] - 1s 14us/step - loss: 0.5216 - acc: 0.8501 - val_loss: 0.5498 - val_acc: 0.8386\n",
      "Epoch 80/200\n",
      "60000/60000 [==============================] - 1s 14us/step - loss: 0.5219 - acc: 0.8496 - val_loss: 0.5440 - val_acc: 0.8431\n",
      "Epoch 81/200\n",
      "60000/60000 [==============================] - 1s 14us/step - loss: 0.5213 - acc: 0.8495 - val_loss: 0.5496 - val_acc: 0.8408\n",
      "Epoch 82/200\n",
      "60000/60000 [==============================] - 1s 14us/step - loss: 0.5211 - acc: 0.8493 - val_loss: 0.5451 - val_acc: 0.8439\n",
      "Epoch 83/200\n",
      "60000/60000 [==============================] - 1s 14us/step - loss: 0.5213 - acc: 0.8492 - val_loss: 0.5514 - val_acc: 0.8389\n",
      "Epoch 84/200\n",
      "60000/60000 [==============================] - 1s 14us/step - loss: 0.5196 - acc: 0.8491 - val_loss: 0.5484 - val_acc: 0.8403\n",
      "Epoch 85/200\n",
      "60000/60000 [==============================] - 1s 14us/step - loss: 0.5197 - acc: 0.8495 - val_loss: 0.5447 - val_acc: 0.8437\n",
      "Epoch 86/200\n",
      "60000/60000 [==============================] - 1s 14us/step - loss: 0.5194 - acc: 0.8501 - val_loss: 0.5501 - val_acc: 0.8398\n",
      "Epoch 87/200\n",
      "60000/60000 [==============================] - 1s 14us/step - loss: 0.5192 - acc: 0.8504 - val_loss: 0.5485 - val_acc: 0.8416\n",
      "Epoch 88/200\n",
      "60000/60000 [==============================] - ETA: 0s - loss: 0.5185 - acc: 0.850 - 1s 14us/step - loss: 0.5174 - acc: 0.8504 - val_loss: 0.5464 - val_acc: 0.8417\n",
      "Epoch 89/200\n",
      "60000/60000 [==============================] - 1s 14us/step - loss: 0.5182 - acc: 0.8503 - val_loss: 0.5449 - val_acc: 0.8448\n",
      "Epoch 90/200\n",
      "60000/60000 [==============================] - 1s 14us/step - loss: 0.5173 - acc: 0.8509 - val_loss: 0.5410 - val_acc: 0.8433\n",
      "Epoch 91/200\n",
      "60000/60000 [==============================] - 1s 14us/step - loss: 0.5174 - acc: 0.8504 - val_loss: 0.5473 - val_acc: 0.8424\n",
      "\n",
      "Epoch 00091: ReduceLROnPlateau reducing learning rate to 0.00010000000474974513.\n",
      "Epoch 92/200\n",
      "60000/60000 [==============================] - 1s 14us/step - loss: 0.5100 - acc: 0.8534 - val_loss: 0.5385 - val_acc: 0.8460\n",
      "Epoch 93/200\n",
      "60000/60000 [==============================] - 1s 14us/step - loss: 0.5064 - acc: 0.8542 - val_loss: 0.5392 - val_acc: 0.8456\n",
      "Epoch 94/200\n",
      "60000/60000 [==============================] - 1s 14us/step - loss: 0.5078 - acc: 0.8546 - val_loss: 0.5384 - val_acc: 0.8469\n",
      "Epoch 95/200\n",
      "60000/60000 [==============================] - 1s 14us/step - loss: 0.5066 - acc: 0.8545 - val_loss: 0.5383 - val_acc: 0.8466\n",
      "Epoch 96/200\n",
      "60000/60000 [==============================] - 1s 14us/step - loss: 0.5067 - acc: 0.8547 - val_loss: 0.5383 - val_acc: 0.8464\n",
      "\n",
      "Epoch 00096: ReduceLROnPlateau reducing learning rate to 1.0000000474974514e-05.\n",
      "Epoch 97/200\n",
      "60000/60000 [==============================] - 1s 14us/step - loss: 0.5065 - acc: 0.8539 - val_loss: 0.5383 - val_acc: 0.8464\n",
      "Epoch 98/200\n",
      "60000/60000 [==============================] - 1s 14us/step - loss: 0.5057 - acc: 0.8556 - val_loss: 0.5384 - val_acc: 0.8467\n",
      "Epoch 99/200\n",
      "60000/60000 [==============================] - 1s 14us/step - loss: 0.5050 - acc: 0.8553 - val_loss: 0.5384 - val_acc: 0.8467\n",
      "Epoch 100/200\n",
      "60000/60000 [==============================] - 1s 14us/step - loss: 0.5054 - acc: 0.8554 - val_loss: 0.5384 - val_acc: 0.8466\n",
      "Epoch 101/200\n",
      "60000/60000 [==============================] - 1s 14us/step - loss: 0.5058 - acc: 0.8553 - val_loss: 0.5383 - val_acc: 0.8469\n",
      "Epoch 102/200\n",
      "60000/60000 [==============================] - 1s 14us/step - loss: 0.5052 - acc: 0.8552 - val_loss: 0.5383 - val_acc: 0.8466\n",
      "\n",
      "Epoch 00102: ReduceLROnPlateau reducing learning rate to 1.0000000656873453e-06.\n",
      "Epoch 103/200\n",
      "60000/60000 [==============================] - 1s 14us/step - loss: 0.5053 - acc: 0.8547 - val_loss: 0.5382 - val_acc: 0.8463\n",
      "Epoch 104/200\n",
      "60000/60000 [==============================] - 1s 14us/step - loss: 0.5047 - acc: 0.8547 - val_loss: 0.5383 - val_acc: 0.8466\n",
      "Epoch 105/200\n",
      "60000/60000 [==============================] - 1s 14us/step - loss: 0.5049 - acc: 0.8556 - val_loss: 0.5382 - val_acc: 0.8463\n",
      "Epoch 106/200\n",
      "60000/60000 [==============================] - 1s 14us/step - loss: 0.5062 - acc: 0.8551 - val_loss: 0.5382 - val_acc: 0.8467\n",
      "Epoch 107/200\n",
      "60000/60000 [==============================] - 1s 14us/step - loss: 0.5055 - acc: 0.8558 - val_loss: 0.5383 - val_acc: 0.8469\n",
      "\n",
      "Epoch 00107: ReduceLROnPlateau reducing learning rate to 1.0000001111620805e-07.\n",
      "Epoch 108/200\n",
      "60000/60000 [==============================] - 1s 14us/step - loss: 0.5051 - acc: 0.8551 - val_loss: 0.5382 - val_acc: 0.8466\n",
      "Epoch 109/200\n",
      "60000/60000 [==============================] - 1s 14us/step - loss: 0.5051 - acc: 0.8556 - val_loss: 0.5383 - val_acc: 0.8470\n",
      "Epoch 110/200\n",
      "60000/60000 [==============================] - 1s 14us/step - loss: 0.5054 - acc: 0.8544 - val_loss: 0.5383 - val_acc: 0.8470\n",
      "\n",
      "Epoch 00110: ReduceLROnPlateau reducing learning rate to 1.000000082740371e-08.\n",
      "Epoch 111/200\n",
      "60000/60000 [==============================] - 1s 14us/step - loss: 0.5044 - acc: 0.8547 - val_loss: 0.5383 - val_acc: 0.8465\n",
      "Epoch 112/200\n",
      "60000/60000 [==============================] - 1s 14us/step - loss: 0.5051 - acc: 0.8549 - val_loss: 0.5382 - val_acc: 0.8466\n",
      "Epoch 113/200\n",
      "60000/60000 [==============================] - 1s 14us/step - loss: 0.5047 - acc: 0.8553 - val_loss: 0.5383 - val_acc: 0.8464\n",
      "Epoch 114/200\n",
      "60000/60000 [==============================] - 1s 14us/step - loss: 0.5059 - acc: 0.8553 - val_loss: 0.5383 - val_acc: 0.8466\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Epoch 00114: ReduceLROnPlateau reducing learning rate to 1.000000082740371e-09.\n",
      "Epoch 115/200\n",
      "60000/60000 [==============================] - 1s 14us/step - loss: 0.5059 - acc: 0.8552 - val_loss: 0.5383 - val_acc: 0.8467\n",
      "Epoch 116/200\n",
      "60000/60000 [==============================] - 1s 14us/step - loss: 0.5061 - acc: 0.8555 - val_loss: 0.5383 - val_acc: 0.8465\n",
      "Epoch 117/200\n",
      "60000/60000 [==============================] - 1s 14us/step - loss: 0.5046 - acc: 0.8551 - val_loss: 0.5383 - val_acc: 0.8463\n",
      "\n",
      "Epoch 00117: ReduceLROnPlateau reducing learning rate to 1.000000082740371e-10.\n",
      "Epoch 118/200\n",
      "60000/60000 [==============================] - 1s 14us/step - loss: 0.5058 - acc: 0.8550 - val_loss: 0.5383 - val_acc: 0.8466\n",
      "Epoch 119/200\n",
      "60000/60000 [==============================] - 1s 14us/step - loss: 0.5044 - acc: 0.8558 - val_loss: 0.5382 - val_acc: 0.8466\n",
      "Epoch 120/200\n",
      "60000/60000 [==============================] - 1s 14us/step - loss: 0.5050 - acc: 0.8560 - val_loss: 0.5382 - val_acc: 0.8468\n",
      "\n",
      "Epoch 00120: ReduceLROnPlateau reducing learning rate to 1.000000082740371e-11.\n",
      "Epoch 121/200\n",
      "60000/60000 [==============================] - 1s 14us/step - loss: 0.5057 - acc: 0.8549 - val_loss: 0.5382 - val_acc: 0.8467\n",
      "Epoch 00121: early stopping\n",
      "CPU times: user 1min 53s, sys: 28 s, total: 2min 21s\n",
      "Wall time: 1min 43s\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0x7f0c216448d0>"
      ]
     },
     "execution_count": 54,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%time\n",
    "M.fit(X_train, Y_train, validation_data=(X_test, Y_test), batch_size=512, epochs=200, callbacks=[\n",
    "    keras.callbacks.ReduceLROnPlateau('loss', patience=3, verbose=1),\n",
    "    keras.callbacks.EarlyStopping('loss', patience=10, verbose=1)\n",
    "])"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "ml",
   "language": "python",
   "name": "ml"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}