dynamic-weights.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import keras\n",
    "import keras.backend as K"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "(x_train, y_train), (x_test, y_test) = keras.datasets.mnist.load_data()\n",
    "x_train = x_train[:,:,:,np.newaxis] / 255.\n",
    "x_test = x_test[:,:,:,np.newaxis] / 255."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "input_1 (InputLayer)            (None, 28, 28, 1)    0                                            \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_1 (Conv2D)               (None, 28, 28, 4)    40          input_1[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_1 (BatchNor (None, 28, 28, 4)    16          conv2d_1[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "leaky_re_lu_1 (LeakyReLU)       (None, 28, 28, 4)    0           batch_normalization_1[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "max_pooling2d_1 (MaxPooling2D)  (None, 14, 14, 4)    0           leaky_re_lu_1[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_2 (Conv2D)               (None, 14, 14, 8)    296         max_pooling2d_1[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_2 (BatchNor (None, 14, 14, 8)    32          conv2d_2[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "leaky_re_lu_2 (LeakyReLU)       (None, 14, 14, 8)    0           batch_normalization_2[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "max_pooling2d_2 (MaxPooling2D)  (None, 7, 7, 8)      0           leaky_re_lu_2[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_3 (Conv2D)               (None, 7, 7, 16)     1168        max_pooling2d_2[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_3 (BatchNor (None, 7, 7, 16)     64          conv2d_3[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "leaky_re_lu_3 (LeakyReLU)       (None, 7, 7, 16)     0           batch_normalization_3[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "max_pooling2d_3 (MaxPooling2D)  (None, 3, 3, 16)     0           leaky_re_lu_3[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "flatten_1 (Flatten)             (None, 144)          0           max_pooling2d_3[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "lambda_1 (Lambda)               (1, 1441, 32)        0                                            \n",
      "__________________________________________________________________________________________________\n",
      "lambda_2 (Lambda)               (None, 1441, 32)     0           flatten_1[0][0]                  \n",
      "                                                                 lambda_1[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "lambda_3 (Lambda)               (None, 1441, 176)    0           flatten_1[0][0]                  \n",
      "                                                                 lambda_2[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "time_distributed_1 (TimeDistrib (None, 1441, 1)      177         lambda_3[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "time_distributed_2 (TimeDistrib (None, 1441, 1)      0           time_distributed_1[0][0]         \n",
      "__________________________________________________________________________________________________\n",
      "lambda_4 (Lambda)               (None, 1440, 1)      0           time_distributed_2[0][0]         \n",
      "__________________________________________________________________________________________________\n",
      "lambda_5 (Lambda)               (None, 1, 1)         0           time_distributed_2[0][0]         \n",
      "__________________________________________________________________________________________________\n",
      "reshape_1 (Reshape)             (None, 10, 144)      0           lambda_4[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "reshape_2 (Reshape)             (None, 1)            0           lambda_5[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "lambda_6 (Lambda)               (None, 10)           0           flatten_1[0][0]                  \n",
      "                                                                 reshape_1[0][0]                  \n",
      "                                                                 reshape_2[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "softmax_1 (Softmax)             (None, 10)           0           lambda_6[0][0]                   \n",
      "==================================================================================================\n",
      "Total params: 1,793\n",
      "Trainable params: 1,737\n",
      "Non-trainable params: 56\n",
      "__________________________________________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "L = L0 = keras.layers.Input((28,28,1))\n",
    "L = keras.layers.Conv2D(4, (3,3), padding='same')(L)\n",
    "L = keras.layers.BatchNormalization()(L)\n",
    "L = keras.layers.LeakyReLU()(L)\n",
    "L = keras.layers.MaxPool2D()(L)\n",
    "L = keras.layers.Conv2D(8, (3,3), padding='same')(L)\n",
    "L = keras.layers.BatchNormalization()(L)\n",
    "L = keras.layers.LeakyReLU()(L)\n",
    "L = keras.layers.MaxPool2D()(L)\n",
    "L = keras.layers.Conv2D(16, (3,3), padding='same')(L)\n",
    "L = keras.layers.BatchNormalization()(L)\n",
    "L = keras.layers.LeakyReLU()(L)\n",
    "L = keras.layers.MaxPool2D()(L)\n",
    "L = keras.layers.Flatten()(L)\n",
    "\n",
    "L_dyn = keras.layers.Lambda(lambda _: K.variable(np.random.randn(1, 3*3*16 * 10 + 1, 32)))([])\n",
    "L_dyn = keras.layers.Lambda(lambda x: K.tile(x[1], [K.shape(x[0])[0],1,1]))([L, L_dyn])\n",
    "L_dyn = keras.layers.Lambda(lambda x: K.concatenate([\n",
    "    K.concatenate([\n",
    "        K.repeat(x[0], 3*3*16 * 10),\n",
    "        K.ones([K.shape(x[0])[0],1,3*3*16])], axis=-2),\n",
    "    x[1]]))([L, L_dyn])\n",
    "L_dyn = keras.layers.TimeDistributed(keras.layers.Dense(1))(L_dyn)\n",
    "L_dyn = keras.layers.TimeDistributed(keras.layers.LeakyReLU())(L_dyn)\n",
    "L_dyn_w = keras.layers.Lambda(lambda x: x[:,:-1])(L_dyn)\n",
    "L_dyn_w = keras.layers.Reshape((10, 3*3*16))(L_dyn_w)\n",
    "L_dyn_b = keras.layers.Lambda(lambda x: x[:,-1:])(L_dyn)\n",
    "L_dyn_b = keras.layers.Reshape((1,))(L_dyn_b)\n",
    "\n",
    "L = keras.layers.Lambda(lambda x: K.batch_dot(x[1], x[0]) + x[2])([L, L_dyn_w, L_dyn_b])\n",
    "L = keras.layers.Softmax()(L)\n",
    "\n",
    "M = keras.models.Model([L0], [L])\n",
    "M.compile('adam', 'sparse_categorical_crossentropy', metrics=['accuracy'])\n",
    "M.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 60000 samples, validate on 10000 samples\n",
      "Epoch 1/10\n",
      "60000/60000 [==============================] - 106s 2ms/step - loss: 0.4339 - acc: 0.8685 - val_loss: 0.1663 - val_acc: 0.9469\n",
      "Epoch 2/10\n",
      "60000/60000 [==============================] - 103s 2ms/step - loss: 0.1482 - acc: 0.9553 - val_loss: 0.1243 - val_acc: 0.9608\n",
      "Epoch 3/10\n",
      "60000/60000 [==============================] - 102s 2ms/step - loss: 0.1177 - acc: 0.9630 - val_loss: 0.0991 - val_acc: 0.9681\n",
      "Epoch 4/10\n",
      "60000/60000 [==============================] - 101s 2ms/step - loss: 0.1029 - acc: 0.9684 - val_loss: 0.0876 - val_acc: 0.9717\n",
      "Epoch 5/10\n",
      "60000/60000 [==============================] - 100s 2ms/step - loss: 0.0937 - acc: 0.9714 - val_loss: 0.0900 - val_acc: 0.9693\n",
      "Epoch 6/10\n",
      "60000/60000 [==============================] - 100s 2ms/step - loss: 0.0877 - acc: 0.9728 - val_loss: 0.0756 - val_acc: 0.9761\n",
      "Epoch 7/10\n",
      "60000/60000 [==============================] - 99s 2ms/step - loss: 0.0820 - acc: 0.9741 - val_loss: 0.0773 - val_acc: 0.9739\n",
      "Epoch 8/10\n",
      "60000/60000 [==============================] - 99s 2ms/step - loss: 0.0794 - acc: 0.9751 - val_loss: 0.0870 - val_acc: 0.9731\n",
      "Epoch 9/10\n",
      "60000/60000 [==============================] - 99s 2ms/step - loss: 0.0750 - acc: 0.9766 - val_loss: 0.0854 - val_acc: 0.9751\n",
      "Epoch 10/10\n",
      "60000/60000 [==============================] - 99s 2ms/step - loss: 0.0733 - acc: 0.9771 - val_loss: 0.0754 - val_acc: 0.9755\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0x7fae288ceef0>"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "M.fit(x_train, y_train, validation_data=(x_test, y_test), epochs=10)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "ml",
   "language": "python",
   "name": "ml"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}