training/training.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.6/dist-packages/h5py/__init__.py:36: FutureWarning: Conversion of the second argument of issubdtype from `float` to `np.floating` is deprecated. In future, it will be treated as `np.float64 == np.dtype(float).type`.\n",
      "  from ._conv import register_converters as _register_converters\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "import tensorflow as tf\n",
    "import json\n",
    "import os\n",
    "from enum import unique, Enum\n",
    "from multiprocessing import Queue, Process\n",
    "\n",
    "## Deep Neural Network Class\n",
    "# Abstract class which wraps a Keras model, providing common model functionality\n",
    "class DNNModel(object):\n",
    "    def __init__(self, name: str):\n",
    "        self._model = None\n",
    "        self.name = name\n",
    "\n",
    "    ## @brief Fits a Keras model\n",
    "    # @param data Network Input\n",
    "    # @param labels Network Output\n",
    "    # @param epochs Number of training epochs\n",
    "    # @param batch_size Size of training batches\n",
    "    # @param validation_split Amount of data / labels to save for validation\n",
    "    # @param callbacks Keras training callbacks\n",
    "    def train(self, data: np.ndarray, labels: np.ndarray, epochs: int = 1, batch_size: int = 128,\n",
    "              validation_split=0.2, callbacks=None):\n",
    "\n",
    "        self._model.fit(data, labels, epochs=epochs, batch_size=batch_size,\n",
    "                        validation_split=validation_split, callbacks=callbacks)\n",
    "\n",
    "    ## @brief Predict a network output based on network input\n",
    "    # @param samples Network input\n",
    "    # @return Predicted network output\n",
    "    def predict(self, samples) -> np.ndarray:\n",
    "        return self._model.predict(samples)\n",
    "\n",
    "    ## @brief Gets the shape of the network's input and output\n",
    "    # @return Dict containing the keys 'input' and 'output' mapped to the network shape\n",
    "    def shape(self) -> dict:\n",
    "        return {'input': self._model.input_shape, 'output': self._model.output_shape}\n",
    "\n",
    "    ## @brief Returns a string representation of the neural network\n",
    "    # @return String representation of neural network\n",
    "    def summary(self) -> str:\n",
    "        return self._model.summary()\n",
    "\n",
    "    ## @brief Loads a network from a saved weights file\n",
    "    # @param path Path to the .h5 saved weights file\n",
    "    # @return Boolean value representing call success\n",
    "    def load(self, path: str) -> bool:\n",
    "        return self._model.load_weights(path)\n",
    "\n",
    "    ## @brief Saves network weights to a file\n",
    "    # @param path Path to the .h5 weights file to save\n",
    "    # @return Boolean value representing call success\n",
    "    def save(self, path: str) -> bool:\n",
    "        return self._model.save_weights(path)\n",
    "\n",
    "    ## @brief Export and freezes the graph to a protobuff file\n",
    "    # @param path Path to the .pb file\n",
    "    # @return Boolean value representing call success\n",
    "    def export(self, path: str) -> bool:\n",
    "        import tensorflow as tf\n",
    "        import keras.backend as K\n",
    "        from tensorflow.python.framework import graph_io\n",
    "        from tensorflow.python.tools import freeze_graph\n",
    "        from tensorflow.core.protobuf import saver_pb2\n",
    "        from tensorflow.python.training import saver as saver_lib\n",
    "\n",
    "        with tf.Session() as sess:\n",
    "            checkpoints_path = './checkpoints'\n",
    "\n",
    "            sess.run(tf.global_variables_initializer())\n",
    "\n",
    "            K.set_learning_phase(0)\n",
    "            sess = K.get_session()\n",
    "            saver = saver_lib.Saver(write_version=saver_pb2.SaverDef.V2)\n",
    "\n",
    "            if not os.path.exists(checkpoints_path):\n",
    "                os.mkdir(checkpoints_path)\n",
    "\n",
    "            checkpoint_path = saver.save(sess, './checkpoints/%s.ckpt' % self.name, global_step=0)\n",
    "            graph_io.write_graph(sess.graph, '.', 'tmp.pb')\n",
    "\n",
    "            out_names = [node.name for node in tf.get_default_graph().as_graph_def().node]\n",
    "\n",
    "            ret = freeze_graph.freeze_graph('./tmp.pb', '',\n",
    "                                            False, checkpoint_path, ','.join(out_names),\n",
    "                                            \"save/restore_all\", \"save/Const:0\",\n",
    "                                            path, False, \"\")\n",
    "\n",
    "            os.unlink('tmp.pb')\n",
    "            return ret\n",
    "        \n",
    "\n",
    "class MicroFeatureMapModel(DNNModel):\n",
    "    INPUT_CHANNELS = 1\n",
    "    INPUT_RESOLUTION_X = 40\n",
    "    INPUT_RESOLUTION_Y = 30\n",
    "    CONV_DIM = 2\n",
    "    \n",
    "    INPUT_SHAPE = (int(INPUT_RESOLUTION_Y), int(INPUT_RESOLUTION_X), INPUT_CHANNELS)\n",
    "\n",
    "    def __init__(self):\n",
    "\n",
    "        DNNModel.__init__(self, name='MicroFeatureMapModel')\n",
    "        from keras.layers import SeparableConv2D, Input, BatchNormalization, Conv2D, Activation, Dense, MaxPooling2D, AveragePooling2D, Flatten, concatenate, Reshape\n",
    "        from keras.models import Model\n",
    "        from keras.backend import set_session\n",
    "\n",
    "        def mobilenet_layer(input_layer, strides, dim, batchnorm=False):\n",
    "            mn_layer = input_layer\n",
    "            mn_layer = SeparableConv2D(dim, strides)(mn_layer)\n",
    "            if batchnorm:\n",
    "                mn_layer = BatchNormalization()(mn_layer)\n",
    "            mn_layer = Activation('relu')(mn_layer)\n",
    "            mn_layer = Conv2D(dim * 2, (1, 1))(mn_layer)\n",
    "            if batchnorm:\n",
    "                mn_layer = BatchNormalization()(mn_layer)\n",
    "            mn_layer = Activation('relu')(mn_layer)\n",
    "            return mn_layer\n",
    "\n",
    "        branch_1_input = Input(shape=MicroFeatureMapModel.INPUT_SHAPE)\n",
    "        branch_2_input = Input(shape=MicroFeatureMapModel.INPUT_SHAPE)\n",
    "\n",
    "        # Large Features\n",
    "        branch_1 = branch_1_input\n",
    "        branch_1 = mobilenet_layer(branch_1, (6, 6), np.power(MicroFeatureMapModel.CONV_DIM, 1))\n",
    "        branch_1 = mobilenet_layer(branch_1, (4, 4), np.power(MicroFeatureMapModel.CONV_DIM, 2))\n",
    "        branch_1 = mobilenet_layer(branch_1, (2, 2), np.power(MicroFeatureMapModel.CONV_DIM, 3))\n",
    "        \n",
    "        # Small Features\n",
    "        branch_2 = branch_2_input\n",
    "        branch_2 = mobilenet_layer(branch_2, (2, 2), np.power(MicroFeatureMapModel.CONV_DIM, 1))\n",
    "        branch_2 = mobilenet_layer(branch_2, (4, 4), np.power(MicroFeatureMapModel.CONV_DIM, 2))\n",
    "        branch_2 = mobilenet_layer(branch_2, (6, 6), np.power(MicroFeatureMapModel.CONV_DIM, 3))\n",
    "\n",
    "        branch_out = concatenate([branch_1, branch_2])\n",
    "        branch_out = Flatten()(branch_out)\n",
    "\n",
    "        # Dimensionality Reduction / Heat Map\n",
    "        branch_out = Dense(1)(branch_out)\n",
    "\n",
    "        model = Model(inputs=[branch_1_input, branch_2_input], outputs=[branch_out])\n",
    "        model.compile(optimizer='adam', loss='mse', metrics=['mse'])\n",
    "\n",
    "        self._model = model\n",
    "\n",
    "        import tensorflow as tf\n",
    "\n",
    "        config = tf.ConfigProto()\n",
    "        config.gpu_options.allow_growth = True\n",
    "        set_session(tf.Session(config=config))\n",
    "\n",
    "    ## @brief Fits a Keras model\n",
    "    # @param data Network Input\n",
    "    # @param labels Network Output\n",
    "    # @param epochs Number of training epochs\n",
    "    # @param batch_size Size of training batches\n",
    "    # @param validation_split Amount of data / labels to save for validation\n",
    "    def train(self, data: np.ndarray, labels: np.ndarray, epochs: int = 1, batch_size: int = 128,\n",
    "              validation_split=0.2, callbacks=None):\n",
    "\n",
    "        from keras.callbacks import ModelCheckpoint\n",
    "\n",
    "        filepath = \"weights-{epoch:02d}-{val_loss:.4f}.hdf5\"\n",
    "        checkpoint = ModelCheckpoint(filepath, monitor='val_loss', verbose=1, save_best_only=True, mode='min')\n",
    "\n",
    "        DNNModel.train(self, data, labels, epochs, batch_size, validation_split, [checkpoint])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loading Data\n",
      "Converting to ndarrays\n",
      "(63432, 40, 30, 1)\n",
      "(63432, 30, 40, 1)\n",
      "Done\n"
     ]
    }
   ],
   "source": [
    "print(\"Loading Data\")\n",
    "training_data = json.loads(open('training.json', 'r').read())\n",
    "\n",
    "samples = training_data['samples']\n",
    "labels = training_data['labels']\n",
    "\n",
    "print(\"Converting to ndarrays\")\n",
    "samples = np.array(samples)\n",
    "print(samples.shape)\n",
    "samples = samples.reshape(samples.shape[0], samples.shape[2], samples.shape[1], samples.shape[3])\n",
    "print(samples.shape)\n",
    "labels = np.array(labels)\n",
    "\n",
    "print(\"Done\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training\n",
      "Train on 50745 samples, validate on 12687 samples\n",
      "Epoch 1/10\n",
      "50745/50745 [==============================] - 27s 530us/step - loss: 0.0102 - mean_squared_error: 0.0102 - val_loss: 0.0013 - val_mean_squared_error: 0.0013\n",
      "\n",
      "Epoch 00001: val_loss improved from inf to 0.00128, saving model to weights-01-0.00.hdf5\n",
      "Epoch 2/10\n",
      "50745/50745 [==============================] - 26s 503us/step - loss: 0.0081 - mean_squared_error: 0.0081 - val_loss: 0.0020 - val_mean_squared_error: 0.0020\n",
      "\n",
      "Epoch 00002: val_loss did not improve from 0.00128\n",
      "Epoch 3/10\n",
      "50745/50745 [==============================] - 26s 507us/step - loss: 0.0073 - mean_squared_error: 0.0073 - val_loss: 0.0040 - val_mean_squared_error: 0.0040\n",
      "\n",
      "Epoch 00003: val_loss did not improve from 0.00128\n",
      "Epoch 4/10\n",
      "50745/50745 [==============================] - 26s 509us/step - loss: 0.0070 - mean_squared_error: 0.0070 - val_loss: 0.0050 - val_mean_squared_error: 0.0050\n",
      "\n",
      "Epoch 00004: val_loss did not improve from 0.00128\n",
      "Epoch 5/10\n",
      "50745/50745 [==============================] - 27s 541us/step - loss: 0.0067 - mean_squared_error: 0.0067 - val_loss: 0.0019 - val_mean_squared_error: 0.0019\n",
      "\n",
      "Epoch 00005: val_loss did not improve from 0.00128\n",
      "Epoch 6/10\n",
      "50745/50745 [==============================] - 28s 547us/step - loss: 0.0068 - mean_squared_error: 0.0068 - val_loss: 0.0022 - val_mean_squared_error: 0.0022\n",
      "\n",
      "Epoch 00006: val_loss did not improve from 0.00128\n",
      "Epoch 7/10\n",
      "50745/50745 [==============================] - 28s 553us/step - loss: 0.0062 - mean_squared_error: 0.0062 - val_loss: 0.0023 - val_mean_squared_error: 0.0023\n",
      "\n",
      "Epoch 00007: val_loss did not improve from 0.00128\n",
      "Epoch 8/10\n",
      "50745/50745 [==============================] - 28s 555us/step - loss: 0.0060 - mean_squared_error: 0.0060 - val_loss: 0.0026 - val_mean_squared_error: 0.0026\n",
      "\n",
      "Epoch 00008: val_loss did not improve from 0.00128\n",
      "Epoch 9/10\n",
      "50745/50745 [==============================] - 28s 559us/step - loss: 0.0061 - mean_squared_error: 0.0061 - val_loss: 0.0022 - val_mean_squared_error: 0.0022\n",
      "\n",
      "Epoch 00009: val_loss did not improve from 0.00128\n",
      "Epoch 10/10\n",
      "50745/50745 [==============================] - 29s 562us/step - loss: 0.0058 - mean_squared_error: 0.0058 - val_loss: 0.0021 - val_mean_squared_error: 0.0021\n",
      "\n",
      "Epoch 00010: val_loss did not improve from 0.00128\n"
     ]
    }
   ],
   "source": [
    "# Training\n",
    "with tf.Session() as sess:\n",
    "    sess.run(tf.global_variables_initializer())\n",
    "\n",
    "    print(\"Training\")\n",
    "    model = MicroFeatureMapModel()\n",
    "    model.train([samples, samples], labels, epochs=10)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Exporting Graph\n",
      "INFO:tensorflow:Restoring parameters from ./checkpoints/MicroFeatureMapModel.ckpt-0\n",
      "INFO:tensorflow:Froze 37 variables.\n",
      "Converted 37 variables to const ops.\n",
      "Done\n"
     ]
    }
   ],
   "source": [
    "# Export\n",
    "with tf.Session() as sess:\n",
    "    sess.run(tf.global_variables_initializer())\n",
    "    model = MicroFeatureMapModel()\n",
    "\n",
    "    print(\"Exporting Graph\")\n",
    "    \n",
    "    # Export Graph\n",
    "    model.load('weights-01-0.00.hdf5')\n",
    "    model.export('graph.pb')\n",
    "    \n",
    "    print(\"Done\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "input_1 (InputLayer)            (None, 30, 40, 1)    0                                            \n",
      "__________________________________________________________________________________________________\n",
      "input_2 (InputLayer)            (None, 30, 40, 1)    0                                            \n",
      "__________________________________________________________________________________________________\n",
      "separable_conv2d_1 (SeparableCo (None, 25, 35, 2)    40          input_1[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "separable_conv2d_4 (SeparableCo (None, 29, 39, 2)    8           input_2[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "activation_1 (Activation)       (None, 25, 35, 2)    0           separable_conv2d_1[0][0]         \n",
      "__________________________________________________________________________________________________\n",
      "activation_7 (Activation)       (None, 29, 39, 2)    0           separable_conv2d_4[0][0]         \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_1 (Conv2D)               (None, 25, 35, 4)    12          activation_1[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_4 (Conv2D)               (None, 29, 39, 4)    12          activation_7[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "activation_2 (Activation)       (None, 25, 35, 4)    0           conv2d_1[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "activation_8 (Activation)       (None, 29, 39, 4)    0           conv2d_4[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "separable_conv2d_2 (SeparableCo (None, 22, 32, 4)    84          activation_2[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "separable_conv2d_5 (SeparableCo (None, 26, 36, 4)    84          activation_8[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "activation_3 (Activation)       (None, 22, 32, 4)    0           separable_conv2d_2[0][0]         \n",
      "__________________________________________________________________________________________________\n",
      "activation_9 (Activation)       (None, 26, 36, 4)    0           separable_conv2d_5[0][0]         \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_2 (Conv2D)               (None, 22, 32, 8)    40          activation_3[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_5 (Conv2D)               (None, 26, 36, 8)    40          activation_9[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "activation_4 (Activation)       (None, 22, 32, 8)    0           conv2d_2[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "activation_10 (Activation)      (None, 26, 36, 8)    0           conv2d_5[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "separable_conv2d_3 (SeparableCo (None, 21, 31, 8)    104         activation_4[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "separable_conv2d_6 (SeparableCo (None, 21, 31, 8)    360         activation_10[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "activation_5 (Activation)       (None, 21, 31, 8)    0           separable_conv2d_3[0][0]         \n",
      "__________________________________________________________________________________________________\n",
      "activation_11 (Activation)      (None, 21, 31, 8)    0           separable_conv2d_6[0][0]         \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_3 (Conv2D)               (None, 21, 31, 16)   144         activation_5[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_6 (Conv2D)               (None, 21, 31, 16)   144         activation_11[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "activation_6 (Activation)       (None, 21, 31, 16)   0           conv2d_3[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "activation_12 (Activation)      (None, 21, 31, 16)   0           conv2d_6[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_1 (Concatenate)     (None, 21, 31, 32)   0           activation_6[0][0]               \n",
      "                                                                 activation_12[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "flatten_1 (Flatten)             (None, 20832)        0           concatenate_1[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "dense_1 (Dense)                 (None, 1)            20833       flatten_1[0][0]                  \n",
      "==================================================================================================\n",
      "Total params: 21,905\n",
      "Trainable params: 21,905\n",
      "Non-trainable params: 0\n",
      "__________________________________________________________________________________________________\n",
      "None\n"
     ]
    }
   ],
   "source": [
    "with tf.Session() as sess:\n",
    "    sess.run(tf.global_variables_initializer())\n",
    "    model = MicroFeatureMapModel()\n",
    "\n",
    "    print(model.summary())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.02673816680908203\n"
     ]
    }
   ],
   "source": [
    "with tf.Session() as sess:\n",
    "    sess.run(tf.global_variables_initializer())\n",
    "    \n",
    "    model = MicroFeatureMapModel()\n",
    "    model.load('weights-01-0.00.hdf5')\n",
    "\n",
    "    from time import time\n",
    "\n",
    "    i = np.zeros((16*16, 30, 40, 1))\n",
    "    model.predict([i, i])\n",
    "\n",
    "    t_i = time()\n",
    "    model.predict([i, i])\n",
    "    t_f = time()\n",
    "                  \n",
    "    print(t_f - t_i)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}