DeepSense_TFLite_Decompose.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "considerable-driving",
   "metadata": {},
   "source": [
    "In order to run DeepSense on an embedded system with proper HW-SW Co-design, Deepsense is decomposed into 3 main sub-models:\n",
    "The CNN, the RNN and the FC"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "above-council",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2.3.0-rc0\n"
     ]
    }
   ],
   "source": [
    "import tensorflow as tf \n",
    "from tensorflow import keras\n",
    "from tensorflow.keras import layers\n",
    "import numpy as np\n",
    "from functools import partial\n",
    "\n",
    "import time\n",
    "import pathlib\n",
    "import math\n",
    "import os\n",
    "import sys\n",
    "print(tf.__version__)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "employed-crime",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"DeepSense\"\n",
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "I1 (InputLayer)                 [(None, 20, 60, 1)]  0                                            \n",
      "__________________________________________________________________________________________________\n",
      "I2 (InputLayer)                 [(None, 20, 60, 1)]  0                                            \n",
      "__________________________________________________________________________________________________\n",
      "I1_conv1 (Conv2D)               (None, 20, 12, 64)   1088        I1[0][0]                         \n",
      "__________________________________________________________________________________________________\n",
      "I2_conv1 (Conv2D)               (None, 20, 12, 64)   1088        I2[0][0]                         \n",
      "__________________________________________________________________________________________________\n",
      "I1_BN1 (BatchNormalization)     (None, 20, 12, 64)   256         I1_conv1[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "I2_BN1 (BatchNormalization)     (None, 20, 12, 64)   256         I2_conv1[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "tf_op_layer_Relu (TensorFlowOpL (None, 20, 12, 64)   0           I1_BN1[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "tf_op_layer_Relu_3 (TensorFlowO (None, 20, 12, 64)   0           I2_BN1[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "I1_dropout1 (Dropout)           (None, 20, 12, 64)   0           tf_op_layer_Relu[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "I2_dropout1 (Dropout)           (None, 20, 12, 64)   0           tf_op_layer_Relu_3[0][0]         \n",
      "__________________________________________________________________________________________________\n",
      "I1_conv2 (Conv2D)               (None, 20, 6, 64)    28736       I1_dropout1[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "I2_conv2 (Conv2D)               (None, 20, 6, 64)    28736       I2_dropout1[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "I1_BN2 (BatchNormalization)     (None, 20, 6, 64)    256         I1_conv2[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "I2_BN2 (BatchNormalization)     (None, 20, 6, 64)    256         I2_conv2[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "tf_op_layer_Relu_1 (TensorFlowO (None, 20, 6, 64)    0           I1_BN2[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "tf_op_layer_Relu_4 (TensorFlowO (None, 20, 6, 64)    0           I2_BN2[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "I1_dropout2 (Dropout)           (None, 20, 6, 64)    0           tf_op_layer_Relu_1[0][0]         \n",
      "__________________________________________________________________________________________________\n",
      "I2_dropout2 (Dropout)           (None, 20, 6, 64)    0           tf_op_layer_Relu_4[0][0]         \n",
      "__________________________________________________________________________________________________\n",
      "I1_conv3 (Conv2D)               (None, 20, 4, 64)    12352       I1_dropout2[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "I2_conv3 (Conv2D)               (None, 20, 4, 64)    12352       I2_dropout2[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "I1_BN3 (BatchNormalization)     (None, 20, 4, 64)    256         I1_conv3[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "I2_BN3 (BatchNormalization)     (None, 20, 4, 64)    256         I2_conv3[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "tf_op_layer_Relu_2 (TensorFlowO (None, 20, 4, 64)    0           I1_BN3[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "tf_op_layer_Relu_5 (TensorFlowO (None, 20, 4, 64)    0           I2_BN3[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "I1_output (Reshape)             (None, 20, 4, 1, 64) 0           tf_op_layer_Relu_2[0][0]         \n",
      "__________________________________________________________________________________________________\n",
      "I2_output (Reshape)             (None, 20, 4, 1, 64) 0           tf_op_layer_Relu_5[0][0]         \n",
      "__________________________________________________________________________________________________\n",
      "concatenate (Concatenate)       (None, 20, 4, 2, 64) 0           I1_output[0][0]                  \n",
      "                                                                 I2_output[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "Merge_dropout1 (Dropout)        (None, 20, 4, 2, 64) 0           concatenate[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "reshape (Reshape)               (None, 20, 8, 64)    0           Merge_dropout1[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "Merge_conv1 (Conv2D)            (None, 20, 8, 64)    65600       reshape[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "Merge_BN1 (BatchNormalization)  (None, 20, 8, 64)    256         Merge_conv1[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "tf_op_layer_Relu_6 (TensorFlowO (None, 20, 8, 64)    0           Merge_BN1[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "Merge_dropout2 (Dropout)        (None, 20, 8, 64)    0           tf_op_layer_Relu_6[0][0]         \n",
      "__________________________________________________________________________________________________\n",
      "Merge_conv2 (Conv2D)            (None, 20, 8, 64)    49216       Merge_dropout2[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "Merge_BN2 (BatchNormalization)  (None, 20, 8, 64)    256         Merge_conv2[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "tf_op_layer_Relu_7 (TensorFlowO (None, 20, 8, 64)    0           Merge_BN2[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "Merge_dropout3 (Dropout)        (None, 20, 8, 64)    0           tf_op_layer_Relu_7[0][0]         \n",
      "__________________________________________________________________________________________________\n",
      "Merge_conv3 (Conv2D)            (None, 20, 8, 64)    32832       Merge_dropout3[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "Merge_BN3 (BatchNormalization)  (None, 20, 8, 64)    256         Merge_conv3[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "tf_op_layer_Relu_8 (TensorFlowO (None, 20, 8, 64)    0           Merge_BN3[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "Merge_output (Reshape)          (None, 20, 512)      0           tf_op_layer_Relu_8[0][0]         \n",
      "__________________________________________________________________________________________________\n",
      "GRU (RNN)                       (None, 120)          315360      Merge_output[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "tf_op_layer_RealDiv (TensorFlow (None, 120)          0           GRU[0][0]                        \n",
      "__________________________________________________________________________________________________\n",
      "Output (Dense)                  (None, 6)            726         tf_op_layer_RealDiv[0][0]        \n",
      "==================================================================================================\n",
      "Total params: 550,390\n",
      "Trainable params: 549,238\n",
      "Non-trainable params: 1,152\n",
      "__________________________________________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "# Load the complete DeepSense Model\n",
    "new_model = tf.keras.models.load_model('DeepSense')\n",
    "\n",
    "# Show the model architecture\n",
    "new_model.summary()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "dangerous-inventory",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"DeepSense_CNN\"\n",
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "I1 (InputLayer)                 [(None, 20, 60, 1)]  0                                            \n",
      "__________________________________________________________________________________________________\n",
      "I2 (InputLayer)                 [(None, 20, 60, 1)]  0                                            \n",
      "__________________________________________________________________________________________________\n",
      "I1_conv1 (Conv2D)               (None, 20, 12, 64)   1088        I1[0][0]                         \n",
      "__________________________________________________________________________________________________\n",
      "I2_conv1 (Conv2D)               (None, 20, 12, 64)   1088        I2[0][0]                         \n",
      "__________________________________________________________________________________________________\n",
      "I1_BN1 (BatchNormalization)     (None, 20, 12, 64)   256         I1_conv1[1][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "I2_BN1 (BatchNormalization)     (None, 20, 12, 64)   256         I2_conv1[1][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "I1_ReLU1 (ReLU)                 (None, 20, 12, 64)   0           I1_BN1[1][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "I2_ReLU1 (ReLU)                 (None, 20, 12, 64)   0           I2_BN1[1][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "I1_dropout1 (Dropout)           (None, 20, 12, 64)   0           I1_ReLU1[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "I2_dropout1 (Dropout)           (None, 20, 12, 64)   0           I2_ReLU1[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "I1_conv2 (Conv2D)               (None, 20, 6, 64)    28736       I1_dropout1[1][0]                \n",
      "__________________________________________________________________________________________________\n",
      "I2_conv2 (Conv2D)               (None, 20, 6, 64)    28736       I2_dropout1[1][0]                \n",
      "__________________________________________________________________________________________________\n",
      "I1_BN2 (BatchNormalization)     (None, 20, 6, 64)    256         I1_conv2[1][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "I2_BN2 (BatchNormalization)     (None, 20, 6, 64)    256         I2_conv2[1][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "I1_ReLU2 (ReLU)                 (None, 20, 6, 64)    0           I1_BN2[1][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "I2_ReLU2 (ReLU)                 (None, 20, 6, 64)    0           I2_BN2[1][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "I1_dropout2 (Dropout)           (None, 20, 6, 64)    0           I1_ReLU2[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "I2_dropout2 (Dropout)           (None, 20, 6, 64)    0           I2_ReLU2[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "I1_conv3 (Conv2D)               (None, 20, 4, 64)    12352       I1_dropout2[1][0]                \n",
      "__________________________________________________________________________________________________\n",
      "I2_conv3 (Conv2D)               (None, 20, 4, 64)    12352       I2_dropout2[1][0]                \n",
      "__________________________________________________________________________________________________\n",
      "I1_BN3 (BatchNormalization)     (None, 20, 4, 64)    256         I1_conv3[1][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "I2_BN3 (BatchNormalization)     (None, 20, 4, 64)    256         I2_conv3[1][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "I1_ReLU3 (ReLU)                 (None, 20, 4, 64)    0           I1_BN3[1][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "I2_ReLU3 (ReLU)                 (None, 20, 4, 64)    0           I2_BN3[1][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "I1_output (Reshape)             (None, 20, 4, 1, 64) 0           I1_ReLU3[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "I2_output (Reshape)             (None, 20, 4, 1, 64) 0           I2_ReLU3[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "concatenate (Concatenate)       (None, 20, 4, 2, 64) 0           I1_output[1][0]                  \n",
      "                                                                 I2_output[1][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "Merge_dropout1 (Dropout)        (None, 20, 4, 2, 64) 0           concatenate[1][0]                \n",
      "__________________________________________________________________________________________________\n",
      "reshape (Reshape)               (None, 20, 8, 64)    0           Merge_dropout1[1][0]             \n",
      "__________________________________________________________________________________________________\n",
      "Merge_conv1 (Conv2D)            (None, 20, 8, 64)    65600       reshape[1][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "Merge_BN1 (BatchNormalization)  (None, 20, 8, 64)    256         Merge_conv1[1][0]                \n",
      "__________________________________________________________________________________________________\n",
      "Merge_ReLU1 (ReLU)              (None, 20, 8, 64)    0           Merge_BN1[1][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "Merge_dropout2 (Dropout)        (None, 20, 8, 64)    0           Merge_ReLU1[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "Merge_conv2 (Conv2D)            (None, 20, 8, 64)    49216       Merge_dropout2[1][0]             \n",
      "__________________________________________________________________________________________________\n",
      "Merge_BN2 (BatchNormalization)  (None, 20, 8, 64)    256         Merge_conv2[1][0]                \n",
      "__________________________________________________________________________________________________\n",
      "Merge_ReLU2 (ReLU)              (None, 20, 8, 64)    0           Merge_BN2[1][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "Merge_dropout3 (Dropout)        (None, 20, 8, 64)    0           Merge_ReLU2[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "Merge_conv3 (Conv2D)            (None, 20, 8, 64)    32832       Merge_dropout3[1][0]             \n",
      "__________________________________________________________________________________________________\n",
      "Merge_BN3 (BatchNormalization)  (None, 20, 8, 64)    256         Merge_conv3[1][0]                \n",
      "__________________________________________________________________________________________________\n",
      "Merge_ReLU3 (ReLU)              (None, 20, 8, 64)    0           Merge_BN3[1][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "Merge_output (Reshape)          (None, 20, 512)      0           Merge_ReLU3[0][0]                \n",
      "==================================================================================================\n",
      "Total params: 234,304\n",
      "Trainable params: 233,152\n",
      "Non-trainable params: 1,152\n",
      "__________________________________________________________________________________________________\n",
      "Model: \"DeepSense_RNN\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "RNN_Input (InputLayer)       [(None, 20, 512)]         0         \n",
      "_________________________________________________________________\n",
      "GRU (RNN)                    (None, 120)               315360    \n",
      "_________________________________________________________________\n",
      "tf_op_layer_RealDiv (TensorF (None, 120)               0         \n",
      "=================================================================\n",
      "Total params: 315,360\n",
      "Trainable params: 315,360\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "Model: \"DeepSense_FC\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "FC_Input (InputLayer)        [(None, 120)]             0         \n",
      "_________________________________________________________________\n",
      "Output (Dense)               (None, 6)                 726       \n",
      "=================================================================\n",
      "Total params: 726\n",
      "Trainable params: 726\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "# Recreate Inputs from the complete DeepSense Model\n",
    "input1 = new_model.get_layer(\"I1\")\n",
    "input2 = new_model.get_layer(\"I2\")\n",
    "\n",
    "#Recreate a sub-model for CNN from the complete DeepSense Model\n",
    "cnn1 = new_model.get_layer(\"I1_conv1\")(input1.input)\n",
    "cnn1 = new_model.get_layer(\"I1_BN1\")(cnn1)\n",
    "cnn1 = layers.ReLU(name=\"I1_ReLU1\")(cnn1)\n",
    "cnn1 = new_model.get_layer(\"I1_dropout1\")(cnn1)\n",
    "cnn1 = new_model.get_layer(\"I1_conv2\")(cnn1)\n",
    "cnn1 = new_model.get_layer(\"I1_BN2\")(cnn1)\n",
    "cnn1 = layers.ReLU(name=\"I1_ReLU2\")(cnn1)\n",
    "cnn1 = new_model.get_layer(\"I1_dropout2\")(cnn1)\n",
    "cnn1 = new_model.get_layer(\"I1_conv3\")(cnn1)\n",
    "cnn1 = new_model.get_layer(\"I1_BN3\")(cnn1)\n",
    "cnn1 = layers.ReLU(name=\"I1_ReLU3\")(cnn1)\n",
    "cnn1 = new_model.get_layer(\"I1_output\")(cnn1)\n",
    "\n",
    "cnn2 = new_model.get_layer(\"I2_conv1\")(input2.input)\n",
    "cnn2 = new_model.get_layer(\"I2_BN1\")(cnn2)\n",
    "cnn2 = layers.ReLU(name=\"I2_ReLU1\")(cnn2)\n",
    "cnn2 = new_model.get_layer(\"I2_dropout1\")(cnn2)\n",
    "cnn2 = new_model.get_layer(\"I2_conv2\")(cnn2)\n",
    "cnn2 = new_model.get_layer(\"I2_BN2\")(cnn2)\n",
    "cnn2 = layers.ReLU(name=\"I2_ReLU2\")(cnn2)\n",
    "cnn2 = new_model.get_layer(\"I2_dropout2\")(cnn2)\n",
    "cnn2 = new_model.get_layer(\"I2_conv3\")(cnn2)\n",
    "cnn2 = new_model.get_layer(\"I2_BN3\")(cnn2)\n",
    "cnn2 = layers.ReLU(name=\"I2_ReLU3\")(cnn2)\n",
    "cnn2 = new_model.get_layer(\"I2_output\")(cnn2)\n",
    "\n",
    "cnn = new_model.get_layer(\"concatenate\")((cnn1,cnn2))\n",
    "\n",
    "cnn = new_model.get_layer(\"Merge_dropout1\")(cnn)\n",
    "cnn = new_model.get_layer(\"reshape\")(cnn)\n",
    "cnn = new_model.get_layer(\"Merge_conv1\")(cnn)\n",
    "cnn = new_model.get_layer(\"Merge_BN1\")(cnn)\n",
    "cnn = layers.ReLU(name=\"Merge_ReLU1\")(cnn)\n",
    "cnn = new_model.get_layer(\"Merge_dropout2\")(cnn)\n",
    "cnn = new_model.get_layer(\"Merge_conv2\")(cnn)\n",
    "cnn = new_model.get_layer(\"Merge_BN2\")(cnn)\n",
    "cnn = layers.ReLU(name=\"Merge_ReLU2\")(cnn)\n",
    "cnn = new_model.get_layer(\"Merge_dropout3\")(cnn)\n",
    "cnn = new_model.get_layer(\"Merge_conv3\")(cnn)\n",
    "cnn = new_model.get_layer(\"Merge_BN3\")(cnn)\n",
    "cnn = layers.ReLU(name=\"Merge_ReLU3\")(cnn)\n",
    "cnn = new_model.get_layer(\"Merge_output\")(cnn)\n",
    "sub_cnn_model = keras.Model(inputs=(input1.input,input2.input), outputs=cnn, name=\"DeepSense_CNN\")\n",
    "\n",
    "#Recreate a sub-model for RNN from the complete DeepSense Model\n",
    "rnn_input = keras.Input(shape=(cnn[0].shape),name=\"RNN_Input\")\n",
    "rnn = new_model.get_layer(\"GRU\")(rnn_input)\n",
    "rnn = new_model.get_layer(\"tf_op_layer_RealDiv\")(rnn)\n",
    "sub_rnn_model = keras.Model(inputs=rnn_input, outputs=rnn, name=\"DeepSense_RNN\")\n",
    "\n",
    "#Recreate a sub-model for FC from the complete DeepSense Model\n",
    "fc_input = keras.Input(shape=(rnn[0].shape),name=\"FC_Input\")\n",
    "fc = new_model.get_layer(\"Output\")(fc_input)\n",
    "sub_fc_model = keras.Model(inputs=fc_input, outputs=fc, name=\"DeepSense_FC\")\n",
    "\n",
    "# Check all sub-models\n",
    "sub_cnn_model.summary()\n",
    "sub_rnn_model.summary()\n",
    "sub_fc_model.summary()\n",
    "keras.utils.plot_model(sub_cnn_model, \"DeepSense_CNN.png\", show_shapes=True)\n",
    "keras.utils.plot_model(sub_rnn_model, \"DeepSense_RNN.png\", show_shapes=True)\n",
    "keras.utils.plot_model(sub_fc_model, \"DeepSense_FC.png\", show_shapes=True)\n",
    "sub_cnn_model.save(\"DeepSense_CNN.h5\")\n",
    "sub_rnn_model.save(\"DeepSense_RNN.h5\")\n",
    "sub_fc_model.save(\"DeepSense_FC.h5\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "advanced-membrane",
   "metadata": {},
   "source": [
    "Load the DataSet to test the converted sub-models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "ultimate-egyptian",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(20, 120, 1)\n",
      "(20, 60, 1)\n",
      "(20, 60, 1)\n",
      "(20, 60, 1)\n",
      "[0. 0. 1. 0. 0. 0.]\n"
     ]
    }
   ],
   "source": [
    "SEPCTURAL_SAMPLES = 10\n",
    "FEATURE_DIM = SEPCTURAL_SAMPLES*6*2\n",
    "CONV_LEN = 3\n",
    "CONV_LEN_INTE = 3#4\n",
    "CONV_LEN_LAST = 3#5\n",
    "CONV_NUM = 64\n",
    "CONV_MERGE_LEN = 8\n",
    "CONV_MERGE_LEN2 = 6\n",
    "CONV_MERGE_LEN3 = 4\n",
    "CONV_NUM2 = 64\n",
    "INTER_DIM = 120\n",
    "OUT_DIM = 6#len(idDict)\n",
    "WIDE = 20\n",
    "CONV_DROP_PROB = 0.2\n",
    "\n",
    "BATCH_SIZE = 64\n",
    "TOTAL_ITER_NUM = 1000000000\n",
    "\n",
    "AUTOTUNE = tf.data.experimental.AUTOTUNE\n",
    "VALID_FILENAMES = tf.io.gfile.glob(\"./DeepSenseData/eval.tfrecord\")\n",
    "\n",
    "def read_tfrecord(example):\n",
    "    tfrecord_format = (\n",
    "        {\n",
    "            #\"image\": tf.io.FixedLenFeature([], tf.string),\n",
    "            'label': tf.io.FixedLenFeature([OUT_DIM], tf.float32),\n",
    "            'example': tf.io.FixedLenFeature([WIDE*FEATURE_DIM], tf.float32),\n",
    "            #\"target\": tf.io.FixedLenFeature([], tf.int64),\n",
    "        }\n",
    "    )\n",
    "    example = tf.io.parse_single_example(example, tfrecord_format)\n",
    "    features = tf.expand_dims(example['example'], axis=0)\n",
    "    features = tf.reshape(features, shape=(WIDE, FEATURE_DIM))\n",
    "    features = tf.expand_dims(features, axis=2)\n",
    "    print(features.shape)\n",
    "    input1, input2 = tf.split(features, num_or_size_splits=2, axis=1)\n",
    "    print(input1.shape)\n",
    "    label = example['label']\n",
    "    return (input1, input2) , label\n",
    "\n",
    "def load_dataset(filenames):\n",
    "    ignore_order = tf.data.Options()\n",
    "    ignore_order.experimental_deterministic = False  # disable order, increase speed\n",
    "    dataset = tf.data.TFRecordDataset(\n",
    "        filenames\n",
    "    )  # automatically interleaves reads from multiple files\n",
    "    dataset = dataset.with_options(\n",
    "        ignore_order\n",
    "    )  # uses data as soon as it streams in, rather than in its original order\n",
    "    dataset = dataset.map(\n",
    "        partial(read_tfrecord), num_parallel_calls=AUTOTUNE\n",
    "    )\n",
    "    # returns a dataset of (image, label) pairs if labeled=True or just images if labeled=False\n",
    "    return dataset\n",
    "\n",
    "def get_dataset(filenames):\n",
    "    dataset = load_dataset(filenames)\n",
    "    dataset = dataset.shuffle(1192)\n",
    "    dataset = dataset.prefetch(buffer_size=AUTOTUNE)\n",
    "    dataset = dataset.batch(BATCH_SIZE)\n",
    "    return dataset\n",
    "\n",
    "valid_dataset = get_dataset(VALID_FILENAMES)\n",
    "\n",
    "for features, labels in valid_dataset.take(1):\n",
    "    print(features[0].numpy()[0].shape)\n",
    "    print(features[1].numpy()[0].shape)\n",
    "    print(labels.numpy()[0])\n",
    "    samples1=tf.expand_dims(features[0].numpy()[0], axis=0)\n",
    "    samples2=tf.expand_dims(features[1].numpy()[0], axis=0)\n",
    "    label=labels.numpy()[0]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "analyzed-migration",
   "metadata": {},
   "source": [
    "Run a sample to check the accuracy of the sub-models chained together"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "guided-practice",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[2]\n",
      "2\n"
     ]
    }
   ],
   "source": [
    "cnn_output = sub_cnn_model.predict((samples1,samples2))\n",
    "rnn_output = sub_rnn_model.predict(cnn_output)\n",
    "output = sub_fc_model.predict(rnn_output)\n",
    "\n",
    "print(np.argmax(output, axis=1))\n",
    "print(np.argmax(label, axis=0))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "split-alfred",
   "metadata": {},
   "source": [
    "Convert the sub-models into TFLite model files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "acute-scott",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From c:\\users\\mehdi\\.conda\\envs\\tensorflow2\\lib\\site-packages\\tensorflow\\python\\training\\tracking\\tracking.py:111: Model.state_updates (from tensorflow.python.keras.engine.training) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "This property should not be used in TensorFlow 2.0, as updates are applied automatically.\n",
      "WARNING:tensorflow:From c:\\users\\mehdi\\.conda\\envs\\tensorflow2\\lib\\site-packages\\tensorflow\\python\\training\\tracking\\tracking.py:111: Layer.updates (from tensorflow.python.keras.engine.base_layer) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "This property should not be used in TensorFlow 2.0, as updates are applied automatically.\n",
      "INFO:tensorflow:Assets written to: C:\\Users\\mehdi\\AppData\\Local\\Temp\\tmpjz5rqxpu\\assets\n",
      "INFO:tensorflow:Assets written to: C:\\Users\\mehdi\\AppData\\Local\\Temp\\tmpqb28ftur\\assets\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Assets written to: C:\\Users\\mehdi\\AppData\\Local\\Temp\\tmpqb28ftur\\assets\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Assets written to: C:\\Users\\mehdi\\AppData\\Local\\Temp\\tmpalzpf4c_\\assets\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Assets written to: C:\\Users\\mehdi\\AppData\\Local\\Temp\\tmpalzpf4c_\\assets\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "3904"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cnn_converter = tf.lite.TFLiteConverter.from_keras_model(sub_cnn_model)\n",
    "cnn_converter.optimizations = [tf.lite.Optimize.DEFAULT]\n",
    "\n",
    "rnn_converter = tf.lite.TFLiteConverter.from_keras_model(sub_rnn_model)\n",
    "rnn_converter.optimizations = [tf.lite.Optimize.DEFAULT]\n",
    "\n",
    "fc_converter = tf.lite.TFLiteConverter.from_keras_model(sub_fc_model)\n",
    "fc_converter.optimizations = [tf.lite.Optimize.DEFAULT]\n",
    "\n",
    "tflite_model = cnn_converter.convert()\n",
    "tflite_model_file = pathlib.Path('DeepSense_cnn.tflite')\n",
    "tflite_model_file.write_bytes(tflite_model)\n",
    "\n",
    "tflite_model = rnn_converter.convert()\n",
    "tflite_model_file = pathlib.Path('DeepSense_rnn.tflite')\n",
    "tflite_model_file.write_bytes(tflite_model)\n",
    "\n",
    "tflite_model = fc_converter.convert()\n",
    "tflite_model_file = pathlib.Path('DeepSense_fc.tflite')\n",
    "tflite_model_file.write_bytes(tflite_model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "accomplished-adjustment",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}