DeepSense_Training.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "roxxUX0a0F5J"
   },
   "source": [
    "Install pre-requirements"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 1000
    },
    "id": "3gz6ukAFzTff",
    "outputId": "5c8ace73-2a5a-45f2-8080-cad6a10ea4ad",
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Collecting tensorflow==2.3.0rc0\n",
      "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/d6/f1/c0a2153f2792b68e1a69a679bc1b1fb024c173f5ee41c23afcfde34ce590/tensorflow-2.3.0rc0-cp37-cp37m-manylinux2010_x86_64.whl (320.4MB)\n",
      "\u001b[K     |████████████████████████████████| 320.4MB 53kB/s \n",
      "\u001b[?25hCollecting tensorboard<2.3.0,>=2.2.0\n",
      "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/1d/74/0a6fcb206dcc72a6da9a62dd81784bfdbff5fedb099982861dc2219014fb/tensorboard-2.2.2-py3-none-any.whl (3.0MB)\n",
      "\u001b[K     |████████████████████████████████| 3.0MB 54.8MB/s \n",
      "\u001b[?25hRequirement already satisfied: scipy==1.4.1 in /usr/local/lib/python3.7/dist-packages (from tensorflow==2.3.0rc0) (1.4.1)\n",
      "Requirement already satisfied: wrapt>=1.11.1 in /usr/local/lib/python3.7/dist-packages (from tensorflow==2.3.0rc0) (1.12.1)\n",
      "Requirement already satisfied: six>=1.12.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow==2.3.0rc0) (1.15.0)\n",
      "Requirement already satisfied: absl-py>=0.7.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow==2.3.0rc0) (0.10.0)\n",
      "Requirement already satisfied: google-pasta>=0.1.8 in /usr/local/lib/python3.7/dist-packages (from tensorflow==2.3.0rc0) (0.2.0)\n",
      "Collecting tf-estimator-nightly<2.3.0.dev2020062302,>=2.3.0.dev2020062301\n",
      "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/17/3b/fb9aafd734da258411bff2a600cabff65c7d201782318791b72422bd973d/tf_estimator_nightly-2.3.0.dev2020062301-py2.py3-none-any.whl (459kB)\n",
      "\u001b[K     |████████████████████████████████| 460kB 51.9MB/s \n",
      "\u001b[?25hRequirement already satisfied: termcolor>=1.1.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow==2.3.0rc0) (1.1.0)\n",
      "Requirement already satisfied: grpcio>=1.8.6 in /usr/local/lib/python3.7/dist-packages (from tensorflow==2.3.0rc0) (1.32.0)\n",
      "Requirement already satisfied: h5py<2.11.0,>=2.10.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow==2.3.0rc0) (2.10.0)\n",
      "Requirement already satisfied: opt-einsum>=2.3.2 in /usr/local/lib/python3.7/dist-packages (from tensorflow==2.3.0rc0) (3.3.0)\n",
      "Requirement already satisfied: keras-preprocessing<1.2,>=1.1.1 in /usr/local/lib/python3.7/dist-packages (from tensorflow==2.3.0rc0) (1.1.2)\n",
      "Requirement already satisfied: astunparse==1.6.3 in /usr/local/lib/python3.7/dist-packages (from tensorflow==2.3.0rc0) (1.6.3)\n",
      "Requirement already satisfied: protobuf>=3.9.2 in /usr/local/lib/python3.7/dist-packages (from tensorflow==2.3.0rc0) (3.12.4)\n",
      "Requirement already satisfied: gast==0.3.3 in /usr/local/lib/python3.7/dist-packages (from tensorflow==2.3.0rc0) (0.3.3)\n",
      "Collecting numpy<1.19.0,>=1.16.0\n",
      "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/d6/c6/58e517e8b1fb192725cfa23c01c2e60e4e6699314ee9684a1c5f5c9b27e1/numpy-1.18.5-cp37-cp37m-manylinux1_x86_64.whl (20.1MB)\n",
      "\u001b[K     |████████████████████████████████| 20.1MB 1.4MB/s \n",
      "\u001b[?25hRequirement already satisfied: wheel>=0.26 in /usr/local/lib/python3.7/dist-packages (from tensorflow==2.3.0rc0) (0.36.2)\n",
      "Requirement already satisfied: tensorboard-plugin-wit>=1.6.0 in /usr/local/lib/python3.7/dist-packages (from tensorboard<2.3.0,>=2.2.0->tensorflow==2.3.0rc0) (1.8.0)\n",
      "Requirement already satisfied: werkzeug>=0.11.15 in /usr/local/lib/python3.7/dist-packages (from tensorboard<2.3.0,>=2.2.0->tensorflow==2.3.0rc0) (1.0.1)\n",
      "Requirement already satisfied: google-auth-oauthlib<0.5,>=0.4.1 in /usr/local/lib/python3.7/dist-packages (from tensorboard<2.3.0,>=2.2.0->tensorflow==2.3.0rc0) (0.4.2)\n",
      "Requirement already satisfied: setuptools>=41.0.0 in /usr/local/lib/python3.7/dist-packages (from tensorboard<2.3.0,>=2.2.0->tensorflow==2.3.0rc0) (54.0.0)\n",
      "Requirement already satisfied: google-auth<2,>=1.6.3 in /usr/local/lib/python3.7/dist-packages (from tensorboard<2.3.0,>=2.2.0->tensorflow==2.3.0rc0) (1.27.0)\n",
      "Requirement already satisfied: markdown>=2.6.8 in /usr/local/lib/python3.7/dist-packages (from tensorboard<2.3.0,>=2.2.0->tensorflow==2.3.0rc0) (3.3.4)\n",
      "Requirement already satisfied: requests<3,>=2.21.0 in /usr/local/lib/python3.7/dist-packages (from tensorboard<2.3.0,>=2.2.0->tensorflow==2.3.0rc0) (2.23.0)\n",
      "Requirement already satisfied: requests-oauthlib>=0.7.0 in /usr/local/lib/python3.7/dist-packages (from google-auth-oauthlib<0.5,>=0.4.1->tensorboard<2.3.0,>=2.2.0->tensorflow==2.3.0rc0) (1.3.0)\n",
      "Requirement already satisfied: rsa<5,>=3.1.4; python_version >= \"3.6\" in /usr/local/lib/python3.7/dist-packages (from google-auth<2,>=1.6.3->tensorboard<2.3.0,>=2.2.0->tensorflow==2.3.0rc0) (4.7.2)\n",
      "Requirement already satisfied: cachetools<5.0,>=2.0.0 in /usr/local/lib/python3.7/dist-packages (from google-auth<2,>=1.6.3->tensorboard<2.3.0,>=2.2.0->tensorflow==2.3.0rc0) (4.2.1)\n",
      "Requirement already satisfied: pyasn1-modules>=0.2.1 in /usr/local/lib/python3.7/dist-packages (from google-auth<2,>=1.6.3->tensorboard<2.3.0,>=2.2.0->tensorflow==2.3.0rc0) (0.2.8)\n",
      "Requirement already satisfied: importlib-metadata; python_version < \"3.8\" in /usr/local/lib/python3.7/dist-packages (from markdown>=2.6.8->tensorboard<2.3.0,>=2.2.0->tensorflow==2.3.0rc0) (3.7.0)\n",
      "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests<3,>=2.21.0->tensorboard<2.3.0,>=2.2.0->tensorflow==2.3.0rc0) (1.24.3)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests<3,>=2.21.0->tensorboard<2.3.0,>=2.2.0->tensorflow==2.3.0rc0) (2020.12.5)\n",
      "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests<3,>=2.21.0->tensorboard<2.3.0,>=2.2.0->tensorflow==2.3.0rc0) (3.0.4)\n",
      "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests<3,>=2.21.0->tensorboard<2.3.0,>=2.2.0->tensorflow==2.3.0rc0) (2.10)\n",
      "Requirement already satisfied: oauthlib>=3.0.0 in /usr/local/lib/python3.7/dist-packages (from requests-oauthlib>=0.7.0->google-auth-oauthlib<0.5,>=0.4.1->tensorboard<2.3.0,>=2.2.0->tensorflow==2.3.0rc0) (3.1.0)\n",
      "Requirement already satisfied: pyasn1>=0.1.3 in /usr/local/lib/python3.7/dist-packages (from rsa<5,>=3.1.4; python_version >= \"3.6\"->google-auth<2,>=1.6.3->tensorboard<2.3.0,>=2.2.0->tensorflow==2.3.0rc0) (0.4.8)\n",
      "Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.7/dist-packages (from importlib-metadata; python_version < \"3.8\"->markdown>=2.6.8->tensorboard<2.3.0,>=2.2.0->tensorflow==2.3.0rc0) (3.4.0)\n",
      "Requirement already satisfied: typing-extensions>=3.6.4; python_version < \"3.8\" in /usr/local/lib/python3.7/dist-packages (from importlib-metadata; python_version < \"3.8\"->markdown>=2.6.8->tensorboard<2.3.0,>=2.2.0->tensorflow==2.3.0rc0) (3.7.4.3)\n",
      "\u001b[31mERROR: datascience 0.10.6 has requirement folium==0.2.1, but you'll have folium 0.8.3 which is incompatible.\u001b[0m\n",
      "\u001b[31mERROR: albumentations 0.1.12 has requirement imgaug<0.2.7,>=0.2.5, but you'll have imgaug 0.2.9 which is incompatible.\u001b[0m\n",
      "Installing collected packages: numpy, tensorboard, tf-estimator-nightly, tensorflow\n",
      "  Found existing installation: numpy 1.19.5\n",
      "    Uninstalling numpy-1.19.5:\n",
      "      Successfully uninstalled numpy-1.19.5\n",
      "  Found existing installation: tensorboard 2.4.1\n",
      "    Uninstalling tensorboard-2.4.1:\n",
      "      Successfully uninstalled tensorboard-2.4.1\n",
      "  Found existing installation: tensorflow 2.4.1\n",
      "    Uninstalling tensorflow-2.4.1:\n",
      "      Successfully uninstalled tensorflow-2.4.1\n",
      "Successfully installed numpy-1.18.5 tensorboard-2.2.2 tensorflow-2.3.0rc0 tf-estimator-nightly-2.3.0.dev2020062301\n"
     ]
    },
    {
     "data": {
      "application/vnd.colab-display-data+json": {
       "pip_warning": {
        "packages": [
         "numpy"
        ]
       }
      }
     },
     "metadata": {
      "tags": []
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "!pip install tensorflow==2.3.0rc0"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "XSEVA34Z0OiN"
   },
   "source": [
    "Import libraries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "H-YigM11zyqW",
    "outputId": "bce358ff-2d83-4f99-8469-0026261fe0ee"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2.3.0-rc0\n"
     ]
    }
   ],
   "source": [
    "import tensorflow as tf \n",
    "from tensorflow import keras\n",
    "from tensorflow.keras import layers\n",
    "import numpy as np\n",
    "from functools import partial\n",
    "\n",
    "import time\n",
    "import pathlib\n",
    "import math\n",
    "import os\n",
    "import sys\n",
    "print(tf.__version__)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "QwXsQKEw0TSi"
   },
   "source": [
    "Define Global VARIABLES\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "id": "UtEkEqtd0X3y"
   },
   "outputs": [],
   "source": [
    "SEPCTURAL_SAMPLES = 10\n",
    "FEATURE_DIM = SEPCTURAL_SAMPLES*6*2\n",
    "CONV_LEN = 3\n",
    "CONV_LEN_INTE = 3#4\n",
    "CONV_LEN_LAST = 3#5\n",
    "CONV_NUM = 64\n",
    "CONV_MERGE_LEN = 8\n",
    "CONV_MERGE_LEN2 = 6\n",
    "CONV_MERGE_LEN3 = 4\n",
    "CONV_NUM2 = 64\n",
    "INTER_DIM = 120\n",
    "OUT_DIM = 6#len(idDict)\n",
    "WIDE = 20\n",
    "CONV_DROP_PROB = 0.2\n",
    "\n",
    "BATCH_SIZE = 64\n",
    "TOTAL_ITER_NUM = 1000000000"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "TuoSRpJNMFhi"
   },
   "source": [
    "Load the Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "qVGY0TcNwbrp",
    "outputId": "e94b97db-b3ad-494f-a159-a0fd4dd41de4",
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(20, 120, 1)\n",
      "(20, 60, 1)\n",
      "(20, 120, 1)\n",
      "(20, 60, 1)\n"
     ]
    }
   ],
   "source": [
    "AUTOTUNE = tf.data.experimental.AUTOTUNE\n",
    "TRAINING_FILENAMES = tf.io.gfile.glob(\"./DeepSenseData/train.tfrecord\")\n",
    "VALID_FILENAMES = tf.io.gfile.glob(\"./DeepSenseData/eval.tfrecord\")\n",
    "def read_tfrecord(example):\n",
    "    tfrecord_format = (\n",
    "        {\n",
    "            'label': tf.io.FixedLenFeature([OUT_DIM], tf.float32),\n",
    "            'example': tf.io.FixedLenFeature([WIDE*FEATURE_DIM], tf.float32),\n",
    "        }\n",
    "    )\n",
    "    example = tf.io.parse_single_example(example, tfrecord_format)\n",
    "    features = tf.expand_dims(example['example'], axis=0)\n",
    "    features = tf.reshape(features, shape=(WIDE, FEATURE_DIM))\n",
    "    features = tf.expand_dims(features, axis=2)\n",
    "    print(features.shape)\n",
    "    input1, input2 = tf.split(features, num_or_size_splits=2, axis=1)\n",
    "    print(input1.shape)\n",
    "    label = example['label']\n",
    "    return (input1, input2) , label\n",
    "\n",
    "def load_dataset(filenames):\n",
    "    ignore_order = tf.data.Options()\n",
    "    ignore_order.experimental_deterministic = False  # disable order, increase speed\n",
    "    dataset = tf.data.TFRecordDataset(\n",
    "        filenames\n",
    "    )  # automatically interleaves reads from multiple files\n",
    "    dataset = dataset.with_options(\n",
    "        ignore_order\n",
    "    )  # uses data as soon as it streams in, rather than in its original order\n",
    "    dataset = dataset.map(\n",
    "        partial(read_tfrecord), num_parallel_calls=AUTOTUNE\n",
    "    )\n",
    "    # returns a dataset of (image, label) pairs if labeled=True or just images if labeled=False\n",
    "    return dataset\n",
    "\n",
    "def get_dataset(filenames):\n",
    "    dataset = load_dataset(filenames)\n",
    "    dataset = dataset.shuffle(1192)\n",
    "    dataset = dataset.prefetch(buffer_size=AUTOTUNE)\n",
    "    dataset = dataset.batch(BATCH_SIZE)\n",
    "    return dataset\n",
    "\n",
    "train_dataset = get_dataset(TRAINING_FILENAMES)\n",
    "valid_dataset = get_dataset(VALID_FILENAMES)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Define a class for every Sensor input"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "id": "pCi00iPh2bsd"
   },
   "outputs": [],
   "source": [
    "class Sensor:\n",
    "  def __init__(self, name, shape, filters1, kernel_size1, strides1, filters2, kernel_size2, strides2, filters3, kernel_size3, strides3):\n",
    "    self.shape = shape #list of integers\n",
    "    self.name = name #string with input name\n",
    "\n",
    "    self.filters1=filters1\n",
    "    self.kernel_size1=kernel_size1\n",
    "    self.strides1=strides1\n",
    "\n",
    "    self.filters2=filters2\n",
    "    self.kernel_size2=kernel_size2\n",
    "    self.strides2=strides2\n",
    "\n",
    "    self.filters3=filters3\n",
    "    self.kernel_size3=kernel_size3\n",
    "    self.strides3=strides3"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Define the Sensor parameters and the complete network"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "2ImHuk4B1bvn",
    "outputId": "c8b8384f-d804-4e70-de31-f0fa25ce2b3e"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"DeepSense\"\n",
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "I1 (InputLayer)                 [(None, 20, 60, 1)]  0                                            \n",
      "__________________________________________________________________________________________________\n",
      "I2 (InputLayer)                 [(None, 20, 60, 1)]  0                                            \n",
      "__________________________________________________________________________________________________\n",
      "I1_conv1 (Conv2D)               (None, 20, 12, 64)   1088        I1[0][0]                         \n",
      "__________________________________________________________________________________________________\n",
      "I2_conv1 (Conv2D)               (None, 20, 12, 64)   1088        I2[0][0]                         \n",
      "__________________________________________________________________________________________________\n",
      "I1_BN1 (BatchNormalization)     (None, 20, 12, 64)   256         I1_conv1[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "I2_BN1 (BatchNormalization)     (None, 20, 12, 64)   256         I2_conv1[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "tf_op_layer_Relu (TensorFlowOpL [(None, 20, 12, 64)] 0           I1_BN1[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "tf_op_layer_Relu_3 (TensorFlowO [(None, 20, 12, 64)] 0           I2_BN1[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "I1_dropout1 (Dropout)           (None, 20, 12, 64)   0           tf_op_layer_Relu[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "I2_dropout1 (Dropout)           (None, 20, 12, 64)   0           tf_op_layer_Relu_3[0][0]         \n",
      "__________________________________________________________________________________________________\n",
      "I1_conv2 (Conv2D)               (None, 20, 6, 64)    28736       I1_dropout1[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "I2_conv2 (Conv2D)               (None, 20, 6, 64)    28736       I2_dropout1[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "I1_BN2 (BatchNormalization)     (None, 20, 6, 64)    256         I1_conv2[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "I2_BN2 (BatchNormalization)     (None, 20, 6, 64)    256         I2_conv2[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "tf_op_layer_Relu_1 (TensorFlowO [(None, 20, 6, 64)]  0           I1_BN2[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "tf_op_layer_Relu_4 (TensorFlowO [(None, 20, 6, 64)]  0           I2_BN2[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "I1_dropout2 (Dropout)           (None, 20, 6, 64)    0           tf_op_layer_Relu_1[0][0]         \n",
      "__________________________________________________________________________________________________\n",
      "I2_dropout2 (Dropout)           (None, 20, 6, 64)    0           tf_op_layer_Relu_4[0][0]         \n",
      "__________________________________________________________________________________________________\n",
      "I1_conv3 (Conv2D)               (None, 20, 4, 64)    12352       I1_dropout2[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "I2_conv3 (Conv2D)               (None, 20, 4, 64)    12352       I2_dropout2[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "I1_BN3 (BatchNormalization)     (None, 20, 4, 64)    256         I1_conv3[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "I2_BN3 (BatchNormalization)     (None, 20, 4, 64)    256         I2_conv3[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "tf_op_layer_Relu_2 (TensorFlowO [(None, 20, 4, 64)]  0           I1_BN3[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "tf_op_layer_Relu_5 (TensorFlowO [(None, 20, 4, 64)]  0           I2_BN3[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "I1_output (Reshape)             (None, 20, 4, 1, 64) 0           tf_op_layer_Relu_2[0][0]         \n",
      "__________________________________________________________________________________________________\n",
      "I2_output (Reshape)             (None, 20, 4, 1, 64) 0           tf_op_layer_Relu_5[0][0]         \n",
      "__________________________________________________________________________________________________\n",
      "concatenate (Concatenate)       (None, 20, 4, 2, 64) 0           I1_output[0][0]                  \n",
      "                                                                 I2_output[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "Merge_dropout1 (Dropout)        (None, 20, 4, 2, 64) 0           concatenate[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "reshape (Reshape)               (None, 20, 8, 64)    0           Merge_dropout1[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "Merge_conv1 (Conv2D)            (None, 20, 8, 64)    65600       reshape[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "Merge_BN1 (BatchNormalization)  (None, 20, 8, 64)    256         Merge_conv1[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "tf_op_layer_Relu_6 (TensorFlowO [(None, 20, 8, 64)]  0           Merge_BN1[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "Merge_dropout2 (Dropout)        (None, 20, 8, 64)    0           tf_op_layer_Relu_6[0][0]         \n",
      "__________________________________________________________________________________________________\n",
      "Merge_conv2 (Conv2D)            (None, 20, 8, 64)    49216       Merge_dropout2[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "Merge_BN2 (BatchNormalization)  (None, 20, 8, 64)    256         Merge_conv2[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "tf_op_layer_Relu_7 (TensorFlowO [(None, 20, 8, 64)]  0           Merge_BN2[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "Merge_dropout3 (Dropout)        (None, 20, 8, 64)    0           tf_op_layer_Relu_7[0][0]         \n",
      "__________________________________________________________________________________________________\n",
      "Merge_conv3 (Conv2D)            (None, 20, 8, 64)    32832       Merge_dropout3[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "Merge_BN3 (BatchNormalization)  (None, 20, 8, 64)    256         Merge_conv3[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "tf_op_layer_Relu_8 (TensorFlowO [(None, 20, 8, 64)]  0           Merge_BN3[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "Merge_output (Reshape)          (None, 20, 512)      0           tf_op_layer_Relu_8[0][0]         \n",
      "__________________________________________________________________________________________________\n",
      "GRU (RNN)                       (None, 120)          315360      Merge_output[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "tf_op_layer_RealDiv (TensorFlow [(None, 120)]        0           GRU[0][0]                        \n",
      "__________________________________________________________________________________________________\n",
      "Output (Dense)                  (None, 6)            726         tf_op_layer_RealDiv[0][0]        \n",
      "==================================================================================================\n",
      "Total params: 550,390\n",
      "Trainable params: 549,238\n",
      "Non-trainable params: 1,152\n",
      "__________________________________________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "K = [Sensor(\"I1\",(20,60,1),64,(1,16),(1,4),64,(1,7),(1,1),64,(1,3),(1,1)),Sensor(\"I2\",(20,60,1),64,(1,16),(1,4),64,(1,7),(1,1),64,(1,3),(1,1))]\n",
    "inputs=[]\n",
    "sensors_outputs=[]\n",
    "#from tensorflow.compat import v1\n",
    "avgNum = np.full((1, 120), 20,dtype=np.float32)\n",
    "\n",
    "# Create the individual CNN branches\n",
    "for k in K: \n",
    "  inputs.append(keras.Input(shape=(k.shape),name=k.name))\n",
    "\n",
    "  cnn=layers.Conv2D(k.filters1,k.kernel_size1,k.strides1,\"valid\", name=k.name+\"_conv1\")(inputs[-1])\n",
    "  bn=layers.BatchNormalization(name=k.name+\"_BN1\")(cnn)\n",
    "  relu=keras.activations.relu(bn)\n",
    "  dropout=layers.Dropout(CONV_DROP_PROB,name=k.name+\"_dropout1\")(relu)\n",
    "\n",
    "  cnn=layers.Conv2D(k.filters2,k.kernel_size2,k.strides2,\"valid\", name=k.name+\"_conv2\")(dropout)\n",
    "  bn=layers.BatchNormalization(name=k.name+\"_BN2\")(cnn)\n",
    "  relu=keras.activations.relu(bn)\n",
    "  dropout=layers.Dropout(CONV_DROP_PROB,name=k.name+\"_dropout2\")(relu)\n",
    "\n",
    "  cnn=layers.Conv2D(k.filters3,k.kernel_size3,k.strides3,\"valid\", name=k.name+\"_conv3\")(dropout)\n",
    "  bn=layers.BatchNormalization(name=k.name+\"_BN3\")(cnn)\n",
    "  relu=keras.activations.relu(bn)\n",
    "  output=layers.Reshape((relu.shape[1], relu.shape[2], 1, relu.shape[3]), name=k.name+\"_output\")(relu)\n",
    "\n",
    "  sensors_outputs.append(output)\n",
    "    \n",
    "# Concatenate the outputs of all branches\n",
    "merge=layers.Concatenate(axis=3)(sensors_outputs)\n",
    "merge=layers.Dropout(CONV_DROP_PROB,name=\"Merge_dropout1\")(merge)\n",
    "merge = layers.Reshape((merge.shape[1], merge.shape[2]*2, merge.shape[4]))(merge)\n",
    "\n",
    "# Create the Merge CNN\n",
    "merge=layers.Conv2D(64,(1,16),(1,1),\"same\", name=\"Merge_conv1\")(merge)\n",
    "merge=layers.BatchNormalization(name=\"Merge_BN1\")(merge)\n",
    "merge=keras.activations.relu(merge)\n",
    "merge=layers.Dropout(CONV_DROP_PROB,name=\"Merge_dropout2\")(merge)\n",
    "\n",
    "merge=layers.Conv2D(64,(1,12),(1,1),\"same\", name=\"Merge_conv2\")(merge)\n",
    "merge=layers.BatchNormalization(name=\"Merge_BN2\")(merge)\n",
    "merge=keras.activations.relu(merge)\n",
    "merge=layers.Dropout(CONV_DROP_PROB,name=\"Merge_dropout3\")(merge)\n",
    "\n",
    "merge=layers.Conv2D(64,(1,8),(1,1),\"same\", name=\"Merge_conv3\")(merge)\n",
    "merge=layers.BatchNormalization(name=\"Merge_BN3\")(merge)\n",
    "merge=keras.activations.relu(merge)\n",
    "merge_output=layers.Reshape((merge.shape[1], merge.shape[2]*merge.shape[3]), name=\"Merge_output\")(merge)\n",
    "\n",
    "# Create the RNN network with 2 Stacked GRUs\n",
    "rnn=layers.StackedRNNCells([layers.GRUCell(INTER_DIM, dropout=0.5),layers.GRUCell(INTER_DIM,dropout=0.5)])\n",
    "rnn_output=layers.RNN(rnn,time_major = False,name=\"GRU\")(merge_output)\n",
    "\n",
    "# Create the final Dense Layer\n",
    "rnn_output = rnn_output/avgNum\n",
    "outputs=layers.Dense(6,name=\"Output\",activation=\"softmax\")(rnn_output)\n",
    "\n",
    "# Define the model\n",
    "model = keras.Model(inputs=inputs, outputs=outputs, name=\"DeepSense\")\n",
    "\n",
    "#Print summary and plot the model\n",
    "model.summary()\n",
    "keras.utils.plot_model(model, \"DeepSense.png\", show_shapes=True)\n",
    "\n",
    "\n",
    "# Compile the model\n",
    "model.compile(\n",
    "    loss=keras.losses.CategoricalCrossentropy(from_logits=True),\n",
    "    #loss=loss,\n",
    "    optimizer=tf.keras.optimizers.Adam(learning_rate=1e-4, beta_1=0.5,\tbeta_2=0.9),\n",
    "    metrics=[\"accuracy\"],\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Train the model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "IoQ4LefXLLnq",
    "outputId": "be9b4230-838b-49c6-819c-153e4ea3cb9d"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/5000\n",
      "   1861/Unknown - 373s 201ms/step - loss: 1.6527 - accuracy: 0.7145{'loss': 1.652748942375183, 'accuracy': 0.7144860625267029, 'val_loss': 1.5981473922729492, 'val_accuracy': 0.714165985584259}\n",
      "1861/1861 [==============================] - 380s 204ms/step - loss: 1.6527 - accuracy: 0.7145 - val_loss: 1.5981 - val_accuracy: 0.7142\n",
      "Epoch 2/5000\n",
      "1861/1861 [==============================] - ETA: 0s - loss: 1.4540 - accuracy: 0.8782{'loss': 1.4539544582366943, 'accuracy': 0.878174364566803, 'val_loss': 1.4650440216064453, 'val_accuracy': 0.771165132522583}\n",
      "1861/1861 [==============================] - 380s 204ms/step - loss: 1.4540 - accuracy: 0.8782 - val_loss: 1.4650 - val_accuracy: 0.7712\n",
      "Epoch 3/5000\n",
      "1861/1861 [==============================] - ETA: 0s - loss: 1.3305 - accuracy: 0.9240{'loss': 1.3304744958877563, 'accuracy': 0.9240006804466248, 'val_loss': 1.4273227453231812, 'val_accuracy': 0.7393126487731934}\n",
      "1861/1861 [==============================] - 377s 203ms/step - loss: 1.3305 - accuracy: 0.9240 - val_loss: 1.4273 - val_accuracy: 0.7393\n",
      "Epoch 4/5000\n",
      "1861/1861 [==============================] - ETA: 0s - loss: 1.2482 - accuracy: 0.9399{'loss': 1.2482454776763916, 'accuracy': 0.9399227499961853, 'val_loss': 1.3595669269561768, 'val_accuracy': 0.7652975916862488}\n",
      "1861/1861 [==============================] - 374s 201ms/step - loss: 1.2482 - accuracy: 0.9399 - val_loss: 1.3596 - val_accuracy: 0.7653\n",
      "Epoch 5/5000\n",
      "1861/1861 [==============================] - ETA: 0s - loss: 1.1935 - accuracy: 0.9465{'loss': 1.193539023399353, 'accuracy': 0.9464981555938721, 'val_loss': 1.3211085796356201, 'val_accuracy': 0.7795473337173462}\n",
      "1861/1861 [==============================] - 351s 188ms/step - loss: 1.1935 - accuracy: 0.9465 - val_loss: 1.3211 - val_accuracy: 0.7795\n",
      "Epoch 6/5000\n",
      "1861/1861 [==============================] - ETA: 0s - loss: 1.1493 - accuracy: 0.9585{'loss': 1.1492904424667358, 'accuracy': 0.958523690700531, 'val_loss': 1.3192793130874634, 'val_accuracy': 0.7585917711257935}\n",
      "1861/1861 [==============================] - 377s 202ms/step - loss: 1.1493 - accuracy: 0.9585 - val_loss: 1.3193 - val_accuracy: 0.7586\n",
      "Epoch 7/5000\n",
      "1861/1861 [==============================] - ETA: 0s - loss: 1.1208 - accuracy: 0.9621{'loss': 1.1207966804504395, 'accuracy': 0.96205073595047, 'val_loss': 1.3011008501052856, 'val_accuracy': 0.7594299912452698}\n",
      "1861/1861 [==============================] - 369s 198ms/step - loss: 1.1208 - accuracy: 0.9621 - val_loss: 1.3011 - val_accuracy: 0.7594\n",
      "Epoch 8/5000\n",
      "1861/1861 [==============================] - ETA: 0s - loss: 1.1049 - accuracy: 0.9618{'loss': 1.1048864126205444, 'accuracy': 0.9618491530418396, 'val_loss': 1.303556203842163, 'val_accuracy': 0.7443419694900513}\n",
      "1861/1861 [==============================] - 367s 197ms/step - loss: 1.1049 - accuracy: 0.9618 - val_loss: 1.3036 - val_accuracy: 0.7443\n",
      "Epoch 9/5000\n",
      "1861/1861 [==============================] - ETA: 0s - loss: 1.0909 - accuracy: 0.9668{'loss': 1.0909050703048706, 'accuracy': 0.9667618274688721, 'val_loss': 1.2776734828948975, 'val_accuracy': 0.7703269124031067}\n",
      "1861/1861 [==============================] - 363s 195ms/step - loss: 1.0909 - accuracy: 0.9668 - val_loss: 1.2777 - val_accuracy: 0.7703\n",
      "Epoch 10/5000\n",
      "1861/1861 [==============================] - ETA: 0s - loss: 1.0827 - accuracy: 0.9695{'loss': 1.0826654434204102, 'accuracy': 0.9694659113883972, 'val_loss': 1.2953386306762695, 'val_accuracy': 0.7476949095726013}\n",
      "1861/1861 [==============================] - 355s 191ms/step - loss: 1.0827 - accuracy: 0.9695 - val_loss: 1.2953 - val_accuracy: 0.7477\n",
      "Epoch 11/5000\n",
      "1861/1861 [==============================] - ETA: 0s - loss: 1.0795 - accuracy: 0.9693{'loss': 1.079535722732544, 'accuracy': 0.9692895412445068, 'val_loss': 1.3083510398864746, 'val_accuracy': 0.7342833280563354}\n",
      "1861/1861 [==============================] - 351s 189ms/step - loss: 1.0795 - accuracy: 0.9693 - val_loss: 1.3084 - val_accuracy: 0.7343\n",
      "Epoch 12/5000\n",
      "1861/1861 [==============================] - ETA: 0s - loss: 1.0739 - accuracy: 0.9731{'loss': 1.0738993883132935, 'accuracy': 0.9731189012527466, 'val_loss': 1.2735416889190674, 'val_accuracy': 0.7728415727615356}\n",
      "1861/1861 [==============================] - 350s 188ms/step - loss: 1.0739 - accuracy: 0.9731 - val_loss: 1.2735 - val_accuracy: 0.7728\n",
      "Epoch 13/5000\n",
      "1861/1861 [==============================] - ETA: 0s - loss: 1.0721 - accuracy: 0.9737{'loss': 1.0720897912979126, 'accuracy': 0.9736731648445129, 'val_loss': 1.325210452079773, 'val_accuracy': 0.7158424258232117}\n",
      "1861/1861 [==============================] - 358s 192ms/step - loss: 1.0721 - accuracy: 0.9737 - val_loss: 1.3252 - val_accuracy: 0.7158\n",
      "Epoch 14/5000\n",
      "1861/1861 [==============================] - ETA: 0s - loss: 1.0707 - accuracy: 0.9743{'loss': 1.0706783533096313, 'accuracy': 0.9742946028709412, 'val_loss': 1.3489294052124023, 'val_accuracy': 0.6923721432685852}\n",
      "1861/1861 [==============================] - 353s 190ms/step - loss: 1.0707 - accuracy: 0.9743 - val_loss: 1.3489 - val_accuracy: 0.6924\n",
      "Epoch 15/5000\n",
      "1861/1861 [==============================] - ETA: 0s - loss: 1.0691 - accuracy: 0.9754{'loss': 1.0690929889678955, 'accuracy': 0.9754198789596558, 'val_loss': 1.2762609720230103, 'val_accuracy': 0.7669740319252014}\n",
      "1861/1861 [==============================] - 364s 196ms/step - loss: 1.0691 - accuracy: 0.9754 - val_loss: 1.2763 - val_accuracy: 0.7670\n",
      "Epoch 16/5000\n",
      "1861/1861 [==============================] - ETA: 0s - loss: 1.0675 - accuracy: 0.9768{'loss': 1.067478895187378, 'accuracy': 0.9768391251564026, 'val_loss': 1.3346059322357178, 'val_accuracy': 0.7066219449043274}\n",
      "1861/1861 [==============================] - 378s 203ms/step - loss: 1.0675 - accuracy: 0.9768 - val_loss: 1.3346 - val_accuracy: 0.7066\n",
      "Epoch 17/5000\n",
      "1861/1861 [==============================] - ETA: 0s - loss: 1.0653 - accuracy: 0.9789{'loss': 1.0652908086776733, 'accuracy': 0.9789133071899414, 'val_loss': 1.2973487377166748, 'val_accuracy': 0.740989089012146}\n",
      "1861/1861 [==============================] - 382s 205ms/step - loss: 1.0653 - accuracy: 0.9789 - val_loss: 1.2973 - val_accuracy: 0.7410\n",
      "Epoch 18/5000\n",
      "1861/1861 [==============================] - ETA: 0s - loss: 1.0644 - accuracy: 0.9796{'loss': 1.0643541812896729, 'accuracy': 0.9796271324157715, 'val_loss': 1.3025474548339844, 'val_accuracy': 0.738474428653717}\n",
      "1861/1861 [==============================] - 362s 194ms/step - loss: 1.0644 - accuracy: 0.9796 - val_loss: 1.3025 - val_accuracy: 0.7385\n",
      "Epoch 19/5000\n",
      "1861/1861 [==============================] - ETA: 0s - loss: 1.0634 - accuracy: 0.9804{'loss': 1.0634204149246216, 'accuracy': 0.9804249405860901, 'val_loss': 1.2613073587417603, 'val_accuracy': 0.7812238335609436}\n",
      "1861/1861 [==============================] - 361s 194ms/step - loss: 1.0634 - accuracy: 0.9804 - val_loss: 1.2613 - val_accuracy: 0.7812\n",
      "Epoch 20/5000\n",
      "1861/1861 [==============================] - ETA: 0s - loss: 1.0629 - accuracy: 0.9809{'loss': 1.0629395246505737, 'accuracy': 0.9809455871582031, 'val_loss': 1.2697252035140991, 'val_accuracy': 0.7728415727615356}\n",
      "1861/1861 [==============================] - 355s 191ms/step - loss: 1.0629 - accuracy: 0.9809 - val_loss: 1.2697 - val_accuracy: 0.7728\n",
      "Epoch 21/5000\n",
      "1861/1861 [==============================] - ETA: 0s - loss: 1.0619 - accuracy: 0.9819{'loss': 1.0618760585784912, 'accuracy': 0.9819449186325073, 'val_loss': 1.330489993095398, 'val_accuracy': 0.7091366052627563}\n",
      "1861/1861 [==============================] - 358s 193ms/step - loss: 1.0619 - accuracy: 0.9819 - val_loss: 1.3305 - val_accuracy: 0.7091\n",
      "Epoch 22/5000\n",
      "1861/1861 [==============================] - ETA: 0s - loss: 1.0602 - accuracy: 0.9835{'loss': 1.06022310256958, 'accuracy': 0.9835068583488464, 'val_loss': 1.3380188941955566, 'val_accuracy': 0.7007544040679932}\n",
      "1861/1861 [==============================] - 355s 191ms/step - loss: 1.0602 - accuracy: 0.9835 - val_loss: 1.3380 - val_accuracy: 0.7008\n",
      "Epoch 23/5000\n",
      "1861/1861 [==============================] - ETA: 0s - loss: 1.0624 - accuracy: 0.9813{'loss': 1.0624088048934937, 'accuracy': 0.9812731146812439, 'val_loss': 1.3129242658615112, 'val_accuracy': 0.7259010672569275}\n",
      "1861/1861 [==============================] - 354s 190ms/step - loss: 1.0624 - accuracy: 0.9813 - val_loss: 1.3129 - val_accuracy: 0.7259\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 24/5000\n",
      "1861/1861 [==============================] - ETA: 0s - loss: 1.0592 - accuracy: 0.9844{'loss': 1.0592314004898071, 'accuracy': 0.9844138622283936, 'val_loss': 1.2374114990234375, 'val_accuracy': 0.803855836391449}\n",
      "1861/1861 [==============================] - 354s 190ms/step - loss: 1.0592 - accuracy: 0.9844 - val_loss: 1.2374 - val_accuracy: 0.8039\n",
      "Epoch 25/5000\n",
      "1861/1861 [==============================] - ETA: 0s - loss: 1.0583 - accuracy: 0.9854{'loss': 1.0583027601242065, 'accuracy': 0.9854299426078796, 'val_loss': 1.2628148794174194, 'val_accuracy': 0.7787091135978699}\n",
      "1861/1861 [==============================] - 353s 190ms/step - loss: 1.0583 - accuracy: 0.9854 - val_loss: 1.2628 - val_accuracy: 0.7787\n",
      "Epoch 26/5000\n",
      "1861/1861 [==============================] - ETA: 0s - loss: 1.0582 - accuracy: 0.9855{'loss': 1.0581525564193726, 'accuracy': 0.985488772392273, 'val_loss': 1.2325984239578247, 'val_accuracy': 0.8080469369888306}\n",
      "1861/1861 [==============================] - 360s 193ms/step - loss: 1.0582 - accuracy: 0.9855 - val_loss: 1.2326 - val_accuracy: 0.8080\n",
      "Epoch 27/5000\n",
      "1861/1861 [==============================] - ETA: 0s - loss: 1.0567 - accuracy: 0.9869{'loss': 1.0567412376403809, 'accuracy': 0.9869247674942017, 'val_loss': 1.303085446357727, 'val_accuracy': 0.7376362085342407}\n",
      "1861/1861 [==============================] - 351s 189ms/step - loss: 1.0567 - accuracy: 0.9869 - val_loss: 1.3031 - val_accuracy: 0.7376\n",
      "Epoch 28/5000\n",
      "1861/1861 [==============================] - ETA: 0s - loss: 1.0577 - accuracy: 0.9860{'loss': 1.0576704740524292, 'accuracy': 0.985984206199646, 'val_loss': 1.241335153579712, 'val_accuracy': 0.7996647357940674}\n",
      "1861/1861 [==============================] - 355s 191ms/step - loss: 1.0577 - accuracy: 0.9860 - val_loss: 1.2413 - val_accuracy: 0.7997\n",
      "Epoch 29/5000\n",
      "1861/1861 [==============================] - ETA: 0s - loss: 1.0573 - accuracy: 0.9865{'loss': 1.0572584867477417, 'accuracy': 0.9864544868469238, 'val_loss': 1.2587395906448364, 'val_accuracy': 0.7829002737998962}\n",
      "1861/1861 [==============================] - 358s 192ms/step - loss: 1.0573 - accuracy: 0.9865 - val_loss: 1.2587 - val_accuracy: 0.7829\n",
      "Epoch 30/5000\n",
      "1861/1861 [==============================] - ETA: 0s - loss: 1.0577 - accuracy: 0.9859{'loss': 1.0577410459518433, 'accuracy': 0.9858582615852356, 'val_loss': 1.268962025642395, 'val_accuracy': 0.7720033526420593}\n",
      "1861/1861 [==============================] - 358s 192ms/step - loss: 1.0577 - accuracy: 0.9859 - val_loss: 1.2690 - val_accuracy: 0.7720\n",
      "Epoch 31/5000\n",
      "1861/1861 [==============================] - ETA: 0s - loss: 1.0565 - accuracy: 0.9872{'loss': 1.0564662218093872, 'accuracy': 0.9871934652328491, 'val_loss': 1.270146131515503, 'val_accuracy': 0.771165132522583}\n",
      "1861/1861 [==============================] - 359s 193ms/step - loss: 1.0565 - accuracy: 0.9872 - val_loss: 1.2701 - val_accuracy: 0.7712\n",
      "Epoch 32/5000\n",
      "1861/1861 [==============================] - ETA: 0s - loss: 1.0564 - accuracy: 0.9872{'loss': 1.0564335584640503, 'accuracy': 0.9871767163276672, 'val_loss': 1.2241538763046265, 'val_accuracy': 0.8172674179077148}\n",
      "1861/1861 [==============================] - 359s 193ms/step - loss: 1.0564 - accuracy: 0.9872 - val_loss: 1.2242 - val_accuracy: 0.8173\n",
      "Epoch 33/5000\n",
      "1861/1861 [==============================] - ETA: 0s - loss: 1.0560 - accuracy: 0.9877{'loss': 1.056044101715088, 'accuracy': 0.9876721501350403, 'val_loss': 1.340372085571289, 'val_accuracy': 0.7024308443069458}\n",
      "1861/1861 [==============================] - 359s 193ms/step - loss: 1.0560 - accuracy: 0.9877 - val_loss: 1.3404 - val_accuracy: 0.7024\n",
      "Epoch 34/5000\n",
      "1861/1861 [==============================] - ETA: 0s - loss: 1.0559 - accuracy: 0.9877{'loss': 1.0559172630310059, 'accuracy': 0.9876973628997803, 'val_loss': 1.2102305889129639, 'val_accuracy': 0.8323553800582886}\n",
      "1861/1861 [==============================] - 360s 193ms/step - loss: 1.0559 - accuracy: 0.9877 - val_loss: 1.2102 - val_accuracy: 0.8324\n",
      "Epoch 35/5000\n",
      "1861/1861 [==============================] - ETA: 0s - loss: 1.0548 - accuracy: 0.9889{'loss': 1.0547784566879272, 'accuracy': 0.9888730049133301, 'val_loss': 1.2009201049804688, 'val_accuracy': 0.8415758609771729}\n",
      "1861/1861 [==============================] - 358s 192ms/step - loss: 1.0548 - accuracy: 0.9889 - val_loss: 1.2009 - val_accuracy: 0.8416\n",
      "Epoch 36/5000\n",
      "1861/1861 [==============================] - ETA: 0s - loss: 1.0563 - accuracy: 0.9874{'loss': 1.056316614151001, 'accuracy': 0.9873614311218262, 'val_loss': 1.2125893831253052, 'val_accuracy': 0.8290024995803833}\n",
      "1861/1861 [==============================] - 355s 191ms/step - loss: 1.0563 - accuracy: 0.9874 - val_loss: 1.2126 - val_accuracy: 0.8290\n",
      "Epoch 37/5000\n",
      "1861/1861 [==============================] - ETA: 0s - loss: 1.0548 - accuracy: 0.9888{'loss': 1.0548187494277954, 'accuracy': 0.9888058304786682, 'val_loss': 1.2161954641342163, 'val_accuracy': 0.825649619102478}\n",
      "1861/1861 [==============================] - 355s 191ms/step - loss: 1.0548 - accuracy: 0.9888 - val_loss: 1.2162 - val_accuracy: 0.8256\n",
      "Epoch 38/5000\n",
      "1861/1861 [==============================] - ETA: 0s - loss: 1.0562 - accuracy: 0.9875{'loss': 1.0561726093292236, 'accuracy': 0.987453818321228, 'val_loss': 1.230738639831543, 'val_accuracy': 0.8113998174667358}\n",
      "1861/1861 [==============================] - 359s 193ms/step - loss: 1.0562 - accuracy: 0.9875 - val_loss: 1.2307 - val_accuracy: 0.8114\n",
      "Epoch 39/5000\n",
      "1861/1861 [==============================] - ETA: 0s - loss: 1.0554 - accuracy: 0.9883{'loss': 1.0553522109985352, 'accuracy': 0.9882599711418152, 'val_loss': 1.2227344512939453, 'val_accuracy': 0.8181056380271912}\n",
      "1861/1861 [==============================] - 357s 192ms/step - loss: 1.0554 - accuracy: 0.9883 - val_loss: 1.2227 - val_accuracy: 0.8181\n",
      "Epoch 40/5000\n",
      "1861/1861 [==============================] - ETA: 0s - loss: 1.0545 - accuracy: 0.9891{'loss': 1.0545084476470947, 'accuracy': 0.9891249537467957, 'val_loss': 1.2462788820266724, 'val_accuracy': 0.7963117957115173}\n",
      "1861/1861 [==============================] - 355s 191ms/step - loss: 1.0545 - accuracy: 0.9891 - val_loss: 1.2463 - val_accuracy: 0.7963\n",
      "Epoch 41/5000\n",
      "1861/1861 [==============================] - ETA: 0s - loss: 1.0542 - accuracy: 0.9894{'loss': 1.054193139076233, 'accuracy': 0.9894020557403564, 'val_loss': 1.230379581451416, 'val_accuracy': 0.8122380375862122}\n",
      "1861/1861 [==============================] - 355s 191ms/step - loss: 1.0542 - accuracy: 0.9894 - val_loss: 1.2304 - val_accuracy: 0.8122\n",
      "Epoch 42/5000\n",
      "1861/1861 [==============================] - ETA: 0s - loss: 1.0559 - accuracy: 0.9877{'loss': 1.055888295173645, 'accuracy': 0.9877057671546936, 'val_loss': 1.1885385513305664, 'val_accuracy': 0.8541492223739624}\n",
      "1861/1861 [==============================] - 362s 195ms/step - loss: 1.0559 - accuracy: 0.9877 - val_loss: 1.1885 - val_accuracy: 0.8541\n",
      "Epoch 43/5000\n",
      "1861/1861 [==============================] - ETA: 0s - loss: 1.0536 - accuracy: 0.9900{'loss': 1.0536075830459595, 'accuracy': 0.9899563193321228, 'val_loss': 1.1811071634292603, 'val_accuracy': 0.8633696436882019}\n",
      "1861/1861 [==============================] - 362s 195ms/step - loss: 1.0536 - accuracy: 0.9900 - val_loss: 1.1811 - val_accuracy: 0.8634\n",
      "Epoch 44/5000\n",
      "1861/1861 [==============================] - ETA: 0s - loss: 1.0539 - accuracy: 0.9897{'loss': 1.0538817644119263, 'accuracy': 0.989679217338562, 'val_loss': 1.2063789367675781, 'val_accuracy': 0.8365465402603149}\n",
      "1861/1861 [==============================] - 364s 196ms/step - loss: 1.0539 - accuracy: 0.9897 - val_loss: 1.2064 - val_accuracy: 0.8365\n",
      "Epoch 45/5000\n",
      "1861/1861 [==============================] - ETA: 0s - loss: 1.0530 - accuracy: 0.9906{'loss': 1.0529594421386719, 'accuracy': 0.9906197786331177, 'val_loss': 1.2127710580825806, 'val_accuracy': 0.8298407196998596}\n",
      "1861/1861 [==============================] - 364s 196ms/step - loss: 1.0530 - accuracy: 0.9906 - val_loss: 1.2128 - val_accuracy: 0.8298\n",
      "Epoch 46/5000\n",
      "1861/1861 [==============================] - ETA: 0s - loss: 1.0536 - accuracy: 0.9900 ETA: 0s - loss: 1.0536 - accuracy: 0.99{'loss': 1.0536091327667236, 'accuracy': 0.9899647235870361, 'val_loss': 1.1909525394439697, 'val_accuracy': 0.8533110022544861}\n",
      "1861/1861 [==============================] - 364s 195ms/step - loss: 1.0536 - accuracy: 0.9900 - val_loss: 1.1910 - val_accuracy: 0.8533\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 47/5000\n",
      "1861/1861 [==============================] - ETA: 0s - loss: 1.0534 - accuracy: 0.9902{'loss': 1.05340576171875, 'accuracy': 0.9902166724205017, 'val_loss': 1.2058323621749878, 'val_accuracy': 0.8348701000213623}\n",
      "1861/1861 [==============================] - 364s 196ms/step - loss: 1.0534 - accuracy: 0.9902 - val_loss: 1.2058 - val_accuracy: 0.8349\n",
      "Epoch 48/5000\n",
      "1861/1861 [==============================] - ETA: 0s - loss: 1.0596 - accuracy: 0.9840{'loss': 1.0596139430999756, 'accuracy': 0.9839939475059509, 'val_loss': 1.2258411645889282, 'val_accuracy': 0.8172674179077148}\n",
      "1861/1861 [==============================] - 366s 196ms/step - loss: 1.0596 - accuracy: 0.9840 - val_loss: 1.2258 - val_accuracy: 0.8173\n",
      "Epoch 49/5000\n",
      "1861/1861 [==============================] - ETA: 0s - loss: 1.0537 - accuracy: 0.9899{'loss': 1.0536606311798096, 'accuracy': 0.9899227619171143, 'val_loss': 1.193811297416687, 'val_accuracy': 0.8491198420524597}\n",
      "1861/1861 [==============================] - 365s 196ms/step - loss: 1.0537 - accuracy: 0.9899 - val_loss: 1.1938 - val_accuracy: 0.8491\n",
      "Epoch 50/5000\n",
      "1861/1861 [==============================] - ETA: 0s - loss: 1.0579 - accuracy: 0.9856{'loss': 1.0579423904418945, 'accuracy': 0.9855979084968567, 'val_loss': 1.1840426921844482, 'val_accuracy': 0.8600167632102966}\n",
      "1861/1861 [==============================] - 364s 196ms/step - loss: 1.0579 - accuracy: 0.9856 - val_loss: 1.1840 - val_accuracy: 0.8600\n",
      "Epoch 51/5000\n",
      "1861/1861 [==============================] - ETA: 0s - loss: 1.0528 - accuracy: 0.9906{'loss': 1.0528485774993896, 'accuracy': 0.9906449317932129, 'val_loss': 1.1898696422576904, 'val_accuracy': 0.8533110022544861}\n",
      "1861/1861 [==============================] - 365s 196ms/step - loss: 1.0528 - accuracy: 0.9906 - val_loss: 1.1899 - val_accuracy: 0.8533\n",
      "Epoch 52/5000\n",
      "1861/1861 [==============================] - ETA: 0s - loss: 1.0522 - accuracy: 0.9914{'loss': 1.0522339344024658, 'accuracy': 0.9913503527641296, 'val_loss': 1.1912386417388916, 'val_accuracy': 0.8507962822914124}\n",
      "1861/1861 [==============================] - 364s 196ms/step - loss: 1.0522 - accuracy: 0.9914 - val_loss: 1.1912 - val_accuracy: 0.8508\n",
      "Epoch 53/5000\n",
      "1861/1861 [==============================] - ETA: 0s - loss: 1.0530 - accuracy: 0.9906{'loss': 1.0530130863189697, 'accuracy': 0.9905861616134644, 'val_loss': 1.214432954788208, 'val_accuracy': 0.8264878392219543}\n",
      "1861/1861 [==============================] - 365s 196ms/step - loss: 1.0530 - accuracy: 0.9906 - val_loss: 1.2144 - val_accuracy: 0.8265\n",
      "Epoch 54/5000\n",
      "1861/1861 [==============================] - ETA: 0s - loss: 1.0528 - accuracy: 0.9907{'loss': 1.0528457164764404, 'accuracy': 0.9907037019729614, 'val_loss': 1.2005321979522705, 'val_accuracy': 0.8415758609771729}\n",
      "1861/1861 [==============================] - 365s 196ms/step - loss: 1.0528 - accuracy: 0.9907 - val_loss: 1.2005 - val_accuracy: 0.8416\n",
      "Epoch 55/5000\n",
      "1861/1861 [==============================] - ETA: 0s - loss: 1.0527 - accuracy: 0.9908{'loss': 1.0527453422546387, 'accuracy': 0.9908213019371033, 'val_loss': 1.2279067039489746, 'val_accuracy': 0.8130762577056885}\n",
      "1861/1861 [==============================] - 362s 195ms/step - loss: 1.0527 - accuracy: 0.9908 - val_loss: 1.2279 - val_accuracy: 0.8131\n",
      "Epoch 56/5000\n",
      "1861/1861 [==============================] - ETA: 0s - loss: 1.0526 - accuracy: 0.9910{'loss': 1.0525896549224854, 'accuracy': 0.9909640550613403, 'val_loss': 1.2003800868988037, 'val_accuracy': 0.8424140810966492}\n",
      "1861/1861 [==============================] - 361s 194ms/step - loss: 1.0526 - accuracy: 0.9910 - val_loss: 1.2004 - val_accuracy: 0.8424\n",
      "Epoch 57/5000\n",
      "1861/1861 [==============================] - ETA: 0s - loss: 1.0525 - accuracy: 0.9911{'loss': 1.0524609088897705, 'accuracy': 0.9910984039306641, 'val_loss': 1.170665979385376, 'val_accuracy': 0.8717519044876099}\n",
      "1861/1861 [==============================] - 363s 195ms/step - loss: 1.0525 - accuracy: 0.9911 - val_loss: 1.1707 - val_accuracy: 0.8718\n",
      "Epoch 58/5000\n",
      "1861/1861 [==============================] - ETA: 0s - loss: 1.0523 - accuracy: 0.9913{'loss': 1.0522810220718384, 'accuracy': 0.9913251399993896, 'val_loss': 1.2049072980880737, 'val_accuracy': 0.8373847603797913}\n",
      "1861/1861 [==============================] - 364s 196ms/step - loss: 1.0523 - accuracy: 0.9913 - val_loss: 1.2049 - val_accuracy: 0.8374\n",
      "Epoch 59/5000\n",
      "1861/1861 [==============================] - ETA: 0s - loss: 1.0517 - accuracy: 0.9919{'loss': 1.0517407655715942, 'accuracy': 0.9918625950813293, 'val_loss': 1.1963385343551636, 'val_accuracy': 0.8474434018135071}\n",
      "1861/1861 [==============================] - 363s 195ms/step - loss: 1.0517 - accuracy: 0.9919 - val_loss: 1.1963 - val_accuracy: 0.8474\n",
      "Epoch 60/5000\n",
      "1861/1861 [==============================] - ETA: 0s - loss: 1.0520 - accuracy: 0.9916{'loss': 1.0520442724227905, 'accuracy': 0.9915770888328552, 'val_loss': 1.206074833869934, 'val_accuracy': 0.834031879901886}\n",
      "1861/1861 [==============================] - 362s 194ms/step - loss: 1.0520 - accuracy: 0.9916 - val_loss: 1.2061 - val_accuracy: 0.8340\n",
      "Epoch 61/5000\n",
      "1861/1861 [==============================] - ETA: 0s - loss: 1.0517 - accuracy: 0.9919{'loss': 1.0517369508743286, 'accuracy': 0.991879403591156, 'val_loss': 1.14002525806427, 'val_accuracy': 0.9019278883934021}\n",
      "1861/1861 [==============================] - 372s 200ms/step - loss: 1.0517 - accuracy: 0.9919 - val_loss: 1.1400 - val_accuracy: 0.9019\n",
      "Epoch 62/5000\n",
      "1861/1861 [==============================] - ETA: 0s - loss: 1.0519 - accuracy: 0.9917{'loss': 1.0518983602523804, 'accuracy': 0.991736650466919, 'val_loss': 1.174696683883667, 'val_accuracy': 0.8667225241661072}\n",
      "1861/1861 [==============================] - 376s 202ms/step - loss: 1.0519 - accuracy: 0.9917 - val_loss: 1.1747 - val_accuracy: 0.8667\n",
      "Epoch 63/5000\n",
      "1861/1861 [==============================] - ETA: 0s - loss: 1.0516 - accuracy: 0.9919{'loss': 1.0516417026519775, 'accuracy': 0.991879403591156, 'val_loss': 1.2080674171447754, 'val_accuracy': 0.8315171599388123}\n",
      "1861/1861 [==============================] - 372s 200ms/step - loss: 1.0516 - accuracy: 0.9919 - val_loss: 1.2081 - val_accuracy: 0.8315\n",
      "Epoch 64/5000\n",
      "1861/1861 [==============================] - ETA: 0s - loss: 1.0515 - accuracy: 0.9922{'loss': 1.0514662265777588, 'accuracy': 0.9921565055847168, 'val_loss': 1.1630032062530518, 'val_accuracy': 0.8809723258018494}\n",
      "1861/1861 [==============================] - 372s 200ms/step - loss: 1.0515 - accuracy: 0.9922 - val_loss: 1.1630 - val_accuracy: 0.8810\n",
      "Epoch 65/5000\n",
      "1861/1861 [==============================] - ETA: 0s - loss: 1.0564 - accuracy: 0.9872{'loss': 1.0564454793930054, 'accuracy': 0.9871515035629272, 'val_loss': 1.1551321744918823, 'val_accuracy': 0.8876781463623047}\n",
      "1861/1861 [==============================] - 374s 201ms/step - loss: 1.0564 - accuracy: 0.9872 - val_loss: 1.1551 - val_accuracy: 0.8877\n",
      "Epoch 66/5000\n",
      "1861/1861 [==============================] - ETA: 0s - loss: 1.0514 - accuracy: 0.9922 ETA: 7s - l{'loss': 1.0513834953308105, 'accuracy': 0.9921817183494568, 'val_loss': 1.1521437168121338, 'val_accuracy': 0.888516366481781}\n",
      "1861/1861 [==============================] - 371s 199ms/step - loss: 1.0514 - accuracy: 0.9922 - val_loss: 1.1521 - val_accuracy: 0.8885\n",
      "Epoch 67/5000\n",
      "1861/1861 [==============================] - ETA: 0s - loss: 1.0510 - accuracy: 0.9926{'loss': 1.0510048866271973, 'accuracy': 0.9926016330718994, 'val_loss': 1.164014458656311, 'val_accuracy': 0.8784576654434204}\n",
      "1861/1861 [==============================] - 370s 199ms/step - loss: 1.0510 - accuracy: 0.9926 - val_loss: 1.1640 - val_accuracy: 0.8785\n",
      "Epoch 68/5000\n",
      "1861/1861 [==============================] - ETA: 0s - loss: 1.0511 - accuracy: 0.9925{'loss': 1.0510576963424683, 'accuracy': 0.9925176501274109, 'val_loss': 1.147638201713562, 'val_accuracy': 0.8943839073181152}\n",
      "1861/1861 [==============================] - 393s 211ms/step - loss: 1.0511 - accuracy: 0.9925 - val_loss: 1.1476 - val_accuracy: 0.8944\n",
      "Epoch 69/5000\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1861/1861 [==============================] - ETA: 0s - loss: 1.0513 - accuracy: 0.9922{'loss': 1.051285743713379, 'accuracy': 0.9922488927841187, 'val_loss': 1.1218481063842773, 'val_accuracy': 0.921207070350647}\n",
      "1861/1861 [==============================] - 383s 206ms/step - loss: 1.0513 - accuracy: 0.9922 - val_loss: 1.1218 - val_accuracy: 0.9212\n",
      "Epoch 70/5000\n",
      "1861/1861 [==============================] - ETA: 0s - loss: 1.0511 - accuracy: 0.9925{'loss': 1.051092267036438, 'accuracy': 0.9925176501274109, 'val_loss': 1.1428139209747314, 'val_accuracy': 0.9002514481544495}\n",
      "1861/1861 [==============================] - 380s 204ms/step - loss: 1.0511 - accuracy: 0.9925 - val_loss: 1.1428 - val_accuracy: 0.9003\n",
      "Epoch 71/5000\n",
      "1861/1861 [==============================] - ETA: 0s - loss: 1.0510 - accuracy: 0.9926{'loss': 1.050974726676941, 'accuracy': 0.9926267862319946, 'val_loss': 1.145778775215149, 'val_accuracy': 0.8952221274375916}\n",
      "1861/1861 [==============================] - 384s 206ms/step - loss: 1.0510 - accuracy: 0.9926 - val_loss: 1.1458 - val_accuracy: 0.8952\n",
      "Epoch 72/5000\n",
      "1861/1861 [==============================] - ETA: 0s - loss: 1.0508 - accuracy: 0.9928{'loss': 1.05076265335083, 'accuracy': 0.9928367733955383, 'val_loss': 1.1519538164138794, 'val_accuracy': 0.89103102684021}\n",
      "1861/1861 [==============================] - 374s 201ms/step - loss: 1.0508 - accuracy: 0.9928 - val_loss: 1.1520 - val_accuracy: 0.8910\n",
      "Epoch 73/5000\n",
      "1861/1861 [==============================] - ETA: 0s - loss: 1.0502 - accuracy: 0.9934{'loss': 1.0502482652664185, 'accuracy': 0.9933574199676514, 'val_loss': 1.1387494802474976, 'val_accuracy': 0.9044426083564758}\n",
      "1861/1861 [==============================] - 375s 202ms/step - loss: 1.0502 - accuracy: 0.9934 - val_loss: 1.1387 - val_accuracy: 0.9044\n",
      "Epoch 74/5000\n",
      "1861/1861 [==============================] - ETA: 0s - loss: 1.0507 - accuracy: 0.9929{'loss': 1.050658106803894, 'accuracy': 0.9929206967353821, 'val_loss': 1.1296149492263794, 'val_accuracy': 0.912824809551239}\n",
      "1861/1861 [==============================] - 366s 196ms/step - loss: 1.0507 - accuracy: 0.9929 - val_loss: 1.1296 - val_accuracy: 0.9128\n",
      "Epoch 75/5000\n",
      "1861/1861 [==============================] - ETA: 0s - loss: 1.0506 - accuracy: 0.9929{'loss': 1.0505969524383545, 'accuracy': 0.9929291009902954, 'val_loss': 1.1411763429641724, 'val_accuracy': 0.9027661085128784}\n",
      "1861/1861 [==============================] - 363s 195ms/step - loss: 1.0506 - accuracy: 0.9929 - val_loss: 1.1412 - val_accuracy: 0.9028\n",
      "Epoch 76/5000\n",
      "1861/1861 [==============================] - ETA: 0s - loss: 1.0509 - accuracy: 0.9927{'loss': 1.0508708953857422, 'accuracy': 0.9926856160163879, 'val_loss': 1.144479751586914, 'val_accuracy': 0.8985750079154968}\n",
      "1861/1861 [==============================] - 368s 198ms/step - loss: 1.0509 - accuracy: 0.9927 - val_loss: 1.1445 - val_accuracy: 0.8986\n",
      "Epoch 77/5000\n",
      "1861/1861 [==============================] - ETA: 0s - loss: 1.0504 - accuracy: 0.9932{'loss': 1.050374984741211, 'accuracy': 0.993206262588501, 'val_loss': 1.153351068496704, 'val_accuracy': 0.8893545866012573}\n",
      "1861/1861 [==============================] - 361s 194ms/step - loss: 1.0504 - accuracy: 0.9932 - val_loss: 1.1534 - val_accuracy: 0.8894\n",
      "Epoch 78/5000\n",
      "1861/1861 [==============================] - ETA: 0s - loss: 1.0509 - accuracy: 0.9927{'loss': 1.0508538484573364, 'accuracy': 0.9927107691764832, 'val_loss': 1.1270500421524048, 'val_accuracy': 0.9161776900291443}\n",
      "1861/1861 [==============================] - 359s 193ms/step - loss: 1.0509 - accuracy: 0.9927 - val_loss: 1.1271 - val_accuracy: 0.9162\n",
      "Epoch 79/5000\n",
      "1861/1861 [==============================] - ETA: 0s - loss: 1.0504 - accuracy: 0.9932{'loss': 1.0504010915756226, 'accuracy': 0.9931726455688477, 'val_loss': 1.164699673652649, 'val_accuracy': 0.8776194453239441}\n",
      "1861/1861 [==============================] - 357s 192ms/step - loss: 1.0504 - accuracy: 0.9932 - val_loss: 1.1647 - val_accuracy: 0.8776\n",
      "Epoch 80/5000\n",
      "1861/1861 [==============================] - ETA: 0s - loss: 1.0505 - accuracy: 0.9931{'loss': 1.0505355596542358, 'accuracy': 0.9930719137191772, 'val_loss': 1.1577458381652832, 'val_accuracy': 0.886001706123352}\n",
      "1861/1861 [==============================] - 356s 191ms/step - loss: 1.0505 - accuracy: 0.9931 - val_loss: 1.1577 - val_accuracy: 0.8860\n",
      "Epoch 81/5000\n",
      "1861/1861 [==============================] - ETA: 0s - loss: 1.0503 - accuracy: 0.9933{'loss': 1.0502667427062988, 'accuracy': 0.9932818412780762, 'val_loss': 1.181410551071167, 'val_accuracy': 0.8642078638076782}\n",
      "1861/1861 [==============================] - 356s 191ms/step - loss: 1.0503 - accuracy: 0.9933 - val_loss: 1.1814 - val_accuracy: 0.8642\n",
      "Epoch 82/5000\n",
      "1861/1861 [==============================] - ETA: 0s - loss: 1.0503 - accuracy: 0.9933{'loss': 1.0502629280090332, 'accuracy': 0.9933153986930847, 'val_loss': 1.1300302743911743, 'val_accuracy': 0.912824809551239}\n",
      "1861/1861 [==============================] - 354s 190ms/step - loss: 1.0503 - accuracy: 0.9933 - val_loss: 1.1300 - val_accuracy: 0.9128\n",
      "Epoch 83/5000\n",
      "1861/1861 [==============================] - ETA: 0s - loss: 1.0504 - accuracy: 0.9932{'loss': 1.0503743886947632, 'accuracy': 0.9932146668434143, 'val_loss': 1.134507656097412, 'val_accuracy': 0.9069572687149048}\n",
      "1861/1861 [==============================] - 357s 192ms/step - loss: 1.0504 - accuracy: 0.9932 - val_loss: 1.1345 - val_accuracy: 0.9070\n",
      "Epoch 84/5000\n",
      "1861/1861 [==============================] - ETA: 0s - loss: 1.0509 - accuracy: 0.9927{'loss': 1.050939679145813, 'accuracy': 0.992660403251648, 'val_loss': 1.1325268745422363, 'val_accuracy': 0.9111483693122864}\n",
      "1861/1861 [==============================] - 354s 190ms/step - loss: 1.0509 - accuracy: 0.9927 - val_loss: 1.1325 - val_accuracy: 0.9111\n",
      "Epoch 85/5000\n",
      "1861/1861 [==============================] - ETA: 0s - loss: 1.0496 - accuracy: 0.9940{'loss': 1.0495558977127075, 'accuracy': 0.9940208196640015, 'val_loss': 1.188995361328125, 'val_accuracy': 0.8533110022544861}\n",
      "1861/1861 [==============================] - 356s 191ms/step - loss: 1.0496 - accuracy: 0.9940 - val_loss: 1.1890 - val_accuracy: 0.8533\n",
      "Epoch 86/5000\n",
      "1861/1861 [==============================] - ETA: 0s - loss: 1.0502 - accuracy: 0.9933{'loss': 1.0502238273620605, 'accuracy': 0.993323802947998, 'val_loss': 1.1253597736358643, 'val_accuracy': 0.9170159101486206}\n",
      "1861/1861 [==============================] - 355s 191ms/step - loss: 1.0502 - accuracy: 0.9933 - val_loss: 1.1254 - val_accuracy: 0.9170\n",
      "Epoch 87/5000\n",
      "1861/1861 [==============================] - ETA: 0s - loss: 1.0499 - accuracy: 0.9936{'loss': 1.0499162673950195, 'accuracy': 0.9936261177062988, 'val_loss': 1.1222705841064453, 'val_accuracy': 0.921207070350647}\n",
      "1861/1861 [==============================] - 364s 195ms/step - loss: 1.0499 - accuracy: 0.9936 - val_loss: 1.1223 - val_accuracy: 0.9212\n",
      "Epoch 88/5000\n",
      "1861/1861 [==============================] - ETA: 0s - loss: 1.0499 - accuracy: 0.9936{'loss': 1.0499242544174194, 'accuracy': 0.9936345219612122, 'val_loss': 1.1387466192245483, 'val_accuracy': 0.9044426083564758}\n",
      "1861/1861 [==============================] - 360s 193ms/step - loss: 1.0499 - accuracy: 0.9936 - val_loss: 1.1387 - val_accuracy: 0.9044\n",
      "Epoch 89/5000\n",
      "1861/1861 [==============================] - ETA: 0s - loss: 1.0498 - accuracy: 0.9938{'loss': 1.0498088598251343, 'accuracy': 0.993752121925354, 'val_loss': 1.1577634811401367, 'val_accuracy': 0.8834869861602783}\n",
      "1861/1861 [==============================] - 356s 192ms/step - loss: 1.0498 - accuracy: 0.9938 - val_loss: 1.1578 - val_accuracy: 0.8835\n",
      "Epoch 90/5000\n",
      "1861/1861 [==============================] - ETA: 0s - loss: 1.0496 - accuracy: 0.9939{'loss': 1.0496271848678589, 'accuracy': 0.9939116835594177, 'val_loss': 1.1640440225601196, 'val_accuracy': 0.8784576654434204}\n",
      "1861/1861 [==============================] - 359s 193ms/step - loss: 1.0496 - accuracy: 0.9939 - val_loss: 1.1640 - val_accuracy: 0.8785\n",
      "Epoch 91/5000\n",
      "1861/1861 [==============================] - ETA: 0s - loss: 1.0494 - accuracy: 0.9942{'loss': 1.0493574142456055, 'accuracy': 0.9942391514778137, 'val_loss': 1.1343218088150024, 'val_accuracy': 0.9077954888343811}\n",
      "1861/1861 [==============================] - 357s 192ms/step - loss: 1.0494 - accuracy: 0.9942 - val_loss: 1.1343 - val_accuracy: 0.9078\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 92/5000\n",
      "1861/1861 [==============================] - ETA: 0s - loss: 1.0494 - accuracy: 0.9942{'loss': 1.049401044845581, 'accuracy': 0.9941803812980652, 'val_loss': 1.165422797203064, 'val_accuracy': 0.8792958855628967}\n",
      "1861/1861 [==============================] - 358s 192ms/step - loss: 1.0494 - accuracy: 0.9942 - val_loss: 1.1654 - val_accuracy: 0.8793\n",
      "Epoch 93/5000\n",
      "1861/1861 [==============================] - ETA: 0s - loss: 1.0495 - accuracy: 0.9940{'loss': 1.0495420694351196, 'accuracy': 0.9940376281738281, 'val_loss': 1.125815749168396, 'val_accuracy': 0.9170159101486206}\n",
      "1861/1861 [==============================] - 355s 191ms/step - loss: 1.0495 - accuracy: 0.9940 - val_loss: 1.1258 - val_accuracy: 0.9170\n",
      "Epoch 94/5000\n",
      "1861/1861 [==============================] - ETA: 0s - loss: 1.0494 - accuracy: 0.9941 ETA: 2s - los{'loss': 1.049446702003479, 'accuracy': 0.99413001537323, 'val_loss': 1.139039158821106, 'val_accuracy': 0.9044426083564758}\n",
      "1861/1861 [==============================] - 354s 190ms/step - loss: 1.0494 - accuracy: 0.9941 - val_loss: 1.1390 - val_accuracy: 0.9044\n",
      "Epoch 95/5000\n",
      "1861/1861 [==============================] - ETA: 0s - loss: 1.0496 - accuracy: 0.9940{'loss': 1.0495960712432861, 'accuracy': 0.9939536452293396, 'val_loss': 1.1424084901809692, 'val_accuracy': 0.8994132280349731}\n",
      "1861/1861 [==============================] - 356s 191ms/step - loss: 1.0496 - accuracy: 0.9940 - val_loss: 1.1424 - val_accuracy: 0.8994\n",
      "Epoch 96/5000\n",
      "1861/1861 [==============================] - ETA: 0s - loss: 1.0495 - accuracy: 0.9941{'loss': 1.0495429039001465, 'accuracy': 0.9940544366836548, 'val_loss': 1.1388602256774902, 'val_accuracy': 0.9027661085128784}\n",
      "1861/1861 [==============================] - 355s 191ms/step - loss: 1.0495 - accuracy: 0.9941 - val_loss: 1.1389 - val_accuracy: 0.9028\n",
      "Epoch 97/5000\n",
      "1861/1861 [==============================] - ETA: 0s - loss: 1.0498 - accuracy: 0.9937{'loss': 1.049845814704895, 'accuracy': 0.993726909160614, 'val_loss': 1.1507563591003418, 'val_accuracy': 0.8918692469596863}\n",
      "1861/1861 [==============================] - 357s 192ms/step - loss: 1.0498 - accuracy: 0.9937 - val_loss: 1.1508 - val_accuracy: 0.8919\n",
      "Epoch 98/5000\n",
      "1861/1861 [==============================] - ETA: 0s - loss: 1.0498 - accuracy: 0.9938{'loss': 1.0497915744781494, 'accuracy': 0.9937605261802673, 'val_loss': 1.1365443468093872, 'val_accuracy': 0.9044426083564758}\n",
      "1861/1861 [==============================] - 356s 191ms/step - loss: 1.0498 - accuracy: 0.9938 - val_loss: 1.1365 - val_accuracy: 0.9044\n",
      "Epoch 99/5000\n",
      "1861/1861 [==============================] - ETA: 0s - loss: 1.0493 - accuracy: 0.9942{'loss': 1.0493122339248657, 'accuracy': 0.9942139983177185, 'val_loss': 1.1322407722473145, 'val_accuracy': 0.9111483693122864}\n",
      "1861/1861 [==============================] - 357s 192ms/step - loss: 1.0493 - accuracy: 0.9942 - val_loss: 1.1322 - val_accuracy: 0.9111\n",
      "Epoch 100/5000\n",
      "1861/1861 [==============================] - ETA: 0s - loss: 1.0492 - accuracy: 0.9943{'loss': 1.049228549003601, 'accuracy': 0.9942811727523804, 'val_loss': 1.146117091178894, 'val_accuracy': 0.8977367877960205}\n",
      "1861/1861 [==============================] - 356s 191ms/step - loss: 1.0492 - accuracy: 0.9943 - val_loss: 1.1461 - val_accuracy: 0.8977\n",
      "Epoch 101/5000\n",
      "1861/1861 [==============================] - ETA: 0s - loss: 1.0495 - accuracy: 0.9941{'loss': 1.0495274066925049, 'accuracy': 0.9940628409385681, 'val_loss': 1.1426103115081787, 'val_accuracy': 0.9010896682739258}\n",
      "1861/1861 [==============================] - 357s 192ms/step - loss: 1.0495 - accuracy: 0.9941 - val_loss: 1.1426 - val_accuracy: 0.9011\n",
      "Epoch 102/5000\n",
      "1861/1861 [==============================] - ETA: 0s - loss: 1.0493 - accuracy: 0.9943{'loss': 1.049296259880066, 'accuracy': 0.994272768497467, 'val_loss': 1.1238938570022583, 'val_accuracy': 0.9203687906265259}\n",
      "1861/1861 [==============================] - 353s 190ms/step - loss: 1.0493 - accuracy: 0.9943 - val_loss: 1.1239 - val_accuracy: 0.9204\n",
      "Epoch 103/5000\n",
      "1861/1861 [==============================] - ETA: 0s - loss: 1.0498 - accuracy: 0.9937{'loss': 1.0498207807540894, 'accuracy': 0.9937353134155273, 'val_loss': 1.1661313772201538, 'val_accuracy': 0.8751047849655151}\n",
      "1861/1861 [==============================] - 353s 190ms/step - loss: 1.0498 - accuracy: 0.9937 - val_loss: 1.1661 - val_accuracy: 0.8751\n",
      "Epoch 104/5000\n",
      "1861/1861 [==============================] - ETA: 0s - loss: 1.0493 - accuracy: 0.9943{'loss': 1.0492671728134155, 'accuracy': 0.9942811727523804, 'val_loss': 1.1522918939590454, 'val_accuracy': 0.89103102684021}\n",
      "1861/1861 [==============================] - 351s 188ms/step - loss: 1.0493 - accuracy: 0.9943 - val_loss: 1.1523 - val_accuracy: 0.8910\n",
      "Epoch 105/5000\n",
      "1861/1861 [==============================] - ETA: 0s - loss: 1.0489 - accuracy: 0.9947{'loss': 1.048938512802124, 'accuracy': 0.994650661945343, 'val_loss': 1.176471471786499, 'val_accuracy': 0.8667225241661072}\n",
      "1861/1861 [==============================] - 352s 189ms/step - loss: 1.0489 - accuracy: 0.9947 - val_loss: 1.1765 - val_accuracy: 0.8667\n",
      "Epoch 106/5000\n",
      "1861/1861 [==============================] - ETA: 0s - loss: 1.0489 - accuracy: 0.9947{'loss': 1.048878788948059, 'accuracy': 0.9946842193603516, 'val_loss': 1.1757240295410156, 'val_accuracy': 0.8667225241661072}\n",
      "1861/1861 [==============================] - 353s 190ms/step - loss: 1.0489 - accuracy: 0.9947 - val_loss: 1.1757 - val_accuracy: 0.8667\n",
      "Epoch 107/5000\n",
      "1861/1861 [==============================] - ETA: 0s - loss: 1.0493 - accuracy: 0.9943{'loss': 1.0492751598358154, 'accuracy': 0.9942811727523804, 'val_loss': 1.1571476459503174, 'val_accuracy': 0.886001706123352}\n",
      "1861/1861 [==============================] - 352s 189ms/step - loss: 1.0493 - accuracy: 0.9943 - val_loss: 1.1571 - val_accuracy: 0.8860\n",
      "Epoch 108/5000\n",
      "1861/1861 [==============================] - ETA: 0s - loss: 1.0489 - accuracy: 0.9946{'loss': 1.0489131212234497, 'accuracy': 0.9946422576904297, 'val_loss': 1.133823037147522, 'val_accuracy': 0.9094719290733337}\n",
      "1861/1861 [==============================] - 353s 190ms/step - loss: 1.0489 - accuracy: 0.9946 - val_loss: 1.1338 - val_accuracy: 0.9095\n",
      "Epoch 109/5000\n",
      "1861/1861 [==============================] - ETA: 0s - loss: 1.0495 - accuracy: 0.9941{'loss': 1.0494773387908936, 'accuracy': 0.9940963983535767, 'val_loss': 1.1420753002166748, 'val_accuracy': 0.9010896682739258}\n",
      "1861/1861 [==============================] - 353s 190ms/step - loss: 1.0495 - accuracy: 0.9941 - val_loss: 1.1421 - val_accuracy: 0.9011\n",
      "Epoch 110/5000\n",
      "1861/1861 [==============================] - ETA: 0s - loss: 1.0492 - accuracy: 0.9944{'loss': 1.0491710901260376, 'accuracy': 0.9944071173667908, 'val_loss': 1.1671103239059448, 'val_accuracy': 0.8767812252044678}\n",
      "1861/1861 [==============================] - 355s 191ms/step - loss: 1.0492 - accuracy: 0.9944 - val_loss: 1.1671 - val_accuracy: 0.8768\n",
      "Epoch 111/5000\n",
      "1861/1861 [==============================] - ETA: 0s - loss: 1.0494 - accuracy: 0.9942{'loss': 1.0493748188018799, 'accuracy': 0.9941635727882385, 'val_loss': 1.1546778678894043, 'val_accuracy': 0.8893545866012573}\n",
      "1861/1861 [==============================] - 352s 189ms/step - loss: 1.0494 - accuracy: 0.9942 - val_loss: 1.1547 - val_accuracy: 0.8894\n",
      "Epoch 112/5000\n",
      "1861/1861 [==============================] - ETA: 0s - loss: 1.0496 - accuracy: 0.9940{'loss': 1.0495517253875732, 'accuracy': 0.9940208196640015, 'val_loss': 1.161387324333191, 'val_accuracy': 0.8809723258018494}\n",
      "1861/1861 [==============================] - 353s 190ms/step - loss: 1.0496 - accuracy: 0.9940 - val_loss: 1.1614 - val_accuracy: 0.8810\n",
      "Epoch 113/5000\n",
      "1861/1861 [==============================] - ETA: 0s - loss: 1.0490 - accuracy: 0.9945{'loss': 1.0490211248397827, 'accuracy': 0.9945247173309326, 'val_loss': 1.1171013116836548, 'val_accuracy': 0.9262363910675049}\n",
      "1861/1861 [==============================] - 351s 189ms/step - loss: 1.0490 - accuracy: 0.9945 - val_loss: 1.1171 - val_accuracy: 0.9262\n",
      "Epoch 114/5000\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1861/1861 [==============================] - ETA: 0s - loss: 1.0491 - accuracy: 0.9945{'loss': 1.0490506887435913, 'accuracy': 0.994507908821106, 'val_loss': 1.1821327209472656, 'val_accuracy': 0.860854983329773}\n",
      "1861/1861 [==============================] - 351s 188ms/step - loss: 1.0491 - accuracy: 0.9945 - val_loss: 1.1821 - val_accuracy: 0.8609\n",
      "Epoch 115/5000\n",
      "1861/1861 [==============================] - ETA: 0s - loss: 1.0488 - accuracy: 0.9948{'loss': 1.0487605333328247, 'accuracy': 0.9947934150695801, 'val_loss': 1.1410342454910278, 'val_accuracy': 0.9019278883934021}\n",
      "1861/1861 [==============================] - 352s 189ms/step - loss: 1.0488 - accuracy: 0.9948 - val_loss: 1.1410 - val_accuracy: 0.9019\n",
      "Epoch 116/5000\n",
      "1861/1861 [==============================] - ETA: 0s - loss: 1.0489 - accuracy: 0.9946{'loss': 1.0489345788955688, 'accuracy': 0.9946338534355164, 'val_loss': 1.149381399154663, 'val_accuracy': 0.8943839073181152}\n",
      "1861/1861 [==============================] - 351s 189ms/step - loss: 1.0489 - accuracy: 0.9946 - val_loss: 1.1494 - val_accuracy: 0.8944\n",
      "Epoch 117/5000\n",
      "1861/1861 [==============================] - ETA: 0s - loss: 1.0494 - accuracy: 0.9941{'loss': 1.0494465827941895, 'accuracy': 0.9941467642784119, 'val_loss': 1.1283848285675049, 'val_accuracy': 0.9145012497901917}\n",
      "1861/1861 [==============================] - 350s 188ms/step - loss: 1.0494 - accuracy: 0.9941 - val_loss: 1.1284 - val_accuracy: 0.9145\n",
      "Epoch 118/5000\n",
      "1861/1861 [==============================] - ETA: 0s - loss: 1.0488 - accuracy: 0.9948{'loss': 1.048781156539917, 'accuracy': 0.9947934150695801, 'val_loss': 1.1288102865219116, 'val_accuracy': 0.9136630296707153}\n",
      "1861/1861 [==============================] - 352s 189ms/step - loss: 1.0488 - accuracy: 0.9948 - val_loss: 1.1288 - val_accuracy: 0.9137\n",
      "Epoch 119/5000\n",
      "1861/1861 [==============================] - ETA: 0s - loss: 1.0485 - accuracy: 0.9952{'loss': 1.0484565496444702, 'accuracy': 0.9951545000076294, 'val_loss': 1.132856011390686, 'val_accuracy': 0.9094719290733337}\n",
      "1861/1861 [==============================] - 352s 189ms/step - loss: 1.0485 - accuracy: 0.9952 - val_loss: 1.1329 - val_accuracy: 0.9095\n",
      "Epoch 120/5000\n",
      "1861/1861 [==============================] - ETA: 0s - loss: 1.0489 - accuracy: 0.9947{'loss': 1.048888921737671, 'accuracy': 0.9946842193603516, 'val_loss': 1.148036241531372, 'val_accuracy': 0.8952221274375916}\n",
      "1861/1861 [==============================] - 352s 189ms/step - loss: 1.0489 - accuracy: 0.9947 - val_loss: 1.1480 - val_accuracy: 0.8952\n",
      "Epoch 121/5000\n",
      "1861/1861 [==============================] - ETA: 0s - loss: 1.0486 - accuracy: 0.9949{'loss': 1.0486122369766235, 'accuracy': 0.9949445724487305, 'val_loss': 1.1447473764419556, 'val_accuracy': 0.8977367877960205}\n",
      "1861/1861 [==============================] - 356s 191ms/step - loss: 1.0486 - accuracy: 0.9949 - val_loss: 1.1447 - val_accuracy: 0.8977\n",
      "Epoch 122/5000\n",
      "1861/1861 [==============================] - ETA: 0s - loss: 1.0490 - accuracy: 0.9945{'loss': 1.0490446090698242, 'accuracy': 0.9945498704910278, 'val_loss': 1.1507163047790527, 'val_accuracy': 0.89103102684021}\n",
      "1861/1861 [==============================] - 352s 189ms/step - loss: 1.0490 - accuracy: 0.9945 - val_loss: 1.1507 - val_accuracy: 0.8910\n",
      "Epoch 123/5000\n",
      "1861/1861 [==============================] - ETA: 0s - loss: 1.0488 - accuracy: 0.9948 ETA: 1s - loss: 1.048{'loss': 1.0487951040267944, 'accuracy': 0.9947850108146667, 'val_loss': 1.1477351188659668, 'val_accuracy': 0.8960603475570679}\n",
      "1861/1861 [==============================] - 355s 191ms/step - loss: 1.0488 - accuracy: 0.9948 - val_loss: 1.1477 - val_accuracy: 0.8961\n",
      "Epoch 124/5000\n",
      "1861/1861 [==============================] - ETA: 0s - loss: 1.0486 - accuracy: 0.9950{'loss': 1.0485775470733643, 'accuracy': 0.995003342628479, 'val_loss': 1.1647887229919434, 'val_accuracy': 0.8767812252044678}\n",
      "1861/1861 [==============================] - 353s 190ms/step - loss: 1.0486 - accuracy: 0.9950 - val_loss: 1.1648 - val_accuracy: 0.8768\n",
      "Epoch 125/5000\n",
      "1861/1861 [==============================] - ETA: 0s - loss: 1.0488 - accuracy: 0.9948{'loss': 1.0488276481628418, 'accuracy': 0.9947514533996582, 'val_loss': 1.2124035358428955, 'val_accuracy': 0.8298407196998596}\n",
      "1861/1861 [==============================] - 357s 192ms/step - loss: 1.0488 - accuracy: 0.9948 - val_loss: 1.2124 - val_accuracy: 0.8298\n",
      "Epoch 126/5000\n",
      "1861/1861 [==============================] - ETA: 0s - loss: 1.0489 - accuracy: 0.9947{'loss': 1.0488581657409668, 'accuracy': 0.9947178363800049, 'val_loss': 1.1549568176269531, 'val_accuracy': 0.8868399262428284}\n",
      "1861/1861 [==============================] - 354s 190ms/step - loss: 1.0489 - accuracy: 0.9947 - val_loss: 1.1550 - val_accuracy: 0.8868\n",
      "Epoch 127/5000\n",
      "1861/1861 [==============================] - ETA: 0s - loss: 1.0488 - accuracy: 0.9948{'loss': 1.0487902164459229, 'accuracy': 0.9947766065597534, 'val_loss': 1.1855247020721436, 'val_accuracy': 0.8549874424934387}\n",
      "1861/1861 [==============================] - 354s 190ms/step - loss: 1.0488 - accuracy: 0.9948 - val_loss: 1.1855 - val_accuracy: 0.8550\n",
      "Epoch 128/5000\n",
      "1861/1861 [==============================] - ETA: 0s - loss: 1.0485 - accuracy: 0.9950{'loss': 1.0485327243804932, 'accuracy': 0.9950453639030457, 'val_loss': 1.1337330341339111, 'val_accuracy': 0.9094719290733337}\n",
      "1861/1861 [==============================] - 354s 190ms/step - loss: 1.0485 - accuracy: 0.9950 - val_loss: 1.1337 - val_accuracy: 0.9095\n",
      "Epoch 129/5000\n",
      "1861/1861 [==============================] - ETA: 0s - loss: 1.0486 - accuracy: 0.9950{'loss': 1.0486148595809937, 'accuracy': 0.9949697852134705, 'val_loss': 1.1367383003234863, 'val_accuracy': 0.9061190485954285}\n",
      "1861/1861 [==============================] - 353s 190ms/step - loss: 1.0486 - accuracy: 0.9950 - val_loss: 1.1367 - val_accuracy: 0.9061\n",
      "Epoch 130/5000\n",
      "1861/1861 [==============================] - ETA: 0s - loss: 1.0482 - accuracy: 0.9953{'loss': 1.048216700553894, 'accuracy': 0.9953476786613464, 'val_loss': 1.1297544240951538, 'val_accuracy': 0.9145012497901917}\n",
      "1861/1861 [==============================] - 355s 191ms/step - loss: 1.0482 - accuracy: 0.9953 - val_loss: 1.1298 - val_accuracy: 0.9145\n",
      "Epoch 131/5000\n",
      "1861/1861 [==============================] - ETA: 0s - loss: 1.0486 - accuracy: 0.9949{'loss': 1.048642873764038, 'accuracy': 0.9949193596839905, 'val_loss': 1.1398367881774902, 'val_accuracy': 0.9027661085128784}\n",
      "1861/1861 [==============================] - 352s 189ms/step - loss: 1.0486 - accuracy: 0.9949 - val_loss: 1.1398 - val_accuracy: 0.9028\n",
      "Epoch 132/5000\n",
      "1861/1861 [==============================] - ETA: 0s - loss: 1.0484 - accuracy: 0.9951{'loss': 1.0484219789505005, 'accuracy': 0.9951460957527161, 'val_loss': 1.1616755723953247, 'val_accuracy': 0.882648766040802}\n",
      "1861/1861 [==============================] - 354s 190ms/step - loss: 1.0484 - accuracy: 0.9951 - val_loss: 1.1617 - val_accuracy: 0.8826\n",
      "Epoch 133/5000\n",
      "1861/1861 [==============================] - ETA: 0s - loss: 1.0482 - accuracy: 0.9953{'loss': 1.0482438802719116, 'accuracy': 0.9952720999717712, 'val_loss': 1.1432595252990723, 'val_accuracy': 0.9010896682739258}\n",
      "1861/1861 [==============================] - 352s 189ms/step - loss: 1.0482 - accuracy: 0.9953 - val_loss: 1.1433 - val_accuracy: 0.9011\n",
      "Epoch 134/5000\n",
      "1861/1861 [==============================] - ETA: 0s - loss: 1.0479 - accuracy: 0.99 - ETA: 0s - loss: 1.0479 - accuracy: 0.9956{'loss': 1.0479189157485962, 'accuracy': 0.9956499934196472, 'val_loss': 1.1306631565093994, 'val_accuracy': 0.912824809551239}\n",
      "1861/1861 [==============================] - 356s 191ms/step - loss: 1.0479 - accuracy: 0.9956 - val_loss: 1.1307 - val_accuracy: 0.9128\n",
      "Epoch 135/5000\n",
      "1861/1861 [==============================] - ETA: 0s - loss: 1.0500 - accuracy: 0.9935{'loss': 1.0500375032424927, 'accuracy': 0.9935253858566284, 'val_loss': 1.1291762590408325, 'val_accuracy': 0.9136630296707153}\n",
      "1861/1861 [==============================] - 355s 191ms/step - loss: 1.0500 - accuracy: 0.9935 - val_loss: 1.1292 - val_accuracy: 0.9137\n",
      "Epoch 136/5000\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1861/1861 [==============================] - ETA: 0s - loss: 1.0482 - accuracy: 0.9953{'loss': 1.0482279062271118, 'accuracy': 0.9953308701515198, 'val_loss': 1.1405093669891357, 'val_accuracy': 0.9019278883934021}\n",
      "1861/1861 [==============================] - 355s 191ms/step - loss: 1.0482 - accuracy: 0.9953 - val_loss: 1.1405 - val_accuracy: 0.9019\n",
      "Epoch 137/5000\n",
      "1861/1861 [==============================] - ETA: 0s - loss: 1.0482 - accuracy: 0.9953 ETA: 0s - loss: 1.0482 - accuracy: 0.99{'loss': 1.0482115745544434, 'accuracy': 0.9953224658966064, 'val_loss': 1.1499285697937012, 'val_accuracy': 0.8918692469596863}\n",
      "1861/1861 [==============================] - 353s 190ms/step - loss: 1.0482 - accuracy: 0.9953 - val_loss: 1.1499 - val_accuracy: 0.8919\n",
      "Epoch 138/5000\n",
      "1861/1861 [==============================] - ETA: 0s - loss: 1.0482 - accuracy: 0.9954{'loss': 1.0482087135314941, 'accuracy': 0.995381236076355, 'val_loss': 1.1567827463150024, 'val_accuracy': 0.8868399262428284}\n",
      "1861/1861 [==============================] - 353s 190ms/step - loss: 1.0482 - accuracy: 0.9954 - val_loss: 1.1568 - val_accuracy: 0.8868\n",
      "Epoch 139/5000\n",
      "1861/1861 [==============================] - ETA: 0s - loss: 1.0481 - accuracy: 0.9955{'loss': 1.0480825901031494, 'accuracy': 0.9955156445503235, 'val_loss': 1.1432485580444336, 'val_accuracy': 0.9002514481544495}\n",
      "1861/1861 [==============================] - 352s 189ms/step - loss: 1.0481 - accuracy: 0.9955 - val_loss: 1.1432 - val_accuracy: 0.9003\n",
      "Epoch 140/5000\n",
      "1861/1861 [==============================] - ETA: 0s - loss: 1.0479 - accuracy: 0.9957{'loss': 1.0478602647781372, 'accuracy': 0.9957087635993958, 'val_loss': 1.1322643756866455, 'val_accuracy': 0.9111483693122864}\n",
      "1861/1861 [==============================] - 354s 190ms/step - loss: 1.0479 - accuracy: 0.9957 - val_loss: 1.1323 - val_accuracy: 0.9111\n",
      "Epoch 141/5000\n",
      "1861/1861 [==============================] - ETA: 0s - loss: 1.0479 - accuracy: 0.9957{'loss': 1.0478986501693726, 'accuracy': 0.9957003593444824, 'val_loss': 1.134781002998352, 'val_accuracy': 0.9069572687149048}\n",
      "1861/1861 [==============================] - 352s 189ms/step - loss: 1.0479 - accuracy: 0.9957 - val_loss: 1.1348 - val_accuracy: 0.9070\n",
      "Epoch 142/5000\n",
      "1861/1861 [==============================] - ETA: 0s - loss: 1.0478 - accuracy: 0.9957{'loss': 1.0478031635284424, 'accuracy': 0.9957423806190491, 'val_loss': 1.1427836418151855, 'val_accuracy': 0.9002514481544495}\n",
      "1861/1861 [==============================] - 354s 190ms/step - loss: 1.0478 - accuracy: 0.9957 - val_loss: 1.1428 - val_accuracy: 0.9003\n",
      "Epoch 143/5000\n",
      "1861/1861 [==============================] - ETA: 0s - loss: 1.0477 - accuracy: 0.9959{'loss': 1.0476652383804321, 'accuracy': 0.9959186911582947, 'val_loss': 1.149792194366455, 'val_accuracy': 0.8935456871986389}\n",
      "1861/1861 [==============================] - 354s 190ms/step - loss: 1.0477 - accuracy: 0.9959 - val_loss: 1.1498 - val_accuracy: 0.8935\n",
      "Epoch 144/5000\n",
      "1861/1861 [==============================] - ETA: 0s - loss: 1.0474 - accuracy: 0.9961{'loss': 1.0474464893341064, 'accuracy': 0.9960866570472717, 'val_loss': 1.133393406867981, 'val_accuracy': 0.9094719290733337}\n",
      "1861/1861 [==============================] - 353s 190ms/step - loss: 1.0474 - accuracy: 0.9961 - val_loss: 1.1334 - val_accuracy: 0.9095\n",
      "Epoch 145/5000\n",
      "1861/1861 [==============================] - ETA: 0s - loss: 1.0477 - accuracy: 0.9959{'loss': 1.0476690530776978, 'accuracy': 0.9958935379981995, 'val_loss': 1.116633653640747, 'val_accuracy': 0.9262363910675049}\n",
      "1861/1861 [==============================] - 357s 192ms/step - loss: 1.0477 - accuracy: 0.9959 - val_loss: 1.1166 - val_accuracy: 0.9262\n",
      "Epoch 146/5000\n",
      "1861/1861 [==============================] - ETA: 0s - loss: 1.0474 - accuracy: 0.9962{'loss': 1.0473743677139282, 'accuracy': 0.9961790442466736, 'val_loss': 1.1363310813903809, 'val_accuracy': 0.9061190485954285}\n",
      "1861/1861 [==============================] - 355s 191ms/step - loss: 1.0474 - accuracy: 0.9962 - val_loss: 1.1363 - val_accuracy: 0.9061\n",
      "Epoch 147/5000\n",
      "1861/1861 [==============================] - ETA: 0s - loss: 1.0475 - accuracy: 0.9961{'loss': 1.0474690198898315, 'accuracy': 0.9960782527923584, 'val_loss': 1.142348289489746, 'val_accuracy': 0.8994132280349731}\n",
      "1861/1861 [==============================] - 357s 192ms/step - loss: 1.0475 - accuracy: 0.9961 - val_loss: 1.1423 - val_accuracy: 0.8994\n",
      "Epoch 148/5000\n",
      "1861/1861 [==============================] - ETA: 0s - loss: 1.0474 - accuracy: 0.9961{'loss': 1.047430157661438, 'accuracy': 0.9961454272270203, 'val_loss': 1.1434308290481567, 'val_accuracy': 0.9002514481544495}\n",
      "1861/1861 [==============================] - 356s 191ms/step - loss: 1.0474 - accuracy: 0.9961 - val_loss: 1.1434 - val_accuracy: 0.9003\n",
      "Epoch 149/5000\n",
      "1861/1861 [==============================] - ETA: 0s - loss: 1.0476 - accuracy: 0.9959{'loss': 1.0476093292236328, 'accuracy': 0.9959354996681213, 'val_loss': 1.1537162065505981, 'val_accuracy': 0.8901928067207336}\n",
      "1861/1861 [==============================] - 354s 190ms/step - loss: 1.0476 - accuracy: 0.9959 - val_loss: 1.1537 - val_accuracy: 0.8902\n",
      "Epoch 150/5000\n",
      "1861/1861 [==============================] - ETA: 0s - loss: 1.0472 - accuracy: 0.9964 E{'loss': 1.047203540802002, 'accuracy': 0.9963638186454773, 'val_loss': 1.132871389389038, 'val_accuracy': 0.9094719290733337}\n",
      "1861/1861 [==============================] - 356s 191ms/step - loss: 1.0472 - accuracy: 0.9964 - val_loss: 1.1329 - val_accuracy: 0.9095\n",
      "Epoch 151/5000\n",
      "1861/1861 [==============================] - ETA: 0s - loss: 1.0471 - accuracy: 0.9965{'loss': 1.0471456050872803, 'accuracy': 0.9964561462402344, 'val_loss': 1.1407647132873535, 'val_accuracy': 0.9019278883934021}\n",
      "1861/1861 [==============================] - 353s 190ms/step - loss: 1.0471 - accuracy: 0.9965 - val_loss: 1.1408 - val_accuracy: 0.9019\n",
      "Epoch 152/5000\n",
      "1861/1861 [==============================] - ETA: 0s - loss: 1.0475 - accuracy: 0.9961{'loss': 1.047524333000183, 'accuracy': 0.9960698485374451, 'val_loss': 1.1106488704681396, 'val_accuracy': 0.9337803721427917}\n",
      "1861/1861 [==============================] - 357s 192ms/step - loss: 1.0475 - accuracy: 0.9961 - val_loss: 1.1106 - val_accuracy: 0.9338\n",
      "Epoch 153/5000\n",
      "1861/1861 [==============================] - ETA: 0s - loss: 1.0474 - accuracy: 0.9962{'loss': 1.0473663806915283, 'accuracy': 0.9962042570114136, 'val_loss': 1.1292526721954346, 'val_accuracy': 0.912824809551239}\n",
      "1861/1861 [==============================] - 354s 190ms/step - loss: 1.0474 - accuracy: 0.9962 - val_loss: 1.1293 - val_accuracy: 0.9128\n",
      "Epoch 154/5000\n",
      "1861/1861 [==============================] - ETA: 0s - loss: 1.0474 - accuracy: 0.9962{'loss': 1.0473690032958984, 'accuracy': 0.9962042570114136, 'val_loss': 1.129561185836792, 'val_accuracy': 0.912824809551239}\n",
      "1861/1861 [==============================] - 357s 192ms/step - loss: 1.0474 - accuracy: 0.9962 - val_loss: 1.1296 - val_accuracy: 0.9128\n",
      "Epoch 155/5000\n",
      "1861/1861 [==============================] - ETA: 0s - loss: 1.0472 - accuracy: 0.9963{'loss': 1.0472218990325928, 'accuracy': 0.9963386058807373, 'val_loss': 1.1213279962539673, 'val_accuracy': 0.9228835105895996}\n",
      "1861/1861 [==============================] - 356s 191ms/step - loss: 1.0472 - accuracy: 0.9963 - val_loss: 1.1213 - val_accuracy: 0.9229\n",
      "Epoch 156/5000\n",
      "1861/1861 [==============================] - ETA: 0s - loss: 1.0476 - accuracy: 0.9960{'loss': 1.0475952625274658, 'accuracy': 0.995977520942688, 'val_loss': 1.1566981077194214, 'val_accuracy': 0.886001706123352}\n",
      "1861/1861 [==============================] - 354s 190ms/step - loss: 1.0476 - accuracy: 0.9960 - val_loss: 1.1567 - val_accuracy: 0.8860\n",
      "Epoch 157/5000\n",
      "1861/1861 [==============================] - ETA: 0s - loss: 1.0476 - accuracy: 0.9960{'loss': 1.0475716590881348, 'accuracy': 0.9960026741027832, 'val_loss': 1.1184078454971313, 'val_accuracy': 0.9245599508285522}\n",
      "1861/1861 [==============================] - 358s 192ms/step - loss: 1.0476 - accuracy: 0.9960 - val_loss: 1.1184 - val_accuracy: 0.9246\n",
      "Epoch 158/5000\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1861/1861 [==============================] - ETA: 0s - loss: 1.0472 - accuracy: 0.9964{'loss': 1.0471525192260742, 'accuracy': 0.9964141845703125, 'val_loss': 1.1191596984863281, 'val_accuracy': 0.9237217307090759}\n",
      "1861/1861 [==============================] - 355s 191ms/step - loss: 1.0472 - accuracy: 0.9964 - val_loss: 1.1192 - val_accuracy: 0.9237\n",
      "Epoch 159/5000\n",
      "1861/1861 [==============================] - ETA: 0s - loss: 1.0477 - accuracy: 0.9958{'loss': 1.0477304458618164, 'accuracy': 0.9958431124687195, 'val_loss': 1.124831199645996, 'val_accuracy': 0.9186923503875732}\n",
      "1861/1861 [==============================] - 354s 190ms/step - loss: 1.0477 - accuracy: 0.9958 - val_loss: 1.1248 - val_accuracy: 0.9187\n",
      "Epoch 160/5000\n",
      "1861/1861 [==============================] - ETA: 0s - loss: 1.0471 - accuracy: 0.9965{'loss': 1.0471246242523193, 'accuracy': 0.996472954750061, 'val_loss': 1.112417221069336, 'val_accuracy': 0.9312657117843628}\n",
      "1861/1861 [==============================] - 355s 191ms/step - loss: 1.0471 - accuracy: 0.9965 - val_loss: 1.1124 - val_accuracy: 0.9313\n",
      "Epoch 161/5000\n",
      "1861/1861 [==============================] - ETA: 0s - loss: 1.0473 - accuracy: 0.9963{'loss': 1.0473086833953857, 'accuracy': 0.9962881803512573, 'val_loss': 1.1182982921600342, 'val_accuracy': 0.9245599508285522}\n",
      "1861/1861 [==============================] - 355s 191ms/step - loss: 1.0473 - accuracy: 0.9963 - val_loss: 1.1183 - val_accuracy: 0.9246\n",
      "Epoch 162/5000\n",
      "1861/1861 [==============================] - ETA: 0s - loss: 1.0471 - accuracy: 0.9965{'loss': 1.0471171140670776, 'accuracy': 0.9964645504951477, 'val_loss': 1.1045103073120117, 'val_accuracy': 0.9379714727401733}\n",
      "1861/1861 [==============================] - 356s 191ms/step - loss: 1.0471 - accuracy: 0.9965 - val_loss: 1.1045 - val_accuracy: 0.9380\n",
      "Epoch 163/5000\n",
      "1861/1861 [==============================] - ETA: 0s - loss: 1.0471 - accuracy: 0.9965{'loss': 1.0470764636993408, 'accuracy': 0.9965317249298096, 'val_loss': 1.136293888092041, 'val_accuracy': 0.9069572687149048}\n",
      "1861/1861 [==============================] - 357s 192ms/step - loss: 1.0471 - accuracy: 0.9965 - val_loss: 1.1363 - val_accuracy: 0.9070\n",
      "Epoch 164/5000\n",
      "1861/1861 [==============================] - ETA: 0s - loss: 1.0472 - accuracy: 0.9964{'loss': 1.047217607498169, 'accuracy': 0.9963638186454773, 'val_loss': 1.13941490650177, 'val_accuracy': 0.9044426083564758}\n",
      "1861/1861 [==============================] - 359s 193ms/step - loss: 1.0472 - accuracy: 0.9964 - val_loss: 1.1394 - val_accuracy: 0.9044\n",
      "Epoch 165/5000\n",
      "1861/1861 [==============================] - ETA: 0s - loss: 1.0475 - accuracy: 0.9961{'loss': 1.0474505424499512, 'accuracy': 0.9961286783218384, 'val_loss': 1.1218595504760742, 'val_accuracy': 0.9220452904701233}\n",
      "1861/1861 [==============================] - 356s 191ms/step - loss: 1.0475 - accuracy: 0.9961 - val_loss: 1.1219 - val_accuracy: 0.9220\n",
      "Epoch 166/5000\n",
      "1861/1861 [==============================] - ETA: 0s - loss: 1.0474 - accuracy: 0.9962{'loss': 1.047385573387146, 'accuracy': 0.9962210059165955, 'val_loss': 1.10165536403656, 'val_accuracy': 0.9413244128227234}\n",
      "\n",
      "Reached 94% accuracy so cancelling training!\n",
      "1861/1861 [==============================] - 356s 191ms/step - loss: 1.0474 - accuracy: 0.9962 - val_loss: 1.1017 - val_accuracy: 0.9413\n",
      "WARNING:tensorflow:From c:\\users\\mehdi\\.conda\\envs\\tensorflow2\\lib\\site-packages\\tensorflow\\python\\training\\tracking\\tracking.py:111: Model.state_updates (from tensorflow.python.keras.engine.training) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "This property should not be used in TensorFlow 2.0, as updates are applied automatically.\n",
      "WARNING:tensorflow:From c:\\users\\mehdi\\.conda\\envs\\tensorflow2\\lib\\site-packages\\tensorflow\\python\\training\\tracking\\tracking.py:111: Layer.updates (from tensorflow.python.keras.engine.base_layer) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "This property should not be used in TensorFlow 2.0, as updates are applied automatically.\n",
      "INFO:tensorflow:Assets written to: DeepSense\\assets\n"
     ]
    }
   ],
   "source": [
    "# Create a callback function to stop the training after reaching a  specific accuracy\n",
    "class myCallback(tf.keras.callbacks.Callback):\n",
    "  def on_epoch_end(self, epoch, logs={}):\n",
    "    print(logs)\n",
    "    if(logs.get('val_accuracy')>0.94):\n",
    "      print(\"\\nReached 94% accuracy so cancelling training!\")\n",
    "      self.model.stop_training = True\n",
    "callbacks = myCallback()\n",
    "\n",
    "# Train the model\n",
    "history = model.fit(\n",
    "    train_dataset,\n",
    "    epochs=5000,\n",
    "    validation_data=valid_dataset,\n",
    "    verbose = 1,\n",
    "    callbacks=[callbacks]\n",
    ")\n",
    "\n",
    "#Save the model\n",
    "model.save(\"DeepSense\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Convert a TF Lite version of the saved model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "ALhluBzpL-FF",
    "outputId": "b6556a91-c914-4353-bcfd-199d4f478fea"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "594128"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "converter = tf.lite.TFLiteConverter.from_saved_model(\"DeepSense\")\n",
    "converter.optimizations = [tf.lite.Optimize.DEFAULT]\n",
    "tflite_model = converter.convert()\n",
    "\n",
    "\n",
    "tflite_model_file = pathlib.Path('model.tflite')\n",
    "tflite_model_file.write_bytes(tflite_model)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Test the TFLite Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "-g0WX2W82Snx",
    "outputId": "287fb480-7a43-4cca-ebc1-863107c94636"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(20, 60, 1)\n",
      "(20, 60, 1)\n",
      "[0. 0. 1. 0. 0. 0.]\n",
      "[2]\n",
      "2\n"
     ]
    }
   ],
   "source": [
    "#Load a sample from the validation dataset\n",
    "for features, labels in valid_dataset.take(1):\n",
    "    #print(features[0].numpy()[0].shape)\n",
    "    #print(features[1].numpy()[0].shape)\n",
    "    #print(labels.numpy()[0])\n",
    "    samples1=tf.expand_dims(features[0].numpy()[0], axis=0)\n",
    "    samples2=tf.expand_dims(features[1].numpy()[0], axis=0)\n",
    "    label=labels.numpy()[0]    \n",
    "    \n",
    "#Define the TFLite interpreter and the inputs/outputs\n",
    "interpreter = tf.lite.Interpreter(model_path=\"model.tflite\")\n",
    "interpreter.allocate_tensors()\n",
    "\n",
    "input_details = interpreter.get_input_details()\n",
    "output_details = interpreter.get_output_details()\n",
    "\n",
    "interpreter.set_tensor(input_details[0]['index'], samples1)\n",
    "interpreter.set_tensor(input_details[1]['index'], samples2)\n",
    "\n",
    "#Run the inference\n",
    "interpreter.invoke()\n",
    "\n",
    "output_data = interpreter.get_tensor(output_details[0]['index'])\n",
    "output = np.argmax(output_data, axis=1)\n",
    "\n",
    "#Compare the outputs\n",
    "print(output)\n",
    "print(np.argmax(label, axis=0))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "collapsed_sections": [],
   "name": "DeepSenseKeras.ipynb",
   "provenance": [],
   "toc_visible": true
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}