code_sample/resnet/resnet_cifar10.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 使用 TensorFlow 2.2 在CIFAR-10上训练 ResNet\n",
    "\n",
    "适配修改自：https://keras.io/examples/cifar10_resnet/"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "\n",
    "# Destroys the current TF graph and creates a new one.\n",
    "tf.keras.backend.clear_session()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[PhysicalDevice(name='/physical_device:GPU:0', device_type='GPU')]\n"
     ]
    }
   ],
   "source": [
    "# 显式使用 GPU\n",
    "gpus = tf.config.list_physical_devices(device_type='GPU')\n",
    "print(gpus)\n",
    "tf.config.experimental.set_memory_growth(gpus[0], True)\n",
    "tf.config.set_visible_devices(devices=gpus[0], device_type='GPU')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 超参数定义与 CIFAR-10 数据集下载"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "x_train shape: (50000, 32, 32, 3)\n",
      "50000 train samples\n",
      "10000 test samples\n",
      "y_train shape: (50000, 1)\n"
     ]
    }
   ],
   "source": [
    "from __future__ import print_function\n",
    "\n",
    "from tensorflow import keras as keras\n",
    "\n",
    "from tensorflow.keras.layers import Dense, Conv2D, BatchNormalization, Activation\n",
    "from tensorflow.keras.layers import AveragePooling2D, Input, Flatten\n",
    "from tensorflow.keras.optimizers import Adam\n",
    "from tensorflow.keras.callbacks import ModelCheckpoint, LearningRateScheduler\n",
    "from tensorflow.keras.callbacks import ReduceLROnPlateau\n",
    "from tensorflow.keras.preprocessing.image import ImageDataGenerator\n",
    "from tensorflow.keras.regularizers import l2\n",
    "from tensorflow.keras import backend as K\n",
    "from tensorflow.keras.models import Model\n",
    "from tensorflow.keras.datasets import cifar10\n",
    "import numpy as np\n",
    "import os\n",
    "\n",
    "# Training parameters\n",
    "batch_size = 32  # orig paper trained all networks with batch_size=128\n",
    "epochs = 100\n",
    "steps_per_epoch = 2000\n",
    "data_augmentation = True\n",
    "num_classes = 10\n",
    "\n",
    "# Subtracting pixel mean improves accuracy\n",
    "subtract_pixel_mean = True\n",
    "\n",
    "# Model parameter\n",
    "# ----------------------------------------------------------------------------\n",
    "#           |      | 200-epoch | Orig Paper| 200-epoch | Orig Paper| sec/epoch\n",
    "# Model     |  n   | ResNet v1 | ResNet v1 | ResNet v2 | ResNet v2 | GTX1080Ti\n",
    "#           |v1(v2)| %Accuracy | %Accuracy | %Accuracy | %Accuracy | v1 (v2)\n",
    "# ----------------------------------------------------------------------------\n",
    "# ResNet20  | 3 (2)| 92.16     | 91.25     | -----     | -----     | 35 (---)\n",
    "# ResNet32  | 5(NA)| 92.46     | 92.49     | NA        | NA        | 50 ( NA)\n",
    "# ResNet44  | 7(NA)| 92.50     | 92.83     | NA        | NA        | 70 ( NA)\n",
    "# ResNet56  | 9 (6)| 92.71     | 93.03     | 93.01     | NA        | 90 (100)\n",
    "# ResNet110 |18(12)| 92.65     | 93.39+-.16| 93.15     | 93.63     | 165(180)\n",
    "# ResNet164 |27(18)| -----     | 94.07     | -----     | 94.54     | ---(---)\n",
    "# ResNet1001| (111)| -----     | 92.39     | -----     | 95.08+-.14| ---(---)\n",
    "# ---------------------------------------------------------------------------\n",
    "n = 3\n",
    "\n",
    "# Model version\n",
    "# Orig paper: version = 1 (ResNet v1), Improved ResNet: version = 2 (ResNet v2)\n",
    "version = 1\n",
    "\n",
    "# Computed depth from supplied model parameter n\n",
    "if version == 1:\n",
    "    depth = n * 6 + 2\n",
    "elif version == 2:\n",
    "    depth = n * 9 + 2\n",
    "\n",
    "# Model name, depth and version\n",
    "model_type = 'ResNet%dv%d' % (depth, version)\n",
    "\n",
    "# Load the CIFAR10 data.\n",
    "(x_train, y_train), (x_test, y_test) = cifar10.load_data()\n",
    "\n",
    "# Input image dimensions.\n",
    "input_shape = x_train.shape[1:]\n",
    "\n",
    "# Normalize data.\n",
    "x_train = x_train.astype('float32') / 255\n",
    "x_test = x_test.astype('float32') / 255\n",
    "\n",
    "# If subtract pixel mean is enabled\n",
    "if subtract_pixel_mean:\n",
    "    x_train_mean = np.mean(x_train, axis=0)\n",
    "    x_train -= x_train_mean\n",
    "    x_test -= x_train_mean\n",
    "\n",
    "print('x_train shape:', x_train.shape)\n",
    "print(x_train.shape[0], 'train samples')\n",
    "print(x_test.shape[0], 'test samples')\n",
    "print('y_train shape:', y_train.shape)\n",
    "\n",
    "# Convert class vectors to binary class matrices.\n",
    "y_train = keras.utils.to_categorical(y_train, num_classes)\n",
    "y_test = keras.utils.to_categorical(y_test, num_classes)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 定义主要函数\n",
    "\n",
    "- 学习率衰减(lr_schedule)\n",
    "- 残差层（resnet_layer)\n",
    "- resnet_v1\n",
    "- resnet_v2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def lr_schedule(epoch):\n",
    "    \"\"\"Learning Rate Schedule\n",
    "\n",
    "    Learning rate is scheduled to be reduced after 80, 120, 160, 180 epochs.\n",
    "    Called automatically every epoch as part of callbacks during training.\n",
    "\n",
    "    # Arguments\n",
    "        epoch (int): The number of epochs\n",
    "\n",
    "    # Returns\n",
    "        lr (float32): learning rate\n",
    "    \"\"\"\n",
    "    lr = 1e-3\n",
    "    if epoch > 180:\n",
    "        lr *= 0.5e-3\n",
    "    elif epoch > 160:\n",
    "        lr *= 1e-3\n",
    "    elif epoch > 120:\n",
    "        lr *= 1e-2\n",
    "    elif epoch > 80:\n",
    "        lr *= 1e-1\n",
    "    print('Learning rate: ', lr)\n",
    "    return lr\n",
    "\n",
    "\n",
    "def resnet_layer(inputs,\n",
    "                 num_filters=16,\n",
    "                 kernel_size=3,\n",
    "                 strides=1,\n",
    "                 activation='relu',\n",
    "                 batch_normalization=True,\n",
    "                 conv_first=True):\n",
    "    \"\"\"2D Convolution-Batch Normalization-Activation stack builder\n",
    "\n",
    "    # Arguments\n",
    "        inputs (tensor): input tensor from input image or previous layer\n",
    "        num_filters (int): Conv2D number of filters\n",
    "        kernel_size (int): Conv2D square kernel dimensions\n",
    "        strides (int): Conv2D square stride dimensions\n",
    "        activation (string): activation name\n",
    "        batch_normalization (bool): whether to include batch normalization\n",
    "        conv_first (bool): conv-bn-activation (True) or\n",
    "            bn-activation-conv (False)\n",
    "\n",
    "    # Returns\n",
    "        x (tensor): tensor as input to the next layer\n",
    "    \"\"\"\n",
    "    conv = Conv2D(num_filters,\n",
    "                  kernel_size=kernel_size,\n",
    "                  strides=strides,\n",
    "                  padding='same',\n",
    "                  kernel_initializer='he_normal',\n",
    "                  kernel_regularizer=l2(1e-4))\n",
    "\n",
    "    x = inputs\n",
    "    if conv_first:\n",
    "        x = conv(x)\n",
    "        if batch_normalization:\n",
    "            x = BatchNormalization()(x)\n",
    "        if activation is not None:\n",
    "            x = Activation(activation)(x)\n",
    "    else:\n",
    "        if batch_normalization:\n",
    "            x = BatchNormalization()(x)\n",
    "        if activation is not None:\n",
    "            x = Activation(activation)(x)\n",
    "        x = conv(x)\n",
    "    return x\n",
    "\n",
    "\n",
    "def resnet_v1(input_shape, depth, num_classes=10):\n",
    "    \"\"\"ResNet Version 1 Model builder [a]\n",
    "\n",
    "    Stacks of 2 x (3 x 3) Conv2D-BN-ReLU\n",
    "    Last ReLU is after the shortcut connection.\n",
    "    At the beginning of each stage, the feature map size is halved (downsampled)\n",
    "    by a convolutional layer with strides=2, while the number of filters is\n",
    "    doubled. Within each stage, the layers have the same number filters and the\n",
    "    same number of filters.\n",
    "    Features maps sizes:\n",
    "    stage 0: 32x32, 16\n",
    "    stage 1: 16x16, 32\n",
    "    stage 2:  8x8,  64\n",
    "    The Number of parameters is approx the same as Table 6 of [a]:\n",
    "    ResNet20 0.27M\n",
    "    ResNet32 0.46M\n",
    "    ResNet44 0.66M\n",
    "    ResNet56 0.85M\n",
    "    ResNet110 1.7M\n",
    "\n",
    "    # Arguments\n",
    "        input_shape (tensor): shape of input image tensor\n",
    "        depth (int): number of core convolutional layers\n",
    "        num_classes (int): number of classes (CIFAR10 has 10)\n",
    "\n",
    "    # Returns\n",
    "        model (Model): tensorflow.keras model instance\n",
    "    \"\"\"\n",
    "    if (depth - 2) % 6 != 0:\n",
    "        raise ValueError('depth should be 6n+2 (eg 20, 32, 44 in [a])')\n",
    "    # Start model definition.\n",
    "    num_filters = 16\n",
    "    num_res_blocks = int((depth - 2) / 6)\n",
    "\n",
    "    inputs = Input(shape=input_shape)\n",
    "    x = resnet_layer(inputs=inputs)\n",
    "    # Instantiate the stack of residual units\n",
    "    for stack in range(3):\n",
    "        for res_block in range(num_res_blocks):\n",
    "            strides = 1\n",
    "            if stack > 0 and res_block == 0:  # first layer but not first stack\n",
    "                strides = 2  # downsample\n",
    "            y = resnet_layer(inputs=x,\n",
    "                             num_filters=num_filters,\n",
    "                             strides=strides)\n",
    "            y = resnet_layer(inputs=y,\n",
    "                             num_filters=num_filters,\n",
    "                             activation=None)\n",
    "            if stack > 0 and res_block == 0:  # first layer but not first stack\n",
    "                # linear projection residual shortcut connection to match\n",
    "                # changed dims\n",
    "                x = resnet_layer(inputs=x,\n",
    "                                 num_filters=num_filters,\n",
    "                                 kernel_size=1,\n",
    "                                 strides=strides,\n",
    "                                 activation=None,\n",
    "                                 batch_normalization=False)\n",
    "            x = keras.layers.add([x, y])\n",
    "            x = Activation('relu')(x)\n",
    "        num_filters *= 2\n",
    "\n",
    "    # Add classifier on top.\n",
    "    # v1 does not use BN after last shortcut connection-ReLU\n",
    "    x = AveragePooling2D(pool_size=8)(x)\n",
    "    y = Flatten()(x)\n",
    "    outputs = Dense(num_classes,\n",
    "                    activation='softmax',\n",
    "                    kernel_initializer='he_normal')(y)\n",
    "\n",
    "    # Instantiate model.\n",
    "    model = Model(inputs=inputs, outputs=outputs)\n",
    "    return model\n",
    "\n",
    "\n",
    "def resnet_v2(input_shape, depth, num_classes=10):\n",
    "    \"\"\"ResNet Version 2 Model builder [b]\n",
    "\n",
    "    Stacks of (1 x 1)-(3 x 3)-(1 x 1) BN-ReLU-Conv2D or also known as\n",
    "    bottleneck layer\n",
    "    First shortcut connection per layer is 1 x 1 Conv2D.\n",
    "    Second and onwards shortcut connection is identity.\n",
    "    At the beginning of each stage, the feature map size is halved (downsampled)\n",
    "    by a convolutional layer with strides=2, while the number of filter maps is\n",
    "    doubled. Within each stage, the layers have the same number filters and the\n",
    "    same filter map sizes.\n",
    "    Features maps sizes:\n",
    "    conv1  : 32x32,  16\n",
    "    stage 0: 32x32,  64\n",
    "    stage 1: 16x16, 128\n",
    "    stage 2:  8x8,  256\n",
    "\n",
    "    # Arguments\n",
    "        input_shape (tensor): shape of input image tensor\n",
    "        depth (int): number of core convolutional layers\n",
    "        num_classes (int): number of classes (CIFAR10 has 10)\n",
    "\n",
    "    # Returns\n",
    "        model (Model): tensorflow.keras model instance\n",
    "    \"\"\"\n",
    "    if (depth - 2) % 9 != 0:\n",
    "        raise ValueError('depth should be 9n+2 (eg 56 or 110 in [b])')\n",
    "    # Start model definition.\n",
    "    num_filters_in = 16\n",
    "    num_res_blocks = int((depth - 2) / 9)\n",
    "\n",
    "    inputs = Input(shape=input_shape)\n",
    "    # v2 performs Conv2D with BN-ReLU on input before splitting into 2 paths\n",
    "    x = resnet_layer(inputs=inputs,\n",
    "                     num_filters=num_filters_in,\n",
    "                     conv_first=True)\n",
    "\n",
    "    # Instantiate the stack of residual units\n",
    "    for stage in range(3):\n",
    "        for res_block in range(num_res_blocks):\n",
    "            activation = 'relu'\n",
    "            batch_normalization = True\n",
    "            strides = 1\n",
    "            if stage == 0:\n",
    "                num_filters_out = num_filters_in * 4\n",
    "                if res_block == 0:  # first layer and first stage\n",
    "                    activation = None\n",
    "                    batch_normalization = False\n",
    "            else:\n",
    "                num_filters_out = num_filters_in * 2\n",
    "                if res_block == 0:  # first layer but not first stage\n",
    "                    strides = 2    # downsample\n",
    "\n",
    "            # bottleneck residual unit\n",
    "            y = resnet_layer(inputs=x,\n",
    "                             num_filters=num_filters_in,\n",
    "                             kernel_size=1,\n",
    "                             strides=strides,\n",
    "                             activation=activation,\n",
    "                             batch_normalization=batch_normalization,\n",
    "                             conv_first=False)\n",
    "            y = resnet_layer(inputs=y,\n",
    "                             num_filters=num_filters_in,\n",
    "                             conv_first=False)\n",
    "            y = resnet_layer(inputs=y,\n",
    "                             num_filters=num_filters_out,\n",
    "                             kernel_size=1,\n",
    "                             conv_first=False)\n",
    "            if res_block == 0:\n",
    "                # linear projection residual shortcut connection to match\n",
    "                # changed dims\n",
    "                x = resnet_layer(inputs=x,\n",
    "                                 num_filters=num_filters_out,\n",
    "                                 kernel_size=1,\n",
    "                                 strides=strides,\n",
    "                                 activation=None,\n",
    "                                 batch_normalization=False)\n",
    "            x = keras.layers.add([x, y])\n",
    "\n",
    "        num_filters_in = num_filters_out\n",
    "\n",
    "    # Add classifier on top.\n",
    "    # v2 has BN-ReLU before Pooling\n",
    "    x = BatchNormalization()(x)\n",
    "    x = Activation('relu')(x)\n",
    "    x = AveragePooling2D(pool_size=8)(x)\n",
    "    y = Flatten()(x)\n",
    "    outputs = Dense(num_classes,\n",
    "                    activation='softmax',\n",
    "                    kernel_initializer='he_normal')(y)\n",
    "\n",
    "    # Instantiate model.\n",
    "    model = Model(inputs=inputs, outputs=outputs)\n",
    "    return model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### ResNet 模型构建与训练"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 开启算子设备放置日志\n",
    "# tf.debugging.set_log_device_placement(False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Learning rate:  0.001\n",
      "Model: \"model\"\n",
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "input_1 (InputLayer)            [(None, 32, 32, 3)]  0                                            \n",
      "__________________________________________________________________________________________________\n",
      "conv2d (Conv2D)                 (None, 32, 32, 16)   448         input_1[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization (BatchNorma (None, 32, 32, 16)   64          conv2d[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "activation (Activation)         (None, 32, 32, 16)   0           batch_normalization[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_1 (Conv2D)               (None, 32, 32, 16)   2320        activation[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_1 (BatchNor (None, 32, 32, 16)   64          conv2d_1[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "activation_1 (Activation)       (None, 32, 32, 16)   0           batch_normalization_1[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_2 (Conv2D)               (None, 32, 32, 16)   2320        activation_1[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_2 (BatchNor (None, 32, 32, 16)   64          conv2d_2[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "add (Add)                       (None, 32, 32, 16)   0           activation[0][0]                 \n",
      "                                                                 batch_normalization_2[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "activation_2 (Activation)       (None, 32, 32, 16)   0           add[0][0]                        \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_3 (Conv2D)               (None, 32, 32, 16)   2320        activation_2[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_3 (BatchNor (None, 32, 32, 16)   64          conv2d_3[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "activation_3 (Activation)       (None, 32, 32, 16)   0           batch_normalization_3[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_4 (Conv2D)               (None, 32, 32, 16)   2320        activation_3[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_4 (BatchNor (None, 32, 32, 16)   64          conv2d_4[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "add_1 (Add)                     (None, 32, 32, 16)   0           activation_2[0][0]               \n",
      "                                                                 batch_normalization_4[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "activation_4 (Activation)       (None, 32, 32, 16)   0           add_1[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_5 (Conv2D)               (None, 32, 32, 16)   2320        activation_4[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_5 (BatchNor (None, 32, 32, 16)   64          conv2d_5[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "activation_5 (Activation)       (None, 32, 32, 16)   0           batch_normalization_5[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_6 (Conv2D)               (None, 32, 32, 16)   2320        activation_5[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_6 (BatchNor (None, 32, 32, 16)   64          conv2d_6[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "add_2 (Add)                     (None, 32, 32, 16)   0           activation_4[0][0]               \n",
      "                                                                 batch_normalization_6[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "activation_6 (Activation)       (None, 32, 32, 16)   0           add_2[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_7 (Conv2D)               (None, 16, 16, 32)   4640        activation_6[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_7 (BatchNor (None, 16, 16, 32)   128         conv2d_7[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "activation_7 (Activation)       (None, 16, 16, 32)   0           batch_normalization_7[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_8 (Conv2D)               (None, 16, 16, 32)   9248        activation_7[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_9 (Conv2D)               (None, 16, 16, 32)   544         activation_6[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_8 (BatchNor (None, 16, 16, 32)   128         conv2d_8[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "add_3 (Add)                     (None, 16, 16, 32)   0           conv2d_9[0][0]                   \n",
      "                                                                 batch_normalization_8[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "activation_8 (Activation)       (None, 16, 16, 32)   0           add_3[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_10 (Conv2D)              (None, 16, 16, 32)   9248        activation_8[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_9 (BatchNor (None, 16, 16, 32)   128         conv2d_10[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "activation_9 (Activation)       (None, 16, 16, 32)   0           batch_normalization_9[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_11 (Conv2D)              (None, 16, 16, 32)   9248        activation_9[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_10 (BatchNo (None, 16, 16, 32)   128         conv2d_11[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "add_4 (Add)                     (None, 16, 16, 32)   0           activation_8[0][0]               \n",
      "                                                                 batch_normalization_10[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "activation_10 (Activation)      (None, 16, 16, 32)   0           add_4[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_12 (Conv2D)              (None, 16, 16, 32)   9248        activation_10[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_11 (BatchNo (None, 16, 16, 32)   128         conv2d_12[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "activation_11 (Activation)      (None, 16, 16, 32)   0           batch_normalization_11[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_13 (Conv2D)              (None, 16, 16, 32)   9248        activation_11[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_12 (BatchNo (None, 16, 16, 32)   128         conv2d_13[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "add_5 (Add)                     (None, 16, 16, 32)   0           activation_10[0][0]              \n",
      "                                                                 batch_normalization_12[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "activation_12 (Activation)      (None, 16, 16, 32)   0           add_5[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_14 (Conv2D)              (None, 8, 8, 64)     18496       activation_12[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_13 (BatchNo (None, 8, 8, 64)     256         conv2d_14[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "activation_13 (Activation)      (None, 8, 8, 64)     0           batch_normalization_13[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_15 (Conv2D)              (None, 8, 8, 64)     36928       activation_13[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_16 (Conv2D)              (None, 8, 8, 64)     2112        activation_12[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_14 (BatchNo (None, 8, 8, 64)     256         conv2d_15[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "add_6 (Add)                     (None, 8, 8, 64)     0           conv2d_16[0][0]                  \n",
      "                                                                 batch_normalization_14[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "activation_14 (Activation)      (None, 8, 8, 64)     0           add_6[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_17 (Conv2D)              (None, 8, 8, 64)     36928       activation_14[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_15 (BatchNo (None, 8, 8, 64)     256         conv2d_17[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "activation_15 (Activation)      (None, 8, 8, 64)     0           batch_normalization_15[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_18 (Conv2D)              (None, 8, 8, 64)     36928       activation_15[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_16 (BatchNo (None, 8, 8, 64)     256         conv2d_18[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "add_7 (Add)                     (None, 8, 8, 64)     0           activation_14[0][0]              \n",
      "                                                                 batch_normalization_16[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "activation_16 (Activation)      (None, 8, 8, 64)     0           add_7[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_19 (Conv2D)              (None, 8, 8, 64)     36928       activation_16[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_17 (BatchNo (None, 8, 8, 64)     256         conv2d_19[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "activation_17 (Activation)      (None, 8, 8, 64)     0           batch_normalization_17[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_20 (Conv2D)              (None, 8, 8, 64)     36928       activation_17[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_18 (BatchNo (None, 8, 8, 64)     256         conv2d_20[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "add_8 (Add)                     (None, 8, 8, 64)     0           activation_16[0][0]              \n",
      "                                                                 batch_normalization_18[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "activation_18 (Activation)      (None, 8, 8, 64)     0           add_8[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "average_pooling2d (AveragePooli (None, 1, 1, 64)     0           activation_18[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "flatten (Flatten)               (None, 64)           0           average_pooling2d[0][0]          \n",
      "__________________________________________________________________________________________________\n",
      "dense (Dense)                   (None, 10)           650         flatten[0][0]                    \n",
      "==================================================================================================\n",
      "Total params: 274,442\n",
      "Trainable params: 273,066\n",
      "Non-trainable params: 1,376\n",
      "__________________________________________________________________________________________________\n",
      "ResNet20v1\n",
      "Using real-time data augmentation.\n",
      "WARNING:tensorflow:From <ipython-input-7-91bbc829b94a>:99: Model.fit_generator (from tensorflow.python.keras.engine.training) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use Model.fit, which supports generators.\n",
      "Learning rate:  0.001\n",
      "Epoch 1/100\n",
      "1999/2000 [============================>.] - ETA: 0s - loss: 1.5123 - accuracy: 0.5115WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "2000/2000 [==============================] - 65s 32ms/step - loss: 1.5119 - accuracy: 0.5117 - val_loss: 1.2773 - val_accuracy: 0.6056 - lr: 0.0010\n",
      "Learning rate:  0.001\n",
      "Epoch 2/100\n",
      "1999/2000 [============================>.] - ETA: 0s - loss: 1.0730 - accuracy: 0.6789WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "2000/2000 [==============================] - 63s 32ms/step - loss: 1.0731 - accuracy: 0.6790 - val_loss: 0.9875 - val_accuracy: 0.7061 - lr: 0.0010\n",
      "Learning rate:  0.001\n",
      "Epoch 3/100\n",
      "1999/2000 [============================>.] - ETA: 0s - loss: 0.9292 - accuracy: 0.7334WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "2000/2000 [==============================] - 64s 32ms/step - loss: 0.9291 - accuracy: 0.7334 - val_loss: 1.1520 - val_accuracy: 0.6750 - lr: 0.0010\n",
      "Learning rate:  0.001\n",
      "Epoch 4/100\n",
      "1999/2000 [============================>.] - ETA: 0s - loss: 0.8534 - accuracy: 0.7628WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "2000/2000 [==============================] - 67s 33ms/step - loss: 0.8533 - accuracy: 0.7629 - val_loss: 0.9538 - val_accuracy: 0.7437 - lr: 0.0010\n",
      "Learning rate:  0.001\n",
      "Epoch 5/100\n",
      "1999/2000 [============================>.] - ETA: 0s - loss: 0.7871 - accuracy: 0.7884WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "2000/2000 [==============================] - 66s 33ms/step - loss: 0.7871 - accuracy: 0.7884 - val_loss: 0.8583 - val_accuracy: 0.7745 - lr: 0.0010\n",
      "Learning rate:  0.001\n",
      "Epoch 6/100\n",
      "2000/2000 [==============================] - ETA: 0s - loss: 0.7539 - accuracy: 0.8013WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "2000/2000 [==============================] - 67s 33ms/step - loss: 0.7539 - accuracy: 0.8013 - val_loss: 0.8469 - val_accuracy: 0.7791 - lr: 0.0010\n",
      "Learning rate:  0.001\n",
      "Epoch 7/100\n",
      "2000/2000 [==============================] - ETA: 0s - loss: 0.7235 - accuracy: 0.8134WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "2000/2000 [==============================] - 66s 33ms/step - loss: 0.7235 - accuracy: 0.8134 - val_loss: 0.8177 - val_accuracy: 0.7855 - lr: 0.0010\n",
      "Learning rate:  0.001\n",
      "Epoch 8/100\n",
      "1999/2000 [============================>.] - ETA: 0s - loss: 0.6935 - accuracy: 0.8245WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "2000/2000 [==============================] - 67s 33ms/step - loss: 0.6936 - accuracy: 0.8245 - val_loss: 0.8019 - val_accuracy: 0.7894 - lr: 0.0010\n",
      "Learning rate:  0.001\n",
      "Epoch 9/100\n",
      "2000/2000 [==============================] - ETA: 0s - loss: 0.6799 - accuracy: 0.8289WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "2000/2000 [==============================] - 67s 34ms/step - loss: 0.6799 - accuracy: 0.8289 - val_loss: 0.8400 - val_accuracy: 0.7859 - lr: 0.0010\n",
      "Learning rate:  0.001\n",
      "Epoch 10/100\n",
      "1999/2000 [============================>.] - ETA: 0s - loss: 0.6671 - accuracy: 0.8350WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "2000/2000 [==============================] - 69s 35ms/step - loss: 0.6669 - accuracy: 0.8351 - val_loss: 0.7788 - val_accuracy: 0.8103 - lr: 0.0010\n",
      "Learning rate:  0.001\n",
      "Epoch 11/100\n",
      "1999/2000 [============================>.] - ETA: 0s - loss: 0.6445 - accuracy: 0.8438WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "2000/2000 [==============================] - 67s 33ms/step - loss: 0.6447 - accuracy: 0.8438 - val_loss: 0.7938 - val_accuracy: 0.8056 - lr: 0.0010\n",
      "Learning rate:  0.001\n",
      "Epoch 12/100\n",
      "2000/2000 [==============================] - ETA: 0s - loss: 0.6337 - accuracy: 0.8477WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "2000/2000 [==============================] - 66s 33ms/step - loss: 0.6337 - accuracy: 0.8477 - val_loss: 0.6753 - val_accuracy: 0.8359 - lr: 0.0010\n",
      "Learning rate:  0.001\n",
      "Epoch 13/100\n",
      "1998/2000 [============================>.] - ETA: 0s - loss: 0.6265 - accuracy: 0.8521 ETA: 0s - loss: 0.6272 - WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "2000/2000 [==============================] - 67s 33ms/step - loss: 0.6264 - accuracy: 0.8522 - val_loss: 0.7358 - val_accuracy: 0.8155 - lr: 0.0010\n",
      "Learning rate:  0.001\n",
      "Epoch 14/100\n",
      "1999/2000 [============================>.] - ETA: 0s - loss: 0.6194 - accuracy: 0.8542WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "2000/2000 [==============================] - 65s 33ms/step - loss: 0.6195 - accuracy: 0.8542 - val_loss: 0.7099 - val_accuracy: 0.8292 - lr: 0.0010\n",
      "Learning rate:  0.001\n",
      "Epoch 15/100\n",
      "1998/2000 [============================>.] - ETA: 0s - loss: 0.6006 - accuracy: 0.8604WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "2000/2000 [==============================] - 66s 33ms/step - loss: 0.6006 - accuracy: 0.8604 - val_loss: 0.7944 - val_accuracy: 0.8068 - lr: 0.0010\n",
      "Learning rate:  0.001\n",
      "Epoch 16/100\n",
      "1999/2000 [============================>.] - ETA: 0s - loss: 0.5977 - accuracy: 0.8629WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "2000/2000 [==============================] - 67s 33ms/step - loss: 0.5978 - accuracy: 0.8629 - val_loss: 0.7223 - val_accuracy: 0.8314 - lr: 0.0010\n",
      "Learning rate:  0.001\n",
      "Epoch 17/100\n",
      "1999/2000 [============================>.] - ETA: 0s - loss: 0.5935 - accuracy: 0.8649WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "2000/2000 [==============================] - 66s 33ms/step - loss: 0.5934 - accuracy: 0.8649 - val_loss: 0.7218 - val_accuracy: 0.8295 - lr: 3.1623e-04\n",
      "Learning rate:  0.001\n",
      "Epoch 18/100\n",
      "1999/2000 [============================>.] - ETA: 0s - loss: 0.5874 - accuracy: 0.8658WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "2000/2000 [==============================] - 66s 33ms/step - loss: 0.5873 - accuracy: 0.8658 - val_loss: 0.7775 - val_accuracy: 0.8189 - lr: 0.0010\n",
      "Learning rate:  0.001\n",
      "Epoch 19/100\n",
      "1999/2000 [============================>.] - ETA: 0s - loss: 0.5785 - accuracy: 0.8694WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "2000/2000 [==============================] - 66s 33ms/step - loss: 0.5784 - accuracy: 0.8694 - val_loss: 0.6812 - val_accuracy: 0.8396 - lr: 0.0010\n",
      "Learning rate:  0.001\n",
      "Epoch 20/100\n",
      "1999/2000 [============================>.] - ETA: 0s - loss: 0.5716 - accuracy: 0.8711WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "2000/2000 [==============================] - 65s 32ms/step - loss: 0.5716 - accuracy: 0.8711 - val_loss: 0.6863 - val_accuracy: 0.8381 - lr: 0.0010\n",
      "Learning rate:  0.001\n",
      "Epoch 21/100\n",
      "1999/2000 [============================>.] - ETA: 0s - loss: 0.5731 - accuracy: 0.8715WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "2000/2000 [==============================] - 65s 33ms/step - loss: 0.5732 - accuracy: 0.8715 - val_loss: 0.6833 - val_accuracy: 0.8430 - lr: 0.0010\n",
      "Learning rate:  0.001\n",
      "Epoch 22/100\n",
      "1999/2000 [============================>.] - ETA: 0s - loss: 0.5636 - accuracy: 0.8752WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "2000/2000 [==============================] - 66s 33ms/step - loss: 0.5637 - accuracy: 0.8752 - val_loss: 0.7992 - val_accuracy: 0.8083 - lr: 3.1623e-04\n",
      "Learning rate:  0.001\n",
      "Epoch 23/100\n",
      "2000/2000 [==============================] - ETA: 0s - loss: 0.5620 - accuracy: 0.8750 ETA: 0s - loss: 0.5WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "2000/2000 [==============================] - 66s 33ms/step - loss: 0.5620 - accuracy: 0.8750 - val_loss: 0.8196 - val_accuracy: 0.7974 - lr: 0.0010\n",
      "Learning rate:  0.001\n",
      "Epoch 24/100\n",
      "1999/2000 [============================>.] - ETA: 0s - loss: 0.5571 - accuracy: 0.8767WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "2000/2000 [==============================] - 65s 33ms/step - loss: 0.5573 - accuracy: 0.8766 - val_loss: 0.6576 - val_accuracy: 0.8498 - lr: 0.0010\n",
      "Learning rate:  0.001\n",
      "Epoch 25/100\n",
      "1998/2000 [============================>.] - ETA: 0s - loss: 0.5591 - accuracy: 0.8780WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "2000/2000 [==============================] - 66s 33ms/step - loss: 0.5589 - accuracy: 0.8781 - val_loss: 0.7656 - val_accuracy: 0.8160 - lr: 0.0010\n",
      "Learning rate:  0.001\n",
      "Epoch 26/100\n",
      "2000/2000 [==============================] - ETA: 0s - loss: 0.5442 - accuracy: 0.8817 ETA: 0s - losWARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "2000/2000 [==============================] - 66s 33ms/step - loss: 0.5442 - accuracy: 0.8817 - val_loss: 0.6681 - val_accuracy: 0.8478 - lr: 0.0010\n",
      "Learning rate:  0.001\n",
      "Epoch 27/100\n",
      "2000/2000 [==============================] - ETA: 0s - loss: 0.5514 - accuracy: 0.8801WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "2000/2000 [==============================] - 68s 34ms/step - loss: 0.5514 - accuracy: 0.8801 - val_loss: 0.7909 - val_accuracy: 0.8208 - lr: 0.0010\n",
      "Learning rate:  0.001\n",
      "Epoch 28/100\n",
      "1999/2000 [============================>.] - ETA: 0s - loss: 0.5446 - accuracy: 0.8821WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "2000/2000 [==============================] - 70s 35ms/step - loss: 0.5446 - accuracy: 0.8821 - val_loss: 0.9289 - val_accuracy: 0.7924 - lr: 0.0010\n",
      "Learning rate:  0.001\n",
      "Epoch 29/100\n",
      "2000/2000 [==============================] - ETA: 0s - loss: 0.5416 - accuracy: 0.8833WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "2000/2000 [==============================] - 68s 34ms/step - loss: 0.5416 - accuracy: 0.8833 - val_loss: 0.7544 - val_accuracy: 0.8264 - lr: 3.1623e-04\n",
      "Learning rate:  0.001\n",
      "Epoch 30/100\n",
      "2000/2000 [==============================] - ETA: 0s - loss: 0.5356 - accuracy: 0.8850WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "2000/2000 [==============================] - 66s 33ms/step - loss: 0.5356 - accuracy: 0.8850 - val_loss: 0.6211 - val_accuracy: 0.8613 - lr: 0.0010\n",
      "Learning rate:  0.001\n",
      "Epoch 31/100\n",
      "1999/2000 [============================>.] - ETA: 0s - loss: 0.5367 - accuracy: 0.8857WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "2000/2000 [==============================] - 66s 33ms/step - loss: 0.5368 - accuracy: 0.8856 - val_loss: 0.8102 - val_accuracy: 0.8072 - lr: 0.0010\n",
      "Learning rate:  0.001\n",
      "Epoch 32/100\n",
      "1999/2000 [============================>.] - ETA: 0s - loss: 0.5361 - accuracy: 0.8871WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "2000/2000 [==============================] - 66s 33ms/step - loss: 0.5361 - accuracy: 0.8871 - val_loss: 0.7500 - val_accuracy: 0.8264 - lr: 0.0010\n",
      "Learning rate:  0.001\n",
      "Epoch 33/100\n",
      "1999/2000 [============================>.] - ETA: 0s - loss: 0.5360 - accuracy: 0.8871WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "2000/2000 [==============================] - 66s 33ms/step - loss: 0.5359 - accuracy: 0.8871 - val_loss: 0.6196 - val_accuracy: 0.8632 - lr: 0.0010\n",
      "Learning rate:  0.001\n",
      "Epoch 34/100\n",
      "2000/2000 [==============================] - ETA: 0s - loss: 0.5341 - accuracy: 0.8870WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "2000/2000 [==============================] - 66s 33ms/step - loss: 0.5341 - accuracy: 0.8870 - val_loss: 0.6896 - val_accuracy: 0.8446 - lr: 0.0010\n",
      "Learning rate:  0.001\n",
      "Epoch 35/100\n",
      "1998/2000 [============================>.] - ETA: 0s - loss: 0.5267 - accuracy: 0.8889WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "2000/2000 [==============================] - 65s 33ms/step - loss: 0.5266 - accuracy: 0.8890 - val_loss: 0.7639 - val_accuracy: 0.8263 - lr: 0.0010\n",
      "Learning rate:  0.001\n",
      "Epoch 36/100\n",
      "2000/2000 [==============================] - ETA: 0s - loss: 0.5214 - accuracy: 0.8917WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "2000/2000 [==============================] - 67s 33ms/step - loss: 0.5214 - accuracy: 0.8917 - val_loss: 0.6874 - val_accuracy: 0.8452 - lr: 0.0010\n",
      "Learning rate:  0.001\n",
      "Epoch 37/100\n",
      "1999/2000 [============================>.] - ETA: 0s - loss: 0.5232 - accuracy: 0.8909WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "2000/2000 [==============================] - 66s 33ms/step - loss: 0.5231 - accuracy: 0.8909 - val_loss: 0.7355 - val_accuracy: 0.8316 - lr: 0.0010\n",
      "Learning rate:  0.001\n",
      "Epoch 38/100\n",
      "2000/2000 [==============================] - ETA: 0s - loss: 0.5256 - accuracy: 0.8897WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "2000/2000 [==============================] - 66s 33ms/step - loss: 0.5256 - accuracy: 0.8897 - val_loss: 0.6979 - val_accuracy: 0.8476 - lr: 3.1623e-04\n",
      "Learning rate:  0.001\n",
      "Epoch 39/100\n",
      "1999/2000 [============================>.] - ETA: 0s - loss: 0.5169 - accuracy: 0.8929WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "2000/2000 [==============================] - 66s 33ms/step - loss: 0.5168 - accuracy: 0.8930 - val_loss: 0.6897 - val_accuracy: 0.8474 - lr: 0.0010\n",
      "Learning rate:  0.001\n",
      "Epoch 40/100\n",
      "1999/2000 [============================>.] - ETA: 0s - loss: 0.5192 - accuracy: 0.8919WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "2000/2000 [==============================] - 65s 33ms/step - loss: 0.5192 - accuracy: 0.8919 - val_loss: 0.7398 - val_accuracy: 0.8308 - lr: 0.0010\n",
      "Learning rate:  0.001\n",
      "Epoch 41/100\n",
      "2000/2000 [==============================] - ETA: 0s - loss: 0.5162 - accuracy: 0.8932WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "2000/2000 [==============================] - 66s 33ms/step - loss: 0.5162 - accuracy: 0.8932 - val_loss: 0.8664 - val_accuracy: 0.8037 - lr: 0.0010\n",
      "Learning rate:  0.001\n",
      "Epoch 42/100\n",
      "1999/2000 [============================>.] - ETA: 0s - loss: 0.5147 - accuracy: 0.8934WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "2000/2000 [==============================] - 66s 33ms/step - loss: 0.5147 - accuracy: 0.8934 - val_loss: 0.9441 - val_accuracy: 0.7778 - lr: 0.0010\n",
      "Learning rate:  0.001\n",
      "Epoch 43/100\n",
      "1999/2000 [============================>.] - ETA: 0s - loss: 0.5124 - accuracy: 0.8954WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "2000/2000 [==============================] - 67s 33ms/step - loss: 0.5123 - accuracy: 0.8954 - val_loss: 0.6384 - val_accuracy: 0.8639 - lr: 3.1623e-04\n",
      "Learning rate:  0.001\n",
      "Epoch 44/100\n",
      "1999/2000 [============================>.] - ETA: 0s - loss: 0.5115 - accuracy: 0.8936WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "2000/2000 [==============================] - 66s 33ms/step - loss: 0.5114 - accuracy: 0.8936 - val_loss: 0.7372 - val_accuracy: 0.8353 - lr: 0.0010\n",
      "Learning rate:  0.001\n",
      "Epoch 45/100\n",
      "2000/2000 [==============================] - ETA: 0s - loss: 0.5106 - accuracy: 0.8954WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "2000/2000 [==============================] - 65s 32ms/step - loss: 0.5106 - accuracy: 0.8954 - val_loss: 0.7024 - val_accuracy: 0.8440 - lr: 0.0010\n",
      "Learning rate:  0.001\n",
      "Epoch 46/100\n",
      "2000/2000 [==============================] - ETA: 0s - loss: 0.5080 - accuracy: 0.8950WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "2000/2000 [==============================] - 67s 33ms/step - loss: 0.5080 - accuracy: 0.8950 - val_loss: 0.8757 - val_accuracy: 0.8030 - lr: 0.0010\n",
      "Learning rate:  0.001\n",
      "Epoch 47/100\n",
      "1999/2000 [============================>.] - ETA: 0s - loss: 0.5079 - accuracy: 0.8946WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "2000/2000 [==============================] - 66s 33ms/step - loss: 0.5078 - accuracy: 0.8946 - val_loss: 0.6424 - val_accuracy: 0.8594 - lr: 0.0010\n",
      "Learning rate:  0.001\n",
      "Epoch 48/100\n",
      "2000/2000 [==============================] - ETA: 0s - loss: 0.4999 - accuracy: 0.8977 ETA: 1s - lWARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "2000/2000 [==============================] - 66s 33ms/step - loss: 0.4999 - accuracy: 0.8977 - val_loss: 0.6927 - val_accuracy: 0.8428 - lr: 3.1623e-04\n",
      "Learning rate:  0.001\n",
      "Epoch 49/100\n",
      "1999/2000 [============================>.] - ETA: 0s - loss: 0.5032 - accuracy: 0.8968WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "2000/2000 [==============================] - 66s 33ms/step - loss: 0.5033 - accuracy: 0.8968 - val_loss: 0.7879 - val_accuracy: 0.8238 - lr: 0.0010\n",
      "Learning rate:  0.001\n",
      "Epoch 50/100\n",
      "1999/2000 [============================>.] - ETA: 0s - loss: 0.5039 - accuracy: 0.8981WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "2000/2000 [==============================] - 66s 33ms/step - loss: 0.5039 - accuracy: 0.8981 - val_loss: 0.7348 - val_accuracy: 0.8338 - lr: 0.0010\n",
      "Learning rate:  0.001\n",
      "Epoch 51/100\n",
      "1999/2000 [============================>.] - ETA: 0s - loss: 0.4999 - accuracy: 0.8983WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "2000/2000 [==============================] - 69s 34ms/step - loss: 0.5000 - accuracy: 0.8983 - val_loss: 0.6507 - val_accuracy: 0.8563 - lr: 0.0010\n",
      "Learning rate:  0.001\n",
      "Epoch 52/100\n",
      "1999/2000 [============================>.] - ETA: 0s - loss: 0.4988 - accuracy: 0.8982WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "2000/2000 [==============================] - 86s 43ms/step - loss: 0.4988 - accuracy: 0.8982 - val_loss: 0.6344 - val_accuracy: 0.8632 - lr: 0.0010\n",
      "Learning rate:  0.001\n",
      "Epoch 53/100\n",
      "2000/2000 [==============================] - ETA: 0s - loss: 0.5039 - accuracy: 0.8971WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "2000/2000 [==============================] - 88s 44ms/step - loss: 0.5039 - accuracy: 0.8971 - val_loss: 0.6056 - val_accuracy: 0.8689 - lr: 0.0010\n",
      "Learning rate:  0.001\n",
      "Epoch 54/100\n",
      "2000/2000 [==============================] - ETA: 0s - loss: 0.4992 - accuracy: 0.8983WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "2000/2000 [==============================] - 85s 42ms/step - loss: 0.4992 - accuracy: 0.8983 - val_loss: 0.6284 - val_accuracy: 0.8572 - lr: 0.0010\n",
      "Learning rate:  0.001\n",
      "Epoch 55/100\n",
      "1998/2000 [============================>.] - ETA: 0s - loss: 0.4943 - accuracy: 0.9000 ETA: 0s - loss: 0.4943 - accuracy: 0.90WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "2000/2000 [==============================] - 86s 43ms/step - loss: 0.4944 - accuracy: 0.8999 - val_loss: 0.6918 - val_accuracy: 0.8478 - lr: 0.0010\n",
      "Learning rate:  0.001\n",
      "Epoch 56/100\n",
      "2000/2000 [==============================] - ETA: 0s - loss: 0.4947 - accuracy: 0.9009WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "2000/2000 [==============================] - 87s 44ms/step - loss: 0.4947 - accuracy: 0.9009 - val_loss: 0.7568 - val_accuracy: 0.8362 - lr: 0.0010\n",
      "Learning rate:  0.001\n",
      "Epoch 57/100\n",
      "1999/2000 [============================>.] - ETA: 0s - loss: 0.5003 - accuracy: 0.8989WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "2000/2000 [==============================] - 89s 44ms/step - loss: 0.5002 - accuracy: 0.8989 - val_loss: 0.5976 - val_accuracy: 0.8733 - lr: 0.0010\n",
      "Learning rate:  0.001\n",
      "Epoch 58/100\n",
      "2000/2000 [==============================] - ETA: 0s - loss: 0.4934 - accuracy: 0.9011WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "2000/2000 [==============================] - 88s 44ms/step - loss: 0.4934 - accuracy: 0.9011 - val_loss: 0.7487 - val_accuracy: 0.8361 - lr: 0.0010\n",
      "Learning rate:  0.001\n",
      "Epoch 59/100\n",
      "1998/2000 [============================>.] - ETA: 0s - loss: 0.4911 - accuracy: 0.9016WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "2000/2000 [==============================] - 89s 44ms/step - loss: 0.4911 - accuracy: 0.9016 - val_loss: 0.6638 - val_accuracy: 0.8554 - lr: 0.0010\n",
      "Learning rate:  0.001\n",
      "Epoch 60/100\n",
      "1999/2000 [============================>.] - ETA: 0s - loss: 0.4935 - accuracy: 0.9003WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "2000/2000 [==============================] - 85s 42ms/step - loss: 0.4935 - accuracy: 0.9003 - val_loss: 0.6585 - val_accuracy: 0.8546 - lr: 0.0010\n",
      "Learning rate:  0.001\n",
      "Epoch 61/100\n",
      "1998/2000 [============================>.] - ETA: 0s - loss: 0.4922 - accuracy: 0.9006WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "2000/2000 [==============================] - 91s 46ms/step - loss: 0.4923 - accuracy: 0.9006 - val_loss: 0.6410 - val_accuracy: 0.8589 - lr: 0.0010\n",
      "Learning rate:  0.001\n",
      "Epoch 62/100\n",
      "1999/2000 [============================>.] - ETA: 0s - loss: 0.4894 - accuracy: 0.9026WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "2000/2000 [==============================] - 91s 45ms/step - loss: 0.4894 - accuracy: 0.9026 - val_loss: 0.6053 - val_accuracy: 0.8715 - lr: 3.1623e-04\n",
      "Learning rate:  0.001\n",
      "Epoch 63/100\n",
      "1999/2000 [============================>.] - ETA: 0s - loss: 0.4876 - accuracy: 0.9026WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "2000/2000 [==============================] - 87s 43ms/step - loss: 0.4877 - accuracy: 0.9026 - val_loss: 0.8684 - val_accuracy: 0.8027 - lr: 0.0010\n",
      "Learning rate:  0.001\n",
      "Epoch 64/100\n",
      "1998/2000 [============================>.] - ETA: 0s - loss: 0.4890 - accuracy: 0.9015WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "2000/2000 [==============================] - 88s 44ms/step - loss: 0.4890 - accuracy: 0.9015 - val_loss: 0.6872 - val_accuracy: 0.8411 - lr: 0.0010\n",
      "Learning rate:  0.001\n",
      "Epoch 65/100\n",
      "2000/2000 [==============================] - ETA: 0s - loss: 0.4855 - accuracy: 0.9041WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "2000/2000 [==============================] - 90s 45ms/step - loss: 0.4855 - accuracy: 0.9041 - val_loss: 0.5964 - val_accuracy: 0.8747 - lr: 0.0010\n",
      "Learning rate:  0.001\n",
      "Epoch 66/100\n",
      "2000/2000 [==============================] - ETA: 0s - loss: 0.4849 - accuracy: 0.9040WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "2000/2000 [==============================] - 89s 45ms/step - loss: 0.4849 - accuracy: 0.9040 - val_loss: 0.7690 - val_accuracy: 0.8259 - lr: 0.0010\n",
      "Learning rate:  0.001\n",
      "Epoch 67/100\n",
      "1998/2000 [============================>.] - ETA: 0s - loss: 0.4823 - accuracy: 0.9038WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "2000/2000 [==============================] - 88s 44ms/step - loss: 0.4823 - accuracy: 0.9038 - val_loss: 0.7765 - val_accuracy: 0.8295 - lr: 0.0010\n",
      "Learning rate:  0.001\n",
      "Epoch 68/100\n",
      "2000/2000 [==============================] - ETA: 0s - loss: 0.4896 - accuracy: 0.9021WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "2000/2000 [==============================] - 90s 45ms/step - loss: 0.4896 - accuracy: 0.9021 - val_loss: 0.7100 - val_accuracy: 0.8407 - lr: 0.0010\n",
      "Learning rate:  0.001\n",
      "Epoch 69/100\n",
      "1999/2000 [============================>.] - ETA: 0s - loss: 0.4792 - accuracy: 0.9053WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "2000/2000 [==============================] - 88s 44ms/step - loss: 0.4792 - accuracy: 0.9053 - val_loss: 0.6485 - val_accuracy: 0.8582 - lr: 0.0010\n",
      "Learning rate:  0.001\n",
      "Epoch 70/100\n",
      "1999/2000 [============================>.] - ETA: 0s - loss: 0.4841 - accuracy: 0.9041WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "2000/2000 [==============================] - 86s 43ms/step - loss: 0.4840 - accuracy: 0.9041 - val_loss: 0.9206 - val_accuracy: 0.8046 - lr: 3.1623e-04\n",
      "Learning rate:  0.001\n",
      "Epoch 71/100\n",
      "2000/2000 [==============================] - ETA: 0s - loss: 0.4791 - accuracy: 0.9038WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "2000/2000 [==============================] - 90s 45ms/step - loss: 0.4791 - accuracy: 0.9038 - val_loss: 0.6255 - val_accuracy: 0.8652 - lr: 0.0010\n",
      "Learning rate:  0.001\n",
      "Epoch 72/100\n",
      "2000/2000 [==============================] - ETA: 0s - loss: 0.4805 - accuracy: 0.9053WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "2000/2000 [==============================] - 83s 42ms/step - loss: 0.4805 - accuracy: 0.9053 - val_loss: 0.6977 - val_accuracy: 0.8423 - lr: 0.0010\n",
      "Learning rate:  0.001\n",
      "Epoch 73/100\n",
      "1999/2000 [============================>.] - ETA: 0s - loss: 0.4808 - accuracy: 0.9052WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "2000/2000 [==============================] - 65s 33ms/step - loss: 0.4808 - accuracy: 0.9052 - val_loss: 0.6452 - val_accuracy: 0.8560 - lr: 0.0010\n",
      "Learning rate:  0.001\n",
      "Epoch 74/100\n",
      "2000/2000 [==============================] - ETA: 0s - loss: 0.4827 - accuracy: 0.9030 ETA: 0s - loss: 0.4825 - accuracy: 0.WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "2000/2000 [==============================] - 65s 32ms/step - loss: 0.4827 - accuracy: 0.9030 - val_loss: 0.6463 - val_accuracy: 0.8543 - lr: 0.0010\n",
      "Learning rate:  0.001\n",
      "Epoch 75/100\n",
      "1999/2000 [============================>.] - ETA: 0s - loss: 0.4805 - accuracy: 0.9043WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "2000/2000 [==============================] - 64s 32ms/step - loss: 0.4806 - accuracy: 0.9043 - val_loss: 0.6958 - val_accuracy: 0.8439 - lr: 3.1623e-04\n",
      "Learning rate:  0.001\n",
      "Epoch 76/100\n",
      "2000/2000 [==============================] - ETA: 0s - loss: 0.4762 - accuracy: 0.9060WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "2000/2000 [==============================] - 66s 33ms/step - loss: 0.4762 - accuracy: 0.9060 - val_loss: 0.7772 - val_accuracy: 0.8257 - lr: 0.0010\n",
      "Learning rate:  0.001\n",
      "Epoch 77/100\n",
      "2000/2000 [==============================] - ETA: 0s - loss: 0.4764 - accuracy: 0.9059WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "2000/2000 [==============================] - 64s 32ms/step - loss: 0.4764 - accuracy: 0.9059 - val_loss: 0.6448 - val_accuracy: 0.8598 - lr: 0.0010\n",
      "Learning rate:  0.001\n",
      "Epoch 78/100\n",
      "2000/2000 [==============================] - ETA: 0s - loss: 0.4761 - accuracy: 0.9062WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "2000/2000 [==============================] - 64s 32ms/step - loss: 0.4761 - accuracy: 0.9062 - val_loss: 0.5849 - val_accuracy: 0.8727 - lr: 0.0010\n",
      "Learning rate:  0.001\n",
      "Epoch 79/100\n",
      "2000/2000 [==============================] - ETA: 0s - loss: 0.4754 - accuracy: 0.9061WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "2000/2000 [==============================] - 64s 32ms/step - loss: 0.4754 - accuracy: 0.9061 - val_loss: 0.6494 - val_accuracy: 0.8514 - lr: 0.0010\n",
      "Learning rate:  0.001\n",
      "Epoch 80/100\n",
      "1999/2000 [============================>.] - ETA: 0s - loss: 0.4705 - accuracy: 0.9097WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "2000/2000 [==============================] - 63s 32ms/step - loss: 0.4706 - accuracy: 0.9096 - val_loss: 0.6957 - val_accuracy: 0.8453 - lr: 0.0010\n",
      "Learning rate:  0.001\n",
      "Epoch 81/100\n",
      "1999/2000 [============================>.] - ETA: 0s - loss: 0.4756 - accuracy: 0.9060WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "2000/2000 [==============================] - 65s 32ms/step - loss: 0.4757 - accuracy: 0.9059 - val_loss: 0.7770 - val_accuracy: 0.8174 - lr: 0.0010\n",
      "Learning rate:  0.0001\n",
      "Epoch 82/100\n",
      "2000/2000 [==============================] - ETA: 0s - loss: 0.3960 - accuracy: 0.9341WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "2000/2000 [==============================] - 64s 32ms/step - loss: 0.3960 - accuracy: 0.9341 - val_loss: 0.4731 - val_accuracy: 0.9105 - lr: 1.0000e-04\n",
      "Learning rate:  0.0001\n",
      "Epoch 83/100\n",
      "1998/2000 [============================>.] - ETA: 0s - loss: 0.3548 - accuracy: 0.9469 ETA: 0s - loss: 0.3548 - accuracy: WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "2000/2000 [==============================] - 64s 32ms/step - loss: 0.3547 - accuracy: 0.9469 - val_loss: 0.4613 - val_accuracy: 0.9127 - lr: 1.0000e-04\n",
      "Learning rate:  0.0001\n",
      "Epoch 84/100\n",
      "2000/2000 [==============================] - ETA: 0s - loss: 0.3379 - accuracy: 0.9504WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "2000/2000 [==============================] - 64s 32ms/step - loss: 0.3379 - accuracy: 0.9504 - val_loss: 0.4522 - val_accuracy: 0.9139 - lr: 1.0000e-04\n",
      "Learning rate:  0.0001\n",
      "Epoch 85/100\n",
      "2000/2000 [==============================] - ETA: 0s - loss: 0.3248 - accuracy: 0.9539WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "2000/2000 [==============================] - 63s 32ms/step - loss: 0.3248 - accuracy: 0.9539 - val_loss: 0.4441 - val_accuracy: 0.9154 - lr: 1.0000e-04\n",
      "Learning rate:  0.0001\n",
      "Epoch 86/100\n",
      "1999/2000 [============================>.] - ETA: 0s - loss: 0.3161 - accuracy: 0.9557WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "2000/2000 [==============================] - 63s 32ms/step - loss: 0.3160 - accuracy: 0.9557 - val_loss: 0.4568 - val_accuracy: 0.9104 - lr: 1.0000e-04\n",
      "Learning rate:  0.0001\n",
      "Epoch 87/100\n",
      "2000/2000 [==============================] - ETA: 0s - loss: 0.3042 - accuracy: 0.9580WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "2000/2000 [==============================] - 64s 32ms/step - loss: 0.3042 - accuracy: 0.9580 - val_loss: 0.4369 - val_accuracy: 0.9166 - lr: 1.0000e-04\n",
      "Learning rate:  0.0001\n",
      "Epoch 88/100\n",
      "2000/2000 [==============================] - ETA: 0s - loss: 0.2968 - accuracy: 0.9594WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "2000/2000 [==============================] - 64s 32ms/step - loss: 0.2968 - accuracy: 0.9594 - val_loss: 0.4444 - val_accuracy: 0.9143 - lr: 1.0000e-04\n",
      "Learning rate:  0.0001\n",
      "Epoch 89/100\n",
      "2000/2000 [==============================] - ETA: 0s - loss: 0.2912 - accuracy: 0.9603WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "2000/2000 [==============================] - 63s 32ms/step - loss: 0.2912 - accuracy: 0.9603 - val_loss: 0.4449 - val_accuracy: 0.9147 - lr: 1.0000e-04\n",
      "Learning rate:  0.0001\n",
      "Epoch 90/100\n",
      "1999/2000 [============================>.] - ETA: 0s - loss: 0.2823 - accuracy: 0.9618WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "2000/2000 [==============================] - 64s 32ms/step - loss: 0.2823 - accuracy: 0.9618 - val_loss: 0.4407 - val_accuracy: 0.9161 - lr: 1.0000e-04\n",
      "Learning rate:  0.0001\n",
      "Epoch 91/100\n",
      "2000/2000 [==============================] - ETA: 0s - loss: 0.2766 - accuracy: 0.9637WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "2000/2000 [==============================] - 63s 32ms/step - loss: 0.2766 - accuracy: 0.9637 - val_loss: 0.4343 - val_accuracy: 0.9162 - lr: 1.0000e-04\n",
      "Learning rate:  0.0001\n",
      "Epoch 92/100\n",
      "1999/2000 [============================>.] - ETA: 0s - loss: 0.2693 - accuracy: 0.9637 ETA: 1s -WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "2000/2000 [==============================] - 62s 31ms/step - loss: 0.2693 - accuracy: 0.9637 - val_loss: 0.4354 - val_accuracy: 0.9169 - lr: 1.0000e-04\n",
      "Learning rate:  0.0001\n",
      "Epoch 93/100\n",
      "1999/2000 [============================>.] - ETA: 0s - loss: 0.2642 - accuracy: 0.9651WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "2000/2000 [==============================] - 63s 31ms/step - loss: 0.2642 - accuracy: 0.9651 - val_loss: 0.4313 - val_accuracy: 0.9151 - lr: 1.0000e-04\n",
      "Learning rate:  0.0001\n",
      "Epoch 94/100\n",
      "1999/2000 [============================>.] - ETA: 0s - loss: 0.2583 - accuracy: 0.9653WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "2000/2000 [==============================] - 64s 32ms/step - loss: 0.2583 - accuracy: 0.9653 - val_loss: 0.4334 - val_accuracy: 0.9146 - lr: 1.0000e-04\n",
      "Learning rate:  0.0001\n",
      "Epoch 95/100\n",
      "1999/2000 [============================>.] - ETA: 0s - loss: 0.2539 - accuracy: 0.9667WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "2000/2000 [==============================] - 66s 33ms/step - loss: 0.2538 - accuracy: 0.9667 - val_loss: 0.4357 - val_accuracy: 0.9153 - lr: 1.0000e-04\n",
      "Learning rate:  0.0001\n",
      "Epoch 96/100\n",
      "1998/2000 [============================>.] - ETA: 0s - loss: 0.2476 - accuracy: 0.9684WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "2000/2000 [==============================] - 64s 32ms/step - loss: 0.2476 - accuracy: 0.9683 - val_loss: 0.4387 - val_accuracy: 0.9127 - lr: 1.0000e-04\n",
      "Learning rate:  0.0001\n",
      "Epoch 97/100\n",
      "2000/2000 [==============================] - ETA: 0s - loss: 0.2421 - accuracy: 0.9692WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "2000/2000 [==============================] - 64s 32ms/step - loss: 0.2421 - accuracy: 0.9692 - val_loss: 0.4389 - val_accuracy: 0.9103 - lr: 1.0000e-04\n",
      "Learning rate:  0.0001\n",
      "Epoch 98/100\n",
      "2000/2000 [==============================] - ETA: 0s - loss: 0.2420 - accuracy: 0.9689WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "2000/2000 [==============================] - 64s 32ms/step - loss: 0.2420 - accuracy: 0.9689 - val_loss: 0.4368 - val_accuracy: 0.9135 - lr: 3.1623e-05\n",
      "Learning rate:  0.0001\n",
      "Epoch 99/100\n",
      "1999/2000 [============================>.] - ETA: 0s - loss: 0.2389 - accuracy: 0.9689WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "2000/2000 [==============================] - 65s 33ms/step - loss: 0.2390 - accuracy: 0.9689 - val_loss: 0.4474 - val_accuracy: 0.9105 - lr: 1.0000e-04\n",
      "Learning rate:  0.0001\n",
      "Epoch 100/100\n",
      "1999/2000 [============================>.] - ETA: 0s - loss: 0.2341 - accuracy: 0.9689WARNING:tensorflow:Can save best model only with val_acc available, skipping.\n",
      "2000/2000 [==============================] - 65s 32ms/step - loss: 0.2341 - accuracy: 0.9689 - val_loss: 0.4271 - val_accuracy: 0.9156 - lr: 1.0000e-04\n"
     ]
    }
   ],
   "source": [
    "if version == 2:\n",
    "    model = resnet_v2(input_shape=input_shape, depth=depth)\n",
    "else:\n",
    "    model = resnet_v1(input_shape=input_shape, depth=depth)\n",
    "\n",
    "model.compile(loss='categorical_crossentropy',\n",
    "              optimizer=Adam(learning_rate=lr_schedule(0)),\n",
    "              metrics=['accuracy'])\n",
    "model.summary()\n",
    "print(model_type)\n",
    "\n",
    "# Prepare model model saving directory.\n",
    "save_dir = os.path.join(os.getcwd(), 'saved_models')\n",
    "model_name = 'cifar10_%s_model.{epoch:03d}.h5' % model_type\n",
    "if not os.path.isdir(save_dir):\n",
    "    os.makedirs(save_dir)\n",
    "filepath = os.path.join(save_dir, model_name)\n",
    "\n",
    "# Prepare callbacks for model saving and for learning rate adjustment.\n",
    "checkpoint = ModelCheckpoint(filepath=filepath,\n",
    "                             monitor='val_acc',\n",
    "                             verbose=1,\n",
    "                             save_best_only=True)\n",
    "\n",
    "lr_scheduler = LearningRateScheduler(lr_schedule)\n",
    "\n",
    "lr_reducer = ReduceLROnPlateau(factor=np.sqrt(0.1),\n",
    "                               cooldown=0,\n",
    "                               patience=5,\n",
    "                               min_lr=0.5e-6)\n",
    "\n",
    "callbacks = [checkpoint, lr_reducer, lr_scheduler]\n",
    "\n",
    "# Run training, with or without data augmentation.\n",
    "if not data_augmentation:\n",
    "    print('Not using data augmentation.')\n",
    "    model.fit(x_train, y_train,\n",
    "              batch_size=batch_size,\n",
    "              epochs=epochs,\n",
    "              validation_data=(x_test, y_test),\n",
    "              shuffle=True,\n",
    "              callbacks=callbacks)\n",
    "else:\n",
    "    print('Using real-time data augmentation.')\n",
    "    # This will do preprocessing and realtime data augmentation:\n",
    "    datagen = ImageDataGenerator(\n",
    "        # set input mean to 0 over the dataset\n",
    "        featurewise_center=False,\n",
    "        # set each sample mean to 0\n",
    "        samplewise_center=False,\n",
    "        # divide inputs by std of dataset\n",
    "        featurewise_std_normalization=False,\n",
    "        # divide each input by its std\n",
    "        samplewise_std_normalization=False,\n",
    "        # apply ZCA whitening\n",
    "        zca_whitening=False,\n",
    "        # epsilon for ZCA whitening\n",
    "        zca_epsilon=1e-06,\n",
    "        # randomly rotate images in the range (deg 0 to 180)\n",
    "        rotation_range=0,\n",
    "        # randomly shift images horizontally\n",
    "        width_shift_range=0.1,\n",
    "        # randomly shift images vertically\n",
    "        height_shift_range=0.1,\n",
    "        # set range for random shear\n",
    "        shear_range=0.,\n",
    "        # set range for random zoom\n",
    "        zoom_range=0.,\n",
    "        # set range for random channel shifts\n",
    "        channel_shift_range=0.,\n",
    "        # set mode for filling points outside the input boundaries\n",
    "        fill_mode='nearest',\n",
    "        # value used for fill_mode = \"constant\"\n",
    "        cval=0.,\n",
    "        # randomly flip images\n",
    "        horizontal_flip=True,\n",
    "        # randomly flip images\n",
    "        vertical_flip=False,\n",
    "        # set rescaling factor (applied before any other transformation)\n",
    "        rescale=None,\n",
    "        # set function that will be applied on each input\n",
    "        preprocessing_function=None,\n",
    "        # image data format, either \"channels_first\" or \"channels_last\"\n",
    "        data_format=None,\n",
    "        # fraction of images reserved for validation (strictly between 0 and 1)\n",
    "        validation_split=0.0)\n",
    "\n",
    "    # Compute quantities required for featurewise normalization\n",
    "    # (std, mean, and principal components if ZCA whitening is applied).\n",
    "    datagen.fit(x_train)\n",
    "\n",
    "    # Fit the model on the batches generated by datagen.flow().\n",
    "    model.fit_generator(datagen.flow(x_train, y_train, batch_size=batch_size),\n",
    "                        validation_data=(x_test, y_test),\n",
    "                        epochs=epochs,\n",
    "                        steps_per_epoch=steps_per_epoch,\n",
    "                        verbose=1,\n",
    "                        workers=4,\n",
    "                        callbacks=callbacks)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "313/313 [==============================] - 3s 10ms/step - loss: 0.4271 - accuracy: 0.9156\n",
      "Test loss: 0.4270820617675781\n",
      "Test accuracy: 0.9156000018119812\n"
     ]
    }
   ],
   "source": [
    "# Score trained model.\n",
    "scores = model.evaluate(x_test, y_test, verbose=1)\n",
    "print('Test loss:', scores[0])\n",
    "print('Test accuracy:', scores[1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "model_name = \"cifar10_ResNet20v1_model_epoch100.h5\"\n",
    "model.save(model_name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}