train_MS2_DS.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.8.2"
    },
    "colab": {
      "name": "train_MS2_DS.ipynb",
      "provenance": [],
      "collapsed_sections": []
    },
    "accelerator": "GPU",
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "0b7de94fc64245caa43402c8cf1ff264": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_70dd8c194f1c47e49f62a5ab9d2b9c36",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_b945b731ff7b4b0b8bc84d1301b5e5e6",
              "IPY_MODEL_3ba6e210b65446dfa058814372db1d11"
            ]
          }
        },
        "70dd8c194f1c47e49f62a5ab9d2b9c36": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "b945b731ff7b4b0b8bc84d1301b5e5e6": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_f8dd8a1ccc794608a8910607df6adc64",
            "_dom_classes": [],
            "description": "100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 87306240,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 87306240,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_fbe319f189ec470a8e246d3497dd0392"
          }
        },
        "3ba6e210b65446dfa058814372db1d11": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_6863311aa1eb4529ad5b283dc2f9c404",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 83.3M/83.3M [00:00&lt;00:00, 121MB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_ad396d87080f40958b7ee4f209c25ffa"
          }
        },
        "f8dd8a1ccc794608a8910607df6adc64": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "fbe319f189ec470a8e246d3497dd0392": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "6863311aa1eb4529ad5b283dc2f9c404": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "ad396d87080f40958b7ee4f209c25ffa": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "74f8f73a236f4c49b9146664c823f140": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_48d6dc635f72467bb0158b402796b2db",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_d41b0011933846208b24bcecef1a1632",
              "IPY_MODEL_813897393355404faee4365d74282420"
            ]
          }
        },
        "48d6dc635f72467bb0158b402796b2db": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "d41b0011933846208b24bcecef1a1632": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_a3e7f81aa5f149e091c24e992bbea54d",
            "_dom_classes": [],
            "description": "100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 87306240,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 87306240,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_aab8ddf0afef4681ad318550c5afae56"
          }
        },
        "813897393355404faee4365d74282420": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_e50480ed3c4c46b4b1970546a413b54a",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 83.3M/83.3M [00:10&lt;00:00, 8.20MB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_509cd3ca4ab146ff9c497bd2a36b5552"
          }
        },
        "a3e7f81aa5f149e091c24e992bbea54d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "aab8ddf0afef4681ad318550c5afae56": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "e50480ed3c4c46b4b1970546a413b54a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "509cd3ca4ab146ff9c497bd2a36b5552": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "code",
      "metadata": {
        "id": "TCOAcY_D0i9S",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 122
        },
        "outputId": "caedb1c3-2a94-4d02-e828-0723dd50493f"
      },
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/drive')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Go to this URL in a browser: https://accounts.google.com/o/oauth2/auth?client_id=947318989803-6bn6qk8qdgf4n4g3pfee6491hc0brc4i.apps.googleusercontent.com&redirect_uri=urn%3aietf%3awg%3aoauth%3a2.0%3aoob&response_type=code&scope=email%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdocs.test%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdrive%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdrive.photos.readonly%20https%3a%2f%2fwww.googleapis.com%2fauth%2fpeopleapi.readonly\n",
            "\n",
            "Enter your authorization code:\n",
            "··········\n",
            "Mounted at /content/drive\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_hPjIffr0jr7",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 51
        },
        "outputId": "19b691ed-c76d-4354-a10a-10da8de2463c"
      },
      "source": [
        "%cd /content/drive/My\\ Drive/\n",
        "%cd FPAR-Project-MLDL/"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/content/drive/My Drive\n",
            "/content/drive/My Drive/FPAR-Project-MLDL\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "EHJc_H8u0a62",
        "colab_type": "text"
      },
      "source": [
        "# SETUP\n",
        "The following blocks are to be executed first of anything else, to setup import, classes, functions and \n",
        "constants that are needed for all stages"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "ExecuteTime": {
          "end_time": "2020-06-08T15:51:14.755215Z",
          "start_time": "2020-06-08T15:51:14.745003Z"
        },
        "id": "Srq0xZDF0a64",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import os\n",
        "import logging\n",
        "\n",
        "import torch\n",
        "import torch.nn as nn\n",
        "import torch.nn.functional as F\n",
        "import torch.optim as optim\n",
        "from torch.utils.data import DataLoader\n",
        "from torch.backends import cudnn\n",
        "import torchvision\n",
        "from torchvision import transforms\n",
        "from torchvision.models import resnet34\n",
        "\n",
        "from PIL import Image\n",
        "from tqdm import tqdm\n",
        "\n",
        "from logs import Logger, generate_model_checkpoint_name, generate_log_filenames\n",
        "\n",
        "from gtea_dataset import GTEA61, GTEA61_flow, GTEA61_2Stream\n",
        "# from clean_dataset import adjust_flows\n",
        "from AttentMS2 import attention_model_ms\n",
        "from flow_resnet import flow_resnet34\n",
        "from twoStreamModel import twoStreamAttentionModel\n",
        "from spatial_transforms import (Compose, ToTensor, CenterCrop, Scale, Normalize, MultiScaleCornerCrop,\n",
        "                                RandomHorizontalFlip, DownSampling, To1Dimension)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "ExecuteTime": {
          "end_time": "2020-06-08T15:51:15.213199Z",
          "start_time": "2020-06-08T15:51:15.209364Z"
        },
        "id": "tVzjGYRq0a69",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "DEVICE = 'cuda' # 'cuda' or 'cpu'\n",
        "NUM_CLASSES = 61 # 101 + 1: There is am extra Background class that should be removed \n",
        "\n",
        "BATCH_SIZE = 32     # Higher batch sizes allows for larger learning rates. An empirical heuristic suggests that, when changing\n",
        "                     # the batch size, learning rate should change by the same factor to have comparable results\n",
        "\n",
        "MMAP_LENGTH = 49\n",
        "DATA_DIR = '../GTEA61'\n",
        "model_folder = '../saved_models'\n",
        "\n",
        "RUN = '_run01'"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "ExecuteTime": {
          "end_time": "2020-06-08T15:51:15.651373Z",
          "start_time": "2020-06-08T15:51:15.646963Z"
        },
        "id": "NmmQMCPC0a7E",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Data loader\n",
        "normalize = Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225])\n",
        "default_transforms = [\n",
        "    Scale(256),\n",
        "    RandomHorizontalFlip(),\n",
        "    MultiScaleCornerCrop([1, 0.875, 0.75, 0.65625], 224),\n",
        "    ToTensor()\n",
        "]\n",
        "\n",
        "spatial_transform = Compose(default_transforms + [normalize])\n",
        "spatial_transform_mmaps = Compose(default_transforms + [DownSampling(), To1Dimension()])\n",
        "spatial_transform_val = Compose([Scale(256), CenterCrop(224), ToTensor(), normalize])"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "gdn20BFx0a7J",
        "colab_type": "text"
      },
      "source": [
        "# Stage 1 specific setup"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "3lUwlCvL0a7J",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "STAGE = 1\n",
        "\n",
        "LR = 0.001                            # The initial Learning Rate\n",
        "MOMENTUM = 0.9                        # Hyperparameter for SGD, keep this at 0.9 when using SGD\n",
        "WEIGHT_DECAY = 4e-5                   # Regularization, you can keep this at the default\n",
        "\n",
        "NUM_EPOCHS = 150                      # Total number of training epochs (iterations over dataset)\n",
        "STEP_SIZE = [30, 80]             # How many epochs before decreasing learning rate (if using a step-down policy)\n",
        "GAMMA = 0.1                           # Multiplicative factor for learning rate step-down\n",
        "MEM_SIZE = 512\n",
        "SEQ_LEN = 7\n",
        "\n",
        "# this dictionary is needed for the logger class\n",
        "parameters = {\n",
        "    'DEVICE': DEVICE,\n",
        "    'NUM_CLASSES': NUM_CLASSES,\n",
        "    'BATCH_SIZE': BATCH_SIZE,\n",
        "    'LR': LR,\n",
        "    'MOMENTUM': MOMENTUM,\n",
        "    'WEIGHT_DECAY': WEIGHT_DECAY,\n",
        "    'NUM_EPOCHS': NUM_EPOCHS,\n",
        "    'STEP_SIZE': STEP_SIZE,\n",
        "    'GAMMA': GAMMA,\n",
        "    'MEM_SIZE': MEM_SIZE,\n",
        "    'SEQ_LEN': SEQ_LEN\n",
        "}"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "GO5aDlTI0a7O",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 51
        },
        "outputId": "28d9295a-a0eb-436c-8395-09b8686e041b"
      },
      "source": [
        "# Prepare Pytorch train/test Datasets\n",
        "train_dataset = GTEA61(DATA_DIR, split = 'train', transform = spatial_transform, seq_len = SEQ_LEN,\n",
        "                       mmaps = True, mmaps_transform = spatial_transform_mmaps, static_frames = True)\n",
        "test_dataset = GTEA61(DATA_DIR, split = 'test', transform = spatial_transform_val, seq_len = SEQ_LEN)\n",
        "\n",
        "# Check dataset sizes\n",
        "print('Train Dataset: {}'.format(len(train_dataset)))\n",
        "print('Test Dataset: {}'.format(len(test_dataset)))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Train Dataset: 341\n",
            "Test Dataset: 116\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "IaGYOtkm0a7R",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Dataloaders iterate over pytorch datasets and transparently provide useful functions (e.g. parallelization and shuffling)\n",
        "train_loader = DataLoader(train_dataset, batch_size = BATCH_SIZE, shuffle = True, num_workers = 4, drop_last = True)\n",
        "val_loader = DataLoader(test_dataset, batch_size = BATCH_SIZE, shuffle = False, num_workers = 4)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "dlRAcqQW0a7V",
        "colab_type": "text"
      },
      "source": [
        "# Prepare Stage 1"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "nU1P8FOS0a7W",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 83,
          "referenced_widgets": [
            "0b7de94fc64245caa43402c8cf1ff264",
            "70dd8c194f1c47e49f62a5ab9d2b9c36",
            "b945b731ff7b4b0b8bc84d1301b5e5e6",
            "3ba6e210b65446dfa058814372db1d11",
            "f8dd8a1ccc794608a8910607df6adc64",
            "fbe319f189ec470a8e246d3497dd0392",
            "6863311aa1eb4529ad5b283dc2f9c404",
            "ad396d87080f40958b7ee4f209c25ffa"
          ]
        },
        "outputId": "cf2ed7bc-899b-4540-d410-11dbdcccf8b2"
      },
      "source": [
        "validate = True\n",
        "\n",
        "model = attention_model_ms(num_classes = NUM_CLASSES, mem_size = MEM_SIZE)\n",
        "model.train(False)\n",
        "\n",
        "for params in model.parameters():\n",
        "    params.requires_grad = False\n",
        "\n",
        "for params in model.lstm_cell.parameters():\n",
        "    params.requires_grad = True\n",
        "\n",
        "for params in model.classifier.parameters():\n",
        "    params.requires_grad = True\n",
        "\n",
        "model.lstm_cell.train(True)\n",
        "model.classifier.train(True)\n",
        "model.to(DEVICE)\n",
        "\n",
        "trainable_params = [p for p in model.parameters() if p.requires_grad]\n",
        "\n",
        "loss_fn = nn.CrossEntropyLoss()\n",
        "\n",
        "optimizer_fn = optim.Adam(trainable_params, lr = LR, weight_decay = WEIGHT_DECAY, eps = 1e-4)\n",
        "\n",
        "optim_scheduler = optim.lr_scheduler.MultiStepLR(optimizer_fn, milestones = STEP_SIZE, gamma = GAMMA)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Downloading: \"https://download.pytorch.org/models/resnet34-333f7ec4.pth\" to /root/.cache/torch/checkpoints/resnet34-333f7ec4.pth\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "0b7de94fc64245caa43402c8cf1ff264",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, max=87306240.0), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "gYWRFLj_0a7a",
        "colab_type": "text"
      },
      "source": [
        "# Stage 1"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "N3OdMzTO0a7b",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "outputId": "8db28fa1-5a2f-41c0-d56a-fa90c715a08e"
      },
      "source": [
        "train_iter = 0\n",
        "val_iter = 0\n",
        "min_accuracy = 0\n",
        "\n",
        "trainSamples = len(train_dataset) - (len(train_dataset) % BATCH_SIZE)\n",
        "val_samples = len(test_dataset) \n",
        "iterPerEpoch = len(train_loader)\n",
        "val_steps = len(val_loader)\n",
        "\n",
        "cudnn.benchmark\n",
        "\n",
        "train_log, val_log = generate_log_filenames(STAGE, SEQ_LEN, ms_block = True, \\\n",
        "                                            optional = '_2neur_DS_' + RUN)\n",
        "model_checkpoint = generate_model_checkpoint_name(STAGE, SEQ_LEN, ms_block = True, \\\n",
        "                                                  optional = '_2neur_DS_' + RUN)\n",
        "\n",
        "train_log_file = os.path.join(model_folder, train_log)\n",
        "val_log_file = os.path.join(model_folder, val_log)\n",
        "\n",
        "train_logger = Logger(**parameters)\n",
        "val_logger = Logger(**parameters)\n",
        "\n",
        "for epoch in range(NUM_EPOCHS):\n",
        "\n",
        "    epoch_loss = 0\n",
        "    numCorrTrain = 0\n",
        "    \n",
        "    model.lstm_cell.train(True)\n",
        "    model.classifier.train(True)\n",
        "        \n",
        "    for i, (inputs, _, _, targets) in enumerate(train_loader):\n",
        "\n",
        "        train_iter += 1\n",
        "\n",
        "        optimizer_fn.zero_grad()\n",
        "\n",
        "        inputVariable = inputs.permute(1, 0, 2, 3, 4).to(DEVICE)\n",
        "        labelVariable = targets.to(DEVICE)\n",
        "        output_label, _ = model(inputVariable)\n",
        "        \n",
        "        loss = loss_fn(output_label, labelVariable)\n",
        "        loss.backward()\n",
        "        optimizer_fn.step()\n",
        "            \n",
        "        _, predicted = torch.max(output_label.data, 1)\n",
        "        numCorrTrain += torch.sum(predicted == labelVariable.data).data.item()\n",
        "\n",
        "        step_loss = loss.data.item()\n",
        "        epoch_loss += step_loss\n",
        "\n",
        "        train_logger.add_step_data(train_iter, numCorrTrain, step_loss)\n",
        "    \n",
        "    avg_loss = epoch_loss/iterPerEpoch\n",
        "\n",
        "    trainAccuracy = (numCorrTrain / trainSamples) * 100\n",
        "    train_logger.add_epoch_data(epoch+1, trainAccuracy, avg_loss)\n",
        "    print('Train: Epoch = {} | Loss = {:.3f} | Accuracy = {:.3f}'.format(epoch+1, avg_loss, trainAccuracy))\n",
        "\n",
        "    if validate:\n",
        "\n",
        "        if (epoch+1) % 1 == 0:\n",
        "\n",
        "            model.train(False)\n",
        "\n",
        "            val_loss_epoch = 0\n",
        "            numCorr = 0\n",
        "\n",
        "            for j, (inputs, targets) in enumerate(val_loader):\n",
        "\n",
        "                val_iter += 1\n",
        "\n",
        "                inputVariable = inputs.permute(1, 0, 2, 3, 4).to(DEVICE)\n",
        "                labelVariable = targets.to(DEVICE)\n",
        "                \n",
        "                output_label, _ = model(inputVariable)\n",
        "\n",
        "                val_loss = loss_fn(output_label, labelVariable)\n",
        "                val_loss_step = val_loss.data.item()\n",
        "                val_loss_epoch += val_loss_step\n",
        "                \n",
        "                _, predicted = torch.max(output_label.data, 1)\n",
        "                numCorr += torch.sum(predicted == labelVariable.data).data.item()\n",
        "\n",
        "                val_logger.add_step_data(val_iter, numCorr, val_loss_step)\n",
        "                \n",
        "            val_accuracy = (numCorr / val_samples) * 100\n",
        "            avg_val_loss = val_loss_epoch / val_steps\n",
        "            val_logger.add_epoch_data(epoch+1, val_accuracy, avg_val_loss)\n",
        "\n",
        "            print('Val: Epoch = {} | Loss {:.3f} | Accuracy = {:.3f}'.format(epoch + 1, avg_val_loss, val_accuracy))\n",
        "\n",
        "            if val_accuracy > min_accuracy:\n",
        "                print(\"[||| NEW BEST on val||||]\")\n",
        "                save_path_model = os.path.join(model_folder, model_checkpoint)\n",
        "                torch.save(model.state_dict(), save_path_model)\n",
        "                min_accuracy = val_accuracy\n",
        "        \n",
        "    train_logger.save(train_log_file)\n",
        "    val_logger.save(val_log_file)\n",
        "    optim_scheduler.step()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Train: Epoch = 1 | Loss = 4.084 | Accuracy = 4.688\n",
            "Val: Epoch = 1 | Loss 3.914 | Accuracy = 6.897\n",
            "[||| NEW BEST on val||||]\n",
            "Train: Epoch = 2 | Loss = 4.056 | Accuracy = 3.750\n",
            "Val: Epoch = 2 | Loss 3.901 | Accuracy = 5.172\n",
            "Train: Epoch = 3 | Loss = 3.989 | Accuracy = 6.250\n",
            "Val: Epoch = 3 | Loss 3.874 | Accuracy = 5.172\n",
            "Train: Epoch = 4 | Loss = 3.907 | Accuracy = 7.187\n",
            "Val: Epoch = 4 | Loss 3.777 | Accuracy = 6.897\n",
            "Train: Epoch = 5 | Loss = 3.812 | Accuracy = 6.875\n",
            "Val: Epoch = 5 | Loss 3.665 | Accuracy = 10.345\n",
            "[||| NEW BEST on val||||]\n",
            "Train: Epoch = 6 | Loss = 3.647 | Accuracy = 12.500\n",
            "Val: Epoch = 6 | Loss 3.513 | Accuracy = 8.621\n",
            "Train: Epoch = 7 | Loss = 3.578 | Accuracy = 12.812\n",
            "Val: Epoch = 7 | Loss 3.546 | Accuracy = 12.931\n",
            "[||| NEW BEST on val||||]\n",
            "Train: Epoch = 8 | Loss = 3.513 | Accuracy = 11.875\n",
            "Val: Epoch = 8 | Loss 3.596 | Accuracy = 11.207\n",
            "Train: Epoch = 9 | Loss = 3.587 | Accuracy = 12.812\n",
            "Val: Epoch = 9 | Loss 3.506 | Accuracy = 8.621\n",
            "Train: Epoch = 10 | Loss = 3.495 | Accuracy = 12.188\n",
            "Val: Epoch = 10 | Loss 3.426 | Accuracy = 8.621\n",
            "Train: Epoch = 11 | Loss = 3.481 | Accuracy = 10.938\n",
            "Val: Epoch = 11 | Loss 3.242 | Accuracy = 15.517\n",
            "[||| NEW BEST on val||||]\n",
            "Train: Epoch = 12 | Loss = 3.371 | Accuracy = 13.438\n",
            "Val: Epoch = 12 | Loss 3.432 | Accuracy = 12.931\n",
            "Train: Epoch = 13 | Loss = 3.230 | Accuracy = 17.812\n",
            "Val: Epoch = 13 | Loss 3.365 | Accuracy = 12.069\n",
            "Train: Epoch = 14 | Loss = 3.216 | Accuracy = 18.438\n",
            "Val: Epoch = 14 | Loss 3.192 | Accuracy = 24.138\n",
            "[||| NEW BEST on val||||]\n",
            "Train: Epoch = 15 | Loss = 3.188 | Accuracy = 18.438\n",
            "Val: Epoch = 15 | Loss 3.152 | Accuracy = 21.552\n",
            "Train: Epoch = 16 | Loss = 3.114 | Accuracy = 19.688\n",
            "Val: Epoch = 16 | Loss 3.015 | Accuracy = 25.862\n",
            "[||| NEW BEST on val||||]\n",
            "Train: Epoch = 17 | Loss = 3.042 | Accuracy = 18.438\n",
            "Val: Epoch = 17 | Loss 3.023 | Accuracy = 25.000\n",
            "Train: Epoch = 18 | Loss = 3.052 | Accuracy = 18.438\n",
            "Val: Epoch = 18 | Loss 2.901 | Accuracy = 27.586\n",
            "[||| NEW BEST on val||||]\n",
            "Train: Epoch = 19 | Loss = 3.151 | Accuracy = 15.625\n",
            "Val: Epoch = 19 | Loss 3.000 | Accuracy = 19.828\n",
            "Train: Epoch = 20 | Loss = 3.009 | Accuracy = 19.688\n",
            "Val: Epoch = 20 | Loss 2.758 | Accuracy = 23.276\n",
            "Train: Epoch = 21 | Loss = 2.825 | Accuracy = 21.250\n",
            "Val: Epoch = 21 | Loss 2.813 | Accuracy = 24.138\n",
            "Train: Epoch = 22 | Loss = 2.801 | Accuracy = 23.438\n",
            "Val: Epoch = 22 | Loss 2.701 | Accuracy = 22.414\n",
            "Train: Epoch = 23 | Loss = 2.934 | Accuracy = 22.188\n",
            "Val: Epoch = 23 | Loss 3.159 | Accuracy = 14.655\n",
            "Train: Epoch = 24 | Loss = 2.911 | Accuracy = 23.125\n",
            "Val: Epoch = 24 | Loss 2.964 | Accuracy = 23.276\n",
            "Train: Epoch = 25 | Loss = 2.784 | Accuracy = 27.500\n",
            "Val: Epoch = 25 | Loss 2.653 | Accuracy = 29.310\n",
            "[||| NEW BEST on val||||]\n",
            "Train: Epoch = 26 | Loss = 2.704 | Accuracy = 25.000\n",
            "Val: Epoch = 26 | Loss 2.670 | Accuracy = 30.172\n",
            "[||| NEW BEST on val||||]\n",
            "Train: Epoch = 27 | Loss = 2.570 | Accuracy = 29.375\n",
            "Val: Epoch = 27 | Loss 2.785 | Accuracy = 25.862\n",
            "Train: Epoch = 28 | Loss = 2.787 | Accuracy = 22.812\n",
            "Val: Epoch = 28 | Loss 3.113 | Accuracy = 21.552\n",
            "Train: Epoch = 29 | Loss = 2.626 | Accuracy = 27.812\n",
            "Val: Epoch = 29 | Loss 2.847 | Accuracy = 21.552\n",
            "Train: Epoch = 30 | Loss = 2.545 | Accuracy = 29.375\n",
            "Val: Epoch = 30 | Loss 2.758 | Accuracy = 31.897\n",
            "[||| NEW BEST on val||||]\n",
            "Train: Epoch = 31 | Loss = 2.462 | Accuracy = 33.438\n",
            "Val: Epoch = 31 | Loss 2.572 | Accuracy = 31.897\n",
            "Train: Epoch = 32 | Loss = 2.383 | Accuracy = 34.062\n",
            "Val: Epoch = 32 | Loss 2.426 | Accuracy = 32.759\n",
            "[||| NEW BEST on val||||]\n",
            "Train: Epoch = 33 | Loss = 2.398 | Accuracy = 35.000\n",
            "Val: Epoch = 33 | Loss 2.446 | Accuracy = 31.034\n",
            "Train: Epoch = 34 | Loss = 2.389 | Accuracy = 35.938\n",
            "Val: Epoch = 34 | Loss 2.470 | Accuracy = 29.310\n",
            "Train: Epoch = 35 | Loss = 2.317 | Accuracy = 33.438\n",
            "Val: Epoch = 35 | Loss 2.485 | Accuracy = 31.034\n",
            "Train: Epoch = 36 | Loss = 2.291 | Accuracy = 36.562\n",
            "Val: Epoch = 36 | Loss 2.495 | Accuracy = 29.310\n",
            "Train: Epoch = 37 | Loss = 2.367 | Accuracy = 35.312\n",
            "Val: Epoch = 37 | Loss 2.554 | Accuracy = 29.310\n",
            "Train: Epoch = 38 | Loss = 2.299 | Accuracy = 36.875\n",
            "Val: Epoch = 38 | Loss 2.523 | Accuracy = 29.310\n",
            "Train: Epoch = 39 | Loss = 2.218 | Accuracy = 39.688\n",
            "Val: Epoch = 39 | Loss 2.505 | Accuracy = 31.897\n",
            "Train: Epoch = 40 | Loss = 2.122 | Accuracy = 41.562\n",
            "Val: Epoch = 40 | Loss 2.473 | Accuracy = 32.759\n",
            "Train: Epoch = 41 | Loss = 2.196 | Accuracy = 37.500\n",
            "Val: Epoch = 41 | Loss 2.502 | Accuracy = 32.759\n",
            "Train: Epoch = 42 | Loss = 2.078 | Accuracy = 40.625\n",
            "Val: Epoch = 42 | Loss 2.564 | Accuracy = 32.759\n",
            "Train: Epoch = 43 | Loss = 2.334 | Accuracy = 35.625\n",
            "Val: Epoch = 43 | Loss 2.513 | Accuracy = 31.897\n",
            "Train: Epoch = 44 | Loss = 2.169 | Accuracy = 40.312\n",
            "Val: Epoch = 44 | Loss 2.458 | Accuracy = 32.759\n",
            "Train: Epoch = 45 | Loss = 2.133 | Accuracy = 38.438\n",
            "Val: Epoch = 45 | Loss 2.512 | Accuracy = 37.931\n",
            "[||| NEW BEST on val||||]\n",
            "Train: Epoch = 46 | Loss = 2.138 | Accuracy = 39.062\n",
            "Val: Epoch = 46 | Loss 2.467 | Accuracy = 36.207\n",
            "Train: Epoch = 47 | Loss = 2.124 | Accuracy = 38.125\n",
            "Val: Epoch = 47 | Loss 2.515 | Accuracy = 34.483\n",
            "Train: Epoch = 48 | Loss = 2.133 | Accuracy = 41.562\n",
            "Val: Epoch = 48 | Loss 2.476 | Accuracy = 36.207\n",
            "Train: Epoch = 49 | Loss = 2.100 | Accuracy = 42.812\n",
            "Val: Epoch = 49 | Loss 2.502 | Accuracy = 32.759\n",
            "Train: Epoch = 50 | Loss = 2.045 | Accuracy = 40.938\n",
            "Val: Epoch = 50 | Loss 2.534 | Accuracy = 31.034\n",
            "Train: Epoch = 51 | Loss = 2.140 | Accuracy = 38.750\n",
            "Val: Epoch = 51 | Loss 2.518 | Accuracy = 35.345\n",
            "Train: Epoch = 52 | Loss = 2.136 | Accuracy = 38.750\n",
            "Val: Epoch = 52 | Loss 2.555 | Accuracy = 39.655\n",
            "[||| NEW BEST on val||||]\n",
            "Train: Epoch = 53 | Loss = 2.050 | Accuracy = 43.125\n",
            "Val: Epoch = 53 | Loss 2.514 | Accuracy = 37.069\n",
            "Train: Epoch = 54 | Loss = 2.210 | Accuracy = 39.062\n",
            "Val: Epoch = 54 | Loss 2.458 | Accuracy = 40.517\n",
            "[||| NEW BEST on val||||]\n",
            "Train: Epoch = 55 | Loss = 2.038 | Accuracy = 43.750\n",
            "Val: Epoch = 55 | Loss 2.461 | Accuracy = 38.793\n",
            "Train: Epoch = 56 | Loss = 2.036 | Accuracy = 40.312\n",
            "Val: Epoch = 56 | Loss 2.518 | Accuracy = 37.931\n",
            "Train: Epoch = 57 | Loss = 2.037 | Accuracy = 44.375\n",
            "Val: Epoch = 57 | Loss 2.435 | Accuracy = 37.069\n",
            "Train: Epoch = 58 | Loss = 2.044 | Accuracy = 45.312\n",
            "Val: Epoch = 58 | Loss 2.491 | Accuracy = 38.793\n",
            "Train: Epoch = 59 | Loss = 2.059 | Accuracy = 39.375\n",
            "Val: Epoch = 59 | Loss 2.567 | Accuracy = 37.931\n",
            "Train: Epoch = 60 | Loss = 1.941 | Accuracy = 45.312\n",
            "Val: Epoch = 60 | Loss 2.560 | Accuracy = 37.069\n",
            "Train: Epoch = 61 | Loss = 1.876 | Accuracy = 50.313\n",
            "Val: Epoch = 61 | Loss 2.479 | Accuracy = 37.931\n",
            "Train: Epoch = 62 | Loss = 2.066 | Accuracy = 44.375\n",
            "Val: Epoch = 62 | Loss 2.450 | Accuracy = 36.207\n",
            "Train: Epoch = 63 | Loss = 2.113 | Accuracy = 37.500\n",
            "Val: Epoch = 63 | Loss 2.410 | Accuracy = 39.655\n",
            "Train: Epoch = 64 | Loss = 2.066 | Accuracy = 42.812\n",
            "Val: Epoch = 64 | Loss 2.471 | Accuracy = 37.931\n",
            "Train: Epoch = 65 | Loss = 1.950 | Accuracy = 41.562\n",
            "Val: Epoch = 65 | Loss 2.496 | Accuracy = 39.655\n",
            "Train: Epoch = 66 | Loss = 1.839 | Accuracy = 45.312\n",
            "Val: Epoch = 66 | Loss 2.556 | Accuracy = 38.793\n",
            "Train: Epoch = 67 | Loss = 1.985 | Accuracy = 42.500\n",
            "Val: Epoch = 67 | Loss 2.503 | Accuracy = 37.931\n",
            "Train: Epoch = 68 | Loss = 1.854 | Accuracy = 48.438\n",
            "Val: Epoch = 68 | Loss 2.523 | Accuracy = 39.655\n",
            "Train: Epoch = 69 | Loss = 1.940 | Accuracy = 44.062\n",
            "Val: Epoch = 69 | Loss 2.499 | Accuracy = 36.207\n",
            "Train: Epoch = 70 | Loss = 2.094 | Accuracy = 41.250\n",
            "Val: Epoch = 70 | Loss 2.575 | Accuracy = 36.207\n",
            "Train: Epoch = 71 | Loss = 1.967 | Accuracy = 47.188\n",
            "Val: Epoch = 71 | Loss 2.510 | Accuracy = 36.207\n",
            "Train: Epoch = 72 | Loss = 1.909 | Accuracy = 45.625\n",
            "Val: Epoch = 72 | Loss 2.497 | Accuracy = 36.207\n",
            "Train: Epoch = 73 | Loss = 1.963 | Accuracy = 40.625\n",
            "Val: Epoch = 73 | Loss 2.572 | Accuracy = 36.207\n",
            "Train: Epoch = 74 | Loss = 1.952 | Accuracy = 44.688\n",
            "Val: Epoch = 74 | Loss 2.525 | Accuracy = 35.345\n",
            "Train: Epoch = 75 | Loss = 1.935 | Accuracy = 43.438\n",
            "Val: Epoch = 75 | Loss 2.483 | Accuracy = 39.655\n",
            "Train: Epoch = 76 | Loss = 1.897 | Accuracy = 43.125\n",
            "Val: Epoch = 76 | Loss 2.502 | Accuracy = 37.931\n",
            "Train: Epoch = 77 | Loss = 1.947 | Accuracy = 44.062\n",
            "Val: Epoch = 77 | Loss 2.539 | Accuracy = 36.207\n",
            "Train: Epoch = 78 | Loss = 1.743 | Accuracy = 49.688\n",
            "Val: Epoch = 78 | Loss 2.527 | Accuracy = 36.207\n",
            "Train: Epoch = 79 | Loss = 1.746 | Accuracy = 51.250\n",
            "Val: Epoch = 79 | Loss 2.548 | Accuracy = 36.207\n",
            "Train: Epoch = 80 | Loss = 1.695 | Accuracy = 52.188\n",
            "Val: Epoch = 80 | Loss 2.597 | Accuracy = 36.207\n",
            "Train: Epoch = 81 | Loss = 1.934 | Accuracy = 44.375\n",
            "Val: Epoch = 81 | Loss 2.594 | Accuracy = 35.345\n",
            "Train: Epoch = 82 | Loss = 1.936 | Accuracy = 43.750\n",
            "Val: Epoch = 82 | Loss 2.581 | Accuracy = 37.069\n",
            "Train: Epoch = 83 | Loss = 1.839 | Accuracy = 45.938\n",
            "Val: Epoch = 83 | Loss 2.578 | Accuracy = 37.069\n",
            "Train: Epoch = 84 | Loss = 1.967 | Accuracy = 46.875\n",
            "Val: Epoch = 84 | Loss 2.559 | Accuracy = 38.793\n",
            "Train: Epoch = 85 | Loss = 1.820 | Accuracy = 49.062\n",
            "Val: Epoch = 85 | Loss 2.544 | Accuracy = 39.655\n",
            "Train: Epoch = 86 | Loss = 1.762 | Accuracy = 47.812\n",
            "Val: Epoch = 86 | Loss 2.540 | Accuracy = 38.793\n",
            "Train: Epoch = 87 | Loss = 1.791 | Accuracy = 45.000\n",
            "Val: Epoch = 87 | Loss 2.543 | Accuracy = 39.655\n",
            "Train: Epoch = 88 | Loss = 1.896 | Accuracy = 46.875\n",
            "Val: Epoch = 88 | Loss 2.540 | Accuracy = 39.655\n",
            "Train: Epoch = 89 | Loss = 1.776 | Accuracy = 51.875\n",
            "Val: Epoch = 89 | Loss 2.535 | Accuracy = 39.655\n",
            "Train: Epoch = 90 | Loss = 1.753 | Accuracy = 50.313\n",
            "Val: Epoch = 90 | Loss 2.535 | Accuracy = 38.793\n",
            "Train: Epoch = 91 | Loss = 1.791 | Accuracy = 46.562\n",
            "Val: Epoch = 91 | Loss 2.529 | Accuracy = 38.793\n",
            "Train: Epoch = 92 | Loss = 1.731 | Accuracy = 50.313\n",
            "Val: Epoch = 92 | Loss 2.524 | Accuracy = 38.793\n",
            "Train: Epoch = 93 | Loss = 1.865 | Accuracy = 46.250\n",
            "Val: Epoch = 93 | Loss 2.525 | Accuracy = 38.793\n",
            "Train: Epoch = 94 | Loss = 1.707 | Accuracy = 48.438\n",
            "Val: Epoch = 94 | Loss 2.538 | Accuracy = 39.655\n",
            "Train: Epoch = 95 | Loss = 1.803 | Accuracy = 47.500\n",
            "Val: Epoch = 95 | Loss 2.540 | Accuracy = 39.655\n",
            "Train: Epoch = 96 | Loss = 1.720 | Accuracy = 48.438\n",
            "Val: Epoch = 96 | Loss 2.534 | Accuracy = 39.655\n",
            "Train: Epoch = 97 | Loss = 1.731 | Accuracy = 52.500\n",
            "Val: Epoch = 97 | Loss 2.530 | Accuracy = 38.793\n",
            "Train: Epoch = 98 | Loss = 1.731 | Accuracy = 50.000\n",
            "Val: Epoch = 98 | Loss 2.520 | Accuracy = 39.655\n",
            "Train: Epoch = 99 | Loss = 1.714 | Accuracy = 50.625\n",
            "Val: Epoch = 99 | Loss 2.511 | Accuracy = 39.655\n",
            "Train: Epoch = 100 | Loss = 1.722 | Accuracy = 50.938\n",
            "Val: Epoch = 100 | Loss 2.510 | Accuracy = 39.655\n",
            "Train: Epoch = 101 | Loss = 1.746 | Accuracy = 50.000\n",
            "Val: Epoch = 101 | Loss 2.519 | Accuracy = 39.655\n",
            "Train: Epoch = 102 | Loss = 1.650 | Accuracy = 54.688\n",
            "Val: Epoch = 102 | Loss 2.522 | Accuracy = 39.655\n",
            "Train: Epoch = 103 | Loss = 1.735 | Accuracy = 49.375\n",
            "Val: Epoch = 103 | Loss 2.523 | Accuracy = 40.517\n",
            "Train: Epoch = 104 | Loss = 1.784 | Accuracy = 48.438\n",
            "Val: Epoch = 104 | Loss 2.534 | Accuracy = 39.655\n",
            "Train: Epoch = 105 | Loss = 1.766 | Accuracy = 45.000\n",
            "Val: Epoch = 105 | Loss 2.535 | Accuracy = 39.655\n",
            "Train: Epoch = 106 | Loss = 1.809 | Accuracy = 45.000\n",
            "Val: Epoch = 106 | Loss 2.538 | Accuracy = 40.517\n",
            "Train: Epoch = 107 | Loss = 1.717 | Accuracy = 49.688\n",
            "Val: Epoch = 107 | Loss 2.543 | Accuracy = 39.655\n",
            "Train: Epoch = 108 | Loss = 1.725 | Accuracy = 51.562\n",
            "Val: Epoch = 108 | Loss 2.545 | Accuracy = 39.655\n",
            "Train: Epoch = 109 | Loss = 1.822 | Accuracy = 45.000\n",
            "Val: Epoch = 109 | Loss 2.550 | Accuracy = 39.655\n",
            "Train: Epoch = 110 | Loss = 1.693 | Accuracy = 53.438\n",
            "Val: Epoch = 110 | Loss 2.556 | Accuracy = 39.655\n",
            "Train: Epoch = 111 | Loss = 1.817 | Accuracy = 48.750\n",
            "Val: Epoch = 111 | Loss 2.552 | Accuracy = 39.655\n",
            "Train: Epoch = 112 | Loss = 1.767 | Accuracy = 47.812\n",
            "Val: Epoch = 112 | Loss 2.554 | Accuracy = 39.655\n",
            "Train: Epoch = 113 | Loss = 1.703 | Accuracy = 52.188\n",
            "Val: Epoch = 113 | Loss 2.542 | Accuracy = 40.517\n",
            "Train: Epoch = 114 | Loss = 1.726 | Accuracy = 50.000\n",
            "Val: Epoch = 114 | Loss 2.539 | Accuracy = 40.517\n",
            "Train: Epoch = 115 | Loss = 1.768 | Accuracy = 48.750\n",
            "Val: Epoch = 115 | Loss 2.540 | Accuracy = 40.517\n",
            "Train: Epoch = 116 | Loss = 1.754 | Accuracy = 49.062\n",
            "Val: Epoch = 116 | Loss 2.540 | Accuracy = 40.517\n",
            "Train: Epoch = 117 | Loss = 1.673 | Accuracy = 50.938\n",
            "Val: Epoch = 117 | Loss 2.544 | Accuracy = 40.517\n",
            "Train: Epoch = 118 | Loss = 1.650 | Accuracy = 52.500\n",
            "Val: Epoch = 118 | Loss 2.552 | Accuracy = 39.655\n",
            "Train: Epoch = 119 | Loss = 1.684 | Accuracy = 53.438\n",
            "Val: Epoch = 119 | Loss 2.562 | Accuracy = 39.655\n",
            "Train: Epoch = 120 | Loss = 1.764 | Accuracy = 50.938\n",
            "Val: Epoch = 120 | Loss 2.557 | Accuracy = 39.655\n",
            "Train: Epoch = 121 | Loss = 1.747 | Accuracy = 50.000\n",
            "Val: Epoch = 121 | Loss 2.547 | Accuracy = 39.655\n",
            "Train: Epoch = 122 | Loss = 1.647 | Accuracy = 54.062\n",
            "Val: Epoch = 122 | Loss 2.544 | Accuracy = 41.379\n",
            "[||| NEW BEST on val||||]\n",
            "Train: Epoch = 123 | Loss = 1.799 | Accuracy = 49.688\n",
            "Val: Epoch = 123 | Loss 2.548 | Accuracy = 40.517\n",
            "Train: Epoch = 124 | Loss = 1.653 | Accuracy = 51.562\n",
            "Val: Epoch = 124 | Loss 2.540 | Accuracy = 40.517\n",
            "Train: Epoch = 125 | Loss = 1.737 | Accuracy = 49.688\n",
            "Val: Epoch = 125 | Loss 2.539 | Accuracy = 40.517\n",
            "Train: Epoch = 126 | Loss = 1.807 | Accuracy = 46.562\n",
            "Val: Epoch = 126 | Loss 2.531 | Accuracy = 40.517\n",
            "Train: Epoch = 127 | Loss = 1.765 | Accuracy = 47.188\n",
            "Val: Epoch = 127 | Loss 2.531 | Accuracy = 40.517\n",
            "Train: Epoch = 128 | Loss = 1.778 | Accuracy = 51.250\n",
            "Val: Epoch = 128 | Loss 2.526 | Accuracy = 40.517\n",
            "Train: Epoch = 129 | Loss = 1.749 | Accuracy = 49.688\n",
            "Val: Epoch = 129 | Loss 2.514 | Accuracy = 40.517\n",
            "Train: Epoch = 130 | Loss = 1.779 | Accuracy = 47.188\n",
            "Val: Epoch = 130 | Loss 2.511 | Accuracy = 40.517\n",
            "Train: Epoch = 131 | Loss = 1.663 | Accuracy = 53.438\n",
            "Val: Epoch = 131 | Loss 2.500 | Accuracy = 40.517\n",
            "Train: Epoch = 132 | Loss = 1.732 | Accuracy = 52.500\n",
            "Val: Epoch = 132 | Loss 2.504 | Accuracy = 40.517\n",
            "Train: Epoch = 133 | Loss = 1.780 | Accuracy = 47.188\n",
            "Val: Epoch = 133 | Loss 2.505 | Accuracy = 40.517\n",
            "Train: Epoch = 134 | Loss = 1.838 | Accuracy = 45.625\n",
            "Val: Epoch = 134 | Loss 2.513 | Accuracy = 40.517\n",
            "Train: Epoch = 135 | Loss = 1.666 | Accuracy = 54.062\n",
            "Val: Epoch = 135 | Loss 2.526 | Accuracy = 40.517\n",
            "Train: Epoch = 136 | Loss = 1.709 | Accuracy = 53.438\n",
            "Val: Epoch = 136 | Loss 2.537 | Accuracy = 39.655\n",
            "Train: Epoch = 137 | Loss = 1.707 | Accuracy = 50.313\n",
            "Val: Epoch = 137 | Loss 2.546 | Accuracy = 40.517\n",
            "Train: Epoch = 138 | Loss = 1.649 | Accuracy = 50.938\n",
            "Val: Epoch = 138 | Loss 2.538 | Accuracy = 41.379\n",
            "Train: Epoch = 139 | Loss = 1.758 | Accuracy = 48.750\n",
            "Val: Epoch = 139 | Loss 2.530 | Accuracy = 41.379\n",
            "Train: Epoch = 140 | Loss = 1.751 | Accuracy = 48.438\n",
            "Val: Epoch = 140 | Loss 2.536 | Accuracy = 41.379\n",
            "Train: Epoch = 141 | Loss = 1.658 | Accuracy = 53.750\n",
            "Val: Epoch = 141 | Loss 2.530 | Accuracy = 41.379\n",
            "Train: Epoch = 142 | Loss = 1.869 | Accuracy = 48.125\n",
            "Val: Epoch = 142 | Loss 2.524 | Accuracy = 40.517\n",
            "Train: Epoch = 143 | Loss = 1.728 | Accuracy = 52.188\n",
            "Val: Epoch = 143 | Loss 2.522 | Accuracy = 40.517\n",
            "Train: Epoch = 144 | Loss = 1.765 | Accuracy = 50.313\n",
            "Val: Epoch = 144 | Loss 2.524 | Accuracy = 40.517\n",
            "Train: Epoch = 145 | Loss = 1.705 | Accuracy = 51.875\n",
            "Val: Epoch = 145 | Loss 2.530 | Accuracy = 40.517\n",
            "Train: Epoch = 146 | Loss = 1.660 | Accuracy = 53.125\n",
            "Val: Epoch = 146 | Loss 2.518 | Accuracy = 40.517\n",
            "Train: Epoch = 147 | Loss = 1.829 | Accuracy = 47.500\n",
            "Val: Epoch = 147 | Loss 2.511 | Accuracy = 40.517\n",
            "Train: Epoch = 148 | Loss = 1.723 | Accuracy = 49.688\n",
            "Val: Epoch = 148 | Loss 2.510 | Accuracy = 40.517\n",
            "Train: Epoch = 149 | Loss = 1.710 | Accuracy = 50.938\n",
            "Val: Epoch = 149 | Loss 2.506 | Accuracy = 40.517\n",
            "Train: Epoch = 150 | Loss = 1.707 | Accuracy = 52.812\n",
            "Val: Epoch = 150 | Loss 2.501 | Accuracy = 40.517\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "fJdo0P-t0a7e",
        "colab_type": "text"
      },
      "source": [
        "# Stage 2 specific setup"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "ExecuteTime": {
          "end_time": "2020-06-08T15:51:16.753000Z",
          "start_time": "2020-06-08T15:51:16.742346Z"
        },
        "id": "vpa3K6p30a7f",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "STAGE = 2\n",
        "\n",
        "LR = 0.0001            # The initial Learning Rate\n",
        "MOMENTUM = 0.9       # Hyperparameter for SGD, keep this at 0.9 when using SGD\n",
        "WEIGHT_DECAY = 4e-5  # Regularization, you can keep this at the default\n",
        "\n",
        "NUM_EPOCHS = 150      # Total number of training epochs (iterations over dataset)\n",
        "STEP_SIZE = [30, 80] # How many epochs before decreasing learning rate (if using a step-down policy)\n",
        "GAMMA = 0.2          # Multiplicative factor for learning rate step-down\n",
        "MEM_SIZE = 512\n",
        "SEQ_LEN = 7\n",
        "\n",
        "parameters = {\n",
        "    'DEVICE': DEVICE,\n",
        "    'NUM_CLASSES': NUM_CLASSES,\n",
        "    'BATCH_SIZE': BATCH_SIZE,\n",
        "    'LR': LR,\n",
        "    'MOMENTUM': MOMENTUM,\n",
        "    'WEIGHT_DECAY': WEIGHT_DECAY,\n",
        "    'NUM_EPOCHS': NUM_EPOCHS,\n",
        "    'STEP_SIZE': STEP_SIZE,\n",
        "    'GAMMA': GAMMA,\n",
        "    'MEM_SIZE': MEM_SIZE,\n",
        "    'SEQ_LEN': SEQ_LEN\n",
        "}"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "ExecuteTime": {
          "end_time": "2020-06-08T15:51:18.906868Z",
          "start_time": "2020-06-08T15:51:18.872090Z"
        },
        "id": "mmtrkCUx0a7i",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 51
        },
        "outputId": "2f43a676-4280-45c6-a8db-f6202dff07a2"
      },
      "source": [
        "# Prepare Pytorch train/test Datasets\n",
        "train_dataset = GTEA61(DATA_DIR, split='train', transform=spatial_transform,\n",
        "                       seq_len=SEQ_LEN, mmaps = True, mmaps_transform = spatial_transform_mmaps, static_frames = True)\n",
        "test_dataset = GTEA61(DATA_DIR, split='test', transform=spatial_transform_val,\n",
        "                      seq_len=SEQ_LEN)\n",
        "\n",
        "# Check dataset sizes\n",
        "print('Train Dataset: {}'.format(len(train_dataset)))\n",
        "print('Test Dataset: {}'.format(len(test_dataset)))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Train Dataset: 341\n",
            "Test Dataset: 116\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "ExecuteTime": {
          "end_time": "2020-06-08T15:51:30.177645Z",
          "start_time": "2020-06-08T15:51:30.170111Z"
        },
        "id": "veTbYKAP0a7n",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Dataloaders iterate over pytorch datasets and transparently provide useful functions (e.g. parallelization and shuffling)\n",
        "train_loader = DataLoader(train_dataset, batch_size=BATCH_SIZE, shuffle=True, num_workers=4, drop_last=True)\n",
        "val_loader = DataLoader(test_dataset, batch_size=BATCH_SIZE, shuffle=False, num_workers=4)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "kwND2dlG0a7r",
        "colab_type": "text"
      },
      "source": [
        "# Prepare Stage 2"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "ExecuteTime": {
          "end_time": "2020-06-08T15:51:45.195953Z",
          "start_time": "2020-06-08T15:51:41.943238Z"
        },
        "id": "Lf0-uuhw0a7r",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "best_old_stage = generate_model_checkpoint_name(stage=1, n_frames=SEQ_LEN, ms_block = True, \\\n",
        "                                                optional = '_2neur_DS__run01')\n",
        "stage1_dict = os.path.join(model_folder, best_old_stage)\n",
        "validate = True\n",
        "\n",
        "model = attention_model_ms(num_classes=NUM_CLASSES, mem_size=MEM_SIZE)\n",
        "model.load_state_dict(torch.load(stage1_dict))\n",
        "\n",
        "model.train(False)\n",
        "\n",
        "for params in model.parameters():\n",
        "    params.requires_grad = False\n",
        "\n",
        "layers_on_off = [\n",
        "    model.resNet.layer4[0].conv1,\n",
        "    model.resNet.layer4[0].conv2,\n",
        "    model.resNet.layer4[1].conv1,\n",
        "    model.resNet.layer4[1].conv2,\n",
        "    model.resNet.layer4[2].conv1,\n",
        "    model.resNet.layer4[2].conv2,\n",
        "    model.resNet.fc,\n",
        "    model.msBlock\n",
        "]\n",
        "\n",
        "layers_to_train = layers_on_off + [\n",
        "    model.lstm_cell,\n",
        "    model.classifier,\n",
        "    model.classifier_SD\n",
        "]\n",
        "\n",
        "for layer in layers_to_train:\n",
        "    for params in layer.parameters():\n",
        "        params.requires_grad = True\n",
        "\n",
        "for layer in layers_to_train:\n",
        "    layer.train(True)\n",
        "\n",
        "model.to(DEVICE)\n",
        "\n",
        "loss_fn = nn.CrossEntropyLoss()\n",
        "loss_fn_sum = nn.CrossEntropyLoss(reduction = 'sum')\n",
        "\n",
        "trainable_params = [p for p in model.parameters() if p.requires_grad]\n",
        "optimizer_fn = torch.optim.Adam(trainable_params, lr=LR, weight_decay=WEIGHT_DECAY, eps=1e-4)\n",
        "\n",
        "optim_scheduler = torch.optim.lr_scheduler.MultiStepLR(optimizer_fn, milestones=STEP_SIZE, gamma=GAMMA)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "jbtwKMa00a7v",
        "colab_type": "text"
      },
      "source": [
        "# Stage 2"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "ExecuteTime": {
          "end_time": "2020-06-08T08:58:05.318659Z",
          "start_time": "2020-06-08T08:57:45.955602Z"
        },
        "id": "tyw1jvLn0a7v",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "outputId": "f251dbe9-e953-4c00-f7b9-3908928f9f84"
      },
      "source": [
        "RUN = \"_run03\"\n",
        "\n",
        "train_iter = 0\n",
        "val_iter = 0\n",
        "min_accuracy = 0\n",
        "\n",
        "trainSamples = len(train_dataset) - (len(train_dataset) % BATCH_SIZE)\n",
        "val_samples = len(test_dataset)\n",
        "\n",
        "iterPerEpoch = len(train_loader)\n",
        "val_steps = len(val_loader)\n",
        "\n",
        "cudnn.benchmark\n",
        "torch.autograd.set_detect_anomaly(True)\n",
        "train_log, val_log = generate_log_filenames(STAGE, SEQ_LEN, ms_block = True, \\\n",
        "                                            optional = '_2neur_DS_asymmetric_train_SDCtrained_' + RUN)\n",
        "model_checkpoint = generate_model_checkpoint_name(STAGE, SEQ_LEN, ms_block = True, \\\n",
        "                                                  optional = '_2neur_DS_asymmetric_train_SDCtrained_' + RUN)\n",
        "\n",
        "train_log_file = os.path.join(model_folder, train_log)\n",
        "val_log_file = os.path.join(model_folder, val_log)\n",
        "train_logger_2 = Logger(**parameters)\n",
        "val_logger_2 = Logger(**parameters)\n",
        "\n",
        "for epoch in range(NUM_EPOCHS):\n",
        "\n",
        "    epoch_loss = 0\n",
        "    epoch_loss2 = 0\n",
        "    numCorrTrain = 0\n",
        "    \n",
        "    for layer in layers_to_train:\n",
        "        layer.train(True)\n",
        "    \n",
        "    for i, (inputs, inputs_DS, mmaps, targets) in enumerate(train_loader):\n",
        "\n",
        "        mmaps = mmaps.permute(1, 0, 2)\n",
        "\n",
        "        train_iter += 1\n",
        "\n",
        "        optimizer_fn.zero_grad()\n",
        "\n",
        "        inputVariable = inputs.permute(1, 0, 2, 3, 4).to(DEVICE)\n",
        "        inputVariable_DS = inputs_DS.permute(1, 0, 2, 3, 4).to(DEVICE)\n",
        "        labelVariable = targets.to(DEVICE)\n",
        "\n",
        "        output_label, _, output_label_mmaps = model(inputVariable, mmaps = True)\n",
        "\n",
        "        output_label_mmaps = output_label_mmaps.view(SEQ_LEN*BATCH_SIZE*MMAP_LENGTH, 2).to(DEVICE)\n",
        "        mmaps = torch.flatten(mmaps).long().to(DEVICE)\n",
        "\n",
        "        loss = loss_fn_sum(output_label_mmaps, mmaps)/(BATCH_SIZE * SEQ_LEN) + loss_fn(output_label, labelVariable)\n",
        "        loss.backward()\n",
        "\n",
        "        for layer in layers_on_off:\n",
        "            for params in layer.parameters():\n",
        "                params.requires_grad = False\n",
        "\n",
        "        output_label_ds1, _ = model(inputVariable, static_frames = True)\n",
        "        output_label_ds2, _ = model(inputVariable_DS, static_frames = True)\n",
        "        \n",
        "        gt_ds1 = torch.zeros(BATCH_SIZE).long().to(DEVICE)\n",
        "        gt_ds2 = torch.ones(BATCH_SIZE).long().to(DEVICE)\n",
        "\n",
        "        loss2 = loss_fn(output_label_ds1, gt_ds1) + loss_fn(output_label_ds2, gt_ds2)\n",
        "\n",
        "        loss2.backward()\n",
        "\n",
        "        for layer in layers_on_off:\n",
        "            for params in layer.parameters():\n",
        "                params.requires_grad = True\n",
        "\n",
        "        optimizer_fn.step()\n",
        "        \n",
        "        _, predicted = torch.max(output_label.data, 1)\n",
        "        numCorrTrain += torch.sum(predicted == labelVariable.data).data.item()\n",
        "        step_loss = loss.data.item()\n",
        "        step_loss2 = loss2.data.item()\n",
        "        epoch_loss += step_loss\n",
        "        epoch_loss2 += step_loss2\n",
        "        train_logger_2.add_step_data(train_iter, numCorrTrain, step_loss)\n",
        "\n",
        "        \n",
        "    avg_loss = epoch_loss/iterPerEpoch\n",
        "    avg_loss2 = epoch_loss2/iterPerEpoch\n",
        "    trainAccuracy = (numCorrTrain / trainSamples) * 100\n",
        "    train_logger_2.add_epoch_data(epoch+1, trainAccuracy, avg_loss)\n",
        "    \n",
        "    print('Train: Epoch = {} | Loss = {:.3f} | Loss2 = {:.3f} | Accuracy = {:.3f}'.format(epoch+1, avg_loss, avg_loss2, trainAccuracy))\n",
        "\n",
        "    if validate is not None:\n",
        "\n",
        "        if (epoch+1) % 1 == 0:\n",
        "\n",
        "            model.train(False)\n",
        "            val_loss_epoch = 0\n",
        "            numCorr = 0\n",
        "\n",
        "            for j, (inputs, targets) in enumerate(val_loader):\n",
        "\n",
        "                val_iter += 1\n",
        "\n",
        "                inputVariable = inputs.permute(1, 0, 2, 3, 4).to(DEVICE)\n",
        "                labelVariable = targets.to(DEVICE)\n",
        "                \n",
        "                output_label, _ = model(inputVariable)\n",
        "\n",
        "                val_loss = loss_fn(output_label, labelVariable)\n",
        "                val_loss_step = val_loss.data.item()\n",
        "                val_loss_epoch += val_loss_step\n",
        "\n",
        "                _, predicted = torch.max(output_label.data, 1)\n",
        "\n",
        "                numCorr += torch.sum(predicted == labelVariable.data).data.item()\n",
        "                val_logger_2.add_step_data(val_iter, numCorr, val_loss_step)\n",
        "\n",
        "            val_accuracy = (numCorr / val_samples) * 100\n",
        "            avg_val_loss = val_loss_epoch / val_steps\n",
        "\n",
        "            print('Val: Epoch = {} | Loss {:.3f} | Accuracy = {:.3f}'.format(epoch + 1, avg_val_loss, val_accuracy))\n",
        "            \n",
        "            val_logger_2.add_epoch_data(epoch+1, val_accuracy, avg_val_loss)\n",
        "            \n",
        "            if val_accuracy > min_accuracy:\n",
        "                print(\"[||| NEW BEST on val |||]\")\n",
        "                save_path_model = os.path.join(model_folder, model_checkpoint)\n",
        "                torch.save(model.state_dict(), save_path_model)\n",
        "                min_accuracy = val_accuracy\n",
        "            \n",
        "    optim_scheduler.step()\n",
        "\n",
        "    train_logger_2.save(train_log_file)\n",
        "    val_logger_2.save(val_log_file)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Train: Epoch = 1 | Loss = 35.652 | Loss2 = 1.480 | Accuracy = 39.375\n",
            "Val: Epoch = 1 | Loss 2.214 | Accuracy = 40.517\n",
            "[||| NEW BEST on val |||]\n",
            "Train: Epoch = 2 | Loss = 35.605 | Loss2 = 1.451 | Accuracy = 39.062\n",
            "Val: Epoch = 2 | Loss 2.334 | Accuracy = 38.793\n",
            "Train: Epoch = 3 | Loss = 35.473 | Loss2 = 1.407 | Accuracy = 40.938\n",
            "Val: Epoch = 3 | Loss 1.949 | Accuracy = 37.069\n",
            "Train: Epoch = 4 | Loss = 35.412 | Loss2 = 1.368 | Accuracy = 43.438\n",
            "Val: Epoch = 4 | Loss 1.972 | Accuracy = 47.414\n",
            "[||| NEW BEST on val |||]\n",
            "Train: Epoch = 5 | Loss = 35.372 | Loss2 = 1.400 | Accuracy = 47.188\n",
            "Val: Epoch = 5 | Loss 2.022 | Accuracy = 43.103\n",
            "Train: Epoch = 6 | Loss = 35.392 | Loss2 = 1.339 | Accuracy = 44.688\n",
            "Val: Epoch = 6 | Loss 2.096 | Accuracy = 43.966\n",
            "Train: Epoch = 7 | Loss = 35.341 | Loss2 = 1.292 | Accuracy = 50.000\n",
            "Val: Epoch = 7 | Loss 2.014 | Accuracy = 37.931\n",
            "Train: Epoch = 8 | Loss = 35.386 | Loss2 = 1.326 | Accuracy = 46.562\n",
            "Val: Epoch = 8 | Loss 1.855 | Accuracy = 44.828\n",
            "Train: Epoch = 9 | Loss = 35.324 | Loss2 = 1.261 | Accuracy = 48.750\n",
            "Val: Epoch = 9 | Loss 1.852 | Accuracy = 49.138\n",
            "[||| NEW BEST on val |||]\n",
            "Train: Epoch = 10 | Loss = 35.344 | Loss2 = 1.292 | Accuracy = 48.438\n",
            "Val: Epoch = 10 | Loss 1.722 | Accuracy = 47.414\n",
            "Train: Epoch = 11 | Loss = 35.153 | Loss2 = 1.269 | Accuracy = 53.125\n",
            "Val: Epoch = 11 | Loss 1.774 | Accuracy = 47.414\n",
            "Train: Epoch = 12 | Loss = 35.256 | Loss2 = 1.259 | Accuracy = 47.812\n",
            "Val: Epoch = 12 | Loss 1.945 | Accuracy = 50.000\n",
            "[||| NEW BEST on val |||]\n",
            "Train: Epoch = 13 | Loss = 35.247 | Loss2 = 1.171 | Accuracy = 49.688\n",
            "Val: Epoch = 13 | Loss 1.865 | Accuracy = 46.552\n",
            "Train: Epoch = 14 | Loss = 35.211 | Loss2 = 1.277 | Accuracy = 50.938\n",
            "Val: Epoch = 14 | Loss 1.687 | Accuracy = 54.310\n",
            "[||| NEW BEST on val |||]\n",
            "Train: Epoch = 15 | Loss = 35.236 | Loss2 = 1.169 | Accuracy = 49.688\n",
            "Val: Epoch = 15 | Loss 1.790 | Accuracy = 47.414\n",
            "Train: Epoch = 16 | Loss = 35.026 | Loss2 = 1.169 | Accuracy = 53.438\n",
            "Val: Epoch = 16 | Loss 1.881 | Accuracy = 46.552\n",
            "Train: Epoch = 17 | Loss = 35.329 | Loss2 = 1.131 | Accuracy = 45.625\n",
            "Val: Epoch = 17 | Loss 1.727 | Accuracy = 45.690\n",
            "Train: Epoch = 18 | Loss = 35.185 | Loss2 = 1.131 | Accuracy = 49.375\n",
            "Val: Epoch = 18 | Loss 1.650 | Accuracy = 46.552\n",
            "Train: Epoch = 19 | Loss = 35.150 | Loss2 = 1.047 | Accuracy = 47.812\n",
            "Val: Epoch = 19 | Loss 1.751 | Accuracy = 46.552\n",
            "Train: Epoch = 20 | Loss = 35.075 | Loss2 = 1.062 | Accuracy = 52.812\n",
            "Val: Epoch = 20 | Loss 1.687 | Accuracy = 46.552\n",
            "Train: Epoch = 21 | Loss = 35.077 | Loss2 = 1.102 | Accuracy = 55.312\n",
            "Val: Epoch = 21 | Loss 1.810 | Accuracy = 44.828\n",
            "Train: Epoch = 22 | Loss = 34.951 | Loss2 = 1.015 | Accuracy = 61.875\n",
            "Val: Epoch = 22 | Loss 1.760 | Accuracy = 50.862\n",
            "Train: Epoch = 23 | Loss = 35.007 | Loss2 = 1.049 | Accuracy = 56.875\n",
            "Val: Epoch = 23 | Loss 1.701 | Accuracy = 50.000\n",
            "Train: Epoch = 24 | Loss = 34.996 | Loss2 = 1.033 | Accuracy = 54.375\n",
            "Val: Epoch = 24 | Loss 1.691 | Accuracy = 51.724\n",
            "Train: Epoch = 25 | Loss = 35.111 | Loss2 = 1.028 | Accuracy = 52.188\n",
            "Val: Epoch = 25 | Loss 1.744 | Accuracy = 45.690\n",
            "Train: Epoch = 26 | Loss = 35.020 | Loss2 = 0.940 | Accuracy = 52.812\n",
            "Val: Epoch = 26 | Loss 1.769 | Accuracy = 46.552\n",
            "Train: Epoch = 27 | Loss = 35.006 | Loss2 = 0.897 | Accuracy = 55.625\n",
            "Val: Epoch = 27 | Loss 2.052 | Accuracy = 37.931\n",
            "Train: Epoch = 28 | Loss = 34.835 | Loss2 = 0.955 | Accuracy = 61.250\n",
            "Val: Epoch = 28 | Loss 1.657 | Accuracy = 46.552\n",
            "Train: Epoch = 29 | Loss = 35.022 | Loss2 = 0.939 | Accuracy = 56.875\n",
            "Val: Epoch = 29 | Loss 1.868 | Accuracy = 43.966\n",
            "Train: Epoch = 30 | Loss = 34.902 | Loss2 = 0.944 | Accuracy = 59.062\n",
            "Val: Epoch = 30 | Loss 1.936 | Accuracy = 45.690\n",
            "Train: Epoch = 31 | Loss = 35.045 | Loss2 = 0.936 | Accuracy = 54.375\n",
            "Val: Epoch = 31 | Loss 1.800 | Accuracy = 43.103\n",
            "Train: Epoch = 32 | Loss = 34.841 | Loss2 = 0.901 | Accuracy = 60.000\n",
            "Val: Epoch = 32 | Loss 1.709 | Accuracy = 50.000\n",
            "Train: Epoch = 33 | Loss = 34.818 | Loss2 = 0.894 | Accuracy = 60.625\n",
            "Val: Epoch = 33 | Loss 1.721 | Accuracy = 44.828\n",
            "Train: Epoch = 34 | Loss = 34.794 | Loss2 = 0.913 | Accuracy = 66.250\n",
            "Val: Epoch = 34 | Loss 1.675 | Accuracy = 47.414\n",
            "Train: Epoch = 35 | Loss = 34.751 | Loss2 = 0.961 | Accuracy = 64.375\n",
            "Val: Epoch = 35 | Loss 1.656 | Accuracy = 49.138\n",
            "Train: Epoch = 36 | Loss = 34.702 | Loss2 = 0.919 | Accuracy = 66.562\n",
            "Val: Epoch = 36 | Loss 1.605 | Accuracy = 52.586\n",
            "Train: Epoch = 37 | Loss = 34.775 | Loss2 = 0.943 | Accuracy = 63.750\n",
            "Val: Epoch = 37 | Loss 1.594 | Accuracy = 52.586\n",
            "Train: Epoch = 38 | Loss = 34.755 | Loss2 = 0.912 | Accuracy = 62.500\n",
            "Val: Epoch = 38 | Loss 1.547 | Accuracy = 49.138\n",
            "Train: Epoch = 39 | Loss = 34.604 | Loss2 = 0.885 | Accuracy = 65.938\n",
            "Val: Epoch = 39 | Loss 1.477 | Accuracy = 50.000\n",
            "Train: Epoch = 40 | Loss = 34.772 | Loss2 = 0.913 | Accuracy = 62.500\n",
            "Val: Epoch = 40 | Loss 1.502 | Accuracy = 50.862\n",
            "Train: Epoch = 41 | Loss = 34.716 | Loss2 = 0.940 | Accuracy = 61.875\n",
            "Val: Epoch = 41 | Loss 1.526 | Accuracy = 49.138\n",
            "Train: Epoch = 42 | Loss = 34.774 | Loss2 = 0.897 | Accuracy = 62.500\n",
            "Val: Epoch = 42 | Loss 1.557 | Accuracy = 51.724\n",
            "Train: Epoch = 43 | Loss = 34.686 | Loss2 = 0.904 | Accuracy = 66.250\n",
            "Val: Epoch = 43 | Loss 1.549 | Accuracy = 52.586\n",
            "Train: Epoch = 44 | Loss = 34.786 | Loss2 = 0.889 | Accuracy = 61.875\n",
            "Val: Epoch = 44 | Loss 1.567 | Accuracy = 50.000\n",
            "Train: Epoch = 45 | Loss = 34.780 | Loss2 = 0.914 | Accuracy = 61.250\n",
            "Val: Epoch = 45 | Loss 1.566 | Accuracy = 50.000\n",
            "Train: Epoch = 46 | Loss = 34.706 | Loss2 = 0.884 | Accuracy = 64.375\n",
            "Val: Epoch = 46 | Loss 1.572 | Accuracy = 50.000\n",
            "Train: Epoch = 47 | Loss = 34.645 | Loss2 = 0.858 | Accuracy = 66.562\n",
            "Val: Epoch = 47 | Loss 1.544 | Accuracy = 49.138\n",
            "Train: Epoch = 48 | Loss = 34.587 | Loss2 = 0.874 | Accuracy = 67.812\n",
            "Val: Epoch = 48 | Loss 1.501 | Accuracy = 52.586\n",
            "Train: Epoch = 49 | Loss = 34.691 | Loss2 = 0.879 | Accuracy = 62.187\n",
            "Val: Epoch = 49 | Loss 1.472 | Accuracy = 53.448\n",
            "Train: Epoch = 50 | Loss = 34.609 | Loss2 = 0.854 | Accuracy = 68.438\n",
            "Val: Epoch = 50 | Loss 1.609 | Accuracy = 51.724\n",
            "Train: Epoch = 51 | Loss = 34.638 | Loss2 = 0.865 | Accuracy = 68.125\n",
            "Val: Epoch = 51 | Loss 1.595 | Accuracy = 49.138\n",
            "Train: Epoch = 52 | Loss = 34.710 | Loss2 = 0.849 | Accuracy = 66.250\n",
            "Val: Epoch = 52 | Loss 1.524 | Accuracy = 53.448\n",
            "Train: Epoch = 53 | Loss = 34.646 | Loss2 = 0.860 | Accuracy = 64.375\n",
            "Val: Epoch = 53 | Loss 1.456 | Accuracy = 50.862\n",
            "Train: Epoch = 54 | Loss = 34.531 | Loss2 = 0.863 | Accuracy = 73.750\n",
            "Val: Epoch = 54 | Loss 1.471 | Accuracy = 50.000\n",
            "Train: Epoch = 55 | Loss = 34.586 | Loss2 = 0.807 | Accuracy = 66.875\n",
            "Val: Epoch = 55 | Loss 1.518 | Accuracy = 52.586\n",
            "Train: Epoch = 56 | Loss = 34.591 | Loss2 = 0.779 | Accuracy = 68.125\n",
            "Val: Epoch = 56 | Loss 1.546 | Accuracy = 50.000\n",
            "Train: Epoch = 57 | Loss = 34.633 | Loss2 = 0.856 | Accuracy = 66.250\n",
            "Val: Epoch = 57 | Loss 1.473 | Accuracy = 48.276\n",
            "Train: Epoch = 58 | Loss = 34.666 | Loss2 = 0.844 | Accuracy = 66.875\n",
            "Val: Epoch = 58 | Loss 1.534 | Accuracy = 50.000\n",
            "Train: Epoch = 59 | Loss = 34.642 | Loss2 = 0.879 | Accuracy = 65.625\n",
            "Val: Epoch = 59 | Loss 1.510 | Accuracy = 53.448\n",
            "Train: Epoch = 60 | Loss = 34.508 | Loss2 = 0.818 | Accuracy = 71.562\n",
            "Val: Epoch = 60 | Loss 1.485 | Accuracy = 52.586\n",
            "Train: Epoch = 61 | Loss = 34.552 | Loss2 = 0.846 | Accuracy = 65.000\n",
            "Val: Epoch = 61 | Loss 1.562 | Accuracy = 50.862\n",
            "Train: Epoch = 62 | Loss = 34.534 | Loss2 = 0.813 | Accuracy = 69.062\n",
            "Val: Epoch = 62 | Loss 1.496 | Accuracy = 51.724\n",
            "Train: Epoch = 63 | Loss = 34.466 | Loss2 = 0.806 | Accuracy = 69.688\n",
            "Val: Epoch = 63 | Loss 1.476 | Accuracy = 52.586\n",
            "Train: Epoch = 64 | Loss = 34.550 | Loss2 = 0.876 | Accuracy = 69.375\n",
            "Val: Epoch = 64 | Loss 1.523 | Accuracy = 50.862\n",
            "Train: Epoch = 65 | Loss = 34.586 | Loss2 = 0.867 | Accuracy = 68.438\n",
            "Val: Epoch = 65 | Loss 1.538 | Accuracy = 50.000\n",
            "Train: Epoch = 66 | Loss = 34.443 | Loss2 = 0.843 | Accuracy = 73.125\n",
            "Val: Epoch = 66 | Loss 1.604 | Accuracy = 50.862\n",
            "Train: Epoch = 67 | Loss = 34.534 | Loss2 = 0.800 | Accuracy = 67.812\n",
            "Val: Epoch = 67 | Loss 1.598 | Accuracy = 51.724\n",
            "Train: Epoch = 68 | Loss = 34.545 | Loss2 = 0.798 | Accuracy = 69.375\n",
            "Val: Epoch = 68 | Loss 1.639 | Accuracy = 50.000\n",
            "Train: Epoch = 69 | Loss = 34.482 | Loss2 = 0.785 | Accuracy = 70.625\n",
            "Val: Epoch = 69 | Loss 1.541 | Accuracy = 51.724\n",
            "Train: Epoch = 70 | Loss = 34.551 | Loss2 = 0.858 | Accuracy = 64.375\n",
            "Val: Epoch = 70 | Loss 1.619 | Accuracy = 52.586\n",
            "Train: Epoch = 71 | Loss = 34.553 | Loss2 = 0.808 | Accuracy = 70.938\n",
            "Val: Epoch = 71 | Loss 1.592 | Accuracy = 50.862\n",
            "Train: Epoch = 72 | Loss = 34.484 | Loss2 = 0.823 | Accuracy = 72.500\n",
            "Val: Epoch = 72 | Loss 1.498 | Accuracy = 51.724\n",
            "Train: Epoch = 73 | Loss = 34.562 | Loss2 = 0.879 | Accuracy = 69.062\n",
            "Val: Epoch = 73 | Loss 1.571 | Accuracy = 55.172\n",
            "[||| NEW BEST on val |||]\n",
            "Train: Epoch = 74 | Loss = 34.534 | Loss2 = 0.824 | Accuracy = 69.062\n",
            "Val: Epoch = 74 | Loss 1.567 | Accuracy = 51.724\n",
            "Train: Epoch = 75 | Loss = 34.488 | Loss2 = 0.801 | Accuracy = 71.875\n",
            "Val: Epoch = 75 | Loss 1.532 | Accuracy = 54.310\n",
            "Train: Epoch = 76 | Loss = 34.395 | Loss2 = 0.784 | Accuracy = 72.500\n",
            "Val: Epoch = 76 | Loss 1.540 | Accuracy = 51.724\n",
            "Train: Epoch = 77 | Loss = 34.497 | Loss2 = 0.900 | Accuracy = 71.250\n",
            "Val: Epoch = 77 | Loss 1.512 | Accuracy = 50.862\n",
            "Train: Epoch = 78 | Loss = 34.552 | Loss2 = 0.818 | Accuracy = 72.188\n",
            "Val: Epoch = 78 | Loss 1.554 | Accuracy = 56.897\n",
            "[||| NEW BEST on val |||]\n",
            "Train: Epoch = 79 | Loss = 34.447 | Loss2 = 0.722 | Accuracy = 70.625\n",
            "Val: Epoch = 79 | Loss 1.514 | Accuracy = 56.034\n",
            "Train: Epoch = 80 | Loss = 34.441 | Loss2 = 0.822 | Accuracy = 71.875\n",
            "Val: Epoch = 80 | Loss 1.630 | Accuracy = 50.000\n",
            "Train: Epoch = 81 | Loss = 34.461 | Loss2 = 0.775 | Accuracy = 71.250\n",
            "Val: Epoch = 81 | Loss 1.568 | Accuracy = 50.862\n",
            "Train: Epoch = 82 | Loss = 34.555 | Loss2 = 0.803 | Accuracy = 67.812\n",
            "Val: Epoch = 82 | Loss 1.550 | Accuracy = 50.862\n",
            "Train: Epoch = 83 | Loss = 34.512 | Loss2 = 0.792 | Accuracy = 68.438\n",
            "Val: Epoch = 83 | Loss 1.553 | Accuracy = 50.862\n",
            "Train: Epoch = 84 | Loss = 34.504 | Loss2 = 0.758 | Accuracy = 69.062\n",
            "Val: Epoch = 84 | Loss 1.565 | Accuracy = 52.586\n",
            "Train: Epoch = 85 | Loss = 34.446 | Loss2 = 0.794 | Accuracy = 72.500\n",
            "Val: Epoch = 85 | Loss 1.570 | Accuracy = 50.862\n",
            "Train: Epoch = 86 | Loss = 34.420 | Loss2 = 0.746 | Accuracy = 70.312\n",
            "Val: Epoch = 86 | Loss 1.564 | Accuracy = 51.724\n",
            "Train: Epoch = 87 | Loss = 34.449 | Loss2 = 0.792 | Accuracy = 72.812\n",
            "Val: Epoch = 87 | Loss 1.549 | Accuracy = 53.448\n",
            "Train: Epoch = 88 | Loss = 34.427 | Loss2 = 0.790 | Accuracy = 71.250\n",
            "Val: Epoch = 88 | Loss 1.580 | Accuracy = 51.724\n",
            "Train: Epoch = 89 | Loss = 34.382 | Loss2 = 0.792 | Accuracy = 72.812\n",
            "Val: Epoch = 89 | Loss 1.549 | Accuracy = 53.448\n",
            "Train: Epoch = 90 | Loss = 34.484 | Loss2 = 0.793 | Accuracy = 72.812\n",
            "Val: Epoch = 90 | Loss 1.532 | Accuracy = 52.586\n",
            "Train: Epoch = 91 | Loss = 34.419 | Loss2 = 0.807 | Accuracy = 75.312\n",
            "Val: Epoch = 91 | Loss 1.529 | Accuracy = 52.586\n",
            "Train: Epoch = 92 | Loss = 34.309 | Loss2 = 0.705 | Accuracy = 80.625\n",
            "Val: Epoch = 92 | Loss 1.512 | Accuracy = 52.586\n",
            "Train: Epoch = 93 | Loss = 34.396 | Loss2 = 0.846 | Accuracy = 72.188\n",
            "Val: Epoch = 93 | Loss 1.501 | Accuracy = 51.724\n",
            "Train: Epoch = 94 | Loss = 34.353 | Loss2 = 0.826 | Accuracy = 74.062\n",
            "Val: Epoch = 94 | Loss 1.521 | Accuracy = 53.448\n",
            "Train: Epoch = 95 | Loss = 34.445 | Loss2 = 0.808 | Accuracy = 73.125\n",
            "Val: Epoch = 95 | Loss 1.567 | Accuracy = 52.586\n",
            "Train: Epoch = 96 | Loss = 34.413 | Loss2 = 0.769 | Accuracy = 72.812\n",
            "Val: Epoch = 96 | Loss 1.593 | Accuracy = 51.724\n",
            "Train: Epoch = 97 | Loss = 34.331 | Loss2 = 0.795 | Accuracy = 77.188\n",
            "Val: Epoch = 97 | Loss 1.570 | Accuracy = 51.724\n",
            "Train: Epoch = 98 | Loss = 34.531 | Loss2 = 0.763 | Accuracy = 70.938\n",
            "Val: Epoch = 98 | Loss 1.544 | Accuracy = 53.448\n",
            "Train: Epoch = 99 | Loss = 34.398 | Loss2 = 0.773 | Accuracy = 70.938\n",
            "Val: Epoch = 99 | Loss 1.524 | Accuracy = 51.724\n",
            "Train: Epoch = 100 | Loss = 34.364 | Loss2 = 0.828 | Accuracy = 77.188\n",
            "Val: Epoch = 100 | Loss 1.540 | Accuracy = 51.724\n",
            "Train: Epoch = 101 | Loss = 34.358 | Loss2 = 0.791 | Accuracy = 73.750\n",
            "Val: Epoch = 101 | Loss 1.529 | Accuracy = 51.724\n",
            "Train: Epoch = 102 | Loss = 34.394 | Loss2 = 0.757 | Accuracy = 73.438\n",
            "Val: Epoch = 102 | Loss 1.523 | Accuracy = 53.448\n",
            "Train: Epoch = 103 | Loss = 34.333 | Loss2 = 0.784 | Accuracy = 77.188\n",
            "Val: Epoch = 103 | Loss 1.503 | Accuracy = 55.172\n",
            "Train: Epoch = 104 | Loss = 34.468 | Loss2 = 0.772 | Accuracy = 71.875\n",
            "Val: Epoch = 104 | Loss 1.524 | Accuracy = 53.448\n",
            "Train: Epoch = 105 | Loss = 34.388 | Loss2 = 0.759 | Accuracy = 73.125\n",
            "Val: Epoch = 105 | Loss 1.580 | Accuracy = 53.448\n",
            "Train: Epoch = 106 | Loss = 34.382 | Loss2 = 0.735 | Accuracy = 70.938\n",
            "Val: Epoch = 106 | Loss 1.568 | Accuracy = 52.586\n",
            "Train: Epoch = 107 | Loss = 34.368 | Loss2 = 0.765 | Accuracy = 75.312\n",
            "Val: Epoch = 107 | Loss 1.567 | Accuracy = 51.724\n",
            "Train: Epoch = 108 | Loss = 34.306 | Loss2 = 0.760 | Accuracy = 75.312\n",
            "Val: Epoch = 108 | Loss 1.559 | Accuracy = 53.448\n",
            "Train: Epoch = 109 | Loss = 34.323 | Loss2 = 0.730 | Accuracy = 79.062\n",
            "Val: Epoch = 109 | Loss 1.573 | Accuracy = 54.310\n",
            "Train: Epoch = 110 | Loss = 34.375 | Loss2 = 0.748 | Accuracy = 74.375\n",
            "Val: Epoch = 110 | Loss 1.589 | Accuracy = 55.172\n",
            "Train: Epoch = 111 | Loss = 34.420 | Loss2 = 0.726 | Accuracy = 74.688\n",
            "Val: Epoch = 111 | Loss 1.572 | Accuracy = 55.172\n",
            "Train: Epoch = 112 | Loss = 34.382 | Loss2 = 0.824 | Accuracy = 71.562\n",
            "Val: Epoch = 112 | Loss 1.547 | Accuracy = 54.310\n",
            "Train: Epoch = 113 | Loss = 34.476 | Loss2 = 0.738 | Accuracy = 70.000\n",
            "Val: Epoch = 113 | Loss 1.535 | Accuracy = 55.172\n",
            "Train: Epoch = 114 | Loss = 34.360 | Loss2 = 0.740 | Accuracy = 73.125\n",
            "Val: Epoch = 114 | Loss 1.527 | Accuracy = 54.310\n",
            "Train: Epoch = 115 | Loss = 34.411 | Loss2 = 0.773 | Accuracy = 71.562\n",
            "Val: Epoch = 115 | Loss 1.534 | Accuracy = 56.034\n",
            "Train: Epoch = 116 | Loss = 34.394 | Loss2 = 0.769 | Accuracy = 70.625\n",
            "Val: Epoch = 116 | Loss 1.555 | Accuracy = 54.310\n",
            "Train: Epoch = 117 | Loss = 34.333 | Loss2 = 0.802 | Accuracy = 76.875\n",
            "Val: Epoch = 117 | Loss 1.562 | Accuracy = 55.172\n",
            "Train: Epoch = 118 | Loss = 34.496 | Loss2 = 0.749 | Accuracy = 71.875\n",
            "Val: Epoch = 118 | Loss 1.545 | Accuracy = 53.448\n",
            "Train: Epoch = 119 | Loss = 34.287 | Loss2 = 0.790 | Accuracy = 79.375\n",
            "Val: Epoch = 119 | Loss 1.522 | Accuracy = 54.310\n",
            "Train: Epoch = 120 | Loss = 34.448 | Loss2 = 0.784 | Accuracy = 71.875\n",
            "Val: Epoch = 120 | Loss 1.531 | Accuracy = 54.310\n",
            "Train: Epoch = 121 | Loss = 34.348 | Loss2 = 0.827 | Accuracy = 76.250\n",
            "Val: Epoch = 121 | Loss 1.568 | Accuracy = 54.310\n",
            "Train: Epoch = 122 | Loss = 34.339 | Loss2 = 0.743 | Accuracy = 76.875\n",
            "Val: Epoch = 122 | Loss 1.573 | Accuracy = 55.172\n",
            "Train: Epoch = 123 | Loss = 34.268 | Loss2 = 0.853 | Accuracy = 80.000\n",
            "Val: Epoch = 123 | Loss 1.564 | Accuracy = 55.172\n",
            "Train: Epoch = 124 | Loss = 34.314 | Loss2 = 0.708 | Accuracy = 77.188\n",
            "Val: Epoch = 124 | Loss 1.577 | Accuracy = 54.310\n",
            "Train: Epoch = 125 | Loss = 34.408 | Loss2 = 0.741 | Accuracy = 71.562\n",
            "Val: Epoch = 125 | Loss 1.566 | Accuracy = 55.172\n",
            "Train: Epoch = 126 | Loss = 34.374 | Loss2 = 0.761 | Accuracy = 73.438\n",
            "Val: Epoch = 126 | Loss 1.544 | Accuracy = 54.310\n",
            "Train: Epoch = 127 | Loss = 34.472 | Loss2 = 0.801 | Accuracy = 71.875\n",
            "Val: Epoch = 127 | Loss 1.523 | Accuracy = 55.172\n",
            "Train: Epoch = 128 | Loss = 34.336 | Loss2 = 0.707 | Accuracy = 74.375\n",
            "Val: Epoch = 128 | Loss 1.532 | Accuracy = 53.448\n",
            "Train: Epoch = 129 | Loss = 34.242 | Loss2 = 0.706 | Accuracy = 78.125\n",
            "Val: Epoch = 129 | Loss 1.542 | Accuracy = 52.586\n",
            "Train: Epoch = 130 | Loss = 34.433 | Loss2 = 0.693 | Accuracy = 71.250\n",
            "Val: Epoch = 130 | Loss 1.573 | Accuracy = 51.724\n",
            "Train: Epoch = 131 | Loss = 34.257 | Loss2 = 0.749 | Accuracy = 76.562\n",
            "Val: Epoch = 131 | Loss 1.581 | Accuracy = 51.724\n",
            "Train: Epoch = 132 | Loss = 34.353 | Loss2 = 0.752 | Accuracy = 74.688\n",
            "Val: Epoch = 132 | Loss 1.604 | Accuracy = 53.448\n",
            "Train: Epoch = 133 | Loss = 34.370 | Loss2 = 0.759 | Accuracy = 72.188\n",
            "Val: Epoch = 133 | Loss 1.612 | Accuracy = 50.862\n",
            "Train: Epoch = 134 | Loss = 34.324 | Loss2 = 0.733 | Accuracy = 76.875\n",
            "Val: Epoch = 134 | Loss 1.604 | Accuracy = 53.448\n",
            "Train: Epoch = 135 | Loss = 34.317 | Loss2 = 0.689 | Accuracy = 75.625\n",
            "Val: Epoch = 135 | Loss 1.582 | Accuracy = 53.448\n",
            "Train: Epoch = 136 | Loss = 34.317 | Loss2 = 0.707 | Accuracy = 76.875\n",
            "Val: Epoch = 136 | Loss 1.572 | Accuracy = 51.724\n",
            "Train: Epoch = 137 | Loss = 34.360 | Loss2 = 0.773 | Accuracy = 75.625\n",
            "Val: Epoch = 137 | Loss 1.565 | Accuracy = 50.862\n",
            "Train: Epoch = 138 | Loss = 34.273 | Loss2 = 0.750 | Accuracy = 80.000\n",
            "Val: Epoch = 138 | Loss 1.570 | Accuracy = 50.862\n",
            "Train: Epoch = 139 | Loss = 34.327 | Loss2 = 0.729 | Accuracy = 75.938\n",
            "Val: Epoch = 139 | Loss 1.612 | Accuracy = 52.586\n",
            "Train: Epoch = 140 | Loss = 34.349 | Loss2 = 0.700 | Accuracy = 74.062\n",
            "Val: Epoch = 140 | Loss 1.630 | Accuracy = 52.586\n",
            "Train: Epoch = 141 | Loss = 34.331 | Loss2 = 0.694 | Accuracy = 76.250\n",
            "Val: Epoch = 141 | Loss 1.591 | Accuracy = 52.586\n",
            "Train: Epoch = 142 | Loss = 34.397 | Loss2 = 0.736 | Accuracy = 73.750\n",
            "Val: Epoch = 142 | Loss 1.557 | Accuracy = 51.724\n",
            "Train: Epoch = 143 | Loss = 34.358 | Loss2 = 0.694 | Accuracy = 75.312\n",
            "Val: Epoch = 143 | Loss 1.537 | Accuracy = 51.724\n",
            "Train: Epoch = 144 | Loss = 34.381 | Loss2 = 0.697 | Accuracy = 73.750\n",
            "Val: Epoch = 144 | Loss 1.526 | Accuracy = 51.724\n",
            "Train: Epoch = 145 | Loss = 34.516 | Loss2 = 0.767 | Accuracy = 69.062\n",
            "Val: Epoch = 145 | Loss 1.551 | Accuracy = 52.586\n",
            "Train: Epoch = 146 | Loss = 34.367 | Loss2 = 0.835 | Accuracy = 77.188\n",
            "Val: Epoch = 146 | Loss 1.591 | Accuracy = 50.000\n",
            "Train: Epoch = 147 | Loss = 34.353 | Loss2 = 0.756 | Accuracy = 75.625\n",
            "Val: Epoch = 147 | Loss 1.540 | Accuracy = 52.586\n",
            "Train: Epoch = 148 | Loss = 34.329 | Loss2 = 0.745 | Accuracy = 75.625\n",
            "Val: Epoch = 148 | Loss 1.527 | Accuracy = 52.586\n",
            "Train: Epoch = 149 | Loss = 34.385 | Loss2 = 0.798 | Accuracy = 73.125\n",
            "Val: Epoch = 149 | Loss 1.526 | Accuracy = 52.586\n",
            "Train: Epoch = 150 | Loss = 34.385 | Loss2 = 0.688 | Accuracy = 73.750\n",
            "Val: Epoch = 150 | Loss 1.516 | Accuracy = 53.448\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "ds-TpqmA0IYf"
      },
      "source": [
        "# Stage 2 specific setup"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "ExecuteTime": {
          "end_time": "2020-06-08T15:51:16.753000Z",
          "start_time": "2020-06-08T15:51:16.742346Z"
        },
        "colab_type": "code",
        "id": "wyzSjswC0IYk",
        "colab": {}
      },
      "source": [
        "STAGE = 2\n",
        "\n",
        "LR = 0.0001            # The initial Learning Rate\n",
        "MOMENTUM = 0.9       # Hyperparameter for SGD, keep this at 0.9 when using SGD\n",
        "WEIGHT_DECAY = 4e-5  # Regularization, you can keep this at the default\n",
        "\n",
        "NUM_EPOCHS = 150      # Total number of training epochs (iterations over dataset)\n",
        "STEP_SIZE = [30, 80] # How many epochs before decreasing learning rate (if using a step-down policy)\n",
        "GAMMA = 0.2          # Multiplicative factor for learning rate step-down\n",
        "MEM_SIZE = 512\n",
        "SEQ_LEN = 7\n",
        "ALPHA = 0.5\n",
        "\n",
        "parameters = {\n",
        "    'DEVICE': DEVICE,\n",
        "    'NUM_CLASSES': NUM_CLASSES,\n",
        "    'BATCH_SIZE': BATCH_SIZE,\n",
        "    'LR': LR,\n",
        "    'MOMENTUM': MOMENTUM,\n",
        "    'WEIGHT_DECAY': WEIGHT_DECAY,\n",
        "    'NUM_EPOCHS': NUM_EPOCHS,\n",
        "    'STEP_SIZE': STEP_SIZE,\n",
        "    'GAMMA': GAMMA,\n",
        "    'MEM_SIZE': MEM_SIZE,\n",
        "    'SEQ_LEN': SEQ_LEN\n",
        "}"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "ExecuteTime": {
          "end_time": "2020-06-08T15:51:18.906868Z",
          "start_time": "2020-06-08T15:51:18.872090Z"
        },
        "colab_type": "code",
        "id": "6e0AErLb0IYp",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 51
        },
        "outputId": "004a9659-2fb8-490e-bbef-bda85ebd4b15"
      },
      "source": [
        "# Prepare Pytorch train/test Datasets\n",
        "train_dataset = GTEA61(DATA_DIR, split='train', transform=spatial_transform,\n",
        "                       seq_len=SEQ_LEN, mmaps = True, mmaps_transform = spatial_transform_mmaps, static_frames = True)\n",
        "test_dataset = GTEA61(DATA_DIR, split='test', transform=spatial_transform_val,\n",
        "                      seq_len=SEQ_LEN)\n",
        "\n",
        "# Check dataset sizes\n",
        "print('Train Dataset: {}'.format(len(train_dataset)))\n",
        "print('Test Dataset: {}'.format(len(test_dataset)))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Train Dataset: 341\n",
            "Test Dataset: 116\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "ExecuteTime": {
          "end_time": "2020-06-08T15:51:30.177645Z",
          "start_time": "2020-06-08T15:51:30.170111Z"
        },
        "colab_type": "code",
        "id": "RPLIqUM20IYt",
        "colab": {}
      },
      "source": [
        "# Dataloaders iterate over pytorch datasets and transparently provide useful functions (e.g. parallelization and shuffling)\n",
        "train_loader = DataLoader(train_dataset, batch_size=BATCH_SIZE, shuffle=True, num_workers=4, drop_last=True)\n",
        "val_loader = DataLoader(test_dataset, batch_size=BATCH_SIZE, shuffle=False, num_workers=4)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "EbLKUtyM0IYw"
      },
      "source": [
        "# Prepare Stage 2"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "ExecuteTime": {
          "end_time": "2020-06-08T15:51:45.195953Z",
          "start_time": "2020-06-08T15:51:41.943238Z"
        },
        "colab_type": "code",
        "id": "4LsFkJ6y0IYx",
        "colab": {}
      },
      "source": [
        "best_old_stage = generate_model_checkpoint_name(stage=1, n_frames=SEQ_LEN, ms_block = True, \\\n",
        "                                                optional = '_2neur_DS__run01')\n",
        "stage1_dict = os.path.join(model_folder, best_old_stage)\n",
        "validate = True\n",
        "\n",
        "model = attention_model_ms(num_classes=NUM_CLASSES, mem_size=MEM_SIZE)\n",
        "model.load_state_dict(torch.load(stage1_dict))\n",
        "\n",
        "model.train(False)\n",
        "\n",
        "for params in model.parameters():\n",
        "    params.requires_grad = False\n",
        "\n",
        "layers_on_off = [\n",
        "    model.resNet.layer4[0].conv1,\n",
        "    model.resNet.layer4[0].conv2,\n",
        "    model.resNet.layer4[1].conv1,\n",
        "    model.resNet.layer4[1].conv2,\n",
        "    model.resNet.layer4[2].conv1,\n",
        "    model.resNet.layer4[2].conv2,\n",
        "    model.resNet.fc,\n",
        "    model.msBlock\n",
        "]\n",
        "\n",
        "layers_to_train = layers_on_off + [\n",
        "    model.lstm_cell,\n",
        "    model.classifier,\n",
        "    model.classifier_SD\n",
        "]\n",
        "\n",
        "for layer in layers_to_train:\n",
        "    for params in layer.parameters():\n",
        "        params.requires_grad = True\n",
        "\n",
        "for layer in layers_to_train:\n",
        "    layer.train(True)\n",
        "\n",
        "model.to(DEVICE)\n",
        "\n",
        "loss_fn = nn.CrossEntropyLoss()\n",
        "loss_fn_sum = nn.CrossEntropyLoss(reduction = 'sum')\n",
        "\n",
        "trainable_params = [p for p in model.parameters() if p.requires_grad]\n",
        "optimizer_fn = torch.optim.Adam(trainable_params, lr=LR, weight_decay=WEIGHT_DECAY, eps=1e-4)\n",
        "\n",
        "optim_scheduler = torch.optim.lr_scheduler.MultiStepLR(optimizer_fn, milestones=STEP_SIZE, gamma=GAMMA)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "BmZSW6Qw0IY0"
      },
      "source": [
        "# Stage 2"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "ExecuteTime": {
          "end_time": "2020-06-08T08:58:05.318659Z",
          "start_time": "2020-06-08T08:57:45.955602Z"
        },
        "colab_type": "code",
        "id": "5ysxJGKW0IY1",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "outputId": "eabe2dac-13c4-4b03-bbc9-d694e0fe6cb9"
      },
      "source": [
        "RUN = \"_run01\"\n",
        "\n",
        "train_iter = 0\n",
        "val_iter = 0\n",
        "min_accuracy = 0\n",
        "\n",
        "trainSamples = len(train_dataset) - (len(train_dataset) % BATCH_SIZE)\n",
        "val_samples = len(test_dataset)\n",
        "\n",
        "iterPerEpoch = len(train_loader)\n",
        "val_steps = len(val_loader)\n",
        "\n",
        "cudnn.benchmark\n",
        "torch.autograd.set_detect_anomaly(True)\n",
        "train_log, val_log = generate_log_filenames(STAGE, SEQ_LEN, ms_block = True, \\\n",
        "                                            optional = '_2neur_DS_alpha_' + RUN)\n",
        "model_checkpoint = generate_model_checkpoint_name(STAGE, SEQ_LEN, ms_block = True, \\\n",
        "                                                  optional = '_2neur_DS_alpha_' + RUN)\n",
        "\n",
        "train_log_file = os.path.join(model_folder, train_log)\n",
        "val_log_file = os.path.join(model_folder, val_log)\n",
        "train_logger_2 = Logger(**parameters)\n",
        "val_logger_2 = Logger(**parameters)\n",
        "\n",
        "for epoch in range(NUM_EPOCHS):\n",
        "\n",
        "    epoch_loss = 0\n",
        "    epoch_loss2 = 0\n",
        "    numCorrTrain = 0\n",
        "    \n",
        "    for layer in layers_to_train:\n",
        "        layer.train(True)\n",
        "    \n",
        "    for i, (inputs, inputs_DS, mmaps, targets) in enumerate(train_loader):\n",
        "\n",
        "        mmaps = mmaps.permute(1, 0, 2)\n",
        "\n",
        "        train_iter += 1\n",
        "\n",
        "        optimizer_fn.zero_grad()\n",
        "\n",
        "        inputVariable = inputs.permute(1, 0, 2, 3, 4).to(DEVICE)\n",
        "        inputVariable_DS = inputs_DS.permute(1, 0, 2, 3, 4).to(DEVICE)\n",
        "        labelVariable = targets.to(DEVICE)\n",
        "\n",
        "        output_label, _, output_label_mmaps = model(inputVariable, mmaps = True)\n",
        "\n",
        "        output_label_mmaps = output_label_mmaps.view(SEQ_LEN*BATCH_SIZE*MMAP_LENGTH, 2).to(DEVICE)\n",
        "        mmaps = torch.flatten(mmaps).long().to(DEVICE)\n",
        "\n",
        "        loss = loss_fn_sum(output_label_mmaps, mmaps)/(BATCH_SIZE * SEQ_LEN) + loss_fn(output_label, labelVariable)\n",
        "        loss.backward()\n",
        "\n",
        "        for layer in layers_on_off:\n",
        "            for params in layer.parameters():\n",
        "                params.requires_grad = False\n",
        "\n",
        "        output_label_ds1, _ = model(inputVariable, static_frames = True)\n",
        "        output_label_ds2, _ = model(inputVariable_DS, static_frames = True)\n",
        "        \n",
        "        gt_ds1 = torch.zeros(BATCH_SIZE).long().to(DEVICE)\n",
        "        gt_ds2 = torch.ones(BATCH_SIZE).long().to(DEVICE)\n",
        "\n",
        "        loss2 = ALPHA * (loss_fn(output_label_ds1, gt_ds1) + loss_fn(output_label_ds2, gt_ds2))\n",
        "\n",
        "        loss2.backward()\n",
        "\n",
        "        for layer in layers_on_off:\n",
        "            for params in layer.parameters():\n",
        "                params.requires_grad = True\n",
        "\n",
        "        optimizer_fn.step()\n",
        "        \n",
        "        _, predicted = torch.max(output_label.data, 1)\n",
        "        numCorrTrain += torch.sum(predicted == labelVariable.data).data.item()\n",
        "        step_loss = loss.data.item()\n",
        "        step_loss2 = loss2.data.item()\n",
        "        epoch_loss += step_loss\n",
        "        epoch_loss2 += step_loss2\n",
        "        train_logger_2.add_step_data(train_iter, numCorrTrain, step_loss)\n",
        "\n",
        "        \n",
        "    avg_loss = epoch_loss/iterPerEpoch\n",
        "    avg_loss2 = epoch_loss2/iterPerEpoch\n",
        "    trainAccuracy = (numCorrTrain / trainSamples) * 100\n",
        "    train_logger_2.add_epoch_data(epoch+1, trainAccuracy, avg_loss)\n",
        "    \n",
        "    print('Train: Epoch = {} | Loss = {:.3f} | Loss2 = {:.3f} | Accuracy = {:.3f}'.format(epoch+1, avg_loss, avg_loss2, trainAccuracy))\n",
        "\n",
        "    if validate is not None:\n",
        "\n",
        "        if (epoch+1) % 1 == 0:\n",
        "\n",
        "            model.train(False)\n",
        "            val_loss_epoch = 0\n",
        "            numCorr = 0\n",
        "\n",
        "            for j, (inputs, targets) in enumerate(val_loader):\n",
        "\n",
        "                val_iter += 1\n",
        "\n",
        "                inputVariable = inputs.permute(1, 0, 2, 3, 4).to(DEVICE)\n",
        "                labelVariable = targets.to(DEVICE)\n",
        "                \n",
        "                output_label, _ = model(inputVariable)\n",
        "\n",
        "                val_loss = loss_fn(output_label, labelVariable)\n",
        "                val_loss_step = val_loss.data.item()\n",
        "                val_loss_epoch += val_loss_step\n",
        "\n",
        "                _, predicted = torch.max(output_label.data, 1)\n",
        "\n",
        "                numCorr += torch.sum(predicted == labelVariable.data).data.item()\n",
        "                val_logger_2.add_step_data(val_iter, numCorr, val_loss_step)\n",
        "\n",
        "            val_accuracy = (numCorr / val_samples) * 100\n",
        "            avg_val_loss = val_loss_epoch / val_steps\n",
        "\n",
        "            print('Val: Epoch = {} | Loss {:.3f} | Accuracy = {:.3f}'.format(epoch + 1, avg_val_loss, val_accuracy))\n",
        "            \n",
        "            val_logger_2.add_epoch_data(epoch+1, val_accuracy, avg_val_loss)\n",
        "            \n",
        "            if val_accuracy > min_accuracy:\n",
        "                print(\"[||| NEW BEST on val |||]\")\n",
        "                save_path_model = os.path.join(model_folder, model_checkpoint)\n",
        "                torch.save(model.state_dict(), save_path_model)\n",
        "                min_accuracy = val_accuracy\n",
        "            \n",
        "    optim_scheduler.step()\n",
        "\n",
        "    train_logger_2.save(train_log_file)\n",
        "    val_logger_2.save(val_log_file)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Train: Epoch = 1 | Loss = 36.228 | Loss2 = 0.746 | Accuracy = 33.750\n",
            "Val: Epoch = 1 | Loss 2.457 | Accuracy = 32.759\n",
            "[||| NEW BEST on val |||]\n",
            "Train: Epoch = 2 | Loss = 35.774 | Loss2 = 0.769 | Accuracy = 38.125\n",
            "Val: Epoch = 2 | Loss 2.180 | Accuracy = 41.379\n",
            "[||| NEW BEST on val |||]\n",
            "Train: Epoch = 3 | Loss = 35.584 | Loss2 = 0.776 | Accuracy = 39.062\n",
            "Val: Epoch = 3 | Loss 2.091 | Accuracy = 43.966\n",
            "[||| NEW BEST on val |||]\n",
            "Train: Epoch = 4 | Loss = 35.316 | Loss2 = 0.753 | Accuracy = 50.000\n",
            "Val: Epoch = 4 | Loss 2.031 | Accuracy = 40.517\n",
            "Train: Epoch = 5 | Loss = 35.466 | Loss2 = 0.725 | Accuracy = 47.500\n",
            "Val: Epoch = 5 | Loss 1.925 | Accuracy = 48.276\n",
            "[||| NEW BEST on val |||]\n",
            "Train: Epoch = 6 | Loss = 35.349 | Loss2 = 0.758 | Accuracy = 46.562\n",
            "Val: Epoch = 6 | Loss 1.931 | Accuracy = 51.724\n",
            "[||| NEW BEST on val |||]\n",
            "Train: Epoch = 7 | Loss = 35.332 | Loss2 = 0.704 | Accuracy = 49.688\n",
            "Val: Epoch = 7 | Loss 1.973 | Accuracy = 46.552\n",
            "Train: Epoch = 8 | Loss = 35.183 | Loss2 = 0.711 | Accuracy = 48.125\n",
            "Val: Epoch = 8 | Loss 1.997 | Accuracy = 43.966\n",
            "Train: Epoch = 9 | Loss = 35.356 | Loss2 = 0.717 | Accuracy = 45.625\n",
            "Val: Epoch = 9 | Loss 1.978 | Accuracy = 37.931\n",
            "Train: Epoch = 10 | Loss = 35.417 | Loss2 = 0.678 | Accuracy = 45.625\n",
            "Val: Epoch = 10 | Loss 1.882 | Accuracy = 44.828\n",
            "Train: Epoch = 11 | Loss = 35.354 | Loss2 = 0.686 | Accuracy = 46.562\n",
            "Val: Epoch = 11 | Loss 1.798 | Accuracy = 46.552\n",
            "Train: Epoch = 12 | Loss = 35.332 | Loss2 = 0.655 | Accuracy = 45.312\n",
            "Val: Epoch = 12 | Loss 1.973 | Accuracy = 50.862\n",
            "Train: Epoch = 13 | Loss = 35.197 | Loss2 = 0.673 | Accuracy = 50.625\n",
            "Val: Epoch = 13 | Loss 1.756 | Accuracy = 56.034\n",
            "[||| NEW BEST on val |||]\n",
            "Train: Epoch = 14 | Loss = 35.194 | Loss2 = 0.654 | Accuracy = 51.875\n",
            "Val: Epoch = 14 | Loss 1.780 | Accuracy = 52.586\n",
            "Train: Epoch = 15 | Loss = 35.184 | Loss2 = 0.653 | Accuracy = 52.188\n",
            "Val: Epoch = 15 | Loss 1.736 | Accuracy = 51.724\n",
            "Train: Epoch = 16 | Loss = 35.096 | Loss2 = 0.674 | Accuracy = 54.688\n",
            "Val: Epoch = 16 | Loss 1.782 | Accuracy = 48.276\n",
            "Train: Epoch = 17 | Loss = 35.162 | Loss2 = 0.657 | Accuracy = 50.938\n",
            "Val: Epoch = 17 | Loss 1.859 | Accuracy = 46.552\n",
            "Train: Epoch = 18 | Loss = 35.156 | Loss2 = 0.646 | Accuracy = 50.938\n",
            "Val: Epoch = 18 | Loss 1.705 | Accuracy = 53.448\n",
            "Train: Epoch = 19 | Loss = 35.148 | Loss2 = 0.661 | Accuracy = 54.062\n",
            "Val: Epoch = 19 | Loss 1.715 | Accuracy = 50.000\n",
            "Train: Epoch = 20 | Loss = 35.190 | Loss2 = 0.654 | Accuracy = 49.375\n",
            "Val: Epoch = 20 | Loss 1.731 | Accuracy = 48.276\n",
            "Train: Epoch = 21 | Loss = 35.054 | Loss2 = 0.643 | Accuracy = 53.125\n",
            "Val: Epoch = 21 | Loss 1.628 | Accuracy = 51.724\n",
            "Train: Epoch = 22 | Loss = 35.157 | Loss2 = 0.610 | Accuracy = 51.250\n",
            "Val: Epoch = 22 | Loss 1.770 | Accuracy = 48.276\n",
            "Train: Epoch = 23 | Loss = 35.042 | Loss2 = 0.628 | Accuracy = 53.750\n",
            "Val: Epoch = 23 | Loss 1.681 | Accuracy = 52.586\n",
            "Train: Epoch = 24 | Loss = 35.050 | Loss2 = 0.609 | Accuracy = 50.625\n",
            "Val: Epoch = 24 | Loss 1.681 | Accuracy = 50.862\n",
            "Train: Epoch = 25 | Loss = 35.004 | Loss2 = 0.643 | Accuracy = 59.375\n",
            "Val: Epoch = 25 | Loss 1.691 | Accuracy = 49.138\n",
            "Train: Epoch = 26 | Loss = 34.931 | Loss2 = 0.638 | Accuracy = 57.188\n",
            "Val: Epoch = 26 | Loss 1.890 | Accuracy = 44.828\n",
            "Train: Epoch = 27 | Loss = 34.998 | Loss2 = 0.610 | Accuracy = 57.500\n",
            "Val: Epoch = 27 | Loss 1.710 | Accuracy = 50.000\n",
            "Train: Epoch = 28 | Loss = 34.821 | Loss2 = 0.616 | Accuracy = 60.000\n",
            "Val: Epoch = 28 | Loss 1.530 | Accuracy = 53.448\n",
            "Train: Epoch = 29 | Loss = 34.832 | Loss2 = 0.637 | Accuracy = 62.813\n",
            "Val: Epoch = 29 | Loss 1.728 | Accuracy = 49.138\n",
            "Train: Epoch = 30 | Loss = 34.819 | Loss2 = 0.563 | Accuracy = 62.500\n",
            "Val: Epoch = 30 | Loss 1.545 | Accuracy = 51.724\n",
            "Train: Epoch = 31 | Loss = 34.868 | Loss2 = 0.594 | Accuracy = 59.062\n",
            "Val: Epoch = 31 | Loss 1.593 | Accuracy = 53.448\n",
            "Train: Epoch = 32 | Loss = 34.716 | Loss2 = 0.623 | Accuracy = 63.125\n",
            "Val: Epoch = 32 | Loss 1.628 | Accuracy = 52.586\n",
            "Train: Epoch = 33 | Loss = 34.669 | Loss2 = 0.609 | Accuracy = 65.625\n",
            "Val: Epoch = 33 | Loss 1.507 | Accuracy = 52.586\n",
            "Train: Epoch = 34 | Loss = 34.797 | Loss2 = 0.585 | Accuracy = 61.875\n",
            "Val: Epoch = 34 | Loss 1.519 | Accuracy = 50.000\n",
            "Train: Epoch = 35 | Loss = 34.829 | Loss2 = 0.626 | Accuracy = 62.500\n",
            "Val: Epoch = 35 | Loss 1.603 | Accuracy = 52.586\n",
            "Train: Epoch = 36 | Loss = 34.585 | Loss2 = 0.598 | Accuracy = 67.500\n",
            "Val: Epoch = 36 | Loss 1.616 | Accuracy = 52.586\n",
            "Train: Epoch = 37 | Loss = 34.696 | Loss2 = 0.617 | Accuracy = 63.438\n",
            "Val: Epoch = 37 | Loss 1.599 | Accuracy = 53.448\n",
            "Train: Epoch = 38 | Loss = 34.642 | Loss2 = 0.606 | Accuracy = 64.688\n",
            "Val: Epoch = 38 | Loss 1.585 | Accuracy = 51.724\n",
            "Train: Epoch = 39 | Loss = 34.708 | Loss2 = 0.620 | Accuracy = 66.875\n",
            "Val: Epoch = 39 | Loss 1.550 | Accuracy = 51.724\n",
            "Train: Epoch = 40 | Loss = 34.657 | Loss2 = 0.609 | Accuracy = 62.813\n",
            "Val: Epoch = 40 | Loss 1.520 | Accuracy = 55.172\n",
            "Train: Epoch = 41 | Loss = 34.554 | Loss2 = 0.598 | Accuracy = 67.812\n",
            "Val: Epoch = 41 | Loss 1.518 | Accuracy = 50.862\n",
            "Train: Epoch = 42 | Loss = 34.669 | Loss2 = 0.614 | Accuracy = 66.250\n",
            "Val: Epoch = 42 | Loss 1.624 | Accuracy = 54.310\n",
            "Train: Epoch = 43 | Loss = 34.614 | Loss2 = 0.623 | Accuracy = 68.750\n",
            "Val: Epoch = 43 | Loss 1.510 | Accuracy = 51.724\n",
            "Train: Epoch = 44 | Loss = 34.458 | Loss2 = 0.594 | Accuracy = 73.750\n",
            "Val: Epoch = 44 | Loss 1.599 | Accuracy = 50.000\n",
            "Train: Epoch = 45 | Loss = 34.752 | Loss2 = 0.627 | Accuracy = 65.312\n",
            "Val: Epoch = 45 | Loss 1.566 | Accuracy = 51.724\n",
            "Train: Epoch = 46 | Loss = 34.601 | Loss2 = 0.586 | Accuracy = 64.062\n",
            "Val: Epoch = 46 | Loss 1.594 | Accuracy = 51.724\n",
            "Train: Epoch = 47 | Loss = 34.638 | Loss2 = 0.622 | Accuracy = 66.875\n",
            "Val: Epoch = 47 | Loss 1.597 | Accuracy = 52.586\n",
            "Train: Epoch = 48 | Loss = 34.615 | Loss2 = 0.587 | Accuracy = 66.875\n",
            "Val: Epoch = 48 | Loss 1.508 | Accuracy = 52.586\n",
            "Train: Epoch = 49 | Loss = 34.564 | Loss2 = 0.553 | Accuracy = 65.625\n",
            "Val: Epoch = 49 | Loss 1.536 | Accuracy = 52.586\n",
            "Train: Epoch = 50 | Loss = 34.571 | Loss2 = 0.579 | Accuracy = 66.875\n",
            "Val: Epoch = 50 | Loss 1.527 | Accuracy = 52.586\n",
            "Train: Epoch = 51 | Loss = 34.549 | Loss2 = 0.596 | Accuracy = 69.062\n",
            "Val: Epoch = 51 | Loss 1.461 | Accuracy = 56.034\n",
            "Train: Epoch = 52 | Loss = 34.583 | Loss2 = 0.560 | Accuracy = 67.500\n",
            "Val: Epoch = 52 | Loss 1.612 | Accuracy = 50.000\n",
            "Train: Epoch = 53 | Loss = 34.580 | Loss2 = 0.586 | Accuracy = 67.188\n",
            "Val: Epoch = 53 | Loss 1.467 | Accuracy = 52.586\n",
            "Train: Epoch = 54 | Loss = 34.486 | Loss2 = 0.605 | Accuracy = 73.125\n",
            "Val: Epoch = 54 | Loss 1.508 | Accuracy = 56.897\n",
            "[||| NEW BEST on val |||]\n",
            "Train: Epoch = 55 | Loss = 34.589 | Loss2 = 0.595 | Accuracy = 66.250\n",
            "Val: Epoch = 55 | Loss 1.474 | Accuracy = 56.897\n",
            "Train: Epoch = 56 | Loss = 34.515 | Loss2 = 0.594 | Accuracy = 69.062\n",
            "Val: Epoch = 56 | Loss 1.497 | Accuracy = 56.897\n",
            "Train: Epoch = 57 | Loss = 34.722 | Loss2 = 0.572 | Accuracy = 61.562\n",
            "Val: Epoch = 57 | Loss 1.422 | Accuracy = 54.310\n",
            "Train: Epoch = 58 | Loss = 34.643 | Loss2 = 0.576 | Accuracy = 64.688\n",
            "Val: Epoch = 58 | Loss 1.494 | Accuracy = 51.724\n",
            "Train: Epoch = 59 | Loss = 34.531 | Loss2 = 0.592 | Accuracy = 70.000\n",
            "Val: Epoch = 59 | Loss 1.465 | Accuracy = 55.172\n",
            "Train: Epoch = 60 | Loss = 34.570 | Loss2 = 0.543 | Accuracy = 69.062\n",
            "Val: Epoch = 60 | Loss 1.461 | Accuracy = 57.759\n",
            "[||| NEW BEST on val |||]\n",
            "Train: Epoch = 61 | Loss = 34.518 | Loss2 = 0.575 | Accuracy = 72.812\n",
            "Val: Epoch = 61 | Loss 1.454 | Accuracy = 51.724\n",
            "Train: Epoch = 62 | Loss = 34.489 | Loss2 = 0.549 | Accuracy = 70.625\n",
            "Val: Epoch = 62 | Loss 1.472 | Accuracy = 56.897\n",
            "Train: Epoch = 63 | Loss = 34.443 | Loss2 = 0.589 | Accuracy = 70.625\n",
            "Val: Epoch = 63 | Loss 1.441 | Accuracy = 55.172\n",
            "Train: Epoch = 64 | Loss = 34.489 | Loss2 = 0.584 | Accuracy = 69.688\n",
            "Val: Epoch = 64 | Loss 1.544 | Accuracy = 54.310\n",
            "Train: Epoch = 65 | Loss = 34.634 | Loss2 = 0.597 | Accuracy = 65.625\n",
            "Val: Epoch = 65 | Loss 1.582 | Accuracy = 56.034\n",
            "Train: Epoch = 66 | Loss = 34.419 | Loss2 = 0.546 | Accuracy = 72.812\n",
            "Val: Epoch = 66 | Loss 1.506 | Accuracy = 57.759\n",
            "Train: Epoch = 67 | Loss = 34.431 | Loss2 = 0.571 | Accuracy = 73.438\n",
            "Val: Epoch = 67 | Loss 1.481 | Accuracy = 53.448\n",
            "Train: Epoch = 68 | Loss = 34.508 | Loss2 = 0.575 | Accuracy = 71.250\n",
            "Val: Epoch = 68 | Loss 1.528 | Accuracy = 55.172\n",
            "Train: Epoch = 69 | Loss = 34.354 | Loss2 = 0.585 | Accuracy = 73.750\n",
            "Val: Epoch = 69 | Loss 1.439 | Accuracy = 54.310\n",
            "Train: Epoch = 70 | Loss = 34.444 | Loss2 = 0.544 | Accuracy = 69.375\n",
            "Val: Epoch = 70 | Loss 1.426 | Accuracy = 56.897\n",
            "Train: Epoch = 71 | Loss = 34.403 | Loss2 = 0.586 | Accuracy = 73.750\n",
            "Val: Epoch = 71 | Loss 1.465 | Accuracy = 53.448\n",
            "Train: Epoch = 72 | Loss = 34.576 | Loss2 = 0.550 | Accuracy = 68.438\n",
            "Val: Epoch = 72 | Loss 1.514 | Accuracy = 52.586\n",
            "Train: Epoch = 73 | Loss = 34.450 | Loss2 = 0.583 | Accuracy = 72.812\n",
            "Val: Epoch = 73 | Loss 1.581 | Accuracy = 53.448\n",
            "Train: Epoch = 74 | Loss = 34.493 | Loss2 = 0.573 | Accuracy = 68.125\n",
            "Val: Epoch = 74 | Loss 1.473 | Accuracy = 55.172\n",
            "Train: Epoch = 75 | Loss = 34.392 | Loss2 = 0.574 | Accuracy = 76.250\n",
            "Val: Epoch = 75 | Loss 1.648 | Accuracy = 55.172\n",
            "Train: Epoch = 76 | Loss = 34.360 | Loss2 = 0.556 | Accuracy = 72.812\n",
            "Val: Epoch = 76 | Loss 1.596 | Accuracy = 53.448\n",
            "Train: Epoch = 77 | Loss = 34.381 | Loss2 = 0.548 | Accuracy = 74.375\n",
            "Val: Epoch = 77 | Loss 1.541 | Accuracy = 53.448\n",
            "Train: Epoch = 78 | Loss = 34.388 | Loss2 = 0.613 | Accuracy = 74.688\n",
            "Val: Epoch = 78 | Loss 1.589 | Accuracy = 55.172\n",
            "Train: Epoch = 79 | Loss = 34.495 | Loss2 = 0.545 | Accuracy = 72.500\n",
            "Val: Epoch = 79 | Loss 1.485 | Accuracy = 52.586\n",
            "Train: Epoch = 80 | Loss = 34.362 | Loss2 = 0.572 | Accuracy = 75.938\n",
            "Val: Epoch = 80 | Loss 1.570 | Accuracy = 55.172\n",
            "Train: Epoch = 81 | Loss = 34.335 | Loss2 = 0.565 | Accuracy = 77.188\n",
            "Val: Epoch = 81 | Loss 1.561 | Accuracy = 54.310\n",
            "Train: Epoch = 82 | Loss = 34.362 | Loss2 = 0.545 | Accuracy = 75.938\n",
            "Val: Epoch = 82 | Loss 1.561 | Accuracy = 54.310\n",
            "Train: Epoch = 83 | Loss = 34.464 | Loss2 = 0.568 | Accuracy = 71.562\n",
            "Val: Epoch = 83 | Loss 1.625 | Accuracy = 52.586\n",
            "Train: Epoch = 84 | Loss = 34.393 | Loss2 = 0.587 | Accuracy = 74.062\n",
            "Val: Epoch = 84 | Loss 1.672 | Accuracy = 51.724\n",
            "Train: Epoch = 85 | Loss = 34.324 | Loss2 = 0.572 | Accuracy = 78.750\n",
            "Val: Epoch = 85 | Loss 1.639 | Accuracy = 51.724\n",
            "Train: Epoch = 86 | Loss = 34.475 | Loss2 = 0.565 | Accuracy = 71.562\n",
            "Val: Epoch = 86 | Loss 1.607 | Accuracy = 53.448\n",
            "Train: Epoch = 87 | Loss = 34.448 | Loss2 = 0.548 | Accuracy = 75.312\n",
            "Val: Epoch = 87 | Loss 1.592 | Accuracy = 52.586\n",
            "Train: Epoch = 88 | Loss = 34.394 | Loss2 = 0.545 | Accuracy = 74.062\n",
            "Val: Epoch = 88 | Loss 1.590 | Accuracy = 52.586\n",
            "Train: Epoch = 89 | Loss = 34.409 | Loss2 = 0.570 | Accuracy = 73.750\n",
            "Val: Epoch = 89 | Loss 1.610 | Accuracy = 54.310\n",
            "Train: Epoch = 90 | Loss = 34.338 | Loss2 = 0.577 | Accuracy = 75.625\n",
            "Val: Epoch = 90 | Loss 1.599 | Accuracy = 51.724\n",
            "Train: Epoch = 91 | Loss = 34.376 | Loss2 = 0.537 | Accuracy = 73.438\n",
            "Val: Epoch = 91 | Loss 1.558 | Accuracy = 53.448\n",
            "Train: Epoch = 92 | Loss = 34.352 | Loss2 = 0.547 | Accuracy = 75.625\n",
            "Val: Epoch = 92 | Loss 1.532 | Accuracy = 53.448\n",
            "Train: Epoch = 93 | Loss = 34.400 | Loss2 = 0.584 | Accuracy = 71.562\n",
            "Val: Epoch = 93 | Loss 1.547 | Accuracy = 53.448\n",
            "Train: Epoch = 94 | Loss = 34.411 | Loss2 = 0.571 | Accuracy = 73.750\n",
            "Val: Epoch = 94 | Loss 1.572 | Accuracy = 52.586\n",
            "Train: Epoch = 95 | Loss = 34.387 | Loss2 = 0.539 | Accuracy = 78.125\n",
            "Val: Epoch = 95 | Loss 1.593 | Accuracy = 53.448\n",
            "Train: Epoch = 96 | Loss = 34.329 | Loss2 = 0.555 | Accuracy = 75.938\n",
            "Val: Epoch = 96 | Loss 1.587 | Accuracy = 54.310\n",
            "Train: Epoch = 97 | Loss = 34.392 | Loss2 = 0.546 | Accuracy = 72.500\n",
            "Val: Epoch = 97 | Loss 1.579 | Accuracy = 54.310\n",
            "Train: Epoch = 98 | Loss = 34.368 | Loss2 = 0.538 | Accuracy = 72.188\n",
            "Val: Epoch = 98 | Loss 1.568 | Accuracy = 52.586\n",
            "Train: Epoch = 99 | Loss = 34.276 | Loss2 = 0.580 | Accuracy = 77.812\n",
            "Val: Epoch = 99 | Loss 1.591 | Accuracy = 52.586\n",
            "Train: Epoch = 100 | Loss = 34.396 | Loss2 = 0.558 | Accuracy = 73.438\n",
            "Val: Epoch = 100 | Loss 1.605 | Accuracy = 50.862\n",
            "Train: Epoch = 101 | Loss = 34.407 | Loss2 = 0.562 | Accuracy = 72.812\n",
            "Val: Epoch = 101 | Loss 1.611 | Accuracy = 50.862\n",
            "Train: Epoch = 102 | Loss = 34.458 | Loss2 = 0.553 | Accuracy = 71.875\n",
            "Val: Epoch = 102 | Loss 1.590 | Accuracy = 54.310\n",
            "Train: Epoch = 103 | Loss = 34.317 | Loss2 = 0.568 | Accuracy = 73.438\n",
            "Val: Epoch = 103 | Loss 1.582 | Accuracy = 53.448\n",
            "Train: Epoch = 104 | Loss = 34.274 | Loss2 = 0.565 | Accuracy = 79.688\n",
            "Val: Epoch = 104 | Loss 1.583 | Accuracy = 52.586\n",
            "Train: Epoch = 105 | Loss = 34.346 | Loss2 = 0.554 | Accuracy = 75.938\n",
            "Val: Epoch = 105 | Loss 1.578 | Accuracy = 51.724\n",
            "Train: Epoch = 106 | Loss = 34.322 | Loss2 = 0.576 | Accuracy = 74.688\n",
            "Val: Epoch = 106 | Loss 1.563 | Accuracy = 52.586\n",
            "Train: Epoch = 107 | Loss = 34.446 | Loss2 = 0.535 | Accuracy = 72.500\n",
            "Val: Epoch = 107 | Loss 1.545 | Accuracy = 52.586\n",
            "Train: Epoch = 108 | Loss = 34.355 | Loss2 = 0.574 | Accuracy = 75.938\n",
            "Val: Epoch = 108 | Loss 1.572 | Accuracy = 53.448\n",
            "Train: Epoch = 109 | Loss = 34.376 | Loss2 = 0.586 | Accuracy = 75.938\n",
            "Val: Epoch = 109 | Loss 1.572 | Accuracy = 52.586\n",
            "Train: Epoch = 110 | Loss = 34.316 | Loss2 = 0.582 | Accuracy = 73.750\n",
            "Val: Epoch = 110 | Loss 1.579 | Accuracy = 52.586\n",
            "Train: Epoch = 111 | Loss = 34.300 | Loss2 = 0.537 | Accuracy = 77.188\n",
            "Val: Epoch = 111 | Loss 1.569 | Accuracy = 51.724\n",
            "Train: Epoch = 112 | Loss = 34.298 | Loss2 = 0.565 | Accuracy = 75.625\n",
            "Val: Epoch = 112 | Loss 1.588 | Accuracy = 51.724\n",
            "Train: Epoch = 113 | Loss = 34.335 | Loss2 = 0.539 | Accuracy = 74.062\n",
            "Val: Epoch = 113 | Loss 1.554 | Accuracy = 52.586\n",
            "Train: Epoch = 114 | Loss = 34.280 | Loss2 = 0.552 | Accuracy = 77.500\n",
            "Val: Epoch = 114 | Loss 1.561 | Accuracy = 52.586\n",
            "Train: Epoch = 115 | Loss = 34.366 | Loss2 = 0.534 | Accuracy = 75.625\n",
            "Val: Epoch = 115 | Loss 1.581 | Accuracy = 52.586\n",
            "Train: Epoch = 116 | Loss = 34.222 | Loss2 = 0.540 | Accuracy = 77.812\n",
            "Val: Epoch = 116 | Loss 1.609 | Accuracy = 51.724\n",
            "Train: Epoch = 117 | Loss = 34.318 | Loss2 = 0.575 | Accuracy = 77.812\n",
            "Val: Epoch = 117 | Loss 1.589 | Accuracy = 52.586\n",
            "Train: Epoch = 118 | Loss = 34.305 | Loss2 = 0.548 | Accuracy = 76.562\n",
            "Val: Epoch = 118 | Loss 1.588 | Accuracy = 51.724\n",
            "Train: Epoch = 119 | Loss = 34.176 | Loss2 = 0.548 | Accuracy = 80.312\n",
            "Val: Epoch = 119 | Loss 1.576 | Accuracy = 53.448\n",
            "Train: Epoch = 120 | Loss = 34.328 | Loss2 = 0.557 | Accuracy = 77.500\n",
            "Val: Epoch = 120 | Loss 1.573 | Accuracy = 52.586\n",
            "Train: Epoch = 121 | Loss = 34.318 | Loss2 = 0.556 | Accuracy = 76.875\n",
            "Val: Epoch = 121 | Loss 1.599 | Accuracy = 51.724\n",
            "Train: Epoch = 122 | Loss = 34.344 | Loss2 = 0.561 | Accuracy = 73.125\n",
            "Val: Epoch = 122 | Loss 1.564 | Accuracy = 52.586\n",
            "Train: Epoch = 123 | Loss = 34.272 | Loss2 = 0.537 | Accuracy = 78.125\n",
            "Val: Epoch = 123 | Loss 1.610 | Accuracy = 52.586\n",
            "Train: Epoch = 124 | Loss = 34.348 | Loss2 = 0.582 | Accuracy = 75.312\n",
            "Val: Epoch = 124 | Loss 1.623 | Accuracy = 52.586\n",
            "Train: Epoch = 125 | Loss = 34.275 | Loss2 = 0.552 | Accuracy = 76.875\n",
            "Val: Epoch = 125 | Loss 1.614 | Accuracy = 53.448\n",
            "Train: Epoch = 126 | Loss = 34.226 | Loss2 = 0.562 | Accuracy = 77.188\n",
            "Val: Epoch = 126 | Loss 1.634 | Accuracy = 53.448\n",
            "Train: Epoch = 127 | Loss = 34.328 | Loss2 = 0.577 | Accuracy = 72.188\n",
            "Val: Epoch = 127 | Loss 1.653 | Accuracy = 52.586\n",
            "Train: Epoch = 128 | Loss = 34.331 | Loss2 = 0.510 | Accuracy = 78.125\n",
            "Val: Epoch = 128 | Loss 1.633 | Accuracy = 53.448\n",
            "Train: Epoch = 129 | Loss = 34.334 | Loss2 = 0.553 | Accuracy = 75.000\n",
            "Val: Epoch = 129 | Loss 1.603 | Accuracy = 53.448\n",
            "Train: Epoch = 130 | Loss = 34.253 | Loss2 = 0.590 | Accuracy = 78.438\n",
            "Val: Epoch = 130 | Loss 1.595 | Accuracy = 52.586\n",
            "Train: Epoch = 131 | Loss = 34.397 | Loss2 = 0.557 | Accuracy = 71.875\n",
            "Val: Epoch = 131 | Loss 1.633 | Accuracy = 52.586\n",
            "Train: Epoch = 132 | Loss = 34.224 | Loss2 = 0.578 | Accuracy = 79.062\n",
            "Val: Epoch = 132 | Loss 1.621 | Accuracy = 53.448\n",
            "Train: Epoch = 133 | Loss = 34.330 | Loss2 = 0.578 | Accuracy = 77.500\n",
            "Val: Epoch = 133 | Loss 1.597 | Accuracy = 53.448\n",
            "Train: Epoch = 134 | Loss = 34.261 | Loss2 = 0.595 | Accuracy = 76.562\n",
            "Val: Epoch = 134 | Loss 1.591 | Accuracy = 53.448\n",
            "Train: Epoch = 135 | Loss = 34.354 | Loss2 = 0.571 | Accuracy = 77.188\n",
            "Val: Epoch = 135 | Loss 1.570 | Accuracy = 55.172\n",
            "Train: Epoch = 136 | Loss = 34.318 | Loss2 = 0.549 | Accuracy = 76.875\n",
            "Val: Epoch = 136 | Loss 1.568 | Accuracy = 55.172\n",
            "Train: Epoch = 137 | Loss = 34.382 | Loss2 = 0.560 | Accuracy = 74.375\n",
            "Val: Epoch = 137 | Loss 1.550 | Accuracy = 56.034\n",
            "Train: Epoch = 138 | Loss = 34.329 | Loss2 = 0.580 | Accuracy = 77.188\n",
            "Val: Epoch = 138 | Loss 1.567 | Accuracy = 55.172\n",
            "Train: Epoch = 139 | Loss = 34.248 | Loss2 = 0.555 | Accuracy = 80.938\n",
            "Val: Epoch = 139 | Loss 1.566 | Accuracy = 55.172\n",
            "Train: Epoch = 140 | Loss = 34.297 | Loss2 = 0.557 | Accuracy = 78.125\n",
            "Val: Epoch = 140 | Loss 1.581 | Accuracy = 52.586\n",
            "Train: Epoch = 141 | Loss = 34.264 | Loss2 = 0.556 | Accuracy = 78.125\n",
            "Val: Epoch = 141 | Loss 1.557 | Accuracy = 54.310\n",
            "Train: Epoch = 142 | Loss = 34.264 | Loss2 = 0.558 | Accuracy = 77.188\n",
            "Val: Epoch = 142 | Loss 1.570 | Accuracy = 52.586\n",
            "Train: Epoch = 143 | Loss = 34.292 | Loss2 = 0.551 | Accuracy = 78.438\n",
            "Val: Epoch = 143 | Loss 1.594 | Accuracy = 53.448\n",
            "Train: Epoch = 144 | Loss = 34.169 | Loss2 = 0.503 | Accuracy = 80.625\n",
            "Val: Epoch = 144 | Loss 1.566 | Accuracy = 53.448\n",
            "Train: Epoch = 145 | Loss = 34.218 | Loss2 = 0.575 | Accuracy = 76.562\n",
            "Val: Epoch = 145 | Loss 1.549 | Accuracy = 53.448\n",
            "Train: Epoch = 146 | Loss = 34.255 | Loss2 = 0.551 | Accuracy = 77.500\n",
            "Val: Epoch = 146 | Loss 1.555 | Accuracy = 53.448\n",
            "Train: Epoch = 147 | Loss = 34.293 | Loss2 = 0.578 | Accuracy = 76.250\n",
            "Val: Epoch = 147 | Loss 1.542 | Accuracy = 53.448\n",
            "Train: Epoch = 148 | Loss = 34.306 | Loss2 = 0.566 | Accuracy = 77.812\n",
            "Val: Epoch = 148 | Loss 1.518 | Accuracy = 52.586\n",
            "Train: Epoch = 149 | Loss = 34.269 | Loss2 = 0.509 | Accuracy = 75.312\n",
            "Val: Epoch = 149 | Loss 1.555 | Accuracy = 51.724\n",
            "Train: Epoch = 150 | Loss = 34.299 | Loss2 = 0.533 | Accuracy = 74.375\n",
            "Val: Epoch = 150 | Loss 1.552 | Accuracy = 53.448\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "V4Yoyixl15MV"
      },
      "source": [
        "# Stage 2 specific setup"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "ExecuteTime": {
          "end_time": "2020-06-08T15:51:16.753000Z",
          "start_time": "2020-06-08T15:51:16.742346Z"
        },
        "colab_type": "code",
        "id": "XXYaYYaO15Mb",
        "colab": {}
      },
      "source": [
        "STAGE = 2\n",
        "\n",
        "LR = 0.0001            # The initial Learning Rate\n",
        "MOMENTUM = 0.9       # Hyperparameter for SGD, keep this at 0.9 when using SGD\n",
        "WEIGHT_DECAY = 4e-5  # Regularization, you can keep this at the default\n",
        "\n",
        "NUM_EPOCHS = 150      # Total number of training epochs (iterations over dataset)\n",
        "STEP_SIZE = [30, 80] # How many epochs before decreasing learning rate (if using a step-down policy)\n",
        "GAMMA = 0.2          # Multiplicative factor for learning rate step-down\n",
        "MEM_SIZE = 512\n",
        "SEQ_LEN = 7\n",
        "ALPHA = 0.5\n",
        "\n",
        "parameters = {\n",
        "    'DEVICE': DEVICE,\n",
        "    'NUM_CLASSES': NUM_CLASSES,\n",
        "    'BATCH_SIZE': BATCH_SIZE,\n",
        "    'LR': LR,\n",
        "    'MOMENTUM': MOMENTUM,\n",
        "    'WEIGHT_DECAY': WEIGHT_DECAY,\n",
        "    'NUM_EPOCHS': NUM_EPOCHS,\n",
        "    'STEP_SIZE': STEP_SIZE,\n",
        "    'GAMMA': GAMMA,\n",
        "    'MEM_SIZE': MEM_SIZE,\n",
        "    'SEQ_LEN': SEQ_LEN\n",
        "}"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "ExecuteTime": {
          "end_time": "2020-06-08T15:51:18.906868Z",
          "start_time": "2020-06-08T15:51:18.872090Z"
        },
        "colab_type": "code",
        "id": "L2koFcz315Mg",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 51
        },
        "outputId": "a7471b87-682b-455e-9dee-5086ae86413c"
      },
      "source": [
        "# Prepare Pytorch train/test Datasets\n",
        "train_dataset = GTEA61(DATA_DIR, split='train', transform=spatial_transform,\n",
        "                       seq_len=SEQ_LEN, mmaps = True, mmaps_transform = spatial_transform_mmaps, static_frames = True)\n",
        "test_dataset = GTEA61(DATA_DIR, split='test', transform=spatial_transform_val,\n",
        "                      seq_len=SEQ_LEN)\n",
        "\n",
        "# Check dataset sizes\n",
        "print('Train Dataset: {}'.format(len(train_dataset)))\n",
        "print('Test Dataset: {}'.format(len(test_dataset)))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Train Dataset: 341\n",
            "Test Dataset: 116\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "ExecuteTime": {
          "end_time": "2020-06-08T15:51:30.177645Z",
          "start_time": "2020-06-08T15:51:30.170111Z"
        },
        "colab_type": "code",
        "id": "BapthqR315Mj",
        "colab": {}
      },
      "source": [
        "# Dataloaders iterate over pytorch datasets and transparently provide useful functions (e.g. parallelization and shuffling)\n",
        "train_loader = DataLoader(train_dataset, batch_size=BATCH_SIZE, shuffle=True, num_workers=4, drop_last=True)\n",
        "val_loader = DataLoader(test_dataset, batch_size=BATCH_SIZE, shuffle=False, num_workers=4)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "eOVleh9q15Mn"
      },
      "source": [
        "# Prepare Stage 2"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "ExecuteTime": {
          "end_time": "2020-06-08T15:51:45.195953Z",
          "start_time": "2020-06-08T15:51:41.943238Z"
        },
        "colab_type": "code",
        "id": "57Iufg3Z15Mn",
        "colab": {}
      },
      "source": [
        "best_old_stage = generate_model_checkpoint_name(stage=1, n_frames=SEQ_LEN, ms_block = True, \\\n",
        "                                                optional = '_2neur_DS__run01')\n",
        "stage1_dict = os.path.join(model_folder, best_old_stage)\n",
        "validate = True\n",
        "\n",
        "model = attention_model_ms(num_classes=NUM_CLASSES, mem_size=MEM_SIZE)\n",
        "model.load_state_dict(torch.load(stage1_dict))\n",
        "\n",
        "model.train(False)\n",
        "\n",
        "for params in model.parameters():\n",
        "    params.requires_grad = False\n",
        "\n",
        "layers_on_off = [\n",
        "    model.resNet.layer4[0].conv1,\n",
        "    model.resNet.layer4[0].conv2,\n",
        "    model.resNet.layer4[1].conv1,\n",
        "    model.resNet.layer4[1].conv2,\n",
        "    model.resNet.layer4[2].conv1,\n",
        "    model.resNet.layer4[2].conv2,\n",
        "    model.resNet.fc,\n",
        "    model.msBlock\n",
        "]\n",
        "\n",
        "layers_to_train = layers_on_off + [\n",
        "    model.lstm_cell,\n",
        "    model.classifier,\n",
        "    model.classifier_SD\n",
        "]\n",
        "\n",
        "for layer in layers_to_train:\n",
        "    for params in layer.parameters():\n",
        "        params.requires_grad = True\n",
        "\n",
        "for layer in layers_to_train:\n",
        "    layer.train(True)\n",
        "\n",
        "model.to(DEVICE)\n",
        "\n",
        "loss_fn = nn.CrossEntropyLoss()\n",
        "loss_fn_sum = nn.CrossEntropyLoss(reduction = 'sum')\n",
        "\n",
        "trainable_params = [p for p in model.parameters() if p.requires_grad]\n",
        "optimizer_fn = torch.optim.Adam(trainable_params, lr=LR, weight_decay=WEIGHT_DECAY, eps=1e-4)\n",
        "\n",
        "optim_scheduler = torch.optim.lr_scheduler.MultiStepLR(optimizer_fn, milestones=STEP_SIZE, gamma=GAMMA)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "gGtJGHAw15Mq"
      },
      "source": [
        "# Stage 2"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "ExecuteTime": {
          "end_time": "2020-06-08T08:58:05.318659Z",
          "start_time": "2020-06-08T08:57:45.955602Z"
        },
        "colab_type": "code",
        "id": "yn5MVYRC15Mr",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "outputId": "3d8e1ac5-80da-446d-8ae0-792e90cfd3cf"
      },
      "source": [
        "RUN = \"_run02\"\n",
        "\n",
        "train_iter = 0\n",
        "val_iter = 0\n",
        "min_accuracy = 0\n",
        "\n",
        "trainSamples = len(train_dataset) - (len(train_dataset) % BATCH_SIZE)\n",
        "val_samples = len(test_dataset)\n",
        "\n",
        "iterPerEpoch = len(train_loader)\n",
        "val_steps = len(val_loader)\n",
        "\n",
        "cudnn.benchmark\n",
        "torch.autograd.set_detect_anomaly(True)\n",
        "train_log, val_log = generate_log_filenames(STAGE, SEQ_LEN, ms_block = True, \\\n",
        "                                            optional = '_2neur_DS_alpha_' + RUN)\n",
        "model_checkpoint = generate_model_checkpoint_name(STAGE, SEQ_LEN, ms_block = True, \\\n",
        "                                                  optional = '_2neur_DS_alpha_' + RUN)\n",
        "\n",
        "train_log_file = os.path.join(model_folder, train_log)\n",
        "val_log_file = os.path.join(model_folder, val_log)\n",
        "train_logger_2 = Logger(**parameters)\n",
        "val_logger_2 = Logger(**parameters)\n",
        "\n",
        "for epoch in range(NUM_EPOCHS):\n",
        "\n",
        "    epoch_loss = 0\n",
        "    epoch_loss2 = 0\n",
        "    numCorrTrain = 0\n",
        "    \n",
        "    for layer in layers_to_train:\n",
        "        layer.train(True)\n",
        "    \n",
        "    for i, (inputs, inputs_DS, mmaps, targets) in enumerate(train_loader):\n",
        "\n",
        "        mmaps = mmaps.permute(1, 0, 2)\n",
        "\n",
        "        train_iter += 1\n",
        "\n",
        "        optimizer_fn.zero_grad()\n",
        "\n",
        "        inputVariable = inputs.permute(1, 0, 2, 3, 4).to(DEVICE)\n",
        "        inputVariable_DS = inputs_DS.permute(1, 0, 2, 3, 4).to(DEVICE)\n",
        "        labelVariable = targets.to(DEVICE)\n",
        "\n",
        "        output_label, _, output_label_mmaps = model(inputVariable, mmaps = True)\n",
        "\n",
        "        output_label_mmaps = output_label_mmaps.view(SEQ_LEN*BATCH_SIZE*MMAP_LENGTH, 2).to(DEVICE)\n",
        "        mmaps = torch.flatten(mmaps).long().to(DEVICE)\n",
        "\n",
        "        loss = loss_fn_sum(output_label_mmaps, mmaps)/(BATCH_SIZE * SEQ_LEN) + loss_fn(output_label, labelVariable)\n",
        "        loss.backward()\n",
        "\n",
        "        for layer in layers_on_off:\n",
        "            for params in layer.parameters():\n",
        "                params.requires_grad = False\n",
        "\n",
        "        output_label_ds1, _ = model(inputVariable, static_frames = True)\n",
        "        output_label_ds2, _ = model(inputVariable_DS, static_frames = True)\n",
        "        \n",
        "        gt_ds1 = torch.zeros(BATCH_SIZE).long().to(DEVICE)\n",
        "        gt_ds2 = torch.ones(BATCH_SIZE).long().to(DEVICE)\n",
        "\n",
        "        loss2 = ALPHA * (loss_fn(output_label_ds1, gt_ds1) + loss_fn(output_label_ds2, gt_ds2))\n",
        "\n",
        "        loss2.backward()\n",
        "\n",
        "        for layer in layers_on_off:\n",
        "            for params in layer.parameters():\n",
        "                params.requires_grad = True\n",
        "\n",
        "        optimizer_fn.step()\n",
        "        \n",
        "        _, predicted = torch.max(output_label.data, 1)\n",
        "        numCorrTrain += torch.sum(predicted == labelVariable.data).data.item()\n",
        "        step_loss = loss.data.item()\n",
        "        step_loss2 = loss2.data.item()\n",
        "        epoch_loss += step_loss\n",
        "        epoch_loss2 += step_loss2\n",
        "        train_logger_2.add_step_data(train_iter, numCorrTrain, step_loss)\n",
        "\n",
        "        \n",
        "    avg_loss = epoch_loss/iterPerEpoch\n",
        "    avg_loss2 = epoch_loss2/iterPerEpoch\n",
        "    trainAccuracy = (numCorrTrain / trainSamples) * 100\n",
        "    train_logger_2.add_epoch_data(epoch+1, trainAccuracy, avg_loss)\n",
        "    \n",
        "    print('Train: Epoch = {} | Loss = {:.3f} | Loss2 = {:.3f} | Accuracy = {:.3f}'.format(epoch+1, avg_loss, avg_loss2, trainAccuracy))\n",
        "\n",
        "    if validate is not None:\n",
        "\n",
        "        if (epoch+1) % 1 == 0:\n",
        "\n",
        "            model.train(False)\n",
        "            val_loss_epoch = 0\n",
        "            numCorr = 0\n",
        "\n",
        "            for j, (inputs, targets) in enumerate(val_loader):\n",
        "\n",
        "                val_iter += 1\n",
        "\n",
        "                inputVariable = inputs.permute(1, 0, 2, 3, 4).to(DEVICE)\n",
        "                labelVariable = targets.to(DEVICE)\n",
        "                \n",
        "                output_label, _ = model(inputVariable)\n",
        "\n",
        "                val_loss = loss_fn(output_label, labelVariable)\n",
        "                val_loss_step = val_loss.data.item()\n",
        "                val_loss_epoch += val_loss_step\n",
        "\n",
        "                _, predicted = torch.max(output_label.data, 1)\n",
        "\n",
        "                numCorr += torch.sum(predicted == labelVariable.data).data.item()\n",
        "                val_logger_2.add_step_data(val_iter, numCorr, val_loss_step)\n",
        "\n",
        "            val_accuracy = (numCorr / val_samples) * 100\n",
        "            avg_val_loss = val_loss_epoch / val_steps\n",
        "\n",
        "            print('Val: Epoch = {} | Loss {:.3f} | Accuracy = {:.3f}'.format(epoch + 1, avg_val_loss, val_accuracy))\n",
        "            \n",
        "            val_logger_2.add_epoch_data(epoch+1, val_accuracy, avg_val_loss)\n",
        "            \n",
        "            if val_accuracy > min_accuracy:\n",
        "                print(\"[||| NEW BEST on val |||]\")\n",
        "                save_path_model = os.path.join(model_folder, model_checkpoint)\n",
        "                torch.save(model.state_dict(), save_path_model)\n",
        "                min_accuracy = val_accuracy\n",
        "            \n",
        "    optim_scheduler.step()\n",
        "\n",
        "    train_logger_2.save(train_log_file)\n",
        "    val_logger_2.save(val_log_file)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Train: Epoch = 1 | Loss = 35.907 | Loss2 = 0.856 | Accuracy = 35.938\n",
            "Val: Epoch = 1 | Loss 2.798 | Accuracy = 31.897\n",
            "[||| NEW BEST on val |||]\n",
            "Train: Epoch = 2 | Loss = 35.651 | Loss2 = 0.790 | Accuracy = 42.500\n",
            "Val: Epoch = 2 | Loss 2.363 | Accuracy = 33.621\n",
            "[||| NEW BEST on val |||]\n",
            "Train: Epoch = 3 | Loss = 35.557 | Loss2 = 0.763 | Accuracy = 42.500\n",
            "Val: Epoch = 3 | Loss 2.259 | Accuracy = 39.655\n",
            "[||| NEW BEST on val |||]\n",
            "Train: Epoch = 4 | Loss = 35.441 | Loss2 = 0.752 | Accuracy = 45.312\n",
            "Val: Epoch = 4 | Loss 2.543 | Accuracy = 36.207\n",
            "Train: Epoch = 5 | Loss = 35.535 | Loss2 = 0.747 | Accuracy = 40.312\n",
            "Val: Epoch = 5 | Loss 2.564 | Accuracy = 37.931\n",
            "Train: Epoch = 6 | Loss = 35.384 | Loss2 = 0.710 | Accuracy = 45.312\n",
            "Val: Epoch = 6 | Loss 2.243 | Accuracy = 37.931\n",
            "Train: Epoch = 7 | Loss = 35.302 | Loss2 = 0.731 | Accuracy = 44.062\n",
            "Val: Epoch = 7 | Loss 2.385 | Accuracy = 37.069\n",
            "Train: Epoch = 8 | Loss = 35.405 | Loss2 = 0.731 | Accuracy = 43.125\n",
            "Val: Epoch = 8 | Loss 2.322 | Accuracy = 40.517\n",
            "[||| NEW BEST on val |||]\n",
            "Train: Epoch = 9 | Loss = 35.176 | Loss2 = 0.702 | Accuracy = 52.812\n",
            "Val: Epoch = 9 | Loss 2.237 | Accuracy = 40.517\n",
            "Train: Epoch = 10 | Loss = 35.304 | Loss2 = 0.717 | Accuracy = 49.688\n",
            "Val: Epoch = 10 | Loss 2.202 | Accuracy = 38.793\n",
            "Train: Epoch = 11 | Loss = 35.243 | Loss2 = 0.694 | Accuracy = 53.125\n",
            "Val: Epoch = 11 | Loss 1.902 | Accuracy = 42.241\n",
            "[||| NEW BEST on val |||]\n",
            "Train: Epoch = 12 | Loss = 35.215 | Loss2 = 0.666 | Accuracy = 50.313\n",
            "Val: Epoch = 12 | Loss 1.938 | Accuracy = 47.414\n",
            "[||| NEW BEST on val |||]\n",
            "Train: Epoch = 13 | Loss = 35.188 | Loss2 = 0.688 | Accuracy = 51.875\n",
            "Val: Epoch = 13 | Loss 2.116 | Accuracy = 43.103\n",
            "Train: Epoch = 14 | Loss = 35.219 | Loss2 = 0.692 | Accuracy = 51.562\n",
            "Val: Epoch = 14 | Loss 1.949 | Accuracy = 51.724\n",
            "[||| NEW BEST on val |||]\n",
            "Train: Epoch = 15 | Loss = 35.155 | Loss2 = 0.710 | Accuracy = 51.250\n",
            "Val: Epoch = 15 | Loss 1.869 | Accuracy = 48.276\n",
            "Train: Epoch = 16 | Loss = 35.104 | Loss2 = 0.648 | Accuracy = 54.375\n",
            "Val: Epoch = 16 | Loss 1.910 | Accuracy = 44.828\n",
            "Train: Epoch = 17 | Loss = 35.010 | Loss2 = 0.676 | Accuracy = 56.250\n",
            "Val: Epoch = 17 | Loss 1.782 | Accuracy = 51.724\n",
            "Train: Epoch = 18 | Loss = 34.953 | Loss2 = 0.658 | Accuracy = 57.812\n",
            "Val: Epoch = 18 | Loss 2.066 | Accuracy = 41.379\n",
            "Train: Epoch = 19 | Loss = 35.134 | Loss2 = 0.660 | Accuracy = 53.438\n",
            "Val: Epoch = 19 | Loss 1.843 | Accuracy = 45.690\n",
            "Train: Epoch = 20 | Loss = 35.030 | Loss2 = 0.632 | Accuracy = 55.937\n",
            "Val: Epoch = 20 | Loss 2.014 | Accuracy = 44.828\n",
            "Train: Epoch = 21 | Loss = 35.027 | Loss2 = 0.647 | Accuracy = 56.250\n",
            "Val: Epoch = 21 | Loss 1.887 | Accuracy = 46.552\n",
            "Train: Epoch = 22 | Loss = 34.947 | Loss2 = 0.639 | Accuracy = 56.563\n",
            "Val: Epoch = 22 | Loss 1.852 | Accuracy = 43.966\n",
            "Train: Epoch = 23 | Loss = 35.045 | Loss2 = 0.622 | Accuracy = 57.188\n",
            "Val: Epoch = 23 | Loss 2.026 | Accuracy = 43.103\n",
            "Train: Epoch = 24 | Loss = 34.947 | Loss2 = 0.640 | Accuracy = 57.188\n",
            "Val: Epoch = 24 | Loss 1.937 | Accuracy = 44.828\n",
            "Train: Epoch = 25 | Loss = 35.025 | Loss2 = 0.592 | Accuracy = 55.000\n",
            "Val: Epoch = 25 | Loss 1.817 | Accuracy = 47.414\n",
            "Train: Epoch = 26 | Loss = 34.953 | Loss2 = 0.609 | Accuracy = 56.875\n",
            "Val: Epoch = 26 | Loss 1.827 | Accuracy = 45.690\n",
            "Train: Epoch = 27 | Loss = 34.927 | Loss2 = 0.614 | Accuracy = 55.312\n",
            "Val: Epoch = 27 | Loss 1.647 | Accuracy = 46.552\n",
            "Train: Epoch = 28 | Loss = 34.969 | Loss2 = 0.605 | Accuracy = 58.750\n",
            "Val: Epoch = 28 | Loss 1.804 | Accuracy = 44.828\n",
            "Train: Epoch = 29 | Loss = 34.874 | Loss2 = 0.617 | Accuracy = 56.250\n",
            "Val: Epoch = 29 | Loss 1.883 | Accuracy = 46.552\n",
            "Train: Epoch = 30 | Loss = 34.882 | Loss2 = 0.616 | Accuracy = 58.750\n",
            "Val: Epoch = 30 | Loss 1.979 | Accuracy = 43.966\n",
            "Train: Epoch = 31 | Loss = 34.865 | Loss2 = 0.612 | Accuracy = 58.750\n",
            "Val: Epoch = 31 | Loss 1.788 | Accuracy = 45.690\n",
            "Train: Epoch = 32 | Loss = 34.633 | Loss2 = 0.617 | Accuracy = 67.188\n",
            "Val: Epoch = 32 | Loss 1.672 | Accuracy = 46.552\n",
            "Train: Epoch = 33 | Loss = 34.789 | Loss2 = 0.618 | Accuracy = 61.875\n",
            "Val: Epoch = 33 | Loss 1.669 | Accuracy = 46.552\n",
            "Train: Epoch = 34 | Loss = 34.632 | Loss2 = 0.619 | Accuracy = 65.938\n",
            "Val: Epoch = 34 | Loss 1.681 | Accuracy = 43.103\n",
            "Train: Epoch = 35 | Loss = 34.655 | Loss2 = 0.587 | Accuracy = 63.438\n",
            "Val: Epoch = 35 | Loss 1.618 | Accuracy = 47.414\n",
            "Train: Epoch = 36 | Loss = 34.567 | Loss2 = 0.577 | Accuracy = 68.438\n",
            "Val: Epoch = 36 | Loss 1.643 | Accuracy = 47.414\n",
            "Train: Epoch = 37 | Loss = 34.693 | Loss2 = 0.617 | Accuracy = 66.875\n",
            "Val: Epoch = 37 | Loss 1.625 | Accuracy = 47.414\n",
            "Train: Epoch = 38 | Loss = 34.669 | Loss2 = 0.608 | Accuracy = 62.187\n",
            "Val: Epoch = 38 | Loss 1.670 | Accuracy = 50.000\n",
            "Train: Epoch = 39 | Loss = 34.613 | Loss2 = 0.560 | Accuracy = 68.125\n",
            "Val: Epoch = 39 | Loss 1.698 | Accuracy = 49.138\n",
            "Train: Epoch = 40 | Loss = 34.646 | Loss2 = 0.628 | Accuracy = 64.062\n",
            "Val: Epoch = 40 | Loss 1.673 | Accuracy = 44.828\n",
            "Train: Epoch = 41 | Loss = 34.693 | Loss2 = 0.591 | Accuracy = 65.000\n",
            "Val: Epoch = 41 | Loss 1.680 | Accuracy = 50.000\n",
            "Train: Epoch = 42 | Loss = 34.605 | Loss2 = 0.603 | Accuracy = 69.688\n",
            "Val: Epoch = 42 | Loss 1.685 | Accuracy = 51.724\n",
            "Train: Epoch = 43 | Loss = 34.641 | Loss2 = 0.595 | Accuracy = 68.125\n",
            "Val: Epoch = 43 | Loss 1.713 | Accuracy = 49.138\n",
            "Train: Epoch = 44 | Loss = 34.721 | Loss2 = 0.604 | Accuracy = 61.250\n",
            "Val: Epoch = 44 | Loss 1.760 | Accuracy = 46.552\n",
            "Train: Epoch = 45 | Loss = 34.661 | Loss2 = 0.592 | Accuracy = 64.062\n",
            "Val: Epoch = 45 | Loss 1.764 | Accuracy = 52.586\n",
            "[||| NEW BEST on val |||]\n",
            "Train: Epoch = 46 | Loss = 34.543 | Loss2 = 0.596 | Accuracy = 72.812\n",
            "Val: Epoch = 46 | Loss 1.702 | Accuracy = 51.724\n",
            "Train: Epoch = 47 | Loss = 34.563 | Loss2 = 0.605 | Accuracy = 68.438\n",
            "Val: Epoch = 47 | Loss 1.660 | Accuracy = 50.000\n",
            "Train: Epoch = 48 | Loss = 34.692 | Loss2 = 0.588 | Accuracy = 64.688\n",
            "Val: Epoch = 48 | Loss 1.618 | Accuracy = 52.586\n",
            "Train: Epoch = 49 | Loss = 34.537 | Loss2 = 0.608 | Accuracy = 69.062\n",
            "Val: Epoch = 49 | Loss 1.645 | Accuracy = 52.586\n",
            "Train: Epoch = 50 | Loss = 34.536 | Loss2 = 0.590 | Accuracy = 68.438\n",
            "Val: Epoch = 50 | Loss 1.671 | Accuracy = 53.448\n",
            "[||| NEW BEST on val |||]\n",
            "Train: Epoch = 51 | Loss = 34.525 | Loss2 = 0.598 | Accuracy = 69.688\n",
            "Val: Epoch = 51 | Loss 1.704 | Accuracy = 50.862\n",
            "Train: Epoch = 52 | Loss = 34.523 | Loss2 = 0.610 | Accuracy = 68.438\n",
            "Val: Epoch = 52 | Loss 1.688 | Accuracy = 52.586\n",
            "Train: Epoch = 53 | Loss = 34.589 | Loss2 = 0.572 | Accuracy = 65.625\n",
            "Val: Epoch = 53 | Loss 1.647 | Accuracy = 50.862\n",
            "Train: Epoch = 54 | Loss = 34.600 | Loss2 = 0.582 | Accuracy = 69.062\n",
            "Val: Epoch = 54 | Loss 1.705 | Accuracy = 50.862\n",
            "Train: Epoch = 55 | Loss = 34.501 | Loss2 = 0.577 | Accuracy = 71.875\n",
            "Val: Epoch = 55 | Loss 1.654 | Accuracy = 46.552\n",
            "Train: Epoch = 56 | Loss = 34.552 | Loss2 = 0.582 | Accuracy = 66.875\n",
            "Val: Epoch = 56 | Loss 1.706 | Accuracy = 50.862\n",
            "Train: Epoch = 57 | Loss = 34.590 | Loss2 = 0.614 | Accuracy = 69.062\n",
            "Val: Epoch = 57 | Loss 1.739 | Accuracy = 49.138\n",
            "Train: Epoch = 58 | Loss = 34.470 | Loss2 = 0.575 | Accuracy = 73.438\n",
            "Val: Epoch = 58 | Loss 1.728 | Accuracy = 46.552\n",
            "Train: Epoch = 59 | Loss = 34.517 | Loss2 = 0.590 | Accuracy = 68.438\n",
            "Val: Epoch = 59 | Loss 1.725 | Accuracy = 50.000\n",
            "Train: Epoch = 60 | Loss = 34.707 | Loss2 = 0.556 | Accuracy = 63.125\n",
            "Val: Epoch = 60 | Loss 1.829 | Accuracy = 45.690\n",
            "Train: Epoch = 61 | Loss = 34.489 | Loss2 = 0.587 | Accuracy = 69.375\n",
            "Val: Epoch = 61 | Loss 1.697 | Accuracy = 49.138\n",
            "Train: Epoch = 62 | Loss = 34.486 | Loss2 = 0.601 | Accuracy = 68.125\n",
            "Val: Epoch = 62 | Loss 1.764 | Accuracy = 48.276\n",
            "Train: Epoch = 63 | Loss = 34.515 | Loss2 = 0.590 | Accuracy = 68.125\n",
            "Val: Epoch = 63 | Loss 1.697 | Accuracy = 50.000\n",
            "Train: Epoch = 64 | Loss = 34.363 | Loss2 = 0.586 | Accuracy = 75.938\n",
            "Val: Epoch = 64 | Loss 1.836 | Accuracy = 49.138\n",
            "Train: Epoch = 65 | Loss = 34.463 | Loss2 = 0.561 | Accuracy = 73.125\n",
            "Val: Epoch = 65 | Loss 1.738 | Accuracy = 47.414\n",
            "Train: Epoch = 66 | Loss = 34.507 | Loss2 = 0.590 | Accuracy = 70.312\n",
            "Val: Epoch = 66 | Loss 1.718 | Accuracy = 49.138\n",
            "Train: Epoch = 67 | Loss = 34.485 | Loss2 = 0.599 | Accuracy = 70.312\n",
            "Val: Epoch = 67 | Loss 1.751 | Accuracy = 50.862\n",
            "Train: Epoch = 68 | Loss = 34.601 | Loss2 = 0.559 | Accuracy = 69.375\n",
            "Val: Epoch = 68 | Loss 1.781 | Accuracy = 49.138\n",
            "Train: Epoch = 69 | Loss = 34.453 | Loss2 = 0.583 | Accuracy = 70.000\n",
            "Val: Epoch = 69 | Loss 1.754 | Accuracy = 50.000\n",
            "Train: Epoch = 70 | Loss = 34.532 | Loss2 = 0.567 | Accuracy = 70.625\n",
            "Val: Epoch = 70 | Loss 1.767 | Accuracy = 48.276\n",
            "Train: Epoch = 71 | Loss = 34.451 | Loss2 = 0.585 | Accuracy = 71.250\n",
            "Val: Epoch = 71 | Loss 1.734 | Accuracy = 48.276\n",
            "Train: Epoch = 72 | Loss = 34.462 | Loss2 = 0.597 | Accuracy = 71.250\n",
            "Val: Epoch = 72 | Loss 1.764 | Accuracy = 45.690\n",
            "Train: Epoch = 73 | Loss = 34.410 | Loss2 = 0.572 | Accuracy = 71.250\n",
            "Val: Epoch = 73 | Loss 1.793 | Accuracy = 49.138\n",
            "Train: Epoch = 74 | Loss = 34.429 | Loss2 = 0.550 | Accuracy = 75.000\n",
            "Val: Epoch = 74 | Loss 1.875 | Accuracy = 47.414\n",
            "Train: Epoch = 75 | Loss = 34.408 | Loss2 = 0.580 | Accuracy = 73.750\n",
            "Val: Epoch = 75 | Loss 1.834 | Accuracy = 45.690\n",
            "Train: Epoch = 76 | Loss = 34.429 | Loss2 = 0.568 | Accuracy = 72.812\n",
            "Val: Epoch = 76 | Loss 1.728 | Accuracy = 50.862\n",
            "Train: Epoch = 77 | Loss = 34.454 | Loss2 = 0.579 | Accuracy = 71.562\n",
            "Val: Epoch = 77 | Loss 1.776 | Accuracy = 47.414\n",
            "Train: Epoch = 78 | Loss = 34.365 | Loss2 = 0.560 | Accuracy = 75.938\n",
            "Val: Epoch = 78 | Loss 1.901 | Accuracy = 45.690\n",
            "Train: Epoch = 79 | Loss = 34.398 | Loss2 = 0.542 | Accuracy = 73.125\n",
            "Val: Epoch = 79 | Loss 1.757 | Accuracy = 46.552\n",
            "Train: Epoch = 80 | Loss = 34.525 | Loss2 = 0.547 | Accuracy = 68.125\n",
            "Val: Epoch = 80 | Loss 1.704 | Accuracy = 49.138\n",
            "Train: Epoch = 81 | Loss = 34.394 | Loss2 = 0.556 | Accuracy = 75.000\n",
            "Val: Epoch = 81 | Loss 1.708 | Accuracy = 47.414\n",
            "Train: Epoch = 82 | Loss = 34.470 | Loss2 = 0.573 | Accuracy = 72.188\n",
            "Val: Epoch = 82 | Loss 1.724 | Accuracy = 47.414\n",
            "Train: Epoch = 83 | Loss = 34.413 | Loss2 = 0.582 | Accuracy = 73.438\n",
            "Val: Epoch = 83 | Loss 1.727 | Accuracy = 48.276\n",
            "Train: Epoch = 84 | Loss = 34.341 | Loss2 = 0.561 | Accuracy = 74.688\n",
            "Val: Epoch = 84 | Loss 1.721 | Accuracy = 47.414\n",
            "Train: Epoch = 85 | Loss = 34.281 | Loss2 = 0.550 | Accuracy = 75.000\n",
            "Val: Epoch = 85 | Loss 1.724 | Accuracy = 47.414\n",
            "Train: Epoch = 86 | Loss = 34.370 | Loss2 = 0.591 | Accuracy = 73.750\n",
            "Val: Epoch = 86 | Loss 1.703 | Accuracy = 47.414\n",
            "Train: Epoch = 87 | Loss = 34.435 | Loss2 = 0.555 | Accuracy = 70.625\n",
            "Val: Epoch = 87 | Loss 1.720 | Accuracy = 47.414\n",
            "Train: Epoch = 88 | Loss = 34.310 | Loss2 = 0.563 | Accuracy = 77.188\n",
            "Val: Epoch = 88 | Loss 1.692 | Accuracy = 48.276\n",
            "Train: Epoch = 89 | Loss = 34.409 | Loss2 = 0.544 | Accuracy = 70.625\n",
            "Val: Epoch = 89 | Loss 1.697 | Accuracy = 49.138\n",
            "Train: Epoch = 90 | Loss = 34.346 | Loss2 = 0.529 | Accuracy = 75.625\n",
            "Val: Epoch = 90 | Loss 1.690 | Accuracy = 49.138\n",
            "Train: Epoch = 91 | Loss = 34.423 | Loss2 = 0.550 | Accuracy = 72.500\n",
            "Val: Epoch = 91 | Loss 1.685 | Accuracy = 49.138\n",
            "Train: Epoch = 92 | Loss = 34.391 | Loss2 = 0.525 | Accuracy = 71.875\n",
            "Val: Epoch = 92 | Loss 1.732 | Accuracy = 47.414\n",
            "Train: Epoch = 93 | Loss = 34.293 | Loss2 = 0.564 | Accuracy = 75.000\n",
            "Val: Epoch = 93 | Loss 1.737 | Accuracy = 47.414\n",
            "Train: Epoch = 94 | Loss = 34.401 | Loss2 = 0.576 | Accuracy = 72.500\n",
            "Val: Epoch = 94 | Loss 1.730 | Accuracy = 47.414\n",
            "Train: Epoch = 95 | Loss = 34.392 | Loss2 = 0.566 | Accuracy = 74.375\n",
            "Val: Epoch = 95 | Loss 1.701 | Accuracy = 48.276\n",
            "Train: Epoch = 96 | Loss = 34.336 | Loss2 = 0.540 | Accuracy = 73.750\n",
            "Val: Epoch = 96 | Loss 1.701 | Accuracy = 50.000\n",
            "Train: Epoch = 97 | Loss = 34.339 | Loss2 = 0.549 | Accuracy = 74.375\n",
            "Val: Epoch = 97 | Loss 1.632 | Accuracy = 50.862\n",
            "Train: Epoch = 98 | Loss = 34.466 | Loss2 = 0.546 | Accuracy = 71.250\n",
            "Val: Epoch = 98 | Loss 1.734 | Accuracy = 49.138\n",
            "Train: Epoch = 99 | Loss = 34.327 | Loss2 = 0.577 | Accuracy = 75.312\n",
            "Val: Epoch = 99 | Loss 1.776 | Accuracy = 48.276\n",
            "Train: Epoch = 100 | Loss = 34.467 | Loss2 = 0.549 | Accuracy = 71.875\n",
            "Val: Epoch = 100 | Loss 1.797 | Accuracy = 46.552\n",
            "Train: Epoch = 101 | Loss = 34.443 | Loss2 = 0.548 | Accuracy = 70.000\n",
            "Val: Epoch = 101 | Loss 1.684 | Accuracy = 50.000\n",
            "Train: Epoch = 102 | Loss = 34.373 | Loss2 = 0.551 | Accuracy = 78.438\n",
            "Val: Epoch = 102 | Loss 1.665 | Accuracy = 50.000\n",
            "Train: Epoch = 103 | Loss = 34.341 | Loss2 = 0.537 | Accuracy = 75.625\n",
            "Val: Epoch = 103 | Loss 1.655 | Accuracy = 50.000\n",
            "Train: Epoch = 104 | Loss = 34.372 | Loss2 = 0.595 | Accuracy = 74.062\n",
            "Val: Epoch = 104 | Loss 1.639 | Accuracy = 50.862\n",
            "Train: Epoch = 105 | Loss = 34.383 | Loss2 = 0.552 | Accuracy = 73.750\n",
            "Val: Epoch = 105 | Loss 1.649 | Accuracy = 50.862\n",
            "Train: Epoch = 106 | Loss = 34.343 | Loss2 = 0.515 | Accuracy = 73.750\n",
            "Val: Epoch = 106 | Loss 1.648 | Accuracy = 50.862\n",
            "Train: Epoch = 107 | Loss = 34.308 | Loss2 = 0.550 | Accuracy = 75.625\n",
            "Val: Epoch = 107 | Loss 1.694 | Accuracy = 49.138\n",
            "Train: Epoch = 108 | Loss = 34.381 | Loss2 = 0.558 | Accuracy = 73.438\n",
            "Val: Epoch = 108 | Loss 1.679 | Accuracy = 49.138\n",
            "Train: Epoch = 109 | Loss = 34.376 | Loss2 = 0.549 | Accuracy = 71.875\n",
            "Val: Epoch = 109 | Loss 1.661 | Accuracy = 50.862\n",
            "Train: Epoch = 110 | Loss = 34.329 | Loss2 = 0.571 | Accuracy = 74.688\n",
            "Val: Epoch = 110 | Loss 1.638 | Accuracy = 50.862\n",
            "Train: Epoch = 111 | Loss = 34.282 | Loss2 = 0.523 | Accuracy = 75.938\n",
            "Val: Epoch = 111 | Loss 1.643 | Accuracy = 50.862\n",
            "Train: Epoch = 112 | Loss = 34.462 | Loss2 = 0.549 | Accuracy = 71.562\n",
            "Val: Epoch = 112 | Loss 1.644 | Accuracy = 50.000\n",
            "Train: Epoch = 113 | Loss = 34.274 | Loss2 = 0.537 | Accuracy = 79.062\n",
            "Val: Epoch = 113 | Loss 1.652 | Accuracy = 49.138\n",
            "Train: Epoch = 114 | Loss = 34.298 | Loss2 = 0.560 | Accuracy = 76.875\n",
            "Val: Epoch = 114 | Loss 1.665 | Accuracy = 49.138\n",
            "Train: Epoch = 115 | Loss = 34.365 | Loss2 = 0.539 | Accuracy = 72.188\n",
            "Val: Epoch = 115 | Loss 1.693 | Accuracy = 48.276\n",
            "Train: Epoch = 116 | Loss = 34.335 | Loss2 = 0.573 | Accuracy = 75.312\n",
            "Val: Epoch = 116 | Loss 1.689 | Accuracy = 50.862\n",
            "Train: Epoch = 117 | Loss = 34.364 | Loss2 = 0.574 | Accuracy = 74.375\n",
            "Val: Epoch = 117 | Loss 1.657 | Accuracy = 50.862\n",
            "Train: Epoch = 118 | Loss = 34.261 | Loss2 = 0.543 | Accuracy = 76.875\n",
            "Val: Epoch = 118 | Loss 1.658 | Accuracy = 49.138\n",
            "Train: Epoch = 119 | Loss = 34.349 | Loss2 = 0.539 | Accuracy = 74.688\n",
            "Val: Epoch = 119 | Loss 1.654 | Accuracy = 50.000\n",
            "Train: Epoch = 120 | Loss = 34.379 | Loss2 = 0.557 | Accuracy = 75.312\n",
            "Val: Epoch = 120 | Loss 1.635 | Accuracy = 50.862\n",
            "Train: Epoch = 121 | Loss = 34.205 | Loss2 = 0.528 | Accuracy = 79.688\n",
            "Val: Epoch = 121 | Loss 1.661 | Accuracy = 49.138\n",
            "Train: Epoch = 122 | Loss = 34.288 | Loss2 = 0.538 | Accuracy = 78.125\n",
            "Val: Epoch = 122 | Loss 1.632 | Accuracy = 50.000\n",
            "Train: Epoch = 123 | Loss = 34.410 | Loss2 = 0.557 | Accuracy = 74.062\n",
            "Val: Epoch = 123 | Loss 1.611 | Accuracy = 51.724\n",
            "Train: Epoch = 124 | Loss = 34.351 | Loss2 = 0.543 | Accuracy = 73.750\n",
            "Val: Epoch = 124 | Loss 1.641 | Accuracy = 50.862\n",
            "Train: Epoch = 125 | Loss = 34.353 | Loss2 = 0.572 | Accuracy = 75.312\n",
            "Val: Epoch = 125 | Loss 1.655 | Accuracy = 50.000\n",
            "Train: Epoch = 126 | Loss = 34.311 | Loss2 = 0.546 | Accuracy = 77.188\n",
            "Val: Epoch = 126 | Loss 1.656 | Accuracy = 47.414\n",
            "Train: Epoch = 127 | Loss = 34.235 | Loss2 = 0.564 | Accuracy = 78.438\n",
            "Val: Epoch = 127 | Loss 1.658 | Accuracy = 49.138\n",
            "Train: Epoch = 128 | Loss = 34.297 | Loss2 = 0.567 | Accuracy = 77.188\n",
            "Val: Epoch = 128 | Loss 1.660 | Accuracy = 50.000\n",
            "Train: Epoch = 129 | Loss = 34.286 | Loss2 = 0.584 | Accuracy = 76.875\n",
            "Val: Epoch = 129 | Loss 1.656 | Accuracy = 50.000\n",
            "Train: Epoch = 130 | Loss = 34.361 | Loss2 = 0.563 | Accuracy = 76.250\n",
            "Val: Epoch = 130 | Loss 1.652 | Accuracy = 50.000\n",
            "Train: Epoch = 131 | Loss = 34.308 | Loss2 = 0.572 | Accuracy = 75.000\n",
            "Val: Epoch = 131 | Loss 1.648 | Accuracy = 49.138\n",
            "Train: Epoch = 132 | Loss = 34.281 | Loss2 = 0.551 | Accuracy = 75.312\n",
            "Val: Epoch = 132 | Loss 1.643 | Accuracy = 49.138\n",
            "Train: Epoch = 133 | Loss = 34.322 | Loss2 = 0.541 | Accuracy = 74.375\n",
            "Val: Epoch = 133 | Loss 1.643 | Accuracy = 50.000\n",
            "Train: Epoch = 134 | Loss = 34.221 | Loss2 = 0.538 | Accuracy = 76.250\n",
            "Val: Epoch = 134 | Loss 1.637 | Accuracy = 49.138\n",
            "Train: Epoch = 135 | Loss = 34.234 | Loss2 = 0.537 | Accuracy = 78.125\n",
            "Val: Epoch = 135 | Loss 1.605 | Accuracy = 52.586\n",
            "Train: Epoch = 136 | Loss = 34.323 | Loss2 = 0.538 | Accuracy = 73.750\n",
            "Val: Epoch = 136 | Loss 1.656 | Accuracy = 50.862\n",
            "Train: Epoch = 137 | Loss = 34.239 | Loss2 = 0.590 | Accuracy = 75.312\n",
            "Val: Epoch = 137 | Loss 1.678 | Accuracy = 48.276\n",
            "Train: Epoch = 138 | Loss = 34.299 | Loss2 = 0.554 | Accuracy = 76.562\n",
            "Val: Epoch = 138 | Loss 1.702 | Accuracy = 48.276\n",
            "Train: Epoch = 139 | Loss = 34.283 | Loss2 = 0.559 | Accuracy = 77.500\n",
            "Val: Epoch = 139 | Loss 1.698 | Accuracy = 49.138\n",
            "Train: Epoch = 140 | Loss = 34.208 | Loss2 = 0.542 | Accuracy = 78.438\n",
            "Val: Epoch = 140 | Loss 1.682 | Accuracy = 50.000\n",
            "Train: Epoch = 141 | Loss = 34.249 | Loss2 = 0.562 | Accuracy = 77.188\n",
            "Val: Epoch = 141 | Loss 1.668 | Accuracy = 50.000\n",
            "Train: Epoch = 142 | Loss = 34.286 | Loss2 = 0.553 | Accuracy = 75.938\n",
            "Val: Epoch = 142 | Loss 1.659 | Accuracy = 50.000\n",
            "Train: Epoch = 143 | Loss = 34.326 | Loss2 = 0.552 | Accuracy = 74.375\n",
            "Val: Epoch = 143 | Loss 1.670 | Accuracy = 50.000\n",
            "Train: Epoch = 144 | Loss = 34.301 | Loss2 = 0.568 | Accuracy = 77.188\n",
            "Val: Epoch = 144 | Loss 1.674 | Accuracy = 49.138\n",
            "Train: Epoch = 145 | Loss = 34.269 | Loss2 = 0.571 | Accuracy = 78.438\n",
            "Val: Epoch = 145 | Loss 1.644 | Accuracy = 50.000\n",
            "Train: Epoch = 146 | Loss = 34.344 | Loss2 = 0.538 | Accuracy = 73.438\n",
            "Val: Epoch = 146 | Loss 1.606 | Accuracy = 50.862\n",
            "Train: Epoch = 147 | Loss = 34.235 | Loss2 = 0.527 | Accuracy = 80.000\n",
            "Val: Epoch = 147 | Loss 1.622 | Accuracy = 49.138\n",
            "Train: Epoch = 148 | Loss = 34.288 | Loss2 = 0.540 | Accuracy = 76.562\n",
            "Val: Epoch = 148 | Loss 1.635 | Accuracy = 49.138\n",
            "Train: Epoch = 149 | Loss = 34.338 | Loss2 = 0.564 | Accuracy = 76.875\n",
            "Val: Epoch = 149 | Loss 1.642 | Accuracy = 50.000\n",
            "Train: Epoch = 150 | Loss = 34.262 | Loss2 = 0.575 | Accuracy = 78.438\n",
            "Val: Epoch = 150 | Loss 1.640 | Accuracy = 51.724\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "T8L07k5o3qIj"
      },
      "source": [
        "# Stage 2 specific setup"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "ExecuteTime": {
          "end_time": "2020-06-08T15:51:16.753000Z",
          "start_time": "2020-06-08T15:51:16.742346Z"
        },
        "colab_type": "code",
        "id": "fTyWCSfP3qIm",
        "colab": {}
      },
      "source": [
        "STAGE = 2\n",
        "\n",
        "LR = 0.0001            # The initial Learning Rate\n",
        "MOMENTUM = 0.9       # Hyperparameter for SGD, keep this at 0.9 when using SGD\n",
        "WEIGHT_DECAY = 4e-5  # Regularization, you can keep this at the default\n",
        "\n",
        "NUM_EPOCHS = 150      # Total number of training epochs (iterations over dataset)\n",
        "STEP_SIZE = [30, 80] # How many epochs before decreasing learning rate (if using a step-down policy)\n",
        "GAMMA = 0.2          # Multiplicative factor for learning rate step-down\n",
        "MEM_SIZE = 512\n",
        "SEQ_LEN = 7\n",
        "ALPHA = 0.1\n",
        "\n",
        "parameters = {\n",
        "    'DEVICE': DEVICE,\n",
        "    'NUM_CLASSES': NUM_CLASSES,\n",
        "    'BATCH_SIZE': BATCH_SIZE,\n",
        "    'LR': LR,\n",
        "    'MOMENTUM': MOMENTUM,\n",
        "    'WEIGHT_DECAY': WEIGHT_DECAY,\n",
        "    'NUM_EPOCHS': NUM_EPOCHS,\n",
        "    'STEP_SIZE': STEP_SIZE,\n",
        "    'GAMMA': GAMMA,\n",
        "    'MEM_SIZE': MEM_SIZE,\n",
        "    'SEQ_LEN': SEQ_LEN,\n",
        "    'ALPHA': ALPHA\n",
        "}"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "ExecuteTime": {
          "end_time": "2020-06-08T15:51:18.906868Z",
          "start_time": "2020-06-08T15:51:18.872090Z"
        },
        "colab_type": "code",
        "id": "ruyZhdO03qIu",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 51
        },
        "outputId": "1e868bca-c5a4-4ffc-a3ea-3f9f633618a3"
      },
      "source": [
        "# Prepare Pytorch train/test Datasets\n",
        "train_dataset = GTEA61(DATA_DIR, split='train', transform=spatial_transform,\n",
        "                       seq_len=SEQ_LEN, mmaps = True, mmaps_transform = spatial_transform_mmaps, static_frames = True)\n",
        "test_dataset = GTEA61(DATA_DIR, split='test', transform=spatial_transform_val,\n",
        "                      seq_len=SEQ_LEN)\n",
        "\n",
        "# Check dataset sizes\n",
        "print('Train Dataset: {}'.format(len(train_dataset)))\n",
        "print('Test Dataset: {}'.format(len(test_dataset)))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Train Dataset: 341\n",
            "Test Dataset: 116\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "ExecuteTime": {
          "end_time": "2020-06-08T15:51:30.177645Z",
          "start_time": "2020-06-08T15:51:30.170111Z"
        },
        "colab_type": "code",
        "id": "PWyR88gb3qIx",
        "colab": {}
      },
      "source": [
        "# Dataloaders iterate over pytorch datasets and transparently provide useful functions (e.g. parallelization and shuffling)\n",
        "train_loader = DataLoader(train_dataset, batch_size=BATCH_SIZE, shuffle=True, num_workers=4, drop_last=True)\n",
        "val_loader = DataLoader(test_dataset, batch_size=BATCH_SIZE, shuffle=False, num_workers=4)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "NeLDsiIO3qI0"
      },
      "source": [
        "# Prepare Stage 2"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "ExecuteTime": {
          "end_time": "2020-06-08T15:51:45.195953Z",
          "start_time": "2020-06-08T15:51:41.943238Z"
        },
        "colab_type": "code",
        "id": "hk2TKf1v3qI0",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 83,
          "referenced_widgets": [
            "74f8f73a236f4c49b9146664c823f140",
            "48d6dc635f72467bb0158b402796b2db",
            "d41b0011933846208b24bcecef1a1632",
            "813897393355404faee4365d74282420",
            "a3e7f81aa5f149e091c24e992bbea54d",
            "aab8ddf0afef4681ad318550c5afae56",
            "e50480ed3c4c46b4b1970546a413b54a",
            "509cd3ca4ab146ff9c497bd2a36b5552"
          ]
        },
        "outputId": "847c71e2-f048-4775-93cd-a64031ef0124"
      },
      "source": [
        "best_old_stage = generate_model_checkpoint_name(stage=1, n_frames=SEQ_LEN, ms_block = True, \\\n",
        "                                                optional = '_2neur_DS__run01')\n",
        "stage1_dict = os.path.join(model_folder, best_old_stage)\n",
        "validate = True\n",
        "\n",
        "model = attention_model_ms(num_classes=NUM_CLASSES, mem_size=MEM_SIZE)\n",
        "model.load_state_dict(torch.load(stage1_dict))\n",
        "\n",
        "model.train(False)\n",
        "\n",
        "for params in model.parameters():\n",
        "    params.requires_grad = False\n",
        "\n",
        "layers_on_off = [\n",
        "    model.resNet.layer4[0].conv1,\n",
        "    model.resNet.layer4[0].conv2,\n",
        "    model.resNet.layer4[1].conv1,\n",
        "    model.resNet.layer4[1].conv2,\n",
        "    model.resNet.layer4[2].conv1,\n",
        "    model.resNet.layer4[2].conv2,\n",
        "    model.resNet.fc,\n",
        "    model.msBlock\n",
        "]\n",
        "\n",
        "layers_to_train = layers_on_off + [\n",
        "    model.lstm_cell,\n",
        "    model.classifier,\n",
        "    model.classifier_SD\n",
        "]\n",
        "\n",
        "for layer in layers_to_train:\n",
        "    for params in layer.parameters():\n",
        "        params.requires_grad = True\n",
        "\n",
        "for layer in layers_to_train:\n",
        "    layer.train(True)\n",
        "\n",
        "model.to(DEVICE)\n",
        "\n",
        "loss_fn = nn.CrossEntropyLoss()\n",
        "loss_fn_sum = nn.CrossEntropyLoss(reduction = 'sum')\n",
        "\n",
        "trainable_params = [p for p in model.parameters() if p.requires_grad]\n",
        "optimizer_fn = torch.optim.Adam(trainable_params, lr=LR, weight_decay=WEIGHT_DECAY, eps=1e-4)\n",
        "\n",
        "optim_scheduler = torch.optim.lr_scheduler.MultiStepLR(optimizer_fn, milestones=STEP_SIZE, gamma=GAMMA)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Downloading: \"https://download.pytorch.org/models/resnet34-333f7ec4.pth\" to /root/.cache/torch/checkpoints/resnet34-333f7ec4.pth\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "74f8f73a236f4c49b9146664c823f140",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, max=87306240.0), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "PEPejAlK3qI3"
      },
      "source": [
        "# Stage 2"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "ExecuteTime": {
          "end_time": "2020-06-08T08:58:05.318659Z",
          "start_time": "2020-06-08T08:57:45.955602Z"
        },
        "colab_type": "code",
        "id": "1zQ2KEiS3qI4",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "outputId": "5c133987-d1e0-46aa-a171-045b49d7c5d8"
      },
      "source": [
        "# RUN = \"_run05_a0.1\"\n",
        "\n",
        "train_iter = 0\n",
        "val_iter = 0\n",
        "min_accuracy = 0\n",
        "\n",
        "trainSamples = len(train_dataset) - (len(train_dataset) % BATCH_SIZE)\n",
        "val_samples = len(test_dataset)\n",
        "\n",
        "iterPerEpoch = len(train_loader)\n",
        "val_steps = len(val_loader)\n",
        "\n",
        "cudnn.benchmark\n",
        "torch.autograd.set_detect_anomaly(True)\n",
        "train_log, val_log = generate_log_filenames(STAGE, SEQ_LEN, ms_block = True, \\\n",
        "                                            optional = '_dsloss')\n",
        "model_checkpoint = generate_model_checkpoint_name(STAGE, SEQ_LEN, ms_block = True, \\\n",
        "                                                  optional = '_dsloss')\n",
        "\n",
        "train_log_file = os.path.join(model_folder, train_log)\n",
        "val_log_file = os.path.join(model_folder, val_log)\n",
        "train_logger_2 = Logger(**parameters)\n",
        "val_logger_2 = Logger(**parameters)\n",
        "\n",
        "for epoch in range(NUM_EPOCHS):\n",
        "\n",
        "    epoch_loss1 = 0\n",
        "    epoch_loss2 = 0\n",
        "    numCorrTrain = 0\n",
        "    \n",
        "    for layer in layers_to_train:\n",
        "        layer.train(True)\n",
        "    \n",
        "    for i, (inputs, inputs_DS, mmaps, targets) in enumerate(train_loader):\n",
        "\n",
        "        mmaps = mmaps.permute(1, 0, 2)\n",
        "\n",
        "        train_iter += 1\n",
        "\n",
        "        optimizer_fn.zero_grad()\n",
        "\n",
        "        inputVariable = inputs.permute(1, 0, 2, 3, 4).to(DEVICE)\n",
        "        inputVariable_DS = inputs_DS.permute(1, 0, 2, 3, 4).to(DEVICE)\n",
        "        labelVariable = targets.to(DEVICE)\n",
        "\n",
        "        output_label, _, output_label_mmaps = model(inputVariable, mmaps = True)\n",
        "\n",
        "        output_label_mmaps = output_label_mmaps.view(SEQ_LEN*BATCH_SIZE*MMAP_LENGTH, 2).to(DEVICE)\n",
        "        mmaps = torch.flatten(mmaps).long().to(DEVICE)\n",
        "\n",
        "        loss1 = loss_fn(output_label, labelVariable)\n",
        "        loss = loss_fn_sum(output_label_mmaps, mmaps)/(BATCH_SIZE * SEQ_LEN) + loss1\n",
        "        loss.backward()\n",
        "\n",
        "        for layer in layers_on_off:\n",
        "            for params in layer.parameters():\n",
        "                params.requires_grad = False\n",
        "\n",
        "        output_label_ds1, _ = model(inputVariable, static_frames = True)\n",
        "        output_label_ds2, _ = model(inputVariable_DS, static_frames = True)\n",
        "        \n",
        "        gt_ds1 = torch.zeros(BATCH_SIZE).long().to(DEVICE)\n",
        "        gt_ds2 = torch.ones(BATCH_SIZE).long().to(DEVICE)\n",
        "\n",
        "        loss2 = ALPHA * (loss_fn(output_label_ds1, gt_ds1) + loss_fn(output_label_ds2, gt_ds2))\n",
        "\n",
        "        loss2.backward()\n",
        "\n",
        "        for layer in layers_on_off:\n",
        "            for params in layer.parameters():\n",
        "                params.requires_grad = True\n",
        "\n",
        "        optimizer_fn.step()\n",
        "        \n",
        "        _, predicted = torch.max(output_label.data, 1)\n",
        "        numCorrTrain += torch.sum(predicted == labelVariable.data).data.item()\n",
        "        step_loss1 = loss1.data.item()\n",
        "        step_loss2 = loss2.data.item()\n",
        "        epoch_loss1 += step_loss1\n",
        "        epoch_loss2 += step_loss2\n",
        "        # train_logger_2.add_step_data(train_iter, numCorrTrain, step_loss1, step_loss2)\n",
        "\n",
        "        \n",
        "    avg_loss1 = epoch_loss1/iterPerEpoch\n",
        "    avg_loss2 = epoch_loss2/iterPerEpoch\n",
        "    trainAccuracy = (numCorrTrain / trainSamples) * 100\n",
        "    train_logger_2.add_epoch_data(epoch+1, trainAccuracy, avg_loss1, avg_loss2)\n",
        "    \n",
        "    print('Train: Epoch = {} | Loss = {:.3f} | Loss2 = {:.3f} | Accuracy = {:.3f}'.format(epoch+1, avg_loss1, avg_loss2, trainAccuracy))\n",
        "\n",
        "    if validate is not None:\n",
        "\n",
        "        if (epoch+1) % 1 == 0:\n",
        "\n",
        "            model.train(False)\n",
        "            val_loss_epoch = 0\n",
        "            numCorr = 0\n",
        "\n",
        "            for j, (inputs, targets) in enumerate(val_loader):\n",
        "\n",
        "                val_iter += 1\n",
        "\n",
        "                inputVariable = inputs.permute(1, 0, 2, 3, 4).to(DEVICE)\n",
        "                labelVariable = targets.to(DEVICE)\n",
        "                \n",
        "                output_label, _ = model(inputVariable)\n",
        "\n",
        "                val_loss = loss_fn(output_label, labelVariable)\n",
        "                val_loss_step = val_loss.data.item()\n",
        "                val_loss_epoch += val_loss_step\n",
        "\n",
        "                _, predicted = torch.max(output_label.data, 1)\n",
        "\n",
        "                numCorr += torch.sum(predicted == labelVariable.data).data.item()\n",
        "                val_logger_2.add_step_data(val_iter, numCorr, val_loss_step)\n",
        "\n",
        "            val_accuracy = (numCorr / val_samples) * 100\n",
        "            avg_val_loss = val_loss_epoch / val_steps\n",
        "\n",
        "            print('Val: Epoch = {} | Loss {:.3f} | Accuracy = {:.3f}'.format(epoch + 1, avg_val_loss, val_accuracy))\n",
        "            \n",
        "            val_logger_2.add_epoch_data(epoch+1, val_accuracy, avg_val_loss)\n",
        "            \n",
        "            if val_accuracy > min_accuracy:\n",
        "                print(\"[||| NEW BEST on val |||]\")\n",
        "                save_path_model = os.path.join(model_folder, model_checkpoint)\n",
        "                torch.save(model.state_dict(), save_path_model)\n",
        "                min_accuracy = val_accuracy\n",
        "            \n",
        "    optim_scheduler.step()\n",
        "\n",
        "    train_logger_2.save(train_log_file)\n",
        "    val_logger_2.save(val_log_file)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Train: Epoch = 1 | Loss = 2.216 | Loss2 = 0.155 | Accuracy = 35.312\n",
            "Val: Epoch = 1 | Loss 2.283 | Accuracy = 35.345\n",
            "[||| NEW BEST on val |||]\n",
            "Train: Epoch = 2 | Loss = 2.054 | Loss2 = 0.159 | Accuracy = 42.500\n",
            "Val: Epoch = 2 | Loss 2.341 | Accuracy = 37.931\n",
            "[||| NEW BEST on val |||]\n",
            "Train: Epoch = 3 | Loss = 1.892 | Loss2 = 0.154 | Accuracy = 41.875\n",
            "Val: Epoch = 3 | Loss 2.404 | Accuracy = 35.345\n",
            "Train: Epoch = 4 | Loss = 1.948 | Loss2 = 0.151 | Accuracy = 43.125\n",
            "Val: Epoch = 4 | Loss 2.089 | Accuracy = 43.103\n",
            "[||| NEW BEST on val |||]\n",
            "Train: Epoch = 5 | Loss = 1.890 | Loss2 = 0.155 | Accuracy = 47.812\n",
            "Val: Epoch = 5 | Loss 2.052 | Accuracy = 41.379\n",
            "Train: Epoch = 6 | Loss = 1.788 | Loss2 = 0.153 | Accuracy = 47.500\n",
            "Val: Epoch = 6 | Loss 2.105 | Accuracy = 44.828\n",
            "[||| NEW BEST on val |||]\n",
            "Train: Epoch = 7 | Loss = 1.801 | Loss2 = 0.149 | Accuracy = 50.313\n",
            "Val: Epoch = 7 | Loss 2.241 | Accuracy = 41.379\n",
            "Train: Epoch = 8 | Loss = 1.838 | Loss2 = 0.154 | Accuracy = 44.375\n",
            "Val: Epoch = 8 | Loss 2.232 | Accuracy = 42.241\n",
            "Train: Epoch = 9 | Loss = 1.748 | Loss2 = 0.151 | Accuracy = 51.250\n",
            "Val: Epoch = 9 | Loss 1.913 | Accuracy = 43.966\n",
            "Train: Epoch = 10 | Loss = 1.771 | Loss2 = 0.151 | Accuracy = 47.500\n",
            "Val: Epoch = 10 | Loss 2.172 | Accuracy = 41.379\n",
            "Train: Epoch = 11 | Loss = 1.727 | Loss2 = 0.147 | Accuracy = 53.438\n",
            "Val: Epoch = 11 | Loss 2.010 | Accuracy = 43.966\n",
            "Train: Epoch = 12 | Loss = 1.795 | Loss2 = 0.146 | Accuracy = 46.562\n",
            "Val: Epoch = 12 | Loss 1.927 | Accuracy = 48.276\n",
            "[||| NEW BEST on val |||]\n",
            "Train: Epoch = 13 | Loss = 1.614 | Loss2 = 0.153 | Accuracy = 49.375\n",
            "Val: Epoch = 13 | Loss 2.109 | Accuracy = 38.793\n",
            "Train: Epoch = 14 | Loss = 1.654 | Loss2 = 0.141 | Accuracy = 48.438\n",
            "Val: Epoch = 14 | Loss 1.959 | Accuracy = 44.828\n",
            "Train: Epoch = 15 | Loss = 1.669 | Loss2 = 0.147 | Accuracy = 52.188\n",
            "Val: Epoch = 15 | Loss 1.856 | Accuracy = 43.966\n",
            "Train: Epoch = 16 | Loss = 1.570 | Loss2 = 0.151 | Accuracy = 56.250\n",
            "Val: Epoch = 16 | Loss 1.679 | Accuracy = 52.586\n",
            "[||| NEW BEST on val |||]\n",
            "Train: Epoch = 17 | Loss = 1.593 | Loss2 = 0.142 | Accuracy = 54.688\n",
            "Val: Epoch = 17 | Loss 1.852 | Accuracy = 48.276\n",
            "Train: Epoch = 18 | Loss = 1.564 | Loss2 = 0.143 | Accuracy = 54.375\n",
            "Val: Epoch = 18 | Loss 1.976 | Accuracy = 43.966\n",
            "Train: Epoch = 19 | Loss = 1.582 | Loss2 = 0.145 | Accuracy = 52.812\n",
            "Val: Epoch = 19 | Loss 2.062 | Accuracy = 43.103\n",
            "Train: Epoch = 20 | Loss = 1.659 | Loss2 = 0.141 | Accuracy = 53.438\n",
            "Val: Epoch = 20 | Loss 1.839 | Accuracy = 50.862\n",
            "Train: Epoch = 21 | Loss = 1.634 | Loss2 = 0.139 | Accuracy = 50.938\n",
            "Val: Epoch = 21 | Loss 1.761 | Accuracy = 50.000\n",
            "Train: Epoch = 22 | Loss = 1.436 | Loss2 = 0.146 | Accuracy = 58.438\n",
            "Val: Epoch = 22 | Loss 1.906 | Accuracy = 51.724\n",
            "Train: Epoch = 23 | Loss = 1.535 | Loss2 = 0.141 | Accuracy = 56.563\n",
            "Val: Epoch = 23 | Loss 1.994 | Accuracy = 50.862\n",
            "Train: Epoch = 24 | Loss = 1.637 | Loss2 = 0.144 | Accuracy = 52.188\n",
            "Val: Epoch = 24 | Loss 2.111 | Accuracy = 43.103\n",
            "Train: Epoch = 25 | Loss = 1.674 | Loss2 = 0.142 | Accuracy = 51.562\n",
            "Val: Epoch = 25 | Loss 1.692 | Accuracy = 50.000\n",
            "Train: Epoch = 26 | Loss = 1.640 | Loss2 = 0.143 | Accuracy = 51.875\n",
            "Val: Epoch = 26 | Loss 1.973 | Accuracy = 48.276\n",
            "Train: Epoch = 27 | Loss = 1.486 | Loss2 = 0.140 | Accuracy = 60.000\n",
            "Val: Epoch = 27 | Loss 1.780 | Accuracy = 50.862\n",
            "Train: Epoch = 28 | Loss = 1.303 | Loss2 = 0.142 | Accuracy = 60.312\n",
            "Val: Epoch = 28 | Loss 1.654 | Accuracy = 51.724\n",
            "Train: Epoch = 29 | Loss = 1.411 | Loss2 = 0.138 | Accuracy = 56.563\n",
            "Val: Epoch = 29 | Loss 1.926 | Accuracy = 48.276\n",
            "Train: Epoch = 30 | Loss = 1.276 | Loss2 = 0.134 | Accuracy = 61.562\n",
            "Val: Epoch = 30 | Loss 1.967 | Accuracy = 48.276\n",
            "Train: Epoch = 31 | Loss = 1.348 | Loss2 = 0.142 | Accuracy = 60.000\n",
            "Val: Epoch = 31 | Loss 1.798 | Accuracy = 54.310\n",
            "[||| NEW BEST on val |||]\n",
            "Train: Epoch = 32 | Loss = 1.336 | Loss2 = 0.144 | Accuracy = 58.438\n",
            "Val: Epoch = 32 | Loss 1.685 | Accuracy = 53.448\n",
            "Train: Epoch = 33 | Loss = 1.298 | Loss2 = 0.146 | Accuracy = 61.562\n",
            "Val: Epoch = 33 | Loss 1.664 | Accuracy = 56.034\n",
            "[||| NEW BEST on val |||]\n",
            "Train: Epoch = 34 | Loss = 1.308 | Loss2 = 0.141 | Accuracy = 64.375\n",
            "Val: Epoch = 34 | Loss 1.703 | Accuracy = 51.724\n",
            "Train: Epoch = 35 | Loss = 1.214 | Loss2 = 0.142 | Accuracy = 63.750\n",
            "Val: Epoch = 35 | Loss 1.738 | Accuracy = 52.586\n",
            "Train: Epoch = 36 | Loss = 1.126 | Loss2 = 0.145 | Accuracy = 66.875\n",
            "Val: Epoch = 36 | Loss 1.671 | Accuracy = 52.586\n",
            "Train: Epoch = 37 | Loss = 1.144 | Loss2 = 0.146 | Accuracy = 65.938\n",
            "Val: Epoch = 37 | Loss 1.636 | Accuracy = 57.759\n",
            "[||| NEW BEST on val |||]\n",
            "Train: Epoch = 38 | Loss = 1.225 | Loss2 = 0.143 | Accuracy = 64.062\n",
            "Val: Epoch = 38 | Loss 1.665 | Accuracy = 54.310\n",
            "Train: Epoch = 39 | Loss = 1.144 | Loss2 = 0.142 | Accuracy = 62.813\n",
            "Val: Epoch = 39 | Loss 1.700 | Accuracy = 51.724\n",
            "Train: Epoch = 40 | Loss = 1.293 | Loss2 = 0.149 | Accuracy = 61.875\n",
            "Val: Epoch = 40 | Loss 1.769 | Accuracy = 51.724\n",
            "Train: Epoch = 41 | Loss = 1.217 | Loss2 = 0.143 | Accuracy = 64.062\n",
            "Val: Epoch = 41 | Loss 1.680 | Accuracy = 52.586\n",
            "Train: Epoch = 42 | Loss = 1.130 | Loss2 = 0.149 | Accuracy = 65.312\n",
            "Val: Epoch = 42 | Loss 1.636 | Accuracy = 54.310\n",
            "Train: Epoch = 43 | Loss = 1.306 | Loss2 = 0.144 | Accuracy = 60.625\n",
            "Val: Epoch = 43 | Loss 1.708 | Accuracy = 53.448\n",
            "Train: Epoch = 44 | Loss = 1.084 | Loss2 = 0.140 | Accuracy = 66.250\n",
            "Val: Epoch = 44 | Loss 1.705 | Accuracy = 53.448\n",
            "Train: Epoch = 45 | Loss = 1.218 | Loss2 = 0.142 | Accuracy = 60.000\n",
            "Val: Epoch = 45 | Loss 1.809 | Accuracy = 50.000\n",
            "Train: Epoch = 46 | Loss = 1.058 | Loss2 = 0.146 | Accuracy = 70.000\n",
            "Val: Epoch = 46 | Loss 1.654 | Accuracy = 53.448\n",
            "Train: Epoch = 47 | Loss = 1.202 | Loss2 = 0.141 | Accuracy = 65.938\n",
            "Val: Epoch = 47 | Loss 1.769 | Accuracy = 51.724\n",
            "Train: Epoch = 48 | Loss = 1.234 | Loss2 = 0.149 | Accuracy = 62.187\n",
            "Val: Epoch = 48 | Loss 1.685 | Accuracy = 52.586\n",
            "Train: Epoch = 49 | Loss = 1.099 | Loss2 = 0.143 | Accuracy = 66.875\n",
            "Val: Epoch = 49 | Loss 1.596 | Accuracy = 58.621\n",
            "[||| NEW BEST on val |||]\n",
            "Train: Epoch = 50 | Loss = 1.064 | Loss2 = 0.143 | Accuracy = 71.875\n",
            "Val: Epoch = 50 | Loss 1.679 | Accuracy = 54.310\n",
            "Train: Epoch = 51 | Loss = 1.062 | Loss2 = 0.140 | Accuracy = 69.062\n",
            "Val: Epoch = 51 | Loss 1.682 | Accuracy = 51.724\n",
            "Train: Epoch = 52 | Loss = 1.150 | Loss2 = 0.146 | Accuracy = 67.188\n",
            "Val: Epoch = 52 | Loss 1.773 | Accuracy = 53.448\n",
            "Train: Epoch = 53 | Loss = 1.046 | Loss2 = 0.141 | Accuracy = 68.750\n",
            "Val: Epoch = 53 | Loss 1.734 | Accuracy = 49.138\n",
            "Train: Epoch = 54 | Loss = 1.047 | Loss2 = 0.147 | Accuracy = 70.000\n",
            "Val: Epoch = 54 | Loss 1.647 | Accuracy = 52.586\n",
            "Train: Epoch = 55 | Loss = 1.058 | Loss2 = 0.139 | Accuracy = 67.812\n",
            "Val: Epoch = 55 | Loss 1.643 | Accuracy = 51.724\n",
            "Train: Epoch = 56 | Loss = 1.134 | Loss2 = 0.141 | Accuracy = 67.812\n",
            "Val: Epoch = 56 | Loss 1.748 | Accuracy = 52.586\n",
            "Train: Epoch = 57 | Loss = 1.059 | Loss2 = 0.146 | Accuracy = 70.312\n",
            "Val: Epoch = 57 | Loss 1.748 | Accuracy = 51.724\n",
            "Train: Epoch = 58 | Loss = 1.074 | Loss2 = 0.139 | Accuracy = 67.812\n",
            "Val: Epoch = 58 | Loss 1.749 | Accuracy = 50.862\n",
            "Train: Epoch = 59 | Loss = 1.072 | Loss2 = 0.142 | Accuracy = 69.688\n",
            "Val: Epoch = 59 | Loss 1.785 | Accuracy = 50.000\n",
            "Train: Epoch = 60 | Loss = 1.012 | Loss2 = 0.137 | Accuracy = 73.438\n",
            "Val: Epoch = 60 | Loss 1.786 | Accuracy = 51.724\n",
            "Train: Epoch = 61 | Loss = 0.985 | Loss2 = 0.146 | Accuracy = 70.625\n",
            "Val: Epoch = 61 | Loss 1.732 | Accuracy = 54.310\n",
            "Train: Epoch = 62 | Loss = 1.074 | Loss2 = 0.146 | Accuracy = 69.062\n",
            "Val: Epoch = 62 | Loss 1.779 | Accuracy = 54.310\n",
            "Train: Epoch = 63 | Loss = 0.999 | Loss2 = 0.144 | Accuracy = 69.062\n",
            "Val: Epoch = 63 | Loss 1.732 | Accuracy = 51.724\n",
            "Train: Epoch = 64 | Loss = 1.095 | Loss2 = 0.138 | Accuracy = 69.062\n",
            "Val: Epoch = 64 | Loss 1.606 | Accuracy = 59.483\n",
            "[||| NEW BEST on val |||]\n",
            "Train: Epoch = 65 | Loss = 0.958 | Loss2 = 0.140 | Accuracy = 72.188\n",
            "Val: Epoch = 65 | Loss 1.632 | Accuracy = 58.621\n",
            "Train: Epoch = 66 | Loss = 0.882 | Loss2 = 0.140 | Accuracy = 74.375\n",
            "Val: Epoch = 66 | Loss 1.610 | Accuracy = 58.621\n",
            "Train: Epoch = 67 | Loss = 0.999 | Loss2 = 0.147 | Accuracy = 73.125\n",
            "Val: Epoch = 67 | Loss 1.613 | Accuracy = 58.621\n",
            "Train: Epoch = 68 | Loss = 0.967 | Loss2 = 0.142 | Accuracy = 73.750\n",
            "Val: Epoch = 68 | Loss 1.669 | Accuracy = 58.621\n",
            "Train: Epoch = 69 | Loss = 1.004 | Loss2 = 0.145 | Accuracy = 70.312\n",
            "Val: Epoch = 69 | Loss 1.723 | Accuracy = 56.897\n",
            "Train: Epoch = 70 | Loss = 0.992 | Loss2 = 0.147 | Accuracy = 70.312\n",
            "Val: Epoch = 70 | Loss 1.837 | Accuracy = 53.448\n",
            "Train: Epoch = 71 | Loss = 0.940 | Loss2 = 0.141 | Accuracy = 72.188\n",
            "Val: Epoch = 71 | Loss 1.663 | Accuracy = 56.897\n",
            "Train: Epoch = 72 | Loss = 1.005 | Loss2 = 0.149 | Accuracy = 70.000\n",
            "Val: Epoch = 72 | Loss 1.732 | Accuracy = 56.897\n",
            "Train: Epoch = 73 | Loss = 1.045 | Loss2 = 0.140 | Accuracy = 67.188\n",
            "Val: Epoch = 73 | Loss 1.583 | Accuracy = 60.345\n",
            "[||| NEW BEST on val |||]\n",
            "Train: Epoch = 74 | Loss = 0.965 | Loss2 = 0.143 | Accuracy = 72.500\n",
            "Val: Epoch = 74 | Loss 1.659 | Accuracy = 56.897\n",
            "Train: Epoch = 75 | Loss = 0.916 | Loss2 = 0.147 | Accuracy = 75.000\n",
            "Val: Epoch = 75 | Loss 1.688 | Accuracy = 55.172\n",
            "Train: Epoch = 76 | Loss = 0.959 | Loss2 = 0.141 | Accuracy = 70.938\n",
            "Val: Epoch = 76 | Loss 1.657 | Accuracy = 54.310\n",
            "Train: Epoch = 77 | Loss = 0.890 | Loss2 = 0.143 | Accuracy = 72.188\n",
            "Val: Epoch = 77 | Loss 1.821 | Accuracy = 51.724\n",
            "Train: Epoch = 78 | Loss = 0.881 | Loss2 = 0.147 | Accuracy = 74.688\n",
            "Val: Epoch = 78 | Loss 1.639 | Accuracy = 56.034\n",
            "Train: Epoch = 79 | Loss = 0.924 | Loss2 = 0.142 | Accuracy = 73.438\n",
            "Val: Epoch = 79 | Loss 1.752 | Accuracy = 53.448\n",
            "Train: Epoch = 80 | Loss = 0.993 | Loss2 = 0.143 | Accuracy = 72.188\n",
            "Val: Epoch = 80 | Loss 1.654 | Accuracy = 54.310\n",
            "Train: Epoch = 81 | Loss = 0.869 | Loss2 = 0.143 | Accuracy = 74.062\n",
            "Val: Epoch = 81 | Loss 1.672 | Accuracy = 53.448\n",
            "Train: Epoch = 82 | Loss = 0.920 | Loss2 = 0.143 | Accuracy = 70.625\n",
            "Val: Epoch = 82 | Loss 1.705 | Accuracy = 53.448\n",
            "Train: Epoch = 83 | Loss = 0.930 | Loss2 = 0.139 | Accuracy = 71.250\n",
            "Val: Epoch = 83 | Loss 1.672 | Accuracy = 55.172\n",
            "Train: Epoch = 84 | Loss = 0.892 | Loss2 = 0.146 | Accuracy = 73.750\n",
            "Val: Epoch = 84 | Loss 1.649 | Accuracy = 55.172\n",
            "Train: Epoch = 85 | Loss = 0.963 | Loss2 = 0.142 | Accuracy = 72.500\n",
            "Val: Epoch = 85 | Loss 1.653 | Accuracy = 56.034\n",
            "Train: Epoch = 86 | Loss = 0.839 | Loss2 = 0.145 | Accuracy = 73.438\n",
            "Val: Epoch = 86 | Loss 1.663 | Accuracy = 55.172\n",
            "Train: Epoch = 87 | Loss = 0.905 | Loss2 = 0.144 | Accuracy = 72.812\n",
            "Val: Epoch = 87 | Loss 1.658 | Accuracy = 54.310\n",
            "Train: Epoch = 88 | Loss = 0.919 | Loss2 = 0.138 | Accuracy = 72.812\n",
            "Val: Epoch = 88 | Loss 1.682 | Accuracy = 53.448\n",
            "Train: Epoch = 89 | Loss = 0.868 | Loss2 = 0.143 | Accuracy = 72.188\n",
            "Val: Epoch = 89 | Loss 1.670 | Accuracy = 51.724\n",
            "Train: Epoch = 90 | Loss = 0.857 | Loss2 = 0.145 | Accuracy = 74.375\n",
            "Val: Epoch = 90 | Loss 1.670 | Accuracy = 51.724\n",
            "Train: Epoch = 91 | Loss = 1.005 | Loss2 = 0.139 | Accuracy = 68.750\n",
            "Val: Epoch = 91 | Loss 1.648 | Accuracy = 55.172\n",
            "Train: Epoch = 92 | Loss = 0.876 | Loss2 = 0.135 | Accuracy = 75.312\n",
            "Val: Epoch = 92 | Loss 1.646 | Accuracy = 55.172\n",
            "Train: Epoch = 93 | Loss = 0.929 | Loss2 = 0.145 | Accuracy = 71.562\n",
            "Val: Epoch = 93 | Loss 1.630 | Accuracy = 56.034\n",
            "Train: Epoch = 94 | Loss = 0.771 | Loss2 = 0.142 | Accuracy = 75.312\n",
            "Val: Epoch = 94 | Loss 1.638 | Accuracy = 54.310\n",
            "Train: Epoch = 95 | Loss = 0.999 | Loss2 = 0.138 | Accuracy = 73.438\n",
            "Val: Epoch = 95 | Loss 1.666 | Accuracy = 53.448\n",
            "Train: Epoch = 96 | Loss = 0.868 | Loss2 = 0.138 | Accuracy = 73.750\n",
            "Val: Epoch = 96 | Loss 1.682 | Accuracy = 53.448\n",
            "Train: Epoch = 97 | Loss = 0.818 | Loss2 = 0.144 | Accuracy = 74.062\n",
            "Val: Epoch = 97 | Loss 1.707 | Accuracy = 54.310\n",
            "Train: Epoch = 98 | Loss = 0.915 | Loss2 = 0.135 | Accuracy = 73.125\n",
            "Val: Epoch = 98 | Loss 1.666 | Accuracy = 54.310\n",
            "Train: Epoch = 99 | Loss = 0.775 | Loss2 = 0.142 | Accuracy = 75.312\n",
            "Val: Epoch = 99 | Loss 1.639 | Accuracy = 54.310\n",
            "Train: Epoch = 100 | Loss = 0.850 | Loss2 = 0.139 | Accuracy = 77.500\n",
            "Val: Epoch = 100 | Loss 1.623 | Accuracy = 53.448\n",
            "Train: Epoch = 101 | Loss = 0.914 | Loss2 = 0.141 | Accuracy = 72.812\n",
            "Val: Epoch = 101 | Loss 1.601 | Accuracy = 55.172\n",
            "Train: Epoch = 102 | Loss = 0.830 | Loss2 = 0.136 | Accuracy = 76.562\n",
            "Val: Epoch = 102 | Loss 1.618 | Accuracy = 55.172\n",
            "Train: Epoch = 103 | Loss = 0.952 | Loss2 = 0.140 | Accuracy = 71.562\n",
            "Val: Epoch = 103 | Loss 1.621 | Accuracy = 54.310\n",
            "Train: Epoch = 104 | Loss = 0.818 | Loss2 = 0.136 | Accuracy = 76.250\n",
            "Val: Epoch = 104 | Loss 1.616 | Accuracy = 55.172\n",
            "Train: Epoch = 105 | Loss = 0.829 | Loss2 = 0.138 | Accuracy = 77.188\n",
            "Val: Epoch = 105 | Loss 1.627 | Accuracy = 53.448\n",
            "Train: Epoch = 106 | Loss = 1.010 | Loss2 = 0.146 | Accuracy = 72.500\n",
            "Val: Epoch = 106 | Loss 1.635 | Accuracy = 54.310\n",
            "Train: Epoch = 107 | Loss = 0.770 | Loss2 = 0.139 | Accuracy = 76.562\n",
            "Val: Epoch = 107 | Loss 1.624 | Accuracy = 55.172\n",
            "Train: Epoch = 108 | Loss = 0.918 | Loss2 = 0.138 | Accuracy = 72.812\n",
            "Val: Epoch = 108 | Loss 1.634 | Accuracy = 54.310\n",
            "Train: Epoch = 109 | Loss = 0.790 | Loss2 = 0.142 | Accuracy = 76.250\n",
            "Val: Epoch = 109 | Loss 1.639 | Accuracy = 51.724\n",
            "Train: Epoch = 110 | Loss = 0.864 | Loss2 = 0.146 | Accuracy = 74.688\n",
            "Val: Epoch = 110 | Loss 1.637 | Accuracy = 52.586\n",
            "Train: Epoch = 111 | Loss = 0.816 | Loss2 = 0.143 | Accuracy = 76.562\n",
            "Val: Epoch = 111 | Loss 1.626 | Accuracy = 53.448\n",
            "Train: Epoch = 112 | Loss = 0.817 | Loss2 = 0.147 | Accuracy = 75.000\n",
            "Val: Epoch = 112 | Loss 1.627 | Accuracy = 56.034\n",
            "Train: Epoch = 113 | Loss = 0.858 | Loss2 = 0.140 | Accuracy = 73.438\n",
            "Val: Epoch = 113 | Loss 1.635 | Accuracy = 56.034\n",
            "Train: Epoch = 114 | Loss = 0.900 | Loss2 = 0.142 | Accuracy = 76.562\n",
            "Val: Epoch = 114 | Loss 1.640 | Accuracy = 54.310\n",
            "Train: Epoch = 115 | Loss = 0.805 | Loss2 = 0.143 | Accuracy = 75.312\n",
            "Val: Epoch = 115 | Loss 1.656 | Accuracy = 54.310\n",
            "Train: Epoch = 116 | Loss = 0.819 | Loss2 = 0.143 | Accuracy = 78.750\n",
            "Val: Epoch = 116 | Loss 1.640 | Accuracy = 56.034\n",
            "Train: Epoch = 117 | Loss = 0.824 | Loss2 = 0.141 | Accuracy = 76.250\n",
            "Val: Epoch = 117 | Loss 1.589 | Accuracy = 57.759\n",
            "Train: Epoch = 118 | Loss = 0.743 | Loss2 = 0.141 | Accuracy = 79.688\n",
            "Val: Epoch = 118 | Loss 1.601 | Accuracy = 56.034\n",
            "Train: Epoch = 119 | Loss = 0.874 | Loss2 = 0.140 | Accuracy = 73.750\n",
            "Val: Epoch = 119 | Loss 1.640 | Accuracy = 53.448\n",
            "Train: Epoch = 120 | Loss = 0.912 | Loss2 = 0.148 | Accuracy = 73.125\n",
            "Val: Epoch = 120 | Loss 1.656 | Accuracy = 54.310\n",
            "Train: Epoch = 121 | Loss = 0.824 | Loss2 = 0.148 | Accuracy = 77.812\n",
            "Val: Epoch = 121 | Loss 1.653 | Accuracy = 54.310\n",
            "Train: Epoch = 122 | Loss = 0.848 | Loss2 = 0.141 | Accuracy = 77.500\n",
            "Val: Epoch = 122 | Loss 1.662 | Accuracy = 54.310\n",
            "Train: Epoch = 123 | Loss = 0.842 | Loss2 = 0.143 | Accuracy = 76.250\n",
            "Val: Epoch = 123 | Loss 1.664 | Accuracy = 53.448\n",
            "Train: Epoch = 124 | Loss = 0.842 | Loss2 = 0.141 | Accuracy = 74.062\n",
            "Val: Epoch = 124 | Loss 1.645 | Accuracy = 55.172\n",
            "Train: Epoch = 125 | Loss = 0.831 | Loss2 = 0.143 | Accuracy = 76.250\n",
            "Val: Epoch = 125 | Loss 1.635 | Accuracy = 54.310\n",
            "Train: Epoch = 126 | Loss = 0.848 | Loss2 = 0.140 | Accuracy = 74.375\n",
            "Val: Epoch = 126 | Loss 1.632 | Accuracy = 54.310\n",
            "Train: Epoch = 127 | Loss = 0.897 | Loss2 = 0.138 | Accuracy = 71.562\n",
            "Val: Epoch = 127 | Loss 1.634 | Accuracy = 54.310\n",
            "Train: Epoch = 128 | Loss = 0.803 | Loss2 = 0.140 | Accuracy = 75.312\n",
            "Val: Epoch = 128 | Loss 1.622 | Accuracy = 54.310\n",
            "Train: Epoch = 129 | Loss = 0.771 | Loss2 = 0.145 | Accuracy = 76.875\n",
            "Val: Epoch = 129 | Loss 1.639 | Accuracy = 52.586\n",
            "Train: Epoch = 130 | Loss = 0.881 | Loss2 = 0.140 | Accuracy = 76.250\n",
            "Val: Epoch = 130 | Loss 1.648 | Accuracy = 53.448\n",
            "Train: Epoch = 131 | Loss = 0.812 | Loss2 = 0.137 | Accuracy = 75.625\n",
            "Val: Epoch = 131 | Loss 1.641 | Accuracy = 54.310\n",
            "Train: Epoch = 132 | Loss = 0.885 | Loss2 = 0.138 | Accuracy = 74.062\n",
            "Val: Epoch = 132 | Loss 1.636 | Accuracy = 54.310\n",
            "Train: Epoch = 133 | Loss = 0.895 | Loss2 = 0.146 | Accuracy = 73.438\n",
            "Val: Epoch = 133 | Loss 1.673 | Accuracy = 54.310\n",
            "Train: Epoch = 134 | Loss = 0.841 | Loss2 = 0.133 | Accuracy = 76.875\n",
            "Val: Epoch = 134 | Loss 1.710 | Accuracy = 52.586\n",
            "Train: Epoch = 135 | Loss = 0.757 | Loss2 = 0.147 | Accuracy = 76.562\n",
            "Val: Epoch = 135 | Loss 1.727 | Accuracy = 54.310\n",
            "Train: Epoch = 136 | Loss = 0.666 | Loss2 = 0.133 | Accuracy = 80.938\n",
            "Val: Epoch = 136 | Loss 1.680 | Accuracy = 54.310\n",
            "Train: Epoch = 137 | Loss = 0.796 | Loss2 = 0.135 | Accuracy = 74.688\n",
            "Val: Epoch = 137 | Loss 1.654 | Accuracy = 54.310\n",
            "Train: Epoch = 138 | Loss = 0.832 | Loss2 = 0.143 | Accuracy = 79.062\n",
            "Val: Epoch = 138 | Loss 1.679 | Accuracy = 53.448\n",
            "Train: Epoch = 139 | Loss = 0.792 | Loss2 = 0.143 | Accuracy = 73.125\n",
            "Val: Epoch = 139 | Loss 1.661 | Accuracy = 54.310\n",
            "Train: Epoch = 140 | Loss = 0.767 | Loss2 = 0.139 | Accuracy = 78.438\n",
            "Val: Epoch = 140 | Loss 1.637 | Accuracy = 55.172\n",
            "Train: Epoch = 141 | Loss = 0.777 | Loss2 = 0.138 | Accuracy = 77.188\n",
            "Val: Epoch = 141 | Loss 1.653 | Accuracy = 54.310\n",
            "Train: Epoch = 142 | Loss = 0.767 | Loss2 = 0.147 | Accuracy = 80.312\n",
            "Val: Epoch = 142 | Loss 1.660 | Accuracy = 54.310\n",
            "Train: Epoch = 143 | Loss = 0.707 | Loss2 = 0.135 | Accuracy = 77.188\n",
            "Val: Epoch = 143 | Loss 1.647 | Accuracy = 54.310\n",
            "Train: Epoch = 144 | Loss = 0.830 | Loss2 = 0.140 | Accuracy = 76.562\n",
            "Val: Epoch = 144 | Loss 1.665 | Accuracy = 54.310\n",
            "Train: Epoch = 145 | Loss = 0.881 | Loss2 = 0.135 | Accuracy = 74.375\n",
            "Val: Epoch = 145 | Loss 1.684 | Accuracy = 54.310\n",
            "Train: Epoch = 146 | Loss = 0.762 | Loss2 = 0.145 | Accuracy = 76.875\n",
            "Val: Epoch = 146 | Loss 1.634 | Accuracy = 55.172\n",
            "Train: Epoch = 147 | Loss = 0.863 | Loss2 = 0.142 | Accuracy = 74.062\n",
            "Val: Epoch = 147 | Loss 1.640 | Accuracy = 56.034\n",
            "Train: Epoch = 148 | Loss = 0.806 | Loss2 = 0.138 | Accuracy = 76.250\n",
            "Val: Epoch = 148 | Loss 1.652 | Accuracy = 55.172\n",
            "Train: Epoch = 149 | Loss = 0.898 | Loss2 = 0.142 | Accuracy = 71.875\n",
            "Val: Epoch = 149 | Loss 1.664 | Accuracy = 54.310\n",
            "Train: Epoch = 150 | Loss = 0.824 | Loss2 = 0.133 | Accuracy = 75.312\n",
            "Val: Epoch = 150 | Loss 1.688 | Accuracy = 53.448\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "ExecuteTime": {
          "end_time": "2020-06-08T08:58:05.318659Z",
          "start_time": "2020-06-08T08:57:45.955602Z"
        },
        "colab_type": "code",
        "id": "O8Lrj3X2h9VC",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "outputId": "ba81ffd0-09fa-4d80-df10-a9c56b458e33"
      },
      "source": [
        "RUN = \"_run06_a0.1\"\n",
        "\n",
        "train_iter = 0\n",
        "val_iter = 0\n",
        "min_accuracy = 0\n",
        "\n",
        "trainSamples = len(train_dataset) - (len(train_dataset) % BATCH_SIZE)\n",
        "val_samples = len(test_dataset)\n",
        "\n",
        "iterPerEpoch = len(train_loader)\n",
        "val_steps = len(val_loader)\n",
        "\n",
        "cudnn.benchmark\n",
        "torch.autograd.set_detect_anomaly(True)\n",
        "train_log, val_log = generate_log_filenames(STAGE, SEQ_LEN, ms_block = True, \\\n",
        "                                            optional = '_2neur_DS_alpha_' + RUN)\n",
        "model_checkpoint = generate_model_checkpoint_name(STAGE, SEQ_LEN, ms_block = True, \\\n",
        "                                                  optional = '_2neur_DS_alpha_' + RUN)\n",
        "\n",
        "train_log_file = os.path.join(model_folder, train_log)\n",
        "val_log_file = os.path.join(model_folder, val_log)\n",
        "train_logger_2 = Logger(**parameters)\n",
        "val_logger_2 = Logger(**parameters)\n",
        "\n",
        "for epoch in range(NUM_EPOCHS):\n",
        "\n",
        "    epoch_loss = 0\n",
        "    epoch_loss2 = 0\n",
        "    numCorrTrain = 0\n",
        "    \n",
        "    for layer in layers_to_train:\n",
        "        layer.train(True)\n",
        "    \n",
        "    for i, (inputs, inputs_DS, mmaps, targets) in enumerate(train_loader):\n",
        "\n",
        "        mmaps = mmaps.permute(1, 0, 2)\n",
        "\n",
        "        train_iter += 1\n",
        "\n",
        "        optimizer_fn.zero_grad()\n",
        "\n",
        "        inputVariable = inputs.permute(1, 0, 2, 3, 4).to(DEVICE)\n",
        "        inputVariable_DS = inputs_DS.permute(1, 0, 2, 3, 4).to(DEVICE)\n",
        "        labelVariable = targets.to(DEVICE)\n",
        "\n",
        "        output_label, _, output_label_mmaps = model(inputVariable, mmaps = True)\n",
        "\n",
        "        output_label_mmaps = output_label_mmaps.view(SEQ_LEN*BATCH_SIZE*MMAP_LENGTH, 2).to(DEVICE)\n",
        "        mmaps = torch.flatten(mmaps).long().to(DEVICE)\n",
        "\n",
        "        loss = loss_fn_sum(output_label_mmaps, mmaps)/(BATCH_SIZE * SEQ_LEN) + loss_fn(output_label, labelVariable)\n",
        "        loss.backward()\n",
        "\n",
        "        for layer in layers_on_off:\n",
        "            for params in layer.parameters():\n",
        "                params.requires_grad = False\n",
        "\n",
        "        output_label_ds1, _ = model(inputVariable, static_frames = True)\n",
        "        output_label_ds2, _ = model(inputVariable_DS, static_frames = True)\n",
        "        \n",
        "        gt_ds1 = torch.zeros(BATCH_SIZE).long().to(DEVICE)\n",
        "        gt_ds2 = torch.ones(BATCH_SIZE).long().to(DEVICE)\n",
        "\n",
        "        loss2 = ALPHA * (loss_fn(output_label_ds1, gt_ds1) + loss_fn(output_label_ds2, gt_ds2))\n",
        "\n",
        "        loss2.backward()\n",
        "\n",
        "        for layer in layers_on_off:\n",
        "            for params in layer.parameters():\n",
        "                params.requires_grad = True\n",
        "\n",
        "        optimizer_fn.step()\n",
        "        \n",
        "        _, predicted = torch.max(output_label.data, 1)\n",
        "        numCorrTrain += torch.sum(predicted == labelVariable.data).data.item()\n",
        "        step_loss = loss.data.item()\n",
        "        step_loss2 = loss2.data.item()\n",
        "        epoch_loss += step_loss\n",
        "        epoch_loss2 += step_loss2\n",
        "        train_logger_2.add_step_data(train_iter, numCorrTrain, step_loss)\n",
        "\n",
        "        \n",
        "    avg_loss = epoch_loss/iterPerEpoch\n",
        "    avg_loss2 = epoch_loss2/iterPerEpoch\n",
        "    trainAccuracy = (numCorrTrain / trainSamples) * 100\n",
        "    train_logger_2.add_epoch_data(epoch+1, trainAccuracy, avg_loss)\n",
        "    \n",
        "    print('Train: Epoch = {} | Loss = {:.3f} | Loss2 = {:.3f} | Accuracy = {:.3f}'.format(epoch+1, avg_loss, avg_loss2, trainAccuracy))\n",
        "\n",
        "    if validate is not None:\n",
        "\n",
        "        if (epoch+1) % 1 == 0:\n",
        "\n",
        "            model.train(False)\n",
        "            val_loss_epoch = 0\n",
        "            numCorr = 0\n",
        "\n",
        "            for j, (inputs, targets) in enumerate(val_loader):\n",
        "\n",
        "                val_iter += 1\n",
        "\n",
        "                inputVariable = inputs.permute(1, 0, 2, 3, 4).to(DEVICE)\n",
        "                labelVariable = targets.to(DEVICE)\n",
        "                \n",
        "                output_label, _ = model(inputVariable)\n",
        "\n",
        "                val_loss = loss_fn(output_label, labelVariable)\n",
        "                val_loss_step = val_loss.data.item()\n",
        "                val_loss_epoch += val_loss_step\n",
        "\n",
        "                _, predicted = torch.max(output_label.data, 1)\n",
        "\n",
        "                numCorr += torch.sum(predicted == labelVariable.data).data.item()\n",
        "                val_logger_2.add_step_data(val_iter, numCorr, val_loss_step)\n",
        "\n",
        "            val_accuracy = (numCorr / val_samples) * 100\n",
        "            avg_val_loss = val_loss_epoch / val_steps\n",
        "\n",
        "            print('Val: Epoch = {} | Loss {:.3f} | Accuracy = {:.3f}'.format(epoch + 1, avg_val_loss, val_accuracy))\n",
        "            \n",
        "            val_logger_2.add_epoch_data(epoch+1, val_accuracy, avg_val_loss)\n",
        "            \n",
        "            if val_accuracy > min_accuracy:\n",
        "                print(\"[||| NEW BEST on val |||]\")\n",
        "                save_path_model = os.path.join(model_folder, model_checkpoint)\n",
        "                torch.save(model.state_dict(), save_path_model)\n",
        "                min_accuracy = val_accuracy\n",
        "            \n",
        "    optim_scheduler.step()\n",
        "\n",
        "    train_logger_2.save(train_log_file)\n",
        "    val_logger_2.save(val_log_file)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Train: Epoch = 1 | Loss = 34.050 | Loss2 = 0.145 | Accuracy = 86.250\n",
            "Val: Epoch = 1 | Loss 1.509 | Accuracy = 62.069\n",
            "[||| NEW BEST on val |||]\n",
            "Train: Epoch = 2 | Loss = 34.105 | Loss2 = 0.140 | Accuracy = 80.938\n",
            "Val: Epoch = 2 | Loss 1.516 | Accuracy = 61.207\n",
            "Train: Epoch = 3 | Loss = 34.112 | Loss2 = 0.140 | Accuracy = 81.250\n",
            "Val: Epoch = 3 | Loss 1.506 | Accuracy = 63.793\n",
            "[||| NEW BEST on val |||]\n",
            "Train: Epoch = 4 | Loss = 34.064 | Loss2 = 0.137 | Accuracy = 82.812\n",
            "Val: Epoch = 4 | Loss 1.487 | Accuracy = 62.931\n",
            "Train: Epoch = 5 | Loss = 34.042 | Loss2 = 0.144 | Accuracy = 81.250\n",
            "Val: Epoch = 5 | Loss 1.517 | Accuracy = 63.793\n",
            "Train: Epoch = 6 | Loss = 34.127 | Loss2 = 0.138 | Accuracy = 81.875\n",
            "Val: Epoch = 6 | Loss 1.554 | Accuracy = 59.483\n",
            "Train: Epoch = 7 | Loss = 34.148 | Loss2 = 0.141 | Accuracy = 80.000\n",
            "Val: Epoch = 7 | Loss 1.567 | Accuracy = 59.483\n",
            "Train: Epoch = 8 | Loss = 34.008 | Loss2 = 0.143 | Accuracy = 84.375\n",
            "Val: Epoch = 8 | Loss 1.546 | Accuracy = 60.345\n",
            "Train: Epoch = 9 | Loss = 34.072 | Loss2 = 0.142 | Accuracy = 82.500\n",
            "Val: Epoch = 9 | Loss 1.548 | Accuracy = 61.207\n",
            "Train: Epoch = 10 | Loss = 34.111 | Loss2 = 0.139 | Accuracy = 82.188\n",
            "Val: Epoch = 10 | Loss 1.561 | Accuracy = 60.345\n",
            "Train: Epoch = 11 | Loss = 34.165 | Loss2 = 0.142 | Accuracy = 80.938\n",
            "Val: Epoch = 11 | Loss 1.574 | Accuracy = 61.207\n",
            "Train: Epoch = 12 | Loss = 34.130 | Loss2 = 0.147 | Accuracy = 80.938\n",
            "Val: Epoch = 12 | Loss 1.537 | Accuracy = 59.483\n",
            "Train: Epoch = 13 | Loss = 34.074 | Loss2 = 0.145 | Accuracy = 81.562\n",
            "Val: Epoch = 13 | Loss 1.526 | Accuracy = 61.207\n",
            "Train: Epoch = 14 | Loss = 34.076 | Loss2 = 0.141 | Accuracy = 81.250\n",
            "Val: Epoch = 14 | Loss 1.561 | Accuracy = 59.483\n",
            "Train: Epoch = 15 | Loss = 34.009 | Loss2 = 0.138 | Accuracy = 82.812\n",
            "Val: Epoch = 15 | Loss 1.567 | Accuracy = 60.345\n",
            "Train: Epoch = 16 | Loss = 34.091 | Loss2 = 0.140 | Accuracy = 83.125\n",
            "Val: Epoch = 16 | Loss 1.555 | Accuracy = 59.483\n",
            "Train: Epoch = 17 | Loss = 34.067 | Loss2 = 0.142 | Accuracy = 83.125\n",
            "Val: Epoch = 17 | Loss 1.549 | Accuracy = 59.483\n",
            "Train: Epoch = 18 | Loss = 34.039 | Loss2 = 0.139 | Accuracy = 83.750\n",
            "Val: Epoch = 18 | Loss 1.534 | Accuracy = 60.345\n",
            "Train: Epoch = 19 | Loss = 34.056 | Loss2 = 0.145 | Accuracy = 84.062\n",
            "Val: Epoch = 19 | Loss 1.542 | Accuracy = 60.345\n",
            "Train: Epoch = 20 | Loss = 33.989 | Loss2 = 0.140 | Accuracy = 85.938\n",
            "Val: Epoch = 20 | Loss 1.547 | Accuracy = 60.345\n",
            "Train: Epoch = 21 | Loss = 34.043 | Loss2 = 0.140 | Accuracy = 84.688\n",
            "Val: Epoch = 21 | Loss 1.536 | Accuracy = 60.345\n",
            "Train: Epoch = 22 | Loss = 34.061 | Loss2 = 0.148 | Accuracy = 84.062\n",
            "Val: Epoch = 22 | Loss 1.536 | Accuracy = 58.621\n",
            "Train: Epoch = 23 | Loss = 34.033 | Loss2 = 0.139 | Accuracy = 83.438\n",
            "Val: Epoch = 23 | Loss 1.542 | Accuracy = 57.759\n",
            "Train: Epoch = 24 | Loss = 34.145 | Loss2 = 0.142 | Accuracy = 79.062\n",
            "Val: Epoch = 24 | Loss 1.548 | Accuracy = 59.483\n",
            "Train: Epoch = 25 | Loss = 34.091 | Loss2 = 0.136 | Accuracy = 83.125\n",
            "Val: Epoch = 25 | Loss 1.546 | Accuracy = 58.621\n",
            "Train: Epoch = 26 | Loss = 34.012 | Loss2 = 0.144 | Accuracy = 84.062\n",
            "Val: Epoch = 26 | Loss 1.548 | Accuracy = 57.759\n",
            "Train: Epoch = 27 | Loss = 34.136 | Loss2 = 0.145 | Accuracy = 81.250\n",
            "Val: Epoch = 27 | Loss 1.576 | Accuracy = 59.483\n",
            "Train: Epoch = 28 | Loss = 34.051 | Loss2 = 0.141 | Accuracy = 85.000\n",
            "Val: Epoch = 28 | Loss 1.584 | Accuracy = 59.483\n",
            "Train: Epoch = 29 | Loss = 34.007 | Loss2 = 0.143 | Accuracy = 84.688\n",
            "Val: Epoch = 29 | Loss 1.547 | Accuracy = 58.621\n",
            "Train: Epoch = 30 | Loss = 34.052 | Loss2 = 0.142 | Accuracy = 81.875\n",
            "Val: Epoch = 30 | Loss 1.517 | Accuracy = 59.483\n",
            "Train: Epoch = 31 | Loss = 34.103 | Loss2 = 0.137 | Accuracy = 82.500\n",
            "Val: Epoch = 31 | Loss 1.541 | Accuracy = 61.207\n",
            "Train: Epoch = 32 | Loss = 34.021 | Loss2 = 0.147 | Accuracy = 84.688\n",
            "Val: Epoch = 32 | Loss 1.550 | Accuracy = 62.069\n",
            "Train: Epoch = 33 | Loss = 33.982 | Loss2 = 0.145 | Accuracy = 86.250\n",
            "Val: Epoch = 33 | Loss 1.542 | Accuracy = 62.931\n",
            "Train: Epoch = 34 | Loss = 34.103 | Loss2 = 0.149 | Accuracy = 80.938\n",
            "Val: Epoch = 34 | Loss 1.521 | Accuracy = 64.655\n",
            "[||| NEW BEST on val |||]\n",
            "Train: Epoch = 35 | Loss = 34.032 | Loss2 = 0.138 | Accuracy = 84.688\n",
            "Val: Epoch = 35 | Loss 1.548 | Accuracy = 61.207\n",
            "Train: Epoch = 36 | Loss = 33.997 | Loss2 = 0.135 | Accuracy = 84.688\n",
            "Val: Epoch = 36 | Loss 1.570 | Accuracy = 61.207\n",
            "Train: Epoch = 37 | Loss = 34.060 | Loss2 = 0.144 | Accuracy = 83.438\n",
            "Val: Epoch = 37 | Loss 1.579 | Accuracy = 59.483\n",
            "Train: Epoch = 38 | Loss = 34.047 | Loss2 = 0.142 | Accuracy = 83.125\n",
            "Val: Epoch = 38 | Loss 1.560 | Accuracy = 58.621\n",
            "Train: Epoch = 39 | Loss = 34.083 | Loss2 = 0.141 | Accuracy = 82.188\n",
            "Val: Epoch = 39 | Loss 1.562 | Accuracy = 59.483\n",
            "Train: Epoch = 40 | Loss = 34.067 | Loss2 = 0.144 | Accuracy = 82.812\n",
            "Val: Epoch = 40 | Loss 1.584 | Accuracy = 59.483\n",
            "Train: Epoch = 41 | Loss = 34.056 | Loss2 = 0.140 | Accuracy = 85.000\n",
            "Val: Epoch = 41 | Loss 1.566 | Accuracy = 61.207\n",
            "Train: Epoch = 42 | Loss = 34.075 | Loss2 = 0.139 | Accuracy = 84.688\n",
            "Val: Epoch = 42 | Loss 1.576 | Accuracy = 60.345\n",
            "Train: Epoch = 43 | Loss = 34.080 | Loss2 = 0.139 | Accuracy = 81.562\n",
            "Val: Epoch = 43 | Loss 1.536 | Accuracy = 60.345\n",
            "Train: Epoch = 44 | Loss = 33.964 | Loss2 = 0.143 | Accuracy = 85.938\n",
            "Val: Epoch = 44 | Loss 1.543 | Accuracy = 59.483\n",
            "Train: Epoch = 45 | Loss = 34.046 | Loss2 = 0.133 | Accuracy = 82.500\n",
            "Val: Epoch = 45 | Loss 1.563 | Accuracy = 58.621\n",
            "Train: Epoch = 46 | Loss = 33.954 | Loss2 = 0.143 | Accuracy = 88.750\n",
            "Val: Epoch = 46 | Loss 1.560 | Accuracy = 58.621\n",
            "Train: Epoch = 47 | Loss = 34.019 | Loss2 = 0.147 | Accuracy = 86.562\n",
            "Val: Epoch = 47 | Loss 1.565 | Accuracy = 58.621\n",
            "Train: Epoch = 48 | Loss = 33.998 | Loss2 = 0.143 | Accuracy = 84.688\n",
            "Val: Epoch = 48 | Loss 1.571 | Accuracy = 59.483\n",
            "Train: Epoch = 49 | Loss = 34.024 | Loss2 = 0.148 | Accuracy = 84.062\n",
            "Val: Epoch = 49 | Loss 1.563 | Accuracy = 61.207\n",
            "Train: Epoch = 50 | Loss = 34.033 | Loss2 = 0.138 | Accuracy = 83.125\n",
            "Val: Epoch = 50 | Loss 1.549 | Accuracy = 59.483\n",
            "Train: Epoch = 51 | Loss = 33.984 | Loss2 = 0.140 | Accuracy = 85.312\n",
            "Val: Epoch = 51 | Loss 1.543 | Accuracy = 61.207\n",
            "Train: Epoch = 52 | Loss = 33.948 | Loss2 = 0.145 | Accuracy = 84.688\n",
            "Val: Epoch = 52 | Loss 1.564 | Accuracy = 62.069\n",
            "Train: Epoch = 53 | Loss = 33.996 | Loss2 = 0.151 | Accuracy = 86.250\n",
            "Val: Epoch = 53 | Loss 1.580 | Accuracy = 60.345\n",
            "Train: Epoch = 54 | Loss = 34.061 | Loss2 = 0.139 | Accuracy = 84.375\n",
            "Val: Epoch = 54 | Loss 1.545 | Accuracy = 59.483\n",
            "Train: Epoch = 55 | Loss = 34.047 | Loss2 = 0.146 | Accuracy = 83.125\n",
            "Val: Epoch = 55 | Loss 1.529 | Accuracy = 62.931\n",
            "Train: Epoch = 56 | Loss = 34.072 | Loss2 = 0.138 | Accuracy = 81.250\n",
            "Val: Epoch = 56 | Loss 1.534 | Accuracy = 62.069\n",
            "Train: Epoch = 57 | Loss = 34.095 | Loss2 = 0.143 | Accuracy = 81.875\n",
            "Val: Epoch = 57 | Loss 1.542 | Accuracy = 62.069\n",
            "Train: Epoch = 58 | Loss = 34.032 | Loss2 = 0.139 | Accuracy = 82.500\n",
            "Val: Epoch = 58 | Loss 1.527 | Accuracy = 61.207\n",
            "Train: Epoch = 59 | Loss = 34.042 | Loss2 = 0.136 | Accuracy = 83.438\n",
            "Val: Epoch = 59 | Loss 1.583 | Accuracy = 59.483\n",
            "Train: Epoch = 60 | Loss = 34.020 | Loss2 = 0.136 | Accuracy = 85.938\n",
            "Val: Epoch = 60 | Loss 1.578 | Accuracy = 59.483\n",
            "Train: Epoch = 61 | Loss = 34.016 | Loss2 = 0.141 | Accuracy = 85.312\n",
            "Val: Epoch = 61 | Loss 1.583 | Accuracy = 58.621\n",
            "Train: Epoch = 62 | Loss = 33.949 | Loss2 = 0.140 | Accuracy = 87.500\n",
            "Val: Epoch = 62 | Loss 1.565 | Accuracy = 58.621\n",
            "Train: Epoch = 63 | Loss = 33.959 | Loss2 = 0.136 | Accuracy = 86.562\n",
            "Val: Epoch = 63 | Loss 1.554 | Accuracy = 59.483\n",
            "Train: Epoch = 64 | Loss = 33.994 | Loss2 = 0.149 | Accuracy = 85.938\n",
            "Val: Epoch = 64 | Loss 1.583 | Accuracy = 60.345\n",
            "Train: Epoch = 65 | Loss = 34.061 | Loss2 = 0.138 | Accuracy = 83.438\n",
            "Val: Epoch = 65 | Loss 1.616 | Accuracy = 60.345\n",
            "Train: Epoch = 66 | Loss = 33.933 | Loss2 = 0.141 | Accuracy = 87.812\n",
            "Val: Epoch = 66 | Loss 1.589 | Accuracy = 62.931\n",
            "Train: Epoch = 67 | Loss = 34.011 | Loss2 = 0.139 | Accuracy = 83.750\n",
            "Val: Epoch = 67 | Loss 1.580 | Accuracy = 61.207\n",
            "Train: Epoch = 68 | Loss = 34.073 | Loss2 = 0.137 | Accuracy = 81.875\n",
            "Val: Epoch = 68 | Loss 1.608 | Accuracy = 58.621\n",
            "Train: Epoch = 69 | Loss = 34.017 | Loss2 = 0.134 | Accuracy = 83.438\n",
            "Val: Epoch = 69 | Loss 1.567 | Accuracy = 59.483\n",
            "Train: Epoch = 70 | Loss = 33.970 | Loss2 = 0.138 | Accuracy = 85.000\n",
            "Val: Epoch = 70 | Loss 1.550 | Accuracy = 59.483\n",
            "Train: Epoch = 71 | Loss = 33.969 | Loss2 = 0.137 | Accuracy = 85.312\n",
            "Val: Epoch = 71 | Loss 1.572 | Accuracy = 58.621\n",
            "Train: Epoch = 72 | Loss = 33.924 | Loss2 = 0.140 | Accuracy = 87.188\n",
            "Val: Epoch = 72 | Loss 1.602 | Accuracy = 61.207\n",
            "Train: Epoch = 73 | Loss = 34.064 | Loss2 = 0.141 | Accuracy = 86.250\n",
            "Val: Epoch = 73 | Loss 1.598 | Accuracy = 61.207\n",
            "Train: Epoch = 74 | Loss = 33.978 | Loss2 = 0.142 | Accuracy = 84.375\n",
            "Val: Epoch = 74 | Loss 1.579 | Accuracy = 62.069\n",
            "Train: Epoch = 75 | Loss = 34.026 | Loss2 = 0.142 | Accuracy = 84.062\n",
            "Val: Epoch = 75 | Loss 1.588 | Accuracy = 59.483\n",
            "Train: Epoch = 76 | Loss = 34.060 | Loss2 = 0.147 | Accuracy = 81.875\n",
            "Val: Epoch = 76 | Loss 1.571 | Accuracy = 60.345\n",
            "Train: Epoch = 77 | Loss = 33.990 | Loss2 = 0.143 | Accuracy = 85.625\n",
            "Val: Epoch = 77 | Loss 1.558 | Accuracy = 60.345\n",
            "Train: Epoch = 78 | Loss = 34.009 | Loss2 = 0.141 | Accuracy = 83.750\n",
            "Val: Epoch = 78 | Loss 1.553 | Accuracy = 61.207\n",
            "Train: Epoch = 79 | Loss = 33.992 | Loss2 = 0.147 | Accuracy = 83.438\n",
            "Val: Epoch = 79 | Loss 1.567 | Accuracy = 62.069\n",
            "Train: Epoch = 80 | Loss = 33.988 | Loss2 = 0.147 | Accuracy = 85.625\n",
            "Val: Epoch = 80 | Loss 1.575 | Accuracy = 61.207\n",
            "Train: Epoch = 81 | Loss = 34.002 | Loss2 = 0.144 | Accuracy = 84.688\n",
            "Val: Epoch = 81 | Loss 1.551 | Accuracy = 59.483\n",
            "Train: Epoch = 82 | Loss = 33.965 | Loss2 = 0.139 | Accuracy = 86.250\n",
            "Val: Epoch = 82 | Loss 1.538 | Accuracy = 60.345\n",
            "Train: Epoch = 83 | Loss = 33.934 | Loss2 = 0.138 | Accuracy = 88.125\n",
            "Val: Epoch = 83 | Loss 1.561 | Accuracy = 61.207\n",
            "Train: Epoch = 84 | Loss = 33.952 | Loss2 = 0.138 | Accuracy = 86.250\n",
            "Val: Epoch = 84 | Loss 1.585 | Accuracy = 57.759\n",
            "Train: Epoch = 85 | Loss = 33.979 | Loss2 = 0.148 | Accuracy = 86.875\n",
            "Val: Epoch = 85 | Loss 1.565 | Accuracy = 59.483\n",
            "Train: Epoch = 86 | Loss = 34.035 | Loss2 = 0.145 | Accuracy = 84.062\n",
            "Val: Epoch = 86 | Loss 1.567 | Accuracy = 60.345\n",
            "Train: Epoch = 87 | Loss = 34.035 | Loss2 = 0.140 | Accuracy = 85.000\n",
            "Val: Epoch = 87 | Loss 1.618 | Accuracy = 58.621\n",
            "Train: Epoch = 88 | Loss = 34.003 | Loss2 = 0.140 | Accuracy = 85.000\n",
            "Val: Epoch = 88 | Loss 1.611 | Accuracy = 60.345\n",
            "Train: Epoch = 89 | Loss = 34.053 | Loss2 = 0.143 | Accuracy = 83.438\n",
            "Val: Epoch = 89 | Loss 1.599 | Accuracy = 60.345\n",
            "Train: Epoch = 90 | Loss = 33.945 | Loss2 = 0.143 | Accuracy = 86.250\n",
            "Val: Epoch = 90 | Loss 1.590 | Accuracy = 56.897\n",
            "Train: Epoch = 91 | Loss = 34.010 | Loss2 = 0.144 | Accuracy = 87.188\n",
            "Val: Epoch = 91 | Loss 1.597 | Accuracy = 58.621\n",
            "Train: Epoch = 92 | Loss = 34.020 | Loss2 = 0.142 | Accuracy = 84.375\n",
            "Val: Epoch = 92 | Loss 1.601 | Accuracy = 59.483\n",
            "Train: Epoch = 93 | Loss = 34.004 | Loss2 = 0.138 | Accuracy = 84.688\n",
            "Val: Epoch = 93 | Loss 1.600 | Accuracy = 58.621\n",
            "Train: Epoch = 94 | Loss = 33.978 | Loss2 = 0.139 | Accuracy = 85.938\n",
            "Val: Epoch = 94 | Loss 1.607 | Accuracy = 59.483\n",
            "Train: Epoch = 95 | Loss = 33.986 | Loss2 = 0.144 | Accuracy = 85.625\n",
            "Val: Epoch = 95 | Loss 1.621 | Accuracy = 58.621\n",
            "Train: Epoch = 96 | Loss = 33.971 | Loss2 = 0.140 | Accuracy = 85.938\n",
            "Val: Epoch = 96 | Loss 1.574 | Accuracy = 59.483\n",
            "Train: Epoch = 97 | Loss = 34.007 | Loss2 = 0.136 | Accuracy = 87.188\n",
            "Val: Epoch = 97 | Loss 1.573 | Accuracy = 59.483\n",
            "Train: Epoch = 98 | Loss = 34.065 | Loss2 = 0.145 | Accuracy = 82.812\n",
            "Val: Epoch = 98 | Loss 1.559 | Accuracy = 60.345\n",
            "Train: Epoch = 99 | Loss = 34.015 | Loss2 = 0.145 | Accuracy = 82.812\n",
            "Val: Epoch = 99 | Loss 1.549 | Accuracy = 60.345\n",
            "Train: Epoch = 100 | Loss = 33.938 | Loss2 = 0.137 | Accuracy = 88.438\n",
            "Val: Epoch = 100 | Loss 1.567 | Accuracy = 59.483\n",
            "Train: Epoch = 101 | Loss = 34.028 | Loss2 = 0.141 | Accuracy = 85.938\n",
            "Val: Epoch = 101 | Loss 1.578 | Accuracy = 60.345\n",
            "Train: Epoch = 102 | Loss = 33.989 | Loss2 = 0.147 | Accuracy = 84.375\n",
            "Val: Epoch = 102 | Loss 1.612 | Accuracy = 59.483\n",
            "Train: Epoch = 103 | Loss = 33.977 | Loss2 = 0.141 | Accuracy = 84.688\n",
            "Val: Epoch = 103 | Loss 1.579 | Accuracy = 59.483\n",
            "Train: Epoch = 104 | Loss = 33.959 | Loss2 = 0.139 | Accuracy = 86.875\n",
            "Val: Epoch = 104 | Loss 1.553 | Accuracy = 61.207\n",
            "Train: Epoch = 105 | Loss = 33.957 | Loss2 = 0.140 | Accuracy = 88.750\n",
            "Val: Epoch = 105 | Loss 1.574 | Accuracy = 62.069\n",
            "Train: Epoch = 106 | Loss = 34.011 | Loss2 = 0.138 | Accuracy = 84.688\n",
            "Val: Epoch = 106 | Loss 1.577 | Accuracy = 60.345\n",
            "Train: Epoch = 107 | Loss = 33.961 | Loss2 = 0.137 | Accuracy = 87.812\n",
            "Val: Epoch = 107 | Loss 1.597 | Accuracy = 60.345\n",
            "Train: Epoch = 108 | Loss = 33.999 | Loss2 = 0.129 | Accuracy = 85.938\n",
            "Val: Epoch = 108 | Loss 1.610 | Accuracy = 60.345\n",
            "Train: Epoch = 109 | Loss = 34.042 | Loss2 = 0.139 | Accuracy = 80.625\n",
            "Val: Epoch = 109 | Loss 1.637 | Accuracy = 59.483\n",
            "Train: Epoch = 110 | Loss = 33.941 | Loss2 = 0.139 | Accuracy = 86.875\n",
            "Val: Epoch = 110 | Loss 1.600 | Accuracy = 60.345\n",
            "Train: Epoch = 111 | Loss = 34.027 | Loss2 = 0.142 | Accuracy = 85.625\n",
            "Val: Epoch = 111 | Loss 1.598 | Accuracy = 59.483\n",
            "Train: Epoch = 112 | Loss = 34.037 | Loss2 = 0.143 | Accuracy = 83.125\n",
            "Val: Epoch = 112 | Loss 1.579 | Accuracy = 58.621\n",
            "Train: Epoch = 113 | Loss = 33.914 | Loss2 = 0.144 | Accuracy = 89.062\n",
            "Val: Epoch = 113 | Loss 1.598 | Accuracy = 58.621\n",
            "Train: Epoch = 114 | Loss = 34.023 | Loss2 = 0.139 | Accuracy = 84.062\n",
            "Val: Epoch = 114 | Loss 1.580 | Accuracy = 61.207\n",
            "Train: Epoch = 115 | Loss = 33.949 | Loss2 = 0.142 | Accuracy = 88.125\n",
            "Val: Epoch = 115 | Loss 1.574 | Accuracy = 62.931\n",
            "Train: Epoch = 116 | Loss = 34.004 | Loss2 = 0.136 | Accuracy = 83.125\n",
            "Val: Epoch = 116 | Loss 1.623 | Accuracy = 60.345\n",
            "Train: Epoch = 117 | Loss = 33.928 | Loss2 = 0.138 | Accuracy = 85.000\n",
            "Val: Epoch = 117 | Loss 1.612 | Accuracy = 59.483\n",
            "Train: Epoch = 118 | Loss = 33.916 | Loss2 = 0.141 | Accuracy = 87.812\n",
            "Val: Epoch = 118 | Loss 1.626 | Accuracy = 58.621\n",
            "Train: Epoch = 119 | Loss = 33.967 | Loss2 = 0.141 | Accuracy = 87.188\n",
            "Val: Epoch = 119 | Loss 1.596 | Accuracy = 57.759\n",
            "Train: Epoch = 120 | Loss = 33.964 | Loss2 = 0.145 | Accuracy = 88.750\n",
            "Val: Epoch = 120 | Loss 1.566 | Accuracy = 57.759\n",
            "Train: Epoch = 121 | Loss = 33.935 | Loss2 = 0.139 | Accuracy = 86.250\n",
            "Val: Epoch = 121 | Loss 1.561 | Accuracy = 59.483\n",
            "Train: Epoch = 122 | Loss = 33.976 | Loss2 = 0.147 | Accuracy = 87.812\n",
            "Val: Epoch = 122 | Loss 1.602 | Accuracy = 58.621\n",
            "Train: Epoch = 123 | Loss = 34.000 | Loss2 = 0.139 | Accuracy = 84.375\n",
            "Val: Epoch = 123 | Loss 1.627 | Accuracy = 58.621\n",
            "Train: Epoch = 124 | Loss = 33.954 | Loss2 = 0.147 | Accuracy = 84.062\n",
            "Val: Epoch = 124 | Loss 1.634 | Accuracy = 58.621\n",
            "Train: Epoch = 125 | Loss = 33.960 | Loss2 = 0.142 | Accuracy = 85.000\n",
            "Val: Epoch = 125 | Loss 1.620 | Accuracy = 58.621\n",
            "Train: Epoch = 126 | Loss = 33.865 | Loss2 = 0.136 | Accuracy = 90.000\n",
            "Val: Epoch = 126 | Loss 1.598 | Accuracy = 58.621\n",
            "Train: Epoch = 127 | Loss = 33.974 | Loss2 = 0.144 | Accuracy = 88.438\n",
            "Val: Epoch = 127 | Loss 1.618 | Accuracy = 58.621\n",
            "Train: Epoch = 128 | Loss = 33.881 | Loss2 = 0.144 | Accuracy = 89.688\n",
            "Val: Epoch = 128 | Loss 1.659 | Accuracy = 57.759\n",
            "Train: Epoch = 129 | Loss = 34.027 | Loss2 = 0.148 | Accuracy = 84.688\n",
            "Val: Epoch = 129 | Loss 1.647 | Accuracy = 56.034\n",
            "Train: Epoch = 130 | Loss = 33.971 | Loss2 = 0.143 | Accuracy = 85.938\n",
            "Val: Epoch = 130 | Loss 1.600 | Accuracy = 59.483\n",
            "Train: Epoch = 131 | Loss = 33.926 | Loss2 = 0.138 | Accuracy = 87.812\n",
            "Val: Epoch = 131 | Loss 1.583 | Accuracy = 58.621\n",
            "Train: Epoch = 132 | Loss = 33.969 | Loss2 = 0.145 | Accuracy = 82.500\n",
            "Val: Epoch = 132 | Loss 1.589 | Accuracy = 60.345\n",
            "Train: Epoch = 133 | Loss = 33.930 | Loss2 = 0.146 | Accuracy = 86.562\n",
            "Val: Epoch = 133 | Loss 1.604 | Accuracy = 58.621\n",
            "Train: Epoch = 134 | Loss = 33.947 | Loss2 = 0.144 | Accuracy = 86.562\n",
            "Val: Epoch = 134 | Loss 1.612 | Accuracy = 59.483\n",
            "Train: Epoch = 135 | Loss = 33.916 | Loss2 = 0.140 | Accuracy = 88.125\n",
            "Val: Epoch = 135 | Loss 1.614 | Accuracy = 56.897\n",
            "Train: Epoch = 136 | Loss = 33.999 | Loss2 = 0.135 | Accuracy = 86.250\n",
            "Val: Epoch = 136 | Loss 1.603 | Accuracy = 59.483\n",
            "Train: Epoch = 137 | Loss = 33.955 | Loss2 = 0.141 | Accuracy = 87.812\n",
            "Val: Epoch = 137 | Loss 1.588 | Accuracy = 58.621\n",
            "Train: Epoch = 138 | Loss = 33.973 | Loss2 = 0.132 | Accuracy = 86.250\n",
            "Val: Epoch = 138 | Loss 1.580 | Accuracy = 58.621\n",
            "Train: Epoch = 139 | Loss = 33.991 | Loss2 = 0.143 | Accuracy = 83.750\n",
            "Val: Epoch = 139 | Loss 1.589 | Accuracy = 58.621\n",
            "Train: Epoch = 140 | Loss = 33.924 | Loss2 = 0.145 | Accuracy = 88.438\n",
            "Val: Epoch = 140 | Loss 1.580 | Accuracy = 59.483\n",
            "Train: Epoch = 141 | Loss = 33.943 | Loss2 = 0.146 | Accuracy = 87.500\n",
            "Val: Epoch = 141 | Loss 1.578 | Accuracy = 60.345\n",
            "Train: Epoch = 142 | Loss = 33.958 | Loss2 = 0.139 | Accuracy = 87.500\n",
            "Val: Epoch = 142 | Loss 1.600 | Accuracy = 56.034\n",
            "Train: Epoch = 143 | Loss = 33.974 | Loss2 = 0.149 | Accuracy = 86.562\n",
            "Val: Epoch = 143 | Loss 1.592 | Accuracy = 59.483\n",
            "Train: Epoch = 144 | Loss = 33.909 | Loss2 = 0.135 | Accuracy = 89.688\n",
            "Val: Epoch = 144 | Loss 1.576 | Accuracy = 60.345\n",
            "Train: Epoch = 145 | Loss = 33.940 | Loss2 = 0.143 | Accuracy = 85.938\n",
            "Val: Epoch = 145 | Loss 1.595 | Accuracy = 60.345\n",
            "Train: Epoch = 146 | Loss = 33.947 | Loss2 = 0.139 | Accuracy = 87.188\n",
            "Val: Epoch = 146 | Loss 1.621 | Accuracy = 59.483\n",
            "Train: Epoch = 147 | Loss = 33.912 | Loss2 = 0.145 | Accuracy = 86.250\n",
            "Val: Epoch = 147 | Loss 1.625 | Accuracy = 60.345\n",
            "Train: Epoch = 148 | Loss = 33.934 | Loss2 = 0.146 | Accuracy = 86.875\n",
            "Val: Epoch = 148 | Loss 1.609 | Accuracy = 60.345\n",
            "Train: Epoch = 149 | Loss = 33.938 | Loss2 = 0.145 | Accuracy = 87.812\n",
            "Val: Epoch = 149 | Loss 1.572 | Accuracy = 59.483\n",
            "Train: Epoch = 150 | Loss = 33.934 | Loss2 = 0.135 | Accuracy = 87.812\n",
            "Val: Epoch = 150 | Loss 1.568 | Accuracy = 62.069\n"
          ],
          "name": "stdout"
        }
      ]
    }
  ]
}