test_pipeline_MS.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 122
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 19359,
     "status": "ok",
     "timestamp": 1591190175602,
     "user": {
      "displayName": "Eros Fanì",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14GhapRbf5lUkHjaoCtjruM95BBemedtJFkNrcJgVKA=s64",
      "userId": "05010694830681132177"
     },
     "user_tz": -120
    },
    "id": "msj1pGs6NzrT",
    "outputId": "e283946d-6643-4b41-9bcf-da81686442a4"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Go to this URL in a browser: https://accounts.google.com/o/oauth2/auth?client_id=947318989803-6bn6qk8qdgf4n4g3pfee6491hc0brc4i.apps.googleusercontent.com&redirect_uri=urn%3aietf%3awg%3aoauth%3a2.0%3aoob&response_type=code&scope=email%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdocs.test%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdrive%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdrive.photos.readonly%20https%3a%2f%2fwww.googleapis.com%2fauth%2fpeopleapi.readonly\n",
      "\n",
      "Enter your authorization code:\n",
      "··········\n",
      "Mounted at /content/drive\n"
     ]
    }
   ],
   "source": [
    "from google.colab import drive\n",
    "drive.mount('/content/drive')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 68
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 22151,
     "status": "ok",
     "timestamp": 1591190178407,
     "user": {
      "displayName": "Eros Fanì",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14GhapRbf5lUkHjaoCtjruM95BBemedtJFkNrcJgVKA=s64",
      "userId": "05010694830681132177"
     },
     "user_tz": -120
    },
    "id": "t201aJ1PN7gL",
    "outputId": "e0c89191-140e-43c0-e465-cdc8547b5c37"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/content/drive/My Drive\n",
      "fatal: destination path 'FPAR-Project-MLDL' already exists and is not an empty directory.\n",
      "/content/drive/My Drive/FPAR-Project-MLDL\n"
     ]
    }
   ],
   "source": [
    "%cd drive/My\\ Drive/\n",
    "!git clone https://[username]:[password]@github.com/Erosinho13/FPAR-Project-MLDL.git\n",
    "%cd FPAR-Project-MLDL/"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "oOsUmN07PH3Y"
   },
   "source": [
    "# SETUP\n",
    "\n",
    "The following blocks are to be executed first of anything else, to setup import, classes, functions and constants that are needed for all stages"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-06-20T17:49:48.192539Z",
     "start_time": "2020-06-20T17:49:46.373763Z"
    },
    "colab": {},
    "colab_type": "code",
    "id": "i4QGG9WMObiq"
   },
   "outputs": [],
   "source": [
    "import os\n",
    "import logging\n",
    "\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "import torch.optim as optim\n",
    "from torch.utils.data import DataLoader\n",
    "from torch.backends import cudnn\n",
    "\n",
    "import torchvision\n",
    "from torchvision import transforms\n",
    "from torchvision.models import resnet34\n",
    "\n",
    "from PIL import Image\n",
    "from tqdm import tqdm\n",
    "\n",
    "from gtea_dataset import GTEA61, GTEA61_flow, GTEA61_2Stream\n",
    "from AttentMS2 import attention_model_ms\n",
    "from flow_resnet import flow_resnet34\n",
    "from twoStreamModel import twoStreamAttentionModel\n",
    "from spatial_transforms import (Compose, ToTensor, CenterCrop, Scale, Normalize, MultiScaleCornerCrop,\n",
    "                                RandomHorizontalFlip)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-06-20T17:50:03.585934Z",
     "start_time": "2020-06-20T17:50:03.578919Z"
    },
    "colab": {},
    "colab_type": "code",
    "id": "O7LUJwx9OcXf"
   },
   "outputs": [],
   "source": [
    "DEVICE = 'cuda' # 'cuda' or 'cpu'\n",
    "NUM_CLASSES = 61 # 101 + 1: There is am extra Background class that should be removed \n",
    "\n",
    "BATCH_SIZE = 1     # Higher batch sizes allows for larger learning rates. An empirical heuristic suggests that, when changing\n",
    "                     # the batch size, learning rate should change by the same factor to have comparable results\n",
    "\n",
    "RGB_PREFIX = 'model_rgb_state_dict'\n",
    "FLOW_PREFIX = 'model_flow_state_dict'\n",
    "JOINT_PREFIX = 'model_twoStream_state_dict'\n",
    "LOG_PREFIX = 'log_stage'\n",
    "VAL_LOG_PREFIX = 'val_log_stage'\n",
    "DATA_DIR = '../GTEA61'\n",
    "model_folder = '../saved_models'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "GSxYa2WIOhA_"
   },
   "outputs": [],
   "source": [
    "from sklearn.metrics import confusion_matrix\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "\n",
    "def plot_confusion_matrix(true, predicted):\n",
    "    true = torch.cat(true, dim=0)\n",
    "    predicted =[p.to('cpu') for p in  predicted]\n",
    "    predicted = torch.cat(predicted, dim=0) \n",
    "    cnf_matrix = confusion_matrix(true, predicted, normalize='true').astype(float)\n",
    "\n",
    "    ticks = np.linspace(0, 60, num=61)\n",
    "    plt.figure(figsize=(8,8))\n",
    "    plt.imshow(cnf_matrix, interpolation='none', cmap='binary')\n",
    "    plt.colorbar()\n",
    "    plt.xticks(ticks, fontsize=6)\n",
    "    plt.yticks(ticks, fontsize=6)\n",
    "    plt.grid(True)\n",
    "    plt.clim(0, 1)\n",
    "    plt.show()\n",
    "    \n",
    "    return cnf_matrix\n",
    "    \n",
    "def generate_model_checkpoint_name(stage, n_frames, ms_block=False, loss=None, optional=''):\n",
    "    name = \"\"\n",
    "    if stage < 3:\n",
    "        name += RGB_PREFIX\n",
    "        if stage == 2:\n",
    "            name += '_stage2'\n",
    "    elif stage == 3:\n",
    "        name += FLOW_PREFIX\n",
    "    else:\n",
    "        name += JOINT_PREFIX\n",
    "    name += '_'+str(n_frames)+'frames'\n",
    "    if loss is not None:\n",
    "            name += '_'+loss\n",
    "    if ms_block:\n",
    "        name += '_msblock'\n",
    "    name += optional+\".pth\"\n",
    "    \n",
    "    return name"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-06-20T17:51:12.498926Z",
     "start_time": "2020-06-20T17:51:12.493780Z"
    },
    "colab": {},
    "colab_type": "code",
    "id": "UIQgPmyfO7P1"
   },
   "outputs": [],
   "source": [
    "# Data loader\n",
    "normalize = Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225])\n",
    "\n",
    "val_transform = Compose([Scale(256), CenterCrop(224), ToTensor(), normalize])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Discriminator specific setup"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-06-20T17:51:01.028493Z",
     "start_time": "2020-06-20T17:51:01.024287Z"
    }
   },
   "outputs": [],
   "source": [
    "STAGE = 2\n",
    "\n",
    "MEM_SIZE = 512\n",
    "SEQ_LEN = 7"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-06-20T17:51:13.478414Z",
     "start_time": "2020-06-20T17:51:13.462386Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test Dataset: 116\n"
     ]
    }
   ],
   "source": [
    "# Prepare Pytorch train/test Datasets\n",
    "test_dataset = GTEA61(DATA_DIR, split='test', transform=val_transform, seq_len=SEQ_LEN)\n",
    "\n",
    "#print('Valid Dataset: {}'.format(len(val_dataset)))\n",
    "print('Test Dataset: {}'.format(len(test_dataset)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Dataloaders iterate over pytorch datasets and transparently provide useful functions (e.g.\n",
    "test_loader = DataLoader(test_dataset, batch_size=BATCH_SIZE, shuffle=False, num_workers=4)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Prepare discriminator evaluation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "best_model_file = generate_model_checkpoint_name(STAGE, SEQ_LEN, ms_block=True, optional='_2neur_DS_alpha_0.1')\n",
    "best_model = os.path.join(model_folder, best_model_file)\n",
    "model = attention_model_ms(num_classes=NUM_CLASSES, mem_size=MEM_SIZE)\n",
    "model.load_state_dict(torch.load(best_model))\n",
    "\n",
    "for params in model.parameters():\n",
    "    params.requires_grad = False\n",
    "\n",
    "model.train(False)\n",
    "model.to(DEVICE)\n",
    "test_samples = len(test_dataset)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-06-20T17:50:48.998633Z",
     "start_time": "2020-06-20T17:50:48.994738Z"
    }
   },
   "source": [
    "# Discriminator evaluation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print('Number of samples = {}'.format(test_samples))\n",
    "print('Evaluating...')\n",
    "numCorr = 0\n",
    "\n",
    "true_labels = []\n",
    "predicted_labels = []\n",
    "for inputs, targets in test_loader:\n",
    "    inputVariable = inputs.permute(1, 0, 2, 3, 4).to(DEVICE)\n",
    "    labelVariable = targets.to(DEVICE)\n",
    "    \n",
    "    output_label, _ = model(inputVariable)\n",
    "    _, predicted = torch.max(output_label.data, 1)\n",
    "    numCorr += torch.sum(predicted == labelVariable.data).data.item()\n",
    "    # store results in a list to plot confusion matrix later\n",
    "    true_labels.append(targets)\n",
    "    predicted_labels.append(predicted)\n",
    "                \n",
    "test_accuracy = (numCorr / test_samples) * 100\n",
    "print('Test Accuracy = {:.3f}%'.format(test_accuracy))\n",
    "mat = plot_confusion_matrix(true_labels, predicted_labels)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Save results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-06-20T17:55:09.843367Z",
     "start_time": "2020-06-20T17:55:09.833378Z"
    }
   },
   "outputs": [],
   "source": [
    "import pickle\n",
    "class StoreMatrix():\n",
    "    def __init__(self, lab_map, matrix):\n",
    "        self.mat = matrix\n",
    "        self.lab_map = lab_map\n",
    "    \n",
    "    def save(self, path):\n",
    "        with open(path, 'wb') as logfile:\n",
    "            pickle.dump(self, logfile)\n",
    "    \n",
    "    @classmethod\n",
    "    def load(cls, path):\n",
    "        with open(path, 'rb') as logfile:\n",
    "            new_instance = pickle.load(logfile)\n",
    "        return new_instance"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "mat = StoreMatrix(test_dataset.label_map, mat)\n",
    "mat.save(os.path.join(model_folder, \"conf_matrix.obj\"))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "6VLSKx94PQQs"
   },
   "source": [
    "# ConvLSTM + attention + MS specific-setup"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "W5ikmsvrO-8y"
   },
   "outputs": [],
   "source": [
    "STAGE = 2\n",
    "\n",
    "MEM_SIZE = 512\n",
    "SEQ_LEN = 7"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 162635,
     "status": "ok",
     "timestamp": 1591190352651,
     "user": {
      "displayName": "Eros Fanì",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14GhapRbf5lUkHjaoCtjruM95BBemedtJFkNrcJgVKA=s64",
      "userId": "05010694830681132177"
     },
     "user_tz": -120
    },
    "id": "5w-KB_IJPY23",
    "outputId": "e16e9b4c-b06c-4da5-e4d6-6090b035e5cf"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test Dataset: 116\n"
     ]
    }
   ],
   "source": [
    "# Prepare Pytorch train/test Datasets\n",
    "test_dataset = GTEA61(DATA_DIR, split='test', transform=val_transform, seq_len=SEQ_LEN)\n",
    "\n",
    "#print('Valid Dataset: {}'.format(len(val_dataset)))\n",
    "print('Test Dataset: {}'.format(len(test_dataset)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "Yfdg0CpgPeBo"
   },
   "outputs": [],
   "source": [
    "# Dataloaders iterate over pytorch datasets and transparently provide useful functions (e.g.\n",
    "test_loader = DataLoader(test_dataset, batch_size=BATCH_SIZE, shuffle=False, num_workers=4)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "6SIR4Z5oP5Zy"
   },
   "source": [
    "# Prepare evaluation for ConvLSTM + attention + MS"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "lEITI06eP2fW"
   },
   "outputs": [],
   "source": [
    "best_model_file = generate_model_checkpoint_name(STAGE, SEQ_LEN, ms_block = True)\n",
    "best_model = os.path.join(model_folder, best_model_file)\n",
    "model = attention_model_ms(num_classes=NUM_CLASSES, mem_size=MEM_SIZE)\n",
    "model.load_state_dict(torch.load(best_model))\n",
    "\n",
    "for params in model.parameters():\n",
    "    params.requires_grad = False\n",
    "\n",
    "model.train(False)\n",
    "model.to(DEVICE)\n",
    "test_samples = len(test_dataset)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "u05dR8GrQW-O"
   },
   "source": [
    "# ConvLSTM + attention + MS evaluation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 527
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 137703,
     "status": "ok",
     "timestamp": 1591190611375,
     "user": {
      "displayName": "Eros Fanì",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14GhapRbf5lUkHjaoCtjruM95BBemedtJFkNrcJgVKA=s64",
      "userId": "05010694830681132177"
     },
     "user_tz": -120
    },
    "id": "uGPTJAueQZU4",
    "outputId": "20881118-5476-4549-8524-c70282d69a00"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of samples = 116\n",
      "Evaluating...\n",
      "Test Accuracy = 64.655%\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAcUAAAHLCAYAAAC5y4P0AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+j8jraAAAgAElEQVR4nO29ebgcVb3u//kKJGEMCcEQCCQhE4QxbBQUvUlUMMgRo0wm3BNEBOWAl5BAMAgnEQdEwyCDB3JAUB5zFC6eiMjkT7MFOQ5kH1AU7gmDoMGARiAKaGT4/v6o6k6l2bV7d++q7lXV7+d56kn1u1eteqvSXatX9VtrmbsjhBBCCHhTuw0IIYQQoaBGUQghhIhRoyiEEELEqFEUQgghYtQoCiGEEDFqFIUQQogYNYpCCCEKh5l9zcz+aGa/Tvm7mdnlZvaYmf3KzA7oT71qFIUQQhSRG4CZffz9cGBivJwC/Ft/KlWjKIQQonC4+z3Ac30U+QDwDY/4GbC9mY2qV68aRSGEEGVkF+D3iddrYq1PNs/NjhBCiNJjZnmNFfob4O+J18vcfVlO+6qiRlEIIUSI/N3dDxzA9k8DuyZej461PtHtUyGEEAPCzDJfMuBWYG6cQj0YWO/ua+ttpJ6iEEKIwmFm/wFMB0aY2RpgMbAFgLtfDdwOvA94DHgZOLFf9WrqKCGEEM1iZv6mN2V/0/H111/vGeDt06bQ7VMhhBAiRrdPhRBCDIiMfgMMAjWKQgghBkSZGkXdPhVCCCFi1FMUQgjRNBk+QhEE6ikKIYQQMeopCiGEGBBl6imqURRCCDEgytQo6vapEEIIEaOeohBCiAGhnqIQQghRQtRTFEIIMSDK1FNUoyiEEKJp9JyiEEIIUVLUUxRCCDEg1FMUQgghSoh6ikIIIQaEeopCCCFECVFPUQghxIAoU09RjaIQQogBUaZGUbdPhRBCiBj1FIUQQjSNHt4XQgghSop6ikIIIQZEmXqKahSFEEIMiDI1irp9KoQQQsSopyiEEGJAqKcohBBClBD1FIUQQgyIMvUU1SgKIYRoGj2nKIQQQpQU9RSFEEIMCPUUhRBCiBKinqIQQogBUaaeYm6Nopl5ZX3UqFGsXbsWgK6urmqZl156ia233voN2/amN1K2XXpIXuSxczyG5EUew/X45JNPsm7dulxarzI1irh7QwtwAPA54BJg6z7KeWVZunRpdT3JypUrvTd60xsp2y49JC9pekhe0vSQvKTp8iKPRfPY1dXl3uD1vj/L5ptv7iNHjsx8AVbl4bfu8TTRjs4GFgFvBw4FVgy4ZRZCCFFYytRTNHevXyq5gdmXgU8BhwDD3X1F4m+nAKcADB06tOv8888HYPTo0axZswbY9Pbpiy++yDbbbPOGffSmN1K2XXpIXuSxczyG5EUew/V41llnsWrVqsxbry222MJHjBiRdbU888wzPe5+YOYV16PRriXQBXwWuBjYpo9yun0aoB6SlzQ9JC9purzIY9E85nX7dIsttvBRo0ZlvtCm26cNP5Lh7j3ufr67L3D3F9PKdXV1VXeSXBeiUSojZvT09FTXy3S7RggRDnokQwghxIAo05fUhnuKZjbJzG4ws1l5GBJCCFEskndwslraRTO3T1cDN2RvRQghhGgvDadPAcxsOrC9J5KnsV5Nn44cObLrW9/6FtD+1FWr9JC8lMljT08PsGmKGTYmmUPw2E49JC/yGK7HvNKngwYN8pEjR2ZdLWvWrClM+nQn4ErgOmBMWrk46dTvZFQ9PaQEWJoekpc0PSQvaXqtRi8pZhJJ5hA8ykvfekhe0vSQvKTpoaZPR48enflCUR7ed/dngNOzapQrHHvssdX1mTNn8tWvfhWAm266KetdiTZz8803V9c333zz6utjjjmm1/Ie383o7u5WglmIAClT0EbpUyGEEE3T7mBM1jTcKJrZO4mGeJsILHT35zJ3JYQQQrSBpoI2AGZ2HrDc3Z9IaE0HbZ54oloNQ4cOZf369QDsvvvu/a6j3XpIXkL2+Pzzz1d1M6veEh02bFgwHuvp8iKPRfOYV9Bm8ODBPmrUqKyr5amnnipG0Ca+gM0Bju+rTKNBm2OOOaa6XHfdddX1Rupotx6SlzQ9BC833XRTdfnOd75TXQ/JYz1dXuSxaB7zCtoMGjTIx4wZk/lCUYZ5M7NjgbnAjmY2JtsmWgghRNEo08P7zaRPbwIyj4QmU6bd3d3V15WTs3TpUmbMmFHxkPXuRQtJpky7u7uZPn16+8wIIQZMmYI2DfcUhRBCiLLSTPp0L+C9wDjgM+6+LnNXQgghCkOZeorNDvN2PHA08BF3X5/QMx/mrbchvpqdqDhvPSQv8tg5HkPyIo/heswzfTp69Oisq+WJJ54oTvo0bkiPAPZJ+3tWw7yR4UTFeesheUnTQ/KSpofkJU2XF3ksmse80qeDBw/28ePHZ75QoPTpTDNbCBwJ/Cm75rl3KkaTExUnE0rJiWeFEEKIgdBM+vRO4M4cvAghhCggZeqUNNNTPMLMbs3DjBBCiOJRpucUG2oUzWwqMAR4ol5ZIYQQomg0lD41s3OBl4EPAvPc/YGav7dkkuFKIhXCSqWGlFKTx87xGJIXeQzXY17p0yFDhviYMdkPbrZ69eripE+By+qVyXOSYRITzYaUSg0ppZamh+QlTQ/JS5ouL/JYNI95pk8nTZqU+UJRJhmOG9J5WTXKTe6/up6ceLbRyWuFECJvKteh5DUpSa1exOtVmYI2mmRYCCFE07Q7GJM1zQzzNguYAfwW+Ionu21CCCFEgWmmp/gS8FdgC6L06muZOhJCCFEoytRTbGrsUwAz+xDwgrv/KKG1JH2apteb0T0rLwPx2G49JC/yWD4v8vhGvXJdSl6TktTqletVUdKnW265pY8bNy7rannkkUeKkT4FpgPnAFcDO6SVyzN9mqbXm9E9Ky8D8dhuPSQvaXpIXtJ0eZHH/uq9XZPSrlXJ61VR0qdDhgzxKVOmZL5QlPSpu3cD3dk1y9mRNnltsmuvyYqFEK2kcl1Km1C7DBNtl+n2qSYZFkIIIWKaSZ+OA04AXgSucfe/Zu5KCCFEYej0nuLHgefi9Vcy9CKEEEK0lYbTp2Z2GfDvwGhgG3e/JfG3tqZP0/RWjZUaUpJOHjvHY0he5DFcj3mlT7faaiufMGFC1tXy0EMPFSZ9uj9wEXAJsEtauXakT9N0WjRWakhJujQ9JC9pekhe0nR5kceiecwrfbrlllv6Pvvsk/lCm9KnDd8+dfcH3f0cd5/v7k9n1jrnSPKAu7q6kg28EKKEVIYe6+npKd0wZCFSpvkUNfapEEKIAVGmLx0N9xTNbJqZzTOzbjMbm70lIYQQoj008/D+j83sJ8Bkd38ye0tCCCGKRJl6ik2NfWpmRwEvu/sdNXqQ6dNW6SF5kcfO8RiSl1A8VhLnjaTNQzpfeXjMM306efLkrKvlwQcfrJs+NbOZwFeAzYBr3f2LNX/fDfg6sH1c5lPufnufO24mnQMsI25Q05aQ0qet0kPykqaH5CVND8lLmi4vYXukibR5SOcrTQ81fTp16tTMF+qkT+NG7nFgd2AQ8EtgSk2ZZcCp8foU4Ml6x9NU0MbdT2lmOyFEOkuWLKmuT548ufo6qee53+Q+W7HfPIkvgnR3dytpnjNtTIu+FXjM3Z+IfXwL+ADwcKKMA9vF60OBP9SrVOlTIYQQRWQX4PeJ12uAg2rKLAHuNrNPAlsD76lXaTPp0/eb2RfM7KtmNr7R7YUQQpSLnJ5THGFmqxJLM3coZwM3uPto4H3AjWbWZ7vXTE/xb8BwYAjwxya2F0IIIeqxzvsO2jwN7Jp4PTrWkpwEzARw95+a2RBgBH20Xc2MfXoacA1wCLCtu9+W+JvSp4F4kcfieVy7dm1VHzx4MBs2bABg1KhRuXqp7De5z+R+Qz1fWeshecnDY17p06233tqnTJmSdbWsWrWqz/SpmW0OrAbeTdQY3g/McfffJMrcAXzb3W8wsz2BHxINT5re8DWaNAKOI7pPeymwW1o5pU/D1EPykqaH5CVNz6PuxYsXV5fly5dX1/P20ts+k/sN9XxlrYfkJU0PMX261VZb+Vve8pbMF/ox9inRLdHVRCnUT8faBcCRvjFxeh9RMvVB4LB6dTbz8P63G92mLCQTVkuXLmXGjBnAxqSbEAMhmfbs7u5m9uzZLd1vK/cpRBZ49Mzh7TXavybWHya6q9lvlD4VQggxIMo0ok3DjaKZHU70fMgORKMDvJy5KyGEEKINNBO0+QqwCDgRWOPu3038rdRBm3qTFYfgsZ4ekhd5LJ8XeQzXY15Bm2222cb33nvvrKvl5z//eWEmGZ4AnAVcAbwvrVwZgzbUmaw4BI/19JC8pOkheUnT5UUei+Yxr6DN1ltv7QcddFDmC22aZLiZ3xQHx//+Ebh7oI2yEEKIYlOm3xQbHtHG3X/j7kvd/bPu/moepkIl+W2iq6urut7bLN9lepMIIURf5DSiTVtouFEUQgghykrd26dmNgk4F1gB/AOYSjTa+DnuekBPCCE6nTLdGetX+tTMphNN0jjN3c80s7nAr9z9wZpypU6fpum9TWgKYaZSQ/Iij+XzIo/heswzfbrffvtlXS3/9V//FW76FJgOzAIujV/PBfbva5sypk/TdHpJpBJoKjUkL2l6SF7SdHmRx6J5zDN9esghh2S+EGr61Mx2Ao4GtgTuMbNziW6f3ph5Cy2EEKJQtDsYkzV1G0V3fwY4vQVeCou7ZvkWojcqF8vkWMGg8YJFuNRNn5rZJDO7wcxmmdlbzWyFme3fCnNCCCHCp6MeyXD31cAN8foviFKoQgghROloKH3q7ivM7CPAg16TPI3LdWT6NEQv8tg5HkPyUqsrmR2Ox7zSp9tuu60fcMABWVfLPffcE2b6FNgJuBK4DpgGLAe+DAzra7tOSp+G6CVND8lLmh6SlzRdXpTMLprHvNKn22yzjU+bNi3zhTalT/tz+/QZdz/d3U9y9x+7+xx3P9vdn8+nmRYie5K/VSSH4xP5UrnQJIdFdIVsRMBokmEhhBADokxfMBsd5u1VYG9gd+A0d38lX3tCCCFE62g0fXqbu38R2AAMyteaEEKI0MnjcYx29jybSZ+eATzi7m+YS1Hp03C8yGPvKUjYNAlZSUGG4LFZPSQv8hiux7zSp9ttt50feGD2IdGVK1cWIn26GPgPYB5Knwaxz0b1kLyk6XmmIKlJQobksQxe0vSQvKTpIXlJ00NMn2677bb+rne9K/OFUMc+dQ3z1hKStwuSQ2K5knqZkDyPGo5PiGwpU9BGkwwLIYQQMY2mT58F3g6MBc5091dzdSeEECJ4Oqqn6JumT38KvAK8GXg9V2dCCCFEi2k4fRq//hiwwt3X1ZRT+jSQdGQeHlulh+SlCB5D8iKP4XrMM3168MEHZ10tP/jBDwqRPj0BOBtYBmzZ13ZKnzamk3E6Mg+PrdJD8pKmy4s8Fs1jXunT7bbbzg877LDMF5Q+7Ww8JR2pVKoQQrQOjX0qhBBiQHRU0MbMJpnZDWY2K34918yuz9+aEEII0Vr6c/t0tZndAGxvZocCTwHr8zYmhBCiGJSpp9hQ+hQ4GHgG+CDwUXd/vKac0qcaszPI81hmjyF5kcdwPeaVPh06dKgfcsghWVfLHXfcUYj06ZhYu6zedkqfZqNT4jE70/SQvKTp8iKPRfOYZ/r08MMPz3yhTenT/jy8/4y7n+7uJ7n7U7E2L5cWWryB5H9WcvZyzSQvhAiFMk0dpbFPhRBCiJhGxz4dCwwF1rj7dflaE0IIETrt7tllTUNjnwLPA/8AhuToSQghhGgLzY59Op9o7NMnasopfdqifTabSg3pfMlj+bzIY7ge80qfbr/99v7Od74z62q57bbbCpE+PQFYFL8e1Nd2Sp/mq9NkKjWk85Wmh+QlTZcXeSyax7zSp0OHDvX3v//9mS9o7FPRCO4aK1UIIbJGY58KIYQYEGUK2jSaPl0DvBf4m7tfkrM3IYQQoqU0OszbYcDDRKnVK7xmYwVt2u+lXgAnBI/19JC8FMFjSF7kMVyPeQZtpk+fnnW1fPe73w0zaBO3e9OBWcB3gG2AjwNdfW2joE17dOoEcELwWE8PyUuaLi/yWDSPeQVttt9+e581a1bmC6EGbcxsJ+BoYEvgu8ACYBjwzawbaCGEEKKdKH1aMjwllSpEGakEPJJJa1DautWUKWijsU+FEEKImEbTp68B44HjgPe4+0v52hNCCBE6ZeopNjzMm5kNAxa5+8Jeyil9GogXeewcjyF5abXHSto6mbSG5tLWIZ2vPDzmlT4dNmyYv/vd7866Wm655Zbw06fx+nxgz3rbKH0aph6SlzQ9JC9puryE4ZFektY0mbYO6Xyl6aGmT4866qjMF4qQPjWzB4DJrgf3hRBCUL6po5pJn348PztCCCFE+2g0aDMSGA5MAOa7+/p87QkhhAidMvUUG51k+GVgx3j9Lzl5EkIIIdpCo2OfjnX3y8zsBKDH3X9dU07p00C8yGPneAzJS6s9Kn3afz2v9Onw4cP90EMPzbpabrrppjDTp2w6yfBiYCFwObBtX9spfRqmHpKXND0kL2m6vIThEaVP+63nlT4dNmyYH3fccZkvhJo+dQ3z1nFoomJRFCrvSQ1pKLJCkwwLIYQYEGUK2jSaPh0G7ADsDCxwfTUTQghRIhpNn77N3ZcSjYG6b46+hBBCFIDKw/tZL207nv509hLp06eBQ4C3Ap9z94dryil9GoiXgXisJPpg01RfJdEXgsd26/Iij0XzmFf6dIcddvCZM2dmXS3Lly8vRPp0GtEkw2fV207p0zD1/pQlkeJLpvpC8thuXV7ksWge80qfDh8+3OfMmZP5QpvSp/25ffqMu5/u7ie5+4/d/WKPbqGKkpJ8g3R1dVXXk7c2enp62n6bQwgRBmW6fapJhoUQQoiY/qRP3wm8HZgI/AgYBwwFznFX+lQIITqdMt0x6s/t03vd/SLgSeAYd/888Gtgv5y9CSGEKABlun3a3/TpHMCAA939TDObC/zK3R+sKaf0aSBe8vDYqlRqSOerCB5D8iKP4XrMM316xBFHZF0tN954Y7Dp02OBO4F5wAlED/JfRNygpi1Kn4apD6QOWpRKDel8penyIo9F85hX+nSHHXbwuXPnZr4Q8NinNwE35dIii0LhibsKybEmNVaqEKIsaOxTIYQQA6KjgjZm9k4zO8fMrjWzt5rZCjPbvxXmhBBChE+ZgjaNpk/XEQ0MLoQQQpSOhtKn7v5NM/sI8KDXJE/jckqfBuKllR6zTqWGdL6K4DEkL/IYrse80qcjRozwI488Mutquf766wuRPp0GLAe+DAzrazulT8PU86ibjFOpIZ2vNF1e5LFoHvNMn5544omZLwQ89ulN7j7T3S/zaOzTOe5+trs/n08zLYpG8g2lsVKF6Dw66jdFIYQQolNodOzTe4Cdgd2B09z9lXztCSGECJl29+yyptH06U/c/YvABmBQzt6EEEIUgHbdPjWzmWb2P2b2mJl9KqXMsWb2sJn9xsyW163TG0+fngE84u5391JO6dNAvITgsdlUakjnqwgeQ/Iij+F6zCt9uuOOO/qsWbOyrpZrr722z/SpmW0GrAYOBdYA9wOz3f3hRJmJRCOyvcvdnzezN7v7H/vccb0kDpumTxcD/xGvK30awD4b1Vu5T5pMpYZ0vtJ0eZHHonnMK306YsQIP/nkkzNfqJM+Bd4G3JV4vQhYVFPmS8DHGjkejX0aOEuWLKmuT548ufo6qWdRfx51u/c+VqoQQmTALsDvE6/XAAfVlJkEYGb3AZsBS9z9zr4q1dinQgghBkROQZsRZrYq8XqZuy9rsI7NiUKi04HRwD1mto+7v9DXBn1Skz69HjgYGAuc6e6vNmhQCCFEycipUVznfY9o8zSwa+L16FhLsgb4uUdPSvzWzFYTtWX3p1Xar6ANgJmdRzSazT8BhxD9oPl6TRkFbTKue+3atVV98ODBbNiwAYBRo0Zl4rFSfxZ1Z6WH9H9aBI8heZHHcD3mGbQ56qijsq6Wa665pl7QZnOioM27iRrD+4E57v6bRJmZRG3VCWY2AngA2N/d/5y64/788AjMAY5PvP4YMKKvbRS0yUZfvHhxdVm+fHl1PSuPWdadlR7S/2maLi/yWDSPeQVtdtxxR//EJz6R+UI/hnkD3hc3jI8Dn461C4Aj43UDLgEeBh4CPlyvzv7cPj0WmAvcaWYnAG8m6n5+s962QgghRF64++3A7TXavybWHZgfL/1C6dPASSZBu7u7mT17di7151F3s16SSdikLvJB510MlDKNaKP0qRBCiAFRpkax7jBvZvZOMzvHzK41s+FmNtfMrm+FOSGEEKKVNJo+/RPw/4APuvu8XsoofRqIlyJ67C0JCxvTsCF4bOc+8/LSivMe0vnqVI95pU/f/OY3+7HHHpt1tVx11VVhTjLsifQp8EWiId5+DIzvaxulT8PUQ/JSq/eWhE2mYUPwWEYvrTjvIZ2vND0kL2l6qOnT0047LfOFNk0y3FD6FPg3d3/KzMa6++O5tdRCCCEKQ5l+U2wqfeq93DoV5eHmm2+urm+++ebV18ccc0yu+01Lwlb2n/TSCj+dQkgJ5EbRe0NkTaPDvD0MbAuscffrcvYmhBAicMo2yXB/eor3AvfGQZttgX8AQ/I2JoQQohiUqVFseJLh+PV8YIW7P1FTTunTQLwMxOPzzz9f1c2sOuXTsGHD2uKx4ifpJekn1PPYiV5a7THL90ZI5ysPj3mlT0eOHOkf/vCHs66Wyy+/vC3p02aGeduZaB6rNbVlPZrWYxnAgQce6NOnTwei3yoq60ka0bOoI289JC8D8Vj7m+Krr0aToSS3a6XH5O9GFS9JP6Gex0700mqPWb43QjpfeXvMmjL1FDXMmxBCCBGjYd7EG0gm91r1TbMvKn5qvfQ2ZqfG6+yssUzT3huitXRUT7EmfboMOBT4m7tfkrM3IYQQBaBMjWLdsU/d/V53vwh4ElgCrAdetTKdBSGEEIIG06fAUUShm+OJhuDpqSmn9GkgXjrBY29jdlbG6wzFYzu8hDiGbEjnq1M95pU+3Wmnnfz444/PulouueSSMMc+BY4lGuJtHnACsBi4DNimr+009mmYekhe0vSBjNkZmsd2eAlxDNmQzleaHpKXND3EsU9Hjhzp8+fPz3wh1LFPXelTEShFHp6sN5K/SCxdupQZM2YAbPL8XX8o23kR4VOmX9OUPhVCCDEgOqpRrEmf3gsMA44D3uPuL+VrTwghhGgdjY59ei/wPLCzGkQhhBBQrp5iw2OfxuOe3uHuj/RSTunTQLzIY/E89vRsDHOPHj2aNWuikRS7urpa7iVrPSQvneoxz/Tp3Llzs66WL3/5y4VIn44BrulPgkfp0zD1kLyk6SF5SdPzqBuoLkuXLq2ut8NL1npIXtL0kLyk6aGmTxcuXJj5QpvSp/15eP8md5/p7pe5+1Pu/vG8Gmgh8qIy55uZ0dPTE+QccMkPZldXV/KLqRCiRSh9KoQQomlC/II5EBpNnz5KNDTcBGC+u6/P154QQojQKVOj2OjYp0OAHeM//SVHX0IIIUTLaXTs0x3d/bJ4suEed/91TTmlTwPxIo/ZJDtb6bFZPSQv8hiux7zSp6NGjfITTzwx62q58MILC5E+XQwsBC4Htu1rO6VPw9RD8pKmh5TsbKXHMnhJ00PykqaH5CVNDzF9utNOO/miRYsyX9DYp0LkhyfuiHR3dyvVWXIqv3ENZAxZ0X/K9Jui0qdCCCEGREc1ijXp00eIflvcGVjg+uolhBCiRDSaPn2vuy8FXgP2zdmbEEKIwEkOjJHl0rbj6U9nL5E+XQ0cArwV+Jy7P1xTTunTQLzIY+d4DMlLKB4raWONIZt/+nTnnXf2k046Ketq+dznPleI9Ok0YAFwVr3tlD4NUw/JS5oekpc0XV7C9ojGkH2Dnlf6dNSoUX7++ednvlCQsU9/7O4Xe3QLVbSRIozlKUS7qFzgNIZsayjT7dO6jaIQQgjRKfQnfboX8F5gHLAKGAFs7e4X5OxNCCFEASjTXar+Bm2OB44GnnT3M83sX4HL3f2FmnIK2rRon5qQVh5D9CKP4XrMM2hzyimnZF0tn/nMZ8IM2iTuxx8B/Ge8fj6wfV/lFbTJV0cT0rZVlxd5LJrHPIM2S5YsyXwh1GHezGwm0TOJ44EVZrYgbkxf6HNDIYQQomD0Z+zTO4keyRAB4a6xPNtBb2NqgsbVFJ1Lu9OiWdNo0OZe4F+AWeopCiGEgHIFbfrznOJvgGeB0cBdQHfOnoQQQoi20K/0KYCZHQH8DjgKuKy3nqLSp+F4kcd8U7/JxC9sTP3qfMljqB7zSp/usssufuqpp2ZdLeeff36Y6VNgJtHEwtcA7yD6fXERsFlf2yl9GqYekpc0PSQvtTq9JH5JpH51vuQxVI95pU933nln/+xnP5v5QqjpU39j0GZmlo2yEEKIYtNRvymKYqExUfOl8m0yOaamK3kqOpwyjX3azDBvuwAj3H1+zt6EEEKIltJo+nSFu38B2D5vY0IIIcKnIycZhmr6dA3wIaLG8YFeyih9Gkg6EnofEzUEj/X0kLwUwWNIXuQxXI95pU9Hjx7tp512WtbVcu655xYifXotcBXRhMNKnwawz1qdOmOihuCxnh6SlzRdXuSxaB7zSp/usssufuGFF2a+UKD0qRBCCFGlTGE+pU9LRvIbT7tnHVcSVghRNJoZ+3QMUfr0nJy9CSGEKABl+rLbzNinWwHb5exLCCGEaDkNj33q7g+Z2dnAJe7+Wk0ZpU8D8RKCx3pJ2BA8NqvLizwWzWOe6dMzzjgj62pZuHBhIdKn/xqvX1FvO6VPw9RbuU/qJGFD8NisLi/yWDSPeaZPv/SlL2W+oPSpEEKIolG2AF3w6VOlF4tL8ttXu5OwQoj8KNOINo2mTz8DzAbGu/u8nL0JIYQQLaXR9OkRwA/zNiWEEKI4lKmn2Gj69Czgu8AHgaPd/U81ZTJPn/Y203m99GK79JC8yGPneAzJizyG6zGv9Omuu+7qZ555ZtbVsmDBgkKkT2pcu/sAACAASURBVHeKtcvqbZdV+pQm0ovt0kPykqaH5CVND8lLmi4v8lg0j3mlT0ePHu2XXHJJ5gtFSp96C39P9Lgn293dXV1fsmRJ9e+TJ0+uvk7qQhQNva9FUSlTADL49KkQQgjRKhpNn24FPAQ87e435+xNCCFE4LQ7GJM1jaZPfwcYMChnX0IIIUTLaXbs00uBhe7+Sk2Zlox9unbt2qo+ePBgNmzYAMCoUaOaqjsPj6HqIXmRx2ze1yGdF3kM12Ne6dPddtvNzzrrrKyr5YwzzihE+nQu8Gngonrb5Tn26eLFi6vL8uXLq+vN1p2Hx1D1kLyk6SF5SdPzqLvZ93VI5yVND8lLmh6SlzQ9xPTprrvu6pdffnnmC0VKn7abZBqvu7ub2bNnt8+MEBmR9r5O/l6zdOlSZsyYAWxMZgshsqNuoyiEEEL0RZmCNo2mT++N//2zu1+bszchhBCipfQraGNmxwNHxy9/CLzs7l/rpZwmGQ7EizyWx2O9yZpDOi9pekheOtVjnkGbc845J+tqOf3008MM2lQWosHAV8brFwI79FVekwyHqYfkJU0PyUua3sp9Umey5pDOS5oekpc0PSQvaXqoQZsrr7wy84VQgzZmNhPYFxgP3GVmC4ieU3w+hzZaCCFEgSjbw/uFTJ+K9qCxOduDJ37iSI4BLETls5f8PCb1VlGmRlFjnwohhBAxdRtFM9vLzOab2RVmdoKZzTOz+1phTgghRPi0a5JhM5tpZv9jZo+Z2af6KHeUmbmZ1Q3uNJo+/QiwE/BP7n5xL+WUPg3ESx4eWzW8XkjnqwgeQ/Iij631WPlMJj+PsPEz2Yr06ZgxY3zRokVZV8upp57aZ/rUzDYDVgOHAmuA+4HZ7v5wTbltge8TZWFOd/dVfe64v4kcovTpPsBFwPB65ZU+DVMfSB2tGl4vpPOVpsuLPIbgsbfPY/Iz2Yr06W677eZXX3115gt10qfA24C7Eq8XAYt6KXdZ3H51AwfWO57+3D6daWYLgSOJEqeD3P25etsJIYToDNp0+3QX4PeJ12tiLenrAGBXd/9+f4+lmfTpmf2tXJQLjTlbDCoXlOQ4qaCxUstI5TNZ0s/jCDNL3upc5u7L+ruxmb0JuIToZ79+0+gwb+uAl4A93f2kRnYkhBCifOT4nOI673tEm6eBXROvR8dahW2BvYHu2N9OwK1mdqT38btio5MMAwwF1tfbTgghhMiR+4GJZjbOzAYBHwZurfzR3de7+wh3H+vuY4GfAX02iND4JMMHuPtnzew84HJ3/0tNGaVPA/Eij53jsVarjJWaHCcV2jtWakjnq1M95pU+HTt2rJ933nlZV8vJJ59cd+xTM3sfUZBmM+Br7v55M7uAKKRza03ZbuCseo1if1KnyUmGryH6TfFS4E19baf0aZh6SF7S9JC8pOkhe6GXcVJp81ipIZ2vND0kL2n6QOrIK306ZswYv/baazNfCHXsU9cwb0IIIToEDfMmBkwyRt3T01O6AYKLRuUbb1dXV+1dHyFyoV0j2uRBo+nTNcDfiR7eX5yzNyGEEKKlNJo+3cPdvwJMMbPt8zYnhBAifMrUU2w0fToBeJVoyJzj3P2vNWWUPg3ESys91psZPgSPWevyIo9F85hn+nTx4uxvHH70ox+tmz7NhXpJHDZNn84GFgAn19tO6dMw9Tzqps7M8CF4zFqXF3ksmse80qdjx47166+/PvMFpU+FEEIUkTIF6+o2ikLUwz38meGT47YmZylv9Qzlom9CmUledC79ahTj3xM/DtwCjAC2dvcL8jQmhBCiGJSpp9ifqaOmAkOAJ4D9PZ5cWOlTIYQQZaNu+tTMzgVeBj4IvO7uM8zsfOAKd3+hpqzSp4F4kcdN9coM5bDpLOWVGcpD8NisHpKXgXpsZCb5dnlspx5i+nTcuHF+wQXZ3zicO3dumOnTykI06OoJROnT8+uVV/o0TD0kL2l6HnUnZyVPzlIekscyeEnT+1u2kZnk2+WxnXqI6dNx48b5jTfemPlCqOnTROM5L59mWQghhAgDpU8bIPljcnJWcw8wbSk2JZleLOks5aWg5DPJl5aOCtpAlD41s1vN7DAz61bIRgghRBnpz4Dg1fSpu99tZm/P35YQQoiiUKaeYqPp03nAB4DLvCZ5Gpctdfq03hifIXisp4fkRR7L50Uew/WYV/p09913989//vNZV8ucOXMKkT49kGjIt0XAZn2VL2P6lDpjfIbgsZ4ekpc0PSQvabq8yGPRPOaZPl2+fHnmC21Kn/Z7kmF3n+fuq9x9prtf6O6vZd1Ah07yxCUncBWiU9CE0qI3yjR1VL8bRSGEEKLsNDr26beBMcAId5+fpzEhhBDh0+6eXdY0mj79Zqx9LW9jQgghRKtpNH06HzgSWOHuD/RSttTpU3mUx073Ui+BHYLHZvWQvOThMa/06fjx4/3CCy/MulqOO+64QqRPrwSuIno0o+PSp/X0kLyk6SF5SdND8pKmd6oX6iSwQ/DYrB6SlzQ9xPTp7rvv7jfddFPmCwVJn57u7qe5+2XegelTITqd5MUjmcBWKlWUBY19KoQQYkCU6QtQM2Ofnm1mF+VtTAghhGg1/QnaTAV2B95JNJfiecBO7n5qL2UVtAnEizx2jscQvGgIxPA95hm0+dKXvpR1tRx99NFhBm2Ac4mCNT8Gpsba2ShoE8Q+G9VD8pKmh+QlTZeXxgI4IXisp4fkJU0PMWgzfvx4v+WWWzJfCHWSYXf/AoCZjQXeYmaHAru6gjZCCCFKRqPp02Xu/iV3/2SepoQQxSL5Tbu3VGoykVqmUIaI0NinQgghRAlpKH0ar3/SzC7L15YQQoiiUKaeYqPp018ADwKnuPu8XsoqfRqIF3nsHI8heanVK6nUZCIVwkylhuQlD495pU8nTJjgF198cdbVMmvWrEKkT+9IrO/Y13ZKn4aph+QlTQ/JS5ouL42lUpOJVAJNpYbkJU0PNX26YsWKzBeKkD71uHcYr/8p+yZaCCFE0ShTeKqh9Glv60IIkUbl23cykepKpYqAaXSS4UeB3wNPu/vNeRoTQggRPmX7QtPQJMPAs4ABg3L2JYQQoiCUqVFsdJLhee7+gJldCix091dqyip9GogXeewcjyF56a/HEFOpIZ2vPDzmlT6dOHGiX3rppVlXy/vf//4w06eVhWiS4bnAp4GL6pVX+jRMPSQvaXpIXtJ0eRmYRwJMpYZ0vtL0ENOnEyZM8Ntuuy3zhVDTp4nGU+EaIYQQpUaTDAshWo7HP9t0d3dX1wGWLFkCwOTJk6vrSV30j97O4x/+8Ifc9lem3xSbmWT4HDP7WN7GhBBCiFbTaPp0LvAz4PWcfQkhhCgIZeopNpo+/Ye7H2pmFwJL3f3PNWWVPg3Eizx2jseQvAzU49q1awEYPHgwGzZsqOqjRo0KxmM79YGcxwULFvCHP/wh89Zr0qRJfvnll2ddLYcffngh0qcfARYAFwNv6qu80qdh6iF5SdND8pKmy0s+HhcvXuyLFy/25cuXV9cXL14clMd26gM5j6NGjXLPIa05ceJEv+OOOzJfUPpUCCFEESnT7VOlT4UQwVBJSnZ3dzN79uz2msmISoOxdOlSZsyYUdW9zk9XA6G383jbbbfltr8y0ejYp98GdgSOcfdD8jQmhBCiGHRUTzGZPnX3b5rZZGCz3J0JIYQoBGVqFBse+xT4MNEwb8/1Ulbp00C8yGPneAzJizy+Uc9ynNcQxz6dNGmSX3XVVVlXy2GHHVaI9Ok2wKX9Ka/0aZh6SF7S9JC8pOnyIo/91clwnNeBeMlr7NOJEyf6D37wg8wXCpQ+PTPTVlkIIYQIhH4N8yaEECFSmeC2p6enuh7a71uVHkhXV1ft3bdSkDzvWS7totH06c+BDcCe7n5SnsaEEEIUg9C+iAyEuj3FmrFPXwOGAutz9iWEEEK0nEbTpyvc/VIzOw+43N3/UlNW6dNAvMhj53gMyUurPYaS7GyVHmL6dPLkyX711VdnXS3vete7CpE+/QxR0OZSNPZpEPtsVA/JS5oekpc0XV7C8Eggyc5W6SGmTydNmuQ/+tGPMl8oUPpUCCGEqFKm3xQ19qkQJaO7uxuIbp1V1gGmT5/eFj954vHPP93d3ZskOivHOmfOnE3GARWiHo2mT38MvAoMd/fFeRoTQghRDDqqp1iTPt3D3U82s5vNbHt3fyF3h0IIIYKl3c8VZk2j6dPbgReBI4Dj3P2vNWWVPg3Eizx2jsda7cUXXwTgtddeY7PNNo7dXykTgse89dWrVwMwfPhwnnsuGqZ50qRJQXlsRg8xfbrHHnv4smXLsq6WadOmFSJ9ejiwADi5XnmlT8PUQ/KSpofkJU0P2cvKlSt95cqV/r3vfa+6niwTgse89WnTpvm0adP8mmuuqa6H5rEZPcT06eTJk/2ee+7JfKFA6dM7Mm+ZhRCZUQmZdHd3lzJc0x8qoZru7u7qevIWX3LCX69zt0x0FkqfCiGEGBBl+k2xP0GbWcAM4LfAo8D+REO9neP6iiWEEB1PmRrF/gRtDgWmEY13urO7n2lmc4FfufuDNWUVtAnEizx2jseQvITssTIkHGw6LFwzQ8K1Sw81aHPddddlXS3veMc7gg/afAh4IF6fC+zfV3kFbcLUQ/KSpofkJU2Xl+J5JDEMXHJYuJA81tNDDdrcd999mS+EGrQxs+nAQcA4YGn8iMZQ4Mbsm2ghhBCifdRtFN29G+jO3YkQQuSIJ34qSg4LV/k9LJlIrS0v0inbw/tKnwohhBgQZWoU+zPJ8Cwz+4qZzTOzg8xshZnt3wpzQgghRCtpNH16CfDPwINekzyNyyp9GogXeewcjyF5KaLHLCcqzlsPMX265557+g033JB1tRx88MGFSJ++C/gIdZKnrvRpsHpIXtL0kLyk6fJSHo9kOFFx3nqI6dM99tjDf/azn2W+UJD06fXAYcBeZvaUuz+ffTMthBCiSJTpN8Vm0qdz8jIj3khyYtQXX9w4aWynjmkpRNa493+i4ko5UV4aHebtcWAvYHfgNHd/JV97QgghQqddPUUzmwl8BdgMuNbdv1jz9/nAx4BXgT8BH3X3p/qqs276FHgJ+CuwBXB7vNMNwKCGj0AIIUSpqDynmPXSj/1uBlxFNKXhFGC2mU2pKfYAcKC77wv8X+BLdetN3i6oY+BDwAvAPsAj7n53L2WUPs247hdffLGqJyeNTW7Xbo9Z6yF5KYLHkLyUyWNvExXDxsmKQ/DYiJ5X+nTKlCn+jW98I+tqectb3tJn+tTM3gYscff3xq8XAbj7hSnlpwJXuvshfe64XhIHmA6cA1wd//sfwDxgWF/bKX2ajb5y5crqkpw0NiSPWesheUnT5aX8HnubqDg5WXEIHhvR80qf7rnnnr5q1arMF+qkT4GjiW6ZVl7/c9zopZW/Ejiv3vFomDchhBAhMsLMViVeL3P3Zc1UZGb/GziQ6Jn7PtEwb70Q0gzdyZRpJ8+kLkSrqaRMu7u7lTitQ05Bm3Xe98P7TwO7Jl6PjrVNMLP3AJ8Gprn7hno7bTR9+nPg7cBY4Ex3f7Xe9kIIIcpNm9Kn9wMTzWwcUWP4YWoeGYx/R7wGmOnuf+xPpY2mT38BvAK8GXi939aFEEKIDIk7ZacDdwGPADe5+2/M7AIzOzIu9mVgG+BmM3vQzG6tV2/D6VN3/5GZfQxY4e7rasqUIn1ab4buEDw2q4fkRR7L50Uew/WYZ/p0+fLlWVfL1KlTwxz7lE3Tp8cBZwPLgC372q7I6VPqzNAdgsdm9ZC8pOkheUnT5UUei+Yxz/TpAw88kPlCqGOfutKnQgghUijbJMP9+U2x40h+a+jq6kr2moMhOfJDT09P6d6YQoji0I4RbfKi0UmGzczmmtn1rTAnhBBCtJL+PKeYTJ8eBjxFNOGwEEIIUaq7VI2mTz8FLAc+SDTa+OM1ZUqRPi2CRyVk5TFEL/IYrse80qd77bWXV671WbLvvvsWIn26Q6xdVm+7IqdPm9VbuU+UkG2rLi/yWDSPeaVPp0yZ4g899FDmC0VKn7r7vMxaZSGEECIQCjn26WOPPVZd37BhQ/X1hAkT2mWp5XjitnftjOFCiHCYOHEiAGeccQYnn3xyVX/00UfbZSlzyvSbYqNjnxqwHbDG3a/L2ZsQQojAafcjFFnT6NinzwH/AIbkaUoIIYRoB82OfTqfaOzTJ2rKtCR9umHDxtk/NmzYwODBgwGq/2a1z4F4DFUPyYs8ls+LPL5R//Wvfw3AyJEjefbZZ6v63nvv3VKPeaVP9957b7/55puzrpYpU6YUIn16ArCIaAbjQX1tl2f69NFHH60ud955Z3W92brz8BiqHpKXND0kL2m6vMhjf/UJEyb4hAkT/IorrqiuT5gwoeUe80qf7rXXXv7www9nvlCk9KkQQghRoUy/KRYyfZpMma5Zs6b6OvlA+8svv1x9nXygXQghWsnZZ58NwPDhw6vrAMuWLavqlXWAU045pbUGM6CjGsWa9Ol9REO9/c3dL8nZmxBCCNFSGh379KPAb4A3mZm56+E4IYTodMrUU2w0fXotsBtwPNGPoD01Zdo69unLL79c1V999VU23zxq87faaqum9pmHx3brIXmRx/J5kcc36uvWrQNgs80247XXXntD+Vp9xIgRuXjMM336ne98J+tqmTx5ciHSp8cBi4HLgG362q4dY5+uWrWqunz/+9+vrje7zzw8tlsPyUuaHpKXNF1e5LG/+jXXXOPXXHON33zzzdX15FKr5+Uxr/Tp3nvv7atXr858QelTIYQQRaRMt08LmT5NI5ky7e7uVupUCNF2KmnS7u5ujj766Df8vVavNDBLly5lxowZwKZjHYt8aTR9+jgwnug26nvc/aV87QkhhAidMvUU6wZtzOxQYBqwHriEaEDwRe6+sJeymmQ4EC/y2DkeQ/IijwPXK89X9zZ5eIhBm3322cdXrFiRdbVMmDAhzKBNZQE+BLwLmA/sWa+8JhkOUw/JS5oekpc0XV7kMS+dPiYPDzVo8/jjj2e+0KagTd1ZMsxsupmdQ/TQ/i+Bye7+SE5ttBBCCNE26jaK7t7t7he5+yfc/c/u/vFWGBPtozI/mpnR09NTuvnShAiZSo+lq6sreacuaJLXjKyWdlGq9KkQQojW0u5GLGsaTZ/+DRgOTADmu/v6fO0JIYQQraPR9Okfgf2AocDHvGZjpU/D8TIQj8nZRnpLwIXgsd26vMhj0TzmlT7dd999/Xvf+17W1TJ27NhCpE/nx+snAHv3VV7p0zD1/pQlTryRkoALwWO7dXmRx6J5zCt9us8++/iTTz6Z+UKow7yZ2XTgIGAcsNbMFgKjgexHgBVCCFE4Ouo3RdfYpx2HJ+6Kd3d3FyL9JkQn0tuQcND6YeE6qlGsCdqsB3YAdgYWuK6WQgghSkTd5xTZdJLht7v7UuA1YN88jQkhhCgGZXpOsdFJhicCG4C3Ap9z94dryih9GogXeewcjyF5kcfWeuxtnFTofazUPNOnt99+e9bVsuuuu4aZPmXTSYYPARYAZ9XbTunTMPWQvKTpIXlJ0+VFHkPwSC8pcVLGSs0rfbrvvvv6mjVrMl8INX3qbwza3JddkyyEEKLolClo05/fFIUQQgRIpXeTHCfV3Xsduzg5KIdIpz/p03FED+u/CPwJGAFs7e4X5OxNCCFEAei0nuLHgefi9f3d/WIAM9s+N1dCCCFEG+jP2KeXAf9ONIrN1e4+zszOB65w9xdqyip9GogXeewcjyF5kccwPPaWSj3rrLNw98y7dPvtt5/fddddWVfLqFGjgk2f7g9cBFxCdBt1AXB+ve2UPg1TD8lLmh6SlzRdXuQxZI+kp1JzSZ+uXbs284WA06cPAg/m0yQLIYQoOmX6TTG3SYZ7enrWmdlT8csRwLpeijWiZ1FH3npIXuSxczyG5EUeA/J41llnJfUxvWw3YNo9Ak3mtKI7Sko3uBE9izry1kPyIo+d4zEkL/JYDI9ZLvvtt58/++yzmS+t8N7bkltPUQghRGdQpp6iHt4XQgghYlrVU1yWgZ5FHXnrIXlJ00PykqaH5CVNlxd5LLLHTClTT7Hfs2QIIYQQtey///7+wx/+MPN6R4wY0ZbnFHX7VAghhIjJ9fapmR0AfAjYiuiB/5difRJwLrDC3Vckyr8TeDvRvI0L3f25WN8LeC8wDviMu69LbHME8HF3PzKhzQJmAL8FvuJxd7hmHNdr3P2vZjYNmArMAj7i7k/GZd8PvA3YHrjY3R+P9cOJ5pPcAfgU0Ug/5wIrgH/EdQ0Frk3of4jXlwAvJ/RXgb2B3YHLgIWx/mx8HsYCXyWaumuFu68ws7nxsV2YqGdsvM9XgEmxtiY+Z38DbkuUfQ0YDxwHfAI4M9ZHAsOBCcBVwP+J9WHxse4M/Gfi/+dH8f/HXsAvY21Z4ji3TZS9J95+d2A5cFCsXw8cHPu/JaEvBP4pPs6vJep5OK53DbA6oS8DDo2P9f6Efm/s/2PAt+L9PEr0ZXACcDOwX1z2EcBin9ex8f22ini839hjRb8X+Bei980uNeV3ibe5rqb8mFj/RkL/DDAbeAvR88DjiD4vDwFPx8ecrGMc8GfgpzX7HAYcQzSiVEVfRzRJ+J5Eg29U9DXA34Hh7r648hmKj69yrD2xdmX8fzrL3V9IlP125XjcfX5CvzI+pyPc/Zzk59PMPgmMd/d5ifKPAr8Hnnb3m2vqmRof69rEPncEjnH3QxJlf040z+ue7n5SQv8x0WdsOPAAG68JjxINSjIU+K+E/lNgEdH7d2xCf5zofb47cBfwv2L952z8nK4EpsX6V4B/JrrW/DrWDNguPvd/TtR9H3AY0Xv3iZp9Vj6nl8X7+W1crvI5/SHRZ+a3wHo2fk6voMXjVZfp9mnePcXZRG+wFUQXLQDcfTVwQ21hd7/X3S8CniRqjCr6b4gaitFEF34AzGwqMITozZTkJeCvwBZseozJcVxfiev+MdGb6JFKgxhTefNtBfwxoc8Evgz8D3BozbEc6u6fJ/ogbFXR3f0X8TnY5Njd/TZ3/yLRB/qphP7T2N+b4/3cEB/voXG59TX7fZ6oQV6f0D4av36V6CJQqft7wNeBe939l4nyLxNdcCC6gFT0t7n7UqLG9C+J/59j4mO9k+hC8STRRbhynMn/y58kjvP+hL42cZz3JOtOHGeynm3j4xxSoy9JHOtPEvq98bHe7u6fi7UhieO8K1H2vYnj3JyN77fqeL9EjVRFv4t4SrWa9+cKd/8CsH2NfhfRe2K7Gv0Ioovbcwntd0QX0UE1ZWcTvbdfr90ncAfwnRodogv/+hp9D3f/CjAl/iJa+QxVjnUnovf+E+5+d+U4k583d/9m5ThrPoc/rBxnUjezOfHfaj+3z1aOtUafGx/r6OQ+K8dZU/a1ynHW6NXjjM9F5ZrwnsTndKeEvio+l7DpNeT2xPv3Hwn9F2x8/76Y0A8jfv8mtOfibYfU1J38nG6yT+LPabxtRf87G9+/f0rob0+8f89H41U3TSuCNg39aBl/eH7r7ps0dO7+TTN7AdiN6Fs0wOFEF/OpZjbV3R+Iy/4A+IGZfYjo29uP4vJDiD6YlYvRLbE+C7i1xspk4HSiiZWnEfW2IGpA/4XoW/JvGzm2lOM9A/ieu7+U/Lbl7peb2ctEF6cK7waeiY93fKLs1+O65gOjiBr0nYh6XMcDB9Ts9kSiXlqSHeJv/CcQfSuu8O9mNo+o9/NK5f+H6FtphcOJ/8/M7H8ljm1OQk8eZ/L/uHqcZnZYXPc+wJaJ4zwoLv/NynGa2e5Evczfxsd3Q+VYzWxyYr/zgesTvnd0989UjtPM9o31u5LHmXi/faxymuNznXwfVknoY+P33RW15eP9nm1mmyX0s4h6eVOBzwMvAL9z94fM7FIzuylZ1t2vNLMLzWyHGi//m2g4xuQ+D3D3883sPDPbLqFPMLPTiHqE7473ORV4PT6ciUS9t6lxI5P8f6583rqAI+PjrOrAvpXjBN5HdKGfSvR5enO8/iLRF6ipwNfd/QEzu5SoF/tirP8jPtZu4jswsZcPx8f5icQ+V7j7F83sPKLP8vOxfnviOH8Y32n5EDAncUw/c/cHE9eKyv/nJtcQM9uH6P17N/C9ip74nP63u98d658luiOyA9HnbD/gBXf/evx+fNzdK3Uviv//jgeec/fzEl72B65390cSXsYmPqdrE+XflHj/vsrG69xpRHeDcguPlO3h/bwbxW8RfYvfClhcEc1sJ+BoYEsze8Ddn4r1Y4m+Id5pZmMS+kxgX6LbCdV64m+qmNnYSoMYv55OdCEdB3w64eeGuP4tgIsT+nuJepFJ1gHnEX0DvTShD47//SNwd/JYgHvM7Nx4m7uJLqhbmtnzRN8e9zKzvyWOfT9gj8iyPZXQhxJdQCYC/1+i/gvc/SkzG0t0sUmW3zkuPyH2+F2icWqHAT8gagi3NLMHgMnufkmN99+Z2UKiD9JPEvo3gM2A/ya61TuXqHf4YHysbyX6snFnfCu6cpxTiW793GlmixPHuUtCPyFxnH8juuV0J/BviePsYuN74oT4OHch+v+teEke68tEF5g7zWwM0cV4TaLs0MRxOtEF8s64TOU4dzOzfyJ6v60wswXx//nBcSM6HriRqFE+1cx+GZ+b8XEdG4gupKMS+tMW3XrfFTg0Uc9sd38mPncnxtp9ZnYkUWPw7kTZu2Ivg4C3xhfq8UQX4UHu/lzNZwUzO5Po4nxIonx3fB7/093/PS43Fnggrv8ed7/QzEbGx3MwcCpwkbu/Fpc9kaiHN61Gf4tFdzR2dfdPVup293mJ9fMS+9zHzN5H1Ah+LqE/GHvpAb4Yv08erRwn8IVE2e0Tx7nA3V+P9V8R9RL/E+gys8o1YWnic/o7Mzsn1q9n4/t3NNGXw3FEX5r2j3ZngxL6yrjxtaU74gAAAodJREFUnxjXMzXWD3f3P5vZO4g+R+OAn5rZIqL37u5mdkysn8rG927Sy6eB4+LP6XQ2Xs/WJt6/f7ToVvE4Nr4f/5vomlG5zi2pvH+9ZgKHLClTo6j0qRBCiKaZOnWqr1y5MvN6hw0b1pb0qUa0EUIIMSDK1FPUIxlCCCFEjHqKQgghBoR6ikIIIUQJUU9RCCFE0+iRDCGEECJBmRpF3T4VQgghYtRTFEIIMSDUUxRCCCFKiHqKQgghBkSZeopqFIUQQgyIMjWKun0qhBBCxKinKIQQomnK9pyieopCCCFEjHqKQgghBkSZeopqFIUQQgyIMjWKun0qhBBCxKinKIQQYkCopyiEEEKUEPUUhRBCDAj1FIUQQogSop6iEEKIpinbw/tqFIUQQgyIMjWKun0qhBCikJjZTDP7HzN7zMw+1cvfB5vZt+O//9zMxtarU42iEEKIAVG5hZrl0o99bgZcBRwOTAFmm9mUmmInAc+7+wTgUuCievWqURRCCFFE3go85u5PuPs/gG8BH6gp8wHg6/H6/wXebXVaXP2mKIQQYkC06TfFXYDfJ16vAQ5KK+Pur5rZemAHYF1apWoUhRBCNE1PT89dZjYih6qHmNmqxOtl7r4sh/1sghpFIYQQTePuM9u066eBXROvR8dab2XWmNnmwFDgz31Vqt8UhRBCFJH7gYlmNs7MBgEfBm6tKXMrcEK8fjTwI3f3vipVT1EIIUThiH8jPB24C9gM+Jq7/8bMLgBWufutwHXAjWb2GPAcUcPZJ1an0RRCCCE6Bt0+FUIIIWLUKAohhBAxahSFEEKIGDWKQgghRIwaRSGEECJGjaIQQggRo0ZRCCGEiFGjKIQQQsT8/wgaVXDmm8zWAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 576x576 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light",
      "tags": []
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "print('Number of samples = {}'.format(test_samples))\n",
    "print('Evaluating...')\n",
    "numCorr = 0\n",
    "true_labels = []\n",
    "predicted_labels = []\n",
    "\n",
    "for inputs, targets in test_loader:\n",
    "    inputVariable = inputs.permute(1, 0, 2, 3, 4).to(DEVICE)\n",
    "    labelVariable = targets.to(DEVICE)\n",
    "    \n",
    "    output_label, _ = model(inputVariable, no_cam=False)\n",
    "    _, predicted = torch.max(output_label.data, 1)\n",
    "    numCorr += torch.sum(predicted == labelVariable.data).data.item()\n",
    "    # store results in a list to plot confusion matrix later\n",
    "    true_labels.append(targets)\n",
    "    predicted_labels.append(predicted)\n",
    "                \n",
    "test_accuracy = (numCorr / test_samples) * 100\n",
    "print('Test Accuracy = {:.3f}%'.format(test_accuracy))\n",
    "plot_confusion_matrix(true_labels, predicted_labels)"
   ]
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "authorship_tag": "ABX9TyPMoOB4LApfdW5BCYwu0ih9",
   "name": "test_pipeline_MS.ipynb",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}