USAD-SWaT.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "etniX_KTlJ5U"
   },
   "source": [
    "# USAD"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "N3jM0qLU8MgZ"
   },
   "source": [
    "## Environment"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "id": "rjheCL2b1Rnw"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "rm: cannot remove 'sample_data': No such file or directory\n"
     ]
    }
   ],
   "source": [
    "!rm -r sample_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 118
    },
    "id": "e3dDxs8LFZdT",
    "outputId": "ebff804d-1c59-4039-d869-f65907b19712"
   },
   "outputs": [],
   "source": [
    "# !git clone https://github.com/manigalati/usad"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "id": "te9stFZtFfZu",
    "outputId": "3ca36b3b-dd9a-413c-873f-ab730285ad51"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/usr/local/data/elisejzh/Projects/Anomaly_Detection/usad\n"
     ]
    }
   ],
   "source": [
    "%cd /usr/local/data/elisejzh/Projects/Anomaly_Detection/usad/"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "id": "6u1DGKsAlLF-"
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "\n",
    "from utils import *\n",
    "from usad import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "id": "4AzWlDBI_djV",
    "outputId": "7a8d0c19-2389-461b-c0be-3427a25dda91"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "GPU 0: NVIDIA GeForce RTX 3090 (UUID: GPU-b7283323-3998-2b76-97e2-3017160af059)\n"
     ]
    }
   ],
   "source": [
    "!nvidia-smi -L\n",
    "\n",
    "device = get_default_device()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "1crx5rGP9ONf"
   },
   "source": [
    "## EDA - Data Pre-Processing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "down_rate=5"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "vxofeE469RhT"
   },
   "source": [
    "### Download dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 84
    },
    "id": "i95DlAZI1G_p",
    "outputId": "5b35771c-356e-4e0b-a997-682d1ea85c6a",
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "mkdir: cannot create directory ‘input’: File exists\n",
      "input/SWaT_Dataset_Normal_v1.csv [Exists]\n",
      "input/SWaT_Dataset_Attack_v0.csv [Exists]\n"
     ]
    }
   ],
   "source": [
    "!mkdir input\n",
    "#normal period\n",
    "!python gdrivedl.py https://drive.google.com/open?id=1rVJ5ry5GG-ZZi5yI4x9lICB8VhErXwCw input/\n",
    "#anomalies\n",
    "!python gdrivedl.py https://drive.google.com/open?id=1iDYc0OEmidN712fquOBRFjln90SbpaE7 input/"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "kfSj4FYL9W8Y"
   },
   "source": [
    "### Normal period"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 87
    },
    "id": "XeDLxV_r1G9n",
    "outputId": "576538dd-64f2-46fa-8e6f-6c2ffdebad15"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/data/elisejzh/anaconda3/envs/USAD/lib/python3.7/site-packages/IPython/core/interactiveshell.py:3457: DtypeWarning: Columns (26) have mixed types.Specify dtype option on import or set low_memory=False.\n",
      "  exec(code_obj, self.user_global_ns, self.user_ns)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(495000, 51)"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Read data\n",
    "normal = pd.read_csv(\"input/SWaT_Dataset_Normal_v1.csv\")#, nrows=1000)\n",
    "normal = normal.drop([\"Timestamp\" , \"Normal/Attack\" ] , axis = 1)\n",
    "normal.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Transform all columns into float64\n",
    "for i in list(normal): \n",
    "    normal[i]=normal[i].apply(lambda x: str(x).replace(\",\" , \".\"))\n",
    "normal = normal.astype(float)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(99000, 51)"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Downsampling\n",
    "normal=normal.groupby(np.arange(len(normal.index)) // down_rate).mean()\n",
    "normal.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "zxFNH5kU9hIE"
   },
   "source": [
    "#### Normalization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "id": "Mfxj4Uxn9kv4"
   },
   "outputs": [],
   "source": [
    "from sklearn import preprocessing\n",
    "min_max_scaler = preprocessing.MinMaxScaler()\n",
    "x = normal.values\n",
    "x_scaled = min_max_scaler.fit_transform(x)\n",
    "normal = pd.DataFrame(x_scaled)\n",
    "\n",
    "# x = normal.values\n",
    "# x_max=np.nanmax(x) \n",
    "# x_min=np.nanmin(x)\n",
    "# x_std = (x - x_min)/(x_max - x_min)\n",
    "# normal = pd.DataFrame(x_std)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 126
    },
    "id": "mQ6_U4jn9nlw",
    "outputId": "f1cc1bd6-f1cc-4764-b1cc-2fd989ac4918"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "      <th>8</th>\n",
       "      <th>9</th>\n",
       "      <th>...</th>\n",
       "      <th>41</th>\n",
       "      <th>42</th>\n",
       "      <th>43</th>\n",
       "      <th>44</th>\n",
       "      <th>45</th>\n",
       "      <th>46</th>\n",
       "      <th>47</th>\n",
       "      <th>48</th>\n",
       "      <th>49</th>\n",
       "      <th>50</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.005241</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.008695</td>\n",
       "      <td>0.075126</td>\n",
       "      <td>0.002009</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.5</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000814</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.001217</td>\n",
       "      <td>0.000147</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.003821</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.008695</td>\n",
       "      <td>0.074071</td>\n",
       "      <td>0.002009</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.5</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000814</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.001217</td>\n",
       "      <td>0.000147</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>2 rows × 51 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "    0         1    2    3    4         5         6         7    8    9   ...  \\\n",
       "0  0.0  0.005241  0.5  0.0  0.0  0.008695  0.075126  0.002009  0.0  0.5  ...   \n",
       "1  0.0  0.003821  0.5  0.0  0.0  0.008695  0.074071  0.002009  0.0  0.5  ...   \n",
       "\n",
       "    41   42   43        44   45        46        47   48   49   50  \n",
       "0  0.0  0.0  0.0  0.000814  0.0  0.001217  0.000147  0.0  0.0  0.0  \n",
       "1  0.0  0.0  0.0  0.000814  0.0  0.001217  0.000147  0.0  0.0  0.0  \n",
       "\n",
       "[2 rows x 51 columns]"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "normal.head(2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "_i71RFAi9spa"
   },
   "source": [
    "### Attack"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 87
    },
    "id": "aN_TFp5x9uTE",
    "outputId": "38d7993d-c9a3-461d-c430-ebde697afbc6"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/data/elisejzh/anaconda3/envs/USAD/lib/python3.7/site-packages/IPython/core/interactiveshell.py:3457: DtypeWarning: Columns (1,9,28,46) have mixed types.Specify dtype option on import or set low_memory=False.\n",
      "  exec(code_obj, self.user_global_ns, self.user_ns)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(449919, 51)"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Read data\n",
    "attack = pd.read_csv(\"input/SWaT_Dataset_Attack_v0.csv\",sep=\";\")#, nrows=1000)\n",
    "labels = [ float(label!= 'Normal' ) for label  in attack[\"Normal/Attack\"].values]\n",
    "attack = attack.drop([\"Timestamp\" , \"Normal/Attack\" ] , axis = 1)\n",
    "attack.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Transform all columns into float64\n",
    "for i in list(attack):\n",
    "    attack[i]=attack[i].apply(lambda x: str(x).replace(\",\" , \".\"))\n",
    "attack = attack.astype(float)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(89984, 51)"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Downsampling the attack data\n",
    "attack=attack.groupby(np.arange(len(attack.index)) // down_rate).mean()\n",
    "attack.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "89984\n"
     ]
    }
   ],
   "source": [
    "#Downsampling the labels\n",
    "labels_down=[]\n",
    "\n",
    "for i in range(len(labels)//down_rate):\n",
    "    if labels[5*i:5*(i+1)].count(1.0):\n",
    "        labels_down.append(1.0) #Attack\n",
    "    else:\n",
    "        labels_down.append(0.0) #Normal\n",
    "\n",
    "#for the last few labels that are not within a full-length window\n",
    "if labels[down_rate*(i+1):].count(1.0):\n",
    "    labels_down.append(1.0) #Attack\n",
    "else:\n",
    "    labels_down.append(0.0) #Normal\n",
    "\n",
    "print(len(labels_down))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "c4cB4v3N-Dhu"
   },
   "source": [
    "#### Normalization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "id": "jZrha9cO-BGK"
   },
   "outputs": [],
   "source": [
    "from sklearn import preprocessing\n",
    "\n",
    "x = attack.values \n",
    "x_scaled = min_max_scaler.transform(x)\n",
    "attack = pd.DataFrame(x_scaled)\n",
    "\n",
    "# x = attack.values \n",
    "# x_std = (x - x_min)/(x_max - x_min)\n",
    "# attack = pd.DataFrame(x_std)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 126
    },
    "id": "z9SwiPco-BUa",
    "outputId": "f2507282-c0f9-4253-ece7-0a802b68240f"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "      <th>8</th>\n",
       "      <th>9</th>\n",
       "      <th>...</th>\n",
       "      <th>41</th>\n",
       "      <th>42</th>\n",
       "      <th>43</th>\n",
       "      <th>44</th>\n",
       "      <th>45</th>\n",
       "      <th>46</th>\n",
       "      <th>47</th>\n",
       "      <th>48</th>\n",
       "      <th>49</th>\n",
       "      <th>50</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.917606</td>\n",
       "      <td>0.577402</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.49767</td>\n",
       "      <td>0.187329</td>\n",
       "      <td>0.064088</td>\n",
       "      <td>0.982593</td>\n",
       "      <td>1.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.975713</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.946163</td>\n",
       "      <td>0.484022</td>\n",
       "      <td>0.944197</td>\n",
       "      <td>0.000074</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.971946</td>\n",
       "      <td>0.579048</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.49767</td>\n",
       "      <td>0.186274</td>\n",
       "      <td>0.064088</td>\n",
       "      <td>0.982202</td>\n",
       "      <td>1.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.976686</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.945687</td>\n",
       "      <td>0.484022</td>\n",
       "      <td>0.943694</td>\n",
       "      <td>0.000074</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>2 rows × 51 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "         0         1    2    3    4        5         6         7         8   \\\n",
       "0  0.917606  0.577402  1.0  1.0  0.0  0.49767  0.187329  0.064088  0.982593   \n",
       "1  0.971946  0.579048  1.0  1.0  0.0  0.49767  0.186274  0.064088  0.982202   \n",
       "\n",
       "    9   ...        41   42   43        44        45        46        47   48  \\\n",
       "0  1.0  ...  0.975713  1.0  0.0  0.946163  0.484022  0.944197  0.000074  0.0   \n",
       "1  1.0  ...  0.976686  1.0  0.0  0.945687  0.484022  0.943694  0.000074  0.0   \n",
       "\n",
       "    49   50  \n",
       "0  0.0  0.0  \n",
       "1  0.0  0.0  \n",
       "\n",
       "[2 rows x 51 columns]"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "attack.head(2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "xXJi503b-j_d"
   },
   "source": [
    "### Windows"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "id": "vyplttZa-BRN"
   },
   "outputs": [],
   "source": [
    "window_size=12"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "id": "dzGJMp6Y-BN5",
    "outputId": "2949d278-1313-442c-f06b-275a8c6c6578"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(98988, 12, 51)"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "windows_normal=normal.values[np.arange(window_size)[None, :] + np.arange(normal.shape[0]-window_size)[:, None]]\n",
    "windows_normal.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "id": "17LdB3c8-pRH",
    "outputId": "721059d4-5937-4dd3-d73c-e5d255fc273c"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(89972, 12, 51)"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "windows_attack=attack.values[np.arange(window_size)[None, :] + np.arange(attack.shape[0]-window_size)[:, None]]\n",
    "windows_attack.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "k70ZFxGs-_7m"
   },
   "source": [
    "## Training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "id": "yi9S0SGnDKNc"
   },
   "outputs": [],
   "source": [
    "import torch.utils.data as data_utils\n",
    "\n",
    "BATCH_SIZE =  7919\n",
    "N_EPOCHS = 25\n",
    "hidden_size = 40\n",
    "\n",
    "w_size=windows_normal.shape[1]*windows_normal.shape[2]\n",
    "z_size=windows_normal.shape[1]*hidden_size\n",
    "\n",
    "windows_normal_train = windows_normal[:int(np.floor(.8 *  windows_normal.shape[0]))]\n",
    "windows_normal_val = windows_normal[int(np.floor(.8 *  windows_normal.shape[0])):int(np.floor(windows_normal.shape[0]))]\n",
    "\n",
    "train_loader = torch.utils.data.DataLoader(data_utils.TensorDataset(\n",
    "    torch.from_numpy(windows_normal_train).float().view(([windows_normal_train.shape[0],w_size]))\n",
    ") , batch_size=BATCH_SIZE, shuffle=False, num_workers=0)\n",
    "\n",
    "val_loader = torch.utils.data.DataLoader(data_utils.TensorDataset(\n",
    "    torch.from_numpy(windows_normal_val).float().view(([windows_normal_val.shape[0],w_size]))\n",
    ") , batch_size=BATCH_SIZE, shuffle=False, num_workers=0)\n",
    "\n",
    "test_loader = torch.utils.data.DataLoader(data_utils.TensorDataset(\n",
    "    torch.from_numpy(windows_attack).float().view(([windows_attack.shape[0],w_size]))\n",
    ") , batch_size=BATCH_SIZE, shuffle=False, num_workers=0)\n",
    "\n",
    "model = UsadModel(w_size, z_size)\n",
    "model = to_device(model,device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 1000
    },
    "id": "So9yjDPEDObC",
    "outputId": "629bcd13-37b1-4907-ef0d-46d9e3ad5398",
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch [0], val_loss1: 0.0490, val_loss2: 0.0461\n",
      "Epoch [1], val_loss1: 0.0436, val_loss2: -0.0008\n",
      "Epoch [2], val_loss1: 0.0380, val_loss2: -0.0134\n",
      "Epoch [3], val_loss1: 0.0341, val_loss2: -0.0179\n",
      "Epoch [4], val_loss1: 0.0317, val_loss2: -0.0198\n",
      "Epoch [5], val_loss1: 0.0306, val_loss2: -0.0212\n",
      "Epoch [6], val_loss1: 0.0371, val_loss2: -0.0278\n",
      "Epoch [7], val_loss1: 0.0370, val_loss2: -0.0289\n",
      "Epoch [8], val_loss1: 0.0362, val_loss2: -0.0291\n",
      "Epoch [9], val_loss1: 0.0359, val_loss2: -0.0296\n",
      "Epoch [10], val_loss1: 0.0354, val_loss2: -0.0297\n",
      "Epoch [11], val_loss1: 0.0349, val_loss2: -0.0298\n",
      "Epoch [12], val_loss1: 0.0344, val_loss2: -0.0298\n",
      "Epoch [13], val_loss1: 0.0341, val_loss2: -0.0298\n",
      "Epoch [14], val_loss1: 0.0338, val_loss2: -0.0298\n",
      "Epoch [15], val_loss1: 0.0334, val_loss2: -0.0298\n",
      "Epoch [16], val_loss1: 0.0332, val_loss2: -0.0298\n",
      "Epoch [17], val_loss1: 0.0329, val_loss2: -0.0297\n",
      "Epoch [18], val_loss1: 0.0327, val_loss2: -0.0297\n",
      "Epoch [19], val_loss1: 0.0325, val_loss2: -0.0297\n",
      "Epoch [20], val_loss1: 0.0326, val_loss2: -0.0299\n",
      "Epoch [21], val_loss1: 0.0327, val_loss2: -0.0301\n",
      "Epoch [22], val_loss1: 0.0333, val_loss2: -0.0308\n",
      "Epoch [23], val_loss1: 0.0335, val_loss2: -0.0312\n",
      "Epoch [24], val_loss1: 0.0336, val_loss2: -0.0313\n"
     ]
    }
   ],
   "source": [
    "history = training(N_EPOCHS,model,train_loader,val_loader)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 295
    },
    "id": "fYwlN0JKVVtN",
    "outputId": "c742ff8b-3b4a-41f5-dd09-effee1be928a"
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZQAAAEWCAYAAABBvWFzAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAAsTAAALEwEAmpwYAABBfUlEQVR4nO3deXxU1fn48c+TyQokYQ8hQXYXNkGiqFWMO26l7isu1S/yU9v6bb9WW9tvbfu11drWvSpWFFyKe0uVikqJSGVHQDYlBJCwB0hIIOvk+f1xb8IQskwmk7kh87xfr3nN3Dvn3PucGZgn99x7zxFVxRhjjGmpGK8DMMYY0z5YQjHGGBMWllCMMcaEhSUUY4wxYWEJxRhjTFhYQjHGGBMWllCMiQIi8h0RWS8iJSLyvTYQTz8RURGJ9ToWEz6WUExEiMgmETnP6zi8JCI5IlImIn0C1p0nIpsisPvfAM+oaidV/XsE9meikCUUYyLrAPBLD/bbF1jtwX5NFLGEYjwlIgki8oSIbHMfT4hIgvtedxH5QEQKRWSviHwuIjHue/eLyFYRKRaRr0XkXHd9jIg8ICIbRGSPiLwlIl3d9xJF5DV3faGILBaRtHpiekBE3qmz7kkRecp9fauI5Ln73igiNzajyU8B14vIoAY+jxPcI5lCEVktIt8NdsMi8l8ikut+VjNEpLe7fgMwAPin2+WVUE/d3iLyrojsdtv0w4D3HhKRd0TkTbfNy0TkxGBiFpEkEfmTiGwWkSIRmSciSQG7vlFEvhWRAhF5MKDeKSKyRET2i8hOEflzsJ+D8ZCq2sMerf4ANgHn1bP+N8ACoCfQA/gC+K373u+B54E493EmIMBxwBagt1uuHzDQfX2vu71MIAF4Afib+96dwD+BDoAPGA2k1BNTX+BgzXtu2e3AqUBHYD9wnPteOjA0yM8gB7gD+DPwmrvuPGCT+zoOyAV+DsQD5wDFNftqYtvnAAXASW67nwbmNvX5u+/FAEuB/3X3OwDIAy50338IqASucmP8H2BjwPfSYMzAs267M9zP8XQ3vn6AAi8CScCJQDlwgltvPjDBfd0JONXrf8P2COLfuNcB2CM6Ho0klA3AxQHLFwb8wP4G+AcwqE6dQcAu98c4rs57a4FzA5bT3R/DWOD7OAlrRBDxzgNudl+fD2xwX3cECoErgaRmfgY1CaUHUAQMrZNQzgR2ADEBdf4GPBTEtl8C/hCw3Mltd7/GPn/3vTHAt3XW/Qx42X39ELAg4L0YnAR7ZmMxu+VKgRPr2WdNQskMWLcIuM59PRf4NdDd63+79gj+YV1exmu9gc0By5vddQCP4fz1+7HbxfQAgKrm4hyJPATsEpHpNd07OEcX77vdL4U4CcYPpAGvArOA6W732h9EJK6BuN4Arndf3+Auo6oHgGuBScB2EflQRI5vToNVdTfwDE7CrPtZbFHV6jqfR0YQmz3sc1TVEmBPkHX7Ar1rPjP3c/s5zmdWY0vAtquBfHefjcXcHUjE+aOhITsCXh/ESYQAtwPHAuvcrslLg2iH8ZglFOO1bTg/aDWOcdehqsWq+hNVHQBcBvy45lyJqr6hqme4dRV41K2/BbhIVTsHPBJVdauqVqrqr1V1CE7Xy6XAzQ3E9TaQLSKZwOW4CcXd9yxVPR/n6GcdTrdNcz0GnI3T7Rb4WfSpOU8U8HlsDWJ7h32OItIR6BZk3S3AxjqfWbKqXhxQJvDKtBicLsVtTcRcAJQBA4OI4TCqul5Vr8fpCn0UeMdtk2nDLKGYSIpzT4zXPGJxukd+ISI9RKQ7Tj/+awAicqmIDBIRwTlv4Qf8InKciJzjnlwuw+lW8bv7eB54WET6utvoISLj3ddni8hwEfG526sMqHcY9ygiB3gZ58d2rbuNNBH5rvvjVg6UNLSNxqhqIfAn4KcBqxfiXAX2UxGJE5FsnEQ6PYhNvgHcJiIj3c/ld8BCVd0URN1FwH73QockEfGJyDAROTmgzGgRucL9zu7FafuCxmJ2j1qmAH92T/r7ROS0+i4KqEtEbhKRHu42Ct3Vzf6cTWRZQjGRNBPnx7/m8RDwf8ASYCXwFbDMXQcwGPgU50d7PvAXVc3BOan7CM5fwDtw/or9uVvnSWAGTjdZMc6P3hj3vV7AOzjJZC3wGW7yasAbOOc43ghYFwP8BOcv873AWcBdACJypoiUBP1pOLHW/kiqagXwXeAit21/wTmPs87d/r9E5Of1bUhVZ+NcjvwuzvmNgcB1wQShqn6cJDAS52R7AfBXIDWg2D9wuvr2AROAK9wjvkZjxjmB/xWwGOfzepTgfnfGAavdz/NJnHMrZcG0x3hHVG2CLWNMw0TkIZwLI27yOhbTttkRijHGmLCwhGKMMSYsrMvLGGNMWNgRijHGmLCIqqGju3fvrv369Qup7oEDB+jYMXovg4/m9lvbo7PtEN3tD2z70qVLC1S1R1N1oiqh9OvXjyVLloRUNycnh+zs7PAGdBSJ5vZb27O9DsMz0dz+wLaLyObGSzusy8sYY0xYWEIxxhgTFpZQjDHGhIWn51BEZBzOsAo+4K+q+kid98V9/2KckUhvVdVl7nubcOZd8ANVqpoVwdCNMe1cZWUlnTp1Yu3atV6HEjGJiYlkZmYSF9fQINyN8yyhuAP0PYsz10Q+sFhEZqjqmoBiF+GM5zQYZzym5zg0LhPA2apaEKGQjTFRJD8/n7S0NDIzM3H+tm3fVJU9e/aQn59P//79Q9qGl11epwC5qprnDjA3HRhfp8x4YJo6FgCdRSQ9UgE+/9kGvthweL76YkMBz3/W2PQOxpj2oKysjNTU1KhIJgAiQrdu3SgrC30MTi+7vDIImLQH5yhlTBBlMnBGU1WcEWUVeEFVJ9e3ExGZCEwESEtLIycnJ+gAdY+fOz8t466RifRJKOW5d2fzl+XOck7OlqY30I6UlJQ067NrT6ztOV6H4YnU1FSqq6spLi72OpSIKisrIycnJ6Tv3suEUl/arzsOTGNlvqOq20SkJ/CJiKxT1blHFHYSzWSArKwsbc415dnAiSMLmPTqUjI7+NhRXs0Lt47h9IHdg95Ge2HX42d7HYYnornta9euxefzkZyc7HUoEZWYmMioUaNC+u697PLKJ2AWOA7NABdUGVWted4FvI/ThRZ2pw/sTr9uHVmzt5oh6SlRmUyMMd7o1KlT04WC9PbbbzN06FBiYmJCvsG7KV4mlMXAYBHpLyLxOJMBzahTZgZwszhOBYpUdbuIdBSRZKid6vQCYFVrBPnFhgK27DtIekdhXm4Bv/sweq74MMYE52g43zps2DDee+89xo4d22r78CyhqGoVcA8wC2f2vLdUdbWITBKRSW6xmUAekIszb/dd7vo0YJ6IrMCZvvRDVf0o3DF+saGAe974kmdvPInffCeJ4RkpTP48j8c/+SbcuzLGHMVGZKZyzxtf1iaVmt+OEZmpTdQMjqpy3333MWzYMIYPH86bb74JwPbt2xk7diwjR45k2LBhfP755/j9fm699dbaso8//jgAJ5xwAscdd1xY4mmIp/ehqOpMnKQRuO75gNcK3F1PvTzgxNaOb2V+Ec/cMIrTB3YnZ4vw5p2n8b1n/8PT/17PSX27cNaxTY6VZoxpB379z9Ws2ba/0TI9kxO4+aVFpKUksHN/OYN6duLJT9fz5Kfr6y0/pHcKv7psaFD7f++991i+fDkrVqygoKCAk08+mbFjx/LGG29w4YUX8uCDD+L3+zl48CDLly9n69atrFrldNoUFhY2q60tYXfKN2KS75+cHnPotpgO8bG8d5GfX3T+mDtfXcLiTXs9jM4Y05akJsWRlpLA1sIy0lISSE0K7ebA+sybN4/rr78en89HWloaZ511FosXL+bkk0/m5Zdf5qGHHuKrr74iOTmZAQMGkJeXxw9+8AM++ugjUlJSwhZHU6JqtOFmyzgJ3r4Vrn7FWd44l04z7uDKy17ktZk+vv/yYv428VSGZYTnsNYY0zYFcyRR0831w3MG8drCb/nReYPDdhFPQxMhjh07lrlz5/Lhhx8yYcIE7rvvPm6++WZWrFjBrFmzePbZZ3nrrbeYMmVKWOJoih2hNKb/WCeZvDmB49f8uTa5pA45l9fuGENKUhy3TFlE7q4SryM1xnioJpk8c8MofnzBcTxzw6jDzqm01NixY3nzzTfx+/3s3r2buXPncsopp7B582Z69uzJf/3Xf3H77bezbNkyCgoKqK6u5sorr+S3v/0ty5YtC0sMwbCE0pT+Y6FLP3rt+gyybneWgd6dk3j19lMQgQkvLSR/30GPAzXGeCXwfCs4txs8c8MoVuYXhWX7l19+OSNGjODEE0/knHPO4Q9/+AO9evUiJyeHkSNHMmrUKN59911+9KMfsXXrVrKzsxk5ciS33norv//97wF4//33yczMZP78+VxyySVceOGFYYktUFTNKZ+VlaXNvv5641x441qoPAhJXeGaqbVJBWDNtv1cO3k+3TrG8/ak0+mRnBDmqNuGaL7Bzdqe7XUYnli7di2ZmZlRd2Pj2rVrOeGEE+pOsLU0mAF47QilMRvnOt1cp9zpLI/7vbO88dAN+UN6p/DKbSezc385E15aSNHBSk9CNcYYr1lCaczWZc45lIFnO8spvZ3lrYf3SY7u25XJN48mb/cBbntlEQfKqyIeqjHGeM0SSmPOuNfp3krNdJb3b3OWz7j3iKJnDu7BU9eP5MtvC7n2hfmUV/lr32trd8waY0xrsIQSjGR3xPyi/EaLjRuWzp1nDWTVtv3c+NeFVPmrw37HrDHGtFV2H0ow4jtQGZtM3P66Y1ce6YGLjudgRRXT5m/m0qfnsWt/Oc/cOMoGlTTGtHt2hBKk8oRuTpdXEH4zfhhj+ndl3Y5i+nRN4rQB3Vo5OmOM8Z4llCA5CWVrUGW/2FDA+l0ljOzTmRX5RfzkrRWtHJ0xpj0K5/D19913H8cffzwjRozg8ssvb5UxviyhBKk8oXtQRyiBd8y+f9fpnHVsd977ciu//HurjK5vjGkL5j1x2O0EgLM87wkvoqnX+eefz6pVq1i5ciXHHnts7Q2P4WQJJUjlCd3gYAFUNj7fcuAdsyLCS7eczMn9uvDqgs28tSS6pg02JmrUjPtXk1Rq7mHLOCksmw/H8PUXXHABsbHOafNTTz2V/PzGLzIKhZ2UD1J5gntSvXg7dO3fYLlJZw08bDnWF8Nrd4zhjqlLeODdlaQkxjJuWHprhmqMCbd/PQA7vmq8THI6vHq581y8HXocDzmPOo/69BoOFz0S1O7DPXz9lClTuPbaa4Pad3N4eoQiIuNE5GsRyRWRB+p5X0TkKff9lSJyUp33fSLypYh80Nqxlie4J9aDPI8SKCHWxwsTRjOyT2d++LflfL5+d5ijM8Z4LrGzk0yKtjjPiZ3DtulwDl//8MMPExsby4033hi2+Gp4doQiIj7gWeB8nLnjF4vIDFVdE1DsImCw+xgDPOc+1/gRzmyPrT7gf+0RSpBXetXVIT6Wl289hWsnz2fitKW8dscYRvftEsYIjTGtJpgjiZpurrE/hSUvQfb9h4371xLhGr5+6tSpfPDBB8yePRsRCUtsgbw8QjkFyFXVPFWtAKYD4+uUGQ9MU8cCoLOIpAOISCZwCfDXSARbntDVeRHCEUqN1A5xTLv9FNJSErjt5UWs3d74DHDGmKNETTK5+hU450Hnuc64fy0RjuHrP/roIx599FFmzJhBhw4dwhJXXV6eQ8kAAs9S53P40UdDZTKA7cATwE+BRocCFZGJwESAtLQ0cnJyQgq2pKyaKl9HdqxdTG5VaNuocc8w5eEFfq59bh4/H5NIr45t/9qIkpKSkD+7o521PcfrMDyRmpqK3++nuLi4ybLxefPxX/IX/N1HQXExdB+F75K/4MubT0X3US2Ko7i4mPPOO4/PPvuM4cOHIyL8+te/pmPHjvz973/nqaeeIi4ujo4dO/LCCy/wzTffcNddd1FdXQ3Ar371K4qLi7nrrruoqKjg3HPPBeDkk0/miSeeOGJ/ZWVl5OTkhPbdq6onD+Bq4K8ByxOAp+uU+RA4I2B5NjAauBT4i7suG/ggmH2OHj1aQzVnzhzVZ8ao/u2GkLcRaP3O/TrqNx/r6b+frdsKD4Zlm61pzpw5XofgGWt7dFqzZo3u37/f6zAibs2aNap6+HcPLNEgfmO9/NM4H+gTsJwJ1D1B0VCZ7wDfFZFNOF1l54jIa60Xqis1o0VdXoEG9Uxm6m2nUFRaySVPfc5Hq7Yf9r4NKGmMOdp4mVAWA4NFpL+IxAPXATPqlJkB3Oxe7XUqUKSq21X1Z6qaqar93Hr/VtWbWj3ilN4hn5Svz/DMVF66JYvi0irufuNLZq/dCWADShpjjkqeJRRVrQLuAWbhXKn1lqquFpFJIjLJLTYTyANygReBuzwJtkZKBpTsgqqKsG1yzIBuvHDzaFSVO19dyiP/Wlt7p70NKGmMtzSKZrSFlrfX07PBqjpTVY9V1YGq+rC77nlVfd59rap6t/v+cFU9Yv5eVc1R1UsjEnBKb0Cdm5bC6Jzj03j82pFUVSvPf5bHxcN6HZXJ5PnPNvDFhoLD1jXVdRdKHWMiITExkaKioqhJKqrKnj17SExMDHkbdqd8c6T0dp73b4MufcO66R7JCSQnxlLlr+a1hd/SuUMc/3Ph8WHdR2sbkZlae3Q1JD2FhRv38sC7K/nDVSdSXFaJL0aIEUEEYsR5PSIj9bAjssCx0IzxUmZmJitWrKCkpMTrUCImMTGRzMzMkOtbQmmOlAznOUwn5mvU/Ii+MGE0A7p34uYpC3lmzga+3lHCMzeOIiHWF9b9tZbTB3bnmRtG8f2XF1NWVV27/r+mHXFgeYQbXlxIh3gfFf5qLjghjfx9pazYUsjgtE50iD/8n+nzn21gRGbqYUdxX2woYGV+0RFD3xgTqri4OEpKSsjKyvI6lKOGJZTmCDxCCaPAASUBPvzhmdw7/Us+/GoHVz8/n2dvOIk+XVvnRqRw27qvtDaZnHVsD846tgfVqqhCtSrV7rO6r/3VzusvNuxhyeZ99ExOYPa6XcxctQMAETimaweOTUvm+F7JHJuWTNeO8dz9+jKevfGkoI5qLAEZExmWUJojMRXik8OeUOr+qMX5Ynj2xtFctmo79729kkufnsfj157IOcenhXW/4bZiSyE/e+8rYmOESWcN4I1FW7jzrAFNng/6YkMBry38lh+eM4jXFn7LS7ecTEaXJL7eUcw3O4v5ekcxX+8s5t/rduGvdvqzfTEw4a8LSUtNZHdxOScd04X3l21l1qoddEiIpUOcz3mO91FSVsnEaUv54bmDOHVAN/L3HuQXf1/NMzc23K1mSciY5rOE0lwpvWF/+Id9rs+4Yekc3yuFu15fxvdfWcLdZw/kv887llhf27uzfndxObe+vIhqVZ67aTQXDu3F6YO6N3nFWuDRxekDu3PqwG61y+OG9WLcsF61Zcur/OTtPsA3O4tZt6OYj1btYGPBAXomJ1BQUs63ew9ysMLPwYoqKv1Hnkj93cx1ta9jBP7nrRWkpSaSnppIWorz3Cs1iV4piaSnJNYeBdWNsyGWhEy0s4TSXGG+F6Up/bp35L27TuehGat5ds4Glm0u5KnrR9EjOSFiMTSloqqau19fRnFZFf/3vWFcONRJAjXnVFbmFzWYUOp29zVWJyHWxwnpKZyQnkKPDQW8uXhL7VHNb7837LDylf7q2uRysMLPwXI/r3yxkXeXbSX7uB4M7Z3CjqJyduwvZd2OYnK+3s3BCv8R8d344kKS4+GgfxGn9e/GZ1/vZvXW/fRITqB7pwT3OZ4uHeIPuyjBuuJMNLKE0lwpGbBhXdPlwigxzscjV45gdN8u/PIfq8h+bA4/vuBYbj9jQG0ZL3+IHv5wDYs27eXJ60YyfmTGYe+dPrB7o11e9cXbVJ3Gjmpq6sX5YkhNiiE1Ka62zpyvd9cmoIljD++KU1WKy6vYWVTG9qIyduwvY0dRGbNW72D1tv2kpcSTV3CAhZv2UhFwwUENX4zQrWM8yQmx3DJlEYN6dmJjwQGuHt2HPSUVLMjbQ49kJwElJ8QiIs1OQGBJyLRtllCaK6U3FO8AfyX44iK666uz+jAsI5XbXl7Mbz9Yy4ZdJTx8+XDm5+3x7FLbt5dsYer8zdxxRv8jkklrac5RDQSXgESElMQ4UhLjGJyWXFvvlS828d2BcczboTx+7QhOG9CN4vIqdheXU1BcTkFJBbuLy9zncgpKyimt9LN2ezExAq8u2MyrCzYfFk9CbExtchnQvSO3vbyYEzM7s2pbERNO7UtJWRUL8vaQmhRHSlIcKYmxdLIkFPVC+R4j/d1bQmmumpsbS3ZCaujXa4fqhPQUPvnxWO6Yupg3Fm1hXu4eCg9W8PyE0RG/GXLFlkIe/PsqTh/YjQcuitw9M809qmluAoLDk1DFllVcd86ww37IUxLjGNijU4P1ao6EHr1iOP26d2R3cTm7S8qd55qHuxwjsGjTXgBemJtXbzwxgptc4khNiuPmlxbRt1sH8veVcv6QNJZt3kfurhInKSbF1pZLSYrj+LTkVu+Ks6TVPKF8XqH8MRFKnZawhNJctfeibPMkoQAkJ8YxfeJp3DJlEXPXO3eZP/PvXCr9ytjB3Vtl4py6dheXc+erS+nRKYFnbjipTV4oUCOUbrXAJJSzpflJ6IgjoUGN17n9jGN4feFmfnXZUAb17MT+0kqKSivZX1bJ/tKqgNeV7C+roqzSz4bdB+gY72P22l18sLLx0Rt8Mc75oNQOcZSUVTE8I5V3lubz6ZpddOkQR+cOcXTuEE/nDnEkJ8Ty/15bxp+vOZEY1Vb74YrmJBTM51Xlr+ZgpXP+72BFFSmJcdx99kAmTlvKWcf24LNvdnP7Gf0pKKngH8u34q9W9zJ88KvWXpJ/2Yh07pi6hKtHZ/LPldtbdVgnSyjNldo6Nzc21/y8Pazatp87xw5g2oLNrN2+n1umLOKE9BQmnTWAi4enE9dKP/I1J+ELSyt4Z9LpdO0Y3yr78VJLk1BN+eZ0xZ0+qOkEFFiv5ijomRtGkdW362EJZ389yejz9btZtXU/fbokUeGvZmHeXopKKykpr6p3P7dPrbkhdSEpibHc9/ZKEuJiSIz1kRgXQ2Kcz30460b26cxtLy9mRGYqq7ftr+2+W7RxL6lJzhFT5w5xJMY5N+pG4iKGSNQJpryqsr+sim2FpbWP7wzszq1TFtMzJYHtRWWkpyZw39srKa30c6C8ivJ6ztXV+PAr5w+IJ2evb7BMXVPnb+aH5wxq1Z4MSyjNVXNzY5F3CaXuD9FZx/Xg7teXcefYAcxet4sfTV/OHz76mjvO7M+1J/c54k7zlgo8CT8sw0ZErhHprrj6zgd171T/1X9f1Lkq7tGrRtTuo6KqmqLSSgoPVlBYWknhwUr2HaxgxvJtzMstYFSfzgzNSKGssprSSj/llX7KKqspq/Szv6yy9nVZZTWqyuJN+4CGu+/iY2Po7CaYHp3iuWXKIvp378jmPQdr//JetHEviXE+kuIOJa+KqmrufHUpPz7/WE7u15V12/fzfx+u5ZErh3OwoorEWB8xMYcfnbe0m6j2M3/9Sx6/7kRKyquorKqm0l9Nhb+aSr/StWM8k15dyv9ceByDenZi3voCXv7PJk4f2I0FeXvcBFJ2ROKO8wlJcT7y95XSp2sSQ9NT6ZDgo2N87KHneB8d3fupOsTHsqmghKf/ncvFI9KZ+dV2fnHJEE46pgsxQu3QRjExgk+EGIGYGGHZ5n3c/+5KbhhzDK8t/JZTB3ZrtaQi0TLwGUBWVpYuWdL0MCD1ycnJITs7G1Thd71h9G0w7nfhDTBIjf1FNPHMAfx73S5emLuBxZv20blDHDef2hcFTqvzD6k53Qs17X97yRbue2cld5zRn19cOqQ1mtfm1H73bUAof3HXTUJ1lxurc0YvZd4OCaqbpKbOjaccw2sLN/PQZUMZ2LMThQed7rvC0gqKSispqll2n9fvKqagpIKO8T5ifTGUVvrrvZIuGPGxMSS5iSgp3jmCqvT72VRwkN6dk9hWWMqgHp3okOCjqlqp8itV1dXus1Llr6aqWimt9FNSVoUIVIf4E9m1Yzy9OyfSOzWJ3p2TyOjsPPfunEhG5yTW7yzhB9O/5Cb3h76pz7gl32Nz6tQI/HcvIktVtckxaOwIpblE3HtRvDtCaeov4fOGpHHekDSWbt7LC5/l8dS/c4mLESbPzeORK4dz+ajMkE7OeXUS3hwS6a64+i5IqE/dH6rTArrvzhjcvO670wd2x1+tlFf5Ka3wU1ZV7TxXOo/XFmzm78u3ceHQXpx7fE9KK/2Uuu+VVvopq6hZrq5df6Dcz7d7D9I7NZHOHeOI88XgixFiY2KIjRFifeI+xxDnE3wxwlf5RazILyKrbxfOGNydOF8M8e77cbExtcuxPiHOF8OHK7czY8U2bjr1GB68eAhJ8Q2PwffFhgJ+ML3xKw9b+j2GWqdFgpnWsb08WjwFcI1XLlV98byQtxVp63cW60/fXqEDf/ah9r3/Az3td5/qcQ/O1D/OWqebCw5odXV1k9v4+0f/1jEPf6qn/3627ikpj0DUbUe0TYP7XE6u/id3t6oeavt/cnfrczm5QdWp0VSd/+Tu1lG/+bi2Xt3lxur8ada6JsuGo84PJs8Kqk5z9xHK5xVpoUwB7PmPfCQfYUso792p+qchIW/LKzuKSvXKv/xH+97/gfZ/4APte7/zGPnrWTrhpYX6x1nr9OPVO3RnUamqHvpHX17p1/MfmanH/WKmvr5gU5v6Rx8J0ZZQArVm25v7o9qSBBRqnTlz5jRZJ5R9HA1CSSiednmJyDjgScAH/FVVH6nzvrjvXwwcBG5V1WUikgjMBRJwuu3eUdVfRSzwlAxnkq1qP8QcHUPLA2zYXUJewQF+eM4gXl2wmfsuPA4FVm4pYkV+IX/JKagdfLFXSiKZXZJ48tP1ZPXtwjf7qrnn7IH88eNvbK4SExaRuIihpXWCuWQ84t1KbZhnCUVEfMCzwPlAPrBYRGao6pqAYhcBg93HGOA597kcOEdVS0QkDpgnIv9S1QURCT6lN6jfmQ44JT0iu2ypxq4OunGMM1lYaYWf1ducfuOV+YWszC+itNLP57kFDEgV3li0xaYmNp4J5fxRJOqEso/2yssjlFOAXFXNAxCR6cB4IDChjAemuYdcC0Sks4ikq+p2oGYatTj3EbnL1QIn2jpKEkowf0UlxfvI6teVrH5da+sVHazk4ZlreGtJPj8855io/E9ijAmOlwklA9gSsJyPc/TRVJkMYLt7hLMUGAQ8q6oL69uJiEwEJgKkpaWRk5MTUrAlJSW1dTsVbyMLWDX/Ywp6HB3Tgx4PVGzZQs6WI9fn1F0ZYO0eP/9aUca4PsrLn+eSVJzPCd2Onm6+cAj87qNNNLcdorv9obTdy4RS3/ggdY8yGiyjqn5gpIh0Bt4XkWGquuqIwqqTgcng3IcS6v0Eh92LcGA4LP1vhvXpAqeGtr2jwRcbCnjx8y954dYxVGxZxc3jai4fPTGqjlTa0n0okRbNbYfobn8obfdyAKZ8oE/AciZQd6KRJsuoaiGQA4wLe4QN6dAVfAmeD7/S2hrrJjPGmLq8TCiLgcEi0l9E4oHrgBl1yswAbhbHqUCRqm4XkR7ukQkikgScB0RukpLamxsjN9GWFyadNfCII5HTB3Zv9wP3GWNC41mXl6pWicg9wCycy4anqOpqEZnkvv88MBPnkuFcnMuGb3OrpwNT3fMoMcBbqvpBRBuQmtnuE4oxxjSHp/ehqOpMnKQRuO75gNcK3F1PvZWAtzdDpPSGb+d7GoIxxrQlbXcSi7YupTfs3w7VoQ1iZ4wx7Y0llFClZEB1JRzY7XUkxhjTJlhCCVXNvCjt/EovY4wJliWUUNUmFDsxb4wxYAkldIFzyxtjjLGEErIO3cEXb11exhjjsoQSqpgYSE63hGKMMS5LKC2RkmFdXsYY47KE0hIezy1vjDFtiSWUlqgZz0sjNxWLMca0VZZQWiIlA/wVcHCP15EYY4znLKG0RGrAzI3GGBPlLKG0hN3caIwxtSyhtETNzY1F+d7GYYwxbYAllJbo2ANiYu0IxRhjsITSMjE+9+ZGSyjGGONpQhGRcSLytYjkisgD9bwvIvKU+/5KETnJXd9HROaIyFoRWS0iP4p89C67F8UYYwAPE4o7fe+zwEXAEOB6ERlSp9hFwGD3MRF4zl1fBfxEVU8ATgXurqduZETB3PLGGBMML49QTgFyVTVPVSuA6cD4OmXGA9PUsQDoLCLpqrpdVZcBqGoxsBbIiGTwtWqGX7GbG40xUc7LOeUzgC0By/nAmCDKZADba1aISD+c+eUX1rcTEZmIc3RDWloaOTk5IQVbUlJSb93MXaUMqipl3qcfUBWXHNK2jwYNtT8aWNtzvA7DM9Hc/lDa7mVCkXrW1f0zv9EyItIJeBe4V1X317cTVZ0MTAbIysrS7OzskILNycmh3rqrC2HDS5wxvB/0Gh7Sto8GDbY/Cljbs70OwzPR3P5Q2u5ll1c+0CdgOROoezKiwTIiEoeTTF5X1fdaMc7G2URbxhgDeJtQFgODRaS/iMQD1wEz6pSZAdzsXu11KlCkqttFRICXgLWq+ufIhl2HzS1vjDGAh11eqlolIvcAswAfMEVVV4vIJPf954GZwMVALnAQuM2t/h1gAvCViCx31/1cVWdGsAmOTmkgMXaEYoyJel6eQ8FNADPrrHs+4LUCd9dTbx71n1+JPF8sdOplCcUYE/XsTvlwSM2wLi9jTNSzhBIOKb2hyBKKMSa6WUIJB7u50RhjLKGERUpvqDwAZUVeR2KMMZ6xhBIONtGWMcZYQgkLu7nRGGMsoYRFis0tb4wxllDCIbkXIHaEYoyJapZQwsEX59wxv9/mljfGRC9LKOFiE20ZY6KcJZRwsYRijIlyllDCpebmRmOMiVKWUMIlpTeU74eyeuf5MsaYds8SSrikZjrPxdsbL2eMMe2UJZRwsYm2jDFRLqiEIiI/EpEUd+bEl0RkmYhc0NKdi8g4EflaRHJF5IF63hcRecp9f6WInBTw3hQR2SUiq1oaR1jUJBQbddgYE6WCPUL5vqruBy4AeuDMnPhIS3YsIj7gWeAiYAhwvYgMqVPsImCw+5gIPBfw3ivAuJbEEFbJ6c6znZg3xkSpYBNKzeyIFwMvq+oKWj5j4ilArqrmqWoFMB0YX6fMeGCaOhYAnUUkHUBV5wJ7WxhD+MQmQMce1uVljIlawU4BvFREPgb6Az8TkWSguoX7zgC2BCznA2OCKJMBBH3mW0Qm4hzdkJaWRk5OTiixUlJS0mTd0ZJCxaZVfBXiPtqyYNrfXlnbc7wOwzPR3P5Q2h5sQrkdGAnkqepBEemK0+3VEvUd4dSdoSqYMo1S1cnAZICsrCzNzs5uTvVaOTk5NFl3+3Gwb3PT5Y5CQbW/nbK2Z3sdhmeiuf2htD3YLq/TgK9VtVBEbgJ+AbR0Nql8oE/AciZQ9wREMGXajhSbW94YE72CTSjPAQdF5ETgp8BmYFoL970YGCwi/UUkHrgOmFGnzAzgZvdqr1OBIlVtuzd6pPSGskKoOOB1JMYYE3HBJpQqVVWck+RPquqTQHJLdqyqVcA9wCxgLfCWqq4WkUkiMsktNhPIA3KBF4G7auqLyN+A+cBxIpIvIre3JJ6wsIm2jDFRLNhzKMUi8jNgAnCme8lvXEt3rqozcZJG4LrnA14rcHcDda9v6f7DLvDmxu6DvY3FGGMiLNgjlGuBcpz7UXbgXGn1WKtFdbSyueWNMVEsqITiJpHXgVQRuRQoU9WWnkNpf2z4FWNMFAt26JVrgEXA1cA1wEIRuao1AzsqxSVBh252hGKMiUrBnkN5EDhZVXcBiEgP4FPgndYK7KhlE20ZY6JUsOdQYmqSiWtPM+pGl5QMGyDSGBOVgj1C+UhEZgF/c5evpc7VWcaV0hu2LPI6CmOMibigEoqq3iciVwLfwRkOZbKqvt+qkR2tUnpD6V6oLHXOqRhjTJQI9ggFVX0XeLcVY2kfAm9u7DbQ21iMMSaCGk0oIlJM/YMxCs59hymtEtXRLPBeFEsoxpgo0mhCUdUWDa8SlVLcueXtSi9jTJSxK7XCLaVm5ka70ssYE10soYRbfEdI7GwJxRgTdSyhtIaUDOvyMsZEHUsorSGltx2hGGOijiWU1mDDrxhjopAllNaQkgEHdkNVudeRGGNMxHiaUERknIh8LSK5IvJAPe+LiDzlvr9SRE4Ktq6nUt2bG4vb7mzFxhgTbp4lFHfWx2eBi4AhwPUiMqROsYuAwe5jIs7c9sHW9Y5NtGWMiUJeHqGcAuSqap6qVgDTceasDzQemKaOBUBnEUkPsq435j0B+90jk5pRhzfOddYbY0w7FvRYXq0gA9gSsJwPjAmiTEaQdQEQkYk4RzekpaWRk5MTUrAlJSVB1e28L4ahOfcTB2xY/jnFG7YzZM1jrBlyH4Uh7rstCLb97ZG1PcfrMDwTze0Ppe1eJhSpZ13dccMaKhNMXWel6mRgMkBWVpZmZ2c3I8RDcnJyCK5uNowcCVO/y8DS5bDjA7jhdUb2HxvSftuK4Nvf/ljbs70OwzPR3P5Q2u5ll1c+0CdgOROoe9KhoTLB1PVO/7HQbRBsXwEn3eIsG2NMO+dlQlkMDBaR/iISD1wHzKhTZgZws3u116lAkapuD7KudzbOhZKdzutFk51lY4xp5zxLKKpaBdwDzALWAm+p6moRmSQik9xiM4E8IBd4EbirsboRbkL9Ns6Ft2+Fa16DLv2h8zHOsiUVY0w75+U5FFR1JnWmElbV5wNeK3B3sHXbhK3L4OpXnG6urNvgk/+F7z7trLeuL2NMO2Z3yofbGfceShwjbwJfgnMu5Yx7vYzKGGNanSWU1tSxGwz9Hqx4E8pLvI7GGGNalSWU1pZ1O1QUw1dveR2JMca0Kksora3PKZA2DBZPAa33VhljjGkXLKG0NhHI+j7s/Aryl3gdjTHGtBpLKJEw4hqI7wRLXvI6EmOMaTWWUCIhIRlGXAur3oODe72OxhhjWoUllEjJ+j74y2H5615HYowxrcISSqT0GgZ9xsCSKVBd7XU0xhgTdpZQIinrdtibBxtzvI7EGGPCzhJKJA0ZD0ldnaMUY4xpZyyhRFJcIoy6CdbNtOmBjTHtjiWUSMu6DdQPy6Z5HYkxxoSVJZRI6zoABp4LS6eCv8rraIwxJmwsoXgh6/tQvA2++ZfXkRhjTNh4klBEpKuIfCIi693nLg2UGyciX4tIrog8ELD+ahFZLSLVIpIVucjD5NhxkJIBi+3OeWNM++HVEcoDwGxVHQzMdpcPIyI+4FngImAIcL2IDHHfXgVcARyd0yD6Yp255vPmwJ4NXkdjjDFh4VVCGQ9MdV9PBb5XT5lTgFxVzVPVCmC6Ww9VXauqX0ci0FZz0s0gPruE2BjTbng1BXCaqm4HUNXtItKznjIZwJaA5XxgTHN3JCITgYkAaWlp5OTkND9aoKSkJOS6DRna7RQ6L57K/NizqPbFh3Xb4dYa7T9aWNtzvA7DM9Hc/lDa3moJRUQ+BXrV89aDwW6innXNnlBEVScDkwGysrI0Ozu7uZsAICcnh1DrNugYYNp4xnbfCydeF95th1mrtP8oYW3P9joMz0Rz+0Npe6slFFU9r6H3RGSniKS7RyfpwK56iuUDfQKWM4H2dTdg/7Og2yDn5HwbTyjGGNMUr86hzABucV/fAvyjnjKLgcEi0l9E4oHr3HrtR83kW/mLYMdXXkdjjDEt4lVCeQQ4X0TWA+e7y4hIbxGZCaCqVcA9wCxgLfCWqq52y10uIvnAacCHIjLLgzaEx4nXQ2yiXUJsjDnqeXJSXlX3AOfWs34bcHHA8kxgZj3l3gfeb80YI6ZDVxh6Bax8C87/DSSmeB2RMcaExO6UbwviO0LlAVj55qF1G+fCvCc8C8kYY5rLEkpbcMJlzj0pXzwDqk4yeftWyDjJ68iMMSZollDaggFnwan/Dwo3wRvXOMnk6leg/1iPAzPGmOBZQmkrzvkldO4L6z+GHsdDvzO9jsgYY5rFEkpbkb8IKkogbRhs/g+8fjX4K72OyhhjgmYJpS2oOWdy9SswaR6MuBZyP4GXLoTyEq+jM8aYoFhCaQu2Ljt0zkQErpgMp90D25bBKxdD8U6vIzTGmCZZQmkLzrj3yBPwFz4MN7wJBevhpfOcZ2OMacMsobRlx14It34IlaXw0vnw7QKvIzLGmAZZQmnrMk6C2z+BpK4w9buwpr5hz4wxxnuWUI4GXfs7SSX9RHjrFljwvNcRGWPMESyhHC06doNbZsDxl8BH9zuJpbr60Ps2VIsxxmOWUI4mcUlwzTQ4/lJY83eYdhlUltlQLcaYNsGrKYBNqGJ8cO1r8MG9sPQVePJE54bIq6bYUC3GGE/ZEcrRSAQuexJO+C6U7HASytu3wfuTIC8Hqv1eR2iMiUKWUI5WG+c6Q7SMvQ8SUqHfd2DdhzBtPDwxHD59CHat8zpKY0wU8aTLS0S6Am8C/YBNwDWquq+ecuOAJwEf8FdVrZnZ8THgMqAC2ADcpqqFkYi9TQgcqqX/WOfx9q1Ot1d5MayYDv95CuY9DukjnVkhD+6B/mce3i22ca5zl/4Z93rSDGNM++LVEcoDwGxVHQzMdpcPIyI+4FngImAIcL2IDHHf/gQYpqojgG+An0Uk6rYicKgWcJ6vfgV2roZhV8CNb8FP1sGFvwfUuSrs8z/Ca1fAnN85N0raiXxjTJh5dVJ+PJDtvp4K5AD31ylzCpCrqnkAIjLdrbdGVT8OKLcAuKo1g21z6juiqDlSqdGpJ5x2l/PYuQZWTodlr8Jnj8Lcx0BiYMR1znz2/krwxUUsfGNM+ySqGvmdihSqaueA5X2q2qVOmauAcap6h7s8ARijqvfUKfdP4E1Vfa2BfU0EJgKkpaWNnj59ekgxl5SU0KlTp5Dqthnq5/i1T9Jr12eUx3UmobIQgCpfEkWpQ9nXZQT7uozgQMe+IDH0+fY9ipMHUdhlRG37O+9bSXJxLluOucLbtkRQu/juQxTNbYfobn9g288+++ylqprVVJ1WO0IRkU+BXvW89WCwm6hn3WHZT0QeBKqA1xvaiKpOBiYDZGVlaXZ2dpC7P1xOTg6h1m0zNs6Fxatg7E9JWPISXPE6VFcSu3Eu3fI+o9uGKU65Dt2co52e6bD8T3DNNHK+7UR23xhY9ARc/QoDo+gS5Xbx3YcomtsO0d3+UNreaglFVc9r6D0R2Ski6aq6XUTSgV31FMsH+gQsZwLbArZxC3ApcK56cZh1tDniRP6Zh5YvfdwpU5TvlNs4F/I+g2L34371exyXdjYsWm5TExtjGuTVSfkZwC3u61uA+kY8XAwMFpH+IhIPXOfWq7n6637gu6p6MALxHv0aOpG/ddmhMqmZMPIGuPx5+PEauGcJXPIniO9E+o7ZzjkXSybGmAZ4lVAeAc4XkfXA+e4yItJbRGYCqGoVcA8wC1gLvKWqq936zwDJwCcislxEbLTEptQ350r/sQ1fMiwC3QdD92NBYqgmBha9ABvmtHakxpijlCdXeanqHuDcetZvAy4OWJ4JzKyn3KBWDdA4arrJrn2Vrxf+mxPWPQ5/uw5ufNuOVIwxR7A75U3DArrJdvbKhpPvgKoy+LLeC+qMMVHOBoc0DavbHXbh72H7CmeIl93fQI9jPQnLGNM22RGKCV5sPFw91bkZ8s2bnGFejDHGZQnFNE9qBlz9MuxZD/+4B+yKbWOMyxKKab7+Y+G8h5xJvuY/63U0xpg2whKKCc3pP4QTLoNP/hc2/cfraIwxbYAlFBMaERj/F+g6wLm0eP92ryMyxnjMEooJXWIKXPsqVByAt2+BqgqvIzLGeMgSimmZnifA+Kdhy0L45JdeR2OM8ZDdh2JabtiVkL8UFjwLGVkw4mqvIzLGeMCOUEx4nP9r6NwX/nG3M6FXjY1zYd4TnoVljIkcSygmPHxxcP5vwV8Br10JZUU2zbAxUca6vEz4DB0PxY84c9g/fwaU7oPLnraBJI2JEnaEYsLr1Ekw+AIo/NYZmuWdW+Hx4fDO7bDwBdj2pTOHfY15TzhHMoGa6iYLpY4xptVZQjHhtXEubF0KZ/wEElMh6w6ny2vzF/Cvn8LkbHjkGHj5Evj011BdBW/dcihBBNNNlnGSU6Y5dYwxrc4SigmfwGmGz/tfuPY1WPM+nHw7/GQt/PdquGoKnHQzVB6EL56Cf/8WSvfCtPHw1Cjn/EuvEbDqXfjoZ07S+ewx+OJpWPxX+PJ1OFAAp94F02+Ef97rJKSrXm64ay1SR0HNrWNxtY+4TC1PEoqIdBWRT0RkvfvcpYFy40TkaxHJFZEHAtb/VkRWurM1fiwivSMXvWlQU9MMp2Y6lxhf9ChMnAMPbIFbZzrjgnUdAHvzICHZeV4305l35YunYM7/wce/gA9/Av+4C965zUlE5fth6ctOQnr9anhiOLx0Abw5AWb+FD7/Myx/w7mr/80JzrD7qq13FNTcOpHYh8XV+nUsAdUS9WC0WBH5A7BXVR9xE0UXVb2/Thkf8A3OFMH5OHPMX6+qa0QkRVX3u+V+CAxR1UlN7TcrK0uXLFkSUsw5OTlkZ2eHVLc9aNX21/yHzbodlrx0eFIC55xLZakzuVflQagsg83/gU8fggHZkDsbjr3QudKseMehR3lRPTsTQCGuA8QmQEycUy8m1n2OA1+s81x5EPbkUhrXlaTKfZB+InTqCTE+EJ9TJybWWa5ZV7ILNsyGtKGwczUMOg9SerujMmudZ6B4G+TlQPfjoOBrGHiuk3glxtmeiLvtmuUYKMp3BubsfRJsWwZDr4Au/ZztBgr8v71vk3O0mD4Sti2H4y5y4qr2O92ONQ+tdp73b4MtCzkQ34OOFbuhzxjolFYnljqxFW+H3E8Ptb3fGdChm3Pln78SqsoPvfZXgL8cSgth/1bnD4nyYujQA+ISAxtx+MuqMjhYAPGdoKIEkro631t9n69WO6+rKpzvMjbB2W9KhvM9xnVwthPfAeI7QlxH5zm+g/PvZ/kb7EoZTs/i1XD2z6Hvd9z3O0FCJ6e+yOFH5v3HHrl8lAr8Py8iS1U1q6k6Xl3lNR7Idl9PBXKA++uUOQXIVdU8ABGZ7tZbU5NMXB054n+SOWrU/c/X/8wj/zP63B99Ug7VmfMwXPd64/+BKw5CSUCCWfYq5P0bMk+BzCznh626EvxV7nPd5WSoKCGpKB+S050f2qItAT/CAc8a+Fqd80i+BNg0z4lFBJAjnwEkFnasdH6gtiyEb+dDdbXzg6h+90fef2i5xmZ328ubMYPmt/Od/X49MyApugkxMEGKD+KT6Vi6FZK6OZ/f/q1uLI3EptVO22OTnPuRYuPBV/OIcz4TXxzEpTrruvR3fqB3r4WeQ6DX8EOfSw0JXBbns9q5CtKGQe+R9X+uEnP4uvwlTvJNGwpdBzrDBVUehP35zr+TyoPOuooDh33GPQvmOy8+eoAjyaEE40uAad9zknTJLhh+FRTvdPbbdQAkdTm8HfOecI54Av+9bpzrHM3XndjuKOLVEUqhqnYOWN6nql3qlLkKGKeqd7jLE4AxqnqPu/wwcDNQBJytqrsb2NdEYCJAWlra6OnTp4cUc0lJCZ06dQqpbnvQWu3v8+17FCcPorDLiNp1nfetJLk4ly3HXBG2Op33rWTImsfY1nscvbd9xJoh9x1Wv7E6m7qfS7+C2c2qE+x+Qopr70qGrH2M7ekXkL79Y9ae8BMKOw8HQKVuaXH38xVD1v6x2XG1ZttDqdPq+1AlprqSLnuXcfzXT7O1yxgy9i1g8zFXU5aUjs9fhs9fGvAoq12XXLyBDqXbqIpJJLa67LDNVvk6UprUq/Yh1VX03v4JXx97F7t7nk7nwtUMWfNYo7FF6v9KjcD/82effXZQRyioaqs8gE+BVfU8xgOFdcruq6f+1cBfA5YnAE/XU+5nwK+DiWn06NEaqjlz5oRctz04qtuf95nqo/2d5/qWm6gzZ86cZtcJaj8tjKvV6kSi7S2Mq9X2UadMs9s/+/+c568/Vt25RnXth6pfPKP6wU9UX71C9clRqr/upvqrlDqPVNU/DFZ9/kzVVy5TnX6T6j/uUZ31C9W5f1Rd9KLqp79V/V2m6sIXVXesVl3+N9VH+qqu/odqaaFqRamq39/y9rsC/88DSzSI39hW6/JS1fMaek9EdopIuqpuF5F0YFc9xfKBPgHLmcC2esq9AXwI/Kol8Zp2rLGLBRrq4w6sszmn+XWC2U9L42qtOpFoe0vjaq19hNL+xrptj7/4yPLVfuc82L6NzgR16z+G3qOg+2DnnFJZIeze4TyXFjrnmgLN/Mnhy29NOHw5Js45X+SLd54l1umOG3GNs69WPLfjVZfXY8AePXRSvquq/rROmVick/LnAltxTsrfoKqrRWSwqq53y/0AOEtVr2pqv3ZSPnTR3H5re7bXYXgmqPaHej6kqYtRalSWHUouZYWwcDKsfheOu8S5GMVf4V7wUO5cgHDYs/vYvhwKvoGxP4VzHmx229v6SflHgLdE5HbgW5zuLdzLf/+qqherapWI3APMAnzAFFVdXVNfRI4DqoHNQJNXeBljTKuoL2n0H9v4UUAwF6PUiEuEuF6Q3MuptzHHSQxLXnJGpmjqaGPjXHh79qE6/c9stSMUTxKKqu7BOfKou34bcHHA8kxgZj3lrmzVAI0xpjWF0hXXnCTUkjotYHfKG2NMpJ1x75E/6P3HNt5F1tSNw+Gq0wI22rAxxhwNQulaC6VOC9gRijHGmLCwhGKMMSYsLKEYY4wJC0soxhhjwsISijHGmLDw5E55r4jIbpwbIUPRHSgIYzhHm2huv7U9ekVz+wPb3ldVezRVIaoSSkuIyJJghh5or6K5/db26Gw7RHf7Q2m7dXkZY4wJC0soxhhjwsISSvAmex2Ax6K5/db26BXN7W922+0cijHGmLCwIxRjjDFhYQnFGGNMWFhCCYKIjBORr0Uk151hMmqIyCYR+UpElotIaNNdHkVEZIqI7BKRVQHruorIJyKy3n3u4mWMraWBtj8kIlvd73+5iNQzp+3RT0T6iMgcEVkrIqtF5Efu+mj57htqf7O+fzuH0gQR8eFMRXw+zjz3i4HrVXWNp4FFiIhsArJUNSpu7hKRsUAJME1Vh7nr/gDsDZiyuouq3u9lnK2hgbY/BJSo6h+9jK21iUg6kK6qy0QkGVgKfA+4lej47htq/zU04/u3I5SmnQLkqmqeqlYA04HxHsdkWomqzgX21lk9Hpjqvp6K8x+t3Wmg7VFBVber6jL3dTGwFsgger77htrfLJZQmpYBbAlYzieED/oopsDHIrJURCZ6HYxH0lR1Ozj/8YCeHscTafeIyEq3S6xddvkEEpF+wChgIVH43ddpPzTj+7eE0jSpZ1009RN+R1VPAi4C7na7RUz0eA4YCIwEtgN/8jSaViYinYB3gXtVdb/X8URaPe1v1vdvCaVp+UCfgOVMYJtHsUScqm5zn3cB7+N0AUabnW4fc01f8y6P44kYVd2pqn5VrQZepB1//yISh/Nj+rqqvueujprvvr72N/f7t4TStMXAYBHpLyLxwHXADI9jiggR6eieoENEOgIXAKsar9UuzQBucV/fAvzDw1giqubH1HU57fT7FxEBXgLWquqfA96Kiu++ofY39/u3q7yC4F4q9wTgA6ao6sPeRhQZIjIA56gEIBZ4o723XUT+BmTjDN29E/gV8HfgLeAY4FvgalVtdyevG2h7Nk53hwKbgDtrzim0JyJyBvA58BVQ7a7+Oc55hGj47htq//U04/u3hGKMMSYsrMvLGGNMWFhCMcYYExaWUIwxxoSFJRRjjDFhYQnFGGNMWFhCMeYoISLZIvKB13EY0xBLKMYYY8LCEooxYSYiN4nIInf+iBdExCciJSLyJxFZJiKzRaSHW3akiCxwB997v2bwPREZJCKfisgKt85Ad/OdROQdEVknIq+7dzgb0yZYQjEmjETkBOBanEE1RwJ+4EagI7DMHWjzM5y70AGmAfer6gicu5Rr1r8OPKuqJwKn4wzMB84osPcCQ4ABwHdauUnGBC3W6wCMaWfOBUYDi92DhyScAQWrgTfdMq8B74lIKtBZVT9z108F3nbHT8tQ1fcBVLUMwN3eIlXNd5eXA/2Aea3eKmOCYAnFmPASYKqq/uywlSK/rFOusTGPGuvGKg947cf+D5s2xLq8jAmv2cBVItITauck74vzf+0qt8wNwDxVLQL2iciZ7voJwGfuPBT5IvI9dxsJItIhko0wJhT2140xYaSqa0TkFzizXMYAlcDdwAFgqIgsBYpwzrOAMyT6827CyANuc9dPAF4Qkd+427g6gs0wJiQ22rAxESAiJarayes4jGlN1uVljDEmLOwIxRhjTFjYEYoxxpiwsIRijDEmLCyhGGOMCQtLKMYYY8LCEooxxpiw+P9ik4ncrKc+SwAAAABJRU5ErkJggg==",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plot_history(history,'SWaT')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "id": "ieObNqKYsOzh"
   },
   "outputs": [],
   "source": [
    "torch.save({\n",
    "            'encoder': model.encoder.state_dict(),\n",
    "            'decoder1': model.decoder1.state_dict(),\n",
    "            'decoder2': model.decoder2.state_dict()\n",
    "            }, \"model_SWaT.pth\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Another way to save the model!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pickle"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "filename = 'usad_model_SWaT.sav'\n",
    "pickle.dump(model, open(filename, 'wb'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "# load the model\n",
    "filename = 'usad_model_SWaT.sav'\n",
    "loaded_model = pickle.load(open(filename, 'rb'))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "ymhjbmvR_DgJ"
   },
   "source": [
    "## Testing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "id": "b7rbm9wdXKeF",
    "outputId": "076309c7-22be-41f6-f916-5f11cb679672"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "IncompatibleKeys(missing_keys=[], unexpected_keys=[])"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "checkpoint = torch.load(\"model_SWaT.pth\")\n",
    "\n",
    "model.encoder.load_state_dict(checkpoint['encoder'])\n",
    "model.decoder1.load_state_dict(checkpoint['decoder1'])\n",
    "model.decoder2.load_state_dict(checkpoint['decoder2'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "training_scores_list=training_scores(model,train_loader,val_loader)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "training_scores_concat=np.concatenate([torch.stack(training_scores_list[:-1]).flatten().detach().cpu().numpy(),\n",
    "                            training_scores_list[-1].flatten().detach().cpu().numpy()])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0.2984913  0.29850903 0.2985179  ... 0.01132083 0.01130209 0.01126255]\n"
     ]
    }
   ],
   "source": [
    "print(training_scores_concat)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.01009495"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "min(training_scores_concat)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "id": "Ry1QTp6V2ny4"
   },
   "outputs": [],
   "source": [
    "results=training_scores(model, test_loader) # testing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[tensor([0.0112, 0.0112, 0.0112,  ..., 0.0127, 0.0126, 0.0126],\n",
      "       grad_fn=<AddBackward0>), tensor([0.0126, 0.0127, 0.0127,  ..., 0.0129, 0.0129, 0.0129],\n",
      "       grad_fn=<AddBackward0>), tensor([0.0128, 0.0128, 0.0129,  ..., 0.0698, 0.0698, 0.0698],\n",
      "       grad_fn=<AddBackward0>), tensor([0.0698, 0.0698, 0.0697,  ..., 0.0743, 0.0742, 0.0741],\n",
      "       grad_fn=<AddBackward0>), tensor([0.0741, 0.0742, 0.0742,  ..., 0.1539, 0.1529, 0.1512],\n",
      "       grad_fn=<AddBackward0>), tensor([0.1498, 0.1482, 0.1454,  ..., 0.5919, 0.5919, 0.5917],\n",
      "       grad_fn=<AddBackward0>), tensor([0.5916, 0.5915, 0.5914,  ..., 0.3654, 0.3621, 0.3585],\n",
      "       grad_fn=<AddBackward0>), tensor([0.3551, 0.3537, 0.3521,  ..., 0.3755, 0.3756, 0.3757],\n",
      "       grad_fn=<AddBackward0>), tensor([0.3757, 0.3758, 0.3758,  ..., 0.3660, 0.3659, 0.3658],\n",
      "       grad_fn=<AddBackward0>), tensor([0.3658, 0.3657, 0.3656,  ..., 0.4377, 0.4377, 0.4376],\n",
      "       grad_fn=<AddBackward0>), tensor([0.4376, 0.4376, 0.4376,  ..., 0.4278, 0.4278, 0.4279],\n",
      "       grad_fn=<AddBackward0>), tensor([0.4278, 0.4278, 0.4277,  ..., 0.4646, 0.4647, 0.4647],\n",
      "       grad_fn=<AddBackward0>)]\n"
     ]
    }
   ],
   "source": [
    "print(results)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "windows_labels=[]\n",
    "for i in range(len(labels_down)-window_size):\n",
    "    windows_labels.append(list(np.int_(labels_down[i:i+window_size])))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_test = [1.0 if (np.sum(window) > 0) else 0 for window in windows_labels] # this is with the point adjust?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {
    "id": "FSWwxheNvxR7"
   },
   "outputs": [],
   "source": [
    "y_pred=np.concatenate([torch.stack(results[:-1]).flatten().detach().cpu().numpy(),\n",
    "                            results[-1].flatten().detach().cpu().numpy()])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "results_threshold=testing_threshold(model,test_loader,alpha=.5, beta=.5, contamination=0.10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.4556198"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "results_threshold"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 279
    },
    "id": "bROUyLM93cG3",
    "outputId": "755359d9-d0fb-4deb-b313-d3c2a2465a26"
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEGCAYAAABo25JHAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAAsTAAALEwEAmpwYAAA2RElEQVR4nO3deXhUVbbw/+9OZU5IICQEMgBhkCEkgESwbVDQtsWh2+HirCgNot2Ndvtzoq99X70/Xxu71XYWnBHtBq+Kil5nMYozs8o8hgxA5jmVpCrr/eNkIoSQhJxUKrU+z1NPzqlzqmrtKqhV++x91jEiglJKKd/l5+kAlFJKeZYmAqWU8nGaCJRSysdpIlBKKR+niUAppXycv6cD6Kjo6GgZOnRopx5bUVFBWFhY1wbUw2mbfYO22TecSJvXr1+fLyIxrW3zukQwdOhQ1q1b16nHpqenM3369K4NqIfTNvsGbbNvOJE2G2MyjrVNDw0ppZSP00SglFI+ThOBUkr5OE0ESinl4zQRKKWUj7MtERhjXjTG5Bpjfj7GdmOMedwYs9sY86Mx5mS7YlFKKXVsdvYIlgIz29h+LjCy/jYfWGxjLEoppY7BtvMIRORLY8zQNna5EFgmVh3s74wxfY0xg0TkoF0xBeXlQUkJREba9RJKKdVhuWVO1u4rosbtptYtuOsEl7uOalcdpVW1jfsFlrqYbsPre/KEsnggs9l6Vv19RyUCY8x8rF4DsbGxpKend+oFk++/n4o77mDtCy+Aw9Gp5/A25eXlnX6/vJW22Tf0ljZvynXx9KZqauqOvY+p//urBLGlzZ5MBKaV+1q9So6IPAs8C5CWliadPbNu/U03MWnAAKafdZZ1R2kpRER06rm8hZ596Ru0zd5nd245T67exTubc0joF8Kjl08kKiwQfz9DgMMPh58hwGGICA7Az8/6urSrzZ5MBFlAYrP1BCDHzhcsGz0aGt7Et9+GefPg888hJcXOl1VK+Zjs4iq2ZJdQ6xZcdXXUuoXDpU6C/P2oqnGz/VAZ//uTdfBj1qQEbv/1KAZGBnssXk8mglXAAmPMCmAKUGLn+MBRRo6E3/4WRo+21kXAtNZJUUqp9nv2yz387f3tbe4TEuBgzi+H8scZI4gOD+qmyI7NtkRgjFkOTAeijTFZwD1AAICILAHeB84DdgOVwBy7YmlVcjK8+KK17HTCGWfArbfCFVd0axhKqd5BRHg6fQ8PfrSDaSOjufH04cT0CcLfYQjw8yPA3xDk7yA8yJ9A/551Cpeds4auPM52Af5o1+t3SEkJ9OkDUVGejkQp5YVc7jpuenU9n27LZeqIaF647pQe92XfFq8rQ22L2Fj45JOmQ0PPPGMNJN92G/h5z4eplOq8ujrh8x25lFTVUuOqo9ZdR1ZRFREhAdS6rXWXW8goqKRvaAA1rjoOlzlx1taxP7+Cgooarp4ymP/+bTL+Du/63tBE0KD5+MCaNVBYCLff7rl4lFLdau3+Qua+fOxrnTj8DP5+BmPAWVtHYlQIgQ4/nLV1pCZEctHEeC6cEN+NEXcdTQSteeUVqKqykkN+vtVDuP12CPL8oI5Syh778isAeOG6NMYMiiDQ348Ahx8hAQ78/UzjFM7eyLv6L93FGAgNtZZXroT//m/YvduzMSmlbLPjUBlLv9kPQHJcJHF9Q4gODyIyJIBAf79enQRAE8HxzZ8P27ZZs4wAVq2CigrPxqSU6jJf7MzjnEe/ZPuhMlLiI4kOD/R0SN1OE0F7DB9u/c3MhP/4D7j/fs/Go5TqMv+zLpMAh2HlH07j3Zunet1Ab1fQMYKOSEyE1ashNdVaz8qC8HDo29ejYSmlOm/HoTLGxkVy8uB+ng7FYzQRdNS0aU3L118PBw/Cjz/6TBE7pXqyGlcdZc5aKmvc1jRQdx1FFTU4a637i6tqqa6tw+myth8qcbI7t5yTB/f1dOgepYngRPz971avoCEJaIlrpbqNiFBcWcOevHI2Z5bw7o85bDxQ3K7HBjgMwQEO+ocFMm1kNAvPHW1vsD2cJoITMWmSdQN46y2riF16uhaxU6qLOGvdHCis5FCJk+ziKnYcKqNOhO/2FrDzcCV89EnjvknRYcz55VCiQgOJjQzG5RYG9Q0m2N9BUIAfsRHBRAT7W9NBfXAcoC2aCLrK6NFwySVNRezq6vSsZKVOwJpdecx7eR3VrqML9Y+Lj+CXcf6cMjaJ0QMjSI6LIDEq1ANR9g6aCLrKmDHw3HPWstNpjSXceitcdZVn41LKC7ncdazckE21q44HZ6WSGBXKoMhgBvQJJiTQOhRr1eY/ycOR9g6aCOxQWmoVsIuO9nQkSnkNEeGfn+xka04p6zKKKKmqZUj/UC5NSzz+g9UJ0URghwED4MMPm+oXLV5sJYc77tDDRUph/eIvqqwlt8xJXlk1B0ucvLk+i3UZRfQNDSA1IZILUgdxbsogT4fqEzQR2KV5Ebtvv4WCArjzTs/Fo1QP8HN2CTcsW8fBEudR22Ijgrj91yfxh+kjen1Jh55GE0F3WLbsyCJ2ixdbvYNgz12aTim7VVS7yCmuoqCihuyiKoqrannl2/0cLHFy85kj8PfzY0j/UBL6hdAvLJBh0WEYvUqgR2gi6C4hIdbft96C++6zZhg11C9SqocTEfLKqtlfUMmBwkrq6oTCyhryyqrJLKykTmBTZhF+xlDrrqPaVUdljfuo5xnQJ4iF547mpjOGe6AV6lg0EXS3G26AX/0KkpKs9bffttbDwz0allItbTtYyjubcjhc6uSr3fnklVUfc9/o8EBGDexDkL+D+L4hBPr70TckgEF9Q4iNCCI00J/4+mX91d/zaCLwhIYkkJkJl15qHSb62988G5PyKQcKKtmcVUxJVS378ysICXRQ466j1iVUu9z8lF3CzsNlOGvriOkTRHJcBKcMjWJgRDCD+4c2lmgOC3IQ5K/lVbydJgJPSkyEL76AceOs9cxMCAvTayerLrf9UCmvr8tiV245+WXVbD1YetQ+Qf5+BDr8CPD3IyoskDGDIrjnN8lMSOzb/QGrbqWJwNNOO61p+frrIScHfv5Zi9ipE5ZVVMnT6XvILKxkza58wJqZc1JsH+ZNTeK0Ef0ZFh1OTJ8gwoL0q8CX6affkzz0kNUraEgCxcVa4lq1y/rDLl7/1wYOlzrJL6+mzOmioKKmcftlaQncfOZILcOgWqWJoCeZONG6gXWJzIYidg3XP1CqFct/OMATG6uBg/QJ9iexXyhj4yJI6BfKf5ycwKiBfTwdourhNBH0VMnJ1kDymDHWuhaxU1iDvD/sLySjoILv9xay7VApZU4X0SGGr/7zHIID9JCi6jhNBD3VqFHwzDPWclUVTJ1qFbG75hrPxqW6hYhQUeNmX14FWw+WUFRZy8oNWew8XN64T1J0GGMHRTBlWH+G1WVpElCdponAG5SXW/WLBg70dCSqG2w4UMStr20io6DyiPsH9AnilrNGMn1UDMNjwokMCWjclp6e091hql5EE4E3iImBDz5oWn/qKWsgeeFCnV3Uy7jrhEXvbyOnuIq7Zo5mUGQw8f1CGBETTt/QAD0ZS9lCE4E3Wr8ecnN1zKCXEBGyi6tYn1HEO5tyWLu/iLvPG8MNpw/zdGjKR2gi8EYvvmhd/MYYyMuzeggLF2oROy9UXu1i7tK1fL+vEAA/AwtmjNAkoLqVJgJv1fCl/847sGgRXHYZjB3r2ZhUh/3vjzl8v6+Q308fzhknxXDy4H4E+mtPT3Uv/Rfn7ebNg507m5LAypVQVubZmFS7fbungKiwQO48ZxSnDuuvSUB5hK3/6owxM40xO4wxu40xC1vZHmmMedcYs9kYs8UYM8fOeHqtIUOsv5mZcPnlWsDOS7y1MYu3N+VwytB+OgisPMq2Q0PGGAfwFHA2kAWsNcasEpGtzXb7I7BVRH5jjIkBdhhj/iUiNa08pTqexERYs6apd3DgAP4lJZ6NSR3T0q/3MywmjAcvHe/pUJSPs7NHMBnYLSJ767/YVwAXtthHgD7G+jkUDhQCLhtj6v1OPRUiIqzlOXOYeMst4D76AiHKs5y1bjZnlTBtRDQRwQHHf4BSNrJzsDgeyGy2ngVMabHPk8AqIAfoA1wuInUtn8gYMx+YDxAbG0t6enqnAiovL+/0Y71R2FVXIRkZVK5ZA4B/aSmuhiTRi3nD5/xpRi0A4VWHSE/PP+Hn84Y2dzVtcxcSEVtuwKXA883WrwWeaLHPLOARwAAjgH1ARFvPO2nSJOmszz//vNOP9VaNbX7jDZHISJHNmz0ZTrfwhs957tIfZMhd70mNy90lz+cNbe5q2uaOAdbJMb5X7ewRZAGJzdYTsH75NzcHeKA+yN3GmH3AaOAHG+PyTampcNVVTeMHWsSuW9W46iipquVAYSXPr9nLp9tyiY0IIsChn4HyPDsTwVpgpDEmCcgGrgCuarHPAeAsYI0xJhYYBey1MSbfNXIkPP20tVxVBb/8Jfz5zzB7tkfD6m1KqmrZm1dOTrGTMmcteWXVbMosZvWOXKxOsHXS2MUT47n3N8meDVaperYlAhFxGWMWAB8BDuBFEdlijLmpfvsS4D5gqTHmJ6zDQ3eJyIkfMFVtq6iAuDiIj/d0JF4jv7yaXYfLcbrcHC5xEuDwo8ZdR3WtmzKni60HSwkJcLByY/ZRj+0T7M+skxMYFx9JbEQwKQmRxPcN8UArlGqdrWcWi8j7wPst7lvSbDkH+LWdMahWREfDe+81rT/5JBQVwX/+pxaxa8UP+wq55oXvqXEdNY+hUUiAg6paN8NjwkgbEsWstATCg/yJiwwhMlRnBameTUtMKNi4UYvYteHjLYeocdWx5JpJDIgIItDhR3CAg/AgfwL9/QgNdOi1AJRX00Sg4IUXoLq6qYjd44/DX/4CoXp9W4DiqlriIoOZOU6vB6F6J/0JqCxBQdbfVavgH/+A/fs9Gk5PkldWTd/QQE+HoZRtNBGoI82dC7t3N00zfeMNKC31bEwe5HLX8f2+AiYM7uvpUJSyjR4aUkdLrD/9IzMTrrwS7rijVxayq3HVUVXrptrlprLaTY27jlp3HbVuweWuo6Ciho0HinHW1jElKcrT4SplG00E6tgSE+Hrr2HMGGs9IwPCwqxZR14is7CSzKJKsoqqMFgXgimqqGHrwTI+3Xa4Xc8R6PDjF8P72xuoUh6kiUC1bfLkpuU5cyAnB7Zs8YpppiVVtUz7x+etbgsO8CPQ4ccNpycxKDKEQH8/qmvdDIgIJsBhCHD44fAzhAQ4GNI/jKgwHSNQvZcmAtV+jz0GBw40JYGCAujfc34pl1e72HGolLWHXBz64QDL11o1D6/7xRAuPjmBqNBAQoMcRIUG4uen9f+VaqCJQLVfSop1A3jzTfjd7+DLL2G85+vpP/PFHhZ9sL3pjk0/AVYS+K8LxuKvNX2UOiZNBKpzJkyAa6+F5Pp6OW63xw4XFZRXs+zbDEICHNx9/hjcubs547RTiQgJ0EM6SrWD/kxSnTN8uFWawt/fKmI3aRIsXdrtYeSXV/ObJ74iu7iKp66eyDWnDmFIhIOh0XpcX6n20h6BOnEVFdZ1kwcP7raXzCur5pOth3nluwxySpw8PzuNM0fHdtvrK9WbaCJQJy46Gt55p2n9iSesgeT/+q8uPVyUV1bN+owiPt5yiFWbc3DVCYlRITx06Xh+NVaTgFKdpYlAdb2ffoJDh7qkiN0nWw/z8Mc72H6o7Ij7r5oymGumDGHMoD5Yl7xWSnWWJgLV9Z59FmpqrCJ2ubnWtNO77+5wEbuKahd/WrGRyho3YwZFcMrQfpyTPJCxgyLop8f/leoymgiUPQLrv6jfew8efhiuuabpDOV22nqwlMoaN8/PTtNDP0rZSGcNKXv97ndWEbuGJPA//wMlJfCvf8HQodbho6FDrfUWdh62DgcNiwnrvniV8kHaI1D2S0iw/mZmWj2DmTPhs8+gstK6PyMD5s+3lq++uvFhKzdkM7R/KEP7ayJQyk6aCFT3SUyEb76BSy5pSgINKivJ/9PtXJg5iLAgB4dKnJQ6Xdx4xjAtB6GUzTQRqC5xsKSK1dtzKaqoIauoiiB/P6pddezLr6BPcAC17jqr3HONm7czs1o9JhlVcJiEfiFU1Lg4d9wghsWEceWU7js3QSlfpYlAdZqz1s3u3HIe/GgHX+zMa7zfz0CdQGxEEIH+fhTnlDJsQDhBDj8iQwIojh5IVP7Bo57PLyGe1278RXc2QSmFJgKFdRWuqlo3zto6Kmtc5JfXcLjUSWFFDbmlTnJKnAT5+1FR7WL9gSLKnS6KKmuPeI4FM0ZwXsoghg8II9Dh1/bc/uAHrTGB5oeHAgOt6yVv2mTVMVJKdRtNBD6gvNrFu5tz2Jdfwe7ccgIdfuw4XEZhRQ0iQqnT1a7nSegXQr/QQIb2D2PsoAgqa9ycNLAPpwztx+iBEe0PqGFA+O67rbLWgwfDn/4E+/bBuHHWNg8WsVPK12gi6KW+2Z3P0+l72JpVSeGHHzXeHxUWSFWNm5Niw+kbGkBkSABJ0WFEBAcQHR5IcICD6D5BxIQH0Tc0gPAgfyJDArr+7N2rrz5ihtARKivh1FPhz3+2pp8qpWyliaAXKK928dm2wxwqcbIuo4hDJU5+yi4hJMDB+Gg/Jo9JIq5vCJemJeLwhhk4VVUwYoR1foFSynaaCHqoujqhxl1HVY27/gLrdThr3RRX1iII1bV1ZBVVsiu3nFWbcyiuP2bfNzSA8Ql9uSwtgT/OGMG+n9YyffooD7emg/r3h5Urm9Yfe8wqYnfPPXq4SCkbaCKwkYiQV1bNoVIn5dUuDpU4OVBYSW5ZNUUVNQQ4rAHYA4WVFFTUEBLgwFnrpqCipkOv86sxA7j+tCRSEyMJD/Q/Yt79vq5ulCds29ZlReyUUkfTRGCD3bnlrPjhAF/uymPn4fKjtgf5++GqE6LCAokKDcThZxjaP5RAfz+Gx4QT4PCjxl1HYj/rvqoaFwn1y646ITo8kJAAB0H+Dob0DyUsqJd/jEuWNBWxO3wYHnkE/vpXCA/3dGRK9Qq9/BukexwudfLVrnxWbc45Yj59dHggd84cRUx4EIMiQxgQEcSQ/qEE+evhjQ5rKGL3wQfWoaLrrutwETulVOs0EXRSrbuOdzfn8PK3GWzOLAYgNNDBjFExDIsJ55pTh5AUrTVyutz118Ovfw1xcdb68uVW7aJ+/TwallLezNZEYIyZCTwGOIDnReSBVvaZDjwKBAD5InKGnTGdqLyyat77MYfnvtzbeKLV6SfFMG9qEpOToggO0F/7tmtIAllZVs/gtttg0SLPxqSUF7MtERhjHMBTwNlAFrDWGLNKRLY226cv8DQwU0QOGGMG2BXPidp+qJSXvtrPW5uyqXHVMXJAOIuvPpkzxwzQQz2ekpAA330HI0da6/v2dfjiN0ope3sEk4HdIrIXwBizArgQ2Npsn6uAlSJyAEBEcm2Mp9NEhLlL15FdXMX5KYNYcOYIRg/USyT2CCef3LQ8d67VS1i82HPxKOWFjIjY88TGzML6pT+vfv1aYIqILGi2z6NYh4SSgT7AYyKyrJXnmg/MB4iNjZ20YsWKTsVUXl5OeCszTepEyCqrY19pHZW14KoTXHVwuLKOYH/DvpI6MkrrmJMcyBmJAZ16bU85Vpt7o9CMDIIPH+bA2LGEh4URUFxMrY+MHfjS59xA29wxM2bMWC8iaa1ts7NH0NrP5ZZZxx+YBJwFhADfGmO+E5GdRzxI5FngWYC0tDSZPn16pwJKT0+n+WNFhFe/P8Bjn+4iv7z6qP0dfgZ3nTA4KpSzx/bhr1dP8o4zc5tp2WZfUJiezvTcXKuH8OWXMHGip0OynS9+ztrmrmNnIsgCEputJwA5reyTLyIVQIUx5ktgPLATG63PKOSLHXl8sSufzZnFDIsJ44ZpSUxI7EtSdBiRoQHHr6CperbJk61EkJJirbtc4K+T5JRqjZ3/M9YCI40xSUA2cAXWmEBz7wBPGmP8gUBgCvCIXQGV1gi3/c9m3tyQBUBwgB8nxYbz7s1TdcC3txk6FB591FqurLQSw623WslBKXUE2xKBiLiMMQuAj7Cmj74oIluMMTfVb18iItuMMR8CPwJ1WFNMf7YjHpe7jvu+rSKvKotzxw3kjnNGMSzGt44v+iyn0zr5bPhwT0eiVI9ka19ZRN4H3m9x35IW6w8CD9oZB8Cu3HLyqoRLJyXw4KXj7X451ZNERcHrrzetP/KIdRGc++7TInZK4UNnFlfWWBdfuWB8nIcjUR63axfk5GgRO6Xq+dz/BB3+VTz9tNVDaChid+edUH50cUClfIXPJQKlAAioPx/kww/hySetE9GU8lE+kwhsOm9OebvrroM9e2D0aGv93/+GwkLPxqRUN/OZRNBATw1QRxk0yPqblWVVN33oIY+Go1R385nBYqWOKyEBfvjBul4ywN69EBLSlCiU6qU63CMwxjiMMVfbEYxSHjdhQtOVz264AU4/Hdxuj4aklN2O2SMwxkQAfwTigVXAJ8AC4HZgE/Cvboivy+gQgeqwxYut0tYOhzXIdPgwDBzo6aiU6nJt9QheAUYBPwHzgI+BWcCFInJhN8RmC6MTSFV7nXQSnHOOtfzaa9Yho40bPRuTUjZoa4xgmIikABhjngfygcEiUtYtkSnVk5x2Gtx4Y1MRu9rapimoSnm5tnoEtQ0LIuIG9mkSUD5r8GB4+GGrgmllpTWW8Nxzno5KqS7RVo9gvDGmlKaTcUOarYuIRNgeXRfS8whUl3E6Ydy4pktkKuXljpkIRKRXVuPS8wjUCYuKssYMGvzzn5CbC/ffr0XslFdqa9ZQMHATMAKrTPSLIuLqrsCU8hr79mkRO+XV2vqX+zKQhjVr6Dzg4W6JSClv88QTVg/BGDh0CG67DUpLPR2VUu3WViIYKyLXiMgzWNNGp3VTTLYQHSRQdmq4DObHH8OSJVYPQSkv0d5ZQ73mkJAOEShbzZ5tlaZoKGL3yitQUODZmJQ6jrYSwQRjTGn9rQxIbViunz2klGpNbKz1Nzsb5s3TInaqx2tr+uhmEZnYbZEo1dvEx8PatTBsmLW+Z49VxC5Or5Knepa2EoEeVFfqRKWmNi3fcANkZsL27TrNVPUobSWCAcaY/+9YG0XknzbEYxvNasrjnnkG9u9vKmJ36JCWuFY9QltjBA4gHOhzjJt30tFi5SkjR8LZZ1vLDUXsNmzwbExK0XaP4KCI/P/dFolSvmTqVPjDH2D8eGtdi9gpD2qrR6C/nZWyS0ICPPigdZioosIaS3jmGU9HpXxUW4ngrG6Lohvo+WSqx6qpgYkTYcwYT0eifFRbRecKuzOQ7qIXplE9Tr9+8O9/N60/9JB1NbQHHtDZRapb6MXrleppDhywTkbTInaqm+i/NKV6mscfP7KI3a23ahE7ZSufSQSiZxIob9JQxO7TT+HZZ+HgQc/Go3o1n0kEDfTCNMqrXHONdRLaqFHW+ssvQ16eR0NSvY/PJQKlvE5MjPU3OxtuvBEeecSz8ahex9ZEYIyZaYzZYYzZbYxZ2MZ+pxhj3MaYWXbGo5RXi4+H9evh7rut9d27ISvLszGpXsG2RGCMcQBPAecCY4ErjTFjj7Hf34GP7IoF0GJDqndIToawMGv5xhth+nRwuz0akvJ+dk4fnQzsFpG9AMaYFcCFwNYW+90MvAmcYmMsjXSIQPUazz57ZBG77Gyr16BUB9mZCOKBzGbrWcCU5jsYY+KBi4EzaSMRGGPmA/MBYmNjSU9P73AwWwusX02bNm2i6oDvnKRTXl7eqffLm/lUmx0OSE8n4v33cZ97Lhsff5zyk07ydFTdwqc+53p2tdnORNDaj++WB2geBe4SEbdpYzqPiDwLPAuQlpYm06dP73AwgbvzYe33TJgwgSnD+nf48d4qPT2dzrxf3swX2/xNXh6OP/+ZtLlzreRQUwOBgZ4Oy1a++Dnb1WY7B4uzgMRm6wlAyyt6pwErjDH7gVnA08aYi+wIRocIVG9WExPTVJKioYjd4sWeDkt5CTt7BGuBkcaYJCAbuAK4qvkOIpLUsGyMWQq8JyJv2xgTbfU8lOoVamogLQ3GjfN0JMpL2JYIRMRljFmANRvIAbwoIluMMTfVb19i12sr5dP69YNXX21af/BB68zkhrLXSrVga9E5EXkfeL/Ffa0mABG53s5YlPJZBw9a10rWJKCOwWfOLNbrESif9c9/wooV1vLBg3DLLVBc7NGQVM/iM4mggQ4RKJ/U0BtYvRpefBFycz0bj+pRfC4RKOXTrr7aOgmt4VyDpUs1KShNBEr5nOho6292Ntx0kxaxU75zhTK9HoFSLcTHw4YNMGSItb5rFwQHQ2Ji249TvY7PJIIGOkSgVDNjm9WBvPFG6zKZO3boDCMf43OJQCl1DC+8APv2HVnELiHB01GpbqBjBEopS1ISnHmmtbx8OYwcaV3/QPV6PpMI9DwCpTpg+nS49VaYMMFar672ZDTKZj6TCBroeQRKtUNcHPztb9ZhovJySEmBp57ydFTKJj6XCJRSHeR2wy9+YVU0Vb2SDhYrpdoWGQkvv9y0/ve/W6UqHn5YZxf1Ej7TI9AhAqW6SG6ulQg0CfQaPpMImugggVIn5OGH4d//tpYPHoQFC6CoyLMxqRPig4lAKXXCGnoDn39uHTbKy/NsPOqEaCJQSnXeVVcdWcTuhRfg8GGPhqQ6zmcSgeiJBErZo39/6292tnWY6LHHPBuP6jCfmzWk5xEoZZP4eNi4EQYPttZ37oSgoKaidqrH8rlEoJSy0ejRTcu//71Vu2jnTvDXr5qeTD8dpZQ9XnwR9u61koCIdd3kht6C6lF8Z4zA0wEo5WuGDIEZM6zl5cutAeV16zwbk2qVzySCBjpEoJQHnHkm3H47TJxorTudno1HHcHnEoFSygMGDoT/+3+PLGL35JOejkrV00SglOpebjdMm9bUO1Ae5zuDxTpIoFTPEBlpDSQ3eOAB6xyERx7R2UUe4nM9AqMnEijVsxQWWiUqNAl4jM8lAqVUD/OPfzQVscvJsc4/KCz0bEw+RhOBUsrz/Oq/ir74Av71Lygo8Gw8PsZnEoHoIIFSPd+VV0JGBowcaa0//7xV6lrZymcSQQMdIVCqh+vXz/qbkwO33AJPPOHZeHyArYnAGDPTGLPDGLPbGLOwle1XG2N+rL99Y4wZb2c8SikvEhcHmzbBX/9qre/YYZW8Vl3OtmF6Y4wDeAo4G8gC1hpjVonI1ma77QPOEJEiY8y5wLPAFLtiUkp5mYbrHIA1iLx/vxaxs4Gd7+ZkYLeI7AUwxqwALgQaE4GIfNNs/++ABLuC0csRKOXlli61qpnWF7ELPnTI0xH1GnYmgnggs9l6Fm3/2p8LfNDaBmPMfGA+QGxsLOnp6R0O5qdcFwAbNqynaI/vXHS7vLy8U++XN9M293Lp6cR+8gmn/OMfrC8upqx56etezq7P2c5E0Nq4bKu/y40xM7ASwdTWtovIs1iHjUhLS5Pp06d3OBj3tsOwYR2TJk0iNaFvhx/vrdLT0+nM++XNtM0+YMwY9mdlMWn+fGvqaVUVhIR4Oirb2fU52zlYnAUkNltPAHJa7mSMSQWeBy4UEZ08rJQ6vthY9s+ZYyWBsjIYNw4ef9zTUXktOxPBWmCkMSbJGBMIXAGsar6DMWYwsBK4VkR22hiLUqq3ErGuezBpUtO66hDbDg2JiMsYswD4CHAAL4rIFmPMTfXblwD/B+gPPF1fA8glImn2xGPHsyqlPC4iwjrxrMGiRVYRu8ce09lF7WTruyQi7wPvt7hvSbPlecA8O2NoyegpZUr1bqWlUFSkSaADfO7MYqVUL/fAA/Dqq9ZyTg7Mn6+1i45DE4FSqvdpKGK3Zg289prVQ1DH5DOJQIcIlPJBl19uFbEbMcJaf+YZq5egjuAziaCBXpdGKR/Tt6/1NycHbr0VnnrKo+H0RDqaopTyDXFx8OOP1l+A7dshMBCGDfNsXD2AJgKllO9oOEQE8Ic/aBG7ej7TetETCZRSzS1bBnv3NhaxY/9+SErydFQe4XNjBEopBUBCApx+urX8r3/BqFGwdq1nY/IQTQRKKXXOOXD33U1lKqqqPBtPN9NEoJRSMTFwzz1NReySk60SFT7CZxKBjhAopdrt7LNh8mRr2QfGF31msLiBnkeglGpTnz7WiWcN/vY3OHDAOv+gl84u8pkegVJKdUplJVRU9NokAJoIlFKqbfffD6+8Yi1nZcG8eZCf79mYupjPJAIfOMynlLJLwzHlb76BN96AkhLPxtPFfCYRNNDrESilOu2yy6widsOHW+uLF1u9BC/nc4lAKaVOSGSk9TcnB26/3UoGXq73jn4opZSd4uLgp59g0CBrfds2CAg4sp6Rl/ChRKCDBEqpLta8cumCBVbtol27vG6GkXdF2wX0PAKllC2WLYN9+5qK2O3Z4zW9Ax0jUEqprhAfD1OnWsuvvgpjxsAPP3g2pnbyuR6BUsdTW1tLVlYWTqfT06G0W2RkJNu2bfN0GN2qR7d54kTresnh4dbYQV1d03WUT0B72hwcHExCQgIBAQHtfl6fSQR6HoFqr6ysLPr06cPQoUMxXnIssaysjD59+ng6jG7lNW12u2HLFoiOhtjYE3qq47VZRCgoKCArK4ukDlxbwecODXnJ/2vlQU6nk/79+3tNElBeIDISwsKsZRt/lRpj6N+/f4d7sz7TI1CqIzQJqC7jcMCQIU3rBw9CTY11nw3/zjrzb9fnegRKKeVRItaYQQ/6seEziUCHCJS3eeuttzDGsH37dgDS09O54IILjtjn+uuv54033gCsQe6FCxcycuRIxo0bx+TJk/nggw/a9VrV1dVcfvnljBgxgilTprB///6j9ikrK2PChAmNt+joaP785z8DcODAAWbMmMHEiRNJTU3l/fffByAjI4NJkyYxYcIEkpOTWbJkSePzTZs2rfG54uLiuOiiiwAoKiri4osvJjU1lcmTJ/Pzzz83Pqa4uJhZs2YxevRo0tLS+PbbbwF4/fXXSU5Oxs/Pj3Xr1jXuX1BQwIwZMwgPD2fBggVHtGf58uWkpKSQmprKzJkzya8vJJeRkcFZZ51Famoq06dPJ6tZCYmZM2fSt2/foz6HuXPnMn78eFJTU5k1axbl5eUAlJSU8Jvf/Ibx48eTnJzMSy+9BPHx7Kipsdo+fjwTxo4lIiKCRx99tPH5nnjiCUaNGkVycjJ33nknAJ988gmnn346KSkpTJo0idWrVx/3c203EfGq26RJk6Qz/vfHHBly13uy/WBppx7vrT7//HNPh9DtTrTNW7du7ZpATtCll14qU6dOlXvuuUdErHadf/75R+xz3XXXyeuvvy6lpaVy1113yezZs8XpdIqIyKFDh+S1115r12s99dRTcuONN4qIyPLly+Wyyy477mNOPvlk+eKLL0RE5IYbbpCnn35aRES2bNkiQ4YMERGR6urqxnjKyspkyJAhkp2dfdRzXXLJJfLyyy+LiMjtt98u9957r4iIbNu2Tc4888zG/WbPni3PPfeciIjk5+dLUVGRiFif2fbt2+WMM86QtWvXNu5fXl4ua9askcWLF8sf//jHxvtra2slJiZG8vLyRETkjjvuaHyfZ82aJUuXLhURkc8++0yuueaaxsd9+umnsmrVqqM+h5KSksblW2+9VRYtWiQiIvfff7/ceeedIiKSm5sr/fr1k+rq6qYHFhSIa+1aiY2Nlf3794uIyOrVq+Wss85qfN8OHz4sIiIbNmyQHTt2iIjITz/9JHFxcUe9jw1a+zcMrJNjfK/qGIFSbfjvd7ewNae0S59zbFwE9/wmuc19ysvL+frrr/n888/57W9/y7333tvm/pWVlTz33HPs27ePoKAgAGJjY7nsssvaFdM777zT+BqzZs1iwYIFiMgxjzfv2rWL3Nxcpk2bBljHpUtLrfeppKSEuLg4AAIDAxsfU11dTV1d3VHPVVZWxurVq61fy8DWrVv5y1/+AsDo0aPZv38/hw8fJiQkhC+//JKlS5c2PnfDDJoxY8a0GmdYWBhTp05l9+7dR9zf8AVYUVFB//79KS0tZUT9yV9bt27lkUceAWDGjBmNPRWAs846i/T09KNeJyIiovF5q6qqGt83YwxlZWWICOXl5URFReHf/KzjqCg++/57hg8fzpAhQyA3l8VPPcXChQsbP8cBAwYAMHHiRMrKygBITk7G6XRSXV3duN+J8JlDQ0p5k7fffpuZM2dy0kknERUVxYYNG9rcf+/evQwePLjxC6mlyy+//IjDOg23ZcuWAZCdnU1iYiIA/v7+REZGUlBQcMzXW758OZdffnnjF969997Lq6++SkJCAueddx5PPPFE476ZmZmkpqaSmJjIXXfd1ZgkGrz11lucddZZjbGPHz+elStXAvDDDz+QkZFBVlYWe/fuJSYmhjlz5jBx4kQWLFhARUVFm+/LsQQEBLB48WJSUlKIi4tj69atzJ07t/H133zzzcbYysrK2nwvGsyZM4eBAweyfft2br75ZgAWLFjAtm3biIuLIyUlhcceewy/FucTrHj9da688kprADkri53btrFmzRqmTJnCGWecwdq1a496rTfffJOJEyd2SRIAH5o1pOcRqM443i93uyxfvrzx+PsVV1zB8uXLjzou3aA9s0Ree+21NrdLK/9B2nreFStW8ErDxVrq473++uu57bbb+Pbbb7n22mv5+eef8fPzIzExkR9//JGcnBwuuugiZs2aRWyz+fTLly9n3rx5jesLFy7kT3/6ExMmTCAlJYWJEyfi7+9PbW0tGzZs4IknnmDKlCn8/ve/54EHHuC+++47bvtbqq2tZfHixWzcuJFhw4Zx8803s2jRIv7617/y0EMPsWDBApYuXcrpp59OfHz8kb/ij+Gll17C7XZz880389prrzFnzhw++ugjJkyYwOrVq9mzZw9nn30206ZNa0x6NTU1rFq1ikWLFkFgICQn4zKGoqIivvv8c9auX89ll13G3r17Gz+PLVu2cNddd/Hxxx93uN3HYmuPwBgz0xizwxiz2xizsJXtxhjzeP32H40xJ9sZj/Wadr+CUiemoKCA1atXM2/ePIYOHcqDDz7Ia6+9RlRUFEVFRUfsW1hYSHR0NMOGDePAgQONhw5aOl6PICEhgczMTABcLhclJSVERUW1+lybN2/G5XIxadKkxvteeOGFxsNQv/jFL3A6nY2Drw3i4uJITk5mzZo1R7T1hx9+4Pzzz2+8LyIigpdeeolNmzaxbNky8vLySEpKIiEhgYSEBKZMmQLARRdddNye0rFs2rQJgOHDh2OM4bLLLuObb75pjHPlypVs3LiR+++/H7DO6G0Ph8PB5Zdf3tijeOmll7jkkkswxjBixAiSkpIaB/8BPvjgA04++eSmxBgUREJCgvWYzEwmR0bi5+fX+F5mZ2dz8cUXs2zZMoY3XBOhC9iWCIwxDuAp4FxgLHClMWZsi93OBUbW3+YD3l/YW6kT9MYbbzB79mwyMjLYv38/mZmZJCUlUVhYSE5OTmOJgYyMDDZv3syECRMIDQ1l7ty53HLLLdTU1ABw8OBBXn31VcDqEWzatOmo2+zZswH47W9/y8svv9z4+meeeeYxewTLly+3DmU0M3jwYD777DMAtm3bhtPpJCYmhqysLKqqqgBrNtDXX3/NqFGjGh/3+uuvc8EFFxAcHNx4X3FxcWMbnn/+eU4//XQiIiIYOHAgiYmJ7NixA7BmUY0d2/IrpX3i4+PZunUreXl5gDUjp2GcIT8/v3EsY9GiRfzud79r87lEpHEMQkR49913GT169FHvy+HDh9mxYwfDmlUsbe29vOiii6wZQUlJ7HS7qampIbp/f4oPHeLSSy9l0aJF/PKXv+xUu9tshB034BfAR83W/wL8pcU+zwBXNlvfAQxq63k7O2vovc3WrKEdh3TWUG/n7bOGzjjjDPnggw+OuO+xxx6Tm266Sb766iuZMmWKjB8/XtLS0uTjjz8WEZHS0lKprq6WO+64Q4YPHy7JyckyefJk+fDDD9v1mlVVVTJr1iwZPny4nHLKKbJnz57GbePHjz9i36SkJNm2bdsR923ZskVOO+00SU1NlfHjx8tHH30kIiIff/yxpKSkSGpqqqSkpMgzzzxz3LZ+8803MmLECBk1apRcfPHFUlhY2Lht48aNMmnSJElJSZHzzz+/cdvKlSslPj5eAgMDZcCAAfLrX/+68TFDhgyRfv36SVhYmMTHx8uWLVtERGTx4sUyevRoSUlJkQsuuEDy8/NFROT111+XESNGyMiRI2Xu3LmNs3dERKZOnSrR0dESHBws8fHx8uGHH4rb7ZbTTjtNxo0bJ8nJyXLVVVc1ziLKzs6Ws88+u3HbK6+80vhcFRUVEhUVJcXFxUe0v7q6Wq6++mpJTk6WiRMnymeffSaSny/3/eEPEhoaKuPHj2+8Ncwoaqmjs4aM2HTw3BgzC5gpIvPq168FpojIgmb7vAc8ICJf1a9/BtwlIutaPNd8rB4DsbGxk1asWNHheHYXufnfPVVckxxK/xDfGSMvLy8nPDzc02F0qxNtc2RkZOMMEm/hdrtxOByeDqNb+VSb3W4cTic1wcHtavPu3bspaXFd5RkzZqwXkbTW9rdzsLi1fmXLrNOefRCRZ4FnAdLS0mT69OkdDmY6MCI9nc481pula5s7bNu2bd5RzKwZrynA1oV8sc3udrY5ODiYiRMntvt57fxpnAUkNltPAHI6sY9SSikb2ZkI1gIjjTFJxphA4ApgVYt9VgGz62cPnQqUiMhBG2NSql3sOmSqlN0682/XtkNDIuIyxiwAPgIcwIsissUYc1P99iXA+8B5wG6gEphjVzxKtVdwcDAFBQVailp5Ham/HkHzWVjtYesJZSLyPtaXffP7ljRbFuCPdsagVEclJCSQlZXVOLXQGzidzg7/5/d22ubWNVyhrCN85sxipdorICCgQ1d36gnS09M7NDjYG2ibu47vzKNUSinVKk0ESinl4zQRKKWUj7PtzGK7GGPygIxOPjwayD/uXr2Lttk3aJt9w4m0eYiIxLS2wesSwYkwxqw71inWvZW22Tdom32DXW3WQ0NKKeXjNBEopZSP87VE8KynA/AAbbNv0Db7Blva7FNjBEoppY7maz0CpZRSLWgiUEopH9crE4ExZqYxZocxZrcxZmEr240x5vH67T8aY072RJxdqR1tvrq+rT8aY74xxoz3RJxd6XhtbrbfKcYYd/1V87xae9psjJlujNlkjNlijPmiu2Psau34tx1pjHnXGLO5vs1eXcXYGPOiMSbXGPPzMbZ3/ffXsa5h6a03rJLXe4BhQCCwGRjbYp/zgA+wrpB2KvC9p+PuhjafBvSrXz7XF9rcbL/VWFVwZ3k67m74nPsCW4HB9esDPB13N7T5P4G/1y/HAIVAoKdjP4E2nw6cDPx8jO1d/v3VG3sEk4HdIrJXRGqAFcCFLfa5EFgmlu+AvsaYQd0daBc6bptF5BsRKapf/Q7ranDerD2fM8DNwJtAbncGZ5P2tPkqYKWIHAAQEW9vd3vaLEAfY108IhwrEbi6N8yuIyJfYrXhWLr8+6s3JoJ4ILPZelb9fR3dx5t0tD1zsX5ReLPjttkYEw9cDCyhd2jP53wS0M8Yk26MWW+Mmd1t0dmjPW1+EhiDdZnbn4A/iUhd94TnEV3+/dUbr0fQ2iWlWs6Rbc8+3qTd7THGzMBKBFNtjch+7Wnzo8BdIuLuJVcaa0+b/YFJwFlACPCtMeY7Edlpd3A2aU+bzwE2AWcCw4FPjDFrRKTU5tg8pcu/v3pjIsgCEputJ2D9UujoPt6kXe0xxqQCzwPnikhBN8Vml/a0OQ1YUZ8EooHzjDEuEXm7WyLseu39t50vIhVAhTHmS2A84K2JoD1tngM8INYB9N3GmH3AaOCH7gmx23X591dvPDS0FhhpjEkyxgQCVwCrWuyzCphdP/p+KlAiIge7O9AudNw2G2MGAyuBa73412Fzx22ziCSJyFARGQq8AfzBi5MAtO/f9jvANGOMvzEmFJgCbOvmOLtSe9p8AKsHhDEmFhgF7O3WKLtXl39/9boegYi4jDELgI+wZhy8KCJbjDE31W9fgjWD5DxgN1CJ9YvCa7Wzzf8H6A88Xf8L2SVeXLmxnW3uVdrTZhHZZoz5EPgRqAOeF5FWpyF6g3Z+zvcBS40xP2EdNrlLRLy2PLUxZjkwHYg2xmQB9wABYN/3l5aYUEopH9cbDw0ppZTqAE0ESinl4zQRKKWUj9NEoJRSPk4TgVJK+ThNBEq1U30F003NbkPrK32WGGM2GmO2GWPuqd+3+f3bjTEPeTp+pY6l151HoJSNqkRkQvM7jDFDgTUicoExJgzYZIx5r35zw/0hwEZjzFsi8nX3hqzU8WmPQKkuUl/WYT1WvZvm91dh1cLx5sKGqhfTRKBU+4U0Oyz0VsuNxpj+WPXht7S4vx8wEviye8JUqmP00JBS7XfUoaF604wxG7FKOjxQXwJhev39P2LVvnlARA51W6RKdYAmAqVO3BoRueBY9xtjTgK+qh8j2NTNsSl1XHpoSCmb1Vd7XQTc5elYlGqNJgKluscS4HRjTJKnA1GqJa0+qpRSPk57BEop5eM0ESillI/TRKCUUj5OE4FSSvk4TQRKKeXjNBEopZSP00SglFI+7v8BY85nJMq2NuwAAAAASUVORK5CYII=",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "array([0.371273], dtype=float32)"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "threshold=ROC(y_test,y_pred)\n",
    "threshold"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": [
    "threshold=results_threshold # Decide on your own threshold\n",
    "y_pred_label = [1.0 if (score > threshold) else 0 for score in y_pred ]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "precision= 0.7488051572746471\n",
      "recall= 0.5945111189551712\n",
      "f1= 0.6627969895223572\n"
     ]
    }
   ],
   "source": [
    "from sklearn.metrics import precision_score, recall_score, f1_score, classification_report\n",
    "prec=precision_score(y_test,y_pred_label,pos_label=1)\n",
    "recall=recall_score(y_test,y_pred_label,pos_label=1)\n",
    "f1=f1_score(y_test,y_pred_label,pos_label=1)\n",
    "print('precision=',prec)\n",
    "print('recall=',recall)\n",
    "print('f1=',f1)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "name": "USAD_test.ipynb",
   "provenance": [],
   "toc_visible": true
  },
  "kernelspec": {
   "display_name": "USAD",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.11"
  },
  "vscode": {
   "interpreter": {
    "hash": "dfdaddf1ba9b6c6e2043e155f92b2b8f5735f0351a3294800b2afd59282eda18"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}