arcface.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "toc": true
   },
   "source": [
    "<h1>Table of Contents<span class=\"tocSkip\"></span></h1>\n",
    "<div class=\"toc\"><ul class=\"toc-item\"><li><span><a href=\"#Images\" data-toc-modified-id=\"Images-1\"><span class=\"toc-item-num\">1&nbsp;&nbsp;</span>Images</a></span><ul class=\"toc-item\"><li><span><a href=\"#Test-thr\" data-toc-modified-id=\"Test-thr-1.1\"><span class=\"toc-item-num\">1.1&nbsp;&nbsp;</span>Test thr</a></span></li><li><span><a href=\"#Test-thr\" data-toc-modified-id=\"Test-thr-1.2\"><span class=\"toc-item-num\">1.2&nbsp;&nbsp;</span>Test thr</a></span></li><li><span><a href=\"#Full-train\" data-toc-modified-id=\"Full-train-1.3\"><span class=\"toc-item-num\">1.3&nbsp;&nbsp;</span>Full train</a></span></li></ul></li><li><span><a href=\"#Vision-transformer\" data-toc-modified-id=\"Vision-transformer-2\"><span class=\"toc-item-num\">2&nbsp;&nbsp;</span>Vision transformer</a></span></li><li><span><a href=\"#Texts\" data-toc-modified-id=\"Texts-3\"><span class=\"toc-item-num\">3&nbsp;&nbsp;</span>Texts</a></span><ul class=\"toc-item\"><li><span><a href=\"#train-with-freeze\" data-toc-modified-id=\"train-with-freeze-3.1\"><span class=\"toc-item-num\">3.1&nbsp;&nbsp;</span>train with freeze</a></span></li><li><span><a href=\"#train-with-freeze-full-data\" data-toc-modified-id=\"train-with-freeze-full-data-3.2\"><span class=\"toc-item-num\">3.2&nbsp;&nbsp;</span>train with freeze full data</a></span></li><li><span><a href=\"#No-train-test\" data-toc-modified-id=\"No-train-test-3.3\"><span class=\"toc-item-num\">3.3&nbsp;&nbsp;</span>No train test</a></span></li></ul></li><li><span><a href=\"#Effnet-image\" data-toc-modified-id=\"Effnet-image-4\"><span class=\"toc-item-num\">4&nbsp;&nbsp;</span>Effnet image</a></span><ul class=\"toc-item\"><li><span><a href=\"#b0\" data-toc-modified-id=\"b0-4.1\"><span class=\"toc-item-num\">4.1&nbsp;&nbsp;</span>b0</a></span></li><li><span><a href=\"#TF-b3\" data-toc-modified-id=\"TF-b3-4.2\"><span class=\"toc-item-num\">4.2&nbsp;&nbsp;</span>TF b3</a></span></li><li><span><a href=\"#TF-b0\" data-toc-modified-id=\"TF-b0-4.3\"><span class=\"toc-item-num\">4.3&nbsp;&nbsp;</span>TF b0</a></span></li><li><span><a href=\"#b2\" data-toc-modified-id=\"b2-4.4\"><span class=\"toc-item-num\">4.4&nbsp;&nbsp;</span>b2</a></span></li><li><span><a href=\"#b3\" data-toc-modified-id=\"b3-4.5\"><span class=\"toc-item-num\">4.5&nbsp;&nbsp;</span>b3</a></span></li></ul></li></ul></div>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Images"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Test thr"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from imports import *\n",
    "from utils import load_data\n",
    "from image_train.data import create_dl, ImageDS\n",
    "from image_train.model import EMBRes\n",
    "from arcface import ArcMarginProduct, compute_centers\n",
    "from image_train.train import *\n",
    "import matplotlib.pyplot as plt\n",
    "np.random.seed(1337)\n",
    "device = torch.device('cuda')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "df, train_df, val_df, train_labels, val_labels = load_data(df_path='data/train_image_cleaned.csv', train_perc=0.3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# creating dataloaders\n",
    "small_images_dir = 'data/small_train_images_224/'\n",
    "\n",
    "tr_dl = create_dl(train_df, small_images_dir)\n",
    "tr_test_dl = create_dl(train_df, small_images_dir, shuffle=False)\n",
    "val_dl = create_dl(val_df, small_images_dir, shuffle=False)\n",
    "full_dl = create_dl(df, small_images_dir, shuffle=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "***Embeddings normalization is not done in the model but in the arcface metric***"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "vision_model = 'resnet50'\n",
    "model = timm.create_model(vision_model, pretrained=True, num_classes=0).to('cuda')\n",
    "train_tfms, val_tfms = get_tfms()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "centers = compute_centers(tr_test_dl, model, val_tfms, train_df)\n",
    "torch.save(centers, 'centers.pth')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "centers = torch.load('centers.pth')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Using center as wieghts\n"
     ]
    }
   ],
   "source": [
    "metric_fc = ArcMarginProduct(2048, train_df['label_group'].nunique(), \n",
    "                             s=30, m=0.5, easy_margin=False, centers=centers).to(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "#freeze(metric_fc)\n",
    "#freeze_resnet_to()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "n_epochs, lf, params, optimizer, sched = get_hparams(tr_dl, model, metric_fc, lr=5e-4, n_epochs=15)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "loss_hist = []\n",
    "best_thr_score={'val': {'thr': None, 'f1': None}, 'train': {'thr': None, 'f1': None}}\n",
    "thr_score_hist=[]\n",
    "ep_start = 0\n",
    "save_path = 'data/test_2ap'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "fcc10b7dcf7f4f4fa28338be5366f6ba",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value=''), FloatProgress(value=0.0, max=15.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value=''), FloatProgress(value=0.0, max=160.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Thresholds'), FloatProgress(value=0.0, max=10.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Checkpoint : saved model to data/test_2ap_ep_0.pth\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value=''), FloatProgress(value=0.0, max=377.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Thresholds'), FloatProgress(value=0.0, max=10.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Saved best model ep 0 with f score : 0.6469142561779476\n",
      "Ep 0: Train loss 7.8184 | Val f score 0.6469 with thresh 0.56, train f score 0.6472 with thresh 0.73\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value=''), FloatProgress(value=0.0, max=160.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Thresholds'), FloatProgress(value=0.0, max=5.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value=''), FloatProgress(value=0.0, max=377.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Thresholds'), FloatProgress(value=0.0, max=5.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Saved best model ep 1 with f score : 0.6575995952749828\n",
      "Ep 1: Train loss 6.7015 | Val f score 0.6576 with thresh 0.61, train f score 0.7029 with thresh 0.78\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c91d745e3da24129b650e35d2d4148e2",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value=''), FloatProgress(value=0.0, max=160.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-10-fcde6b83309c>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[1;32m----> 1\u001b[1;33m best_thr_score, thr_score_hist, losses = train(model, optimizer, lf, sched, metric_fc, tr_dl, val_dl,\n\u001b[0m\u001b[0;32m      2\u001b[0m                                                \u001b[0mn_epochs\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mtrain_df\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mval_df\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mtrain_tfms\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mval_tfms\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      3\u001b[0m                                                \u001b[0msave_path\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0msave_path\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      4\u001b[0m                                                \u001b[0mprev_best_info\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mbest_thr_score\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0minfo_history\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mthr_score_hist\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      5\u001b[0m                                                ep_start=ep_start)\n",
      "\u001b[1;32m~\\Documents\\shopee\\image_train\\train.py\u001b[0m in \u001b[0;36mtrain\u001b[1;34m(model, optimizer, loss_func, sched, metric_fc, train_dl, val_dl, n_epochs, train_df, val_df, train_transforms, val_transforms, save_path, val_first, prev_best_info, info_history, ep_start, half_precision)\u001b[0m\n\u001b[0;32m    184\u001b[0m             \u001b[0mloss\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mloss_func\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mout\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mlabels\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    185\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 186\u001b[1;33m             \u001b[0mloss\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mbackward\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    187\u001b[0m             \u001b[0moptimizer\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mstep\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    188\u001b[0m             \u001b[1;32mif\u001b[0m \u001b[0msched\u001b[0m \u001b[1;32mis\u001b[0m \u001b[1;32mnot\u001b[0m \u001b[1;32mNone\u001b[0m \u001b[1;33m:\u001b[0m \u001b[0msched\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mstep\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\anaconda3\\lib\\site-packages\\torch\\tensor.py\u001b[0m in \u001b[0;36mbackward\u001b[1;34m(self, gradient, retain_graph, create_graph, inputs)\u001b[0m\n\u001b[0;32m    243\u001b[0m                 \u001b[0mcreate_graph\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mcreate_graph\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    244\u001b[0m                 inputs=inputs)\n\u001b[1;32m--> 245\u001b[1;33m         \u001b[0mtorch\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mautograd\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mbackward\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mgradient\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mretain_graph\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mcreate_graph\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0minputs\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0minputs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    246\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    247\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0mregister_hook\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mhook\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\anaconda3\\lib\\site-packages\\torch\\autograd\\__init__.py\u001b[0m in \u001b[0;36mbackward\u001b[1;34m(tensors, grad_tensors, retain_graph, create_graph, grad_variables, inputs)\u001b[0m\n\u001b[0;32m    139\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    140\u001b[0m     \u001b[0mgrad_tensors_\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0m_tensor_or_tensors_to_tuple\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mgrad_tensors\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mlen\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mtensors\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 141\u001b[1;33m     \u001b[0mgrad_tensors_\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0m_make_grads\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mtensors\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mgrad_tensors_\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    142\u001b[0m     \u001b[1;32mif\u001b[0m \u001b[0mretain_graph\u001b[0m \u001b[1;32mis\u001b[0m \u001b[1;32mNone\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    143\u001b[0m         \u001b[0mretain_graph\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mcreate_graph\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\anaconda3\\lib\\site-packages\\torch\\autograd\\__init__.py\u001b[0m in \u001b[0;36m_make_grads\u001b[1;34m(outputs, grads)\u001b[0m\n\u001b[0;32m     49\u001b[0m                 \u001b[1;32mif\u001b[0m \u001b[0mout\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mnumel\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;33m!=\u001b[0m \u001b[1;36m1\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     50\u001b[0m                     \u001b[1;32mraise\u001b[0m \u001b[0mRuntimeError\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m\"grad can be implicitly created only for scalar outputs\"\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 51\u001b[1;33m                 \u001b[0mnew_grads\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mtorch\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mones_like\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mout\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mmemory_format\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mtorch\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mpreserve_format\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     52\u001b[0m             \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     53\u001b[0m                 \u001b[0mnew_grads\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;32mNone\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "best_thr_score, thr_score_hist, losses = train(model, optimizer, lf, sched, metric_fc, tr_dl, val_dl,\n",
    "                                               n_epochs, train_df, val_df, train_tfms, val_tfms, \n",
    "                                               save_path=save_path, \n",
    "                                               prev_best_info=best_thr_score, info_history=thr_score_hist,\n",
    "                                               ep_start=ep_start)\n",
    "loss_hist.append(losses)\n",
    "ep_start += n_epochs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXsAAAD4CAYAAAANbUbJAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAgAElEQVR4nO3deXxU9bnH8c+Tfd8DxIQlKoKACBpBiyJqBdRWUbyK+9Lq7S1axVuvWu2Fi63a3ra2Vqult7RqVaSutFp3EVFUAqLs+xZCJGRPyDp57h9nEoaQkAlMOJnJ83695jVzljnzHIZ85ze/Oed3RFUxxhgT2sLcLsAYY0z3s7A3xphewMLeGGN6AQt7Y4zpBSzsjTGmF4hwu4C2MjIydNCgQW6XYYwxQWXZsmV7VTWzo+U9LuwHDRpEfn6+22UYY0xQEZHth1pu3TjGGNMLWNgbY0wvYGFvjDG9QKd99iIyF/gOsEdVR7SzXIDfARcC+4AbVXW5d9kNwAPeVX+mqk8fTpGNjY0UFBRQV1d3OE83XRATE0NOTg6RkZFul2KMCSB/fqD9K/A48EwHyy8ABntvY4EngbEikgbMBPIABZaJyAJVLetqkQUFBSQmJjJo0CCczxbTHVSVkpISCgoKyM3NdbscY0wAddqNo6qLgNJDrHIJ8Iw6PgNSRCQLmAS8q6ql3oB/F5h8OEXW1dWRnp5uQd/NRIT09HT7BmVMCApEn302sNNnusA7r6P5BxGRW0UkX0Tyi4uL230RC/qjw/6djQlNgTjOvr100EPMP3im6hxgDkBeXp6NuWxMKGhuhvoKqC2DukpQjzO/9S/c50/dd6h1bfa5eQ6cbm4+xHL1rtPmOdpMayRJGIj3Hmnz2LusZTstzztgW9rB407qbHkd39dv75bQF0Zc1i1vRyDCvgDo7zOdAxR6509oM39hAF7PGOOWugoo3wkVO5376m+cMK8tg7py72PvfV0FHbTvTEey83p02C8AbhOReTg/0Fao6m4ReRt4SERSvetNBO4LwOsFhYSEBKqrq90uwxj/qEJDNewrgZq9+8O87X19xYHPk3CITYHYVIhJgbgMSD9+/3RsqrM8JhnCfOPG+8X/gG5D2X8n4Qe2eMPC22kJi7PeQcvk4Oc7O7m/Nd7aYlef+d55rdto0wKnk9b5QXWE799Oy79x228cbW+ttQaeP4devoDTQs8QkQKcI2windr1KeBNnMMuN+EcenmTd1mpiDwILPVuaraqHuqHXnOYmpqaiIjocSNfGDe1hHf1HudWswdqimFfqRPm+0pgn/e+psS599QfvJ3oZEjpD8n9YeC3nPuU/pA8wLmPz2wT2KZDLR9ChLvy8p0mhKpe1clyBaZ3sGwuMPfwSmvf//xjNWsKKwO5SYYdk8TM7w4/5Dr33HMPAwcO5Ic//CEAs2bNQkRYtGgRZWVlNDY28rOf/YxLLrmk09fbvXs3V155JZWVlTQ1NfHkk09y1lln8dZbb/GTn/wEj8dDRkYG77//PqWlpdx8881s2bKFuLg45syZw8iRI5k1axaFhYVs27aNjIwMnn32We69914WLlxIfX0906dP59///d8D8u9jeqCaEtizBvZugKoipzulpnh/sFcXQ1Nt+8+NToa4NIjPgKRs6HcyxKdDXLrTMo9Lh+QcJ8xjko/ufpluY81BP02bNo0777yzNeznz5/PW2+9xYwZM0hKSmLv3r2cfvrpXHzxxZ0e0fL8888zadIk7r//fjweD/v27aO4uJhbbrmFRYsWkZubS2mp8yVo5syZjB49mtdee40PPviA66+/nhUrVgCwbNkyFi9eTGxsLHPmzCE5OZmlS5dSX1/PuHHjmDhxoh0vH+zqKqB4vRPse9Z679c5gd5CwpyAju8DCZmQfpzT4k7os39efB9nXlw6RES5tz/GNUEX9p21wLvL6NGj2bNnD4WFhRQXF5OamkpWVhYzZsxg0aJFhIWFsWvXLr755hv69et3yG2ddtpp3HzzzTQ2NjJlyhRGjRrFwoULGT9+fGs4p6WlAbB48WJefvllAM4991xKSkqoqHD6TS+++GJiY2MBeOedd/j666956aWXAKioqGDjxo0W9sGksQ4KlsLWRVD4pRPulQX7l0fGQ5+hMHgi9DnRuWUOhcR+Tn+xMYcQdGHvpssvv5yXXnqJoqIipk2bxnPPPUdxcTHLli0jMjKSQYMG+XVC0vjx41m0aBFvvPEG1113HXfffTcpKSntfiNQPfhohpb14uPjD1jv97//PZMmTTqCPTRHVVMDFC6HrR/D1o9g5xdOv7mEQZ9hTh95nxOdx32GOv3kYTaclTk8FvZdMG3aNG655Rb27t3LRx99xPz58+nTpw+RkZF8+OGHbN9+yOGkW23fvp3s7GxuueUWampqWL58Offffz/Tp09n69atrd04aWlpjB8/nueee46f/vSnLFy4kIyMDJKSkg7a5qRJk3jyySc599xziYyMZMOGDWRnZx/wgWBc5mmCoq+clvvWj2HHZ9BY4yzrexKc9n3IPcsJeesrNwFmYd8Fw4cPp6qqiuzsbLKysrjmmmv47ne/S15eHqNGjWLo0KF+bWfhwoX87//+L5GRkSQkJPDMM8+QmZnJnDlzuOyyy2hubqZPnz68++67zJo1i5tuuomRI0cSFxfH00+3P5bc97//fbZt28Ypp5yCqpKZmclrr70WyN03h6PZA9s+hpV/h7X/8B57jtP9MupqyB0Pg850fjA1phtJe90EbsrLy9O2V6pau3YtJ554oksV9T72732EVGHXcifgV7/iHCkTlQgnfgeO/zYMOgsS+7pdpQkxIrJMVfM6Wm4te2MCpXg9rHzJCfmyrRAeBSdMghGXO/eRsW5XaHoxC/tutHLlSq677roD5kVHR/P555+7VJEJuPKdTut95d+haKXz42rueBj/Yxj6HefsUWN6AAv7bnTSSSe1HhNvQkjlbljzGqx6BQq+cOZl58HkX8DwS62LxvRIFvbG+KO62An41a/C9k8BdY6gOe+/nYBPO9btCo05JAt7YzqyrxTWLnBa8Ns+dgaqyhgCE+5zAj7zBLcrNMZvFvbGtFW+E978MWx6D5qbnFb7mXc5Q8/2GWYDf5mgZGFvjK+tH8PfbwBPI5wxHYZfBlknW8CboGfnXvupvLycP/zhD11+3oUXXkh5eXk3VGQCShU+nwPPXAKxaXDLB3D+bDhmlAW9CQkW9n7qKOw9Hs8hn/fmm2+SkuL+4Xed1dmrNdbB67fBv+52Bhm75X3IGOx2VcYEVPB14/zrXud45kDqdxJc8MghV7n33nvZvHkzo0aNah3mICsrixUrVrBmzRqmTJnCzp07qaur44477uDWW28FYNCgQeTn51NdXc0FF1zAmWeeyaeffkp2djavv/5666iVbT322GM89dRTREREMGzYMObNm0d1dTW33347+fn5iAgzZ85k6tSpvPDCCzz00EOoKhdddBG/+MUvAOdqWXfddRdvv/02v/71r4mNjeWuu+6iurqajIwM/vrXv5KVlRXYf8tgU1kIL14Lu5bB2ffA2ffaYGMmJAVf2LvkkUceYdWqVaxYsYKFCxdy0UUXsWrVqtYhhOfOnUtaWhq1tbWcdtppTJ06lfT09AO2sXHjRl544QX+9Kc/ccUVV/Dyyy9z7bXXdvh6W7duJTo6urUb6MEHHyQ5OZmVK50Pu7KyMgoLC7nnnntYtmwZqampTJw4kddee40pU6ZQU1PDiBEjmD17No2NjZx99tm8/vrrZGZm8uKLL3L//fczd25Ary0TXHZ8Bi9eB4374Mq/wYnfdbsiY7pN8IV9Jy3wo2XMmDEHjBX/2GOP8eqrrwKwc+dONm7ceFDY5+bmMmrUKABOPfVUtm3b1uH2R44cyTXXXMOUKVOYMmUKAO+99x7z5s1rXSc1NZVFixYxYcIEMjMzAbjmmmtYtGgRU6ZMITw8nKlTpwKwfv16Vq1axfnnnw843Tq9ulWf/xd4827nakw3LHCGEjYmhAVf2PcQvkMHL1y4kPfee48lS5YQFxfHhAkT2h3XPjo6uvVxeHg4tbUdXDYOeOONN1i0aBELFizgwQcfZPXq1ajqQWPeH2ogu5iYGMLDw1vXGz58OEuWLPF7H0NSUwP8679g2V+cQcmm/p9zUWxjQpxfnZMiMllE1ovIJhG5t53lA0XkfRH5WkQWikiOzzKPiKzw3hYEsvijKTExkaqqqnaXVVRUkJqaSlxcHOvWreOzzz47otdqbm5m586dnHPOOfzyl7+kvLyc6upqJk6cyOOPP966XllZGWPHjuWjjz5i7969eDweXnjhBc4+++yDtjlkyBCKi4tbw76xsZHVq1cfUZ1Bp7IQnv6uE/RnzoCr51vQm16j05a9iIQDTwDnAwXAUhFZoKprfFb7FfCMqj4tIucCDwMtI4DVquqoANd91KWnpzNu3DhGjBhBbGwsffvuH/9k8uTJPPXUU4wcOZIhQ4Zw+umnH9FreTwerr32WioqKlBVZsyYQUpKCg888ADTp09nxIgRhIeHM3PmTC677DIefvhhzjnnHFSVCy+8sN2LnkdFRfHSSy/xox/9iIqKCpqamrjzzjsZPtydyzweVUUr4bMnncHKwiLg8r84J0gZ04t0Op69iJwBzFLVSd7p+wBU9WGfdVYDk1S1QJx+hgpVTfIuq1bVBH8LsvHs3RcS/97NzbDhLfjsD85QB5FxMOoaOOOHNo6NCUmBGM8+G9jpM10AjG2zzlfAVOB3wKVAooikq2oJECMi+UAT8IiqHnT5JBG5FbgVYMCAAX6UZEwH6qthxXPw+VNQugWScpyTo0653rpsTK/mT9i3d/pg268DPwYeF5EbgUXALpxwBxigqoUicizwgYisVNXNB2xMdQ4wB5yWfRfqD3rTp0/nk08+OWDeHXfcwU033eRSRUGqfAd8MQeWPQP1FZBzGpz7U+dwyvBIt6szxnX+hH0B0N9nOgco9F1BVQuBywBEJAGYqqoVPstQ1S0ishAYDRwQ9v5o70iUUPDEE0+4XcIBetplKjtVVwH/nOEMPYzAsEvg9B9C/9PcrsyYHsWfsF8KDBaRXJwW+zTgat8VRCQDKFXVZuA+YK53fiqwT1XrveuMA37Z1SJjYmIoKSkhPT09JAO/p1BVSkpKiImJcbsU/9RVwLOXwu6v4YzbYMytznHzxpiDdBr2qtokIrcBbwPhwFxVXS0is4F8VV0ATAAeFhHF6caZ7n36icAfRaQZ5zDPR9ocxeOXnJwcCgoKKC4u7upTTRfFxMSQk5PT+Ypu8w36K56GoRe5XZExPVqnR+Mcbe0djWPMAeoq4NnLYPdXFvTGeAXiaBxjeo7WoF8BVzxjQW+Mn2x4PxM86irhb1OdoP83a9Eb0xUW9iY41FXC3y6Dwi+doD/xO25XZExQsbA3Pd8BQf9XC3pjDoOFvenZWrpuCr90xrSxMeeNOSwW9qbnag365U7QD7vY7YqMCVoW9qZnqquE5y53Lhd4+VwLemOOkB16aXqW5mZY/Qq8PxsqCuDf/uIMgWCMOSIW9qbn2PoxvPtTp3++70lw/euQe5bbVRkTEizsjfv2rIX3ZjnjzyflwJSnYOQVEBbudmXGhAwLe+Oeyt2w8CH48m8QlQDfngVjfwCRsW5XZkzIsbA3R199FXzyGCx5HDyNTsCf9WOIT3e7MmNCloW9OXpUnYt9f/gQ1BTD8MvgvJ/aZQKNOQos7M3R4WmEf9zhXDJw4Di46kXIOdXtqozpNSzsTfdrqIG/3wgb34EJ98HZ94BdhMaYo8rC3nSvmhJ4/grnLNjvPAp5N7tdkTG9koW96T5l253hDsp3wBXP2gBmxrjIwt50j6JVTtA31TonRw08w+2KjOnVbGwcE3hbP4a/XAASBje9ZUFvTA/gV9iLyGQRWS8im0Tk3naWDxSR90XkaxFZKCI5PstuEJGN3tsNgSze9ECrX3PGnk/Mgu+9A32HuV2RMQY/wl5EwoEngAuAYcBVItL2L/hXwDOqOhKYDTzsfW4aMBMYC4wBZopIauDKNz3KF39yjrrJGgU3vwUp/d2uyBjj5U/LfgywSVW3qGoDMA9oOwzhMOB97+MPfZZPAt5V1VJVLQPeBSYfedmmR1GF9x+EN38MJ0x2+ujj0tyuyhjjw5+wzwZ2+kwXeOf5+gqY6n18KZAoIul+PhcRuVVE8kUkv7i42N/aTU/xwc/g41/B6Ovgyr9BVJzbFRlj2vAn7Ns7+0XbTP8YOFtEvgTOBnYBTX4+F1Wdo6p5qpqXmZnpR0mmx9izFhY/CidfBRf/HsLtAC9jeiJ//jILAN/O1xyg0HcFVS0ELgMQkQRgqqpWiEgBMKHNcxceQb2mJ1GFN++G6ESY+HM7K9aYHsyflv1SYLCI5IpIFDANWOC7gohkiEjLtu4D5nofvw1MFJFU7w+zE73zTChY/Sps+9gZzMxGrDSmR+s07FW1CbgNJ6TXAvNVdbWIzBaRlguDTgDWi8gGoC/wc+9zS4EHcT4wlgKzvfNMsKuvhncegH4nwak3uV2NMaYTonpQF7qr8vLyND8/3+0yTGfem+X01d/8DgwY63Y1xvR6IrJMVfM6Wm5n0Jqu27sJPn0cTr7agt6YIGFhb7pGFf71X86lA8//H7erMcb4ycLedM26N2Dz+3DOTyChj9vVGGP8ZGFv/NdYC2/dB32GwWm3uF2NMaYL7AwY47/Fj0LFDrjxDTt5ypggYy1745/SrbD4tzDichh0ptvVGGO6yMLe+Oftn0B4JEz8mduVGGMOg4W96dyGd2D9m3D2f0FSltvVGGMOg4W9ObTGOudQy4wTYOx/uF2NMeYw2a9s5tCWPA5lW+G6VyEiyu1qjDGHyVr2pmPlO2HRr+DEi+G4c92uxhhzBCzsTcfeud+5n/SQu3UYY46Yhb05mCp89EtY8zqM/0+7lqwxIcD67M2BVOH92bD4N85AZ2fe5XZFxpgAsLA3+6k6x9N/9gdnjPqLfgNh9uXPmFBgYW8czc3wxl2w7C/OIZaTH7bLDBoTQizsDTR7YMHtsOI5OHMGnDfTgt6YEGNh39t5GuGVW2H1K3DO/TD+bgt6Y0KQhX1v1lQPL90M6/4J58+GcXe4XZExppv49eubiEwWkfUisklE7m1n+QAR+VBEvhSRr0XkQu/8QSJSKyIrvLenAr0D5jA11sK8a5ygv+B/LeiNCXGdtuxFJBx4AjgfKACWisgCVV3js9oDwHxVfVJEhgFvAoO8yzar6qjAlm2OSEMNvDANtn4M330MTr3B7YqMMd3Mn5b9GGCTqm5R1QZgHnBJm3UUSPI+TgYKA1eiCai6Snj2Mti2GC79owW9Mb2EP2GfDez0mS7wzvM1C7hWRApwWvW3+yzL9XbvfCQiZ7X3AiJyq4jki0h+cXGx/9Wbrmmogecuh135cPlcOPlKtysyxhwl/oR9e4dmaJvpq4C/qmoOcCHwrIiEAbuBAao6GrgLeF5Ekto8F1Wdo6p5qpqXmZnZtT0w/mmqhxevhYKlMPXPMPxStysyxhxF/oR9AeA7OEoOB3fTfA+YD6CqS4AYIENV61W1xDt/GbAZOOFIizZd5GmCl78Hmz+Ai38Pw6e4XZEx5ijzJ+yXAoNFJFdEooBpwII26+wAzgMQkRNxwr5YRDK9P/AiIscCg4EtgSre+KG5Gf7xI1j7D5j8CIy+1u2KjDEu6PRoHFVtEpHbgLeBcGCuqq4WkdlAvqouAP4T+JOIzMDp4rlRVVVExgOzRaQJ8AA/UNXSbtsbcyBVePs+58zYCffB6XalKWN6K1Ft2/3urry8PM3Pz3e7jNDw4UPw0S/g9Okw6ed2ZqwxIUxElqlqXkfLbUjDUPXp407Qj77Wgt4YY2EfkpY/41xlatglzklTFvTG9HoW9qFm9avwjzvg+G/DZf8HYeFuV2SM6QEs7EPJxnfh5Vug/1i44lmIiHK7ImNMD2FhHyq2fwovXgd9ToSrX4SoOLcrMsb0IBb2oWDnUnj+SufC4Ne9CjHJbldkjOlhLOyD3Y7P4NlLIS4drnsN4jPcrsgY0wNZ2AezbZ84I1gm9IGb3oTktuPTGWOMw8I+WG392BnBMjnbCfqkY9yuyBjTg1nYB6MtC+G5f4OUAXDjG5DYz+2KjDE9nIV9sNn0vvNjbNqxcMM/nS4cY4zphIV9MNnwDrxwFaQPhhv+AQk29r8xxj8W9sFi/b/gxWugz1C4YQHEp7tdkTEmiFjYB4O1/3ROmOo7HK5/HeLS3K7IGBNkLOx7ujWvw99vgKyTnePoY1PdrsgYE4Qs7Huy1a/C32+C7FOdM2NjU9yuyBgTpDq9UpVxyfp/wcvfh5zT4NqXIDrR7YqMMUHMWvY90eYPYf710O8kuObvFvTGmCNmYd/TbF8C8652Dq+89hWISXK7ImNMCPAr7EVksoisF5FNInJvO8sHiMiHIvKliHwtIhf6LLvP+7z1IjIpkMWHnF3LnDNjk7Lh+tfsqBtjTMB02mcvIuHAE8D5QAGwVEQWqOoan9UeAOar6pMiMgx4ExjkfTwNGA4cA7wnIieoqifQOxL0ilY5g5rFpTmHV9qZscaYAPKnZT8G2KSqW1S1AZgHXNJmHQVa+huSgULv40uAeapar6pbgU3e7RlfxRvg2SkQGeecMGWjVxpjAsyfsM8GdvpMF3jn+ZoFXCsiBTit+tu78FxE5FYRyReR/OLiYj9LDxFl2+CZS0DVadGnDnK7ImNMCPIn7KWdedpm+irgr6qaA1wIPCsiYX4+F1Wdo6p5qpqXmdmLxnup2AVPXwyN+5ygzzzB7YqMMSHKn+PsC4D+PtM57O+mafE9YDKAqi4RkRggw8/n9k7Ve5wW/b5SuOF16DfC7YqMMSHMn5b9UmCwiOSKSBTOD64L2qyzAzgPQEROBGKAYu9600QkWkRygcHAF4EqPmjtK4VnpkBFAVwz3zlD1hhjulGnLXtVbRKR24C3gXBgrqquFpHZQL6qLgD+E/iTiMzA6aa5UVUVWC0i84E1QBMwvdcfidNYB3+bCiUb4eoXYeC33K7IGNMLiJPJPUdeXp7m5+e7XUb3+fBh+OgRuOJZGHax29UYY0KEiCxT1byOltsZtEfT3o2w+Dcw4nILemPMUWVhf7Sowht3QUQsTHrI7WqMMb2MjXp5tHz9ImxdBBf9GhL7ul2NMaaXsZb90bCvFN6+H7Lz4NSb3a7GGNMLWcv+aHhvJtSWOYObhdnnqzHm6LPk6W47PoPlz8Dp/+GMT2+MMS6wsO9Onkb45wxIyoEJ97ldjTGmF7NunO605HHYswamvQDRCW5XY4zpxaxl313KtsHCX8CQi2DohZ2ubowx3cnCvjuowpt3g4TBhb90uxpjjLGw7xZrXoeN78A5P4HkHLerMcYYC/uAq6uEt+51jrwZ+wO3qzHGGMB+oA28D38OVUVw5XMQbv+8xpiewVr2gVT4JXwxB077HuTYGPXGmJ7Dwj5Qmj3wjzshPhPO+2+3qzHGmANYP0Og5M+F3Stg6p8hJtntaowx5gDWsg+Epnr4+NcwcByMmOp2NcYYcxAL+0D4ej5U7Yaz7gIRt6sxxpiDWNgfqeZm+OR3zqGWx53ndjXGGNMuv8JeRCaLyHoR2SQi97az/FERWeG9bRCRcp9lHp9lCwJZfI+w/g3n4uHj7rRWvTGmx+r0B1oRCQeeAM4HCoClIrJAVde0rKOqM3zWvx0Y7bOJWlUdFbiSexBVWPwopAyEYVPcrsYYYzrkT8t+DLBJVbeoagMwD7jkEOtfBbwQiOJ6vG2LYdcy+NbtdgKVMaZH8yfss4GdPtMF3nkHEZGBQC7wgc/sGBHJF5HPRKTd5q+I3OpdJ7+4uNjP0nuAT34LcRkw+lq3KzHGmEPyJ+zb64jWDtadBrykqh6feQNUNQ+4GvitiBx30MZU56hqnqrmZWZm+lFSD1C0Eja9B6f/ACJj3a7GGGMOyZ+wLwD6+0znAIUdrDuNNl04qlrovd8CLOTA/vzgtfi3EJUAp33f7UqMMaZT/oT9UmCwiOSKSBROoB90VI2IDAFSgSU+81JFJNr7OAMYB6xp+9ygU7YNVr8Cp94IsaluV2OMMZ3q9FdFVW0SkduAt4FwYK6qrhaR2UC+qrYE/1XAPFX17eI5EfijiDTjfLA84nsUT9D69HGQcDhjutuVGGOMX/w6hERV3wTebDPvv9tMz2rneZ8CJx1BfT1PdTF8+SycfCUkHeN2NcYY4xc7g7arvvijMxbOt+5wuxJjjPGbhX1X1Fc549UPvQgyT3C7GmOM8ZuFfVcsexrqKpyhEYwxJohY2PurqQGWPAEDz4T+p7ldjTHGdImFvb9WzoeqQjhzRufrGmNMD2Nh74+WYYz7ngTH2zDGxpjgY2Hvjw3/gr0bYNwdNoyxMSYoWdh3pnUY4wEw/FK3qzHGmMNiYd+Z7Z9CwVL41o9sGGNjTNCysO/M4kedYYxHXeN2JcYYc9gs7A+l8EvY9C6M/QFExbldjTHGHDYL+0P54GcQkwJjb3W7EmOMOSIW9h3ZvsS5OMmZd0JMstvVGGPMEbGwb48qfPAgxPeBMdaqN8YEPwv79mz+ALZ/AuN/DFHxbldjjDFHzMK+rZZWfXJ/50pUxhgTAizs21r3hnMUztn/BRHRbldjjDEBYWHvq9kDH/4c0o6Dk692uxpjjAkYOyXU16pXYM8amPpnO1vWGBNS/GrZi8hkEVkvIptE5N52lj8qIiu8tw0iUu6z7AYR2ei93RDI4gPK0wgLH4I+w2H4ZW5XY4wxAdVp81VEwoEngPOBAmCpiCxQ1TUt66jqDJ/1bwdGex+nATOBPECBZd7nlgV0LwJhxfNQugWmPQ9h1rtljAkt/qTaGGCTqm5R1QZgHnDJIda/CnjB+3gS8K6qlnoD/l1g8pEU3C2a6uGjX0L2qTDkQrerMcaYgPMn7LOBnT7TBd55BxGRgUAu8EFXnisit4pIvojkFxcX+1N3YOX/BSoL4NwHbLx6Y0xI8ifs20s/7WDdacBLqurpynNVdY6q5qlqXmZmph8lBVBDDXz8K+fasseec3Rf2xhjjhJ/wr4A6O8znZt2xdwAABAzSURBVAMUdrDuNPZ34XT1ue74/I9QUwzn/dRa9caYkOVP2C8FBotIrohE4QT6grYricgQIBVY4jP7bWCiiKSKSCow0TuvZ6gtd64te/z5MOB0t6sxxphu0+nROKraJCK34YR0ODBXVVeLyGwgX1Vbgv8qYJ6qqs9zS0XkQZwPDIDZqloa2F04AkuegLpyp6/eGGNCmPhkc4+Ql5en+fn53f9CNXvhdyfDcefClc92/+sZY0w3EpFlqprX0fLee5ro4kedH2fPud/tSkwvoKoUlNUSGxVOWlwUYWHB+fuQqrK3uoGte2vYUlzNvgYPKXGRpMRFkhwbRUpcJKlxUSTFRBAR3vXzVVoan2K/nwVc7wz7yt2w9P9g5JXQZ6jb1ZgQtbuilsUb9/LJpr18srmE4qp6ACLChIyEaDITo+mTGE2fpGgyE2Ocx4nR9EmKoX9qLOkJgRmIr6GpmfVFVeytqSc2Mty5Re2/j4sKJyYi/IAPoNoGD9tKathS7IT61r01bPYGfFVdk1+vmxgT4RP+kXialfomDw2eZhqafG6eZup9HqtCVHgYkeFCRHgYkeFhRIULkRHO44gwISrCuQfn8D5V72F+qj7T6txryzpKszrzWu4VDp7n/cDx3Q74bqdlD/WAbR/wHJ9pgMiW/QkLIyrCeRzp3Tffx4P7JPDAd4Ydztvcqd4Z9osfheYmmHDQyA8mwOoaPazYWc7nW0r5fGsJKwsqyEyKZkjfRIb0S2y9H5geT3iQtnZbVNQ28tmWEj7ZtJfFm/aypbgGgPT4KMYdn8GY3DSaPM3sqapvve0qr+WrgnJKahpo26PaNymaEcckM/yYJIZ573NSYw/Z6vU0K5uLq/lqZzlfF1TwdUE5a3dX0eBp7rT+6Igw4qLCCQ8LY291/QHLspJjODYznimjsjk2M57cjHiOy0wgMSaCitpGyvY1Ur6vwXlc00B5bSPl3nnltY1U1jYSHibERoWTHB5JVEQYURHhRIU74Rcd4dxHhYchAo0epdHTTJOnmQafx40epcHT7J1u+RaA914Q77S0mQYhPAwEISxs/7IwEUT23wvS+nxatyU+r+Fsq3W6vXU4+JuJp9nZhwbvPjR596HBozR6P+hqGjyU7Wvs9H06XL0v7PeVwpfPwklXQFqu29X0CA1Nzewqr2VH6T52lNQ496X7KCyvIy0+igFpcQxMj2NAWhwDvPdxUe3/16lt8LB8Rxmfbynhs62lrNhZTkNTMyIwtF8S3x11DCXV9awrquKt1UWtARcdEcbgvgmc0Hf/B8DxfRLolxRzWN0B4LSuiirrWFdUxbrdVawvqqS6vom+STFkJcfQLzmWfkkx9Et2puOjD/3noKrUNjp/kGU1DZTta6BsXyMbiqpYvGkvXxeU06wQFxXOmNw0rh4zgHHHZzCkb2Kn3TaNnmZKqhvYU1XHnsp6tpXUsGpXBasLK/lw/R6avf9OybGRDMtKYvgxSQzPTuK4zAS2lezja2+4ryqsYF+Dc5pLQnQEI7KTuHHcIEbmJHNMSix1DR5qG723hvbvG5qaOSYlltyM+NZg7+j9BkiJi2JgetfeG3P09b4faBf9yrk4yX98Cn2Hd9/r9DDNzU6f8ZrdlWwurmZHyb7WUN9dUdsaJuAE74C0OLJSYimtqWd7yb6DvrpnJETv/wBIi6PR08znW0v5uqCcRo8SJjD8mGTG5qYx9th0xgxKIzku8oBt1DZ42LSnmnVFlWz4pop1RVVs+KaKbyr3tyrDw4R+STFkp8aSkxpLTkosOalxrdNZybFERYRRXd/E+qIq1hdVsa6oknXexxW1+1tKWckxJMdG8k1lXbstqMSYCLKSY+ibFENmYjR1jR7Kahq9oe4Ee0PTwS3k8DBhVP8Uxh2fwZnHZzCqfwpREYEbX6m2wcO6okpWFzq3NYUVrCuqot6nlqiIMIZlJXFyTjIjc1I4uX8yx2YkBO1vA6brOvuBtneFfVM9PDoC+p0E173SPa8RIE2eZtbsruSbynrS4p1+z/T4aBJjIjr9A97X0NTaml27u5K1u53wq67fH9i+Yd3fG9gt05kJ0Qe8hqpSUdvI9pJ9bC/dx87SfWxv+QZQso/dlXWEiXBSdjJjj03j9Nx0Th2USlJMZHvldaqspoEN31SxZW8Nu8pq2VVey66yWgrK9lFUWXfAB5MIpMZFUVrT0DovITqCE/omMDQriaH9EhnaL4khfRMP+LCpa/TwTWUduyvqKKpoua+lqNKZLq6qJzYqnNS4KFLiolrfg5S4KFLjIlvv0+KjyEqJJaGTbwWB1uRpZnNxDZuLqxmQFseQfolEHuY3IBMaLOx9LX8WFtwG170Gx/WsoRHqGj18uaOcpdtKWbqtlOXby6hp8By0XniYtIZMalwUafHOLTEmkh2lNazdXcW2kprW7pHE6AiGZiVyYlZS621wn4ROuyy6WrsqxEaFB2ybHWn0NFNUUUeB90OgoGwf31TWk50Sw5B+Trh31q9tTCiyQy9bqMKSx6HvCDh2gtvVUFHbyLLtpXyxtYyl2/Z3fwAM7ZfIZafkcFpuGgPT4iivbaS0pp7SGqevuKSmgbKaBkprGti4p5rSGueHseyUWE7MSuSSUcdwYlYSw7I6/0EvEGIiuz/kW0SGh9Hf+23EGOO/3hP2m96D4nVw6R+P+hg4qsrWvTUs31HOlzvKWL6jnHVFlag6h+GdlJPMzeNyOW1QGnmDUkmJizqs17DWrDGmI70n7D99DBKPOSpXoaqqa+SrnRUs31HGlzvK+HJnOeXeHwQToiMY1T+FO84bzJhBaYwakHLIIx38ZUFvjDmU3hH2u7+CrYvg2/8DEV1vNR9Ky5mRn28tJX9bKV/uKGfDnqrWPvPBfRKYNKwfowekcMrAVI7LTAj648mNMcGnd4T9p49DVAKceuMRb0pV2VxcwxdbnZOEvthayu6KOsA5Bnr0gBQuPCmLUwamMDInheTYwzsixRhjAin0w76iAFa9DGN/ALEpXX56c7OyrqiKL7aW8MW2Ur7YWsreaucwv8zEaMbkpnF6bhpjctMZ3MeOazbG9EyhH/afPencn/4Dv1avrm/iq53lLN9exrIdZSzfXkal94Si7JRYxg/OZIz3RKFB6XHWV26MCQqhHfZ1FbDsaRg+BVIGHLS4pb992fay1tu6okqa1TlgZ3CfBC48KYsxuWmMyU0jJ9UO9zPGBKfQDvvlz0BDFZxxW+usbyrreGfNN3yycS/LdpS1jkQYHxXO6AGp3HbuYE4dmMqo/tbfbowJHaEb9p5Gpwtn0FlsjR7C2x9t5u3VRXy5oxyA/mmxnHl8BqcMTOXUAakM6ZdoR8kYY0JWSIa9qlLw8XP0r9zFT5tu5tlfLQTgpOxkfjzxBCYN78fxfRKsv90Y02uETNg3eZpZuq2Mt1cX8e7qIv5Y+yj1HMOm5DOYOe4YJg7vR3ZKrNtlGmOMK/wKexGZDPwO54Lj/6eqj7SzzhXALJyLs3ylqld753uAld7VdqjqxQGo+yBFlXVc9afPiIoI49acnYyo30bNpN/wwhnjuuPljDEmqHQa9iISDjwBnA8UAEtFZIGqrvFZZzBwHzBOVctEpI/PJmpVdVSA6z5ITmocT988hryBqcT/fRrEZxKfd013v6wxxgQFfwbAHgNsUtUtqtoAzAMuabPOLcATqloGoKp7Alumf84+IZP4io2w6V0YcytExrhRhjHG9Dj+hH02sNNnusA7z9cJwAki8omIfObt9mkRIyL53vlT2nsBEbnVu05+cXFxl3bgIEseh4hYyPvekW3HGGNCiD999u0dstL2iicRwGBgApADfCwiI1S1HBigqoUicizwgYisVNXNB2xMdQ4wB5yLl3RxH/arKoKv58Mp10O8XRTTGGNa+NOyLwD6+0znAIXtrPO6qjaq6lZgPU74o6qF3vstwEJg9BHW3LEv5jjH15/+w257CWOMCUb+hP1SYLCI5IpIFDANWNBmndeAcwBEJAOnW2eLiKSKSLTP/HHAGrpDQw0s/TMMvQjSj+uWlzDGmGDVaTeOqjaJyG3A2ziHXs5V1dUiMhvIV9UF3mUTRWQN4AHuVtUSEfkW8EcRacb5YHnE9yiegKqrdC43aK16Y4w5SO+64LgxxoSozi447k83jjHGmCBnYW+MMb2Ahb0xxvQCFvbGGNMLWNgbY0wvYGFvjDG9gIW9Mcb0Ahb2xhjTC/S4k6pEpBjYfgSbyAD2BqicniDU9gdCb59CbX8g9PYp1PYHDt6ngaqa2dHKPS7sj5SI5B/qLLJgE2r7A6G3T6G2PxB6+xRq+wNd3yfrxjHGmF7Awt4YY3qBUAz7OW4XEGChtj8QevsUavsDobdPobY/0MV9Crk+e2OMMQcLxZa9McaYNizsjTGmFwiZsBeRySKyXkQ2ici9btcTCCKyTURWisgKEQm6K7qIyFwR2SMiq3zmpYnIuyKy0Xuf6maNXdXBPs0SkV3e92mFiFzoZo1dISL9ReRDEVkrIqtF5A7v/KB8nw6xP8H8HsWIyBci8pV3n/7HOz9XRD73vkcvei8b2/F2QqHPXkTCgQ3A+TgXP18KXNVtl0A8SkRkG5CnqkF5MoiIjAeqgWdUdYR33i+BUlV9xPuhnKqq97hZZ1d0sE+zgGpV/ZWbtR0OEckCslR1uYgkAsuAKcCNBOH7dIj9uYLgfY8EiFfVahGJBBYDdwB3Aa+o6jwReQr4SlWf7Gg7odKyHwNsUtUtqtoAzAMucbmmXk9VFwGlbWZfAjztffw0zh9i0Ohgn4KWqu5W1eXex1XAWiCbIH2fDrE/QUsd1d7JSO9NgXOBl7zzO32PQiXss4GdPtMFBPkb7KXAOyKyTERudbuYAOmrqrvB+cME+rhcT6DcJiJfe7t5gqLLoy0RGQSMBj4nBN6nNvsDQfweiUi4iKwA9gDvApuBclVt8q7SaeaFSthLO/OCv38KxqnqKcAFwHRvF4LpeZ4EjgNGAbuBX7tbTteJSALwMnCnqla6Xc+Ramd/gvo9UlWPqo4CcnB6Mk5sb7VDbSNUwr4A6O8znQMUulRLwKhqofd+D/Aqzpsc7L7x9qu29K/ucbmeI6aq33j/GJuBPxFk75O3H/hl4DlVfcU7O2jfp/b2J9jfoxaqWg4sBE4HUkQkwruo08wLlbBfCgz2/jodBUwDFrhc0xERkXjvD0yISDwwEVh16GcFhQXADd7HNwCvu1hLQLSEotelBNH75P3x78/AWlX9jc+ioHyfOtqfIH+PMkUkxfs4Fvg2zm8RHwKXe1fr9D0KiaNxALyHUv0WCAfmqurPXS7piIjIsTiteYAI4Plg2ycReQGYgDMU6zfATOA1YD4wANgB/JuqBs0Pnh3s0wSc7gEFtgH/3tLf3dOJyJnAx8BKoNk7+yc4/dxB9z4dYn+uInjfo5E4P8CG4zTQ56vqbG9GzAPSgC+Ba1W1vsPthErYG2OM6ViodOMYY4w5BAt7Y4zpBSzsjTGmF7CwN8aYXsDC3hhjegELe2OM6QUs7I0xphf4f5NpmTphWmpDAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXsAAAD7CAYAAACL+TRnAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAgAElEQVR4nO3de3RV9Z338feXEAg3IZCoSLhEiojIPQNOddTWalGreMEOYDvi6gzTLunjOJfWPtNHLTOu8el0preHacdxAGtFtICCDq1Vq9NOq8AJd1AEEZIQLpFLALkl4fv8sXfiISTkJDnJzjnn81qLxdm3X77bI182v+/+/X7m7oiISHrrFHUAIiLS9pTsRUQygJK9iEgGULIXEckASvYiIhlAyV5EJAMklOzNbLKZbTWz7Wb2cAPHB5vZG2a2wczeMrOCuGM1ZrYu/LU8mcGLiEhirKn37M0sC3gfuBEoA1YD0919S9w5vwBecfenzeyzwP3u/uXw2DF379lWNyAiIk3rnMA5E4Ht7r4DwMwWAVOALXHnXAE8FH5+E3ippQHl5eX5kCFDWnq5iEhGKi4u/sjd8xs7nkiyHwCUxm2XAZPqnbMeuBv4IXAn0MvM+rn7ASDHzGJANfCEu5/3L4IhQ4YQi8USCEtERGqZ2a7zHU+kz94a2Fe/7+dvgevMbC1wHbCbILkDDHL3ImAG8AMzG9pAkLPMLGZmsYqKigRCEhGR5kgk2ZcBA+O2C4Dy+BPcvdzd73L3ccDfh/sqa4+Fv+8A3gLG1f8B7v6kuxe5e1F+fqP/ChERkRZKJNmvBoaZWaGZdQGmAWe9VWNmeWZW29a3gHnh/lwz61p7DnA1Z/f1i4hIO2gy2bt7NTAbeBV4F3jB3Teb2Rwzuz087Xpgq5m9D1wEPB7uHwHEzGw9QeH2ifi3eEREpH00+epleysqKnIVaEVEmsfMisP6aIM0glZEJAMo2YuIZIBE3rOXTHTmDKx6Eo4fiDoSyTSDJsGnPpe89nYXw+mPofDa5LVZshK2v5689mpdcAkU3Z/8dlGyl8Z88Ab86pvhRkNDLUTagkPX3vA370GX7q1v7swZWPwVOHEoaDO7W3LaXPoXcHgXSf+zUVCkZC/trHgB9MiHh7ZA5y5RRyOZYuf/wIJbYctLMHZG69v78L/h0IfB5y3LYMy01re5480g0d/9nzBqauvbayfqs5dzHdkDW38JY+9Vopf2NfhqyLsMYvOT017xAuiWC30vDT4npc350L0fjLgtOe21EyV7OdfaZ8BrYMJ9UUcimcYMJsyEslWwb3Pr2jq2H957JXhomXA/lLwN+99tXZtH98J7K8IHoa6ta6udKdnL2c7UQPHTcOn1wdOQSHsbMx2yurb+6X7tz+FMNYy/L+gSyurS+qf7ugehma1rJwJK9nK27W/AkbLgSUgkCt37whVTYMPzcPp4y9o4cwbWPA2Dr4H8y6BHXtDtsv45qDrR8jaLfxa81dPvnPkcOzwlezlb8XzocSFcfmvUkUgmK7ofTh2BzUtbdv2Hb8GhnWe/2TLhfjhZCZtbuNzGB7+BypKUfRBSspdPVO6G938F4+6FrOyoo5FMNuiPIW94y7tyYvOhW9+zi6hDroF+nwoeaFqieD50z4PLv9Cy6yOmZC+fWPtz8DNBH6dIlGoLtbtjsHdj8649ug+2rgj66eOLqLVtlq6Efc2cj7H2DbVxqfuGmpK9BM7UwJqfwdDPQt/CqKMRCd6Jz+ra/KLqurAw21B3y5gWFmrX/jwozKbwg5CSvQS2vRYWZmdGHYlIoHtfGHkHbHghmO4gEWfOBG+TDfkTyPvUucd79AuKv+sXJV78PVMTFHsLr0vJwmwtJXsJFC+AnhfB8FuijkTkExPCQu2mBAu1O34TjG4930PLhJlwqhI2v5hYmx/8BipL22wag/aiZC9QWQbbXoVxX1JhVjqWQVdB/uWJF1WLFzQ9urV2lG6iXTmx+cHUIcNT+w01JXuBNc+AO4z/s6gjETlbXaG2GPZsOP+5iY5ubc4o3SPlwRtqaTB1SELJ3swmm9lWM9tuZg83cHywmb1hZhvM7C0zK4g7dp+ZbQt/pW51I13VVAejAod+FnKHRB2NyLnGTIPOOU0/iTdndGuio3RrC7NpMHVIk8nezLKAucDNwBXAdDO7ot5p3wN+5u6jgTnAP4XX9gUeBSYBE4FHzSw3eeFLq21/DY7sTvn+SElj3XJh5J1BofbUsYbPOVPTvNGtZ43SbaT4Wzd1yGfSYuqQRJ7sJwLb3X2Hu58GFgFT6p1zBfBG+PnNuOOfB15z94Pufgh4DZjc+rAlaWLzoefFcJm+FunAJtwPp4/CpiUNH//gzeaPbq0bpdtIoXb768EbamnyIJRIsh8AlMZtl4X74q0H7g4/3wn0MrN+CV4rUTlcGjzZqzArHd3AiZA/ovGunOKwiNqc0a1NjdItXhBMHZImb6glkuwbWorF623/LXCdma0FrgN2A9UJXouZzTKzmJnFKioqEghJkmJtWJhNg/5ISXNmwRN2+RrYs/7sYy1df6G2zYZG6dZNHZI+D0KJJPsyYGDcdgFQHn+Cu5e7+13uPg74+3BfZSLXhuc+6e5F7l6Un5/fzFuQFqmpDkbMfupz0GdQ1NGING30nwaF2vpP4nWjW1vwNtnoP214lO7aZ4KpQ9LoQSiRZL8aGGZmhWbWBZgGLI8/wczyzKy2rW8B88LPrwI3mVluWJi9KdwnUdv2azi6RyNmJXV06wMj74KNv/ikUFs7uvXS61s2urV730+Kv7WF2vipQ9LoDbUmk727VwOzCZL0u8AL7r7ZzOaY2e3hadcDW83sfeAi4PHw2oPAPxD8hbEamBPuk6gVz4de/VWYldRSdD+cPgabFgfb298IRre2Ztrh2kJtbfF3W/iGWopOZdyYhBYcd/cVwIp6+x6J+7wYWNzItfP45ElfOoLDJcH/0Nf+HWRpzXlJIQV/BBeODLpyJsz8pDDbmiLqwEnBKN3Y/KArqHh+OHXIzUkLuyPQCNpMtOaZ4PfxX442DpHmqh39umddUJStLaK2ZnSrWfAUX74maHPbr9OqMFtLyT7T1I6YHXajCrOSmkZ/ETp3g6Wzkrf+wpiw+Lt0Vjh1SPoUZmvp3/DpYsdb8PFHTZ/30ftBYfbWf23zkETaRLc+cOXdwbz1yVp/oVtuUPxdvzB4Qy13cOvb7GCU7NPB7jXws/qDms+jzyAYdlPbxSPS1v7oK8Hi4RP/Molt/nkwfUIy2+xAlOzTQWweZHeHr/w6eGe4KT0vVGFWUtuA8fCND4In8mQpmJD8NjsQ/YlPdSfDV8auvBsuHhV1NCLtpy2ScpomelCBNvVtfAGqjqfNZE0i0jaU7FOZO8QWwMWj4ZLxUUcjIh2Ykn0q210M+zYG7x1bQ3POiYgElOxTWfF8yO4Bo+6JOhIR6eCU7FPVyUrYtBRGTYWcC6KORkQ6OCX7VLUhLMxq1koRSYCSfSpyDyZt6j8meN9YRKQJSvapqCwG+zen3RSsItJ2lOxTUfF86NIz6K8XEUmAkn2qOXH4k8Js115RRyMiKULJPtVseAGqT6gLR0SaRck+lbgHXTiXjINLxkYdjYikkISSvZlNNrOtZrbdzB5u4PggM3vTzNaa2QYzuyXcP8TMTpjZuvDXT5N9AxmldBXs36KnehFptiZnvTSzLGAucCNQBqw2s+XuviXutG8TLET+EzO7gmC92iHhsQ/cXY+hyVC8ALr0Cma4FBFphkSe7CcC2919h7ufBhYB9VfKcKB2GGdvoDx5IQoAJw7B5qUw+h7o2jPqaEQkxSSS7AcApXHbZeG+eI8BXzKzMoKn+q/HHSsMu3f+28z+pKEfYGazzCxmZrGKiorEo88k65+H6pPqwhGRFkkk2Tc0naLX254OLHD3AuAW4Bkz6wTsAQa5+zjgr4GFZnbORC7u/qS7F7l7UX5+fvPuIBPUFWbHQ//RUUcjIikokWRfBgyM2y7g3G6arwAvALj720AOkOfup9z9QLi/GPgAuKy1QWec0pVQ8Z4WKBGRFksk2a8GhplZoZl1AaYBy+udUwLcAGBmIwiSfYWZ5YcFXszsUmAYsCNZwWeM2HzoeoEKsyLSYk2+jePu1WY2G3gVyALmuftmM5sDxNx9OfA3wH+Y2UMEXTwz3d3N7FpgjplVAzXAV939YJvdTTo6fhA2vwjjvwxdekQdjYikqIQWHHf3FQSF1/h9j8R93gJc3cB1S4AlrYwxs61fBDWnNJWxiLSKRtB2ZO7Bu/UDiuDiUVFHIyIpLKEne0myyrJgQrOmVLwHH22FKXPbPiYRSWtK9u3twAcwdyKcqU7s/K69YeSdbRuTiKQ9Jfv2tubpoHvmrqegc9emz+/3KRVmRaTVlOzbU/VpWPssDL85mPZARKSdqEDbnt57GY5/pCkPRKTdKdm3p+IF0HsQDP1s1JGISIZRsm8vBz6AD38LE/4MOuk/u4i0L2Wd9lI8Hzp1hnFfjjoSEclASvbtofoUrFsYFGZ7XRx1NCKSgZTs28O7L8PxA5ryQEQio2TfHooXQJ/BcKkKsyISDSX7tvbRdtj5O5hwnwqzIhIZZZ+2VluYHfulqCMRkQymZN+Wqk6GhdlboNdFUUcjIhlMyb4tvfsynDio5QRFJHJK9m2peAHkDoHC6yMOREQyXULJ3swmm9lWM9tuZg83cHyQmb1pZmvNbIOZ3RJ37FvhdVvN7PPJDL5Dq3gfdv0PjFdhVkSi1+Ssl+GC4XOBG4EyYLWZLQ+XIqz1beAFd/+JmV1BsIThkPDzNGAkcAnwupld5u41yb6RDqd4QThiVoVZEYleIo+cE4Ht7r7D3U8Di4Ap9c5x4ILwc2+gPPw8BVjk7qfc/UNge9heeqs6CesXwuW3Qs8Lo45GRCShZD8AKI3bLgv3xXsM+JKZlRE81X+9GddiZrPMLGZmsYqKigRD78DeXQ4nDmkqYxHpMBJJ9tbAPq+3PR1Y4O4FwC3AM2bWKcFrcfcn3b3I3Yvy8/MTCKmDi82H3EIovC7qSEREgMSSfRkwMG67gE+6aWp9BXgBwN3fBnKAvASvTS8VW6HkD8E8OCrMikgHkUg2Wg0MM7NCM+tCUHBdXu+cEuAGADMbQZDsK8LzpplZVzMrBIYBq5IVfIdUvAA6ZcPYe6OORESkTpNv47h7tZnNBl4FsoB57r7ZzOYAMXdfDvwN8B9m9hBBN81Md3dgs5m9AGwBqoEH0vpNnKoTwYjZEV+AnmnQHSUiaSOhBcfdfQVB4TV+3yNxn7cAVzdy7ePA462IMXVsWQYnD6swKyIdjjqVk6l4AfS9FIb8SdSRiIicRck+Wfa/CyVvqzArIh2SslKyqDArIh2Ykn0yVJ2A9c/BiNugR17U0YiInEPJPhk2vwQnKzWVsYh0WEr2yVA8H/oOVWFWRDosJfvW2rcFSlcGhVlraHYIEZHoKdm3VvECyOqiwqyIdGhK9q1x+jisXwQjboce/aKORkSkUUr2rbHlJTilwqyIdHxK9q0Rmw/9hsHgBmeKEBHpMJTsW2rfZihbpcKsiKQEJfuWqivMzog6EhGRJinZt8Tp47D+ebhiCnTvG3U0IiJNUrJvic1Lg8KspjIWkRShZN8SxQsgbzgM/nTUkYiIJCShxUskzt5NULYaPv9PKsy2off2HuG1zfuiDkOkXV3UO4cvFg1s+sQWSCjZm9lk4IcEyxI+5e5P1Dv+feAz4WZ34EJ37xMeqwE2hsdK3P32ZAQemeL5kNUVxkyLOpK09s0lG1lfejjqMETa1diBfaJL9maWBcwFbgTKgNVmtjxcihAAd38o7vyvA+Pimjjh7mOTF3KETn8MG16AkXeoMNuGNpdXsr70MP/nC1dw3x8PjjockbSQyJP9RGC7u+8AMLNFwBSCRcQbMh14NDnhdTCblsKpIyrMtrGFK0vo2rkTU8cX0DlLZSWRZEjkT9IAoDRuuyzcdw4zGwwUAr+J251jZjEze8fM7mjkulnhObGKiooEQ49A8fygMDvoqqgjSVsfn6pm2bpybh3dn97ds6MORyRtJJLsG6pCeiPnTgMWu3tN3L5B7l4EzAB+YGZDz2nM/Ul3L3L3ovz8/ARCisCeDbC7OJgHR4XZNvPy+nKOnarm3kmDog5FJK0kkuzLgPiKQQFQ3si504Dn4ne4e3n4+w7gLc7uz08dxQugc44Ks21s4aoShl/Ui/GDcqMORSStJJLsVwPDzKzQzLoQJPTl9U8ys+FALvB23L5cM+safs4Drqbxvv6O69SxsDB7J3RTEmorm3ZXsqGskukTB2L615NIUjVZoHX3ajObDbxK8OrlPHffbGZzgJi71yb+6cAid4/v4hkB/LuZnSH4i+WJ+Ld4UsamJXD6aDDpmbSZhatKyMnuxJ3jC6IORSTtJPSevbuvAFbU2/dIve3HGrjuD8CoVsTXMRQvgPwRMHBS1JGkrWOnqlm2djdfGH0JvbupMCuSbHqvrSl71kP5GhVm29jydeV8fLqGGSrMirQJJfumxOYHhdnRX4w6krS2cNUuLr+4F+MG9ok6FJG0lD5z4xw/CE/dkPx2D5fCqHtUmG1DG8sq2bT7CHOmjFRhVqSNpE+y79QZBkxIfrsDJ8G1f5v8dqXOwlW76JadxR3jGhyrJyJJkD7JPucCuPupqKOQZjp6sopl68q5bUx/LshRYVakrajPXiK1fH05x0/XMH2iCrMibUnJXiLj7ixcWcKI/hcwVoVZkTalZC+R2VBWyebyI8yYNEiFWZE2pmQvkVm4soRu2VlMGXtJ1KGIpD0le4nE0ZNVLF9fzu1jLlFhVqQdKNlLJF5aV86JKo2YFWkvSvbS7moLsyMvuYDRBb2jDkckIyjZS7tbV3qYd/ccYfpEFWZF2ouSvbS751aV0L2LCrMi7UnJXtrVkZNVvLx+D1PGXkIvFWZF2o2SvbSrl9bu5kSVRsyKtLf0mRtHkm7ljgMcOn46qW0+8/YurhxwAaMLNGJWpD0llOzNbDLwQ4JlCZ9y9yfqHf8+8Jlwsztwobv3CY/dB3w7PPaP7v50MgKXtrW+9DB/+uQ7bdL2d+8e3Sbtikjjmkz2ZpYFzAVuBMqA1Wa2PH4tWXd/KO78rwPjws99gUeBIsCB4vDaQ0m9C0m6Z1fuonuXLBbNuorsrOT19mVnGUPzeyatPRFJTCJP9hOB7e6+A8DMFgFTgMYWDp9OkOABPg+85u4Hw2tfAyYDz7UmaGlb8UVUdbeIpIdEHtkGAKVx22XhvnOY2WCgEPhNc641s1lmFjOzWEVFRSJxSxtapiKqSNpJJNk3NOrFGzl3GrDY3Wuac627P+nuRe5elJ+fn0BI0lbcnWc1ulUk7SSS7MuAgXHbBUB5I+dO4+wumuZcKx3AutLDvLf3qKYdFkkziST71cAwMys0sy4ECX15/ZPMbDiQC7wdt/tV4CYzyzWzXOCmcJ90UAtXBqNbbx+j0a0i6aTJAq27V5vZbIIknQXMc/fNZjYHiLl7beKfDixyd4+79qCZ/QPBXxgAc2qLtdLxVJ6o4uUN5dw5boBGt4qkmYTes3f3FcCKevseqbf9WCPXzgPmtTA+aUfL1u3mZNUZZkwcHHUoIpJkmi5BgE+mHb5ywAWMUmFWJO0o2QsAa0rCwqye6kXSkpK9AMG0wz26ZHG7ph0WSUtK9kLliSpe2VDOlHED6NlVc+OJpCMle+HFNWVhYVYjZkXSlZJ9hnN3nltVyuiC3lw5QIVZkXSlZJ/h1pQcYuu+o3qqF0lzSvYZ7tmVJfTs2pnbNGJWJK0p2WewyuNV/NeGYCrjHirMiqQ1JfsMtnRtGaeqzzBjkrpwRNKdkn2Gqh0xO6agNyMvUWFWJN0p2Weo2K5DbNt/TE/1IhlCyT5DPafCrEhGUbLPQIePn+aVjXu4Y9wldO+iwqxIJlCyz0BL1uzmdLWmMhbJJEr2GSYYMVvC2IF9uOKSC6IOR0Taif4NnySnq89QcvDjqMNo0ta9x9i+/xjfvXt01KGISDtKKNmb2WTghwTLEj7l7k80cM4XgccAB9a7+4xwfw2wMTytxN1vT0LcHc6jyzfx3KrSqMNISK+cznxhTP+owxCRdtRksjezLGAucCNQBqw2s+XuviXunGHAt4Cr3f2QmV0Y18QJdx+b5Lg7lMoTVby4djefG3ERU1JgPvih+T1VmBXJMIn8iZ8IbHf3HQBmtgiYAmyJO+cvgLnufgjA3fcnO9CO7KW1wdqtf/W5YZo5UkQ6pEQKtAOA+P6JsnBfvMuAy8zs92b2TtjtUyvHzGLh/jsa+gFmNis8J1ZRUdGsG4ha7UhUTREsIh1ZIsneGtjn9bY7A8OA64HpwFNm1ic8Nsjdi4AZwA/MbOg5jbk/6e5F7l6Un5+fcPAdgaYIFpFUkEiyLwMGxm0XAOUNnLPM3avc/UNgK0Hyx93Lw993AG8B41oZc4eycGWpRqKKSIeXSLJfDQwzs0Iz6wJMA5bXO+cl4DMAZpZH0K2zw8xyzaxr3P6rObuvP6VVHg/XbtUUwSLSwTWZody92sxmA68SvHo5z903m9kcIObuy8NjN5nZFqAG+Dt3P2Bmnwb+3czOEPzF8kT8WzypTlMEi0iqMPf63e/RKioq8lgsFnUYTXJ3bvr+b+neJYtls6+JOhwRyXBmVhzWRxuk6RJaqFhTBItIClGyb6GFmiJYRFKIkn0LaIpgEUk1SvYtsFRTBItIilGybyZ3Z6GmCBaRFKNk30yrdx5i+/5jGjErIilFyb6ZnltVQq+umiJYRFKLkn0zHPr4NP+1cQ93jh+gwqyIpBQl+2ZYsqaM09VnmK4uHBFJMUr2Capdu3XcoD6M6K/CrIikFiX7BK368CAfVHyswqyIpCQl+wQtXFUSrN06WiNmRST1KNkn4NDHp/nlxr3cNW4A3bpkRR2OiEizKdknYMmaMk7XnGHGJI2YFZHUpPcHm1A7YnbC4FyGX9wr6nBE0kpVVRVlZWWcPHky6lBSRk5ODgUFBWRnZzfrOiX7Jqz88CA7Kj7me/d8KupQRNJOWVkZvXr1YsiQIZg1tNy1xHN3Dhw4QFlZGYWFhc26Vt04TVi4soQLcjrzhdEaMSuSbCdPnqRfv35K9AkyM/r169eifwkl9GRvZpOBHxIsS/iUuz/RwDlfBB4DHFjv7jPC/fcB3w5P+0d3f7rZUSbZj9/Yxr+89n7C58/89BByslWYFWkLSvTN09L/Xk0mezPLAuYCNwJlwGozWx6/lqyZDQO+BVzt7ofM7MJwf1/gUaCI4C+B4vDaQy2KNglOVdcw7/cfMqagN9cNv7DJ8zt3Mo2YFZGUl8iT/URgu7vvADCzRcAUIH7h8L8A5tYmcXffH+7/PPCaux8Mr30NmAw8l5zwm+9Xm/Zy6HgVP5w2nGsvy48qDBGRdpVIn/0AoDRuuyzcF+8y4DIz+72ZvRN2+yR6bbtauLKEQX27c82n8qIMQ0RSUM+ePRs9tnPnThYuXFi3vWDBAmbPnt0eYSUkkSf7hjqIvIF2hgHXAwXA78zsygSvxcxmAbMABg1quy6T7fuPsfLDg3xj8nA6dVI/oUhH8p2XN7Ol/EhS27zikgt49LaRSW2zMbXJfsaMGc26rqamhqystq8JJvJkXwYMjNsuAMobOGeZu1e5+4fAVoLkn8i1uPuT7l7k7kX5+W3XtbJoVQmdOxn3TBjY9Mkikva++c1v8m//9m9124899hjf+c53uOGGGxg/fjyjRo1i2bJlCbX18MMP87vf/Y6xY8fy/e9/H4Dy8nImT57MsGHD+MY3vlF3bs+ePXnkkUeYNGkSb7/9dnJvqjHuft5fBE/tO4BCoAuwHhhZ75zJwNPh5zyCrpt+QF/gQyA3/PUh0Pd8P2/ChAneFk6crvYx33nVv/bzWJu0LyLNt2XLlkh//po1a/zaa6+t2x4xYoTv2rXLKysr3d29oqLChw4d6mfOnHF39x49ejTa1ptvvum33npr3fb8+fO9sLDQDx8+7CdOnPBBgwZ5SUmJu7sD/vzzz7c47ob+uwExP09ubbIbx92rzWw28CrBq5fz3H2zmc0JG18eHrvJzLYANcDfufsBADP7B2B12NwcD4u17e1Xm/Zy+HiVFgkXkTrjxo1j//79lJeXU1FRQW5uLv379+ehhx7it7/9LZ06dWL37t3s27ePiy++uNnt33DDDfTu3RuAK664gl27djFw4ECysrK4++67k30755XQe/buvgJYUW/fI3GfHfjr8Ff9a+cB81oXZustXFXC4H7d+fTQflGHIiIdyNSpU1m8eDF79+5l2rRpPPvss1RUVFBcXEx2djZDhgxp8XQOXbt2rfuclZVFdXU1EEx50B799PEyYgTt9v1HWfXhQab90SAVZkXkLNOmTWPRokUsXryYqVOnUllZyYUXXkh2djZvvvkmu3btSqidXr16cfTo0TaOtuUyYm6chStLyc4y7ikqiDoUEelgRo4cydGjRxkwYAD9+/fn3nvv5bbbbqOoqIixY8dy+eWXJ9TO6NGj6dy5M2PGjGHmzJnk5ua2ceTNk/bJ/mRVDUvWlHHTyIvJ69m16QtEJONs3Lix7nNeXl6jb8gcO3as0Tays7N54403zto3c+bMus+vvPJKQu20lbTvxvnlpj1UnqjScoIiktHS/sl+4coShvTrzh9fqsKsiLTexo0b+fKXv3zWvq5du7Jy5cqIIkpMWif79/cdZfXOQ3zr5stVmBWRpBg1ahTr1q2LOoxmS+tunOdWlZCdZUydoMKsiGS2tE32J6tqWFJcxudHXkw/FWZFJMOlbbL/rw17OHKymhmTVJgVEUnbZL9wVQmFeT1UmBWRRh0+fPisidASdcstt3D48OFmX7dgwQLKyz+ZC3LIkCF89NFHzW6nJdIy2W/de5TiXYeYPnGgljwTkUY1luxramrOe92KFSvo06dPs39e/WSfiNopFlorLd/GeW5VCV2yOjFVUxmLpI5fPgx7NzZ9XnNcPApuPmfJ7DoPP/wwH3zwAWPHjiU7O5uePXvSv39/1q1bx5YtW7jjjjsoLS3l5MmTPPjgg8yaNQsInshjsRjHjh3j5ptv5r0vJc8AAAecSURBVJprruEPf/gDAwYMYNmyZXTr1u2cn7V48WJisRj33nsv3bp1qxu49eMf/5iXX36ZqqoqfvGLX3D55Zfz2GOPUV5ezs6dO8nLyztrUZSWSrsn+xOngxGzk6+8mL49ukQdjoh0YE888QRDhw5l3bp1/PM//zOrVq3i8ccfZ8uWYNXVefPmUVxcTCwW40c/+hEHDhw4p41t27bxwAMPsHnzZvr06cOSJUsa/FlTp06lqKiIZ599lnXr1tX9hZCXl8eaNWv42te+xve+972684uLi1m2bFlSEj2k4ZP9KxvKOarCrEjqOc8TeHuZOHEihYWFdds/+tGPePHFFwEoLS1l27Zt9Ot3dh2wsLCQsWPHAjBhwgR27tzZrJ9511131V27dOnSuv233357g/9CaKm0S/bPrSrh0vweTCrsG3UoIpJievToUff5rbfe4vXXX+ftt9+me/fuXH/99Q1OdVx/GuMTJ04062fWXh8/BXL9WJIhrbpx3tt7hDUlh5kxcZAKsyLSpPNNS1xZWUlubi7du3fnvffe45133mnTn9fW0urJfuHKoDB793iNmBWRpvXr14+rr76aK6+8km7dunHRRRfVHZs8eTI//elPGT16NMOHD+eqq65q9c+bOXMmX/3qV88q0LYXCxaZ6jiKioo8Fos1+7oTp2uY+PjrfHbEhfxw2rg2iExEku3dd99lxIgRUYeRchr672Zmxe5e1Ng1CXXjmNlkM9tqZtvN7OEGjs80swozWxf++vO4YzVx+5c3436a5cjJKq4bns+XrtIasyIi9TXZjWNmWcBc4EagDFhtZsvdfUu9U59399kNNHHC3ce2PtTzu+iCHP7fjPFt/WNERJr0wAMP8Pvf//6sfQ8++CD3339/RBEl1mc/Edju7jsAzGwRMAWon+xFRASYO3du1CGcI5FunAFAadx2WbivvrvNbIOZLTaz+KGrOWYWM7N3zOyOhn6Amc0Kz4lVVFQkHr2IpLyOVjfs6Fr63yuRZN/QO4z1f9rLwBB3Hw28Djwdd2xQWDSYAfzAzIae05j7k+5e5O5F+fn5CYYuIqkuJyeHAwcOKOEnyN05cOAAOTk5zb42kW6cMiD+Sb0AOGsmH3ePH0P8H8D/jTtWHv6+w8zeAsYBHzQ7UhFJOwUFBZSVlaF/0ScuJyeHgoLmv16eSLJfDQwzs0JgNzCN4Cm9jpn1d/c94ebtwLvh/lzguLufMrM84Grgu82OUkTSUnZ29lnTE0jbaTLZu3u1mc0GXgWygHnuvtnM5gAxd18O/C8zux2oBg4CM8PLRwD/bmZnCLqMnmjgLR4REWljaTOoSkQkkyVlUJWIiKS2Dvdkb2YVwK5WNJEHtM86X+0j3e4H0u+e0u1+IP3uKd3uB869p8Hu3ujrjB0u2beWmcXO90+ZVJNu9wPpd0/pdj+QfveUbvcDzb8ndeOIiGQAJXsRkQyQjsn+yagDSLJ0ux9Iv3tKt/uB9LundLsfaOY9pV2fvYiInCsdn+xFRKSetEn2TS2wkorMbKeZbQwXfkm5kWZmNs/M9pvZprh9fc3sNTPbFv6eG2WMzdXIPT1mZrvjFum5JcoYm8PMBprZm2b2rpltNrMHw/0p+T2d535S+TvKMbNVZrY+vKfvhPsLzWxl+B09b2ZdzttOOnTjhAusvE/cAivA9FSfmsHMdgJF7p6S7web2bXAMeBn7n5luO+7wEF3fyL8SznX3b8ZZZzN0cg9PQYcc/fvRRlbS5hZf6C/u68xs15AMXAHwZQnKfc9ned+vkjqfkcG9HD3Y2aWDfwP8CDw18BSd19kZj8F1rv7TxprJ12e7OsWWHH300DtAisSIXf/LcFcSfGm8MkU2E8T/EFMGY3cU8py9z3uvib8fJRgEsMBpOj3dJ77SVkeOBZuZoe/HPgssDjc3+R3lC7JPtEFVlKNA782s2IzmxV1MElyUe0MqeHvF0YcT7LMDhfvmZcqXR71mdkQginIV5IG31O9+4EU/o7MLMvM1gH7gdcIpok/7O7V4SlN5rx0SfaJLLCSiq529/HAzcADYReCdDw/AYYCY4E9wL9EG07zmVlPYAnwV+5+JOp4WquB+0np78jda8K1vAsIejJGNHTa+dpIl2Tf5AIrqShu4Zf9wIsEX3Kq2xf2q9b2r+6POJ5Wc/d94R/GMwSL96TU9xT2Ay8BnnX3peHulP2eGrqfVP+Oarn7YeAt4Cqgj5nVTlPfZM5Ll2Rft8BKWJGeBiyPOKZWMbMeYYEJM+sB3ARsOv9VKWE5cF/4+T5gWYSxJEVtUgzdSQp9T2Hx7z+Bd939X+MOpeT31Nj9pPh3lG9mfcLP3YDPEdQi3gSmhqc1+R2lxds4AOGrVD/gkwVWHo84pFYxs0sJnuYhWGRmYardk5k9B1xPMDvfPuBR4CXgBWAQUALc4+4pU/Bs5J6uJ+gecGAn8JdxK7d1aGZ2DfA7YCNwJtz9vwn6uVPuezrP/Uwndb+j0QQF2CyCB/QX3H1OmCMWAX2BtcCX3P1Uo+2kS7IXEZHGpUs3joiInIeSvYhIBlCyFxHJAEr2IiIZQMleRCQDKNmLiGQAJXsRkQygZC8ikgH+Pz7Rpbd+YcuVAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plot_hist(thr_score_hist)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Test thr"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from imports import *\n",
    "from utils import load_data\n",
    "from image_train.data import create_dl, ImageDS\n",
    "from image_train.model import EMBRes\n",
    "from arcface import ArcMarginProduct, compute_centers\n",
    "from image_train.train import *\n",
    "import matplotlib.pyplot as plt\n",
    "np.random.seed(1337)\n",
    "device = torch.device('cuda')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "df, train_df, val_df, train_labels, val_labels = load_data(train_perc=0.3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# creating dataloaders\n",
    "small_images_dir_train = 'data/small_train_images_300/'\n",
    "small_images_dir_val = 'data/small_train_images_224/'\n",
    "bs = 32\n",
    "\n",
    "tr_dl = create_dl(train_df, small_images_dir_train, batch_size=bs)\n",
    "tr_test_dl = create_dl(train_df, small_images_dir_val, shuffle=False, batch_size=bs)\n",
    "val_dl = create_dl(val_df, small_images_dir_val, shuffle=False, batch_size=bs)\n",
    "#full_dl = create_dl(df, small_images_dir, shuffle=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "***Embeddings normalization is not done in the model but in the arcface metric***"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "vision_model = 'resnet50'\n",
    "model = timm.create_model(vision_model, pretrained=True, num_classes=0).to('cuda')\n",
    "train_tfms, val_tfms = get_tfms(crop=224)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "67a696dd22cd4d059a7f0c0e7e29d721",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=327.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "centers = compute_centers(tr_test_dl, model, val_tfms, train_df)\n",
    "torch.save(centers, 'data/clip/centers_im_0.3_res50_noc.pth')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "centers = torch.load('data/clip/centers_im_0.3_res50_noc.pth')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Using center as wieghts\n"
     ]
    }
   ],
   "source": [
    "metric_fc = ArcMarginProduct(2048, train_df['label_group'].nunique(), \n",
    "                             s=30, m=0.5, easy_margin=False, centers=centers).to(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "#freeze(metric_fc)\n",
    "#freeze_resnet_to()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "n_epochs, lf, params, optimizer, sched = get_hparams(tr_dl, model, metric_fc, lr=5e-4, n_epochs=10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "mnp = list(model.named_parameters())\n",
    "\n",
    "param_groups = [{'params' : [p for n,p in mnp if 'embeddings' in n]}]\n",
    "\n",
    "params_names = []\n",
    "n_blocks = 5\n",
    "for i in range(n_blocks):\n",
    "    ith_block = [p for n, p in mnp if f'layer{i}.' in n]\n",
    "    ith_block_name = [n for n, p in mnp if f'layer{i}.' in n]\n",
    "    params_names += ith_block_name\n",
    "    param_groups.append({'params' : ith_block})\n",
    "\n",
    "param_groups.append({'params' : metric_fc.parameters()})\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "conv1.weight\n",
      "bn1.weight\n",
      "bn1.bias\n"
     ]
    }
   ],
   "source": [
    "for n, p in model.named_parameters() :\n",
    "    if n not in params_names :\n",
    "        print(n)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "loss_hist = []\n",
    "best_thr_score={'val': {'thr': None, 'f1': None}, 'train': {'thr': None, 'f1': None}}\n",
    "thr_score_hist=[]\n",
    "ep_start = 0\n",
    "save_path = 'data/test_2ap'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=10.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=327.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Thresholds', max=10.0, style=ProgressStyle(description_wi…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Checkpoint : saved model to data/test_2ap_ep_0.pth\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=745.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Thresholds', max=10.0, style=ProgressStyle(description_wi…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Saved best model ep 0 with f score : 0.6513117000297622\n",
      "Ep 0: Train loss 10.4605 | Val f score 0.6513 with thresh 0.56, train f score 0.6149 with thresh 0.82\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=327.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Thresholds', max=5.0, style=ProgressStyle(description_wid…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=745.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Thresholds', max=5.0, style=ProgressStyle(description_wid…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Saved best model ep 1 with f score : 0.6588515282754319\n",
      "Ep 1: Train loss 9.6205 | Val f score 0.6589 with thresh 0.61, train f score 0.6725 with thresh 0.82\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=327.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Thresholds', max=5.0, style=ProgressStyle(description_wid…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Checkpoint : saved model to data/test_2ap_ep_2.pth\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=745.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Thresholds', max=5.0, style=ProgressStyle(description_wid…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Saved best model ep 2 with f score : 0.661767417787532\n",
      "Ep 2: Train loss 9.3039 | Val f score 0.6618 with thresh 0.66, train f score 0.7036 with thresh 0.87\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=327.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Thresholds', max=5.0, style=ProgressStyle(description_wid…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=745.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Thresholds', max=5.0, style=ProgressStyle(description_wid…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Saved best model ep 3 with f score : 0.6682932048878832\n",
      "Ep 3: Train loss 7.7428 | Val f score 0.6683 with thresh 0.66, train f score 0.7523 with thresh 0.87\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=327.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Thresholds', max=5.0, style=ProgressStyle(description_wid…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Checkpoint : saved model to data/test_2ap_ep_4.pth\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=745.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Thresholds', max=5.0, style=ProgressStyle(description_wid…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Saved best model ep 4 with f score : 0.6715341393358971\n",
      "Ep 4: Train loss 6.0926 | Val f score 0.6715 with thresh 0.71, train f score 0.7929 with thresh 0.92\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=327.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Thresholds', max=5.0, style=ProgressStyle(description_wid…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=745.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Thresholds', max=5.0, style=ProgressStyle(description_wid…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Saved best model ep 5 with f score : 0.6783450758619092\n",
      "Ep 5: Train loss 4.3277 | Val f score 0.6783 with thresh 0.71, train f score 0.8425 with thresh 0.92\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=327.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Thresholds', max=5.0, style=ProgressStyle(description_wid…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Checkpoint : saved model to data/test_2ap_ep_6.pth\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=745.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Thresholds', max=5.0, style=ProgressStyle(description_wid…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Saved best model ep 6 with f score : 0.6813136996705283\n",
      "Ep 6: Train loss 2.8975 | Val f score 0.6813 with thresh 0.76, train f score 0.8846 with thresh 0.92\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=327.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Thresholds', max=5.0, style=ProgressStyle(description_wid…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=745.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Thresholds', max=5.0, style=ProgressStyle(description_wid…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Saved best model ep 7 with f score : 0.6861139536585138\n",
      "Ep 7: Train loss 1.8875 | Val f score 0.6861 with thresh 0.76, train f score 0.9154 with thresh 0.92\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=327.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Thresholds', max=5.0, style=ProgressStyle(description_wid…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Checkpoint : saved model to data/test_2ap_ep_8.pth\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=745.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Thresholds', max=5.0, style=ProgressStyle(description_wid…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Saved best model ep 8 with f score : 0.6868997090330737\n",
      "Ep 8: Train loss 1.2629 | Val f score 0.6869 with thresh 0.76, train f score 0.9383 with thresh 0.92\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=327.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Thresholds', max=5.0, style=ProgressStyle(description_wid…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=745.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Thresholds', max=5.0, style=ProgressStyle(description_wid…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Saved best model ep 9 with f score : 0.6870138482307787\n",
      "Ep 9: Train loss 1.0368 | Val f score 0.6870 with thresh 0.76, train f score 0.9485 with thresh 0.92\n",
      "\r"
     ]
    }
   ],
   "source": [
    "best_thr_score, thr_score_hist, losses = train(model, optimizer, lf, sched, metric_fc, tr_dl, val_dl,\n",
    "                                               n_epochs, train_df, val_df, train_tfms, val_tfms, \n",
    "                                               save_path=save_path, \n",
    "                                               prev_best_info=best_thr_score, info_history=thr_score_hist,\n",
    "                                               ep_start=ep_start)\n",
    "loss_hist.append(losses)\n",
    "ep_start += n_epochs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for imgs, txts in dl :\n",
    "    feat_img = model.encode_image(imgs)\n",
    "    feat_txt = model.encode_txt(txts)\n",
    "    \n",
    "    loss_img_arc = metric_fc(feat_img)\n",
    "    loss_txt_arc = metric_fc(feat_txt)\n",
    "    loss_arc = loss_img_arc + loss_txt_arc\n",
    "    \n",
    "    # cosine similarity as logits\n",
    "    logit_scale = self.logit_scale.exp()\n",
    "    logits_per_image = logit_scale * image_features @ text_features.t()\n",
    "    logits_per_text = logit_scale * text_features @ image_features.t()\n",
    "    \n",
    "    ground_truth = torch.arange(BATCH_SIZE).half().to(device)\n",
    "    loss_clip = (loss_img(logits_per_image,ground_truth) + loss_txt(logits_per_text,ground_truth))/2\n",
    "    \n",
    "    loss = loss_arc + l * loss_clip\n",
    "    loss.backward()\n",
    "    ..."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXsAAAD4CAYAAAANbUbJAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAgAElEQVR4nO3deXxU9bnH8c+Tfd8DxIQlKoKACBpBiyJqBdRWUbyK+9Lq7S1axVuvWu2Fi63a3ra2Vqult7RqVaSutFp3EVFUAqLs+xZCJGRPyDp57h9nEoaQkAlMOJnJ83695jVzljnzHIZ85ze/Oed3RFUxxhgT2sLcLsAYY0z3s7A3xphewMLeGGN6AQt7Y4zpBSzsjTGmF4hwu4C2MjIydNCgQW6XYYwxQWXZsmV7VTWzo+U9LuwHDRpEfn6+22UYY0xQEZHth1pu3TjGGNMLWNgbY0wvYGFvjDG9QKd99iIyF/gOsEdVR7SzXIDfARcC+4AbVXW5d9kNwAPeVX+mqk8fTpGNjY0UFBRQV1d3OE83XRATE0NOTg6RkZFul2KMCSB/fqD9K/A48EwHyy8ABntvY4EngbEikgbMBPIABZaJyAJVLetqkQUFBSQmJjJo0CCczxbTHVSVkpISCgoKyM3NdbscY0wAddqNo6qLgNJDrHIJ8Iw6PgNSRCQLmAS8q6ql3oB/F5h8OEXW1dWRnp5uQd/NRIT09HT7BmVMCApEn302sNNnusA7r6P5BxGRW0UkX0Tyi4uL230RC/qjw/6djQlNgTjOvr100EPMP3im6hxgDkBeXp6NuWxMKGhuhvoKqC2DukpQjzO/9S/c50/dd6h1bfa5eQ6cbm4+xHL1rtPmOdpMayRJGIj3Hmnz2LusZTstzztgW9rB407qbHkd39dv75bQF0Zc1i1vRyDCvgDo7zOdAxR6509oM39hAF7PGOOWugoo3wkVO5376m+cMK8tg7py72PvfV0FHbTvTEey83p02C8AbhOReTg/0Fao6m4ReRt4SERSvetNBO4LwOsFhYSEBKqrq90uwxj/qEJDNewrgZq9+8O87X19xYHPk3CITYHYVIhJgbgMSD9+/3RsqrM8JhnCfOPG+8X/gG5D2X8n4Qe2eMPC22kJi7PeQcvk4Oc7O7m/Nd7aYlef+d55rdto0wKnk9b5QXWE799Oy79x228cbW+ttQaeP4devoDTQs8QkQKcI2windr1KeBNnMMuN+EcenmTd1mpiDwILPVuaraqHuqHXnOYmpqaiIjocSNfGDe1hHf1HudWswdqimFfqRPm+0pgn/e+psS599QfvJ3oZEjpD8n9YeC3nPuU/pA8wLmPz2wT2KZDLR9ChLvy8p0mhKpe1clyBaZ3sGwuMPfwSmvf//xjNWsKKwO5SYYdk8TM7w4/5Dr33HMPAwcO5Ic//CEAs2bNQkRYtGgRZWVlNDY28rOf/YxLLrmk09fbvXs3V155JZWVlTQ1NfHkk09y1lln8dZbb/GTn/wEj8dDRkYG77//PqWlpdx8881s2bKFuLg45syZw8iRI5k1axaFhYVs27aNjIwMnn32We69914WLlxIfX0906dP59///d8D8u9jeqCaEtizBvZugKoipzulpnh/sFcXQ1Nt+8+NToa4NIjPgKRs6HcyxKdDXLrTMo9Lh+QcJ8xjko/ufpluY81BP02bNo0777yzNeznz5/PW2+9xYwZM0hKSmLv3r2cfvrpXHzxxZ0e0fL8888zadIk7r//fjweD/v27aO4uJhbbrmFRYsWkZubS2mp8yVo5syZjB49mtdee40PPviA66+/nhUrVgCwbNkyFi9eTGxsLHPmzCE5OZmlS5dSX1/PuHHjmDhxoh0vH+zqKqB4vRPse9Z679c5gd5CwpyAju8DCZmQfpzT4k7os39efB9nXlw6RES5tz/GNUEX9p21wLvL6NGj2bNnD4WFhRQXF5OamkpWVhYzZsxg0aJFhIWFsWvXLr755hv69et3yG2ddtpp3HzzzTQ2NjJlyhRGjRrFwoULGT9+fGs4p6WlAbB48WJefvllAM4991xKSkqoqHD6TS+++GJiY2MBeOedd/j666956aWXAKioqGDjxo0W9sGksQ4KlsLWRVD4pRPulQX7l0fGQ5+hMHgi9DnRuWUOhcR+Tn+xMYcQdGHvpssvv5yXXnqJoqIipk2bxnPPPUdxcTHLli0jMjKSQYMG+XVC0vjx41m0aBFvvPEG1113HXfffTcpKSntfiNQPfhohpb14uPjD1jv97//PZMmTTqCPTRHVVMDFC6HrR/D1o9g5xdOv7mEQZ9hTh95nxOdx32GOv3kYTaclTk8FvZdMG3aNG655Rb27t3LRx99xPz58+nTpw+RkZF8+OGHbN9+yOGkW23fvp3s7GxuueUWampqWL58Offffz/Tp09n69atrd04aWlpjB8/nueee46f/vSnLFy4kIyMDJKSkg7a5qRJk3jyySc599xziYyMZMOGDWRnZx/wgWBc5mmCoq+clvvWj2HHZ9BY4yzrexKc9n3IPcsJeesrNwFmYd8Fw4cPp6qqiuzsbLKysrjmmmv47ne/S15eHqNGjWLo0KF+bWfhwoX87//+L5GRkSQkJPDMM8+QmZnJnDlzuOyyy2hubqZPnz68++67zJo1i5tuuomRI0cSFxfH00+3P5bc97//fbZt28Ypp5yCqpKZmclrr70WyN03h6PZA9s+hpV/h7X/8B57jtP9MupqyB0Pg850fjA1phtJe90EbsrLy9O2V6pau3YtJ554oksV9T72732EVGHXcifgV7/iHCkTlQgnfgeO/zYMOgsS+7pdpQkxIrJMVfM6Wm4te2MCpXg9rHzJCfmyrRAeBSdMghGXO/eRsW5XaHoxC/tutHLlSq677roD5kVHR/P555+7VJEJuPKdTut95d+haKXz42rueBj/Yxj6HefsUWN6AAv7bnTSSSe1HhNvQkjlbljzGqx6BQq+cOZl58HkX8DwS62LxvRIFvbG+KO62An41a/C9k8BdY6gOe+/nYBPO9btCo05JAt7YzqyrxTWLnBa8Ns+dgaqyhgCE+5zAj7zBLcrNMZvFvbGtFW+E978MWx6D5qbnFb7mXc5Q8/2GWYDf5mgZGFvjK+tH8PfbwBPI5wxHYZfBlknW8CboGfnXvupvLycP/zhD11+3oUXXkh5eXk3VGQCShU+nwPPXAKxaXDLB3D+bDhmlAW9CQkW9n7qKOw9Hs8hn/fmm2+SkuL+4Xed1dmrNdbB67fBv+52Bhm75X3IGOx2VcYEVPB14/zrXud45kDqdxJc8MghV7n33nvZvHkzo0aNah3mICsrixUrVrBmzRqmTJnCzp07qaur44477uDWW28FYNCgQeTn51NdXc0FF1zAmWeeyaeffkp2djavv/5666iVbT322GM89dRTREREMGzYMObNm0d1dTW33347+fn5iAgzZ85k6tSpvPDCCzz00EOoKhdddBG/+MUvAOdqWXfddRdvv/02v/71r4mNjeWuu+6iurqajIwM/vrXv5KVlRXYf8tgU1kIL14Lu5bB2ffA2ffaYGMmJAVf2LvkkUceYdWqVaxYsYKFCxdy0UUXsWrVqtYhhOfOnUtaWhq1tbWcdtppTJ06lfT09AO2sXHjRl544QX+9Kc/ccUVV/Dyyy9z7bXXdvh6W7duJTo6urUb6MEHHyQ5OZmVK50Pu7KyMgoLC7nnnntYtmwZqampTJw4kddee40pU6ZQU1PDiBEjmD17No2NjZx99tm8/vrrZGZm8uKLL3L//fczd25Ary0TXHZ8Bi9eB4374Mq/wYnfdbsiY7pN8IV9Jy3wo2XMmDEHjBX/2GOP8eqrrwKwc+dONm7ceFDY5+bmMmrUKABOPfVUtm3b1uH2R44cyTXXXMOUKVOYMmUKAO+99x7z5s1rXSc1NZVFixYxYcIEMjMzAbjmmmtYtGgRU6ZMITw8nKlTpwKwfv16Vq1axfnnnw843Tq9ulWf/xd4827nakw3LHCGEjYmhAVf2PcQvkMHL1y4kPfee48lS5YQFxfHhAkT2h3XPjo6uvVxeHg4tbUdXDYOeOONN1i0aBELFizgwQcfZPXq1ajqQWPeH2ogu5iYGMLDw1vXGz58OEuWLPF7H0NSUwP8679g2V+cQcmm/p9zUWxjQpxfnZMiMllE1ovIJhG5t53lA0XkfRH5WkQWikiOzzKPiKzw3hYEsvijKTExkaqqqnaXVVRUkJqaSlxcHOvWreOzzz47otdqbm5m586dnHPOOfzyl7+kvLyc6upqJk6cyOOPP966XllZGWPHjuWjjz5i7969eDweXnjhBc4+++yDtjlkyBCKi4tbw76xsZHVq1cfUZ1Bp7IQnv6uE/RnzoCr51vQm16j05a9iIQDTwDnAwXAUhFZoKprfFb7FfCMqj4tIucCDwMtI4DVquqoANd91KWnpzNu3DhGjBhBbGwsffvuH/9k8uTJPPXUU4wcOZIhQ4Zw+umnH9FreTwerr32WioqKlBVZsyYQUpKCg888ADTp09nxIgRhIeHM3PmTC677DIefvhhzjnnHFSVCy+8sN2LnkdFRfHSSy/xox/9iIqKCpqamrjzzjsZPtydyzweVUUr4bMnncHKwiLg8r84J0gZ04t0Op69iJwBzFLVSd7p+wBU9WGfdVYDk1S1QJx+hgpVTfIuq1bVBH8LsvHs3RcS/97NzbDhLfjsD85QB5FxMOoaOOOHNo6NCUmBGM8+G9jpM10AjG2zzlfAVOB3wKVAooikq2oJECMi+UAT8IiqHnT5JBG5FbgVYMCAAX6UZEwH6qthxXPw+VNQugWScpyTo0653rpsTK/mT9i3d/pg268DPwYeF5EbgUXALpxwBxigqoUicizwgYisVNXNB2xMdQ4wB5yWfRfqD3rTp0/nk08+OWDeHXfcwU033eRSRUGqfAd8MQeWPQP1FZBzGpz7U+dwyvBIt6szxnX+hH0B0N9nOgco9F1BVQuBywBEJAGYqqoVPstQ1S0ishAYDRwQ9v5o70iUUPDEE0+4XcIBetplKjtVVwH/nOEMPYzAsEvg9B9C/9PcrsyYHsWfsF8KDBaRXJwW+zTgat8VRCQDKFXVZuA+YK53fiqwT1XrveuMA37Z1SJjYmIoKSkhPT09JAO/p1BVSkpKiImJcbsU/9RVwLOXwu6v4YzbYMytznHzxpiDdBr2qtokIrcBbwPhwFxVXS0is4F8VV0ATAAeFhHF6caZ7n36icAfRaQZ5zDPR9ocxeOXnJwcCgoKKC4u7upTTRfFxMSQk5PT+Ypu8w36K56GoRe5XZExPVqnR+Mcbe0djWPMAeoq4NnLYPdXFvTGeAXiaBxjeo7WoF8BVzxjQW+Mn2x4PxM86irhb1OdoP83a9Eb0xUW9iY41FXC3y6Dwi+doD/xO25XZExQsbA3Pd8BQf9XC3pjDoOFvenZWrpuCr90xrSxMeeNOSwW9qbnag365U7QD7vY7YqMCVoW9qZnqquE5y53Lhd4+VwLemOOkB16aXqW5mZY/Qq8PxsqCuDf/uIMgWCMOSIW9qbn2PoxvPtTp3++70lw/euQe5bbVRkTEizsjfv2rIX3ZjnjzyflwJSnYOQVEBbudmXGhAwLe+Oeyt2w8CH48m8QlQDfngVjfwCRsW5XZkzIsbA3R199FXzyGCx5HDyNTsCf9WOIT3e7MmNCloW9OXpUnYt9f/gQ1BTD8MvgvJ/aZQKNOQos7M3R4WmEf9zhXDJw4Di46kXIOdXtqozpNSzsTfdrqIG/3wgb34EJ98HZ94BdhMaYo8rC3nSvmhJ4/grnLNjvPAp5N7tdkTG9koW96T5l253hDsp3wBXP2gBmxrjIwt50j6JVTtA31TonRw08w+2KjOnVbGwcE3hbP4a/XAASBje9ZUFvTA/gV9iLyGQRWS8im0Tk3naWDxSR90XkaxFZKCI5PstuEJGN3tsNgSze9ECrX3PGnk/Mgu+9A32HuV2RMQY/wl5EwoEngAuAYcBVItL2L/hXwDOqOhKYDTzsfW4aMBMYC4wBZopIauDKNz3KF39yjrrJGgU3vwUp/d2uyBjj5U/LfgywSVW3qGoDMA9oOwzhMOB97+MPfZZPAt5V1VJVLQPeBSYfedmmR1GF9x+EN38MJ0x2+ujj0tyuyhjjw5+wzwZ2+kwXeOf5+gqY6n18KZAoIul+PhcRuVVE8kUkv7i42N/aTU/xwc/g41/B6Ovgyr9BVJzbFRlj2vAn7Ns7+0XbTP8YOFtEvgTOBnYBTX4+F1Wdo6p5qpqXmZnpR0mmx9izFhY/CidfBRf/HsLtAC9jeiJ//jILAN/O1xyg0HcFVS0ELgMQkQRgqqpWiEgBMKHNcxceQb2mJ1GFN++G6ESY+HM7K9aYHsyflv1SYLCI5IpIFDANWOC7gohkiEjLtu4D5nofvw1MFJFU7w+zE73zTChY/Sps+9gZzMxGrDSmR+s07FW1CbgNJ6TXAvNVdbWIzBaRlguDTgDWi8gGoC/wc+9zS4EHcT4wlgKzvfNMsKuvhncegH4nwak3uV2NMaYTonpQF7qr8vLyND8/3+0yTGfem+X01d/8DgwY63Y1xvR6IrJMVfM6Wm5n0Jqu27sJPn0cTr7agt6YIGFhb7pGFf71X86lA8//H7erMcb4ycLedM26N2Dz+3DOTyChj9vVGGP8ZGFv/NdYC2/dB32GwWm3uF2NMaYL7AwY47/Fj0LFDrjxDTt5ypggYy1745/SrbD4tzDichh0ptvVGGO6yMLe+Oftn0B4JEz8mduVGGMOg4W96dyGd2D9m3D2f0FSltvVGGMOg4W9ObTGOudQy4wTYOx/uF2NMeYw2a9s5tCWPA5lW+G6VyEiyu1qjDGHyVr2pmPlO2HRr+DEi+G4c92uxhhzBCzsTcfeud+5n/SQu3UYY46Yhb05mCp89EtY8zqM/0+7lqwxIcD67M2BVOH92bD4N85AZ2fe5XZFxpgAsLA3+6k6x9N/9gdnjPqLfgNh9uXPmFBgYW8czc3wxl2w7C/OIZaTH7bLDBoTQizsDTR7YMHtsOI5OHMGnDfTgt6YEGNh39t5GuGVW2H1K3DO/TD+bgt6Y0KQhX1v1lQPL90M6/4J58+GcXe4XZExppv49eubiEwWkfUisklE7m1n+QAR+VBEvhSRr0XkQu/8QSJSKyIrvLenAr0D5jA11sK8a5ygv+B/LeiNCXGdtuxFJBx4AjgfKACWisgCVV3js9oDwHxVfVJEhgFvAoO8yzar6qjAlm2OSEMNvDANtn4M330MTr3B7YqMMd3Mn5b9GGCTqm5R1QZgHnBJm3UUSPI+TgYKA1eiCai6Snj2Mti2GC79owW9Mb2EP2GfDez0mS7wzvM1C7hWRApwWvW3+yzL9XbvfCQiZ7X3AiJyq4jki0h+cXGx/9Wbrmmogecuh135cPlcOPlKtysyxhwl/oR9e4dmaJvpq4C/qmoOcCHwrIiEAbuBAao6GrgLeF5Ekto8F1Wdo6p5qpqXmZnZtT0w/mmqhxevhYKlMPXPMPxStysyxhxF/oR9AeA7OEoOB3fTfA+YD6CqS4AYIENV61W1xDt/GbAZOOFIizZd5GmCl78Hmz+Ai38Pw6e4XZEx5ijzJ+yXAoNFJFdEooBpwII26+wAzgMQkRNxwr5YRDK9P/AiIscCg4EtgSre+KG5Gf7xI1j7D5j8CIy+1u2KjDEu6PRoHFVtEpHbgLeBcGCuqq4WkdlAvqouAP4T+JOIzMDp4rlRVVVExgOzRaQJ8AA/UNXSbtsbcyBVePs+58zYCffB6XalKWN6K1Ft2/3urry8PM3Pz3e7jNDw4UPw0S/g9Okw6ed2ZqwxIUxElqlqXkfLbUjDUPXp407Qj77Wgt4YY2EfkpY/41xlatglzklTFvTG9HoW9qFm9avwjzvg+G/DZf8HYeFuV2SM6QEs7EPJxnfh5Vug/1i44lmIiHK7ImNMD2FhHyq2fwovXgd9ToSrX4SoOLcrMsb0IBb2oWDnUnj+SufC4Ne9CjHJbldkjOlhLOyD3Y7P4NlLIS4drnsN4jPcrsgY0wNZ2AezbZ84I1gm9IGb3oTktuPTGWOMw8I+WG392BnBMjnbCfqkY9yuyBjTg1nYB6MtC+G5f4OUAXDjG5DYz+2KjDE9nIV9sNn0vvNjbNqxcMM/nS4cY4zphIV9MNnwDrxwFaQPhhv+AQk29r8xxj8W9sFi/b/gxWugz1C4YQHEp7tdkTEmiFjYB4O1/3ROmOo7HK5/HeLS3K7IGBNkLOx7ujWvw99vgKyTnePoY1PdrsgYE4Qs7Huy1a/C32+C7FOdM2NjU9yuyBgTpDq9UpVxyfp/wcvfh5zT4NqXIDrR7YqMMUHMWvY90eYPYf710O8kuObvFvTGmCNmYd/TbF8C8652Dq+89hWISXK7ImNMCPAr7EVksoisF5FNInJvO8sHiMiHIvKliHwtIhf6LLvP+7z1IjIpkMWHnF3LnDNjk7Lh+tfsqBtjTMB02mcvIuHAE8D5QAGwVEQWqOoan9UeAOar6pMiMgx4ExjkfTwNGA4cA7wnIieoqifQOxL0ilY5g5rFpTmHV9qZscaYAPKnZT8G2KSqW1S1AZgHXNJmHQVa+huSgULv40uAeapar6pbgU3e7RlfxRvg2SkQGeecMGWjVxpjAsyfsM8GdvpMF3jn+ZoFXCsiBTit+tu78FxE5FYRyReR/OLiYj9LDxFl2+CZS0DVadGnDnK7ImNMCPIn7KWdedpm+irgr6qaA1wIPCsiYX4+F1Wdo6p5qpqXmdmLxnup2AVPXwyN+5ygzzzB7YqMMSHKn+PsC4D+PtM57O+mafE9YDKAqi4RkRggw8/n9k7Ve5wW/b5SuOF16DfC7YqMMSHMn5b9UmCwiOSKSBTOD64L2qyzAzgPQEROBGKAYu9600QkWkRygcHAF4EqPmjtK4VnpkBFAVwz3zlD1hhjulGnLXtVbRKR24C3gXBgrqquFpHZQL6qLgD+E/iTiMzA6aa5UVUVWC0i84E1QBMwvdcfidNYB3+bCiUb4eoXYeC33K7IGNMLiJPJPUdeXp7m5+e7XUb3+fBh+OgRuOJZGHax29UYY0KEiCxT1byOltsZtEfT3o2w+Dcw4nILemPMUWVhf7Sowht3QUQsTHrI7WqMMb2MjXp5tHz9ImxdBBf9GhL7ul2NMaaXsZb90bCvFN6+H7Lz4NSb3a7GGNMLWcv+aHhvJtSWOYObhdnnqzHm6LPk6W47PoPlz8Dp/+GMT2+MMS6wsO9Onkb45wxIyoEJ97ldjTGmF7NunO605HHYswamvQDRCW5XY4zpxaxl313KtsHCX8CQi2DohZ2ubowx3cnCvjuowpt3g4TBhb90uxpjjLGw7xZrXoeN78A5P4HkHLerMcYYC/uAq6uEt+51jrwZ+wO3qzHGGMB+oA28D38OVUVw5XMQbv+8xpiewVr2gVT4JXwxB077HuTYGPXGmJ7Dwj5Qmj3wjzshPhPO+2+3qzHGmANYP0Og5M+F3Stg6p8hJtntaowx5gDWsg+Epnr4+NcwcByMmOp2NcYYcxAL+0D4ej5U7Yaz7gIRt6sxxpiDWNgfqeZm+OR3zqGWx53ndjXGGNMuv8JeRCaLyHoR2SQi97az/FERWeG9bRCRcp9lHp9lCwJZfI+w/g3n4uHj7rRWvTGmx+r0B1oRCQeeAM4HCoClIrJAVde0rKOqM3zWvx0Y7bOJWlUdFbiSexBVWPwopAyEYVPcrsYYYzrkT8t+DLBJVbeoagMwD7jkEOtfBbwQiOJ6vG2LYdcy+NbtdgKVMaZH8yfss4GdPtMF3nkHEZGBQC7wgc/sGBHJF5HPRKTd5q+I3OpdJ7+4uNjP0nuAT34LcRkw+lq3KzHGmEPyJ+zb64jWDtadBrykqh6feQNUNQ+4GvitiBx30MZU56hqnqrmZWZm+lFSD1C0Eja9B6f/ACJj3a7GGGMOyZ+wLwD6+0znAIUdrDuNNl04qlrovd8CLOTA/vzgtfi3EJUAp33f7UqMMaZT/oT9UmCwiOSKSBROoB90VI2IDAFSgSU+81JFJNr7OAMYB6xp+9ygU7YNVr8Cp94IsaluV2OMMZ3q9FdFVW0SkduAt4FwYK6qrhaR2UC+qrYE/1XAPFX17eI5EfijiDTjfLA84nsUT9D69HGQcDhjutuVGGOMX/w6hERV3wTebDPvv9tMz2rneZ8CJx1BfT1PdTF8+SycfCUkHeN2NcYY4xc7g7arvvijMxbOt+5wuxJjjPGbhX1X1Fc549UPvQgyT3C7GmOM8ZuFfVcsexrqKpyhEYwxJohY2PurqQGWPAEDz4T+p7ldjTHGdImFvb9WzoeqQjhzRufrGmNMD2Nh74+WYYz7ngTH2zDGxpjgY2Hvjw3/gr0bYNwdNoyxMSYoWdh3pnUY4wEw/FK3qzHGmMNiYd+Z7Z9CwVL41o9sGGNjTNCysO/M4kedYYxHXeN2JcYYc9gs7A+l8EvY9C6M/QFExbldjTHGHDYL+0P54GcQkwJjb3W7EmOMOSIW9h3ZvsS5OMmZd0JMstvVGGPMEbGwb48qfPAgxPeBMdaqN8YEPwv79mz+ALZ/AuN/DFHxbldjjDFHzMK+rZZWfXJ/50pUxhgTAizs21r3hnMUztn/BRHRbldjjDEBYWHvq9kDH/4c0o6Dk692uxpjjAkYOyXU16pXYM8amPpnO1vWGBNS/GrZi8hkEVkvIptE5N52lj8qIiu8tw0iUu6z7AYR2ei93RDI4gPK0wgLH4I+w2H4ZW5XY4wxAdVp81VEwoEngPOBAmCpiCxQ1TUt66jqDJ/1bwdGex+nATOBPECBZd7nlgV0LwJhxfNQugWmPQ9h1rtljAkt/qTaGGCTqm5R1QZgHnDJIda/CnjB+3gS8K6qlnoD/l1g8pEU3C2a6uGjX0L2qTDkQrerMcaYgPMn7LOBnT7TBd55BxGRgUAu8EFXnisit4pIvojkFxcX+1N3YOX/BSoL4NwHbLx6Y0xI8ifs20s/7WDdacBLqurpynNVdY6q5qlqXmZmph8lBVBDDXz8K+fasseec3Rf2xhjjhJ/wr4A6O8znZt2xdwAABAzSURBVAMUdrDuNPZ34XT1ue74/I9QUwzn/dRa9caYkOVP2C8FBotIrohE4QT6grYricgQIBVY4jP7bWCiiKSKSCow0TuvZ6gtd64te/z5MOB0t6sxxphu0+nROKraJCK34YR0ODBXVVeLyGwgX1Vbgv8qYJ6qqs9zS0XkQZwPDIDZqloa2F04AkuegLpyp6/eGGNCmPhkc4+Ql5en+fn53f9CNXvhdyfDcefClc92/+sZY0w3EpFlqprX0fLee5ro4kedH2fPud/tSkwvoKoUlNUSGxVOWlwUYWHB+fuQqrK3uoGte2vYUlzNvgYPKXGRpMRFkhwbRUpcJKlxUSTFRBAR3vXzVVoan2K/nwVc7wz7yt2w9P9g5JXQZ6jb1ZgQtbuilsUb9/LJpr18srmE4qp6ACLChIyEaDITo+mTGE2fpGgyE2Ocx4nR9EmKoX9qLOkJgRmIr6GpmfVFVeytqSc2Mty5Re2/j4sKJyYi/IAPoNoGD9tKathS7IT61r01bPYGfFVdk1+vmxgT4RP+kXialfomDw2eZhqafG6eZup9HqtCVHgYkeFCRHgYkeFhRIULkRHO44gwISrCuQfn8D5V72F+qj7T6txryzpKszrzWu4VDp7n/cDx3Q74bqdlD/WAbR/wHJ9pgMiW/QkLIyrCeRzp3Tffx4P7JPDAd4Ydztvcqd4Z9osfheYmmHDQyA8mwOoaPazYWc7nW0r5fGsJKwsqyEyKZkjfRIb0S2y9H5geT3iQtnZbVNQ28tmWEj7ZtJfFm/aypbgGgPT4KMYdn8GY3DSaPM3sqapvve0qr+WrgnJKahpo26PaNymaEcckM/yYJIZ573NSYw/Z6vU0K5uLq/lqZzlfF1TwdUE5a3dX0eBp7rT+6Igw4qLCCQ8LY291/QHLspJjODYznimjsjk2M57cjHiOy0wgMSaCitpGyvY1Ur6vwXlc00B5bSPl3nnltY1U1jYSHibERoWTHB5JVEQYURHhRIU74Rcd4dxHhYchAo0epdHTTJOnmQafx40epcHT7J1u+RaA914Q77S0mQYhPAwEISxs/7IwEUT23wvS+nxatyU+r+Fsq3W6vXU4+JuJp9nZhwbvPjR596HBozR6P+hqGjyU7Wvs9H06XL0v7PeVwpfPwklXQFqu29X0CA1Nzewqr2VH6T52lNQ496X7KCyvIy0+igFpcQxMj2NAWhwDvPdxUe3/16lt8LB8Rxmfbynhs62lrNhZTkNTMyIwtF8S3x11DCXV9awrquKt1UWtARcdEcbgvgmc0Hf/B8DxfRLolxRzWN0B4LSuiirrWFdUxbrdVawvqqS6vom+STFkJcfQLzmWfkkx9Et2puOjD/3noKrUNjp/kGU1DZTta6BsXyMbiqpYvGkvXxeU06wQFxXOmNw0rh4zgHHHZzCkb2Kn3TaNnmZKqhvYU1XHnsp6tpXUsGpXBasLK/lw/R6avf9OybGRDMtKYvgxSQzPTuK4zAS2lezja2+4ryqsYF+Dc5pLQnQEI7KTuHHcIEbmJHNMSix1DR5qG723hvbvG5qaOSYlltyM+NZg7+j9BkiJi2JgetfeG3P09b4faBf9yrk4yX98Cn2Hd9/r9DDNzU6f8ZrdlWwurmZHyb7WUN9dUdsaJuAE74C0OLJSYimtqWd7yb6DvrpnJETv/wBIi6PR08znW0v5uqCcRo8SJjD8mGTG5qYx9th0xgxKIzku8oBt1DZ42LSnmnVFlWz4pop1RVVs+KaKbyr3tyrDw4R+STFkp8aSkxpLTkosOalxrdNZybFERYRRXd/E+qIq1hdVsa6oknXexxW1+1tKWckxJMdG8k1lXbstqMSYCLKSY+ibFENmYjR1jR7Kahq9oe4Ee0PTwS3k8DBhVP8Uxh2fwZnHZzCqfwpREYEbX6m2wcO6okpWFzq3NYUVrCuqot6nlqiIMIZlJXFyTjIjc1I4uX8yx2YkBO1vA6brOvuBtneFfVM9PDoC+p0E173SPa8RIE2eZtbsruSbynrS4p1+z/T4aBJjIjr9A97X0NTaml27u5K1u53wq67fH9i+Yd3fG9gt05kJ0Qe8hqpSUdvI9pJ9bC/dx87SfWxv+QZQso/dlXWEiXBSdjJjj03j9Nx0Th2USlJMZHvldaqspoEN31SxZW8Nu8pq2VVey66yWgrK9lFUWXfAB5MIpMZFUVrT0DovITqCE/omMDQriaH9EhnaL4khfRMP+LCpa/TwTWUduyvqKKpoua+lqNKZLq6qJzYqnNS4KFLiolrfg5S4KFLjIlvv0+KjyEqJJaGTbwWB1uRpZnNxDZuLqxmQFseQfolEHuY3IBMaLOx9LX8WFtwG170Gx/WsoRHqGj18uaOcpdtKWbqtlOXby6hp8By0XniYtIZMalwUafHOLTEmkh2lNazdXcW2kprW7pHE6AiGZiVyYlZS621wn4ROuyy6WrsqxEaFB2ybHWn0NFNUUUeB90OgoGwf31TWk50Sw5B+Trh31q9tTCiyQy9bqMKSx6HvCDh2gtvVUFHbyLLtpXyxtYyl2/Z3fwAM7ZfIZafkcFpuGgPT4iivbaS0pp7SGqevuKSmgbKaBkprGti4p5rSGueHseyUWE7MSuSSUcdwYlYSw7I6/0EvEGIiuz/kW0SGh9Hf+23EGOO/3hP2m96D4nVw6R+P+hg4qsrWvTUs31HOlzvKWL6jnHVFlag6h+GdlJPMzeNyOW1QGnmDUkmJizqs17DWrDGmI70n7D99DBKPOSpXoaqqa+SrnRUs31HGlzvK+HJnOeXeHwQToiMY1T+FO84bzJhBaYwakHLIIx38ZUFvjDmU3hH2u7+CrYvg2/8DEV1vNR9Ky5mRn28tJX9bKV/uKGfDnqrWPvPBfRKYNKwfowekcMrAVI7LTAj648mNMcGnd4T9p49DVAKceuMRb0pV2VxcwxdbnZOEvthayu6KOsA5Bnr0gBQuPCmLUwamMDInheTYwzsixRhjAin0w76iAFa9DGN/ALEpXX56c7OyrqiKL7aW8MW2Ur7YWsreaucwv8zEaMbkpnF6bhpjctMZ3MeOazbG9EyhH/afPencn/4Dv1avrm/iq53lLN9exrIdZSzfXkal94Si7JRYxg/OZIz3RKFB6XHWV26MCQqhHfZ1FbDsaRg+BVIGHLS4pb992fay1tu6okqa1TlgZ3CfBC48KYsxuWmMyU0jJ9UO9zPGBKfQDvvlz0BDFZxxW+usbyrreGfNN3yycS/LdpS1jkQYHxXO6AGp3HbuYE4dmMqo/tbfbowJHaEb9p5Gpwtn0FlsjR7C2x9t5u3VRXy5oxyA/mmxnHl8BqcMTOXUAakM6ZdoR8kYY0JWSIa9qlLw8XP0r9zFT5tu5tlfLQTgpOxkfjzxBCYN78fxfRKsv90Y02uETNg3eZpZuq2Mt1cX8e7qIv5Y+yj1HMOm5DOYOe4YJg7vR3ZKrNtlGmOMK/wKexGZDPwO54Lj/6eqj7SzzhXALJyLs3ylqld753uAld7VdqjqxQGo+yBFlXVc9afPiIoI49acnYyo30bNpN/wwhnjuuPljDEmqHQa9iISDjwBnA8UAEtFZIGqrvFZZzBwHzBOVctEpI/PJmpVdVSA6z5ITmocT988hryBqcT/fRrEZxKfd013v6wxxgQFfwbAHgNsUtUtqtoAzAMuabPOLcATqloGoKp7Alumf84+IZP4io2w6V0YcytExrhRhjHG9Dj+hH02sNNnusA7z9cJwAki8omIfObt9mkRIyL53vlT2nsBEbnVu05+cXFxl3bgIEseh4hYyPvekW3HGGNCiD999u0dstL2iicRwGBgApADfCwiI1S1HBigqoUicizwgYisVNXNB2xMdQ4wB5yLl3RxH/arKoKv58Mp10O8XRTTGGNa+NOyLwD6+0znAIXtrPO6qjaq6lZgPU74o6qF3vstwEJg9BHW3LEv5jjH15/+w257CWOMCUb+hP1SYLCI5IpIFDANWNBmndeAcwBEJAOnW2eLiKSKSLTP/HHAGrpDQw0s/TMMvQjSj+uWlzDGmGDVaTeOqjaJyG3A2ziHXs5V1dUiMhvIV9UF3mUTRWQN4AHuVtUSEfkW8EcRacb5YHnE9yiegKqrdC43aK16Y4w5SO+64LgxxoSozi447k83jjHGmCBnYW+MMb2Ahb0xxvQCFvbGGNMLWNgbY0wvYGFvjDG9gIW9Mcb0Ahb2xhjTC/S4k6pEpBjYfgSbyAD2BqicniDU9gdCb59CbX8g9PYp1PYHDt6ngaqa2dHKPS7sj5SI5B/qLLJgE2r7A6G3T6G2PxB6+xRq+wNd3yfrxjHGmF7Awt4YY3qBUAz7OW4XEGChtj8QevsUavsDobdPobY/0MV9Crk+e2OMMQcLxZa9McaYNizsjTGmFwiZsBeRySKyXkQ2ici9btcTCCKyTURWisgKEQm6K7qIyFwR2SMiq3zmpYnIuyKy0Xuf6maNXdXBPs0SkV3e92mFiFzoZo1dISL9ReRDEVkrIqtF5A7v/KB8nw6xP8H8HsWIyBci8pV3n/7HOz9XRD73vkcvei8b2/F2QqHPXkTCgQ3A+TgXP18KXNVtl0A8SkRkG5CnqkF5MoiIjAeqgWdUdYR33i+BUlV9xPuhnKqq97hZZ1d0sE+zgGpV/ZWbtR0OEckCslR1uYgkAsuAKcCNBOH7dIj9uYLgfY8EiFfVahGJBBYDdwB3Aa+o6jwReQr4SlWf7Gg7odKyHwNsUtUtqtoAzAMucbmmXk9VFwGlbWZfAjztffw0zh9i0Ohgn4KWqu5W1eXex1XAWiCbIH2fDrE/QUsd1d7JSO9NgXOBl7zzO32PQiXss4GdPtMFBPkb7KXAOyKyTERudbuYAOmrqrvB+cME+rhcT6DcJiJfe7t5gqLLoy0RGQSMBj4nBN6nNvsDQfweiUi4iKwA9gDvApuBclVt8q7SaeaFSthLO/OCv38KxqnqKcAFwHRvF4LpeZ4EjgNGAbuBX7tbTteJSALwMnCnqla6Xc+Ramd/gvo9UlWPqo4CcnB6Mk5sb7VDbSNUwr4A6O8znQMUulRLwKhqofd+D/Aqzpsc7L7x9qu29K/ucbmeI6aq33j/GJuBPxFk75O3H/hl4DlVfcU7O2jfp/b2J9jfoxaqWg4sBE4HUkQkwruo08wLlbBfCgz2/jodBUwDFrhc0xERkXjvD0yISDwwEVh16GcFhQXADd7HNwCvu1hLQLSEotelBNH75P3x78/AWlX9jc+ioHyfOtqfIH+PMkUkxfs4Fvg2zm8RHwKXe1fr9D0KiaNxALyHUv0WCAfmqurPXS7piIjIsTiteYAI4Plg2ycReQGYgDMU6zfATOA1YD4wANgB/JuqBs0Pnh3s0wSc7gEFtgH/3tLf3dOJyJnAx8BKoNk7+yc4/dxB9z4dYn+uInjfo5E4P8CG4zTQ56vqbG9GzAPSgC+Ba1W1vsPthErYG2OM6ViodOMYY4w5BAt7Y4zpBSzsjTGmF7CwN8aYXsDC3hhjegELe2OM6QUs7I0xphf4f5NpmTphWmpDAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXsAAAD7CAYAAACL+TRnAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAgAElEQVR4nO3de3RV9Z338feXEAg3IZCoSLhEiojIPQNOddTWalGreMEOYDvi6gzTLunjOJfWPtNHLTOu8el0preHacdxAGtFtICCDq1Vq9NOq8AJd1AEEZIQLpFLALkl4fv8sXfiISTkJDnJzjnn81qLxdm3X77bI182v+/+/X7m7oiISHrrFHUAIiLS9pTsRUQygJK9iEgGULIXEckASvYiIhlAyV5EJAMklOzNbLKZbTWz7Wb2cAPHB5vZG2a2wczeMrOCuGM1ZrYu/LU8mcGLiEhirKn37M0sC3gfuBEoA1YD0919S9w5vwBecfenzeyzwP3u/uXw2DF379lWNyAiIk3rnMA5E4Ht7r4DwMwWAVOALXHnXAE8FH5+E3ippQHl5eX5kCFDWnq5iEhGKi4u/sjd8xs7nkiyHwCUxm2XAZPqnbMeuBv4IXAn0MvM+rn7ASDHzGJANfCEu5/3L4IhQ4YQi8USCEtERGqZ2a7zHU+kz94a2Fe/7+dvgevMbC1wHbCbILkDDHL3ImAG8AMzG9pAkLPMLGZmsYqKigRCEhGR5kgk2ZcBA+O2C4Dy+BPcvdzd73L3ccDfh/sqa4+Fv+8A3gLG1f8B7v6kuxe5e1F+fqP/ChERkRZKJNmvBoaZWaGZdQGmAWe9VWNmeWZW29a3gHnh/lwz61p7DnA1Z/f1i4hIO2gy2bt7NTAbeBV4F3jB3Teb2Rwzuz087Xpgq5m9D1wEPB7uHwHEzGw9QeH2ifi3eEREpH00+epleysqKnIVaEVEmsfMisP6aIM0glZEJAMo2YuIZIBE3rOXTHTmDKx6Eo4fiDoSyTSDJsGnPpe89nYXw+mPofDa5LVZshK2v5689mpdcAkU3Z/8dlGyl8Z88Ab86pvhRkNDLUTagkPX3vA370GX7q1v7swZWPwVOHEoaDO7W3LaXPoXcHgXSf+zUVCkZC/trHgB9MiHh7ZA5y5RRyOZYuf/wIJbYctLMHZG69v78L/h0IfB5y3LYMy01re5480g0d/9nzBqauvbayfqs5dzHdkDW38JY+9Vopf2NfhqyLsMYvOT017xAuiWC30vDT4npc350L0fjLgtOe21EyV7OdfaZ8BrYMJ9UUcimcYMJsyEslWwb3Pr2jq2H957JXhomXA/lLwN+99tXZtH98J7K8IHoa6ta6udKdnL2c7UQPHTcOn1wdOQSHsbMx2yurb+6X7tz+FMNYy/L+gSyurS+qf7ugehma1rJwJK9nK27W/AkbLgSUgkCt37whVTYMPzcPp4y9o4cwbWPA2Dr4H8y6BHXtDtsv45qDrR8jaLfxa81dPvnPkcOzwlezlb8XzocSFcfmvUkUgmK7ofTh2BzUtbdv2Hb8GhnWe/2TLhfjhZCZtbuNzGB7+BypKUfRBSspdPVO6G938F4+6FrOyoo5FMNuiPIW94y7tyYvOhW9+zi6hDroF+nwoeaFqieD50z4PLv9Cy6yOmZC+fWPtz8DNBH6dIlGoLtbtjsHdj8649ug+2rgj66eOLqLVtlq6Efc2cj7H2DbVxqfuGmpK9BM7UwJqfwdDPQt/CqKMRCd6Jz+ra/KLqurAw21B3y5gWFmrX/jwozKbwg5CSvQS2vRYWZmdGHYlIoHtfGHkHbHghmO4gEWfOBG+TDfkTyPvUucd79AuKv+sXJV78PVMTFHsLr0vJwmwtJXsJFC+AnhfB8FuijkTkExPCQu2mBAu1O34TjG4930PLhJlwqhI2v5hYmx/8BipL22wag/aiZC9QWQbbXoVxX1JhVjqWQVdB/uWJF1WLFzQ9urV2lG6iXTmx+cHUIcNT+w01JXuBNc+AO4z/s6gjETlbXaG2GPZsOP+5iY5ubc4o3SPlwRtqaTB1SELJ3swmm9lWM9tuZg83cHywmb1hZhvM7C0zK4g7dp+ZbQt/pW51I13VVAejAod+FnKHRB2NyLnGTIPOOU0/iTdndGuio3RrC7NpMHVIk8nezLKAucDNwBXAdDO7ot5p3wN+5u6jgTnAP4XX9gUeBSYBE4FHzSw3eeFLq21/DY7sTvn+SElj3XJh5J1BofbUsYbPOVPTvNGtZ43SbaT4Wzd1yGfSYuqQRJ7sJwLb3X2Hu58GFgFT6p1zBfBG+PnNuOOfB15z94Pufgh4DZjc+rAlaWLzoefFcJm+FunAJtwPp4/CpiUNH//gzeaPbq0bpdtIoXb768EbamnyIJRIsh8AlMZtl4X74q0H7g4/3wn0MrN+CV4rUTlcGjzZqzArHd3AiZA/ovGunOKwiNqc0a1NjdItXhBMHZImb6glkuwbWorF623/LXCdma0FrgN2A9UJXouZzTKzmJnFKioqEghJkmJtWJhNg/5ISXNmwRN2+RrYs/7sYy1df6G2zYZG6dZNHZI+D0KJJPsyYGDcdgFQHn+Cu5e7+13uPg74+3BfZSLXhuc+6e5F7l6Un5/fzFuQFqmpDkbMfupz0GdQ1NGING30nwaF2vpP4nWjW1vwNtnoP214lO7aZ4KpQ9LoQSiRZL8aGGZmhWbWBZgGLI8/wczyzKy2rW8B88LPrwI3mVluWJi9KdwnUdv2azi6RyNmJXV06wMj74KNv/ikUFs7uvXS61s2urV730+Kv7WF2vipQ9LoDbUmk727VwOzCZL0u8AL7r7ZzOaY2e3hadcDW83sfeAi4PHw2oPAPxD8hbEamBPuk6gVz4de/VWYldRSdD+cPgabFgfb298IRre2Ztrh2kJtbfF3W/iGWopOZdyYhBYcd/cVwIp6+x6J+7wYWNzItfP45ElfOoLDJcH/0Nf+HWRpzXlJIQV/BBeODLpyJsz8pDDbmiLqwEnBKN3Y/KArqHh+OHXIzUkLuyPQCNpMtOaZ4PfxX442DpHmqh39umddUJStLaK2ZnSrWfAUX74maHPbr9OqMFtLyT7T1I6YHXajCrOSmkZ/ETp3g6Wzkrf+wpiw+Lt0Vjh1SPoUZmvp3/DpYsdb8PFHTZ/30ftBYfbWf23zkETaRLc+cOXdwbz1yVp/oVtuUPxdvzB4Qy13cOvb7GCU7NPB7jXws/qDms+jzyAYdlPbxSPS1v7oK8Hi4RP/Molt/nkwfUIy2+xAlOzTQWweZHeHr/w6eGe4KT0vVGFWUtuA8fCND4In8mQpmJD8NjsQ/YlPdSfDV8auvBsuHhV1NCLtpy2ScpomelCBNvVtfAGqjqfNZE0i0jaU7FOZO8QWwMWj4ZLxUUcjIh2Ykn0q210M+zYG7x1bQ3POiYgElOxTWfF8yO4Bo+6JOhIR6eCU7FPVyUrYtBRGTYWcC6KORkQ6OCX7VLUhLMxq1koRSYCSfSpyDyZt6j8meN9YRKQJSvapqCwG+zen3RSsItJ2lOxTUfF86NIz6K8XEUmAkn2qOXH4k8Js115RRyMiKULJPtVseAGqT6gLR0SaRck+lbgHXTiXjINLxkYdjYikkISSvZlNNrOtZrbdzB5u4PggM3vTzNaa2QYzuyXcP8TMTpjZuvDXT5N9AxmldBXs36KnehFptiZnvTSzLGAucCNQBqw2s+XuviXutG8TLET+EzO7gmC92iHhsQ/cXY+hyVC8ALr0Cma4FBFphkSe7CcC2919h7ufBhYB9VfKcKB2GGdvoDx5IQoAJw7B5qUw+h7o2jPqaEQkxSSS7AcApXHbZeG+eI8BXzKzMoKn+q/HHSsMu3f+28z+pKEfYGazzCxmZrGKiorEo88k65+H6pPqwhGRFkkk2Tc0naLX254OLHD3AuAW4Bkz6wTsAQa5+zjgr4GFZnbORC7u/qS7F7l7UX5+fvPuIBPUFWbHQ//RUUcjIikokWRfBgyM2y7g3G6arwAvALj720AOkOfup9z9QLi/GPgAuKy1QWec0pVQ8Z4WKBGRFksk2a8GhplZoZl1AaYBy+udUwLcAGBmIwiSfYWZ5YcFXszsUmAYsCNZwWeM2HzoeoEKsyLSYk2+jePu1WY2G3gVyALmuftmM5sDxNx9OfA3wH+Y2UMEXTwz3d3N7FpgjplVAzXAV939YJvdTTo6fhA2vwjjvwxdekQdjYikqIQWHHf3FQSF1/h9j8R93gJc3cB1S4AlrYwxs61fBDWnNJWxiLSKRtB2ZO7Bu/UDiuDiUVFHIyIpLKEne0myyrJgQrOmVLwHH22FKXPbPiYRSWtK9u3twAcwdyKcqU7s/K69YeSdbRuTiKQ9Jfv2tubpoHvmrqegc9emz+/3KRVmRaTVlOzbU/VpWPssDL85mPZARKSdqEDbnt57GY5/pCkPRKTdKdm3p+IF0HsQDP1s1JGISIZRsm8vBz6AD38LE/4MOuk/u4i0L2Wd9lI8Hzp1hnFfjjoSEclASvbtofoUrFsYFGZ7XRx1NCKSgZTs28O7L8PxA5ryQEQio2TfHooXQJ/BcKkKsyISDSX7tvbRdtj5O5hwnwqzIhIZZZ+2VluYHfulqCMRkQymZN+Wqk6GhdlboNdFUUcjIhlMyb4tvfsynDio5QRFJHJK9m2peAHkDoHC6yMOREQyXULJ3swmm9lWM9tuZg83cHyQmb1pZmvNbIOZ3RJ37FvhdVvN7PPJDL5Dq3gfdv0PjFdhVkSi1+Ssl+GC4XOBG4EyYLWZLQ+XIqz1beAFd/+JmV1BsIThkPDzNGAkcAnwupld5u41yb6RDqd4QThiVoVZEYleIo+cE4Ht7r7D3U8Di4Ap9c5x4ILwc2+gPPw8BVjk7qfc/UNge9heeqs6CesXwuW3Qs8Lo45GRCShZD8AKI3bLgv3xXsM+JKZlRE81X+9GddiZrPMLGZmsYqKigRD78DeXQ4nDmkqYxHpMBJJ9tbAPq+3PR1Y4O4FwC3AM2bWKcFrcfcn3b3I3Yvy8/MTCKmDi82H3EIovC7qSEREgMSSfRkwMG67gE+6aWp9BXgBwN3fBnKAvASvTS8VW6HkD8E8OCrMikgHkUg2Wg0MM7NCM+tCUHBdXu+cEuAGADMbQZDsK8LzpplZVzMrBIYBq5IVfIdUvAA6ZcPYe6OORESkTpNv47h7tZnNBl4FsoB57r7ZzOYAMXdfDvwN8B9m9hBBN81Md3dgs5m9AGwBqoEH0vpNnKoTwYjZEV+AnmnQHSUiaSOhBcfdfQVB4TV+3yNxn7cAVzdy7ePA462IMXVsWQYnD6swKyIdjjqVk6l4AfS9FIb8SdSRiIicRck+Wfa/CyVvqzArIh2SslKyqDArIh2Ykn0yVJ2A9c/BiNugR17U0YiInEPJPhk2vwQnKzWVsYh0WEr2yVA8H/oOVWFWRDosJfvW2rcFSlcGhVlraHYIEZHoKdm3VvECyOqiwqyIdGhK9q1x+jisXwQjboce/aKORkSkUUr2rbHlJTilwqyIdHxK9q0Rmw/9hsHgBmeKEBHpMJTsW2rfZihbpcKsiKQEJfuWqivMzog6EhGRJinZt8Tp47D+ebhiCnTvG3U0IiJNUrJvic1Lg8KspjIWkRShZN8SxQsgbzgM/nTUkYiIJCShxUskzt5NULYaPv9PKsy2off2HuG1zfuiDkOkXV3UO4cvFg1s+sQWSCjZm9lk4IcEyxI+5e5P1Dv+feAz4WZ34EJ37xMeqwE2hsdK3P32ZAQemeL5kNUVxkyLOpK09s0lG1lfejjqMETa1diBfaJL9maWBcwFbgTKgNVmtjxcihAAd38o7vyvA+Pimjjh7mOTF3KETn8MG16AkXeoMNuGNpdXsr70MP/nC1dw3x8PjjockbSQyJP9RGC7u+8AMLNFwBSCRcQbMh14NDnhdTCblsKpIyrMtrGFK0vo2rkTU8cX0DlLZSWRZEjkT9IAoDRuuyzcdw4zGwwUAr+J251jZjEze8fM7mjkulnhObGKiooEQ49A8fygMDvoqqgjSVsfn6pm2bpybh3dn97ds6MORyRtJJLsG6pCeiPnTgMWu3tN3L5B7l4EzAB+YGZDz2nM/Ul3L3L3ovz8/ARCisCeDbC7OJgHR4XZNvPy+nKOnarm3kmDog5FJK0kkuzLgPiKQQFQ3si504Dn4ne4e3n4+w7gLc7uz08dxQugc44Ks21s4aoShl/Ui/GDcqMORSStJJLsVwPDzKzQzLoQJPTl9U8ys+FALvB23L5cM+safs4Drqbxvv6O69SxsDB7J3RTEmorm3ZXsqGskukTB2L615NIUjVZoHX3ajObDbxK8OrlPHffbGZzgJi71yb+6cAid4/v4hkB/LuZnSH4i+WJ+Ld4UsamJXD6aDDpmbSZhatKyMnuxJ3jC6IORSTtJPSevbuvAFbU2/dIve3HGrjuD8CoVsTXMRQvgPwRMHBS1JGkrWOnqlm2djdfGH0JvbupMCuSbHqvrSl71kP5GhVm29jydeV8fLqGGSrMirQJJfumxOYHhdnRX4w6krS2cNUuLr+4F+MG9ok6FJG0lD5z4xw/CE/dkPx2D5fCqHtUmG1DG8sq2bT7CHOmjFRhVqSNpE+y79QZBkxIfrsDJ8G1f5v8dqXOwlW76JadxR3jGhyrJyJJkD7JPucCuPupqKOQZjp6sopl68q5bUx/LshRYVakrajPXiK1fH05x0/XMH2iCrMibUnJXiLj7ixcWcKI/hcwVoVZkTalZC+R2VBWyebyI8yYNEiFWZE2pmQvkVm4soRu2VlMGXtJ1KGIpD0le4nE0ZNVLF9fzu1jLlFhVqQdKNlLJF5aV86JKo2YFWkvSvbS7moLsyMvuYDRBb2jDkckIyjZS7tbV3qYd/ccYfpEFWZF2ouSvbS751aV0L2LCrMi7UnJXtrVkZNVvLx+D1PGXkIvFWZF2o2SvbSrl9bu5kSVRsyKtLf0mRtHkm7ljgMcOn46qW0+8/YurhxwAaMLNGJWpD0llOzNbDLwQ4JlCZ9y9yfqHf8+8Jlwsztwobv3CY/dB3w7PPaP7v50MgKXtrW+9DB/+uQ7bdL2d+8e3Sbtikjjmkz2ZpYFzAVuBMqA1Wa2PH4tWXd/KO78rwPjws99gUeBIsCB4vDaQ0m9C0m6Z1fuonuXLBbNuorsrOT19mVnGUPzeyatPRFJTCJP9hOB7e6+A8DMFgFTgMYWDp9OkOABPg+85u4Hw2tfAyYDz7UmaGlb8UVUdbeIpIdEHtkGAKVx22XhvnOY2WCgEPhNc641s1lmFjOzWEVFRSJxSxtapiKqSNpJJNk3NOrFGzl3GrDY3Wuac627P+nuRe5elJ+fn0BI0lbcnWc1ulUk7SSS7MuAgXHbBUB5I+dO4+wumuZcKx3AutLDvLf3qKYdFkkziST71cAwMys0sy4ECX15/ZPMbDiQC7wdt/tV4CYzyzWzXOCmcJ90UAtXBqNbbx+j0a0i6aTJAq27V5vZbIIknQXMc/fNZjYHiLl7beKfDixyd4+79qCZ/QPBXxgAc2qLtdLxVJ6o4uUN5dw5boBGt4qkmYTes3f3FcCKevseqbf9WCPXzgPmtTA+aUfL1u3mZNUZZkwcHHUoIpJkmi5BgE+mHb5ywAWMUmFWJO0o2QsAa0rCwqye6kXSkpK9AMG0wz26ZHG7ph0WSUtK9kLliSpe2VDOlHED6NlVc+OJpCMle+HFNWVhYVYjZkXSlZJ9hnN3nltVyuiC3lw5QIVZkXSlZJ/h1pQcYuu+o3qqF0lzSvYZ7tmVJfTs2pnbNGJWJK0p2WewyuNV/NeGYCrjHirMiqQ1JfsMtnRtGaeqzzBjkrpwRNKdkn2Gqh0xO6agNyMvUWFWJN0p2Weo2K5DbNt/TE/1IhlCyT5DPafCrEhGUbLPQIePn+aVjXu4Y9wldO+iwqxIJlCyz0BL1uzmdLWmMhbJJEr2GSYYMVvC2IF9uOKSC6IOR0Taif4NnySnq89QcvDjqMNo0ta9x9i+/xjfvXt01KGISDtKKNmb2WTghwTLEj7l7k80cM4XgccAB9a7+4xwfw2wMTytxN1vT0LcHc6jyzfx3KrSqMNISK+cznxhTP+owxCRdtRksjezLGAucCNQBqw2s+XuviXunGHAt4Cr3f2QmV0Y18QJdx+b5Lg7lMoTVby4djefG3ERU1JgPvih+T1VmBXJMIn8iZ8IbHf3HQBmtgiYAmyJO+cvgLnufgjA3fcnO9CO7KW1wdqtf/W5YZo5UkQ6pEQKtAOA+P6JsnBfvMuAy8zs92b2TtjtUyvHzGLh/jsa+gFmNis8J1ZRUdGsG4ha7UhUTREsIh1ZIsneGtjn9bY7A8OA64HpwFNm1ic8Nsjdi4AZwA/MbOg5jbk/6e5F7l6Un5+fcPAdgaYIFpFUkEiyLwMGxm0XAOUNnLPM3avc/UNgK0Hyx93Lw993AG8B41oZc4eycGWpRqKKSIeXSLJfDQwzs0Iz6wJMA5bXO+cl4DMAZpZH0K2zw8xyzaxr3P6rObuvP6VVHg/XbtUUwSLSwTWZody92sxmA68SvHo5z903m9kcIObuy8NjN5nZFqAG+Dt3P2Bmnwb+3czOEPzF8kT8WzypTlMEi0iqMPf63e/RKioq8lgsFnUYTXJ3bvr+b+neJYtls6+JOhwRyXBmVhzWRxuk6RJaqFhTBItIClGyb6GFmiJYRFKIkn0LaIpgEUk1SvYtsFRTBItIilGybyZ3Z6GmCBaRFKNk30yrdx5i+/5jGjErIilFyb6ZnltVQq+umiJYRFKLkn0zHPr4NP+1cQ93jh+gwqyIpBQl+2ZYsqaM09VnmK4uHBFJMUr2Capdu3XcoD6M6K/CrIikFiX7BK368CAfVHyswqyIpCQl+wQtXFUSrN06WiNmRST1KNkn4NDHp/nlxr3cNW4A3bpkRR2OiEizKdknYMmaMk7XnGHGJI2YFZHUpPcHm1A7YnbC4FyGX9wr6nBE0kpVVRVlZWWcPHky6lBSRk5ODgUFBWRnZzfrOiX7Jqz88CA7Kj7me/d8KupQRNJOWVkZvXr1YsiQIZg1tNy1xHN3Dhw4QFlZGYWFhc26Vt04TVi4soQLcjrzhdEaMSuSbCdPnqRfv35K9AkyM/r169eifwkl9GRvZpOBHxIsS/iUuz/RwDlfBB4DHFjv7jPC/fcB3w5P+0d3f7rZUSbZj9/Yxr+89n7C58/89BByslWYFWkLSvTN09L/Xk0mezPLAuYCNwJlwGozWx6/lqyZDQO+BVzt7ofM7MJwf1/gUaCI4C+B4vDaQy2KNglOVdcw7/cfMqagN9cNv7DJ8zt3Mo2YFZGUl8iT/URgu7vvADCzRcAUIH7h8L8A5tYmcXffH+7/PPCaux8Mr30NmAw8l5zwm+9Xm/Zy6HgVP5w2nGsvy48qDBGRdpVIn/0AoDRuuyzcF+8y4DIz+72ZvRN2+yR6bbtauLKEQX27c82n8qIMQ0RSUM+ePRs9tnPnThYuXFi3vWDBAmbPnt0eYSUkkSf7hjqIvIF2hgHXAwXA78zsygSvxcxmAbMABg1quy6T7fuPsfLDg3xj8nA6dVI/oUhH8p2XN7Ol/EhS27zikgt49LaRSW2zMbXJfsaMGc26rqamhqystq8JJvJkXwYMjNsuAMobOGeZu1e5+4fAVoLkn8i1uPuT7l7k7kX5+W3XtbJoVQmdOxn3TBjY9Mkikva++c1v8m//9m9124899hjf+c53uOGGGxg/fjyjRo1i2bJlCbX18MMP87vf/Y6xY8fy/e9/H4Dy8nImT57MsGHD+MY3vlF3bs+ePXnkkUeYNGkSb7/9dnJvqjHuft5fBE/tO4BCoAuwHhhZ75zJwNPh5zyCrpt+QF/gQyA3/PUh0Pd8P2/ChAneFk6crvYx33nVv/bzWJu0LyLNt2XLlkh//po1a/zaa6+t2x4xYoTv2rXLKysr3d29oqLChw4d6mfOnHF39x49ejTa1ptvvum33npr3fb8+fO9sLDQDx8+7CdOnPBBgwZ5SUmJu7sD/vzzz7c47ob+uwExP09ubbIbx92rzWw28CrBq5fz3H2zmc0JG18eHrvJzLYANcDfufsBADP7B2B12NwcD4u17e1Xm/Zy+HiVFgkXkTrjxo1j//79lJeXU1FRQW5uLv379+ehhx7it7/9LZ06dWL37t3s27ePiy++uNnt33DDDfTu3RuAK664gl27djFw4ECysrK4++67k30755XQe/buvgJYUW/fI3GfHfjr8Ff9a+cB81oXZustXFXC4H7d+fTQflGHIiIdyNSpU1m8eDF79+5l2rRpPPvss1RUVFBcXEx2djZDhgxp8XQOXbt2rfuclZVFdXU1EEx50B799PEyYgTt9v1HWfXhQab90SAVZkXkLNOmTWPRokUsXryYqVOnUllZyYUXXkh2djZvvvkmu3btSqidXr16cfTo0TaOtuUyYm6chStLyc4y7ikqiDoUEelgRo4cydGjRxkwYAD9+/fn3nvv5bbbbqOoqIixY8dy+eWXJ9TO6NGj6dy5M2PGjGHmzJnk5ua2ceTNk/bJ/mRVDUvWlHHTyIvJ69m16QtEJONs3Lix7nNeXl6jb8gcO3as0Tays7N54403zto3c+bMus+vvPJKQu20lbTvxvnlpj1UnqjScoIiktHS/sl+4coShvTrzh9fqsKsiLTexo0b+fKXv3zWvq5du7Jy5cqIIkpMWif79/cdZfXOQ3zr5stVmBWRpBg1ahTr1q2LOoxmS+tunOdWlZCdZUydoMKsiGS2tE32J6tqWFJcxudHXkw/FWZFJMOlbbL/rw17OHKymhmTVJgVEUnbZL9wVQmFeT1UmBWRRh0+fPisidASdcstt3D48OFmX7dgwQLKyz+ZC3LIkCF89NFHzW6nJdIy2W/de5TiXYeYPnGgljwTkUY1luxramrOe92KFSvo06dPs39e/WSfiNopFlorLd/GeW5VCV2yOjFVUxmLpI5fPgx7NzZ9XnNcPApuPmfJ7DoPP/wwH3zwAWPHjiU7O5uePXvSv39/1q1bx5YtW7jjjjsoLS3l5MmTPPjgg8yaNQsInshjsRjHjh3j5ptv5r0vJc8AAAecSURBVJprruEPf/gDAwYMYNmyZXTr1u2cn7V48WJisRj33nsv3bp1qxu49eMf/5iXX36ZqqoqfvGLX3D55Zfz2GOPUV5ezs6dO8nLyztrUZSWSrsn+xOngxGzk6+8mL49ukQdjoh0YE888QRDhw5l3bp1/PM//zOrVq3i8ccfZ8uWYNXVefPmUVxcTCwW40c/+hEHDhw4p41t27bxwAMPsHnzZvr06cOSJUsa/FlTp06lqKiIZ599lnXr1tX9hZCXl8eaNWv42te+xve+972684uLi1m2bFlSEj2k4ZP9KxvKOarCrEjqOc8TeHuZOHEihYWFdds/+tGPePHFFwEoLS1l27Zt9Ot3dh2wsLCQsWPHAjBhwgR27tzZrJ9511131V27dOnSuv233357g/9CaKm0S/bPrSrh0vweTCrsG3UoIpJievToUff5rbfe4vXXX+ftt9+me/fuXH/99Q1OdVx/GuMTJ04062fWXh8/BXL9WJIhrbpx3tt7hDUlh5kxcZAKsyLSpPNNS1xZWUlubi7du3fnvffe45133mnTn9fW0urJfuHKoDB793iNmBWRpvXr14+rr76aK6+8km7dunHRRRfVHZs8eTI//elPGT16NMOHD+eqq65q9c+bOXMmX/3qV88q0LYXCxaZ6jiKioo8Fos1+7oTp2uY+PjrfHbEhfxw2rg2iExEku3dd99lxIgRUYeRchr672Zmxe5e1Ng1CXXjmNlkM9tqZtvN7OEGjs80swozWxf++vO4YzVx+5c3436a5cjJKq4bns+XrtIasyIi9TXZjWNmWcBc4EagDFhtZsvdfUu9U59399kNNHHC3ce2PtTzu+iCHP7fjPFt/WNERJr0wAMP8Pvf//6sfQ8++CD3339/RBEl1mc/Edju7jsAzGwRMAWon+xFRASYO3du1CGcI5FunAFAadx2WbivvrvNbIOZLTaz+KGrOWYWM7N3zOyOhn6Amc0Kz4lVVFQkHr2IpLyOVjfs6Fr63yuRZN/QO4z1f9rLwBB3Hw28Djwdd2xQWDSYAfzAzIae05j7k+5e5O5F+fn5CYYuIqkuJyeHAwcOKOEnyN05cOAAOTk5zb42kW6cMiD+Sb0AOGsmH3ePH0P8H8D/jTtWHv6+w8zeAsYBHzQ7UhFJOwUFBZSVlaF/0ScuJyeHgoLmv16eSLJfDQwzs0JgNzCN4Cm9jpn1d/c94ebtwLvh/lzguLufMrM84Grgu82OUkTSUnZ29lnTE0jbaTLZu3u1mc0GXgWygHnuvtnM5gAxd18O/C8zux2oBg4CM8PLRwD/bmZnCLqMnmjgLR4REWljaTOoSkQkkyVlUJWIiKS2Dvdkb2YVwK5WNJEHtM86X+0j3e4H0u+e0u1+IP3uKd3uB869p8Hu3ujrjB0u2beWmcXO90+ZVJNu9wPpd0/pdj+QfveUbvcDzb8ndeOIiGQAJXsRkQyQjsn+yagDSLJ0ux9Iv3tKt/uB9LundLsfaOY9pV2fvYiInCsdn+xFRKSetEn2TS2wkorMbKeZbQwXfkm5kWZmNs/M9pvZprh9fc3sNTPbFv6eG2WMzdXIPT1mZrvjFum5JcoYm8PMBprZm2b2rpltNrMHw/0p+T2d535S+TvKMbNVZrY+vKfvhPsLzWxl+B09b2ZdzttOOnTjhAusvE/cAivA9FSfmsHMdgJF7p6S7web2bXAMeBn7n5luO+7wEF3fyL8SznX3b8ZZZzN0cg9PQYcc/fvRRlbS5hZf6C/u68xs15AMXAHwZQnKfc9ned+vkjqfkcG9HD3Y2aWDfwP8CDw18BSd19kZj8F1rv7TxprJ12e7OsWWHH300DtAisSIXf/LcFcSfGm8MkU2E8T/EFMGY3cU8py9z3uvib8fJRgEsMBpOj3dJ77SVkeOBZuZoe/HPgssDjc3+R3lC7JPtEFVlKNA782s2IzmxV1MElyUe0MqeHvF0YcT7LMDhfvmZcqXR71mdkQginIV5IG31O9+4EU/o7MLMvM1gH7gdcIpok/7O7V4SlN5rx0SfaJLLCSiq529/HAzcADYReCdDw/AYYCY4E9wL9EG07zmVlPYAnwV+5+JOp4WquB+0np78jda8K1vAsIejJGNHTa+dpIl2Tf5AIrqShu4Zf9wIsEX3Kq2xf2q9b2r+6POJ5Wc/d94R/GMwSL96TU9xT2Ay8BnnX3peHulP2eGrqfVP+Oarn7YeAt4Cqgj5nVTlPfZM5Ll2Rft8BKWJGeBiyPOKZWMbMeYYEJM+sB3ARsOv9VKWE5cF/4+T5gWYSxJEVtUgzdSQp9T2Hx7z+Bd939X+MOpeT31Nj9pPh3lG9mfcLP3YDPEdQi3gSmhqc1+R2lxds4AOGrVD/gkwVWHo84pFYxs0sJnuYhWGRmYardk5k9B1xPMDvfPuBR4CXgBWAQUALc4+4pU/Bs5J6uJ+gecGAn8JdxK7d1aGZ2DfA7YCNwJtz9vwn6uVPuezrP/Uwndb+j0QQF2CyCB/QX3H1OmCMWAX2BtcCX3P1Uo+2kS7IXEZHGpUs3joiInIeSvYhIBlCyFxHJAEr2IiIZQMleRCQDKNmLiGQAJXsRkQygZC8ikgH+Pz7Rpbd+YcuVAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "00"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "n_epochs, lf, params, optimizer, sched = get_hparams(tr_dl, model, metric_fc, lr=8e-5, n_epochs=4)\n",
    "unfreeze_resnet_to(model, 3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=4.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=164.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=373.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Saved best model ep 0 with f score : 0.6813904524741098\n",
      "Ep 0: Train loss 2.2102 | Val f score 0.68 with thresh 0.70, train f score 0.7800 with thresh 0.60\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=164.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=373.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Saved best model ep 1 with f score : 0.6824626754703321\n",
      "Ep 1: Train loss 2.1734 | Val f score 0.68 with thresh 0.70, train f score 0.7690 with thresh 0.60\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=164.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=373.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Saved best model ep 2 with f score : 0.6835254539227448\n",
      "Ep 2: Train loss 1.6577 | Val f score 0.68 with thresh 0.70, train f score 0.7955 with thresh 0.60\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=164.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=373.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Saved best model ep 3 with f score : 0.6848199293332125\n",
      "Ep 3: Train loss 1.3072 | Val f score 0.68 with thresh 0.70, train f score 0.8194 with thresh 0.60\n",
      "\r"
     ]
    }
   ],
   "source": [
    "best_thr_score, thr_score_hist, losses = train(model, optimizer, lf, sched, metric_fc, tr_dl, val_dl,\n",
    "                                               n_epochs, train_df, val_df, train_tfms, val_tfms, \n",
    "                                               save_path=save_path, \n",
    "                                               prev_best_info=best_thr_score, info_history=thr_score_hist,\n",
    "                                               ep_start=ep_start)\n",
    "loss_hist.append(losses)\n",
    "ep_start += n_epochs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "n_epochs, lf, params, optimizer, sched = get_hparams(tr_dl, model, metric_fc, lr=6e-6, n_epochs=3)\n",
    "unfreeze_resnet_to(model, 2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=3.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=164.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=373.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Saved best model ep 0 with f score : 0.6840672685160704\n",
      "Ep 0: Train loss 1.2826 | Val f score 0.68 with thresh 0.70, train f score 0.8173 with thresh 0.60\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=164.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=373.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Saved best model ep 1 with f score : 0.6845468000185544\n",
      "Ep 1: Train loss 1.2622 | Val f score 0.68 with thresh 0.70, train f score 0.8204 with thresh 0.60\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=164.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=373.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Saved best model ep 2 with f score : 0.6848298176747182\n",
      "Ep 2: Train loss 1.2199 | Val f score 0.68 with thresh 0.70, train f score 0.8199 with thresh 0.60\n",
      "\r"
     ]
    }
   ],
   "source": [
    "best_thr_score, thr_score_hist, losses = train(model, optimizer, lf, sched, metric_fc, tr_dl, val_dl,\n",
    "                                               n_epochs, train_df, val_df, train_tfms, val_tfms, \n",
    "                                               save_path=save_path, \n",
    "                                               prev_best_info=best_thr_score, info_history=thr_score_hist,\n",
    "                                               ep_start=ep_start)\n",
    "loss_hist.append(losses)\n",
    "ep_start += n_epochs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "n_epochs, lf, params, optimizer, sched = get_hparams(tr_dl, model, metric_fc, lr=1e-6, n_epochs=2)\n",
    "unfreeze(model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=2.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=164.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=373.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Saved best model ep 0 with f score : 0.6853407440391863\n",
      "Ep 0: Train loss 1.1990 | Val f score 0.69 with thresh 0.70, train f score 0.8228 with thresh 0.60\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=164.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=373.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Ep 1: Train loss 1.2221 | Val f score 0.68 with thresh 0.70, train f score 0.8219 with thresh 0.60\n",
      "\r"
     ]
    }
   ],
   "source": [
    "best_thr_score, thr_score_hist, losses = train(model, optimizer, lf, sched, metric_fc, tr_dl, val_dl,\n",
    "                                               n_epochs, train_df, val_df, train_tfms, val_tfms, \n",
    "                                               save_path=save_path, \n",
    "                                               prev_best_info=best_thr_score, info_history=thr_score_hist,\n",
    "                                               ep_start=ep_start)\n",
    "loss_hist.append(losses)\n",
    "ep_start += n_epochs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<All keys matched successfully>"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.load_state_dict(torch.load('data/tests_model_image/jason_bourne_best.pth'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value=''), FloatProgress(value=0.0, max=159.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "model.eval()\n",
    "with torch.no_grad():\n",
    "    pbar = tqdm(tr_test_dl, leave=False)\n",
    "    embs = []\n",
    "    for imgs, _ in pbar:\n",
    "        imgs = val_tfms(imgs).to('cuda')\n",
    "        feature = model(imgs)\n",
    "        embs.append(feature)\n",
    "    embs = F.normalize(torch.cat(embs, 0))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Thresholds'), FloatProgress(value=0.0, max=10.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "scores = compute_f1(embs, df['label_group'], np.linspace(0.2, 1, 10))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "({0.2: 0.40398369258130423,\n",
       "  0.2888888888888889: 0.3920361139263555,\n",
       "  0.37777777777777777: 0.37773697843174286,\n",
       "  0.4666666666666667: 0.3573378338520364,\n",
       "  0.5555555555555556: 0.31357675637005644,\n",
       "  0.6444444444444445: 0.24473099438605053,\n",
       "  0.7333333333333334: 0.1622613157667755,\n",
       "  0.8222222222222222: 0.08796155705301248,\n",
       "  0.9111111111111112: 0.048940642207343905,\n",
       "  1.0: 0.03878723601529895},\n",
       " 0.2,\n",
       " 0.40398369258130423)"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "scores"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "***Embeddings normalization is not done in the model but in the arcface metric***"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "vision_model = 'resnet50'\n",
    "model = timm.create_model(vision_model, pretrained=True, num_classes=0).to('cuda')\n",
    "train_tfms, val_tfms = get_tfms()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Full train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "from imports import *\n",
    "from utils import load_data\n",
    "from image_train.data import create_dl, ImageDS\n",
    "from image_train.model import EMBRes\n",
    "from arcface import ArcMarginProduct, compute_centers\n",
    "from image_train.train import *\n",
    "import matplotlib.pyplot as plt\n",
    "np.random.seed(1337)\n",
    "device = torch.device('cuda')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "df, train_df, val_df, train_labels, val_labels = load_data(train_perc=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "# creating dataloaders\n",
    "small_images_dir = 'data/small_train_images/'\n",
    "\n",
    "tr_dl = create_dl(train_df, small_images_dir)\n",
    "tr_test_dl = create_dl(train_df, small_images_dir, shuffle=False)\n",
    "full_dl = create_dl(df, small_images_dir, shuffle=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "***Embeddings normalization is not done in the model but in the arcface metric***"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "vision_model = 'resnet50'\n",
    "model = timm.create_model(vision_model, pretrained=True, num_classes=0).to('cuda')\n",
    "train_tfms, val_tfms = get_tfms()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "#centers = compute_centers(tr_test_dl, model, val_tfms, train_df)\n",
    "#torch.save(centers, 'centers_full.pth')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "centers = torch.load('centers_full.pth')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Using center as wieghts\n"
     ]
    }
   ],
   "source": [
    "metric_fc = ArcMarginProduct(2048, train_df['label_group'].nunique(), \n",
    "                             s=30, m=0.5, easy_margin=False, centers=centers).to(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "freeze(model)\n",
    "unfreeze_resnet_to(model, 4)\n",
    "#freeze(metric_fc)\n",
    "#freeze_resnet_to()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "n_epochs, lf, params, optimizer, sched = get_hparams(tr_dl, model, metric_fc, lr=5e-4, n_epochs=5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "loss_hist = []\n",
    "best_thr_score={'train': {'thr': None, 'f1': None}}\n",
    "thr_score_hist=[]\n",
    "ep_start = 0\n",
    "save_path = 'data/tests_model_image/1april/resnet50_full'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=5.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=536.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Thresholds', max=10.0, style=ProgressStyle(description_wi…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Checkpoint : saved model to data/tests_model_image/1april/resnet50_full_ep_0.pth\n",
      "Ep 0: Loss 10.1480 | F score 0.6212 with thresh 0.73\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=536.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Thresholds', max=5.0, style=ProgressStyle(description_wid…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Ep 1: Loss 9.5412 | F score 0.6751 with thresh 0.78\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=536.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Thresholds', max=5.0, style=ProgressStyle(description_wid…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Checkpoint : saved model to data/tests_model_image/1april/resnet50_full_ep_2.pth\n",
      "Ep 2: Loss 7.3100 | F score 0.7259 with thresh 0.83\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=536.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Thresholds', max=5.0, style=ProgressStyle(description_wid…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Ep 3: Loss 5.1566 | F score 0.7665 with thresh 0.88\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=536.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Thresholds', max=5.0, style=ProgressStyle(description_wid…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Checkpoint : saved model to data/tests_model_image/1april/resnet50_full_ep_4.pth\n",
      "Ep 4: Loss 3.7504 | F score 0.8052 with thresh 0.88\n",
      "\r"
     ]
    }
   ],
   "source": [
    "best_thr_score, thr_score_hist, losses = train_full_data(model, optimizer, lf, sched, metric_fc, tr_dl,\n",
    "                                               n_epochs, train_df, train_tfms, val_tfms, \n",
    "                                               save_path=save_path, \n",
    "                                               prev_best_info=best_thr_score, info_history=thr_score_hist,\n",
    "                                               ep_start=ep_start)\n",
    "loss_hist.append(losses)\n",
    "ep_start += n_epochs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "n_epochs, lf, params, optimizer, sched = get_hparams(tr_dl, model, metric_fc, lr=9e-5, n_epochs=4)\n",
    "unfreeze_resnet_to(model, 3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=4.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=536.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Thresholds', max=5.0, style=ProgressStyle(description_wid…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Ep 5: Loss 3.6675 | F score 0.8121 with thresh 0.88\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=536.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Thresholds', max=5.0, style=ProgressStyle(description_wid…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Checkpoint : saved model to data/tests_model_image/1april/resnet50_full_ep_6.pth\n",
      "Ep 6: Loss 3.7566 | F score 0.8147 with thresh 0.88\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=536.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Thresholds', max=5.0, style=ProgressStyle(description_wid…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Ep 7: Loss 3.0844 | F score 0.8391 with thresh 0.88\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=536.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Thresholds', max=5.0, style=ProgressStyle(description_wid…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Checkpoint : saved model to data/tests_model_image/1april/resnet50_full_ep_8.pth\n",
      "Ep 8: Loss 2.5585 | F score 0.8615 with thresh 0.88\n",
      "\r"
     ]
    }
   ],
   "source": [
    "best_thr_score, thr_score_hist, losses = train_full_data(model, optimizer, lf, sched, metric_fc, tr_dl,\n",
    "                                               n_epochs, train_df, train_tfms, val_tfms, \n",
    "                                               save_path=save_path, \n",
    "                                               prev_best_info=best_thr_score, info_history=thr_score_hist,\n",
    "                                               ep_start=ep_start)\n",
    "loss_hist.append(losses)\n",
    "ep_start += n_epochs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "n_epochs, lf, params, optimizer, sched = get_hparams(tr_dl, model, metric_fc, lr=3e-5, n_epochs=3)\n",
    "unfreeze_resnet_to(model, 2)thresh"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=3.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=536.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Thresholds', max=5.0, style=ProgressStyle(description_wid…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Ep 9: Loss 2.5297 | F score 0.8629 with thresh 0.88\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=536.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Thresholds', max=5.0, style=ProgressStyle(description_wid…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Checkpoint : saved model to data/tests_model_image/1april/resnet50_full_ep_10.pth\n",
      "Ep 10: Loss 2.4658 | F score 0.8651 with thresh 0.88\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=536.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Thresholds', max=5.0, style=ProgressStyle(description_wid…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Ep 11: Loss 2.2163 | F score 0.8768 with thresh 0.93\n",
      "\r"
     ]
    }
   ],
   "source": [
    "best_thr_score, thr_score_hist, losses = train_full_data(model, optimizer, lf, sched, metric_fc, tr_dl,\n",
    "                                               n_epochs, train_df, train_tfms, val_tfms, \n",
    "                                               save_path=save_path, \n",
    "                                               prev_best_info=best_thr_score, info_history=thr_score_hist,\n",
    "                                               ep_start=ep_start)\n",
    "loss_hist.append(losses)\n",
    "ep_start += n_epochs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "n_epochs, lf, params, optimizer, sched = get_hparams(tr_dl, model, metric_fc, lr=7e-6, n_epochs=2)\n",
    "unfreeze(model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=2.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=536.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Thresholds', max=5.0, style=ProgressStyle(description_wid…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Checkpoint : saved model to data/tests_model_image/1april/resnet50_full_ep_12.pth\n",
      "Ep 12: Loss 2.1807 | F score 0.8799 with thresh 0.88\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=536.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Thresholds', max=5.0, style=ProgressStyle(description_wid…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Ep 13: Loss 2.1233 | F score 0.8827 with thresh 0.93\n",
      "\r"
     ]
    }
   ],
   "source": [
    "best_thr_score, thr_score_hist, losses = train_full_data(model, optimizer, lf, sched, metric_fc, tr_dl,\n",
    "                                               n_epochs, train_df, train_tfms, val_tfms, \n",
    "                                               save_path=save_path, \n",
    "                                               prev_best_info=best_thr_score, info_history=thr_score_hist,\n",
    "                                               ep_start=ep_start)\n",
    "loss_hist.append(losses)\n",
    "ep_start += n_epochs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "torch.save(model.state_dict(), save_path + 'last.pth')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "ename": "FileNotFoundError",
     "evalue": "[Errno 2] No such file or directory: 'data/tests_model_image/1april/resnet50_fullbest.pth'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mFileNotFoundError\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-18-032eb2b78051>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mmodel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mload_state_dict\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mload\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msave_path\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0;34m'best.pth'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m~/anaconda3/lib/python3.7/site-packages/torch/serialization.py\u001b[0m in \u001b[0;36mload\u001b[0;34m(f, map_location, pickle_module, **pickle_load_args)\u001b[0m\n\u001b[1;32m    579\u001b[0m         \u001b[0mpickle_load_args\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'encoding'\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m'utf-8'\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    580\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 581\u001b[0;31m     \u001b[0;32mwith\u001b[0m \u001b[0m_open_file_like\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mf\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'rb'\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0mopened_file\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    582\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0m_is_zipfile\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mopened_file\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    583\u001b[0m             \u001b[0;31m# The zipfile reader is going to advance the current file position.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.7/site-packages/torch/serialization.py\u001b[0m in \u001b[0;36m_open_file_like\u001b[0;34m(name_or_buffer, mode)\u001b[0m\n\u001b[1;32m    228\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0m_open_file_like\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mname_or_buffer\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmode\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    229\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0m_is_path\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mname_or_buffer\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 230\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0m_open_file\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mname_or_buffer\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmode\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    231\u001b[0m     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    232\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0;34m'w'\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mmode\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.7/site-packages/torch/serialization.py\u001b[0m in \u001b[0;36m__init__\u001b[0;34m(self, name, mode)\u001b[0m\n\u001b[1;32m    209\u001b[0m \u001b[0;32mclass\u001b[0m \u001b[0m_open_file\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0m_opener\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    210\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m__init__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mname\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmode\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 211\u001b[0;31m         \u001b[0msuper\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0m_open_file\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__init__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mopen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mname\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmode\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    212\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    213\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m__exit__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mFileNotFoundError\u001b[0m: [Errno 2] No such file or directory: 'data/tests_model_image/1april/resnet50_fullbest.pth'"
     ]
    }
   ],
   "source": [
    "model.load_state_dict(torch.load(save_path + '§1.pth'))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Vision transformer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from imports import *\n",
    "from utils import load_data\n",
    "from image_train.data import create_dl, ImageDS\n",
    "from image_train.model import EMBRes\n",
    "from arcface import ArcMarginProduct, compute_centers\n",
    "from image_train.train import *\n",
    "import matplotlib.pyplot as plt\n",
    "np.random.seed(1337)\n",
    "device = torch.device('cuda')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "df, train_df, val_df, train_labels, val_labels = load_data(train_perc=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# creating dataloaders\n",
    "small_images_dir = 'data/small_train_images/'\n",
    "\n",
    "tr_dl = create_dl(train_df, small_images_dir)\n",
    "tr_test_dl = create_dl(train_df, small_images_dir, shuffle=False)\n",
    "full_dl = create_dl(df, small_images_dir, shuffle=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "***Embeddings normalization is not done in the model but in the arcface metric***"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "vision_model = 'resnet50'\n",
    "model = timm.create_model(vision_model, pretrained=True, num_classes=0).to('cuda')\n",
    "train_tfms, val_tfms = get_tfms()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "da45184e3ee841c69f8e7c71042a3eed",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=536.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "centers = compute_centers(tr_test_dl, model, val_tfms, train_df)\n",
    "torch.save(centers, 'centers_full_effb3.pth')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "centers = torch.load('centers_full.pth')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Using center as wieghts\n"
     ]
    }
   ],
   "source": [
    "metric_fc = ArcMarginProduct(2048, train_df['label_group'].nunique(), \n",
    "                             s=30, m=0.5, easy_margin=False, centers=centers).to(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "n_epochs, lf, params, optimizer, sched = get_hparams(tr_dl, model, metric_fc, lr=5e-4, n_epochs=30)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "loss_hist = []\n",
    "best_thr_score={'train': {'thr': None, 'f1': None}}\n",
    "thr_score_hist=[]\n",
    "ep_start = 0\n",
    "save_path = 'data/tests_model_image/1april/model_full_nofrezze'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=30.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=536.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Thresholds', max=10.0, style=ProgressStyle(description_wi…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Checkpoint : saved model to data/tests_model_image/1april/model_full_nofrezze_ep_0.pth\n",
      "Ep 0: Loss 9.5217 | F score 0.6199 with thresh 0.64\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=536.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Thresholds', max=5.0, style=ProgressStyle(description_wid…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Ep 1: Loss 8.5159 | F score 0.6520 with thresh 0.69\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=536.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Thresholds', max=5.0, style=ProgressStyle(description_wid…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Checkpoint : saved model to data/tests_model_image/1april/model_full_nofrezze_ep_2.pth\n",
      "Ep 2: Loss 7.8212 | F score 0.6752 with thresh 0.74\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=536.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Thresholds', max=5.0, style=ProgressStyle(description_wid…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Ep 3: Loss 7.3146 | F score 0.7029 with thresh 0.74\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=536.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Thresholds', max=5.0, style=ProgressStyle(description_wid…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Checkpoint : saved model to data/tests_model_image/1april/model_full_nofrezze_ep_4.pth\n",
      "Ep 4: Loss 7.0361 | F score 0.7220 with thresh 0.79\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=536.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Thresholds', max=5.0, style=ProgressStyle(description_wid…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Ep 5: Loss 6.7008 | F score 0.7399 with thresh 0.84\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=536.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Thresholds', max=5.0, style=ProgressStyle(description_wid…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Checkpoint : saved model to data/tests_model_image/1april/model_full_nofrezze_ep_6.pth\n",
      "Ep 6: Loss 6.3737 | F score 0.7545 with thresh 0.84\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=536.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Thresholds', max=5.0, style=ProgressStyle(description_wid…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Ep 7: Loss 5.9146 | F score 0.7735 with thresh 0.89\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=536.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Thresholds', max=5.0, style=ProgressStyle(description_wid…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Checkpoint : saved model to data/tests_model_image/1april/model_full_nofrezze_ep_8.pth\n",
      "Ep 8: Loss 5.3383 | F score 0.7922 with thresh 0.89\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=536.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Thresholds', max=5.0, style=ProgressStyle(description_wid…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Ep 9: Loss 4.6596 | F score 0.8113 with thresh 0.94\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=536.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Thresholds', max=5.0, style=ProgressStyle(description_wid…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Checkpoint : saved model to data/tests_model_image/1april/model_full_nofrezze_ep_10.pth\n",
      "Ep 10: Loss 4.0444 | F score 0.8324 with thresh 0.94\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=536.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Thresholds', max=5.0, style=ProgressStyle(description_wid…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Ep 11: Loss 3.5480 | F score 0.8483 with thresh 0.94\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=536.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Thresholds', max=5.0, style=ProgressStyle(description_wid…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Checkpoint : saved model to data/tests_model_image/1april/model_full_nofrezze_ep_12.pth\n",
      "Ep 12: Loss 3.0807 | F score 0.8652 with thresh 0.94\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=536.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Thresholds', max=5.0, style=ProgressStyle(description_wid…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Ep 13: Loss 2.7150 | F score 0.8766 with thresh 0.94\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=536.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Thresholds', max=5.0, style=ProgressStyle(description_wid…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Checkpoint : saved model to data/tests_model_image/1april/model_full_nofrezze_ep_14.pth\n",
      "Ep 14: Loss 2.2196 | F score 0.8947 with thresh 0.94\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=536.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Thresholds', max=5.0, style=ProgressStyle(description_wid…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Ep 15: Loss 1.8490 | F score 0.9097 with thresh 0.94\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=536.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Thresholds', max=5.0, style=ProgressStyle(description_wid…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Checkpoint : saved model to data/tests_model_image/1april/model_full_nofrezze_ep_16.pth\n",
      "Ep 16: Loss 1.5134 | F score 0.9231 with thresh 0.99\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=536.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Thresholds', max=5.0, style=ProgressStyle(description_wid…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Ep 17: Loss 1.1863 | F score 0.9374 with thresh 0.99\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=536.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Thresholds', max=5.0, style=ProgressStyle(description_wid…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Checkpoint : saved model to data/tests_model_image/1april/model_full_nofrezze_ep_18.pth\n",
      "Ep 18: Loss 0.9500 | F score 0.9476 with thresh 0.99\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=536.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Thresholds', max=5.0, style=ProgressStyle(description_wid…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Ep 19: Loss 0.7685 | F score 0.9559 with thresh 0.99\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=536.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Thresholds', max=5.0, style=ProgressStyle(description_wid…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Checkpoint : saved model to data/tests_model_image/1april/model_full_nofrezze_ep_20.pth\n",
      "Ep 20: Loss 0.6183 | F score 0.9633 with thresh 0.99\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=536.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Thresholds', max=5.0, style=ProgressStyle(description_wid…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Ep 21: Loss 0.5042 | F score 0.9686 with thresh 0.99\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=536.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Thresholds', max=5.0, style=ProgressStyle(description_wid…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Checkpoint : saved model to data/tests_model_image/1april/model_full_nofrezze_ep_22.pth\n",
      "Ep 22: Loss 0.4358 | F score 0.9719 with thresh 0.94\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=536.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Thresholds', max=5.0, style=ProgressStyle(description_wid…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Ep 23: Loss 0.3723 | F score 0.9756 with thresh 0.94\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=536.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Thresholds', max=5.0, style=ProgressStyle(description_wid…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Checkpoint : saved model to data/tests_model_image/1april/model_full_nofrezze_ep_24.pth\n",
      "Ep 24: Loss 0.3357 | F score 0.9782 with thresh 0.99\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=536.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Thresholds', max=5.0, style=ProgressStyle(description_wid…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Ep 25: Loss 0.2992 | F score 0.9794 with thresh 0.99\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=536.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Thresholds', max=5.0, style=ProgressStyle(description_wid…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Checkpoint : saved model to data/tests_model_image/1april/model_full_nofrezze_ep_26.pth\n",
      "Ep 26: Loss 0.2738 | F score 0.9809 with thresh 0.94\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=536.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Thresholds', max=5.0, style=ProgressStyle(description_wid…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Ep 27: Loss 0.2535 | F score 0.9824 with thresh 0.99\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=536.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Thresholds', max=5.0, style=ProgressStyle(description_wid…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Checkpoint : saved model to data/tests_model_image/1april/model_full_nofrezze_ep_28.pth\n",
      "Ep 28: Loss 0.2429 | F score 0.9829 with thresh 0.99\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=536.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Thresholds', max=5.0, style=ProgressStyle(description_wid…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Ep 29: Loss 0.2358 | F score 0.9830 with thresh 0.99\n",
      "\r"
     ]
    }
   ],
   "source": [
    "best_thr_score, thr_score_hist, losses = train_full_data(model, optimizer, lf, sched, metric_fc, tr_dl,\n",
    "                                               n_epochs, train_df, train_tfms, val_tfms, \n",
    "                                               save_path=save_path, \n",
    "                                               prev_best_info=best_thr_score, info_history=thr_score_hist,\n",
    "                                               ep_start=ep_start)\n",
    "loss_hist.append(losses)\n",
    "ep_start += n_epochs"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Texts"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### train with freeze"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "from utils import compute_f1, load_data\n",
    "from text_train.imports import *\n",
    "from text_train.data import TextDS, text_to_device\n",
    "from arcface import ArcMarginProduct\n",
    "from text_train.model import EMBBert\n",
    "from text_train.train import *\n",
    "np.random.seed(1337)\n",
    "device = torch.device('cuda')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from imports import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def plot_hist(history) : \n",
    "    train_scores = [info['train']['f1'] for info in history]\n",
    "    train_thr = [info['train']['thr'] for info in history]\n",
    "    val_scores = [info['val']['f1'] for info in history]\n",
    "    val_thr = [info['val']['thr'] for info in history]\n",
    "    x = range(len(train_scores))\n",
    "    plt.plot(x, val_scores, label='val_score')\n",
    "    plt.plot(x, train_scores, label='train_score')\n",
    "    plt.legend()\n",
    "    plt.show()\n",
    "    plt.plot(x, val_thr, label='val_thr')\n",
    "    plt.plot(x, train_thr, label='train_thr')\n",
    "    plt.legend()\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "df, train_df, val_df, train_labels, val_labels = load_data(train_perc=0.3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "bs = 64\n",
    "language_model = 'bert-base-multilingual-uncased'\n",
    "tokenizer = BertTokenizer.from_pretrained(language_model)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "tr_ds = TextDS(train_df, tokenizer)\n",
    "tr_test_dl = DataLoader(tr_ds, batch_size = bs, shuffle = False, pin_memory = True)\n",
    "\n",
    "tr_ds = TextDS(train_df, tokenizer)\n",
    "tr_dl = DataLoader(tr_ds, batch_size = bs, shuffle = True, pin_memory = True)\n",
    "\n",
    "val_ds = TextDS(val_df, tokenizer)\n",
    "val_dl = DataLoader(val_ds, batch_size = bs, shuffle = False, pin_memory = True)\n",
    "\n",
    "full_ds = TextDS(df, tokenizer)\n",
    "full_dl = DataLoader(full_ds, batch_size = bs, shuffle = False, pin_memory = True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "torch.save(tr_test_dl, 'data/text_models/tr_test_dl_0.3.pth')\n",
    "torch.save(tr_dl, 'data/text_models/tr_dl_0.3.pth')\n",
    "torch.save(val_dl, 'data/text_models/val_dl_0.3.pth')\n",
    "torch.save(full_dl, 'data/text_models/full_dl_0.3.pth')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "tr_dl = torch.load('data/text_models/tr_dl_0.3.pth')\n",
    "val_dl = torch.load('data/text_models/val_dl_0.3.pth')\n",
    "tr_test_dl = torch.load('data/text_models/tr_test_dl_0.3.pth')\n",
    "full_dl = torch.load('data/text_models/full_dl_0.3.pth')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# model\n",
    "model = EMBBert(language_model).to(device)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "centers = compute_centers(tr_test_dl, model, train_df)\n",
    "torch.save(centers, 'data/text_models/centers_0.3.pth')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "centers = torch.load('data/text_models/centers_0.3.pth')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Using center as wieghts\n"
     ]
    }
   ],
   "source": [
    "metric_fc = ArcMarginProduct(512, train_df['label_group'].nunique(), s=30, m=0.5,\n",
    "                             easy_margin=False, centers=centers, device=device).to(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Unfroze 5 parameters\n"
     ]
    }
   ],
   "source": [
    "n_epochs, lf, params, optimizer, sched = get_hparams(tr_dl, model, metric_fc, lr=5e-4, n_epochs=5)\n",
    "freeze_bert_embeddings(model)\n",
    "freeze(model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Unfroze 32 parameters\n"
     ]
    }
   ],
   "source": [
    "unfreeze_bert_to(model, 10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "loss_hist = []\n",
    "best_thr_score={'val': {'thr': None, 'f1': None}, 'train': {'thr': None, 'f1': None}}\n",
    "thr_score_hist=[]\n",
    "ep_start = 0\n",
    "save_path = 'data/text_models/test_3n_epochs'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=5.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=164.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Thresholds', max=10.0, style=ProgressStyle(description_wi…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Checkpoint : saved model to data/text_models/test_3ap_ep_0.pth\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=373.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Thresholds', max=10.0, style=ProgressStyle(description_wi…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Saved best model ep 0 with f score : 0.6001494931589628\n",
      "Ep 0: Train loss 10.6992 | Val f score 0.6001 with thresh 0.56, train f score 0.5741 with thresh 0.64\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=164.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Thresholds', max=5.0, style=ProgressStyle(description_wid…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=373.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Thresholds', max=5.0, style=ProgressStyle(description_wid…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Saved best model ep 1 with f score : 0.6277116685801583\n",
      "Ep 1: Train loss 9.3118 | Val f score 0.6277 with thresh 0.66, train f score 0.6530 with thresh 0.74\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=164.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Thresholds', max=5.0, style=ProgressStyle(description_wid…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Checkpoint : saved model to data/text_models/test_3ap_ep_2.pth\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=373.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Thresholds', max=5.0, style=ProgressStyle(description_wid…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Saved best model ep 2 with f score : 0.6397538938208261\n",
      "Ep 2: Train loss 6.8955 | Val f score 0.6398 with thresh 0.76, train f score 0.7393 with thresh 0.84\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=164.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Thresholds', max=5.0, style=ProgressStyle(description_wid…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=373.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Thresholds', max=5.0, style=ProgressStyle(description_wid…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Saved best model ep 3 with f score : 0.6474627951197889\n",
      "Ep 3: Train loss 4.8839 | Val f score 0.6475 with thresh 0.76, train f score 0.7989 with thresh 0.89\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=164.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Thresholds', max=5.0, style=ProgressStyle(description_wid…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Checkpoint : saved model to data/text_models/test_3ap_ep_4.pth\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=373.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Thresholds', max=5.0, style=ProgressStyle(description_wid…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Saved best model ep 4 with f score : 0.6481900406991484\n",
      "Ep 4: Train loss 3.8192 | Val f score 0.6482 with thresh 0.76, train f score 0.8346 with thresh 0.89\n",
      "\r"
     ]
    }
   ],
   "source": [
    "best_thr_score, thr_score_hist, losses = train(model, optimizer, lf, sched, metric_fc, tr_dl, val_dl,\n",
    "                                               n_epochs, train_df, val_df, \n",
    "                                               save_path=save_path, \n",
    "                                               prev_best_info=best_thr_score, info_history=thr_score_hist,\n",
    "                                               ep_start=ep_start, device=device)\n",
    "loss_hist.append(losses)\n",
    "ep_start += n_epochs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Unfroze 5 parameters\n"
     ]
    }
   ],
   "source": [
    "n_epochs, lf, params, optimizer, sched = get_hparams(tr_dl, model, metric_fc, lr=1e-4, n_epochs=2)\n",
    "freeze_bert_embeddings(model)\n",
    "freeze(model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Unfroze 96 parameters\n"
     ]
    }
   ],
   "source": [
    "unfreeze_bert_to(model, 6)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=2.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=164.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Thresholds', max=5.0, style=ProgressStyle(description_wid…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=373.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Thresholds', max=5.0, style=ProgressStyle(description_wid…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Saved best model ep 5 with f score : 0.6587867599411823\n",
      "Ep 5: Train loss 3.7791 | Val f score 0.6588 with thresh 0.76, train f score 0.8426 with thresh 0.89\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=164.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Thresholds', max=5.0, style=ProgressStyle(description_wid…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Checkpoint : saved model to data/text_models/test_3ap_ep_6.pth\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=373.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Thresholds', max=5.0, style=ProgressStyle(description_wid…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Ep 6: Train loss 3.0978 | Val f score 0.6583 with thresh 0.76, train f score 0.8699 with thresh 0.89\n",
      "\r"
     ]
    }
   ],
   "source": [
    "best_thr_score, thr_score_hist, losses = train(model, optimizer, lf, sched, metric_fc, tr_dl, val_dl,\n",
    "                                               n_epochs, train_df, val_df, \n",
    "                                               save_path=save_path, \n",
    "                                               prev_best_info=best_thr_score, info_history=thr_score_hist,\n",
    "                                               ep_start=ep_start, device=device)\n",
    "loss_hist.append(losses)\n",
    "ep_start += n_epochs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Unfroze 5 parameters\n"
     ]
    }
   ],
   "source": [
    "n_epochs, lf, params, optimizer, sched = get_hparams(tr_dl, model, metric_fc, lr=1e-5, n_epochs=2)\n",
    "freeze_bert_embeddings(model)\n",
    "freeze(model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "unfreeze(model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=2.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=164.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Thresholds', max=5.0, style=ProgressStyle(description_wid…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=373.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Thresholds', max=5.0, style=ProgressStyle(description_wid…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Saved best model ep 7 with f score : 0.6611133421224951\n",
      "Ep 7: Train loss 2.7312 | Val f score 0.6611 with thresh 0.76, train f score 0.8799 with thresh 0.89\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=164.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Thresholds', max=5.0, style=ProgressStyle(description_wid…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Checkpoint : saved model to data/text_models/test_3ap_ep_8.pth\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=373.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Thresholds', max=5.0, style=ProgressStyle(description_wid…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Saved best model ep 8 with f score : 0.6618433112378985\n",
      "Ep 8: Train loss 2.5136 | Val f score 0.6618 with thresh 0.76, train f score 0.8905 with thresh 0.89\n",
      "\r"
     ]
    }
   ],
   "source": [
    "best_thr_score, thr_score_hist, losses = train(model, optimizer, lf, sched, metric_fc, tr_dl, val_dl,\n",
    "                                               n_epochs, train_df, val_df, \n",
    "                                               save_path=save_path, \n",
    "                                               prev_best_info=best_thr_score, info_history=thr_score_hist,\n",
    "                                               ep_start=ep_start, device=device)\n",
    "loss_hist.append(losses)\n",
    "ep_start += n_epochs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXoAAAD4CAYAAADiry33AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAgAElEQVR4nO3deXxU1f3/8dcne8KakLAlYKIiyCZoBC2KuAHiAoJVXHCH9lu0iv210moLX2ytXb+tlWpRcatCKShitbgjLmAJFmUThAAmhCWQEAjZZz6/P+4kmYQAE5jkTiaf5+ORx2TuPffmkxjfOZw5c46oKsYYY8JXhNsFGGOMaVoW9MYYE+Ys6I0xJsxZ0BtjTJizoDfGmDAX5XYB9SUnJ2t6errbZRhjTIuyevXqfaqa0tC5gIJeREYDfwYigWdU9bF6508B5gIpQAFwi6rm+s7dBjzsa/pLVX3hWF8rPT2drKysQMoyxhjjIyI7jnbuuEM3IhIJzAauAPoCN4pI33rNfg+8qKoDgVnAr33XJgEzgKHAEGCGiCSeyDdhjDHmxAQyRj8E2KKq2apaAcwHxtZr0xd43/f5h37nRwHvqmqBqhYC7wKjT75sY4wxgQok6FOBHL/nub5j/r4EJvg+vxZoJyKdArwWEZkiIlkikpWfnx9o7cYYYwIQyBi9NHCs/roJ/w94QkRuB5YDO4GqAK9FVecAcwAyMzOPOF9ZWUlubi5lZWUBlGtORlxcHGlpaURHR7tdijEmSAIJ+lygh9/zNCDPv4Gq5gHjAUSkLTBBVYtEJBcYUe/aZY0tMjc3l3bt2pGeno5IQ387TDCoKvv37yc3N5eMjAy3yzHGBEkgQzergF4ikiEiMcBEYIl/AxFJFpHqe/0UZwYOwNvASBFJ9L0IO9J3rFHKysro1KmThXwTExE6depk/3IyJswcN+hVtQq4ByegNwILVHW9iMwSkWt8zUYAm0RkM9AF+JXv2gLgEZw/FquAWb5jjWYh3zzs52xM+AloHr2qvgW8Ve/YL/w+XwgsPMq1c6nt4RtjjPF3eB/s3QB7NkBULGTeEfQvEXLvjDXGmLBUcRj2fu2E+t4NsGc97N0Ih/fWtkkbYkHfkrRt25bi4mK3yzDGNDdPJezfCnt9Qb5ng/N54Q5qJh1GxUPnPtDrcujcFzqfCV36QdsuTVKSBX0YqKqqIirK/lMa06xUoSi3bu987wbYtxk8FU4biYBOp0O3QXDWTdClrxPsiekQEdlspba4dPjfN9azIe9gUO/Zt3t7Zlzd75htHnzwQU455RR+8IMfADBz5kxEhOXLl1NYWEhlZSW//OUvGTu2/puGj7Rr1y5uuOEGDh48SFVVFU8++SQXXnghS5cu5Wc/+xkej4fk5GTef/99CgoKuPPOO8nOziYhIYE5c+YwcOBAZs6cSV5eHtu3byc5OZmXXnqJ6dOns2zZMsrLy5k6dSrf+973gvLzMabVKymoHUev7qnv3QjlflnUPs3pmZ9+qa+X3heSz4DoOPfq9mlxQe+WiRMncv/999cE/YIFC1i6dCnTpk2jffv27Nu3j/POO49rrrnmuDNXXnnlFUaNGsVDDz2Ex+OhpKSE/Px8Jk+ezPLly8nIyKCgwJmcNGPGDAYPHszixYv54IMPuPXWW1mzZg0Aq1ev5pNPPiE+Pp45c+bQoUMHVq1aRXl5OcOGDWPkyJE2H96Yxqgogfyva3vn1eFevLu2TVxHZ5hl4PW1gd75TIjv6F7dx9Higv54Pe+mMnjwYPbu3UteXh75+fkkJibSrVs3pk2bxvLly4mIiGDnzp3s2bOHrl27HvNe5557LnfeeSeVlZWMGzeOQYMGsWzZMoYPH14TzElJSQB88sknLFq0CIBLLrmE/fv3U1RUBMA111xDfHw8AO+88w5fffUVCxc6k5+Kior45ptvLOiNaYinCgqyj3xhtCCb2nH0OEjpDadd4htD7wud+0G7rtDCpiG3uKB303XXXcfChQvZvXs3EydO5OWXXyY/P5/Vq1cTHR1Nenp6QG82Gj58OMuXL+fNN99k0qRJ/PjHP6Zjx44N/ktA9YgVIWratWnTpk67v/zlL4waNeokvkNjwoynEgq2Ob30fZsgf5Pzef5m8JQ7bSQCkk47speelNGs4+hNyYK+ESZOnMjkyZPZt28fH330EQsWLKBz585ER0fz4YcfsmPHUZeDrmPHjh2kpqYyefJkDh8+zBdffMFDDz3E1KlT2bZtW83QTVJSEsOHD+fll1/m5z//OcuWLSM5OZn27dsfcc9Ro0bx5JNPcskllxAdHc3mzZtJTU2t88fAmLBVVe7MdMn/2i/MN8H+LeCtrG3XsSek9IFTRzi98y7V4+jxblXeLCzoG6Ffv34cOnSI1NRUunXrxs0338zVV19NZmYmgwYNok+fPgHdZ9myZfzud78jOjqatm3b8uKLL5KSksKcOXMYP348Xq+Xzp078+677zJz5kzuuOMOBg4cSEJCAi+80PC+LXfffTfbt2/n7LPPRlVJSUlh8eLFwfz2jXFfRQns/8bpked/XRvoBdmgHqeNRDizWlL6QO/RzmNKbyfQY1pnx0caGhpwU2ZmptbfYWrjxo2ceeaZLlXU+tjP27iuvLjeUIvv0X8uekSUM+SS0tv34Qv0TqeHfQ+9ISKyWlUzGzpnPXpjjHtKDzghXj/Ui/y2sYiMgU69oPvZzlz06lBPOhWiYtyrvQWxoG9Ca9euZdKkSXWOxcbG8vnnn7tUkTEuOby/7lBL9aP/tMWoOGd4pef5kHK7r4fexxmGibSoOhn202tCAwYMqJnzbkyrcnAXZD0LO1Y4oV6yr/ZcTNvaaYv+Qy4de4bNLJdQY0FvjAme3WthxWxYuxC8VZCWCX3G1IZ5Sh9on9ri5qG3dBb0xpiTowpb3oPP/gLbPoLoNpB5J5z3fWcc3bjOgt4Yc2Iqy+Crf8DKvzrDM+26waUznGV24xPdrs74saA3xjTO4X2w6llY9TQczocuA+Dav0G/8TYLJkQFsmesAQ4cOMBf//rXRl83ZswYDhw40AQVGdPM9n0Db9wP/9cPlj0K3QfDra/D9z+GsyZayIcw69EHqDroq1evrObxeIiMPPpMgbfeeuuo55rT8eo0pkGqsP0TWPEEbF4KkbFw1g1w3lRn4wzTIrS8oP/3dOeV/WDqOgCueOyYTaZPn87WrVsZNGhQzdIF3bp1Y82aNWzYsIFx48aRk5NDWVkZ9913H1OmTAEgPT2drKwsiouLueKKK7jgggv47LPPSE1N5fXXX69ZfbK+xx9/nKeeeoqoqCj69u3L/PnzKS4u5t577yUrKwsRYcaMGUyYMIF58+bx6KOPoqpceeWV/OY3vwGcXa4eeOAB3n77bf7whz8QHx/PAw88QHFxMcnJyTz//PN069YtuD9LEx48lbD+NSfgd30JCZ3gogfh3LuhbWe3qzON1PKC3iWPPfYY69atY82aNSxbtowrr7ySdevW1SwDPHfuXJKSkigtLeXcc89lwoQJdOrUqc49vvnmG+bNm8fTTz/N9ddfz6JFi7jllluO+vW2bdtGbGxszdDPI488QocOHVi71vlDV1hYSF5eHg8++CCrV68mMTGRkSNHsnjxYsaNG8fhw4fp378/s2bNorKykosuuojXX3+dlJQU/vGPf/DQQw8xd67t2278lB6AL16Az/8GB3c670i96k/O0EwrXFYgXLS8oD9Oz7u5DBkypM5a748//jivvfYaADk5OXzzzTdHBH1GRgaDBg0C4JxzzmH79u1Hvf/AgQO5+eabGTduHOPGjQPgvffeY/78+TVtEhMTWb58OSNGjCAlJQWAm2++meXLlzNu3DgiIyOZMGECAJs2bWLdunVcfvnlgDOUY715U6NwB3z+FHzxIlQUQ/qFcOUfoddIiLCX8lq6gIJeREYDfwYigWdU9bF653sCLwAdfW2mq+pbIpIObAQ2+ZquVNXvB6d0d/kv/7ts2TLee+89VqxYQUJCAiNGjGhwXfrY2NiazyMjIyktLT3q/d98802WL1/OkiVLeOSRR1i/fj2qesSa9cdalC4uLq5mXF5V6devHytWrAj4ezStQG6WM/994xJn1cd+4+H8qdB9kNuVmSA67p9qEYkEZgNXAH2BG0Wkb71mDwMLVHUwMBHwn56yVVUH+T5abMi3a9eOQ4cONXiuqKiIxMREEhIS+Prrr1m5cuVJfS2v10tOTg4XX3wxv/3tbzlw4ADFxcWMHDmSJ554oqZdYWEhQ4cO5aOPPmLfvn14PB7mzZvHRRdddMQ9e/fuTX5+fk3QV1ZWsn79+pOq07RQXg9sWALPjoJnLoWtH8L598B9X8GEpy3kw1AgPfohwBZVzQYQkfnAWGCDXxsFqnfD6ADkBbPIUNCpUyeGDRtG//79iY+Pp0uXLjXnRo8ezVNPPcXAgQPp3bs355133kl9LY/Hwy233EJRURGqyrRp0+jYsSMPP/wwU6dOpX///kRGRjJjxgzGjx/Pr3/9ay6++GJUlTFjxjS4QXlMTAwLFy7khz/8IUVFRVRVVXH//ffTr587WzMaF1Qchv++7LzBqXCbs7bM6Mdg8C0Q287t6kwTOu569CJyHTBaVe/2PZ8EDFXVe/zadAPeARKBNsBlqrraN3SzHtgMHAQeVtWPj/X1bD1699nPO8wc3AX/mQNZc6HsAKRmwnfugT5X26qQYeRk16NvaPWh+n8dbgSeV9U/iMj5wEsi0h/YBfRU1f0icg6wWET6qerBegVOAaYA9OzZM4CSjDHHVX+BsTOvgvPvhZ5D3a7MNLNAgj4X6OH3PI0jh2buAkYDqOoKEYkDklV1L1DuO75aRLYCZwB1uuyqOgeYA06P/gS+jxZr6tSpfPrpp3WO3Xfffdxxxx0uVWRatOoFxlY8AdnLbIExAwQW9KuAXiKSAezEebH1pnptvgUuBZ4XkTOBOCBfRFKAAlX1iMipQC8g+0QKbWjGSTiYPXu22yXUEWpbS5oAVZbB2gVOD94WGDP1HDfoVbVKRO4B3saZOjlXVdeLyCwgS1WXAD8CnhaRaTjDOrerqorIcGCWiFQBHuD7qlrQ2CLj4uLYv38/nTp1CsuwDxWqyv79+4mLi3O7FHMslWXOVnuF252P/Vth3UJbYMwcVYvYHLyyspLc3NwG56ab4IqLiyMtLY3o6Gi3S2m9VKF4jy/Id9QGeuF2OLADDuZR52WyqDjIGO7Mf8+4yDb1aKVa/Obg0dHRdd6FakyLV17shHZDQV64A6rqvZmuXXdn79SMi5zHxFN8j+nQtouFuzmmFhH0xrQ4Xo/T8z4ixH2fH86v2z6mnRPanU6H0y+rDfHEdOjQA6JtOM2cOAt6Y05U6YGjB/mBHPBW1raVSOiQ5gR37yvqBnnHdEhIsl65aTIW9MYEorIUVj0Duatqw7ysqG6b+CRnSKXbWdB3rC/EfUMsHdIg0l73MO6woDfmWFTh63/B2z+DA99C0mmQlAFp59aGePWYeVwHt6s1pkEW9MYczd6vYemDzhuPOveF295wZrcY08JY0BtTX+kB+Og3zuYbsW3hit857y61dWFMC2W/ucZU83rgv3+H92dByX4453a45OfQptNxLzUmlFnQGwOQ8x9468ewaw30PB+ueNV5UdWYMGBBb1q3Q7vh3Rnw1XznTUkTnoX+E2yqowkrFvSmdaoqh5VPwvLfgacCLvwRXPCAMyZvTJixoDetz+Z3YOl0KNgKvcfAqF/ZEr4mrFnQm9Zj/1ZY+lP45m3o1AtuXgS9LnO7KmOanAW9CX/lh2D575212qPiYOQvYcj3bBlf02pY0JvwpQpfLYB3fwHFu2HQzc5mHO26HP9aY8KIBb0JT3n/hX8/CDmfQ/ezYeLLkNbgUt3GhD0LehNeDu9z3vD0xYvQJhnGzoazboKICLcrM8Y1FvQmPHgqYdWz8OGjUHnY2W3pop/YQmPGYEFvwkH2Mvj3dMjfCKddAqMfg5TebldlTMiwoDctV+EOeOdh2LjEWSp44ivOvHh7V6sxdVjQm5anogQ+/TN8+ieQCGfhsfPvse32jDkKC3rTcqjChtedXnxRDvS/Di6fBR1S3a7MmJBmQW9ahj3rnemS2z+GLgNg/Bw45TtuV2VMixDQnDMRGS0im0Rki4hMb+B8TxH5UET+KyJficgYv3M/9V23SURGBbN40wqUFDjLBz91AexZB1f+Eb73kYW8MY1w3B69iEQCs4HLgVxglYgsUdUNfs0eBhao6pMi0hd4C0j3fT4R6Ad0B94TkTNU1RPsb8SEGa8HvngB3n8Eyg5A5l1w8c8gIcntyoxpcQIZuhkCbFHVbAARmQ+MBfyDXoH2vs87AHm+z8cC81W1HNgmIlt891sRhNpNuPr2c3jrR7B7LZxyAVzxG+ja3+2qjGmxAgn6VCDH73kuMLRem5nAOyJyL9AGqF4SMBVYWe/aI145E5EpwBSAnj17BlK3CVfrX4NFd0PbrnDdc9DvWpsuacxJCmSMvqH/y7Te8xuB51U1DRgDvCQiEQFei6rOUdVMVc1MSUkJoCQTltYuhIV3Qdq58IMV0H+8hbwxQRBIjz4X6OH3PI3aoZlqdwGjAVR1hYjEAckBXmsMrJkHr/8AThkGN863nZ6MCaJAevSrgF4ikiEiMTgvri6p1+Zb4FIAETkTiAPyfe0mikisiGQAvYD/BKt4Eya+eBEW/w9kDIebFljIGxNkx+3Rq2qViNwDvA1EAnNVdb2IzAKyVHUJ8CPgaRGZhjM0c7uqKrBeRBbgvHBbBUy1GTemjlXPwpsPwOmXwQ1/h+h4tysyJuyIk8ehIzMzU7OystwuwzSHz/8G//4JnHEFXP8CRMW6XZExLZaIrFbVBjddsEW6jTs+e8IJ+T5XwfUvWsgb04Qs6E3z+/iP8M5D0HccfPd527vVmCZma92Y5rXsN7DsURjwXRj3FETar6AxTc3+LzPNQxU+/BUs/52ztd/YJyAi0u2qjGkVLOhN01OF92Y668effStc9Wfbw9WYZmRBb5qWKrz9EKyc7SxMNub3FvLGNDMLetN0VJ2ZNf+ZA0O/7+zlaksaGNPsLOhN0/B6nTdCrX7O2eZv5C8t5I1xiQW9CT6vB5b8ENb8HS54AC79hYW8MS6yoDfB5alyFif76h9w0XQYMd1C3hiXWdCb4PFUwWtTYN0iuPhhuOjHbldkjMGC3gSLpxIW3gkbl8Bl/wsX3O92RcYYHwt6c/KqyuGfd8CmN2HUr+H8H7hdkTHGjwW9OTmVZbBgEnzzjjNHfshktysyxtRjQW9OXGUpzL8Jtn4AV/0JMu9wuyJjTAMs6M2JqTgM8ybCto9h7GwYfIvbFRljjsKC3jRe+SF4+XrIWQnX/g3OusHtiowxx2BBbxqn7CC8fB3kZsH4p2HAdW5XZIw5Dgt6E7jSA/D38bDrS/juc9B3rNsVGWMCYEFvAlNSAC+Ngz0bnK3/+lzpdkXGmABZ0JvjO7wPXhwH+zbDxFfgjJFuV2SMaQQLenNsxXvhxbFQkA03zoPTL3W7ImNMIwW0A4SIjBaRTSKyRUSmN3D+/0Rkje9js4gc8Dvn8Tu3JJjFmyZ2aDc8fyUUboebFljIG9NCHbdHLyKRwGzgciAXWCUiS1R1Q3UbVZ3m1/5eYLDfLUpVdVDwSjbNomgnvHC1E/Y3L4T0YW5XZIw5QYH06IcAW1Q1W1UrgPnAsaZb3AjMC0ZxxiUHvoXnxzjDNpNes5A3poULJOhTgRy/57m+Y0cQkVOADOADv8NxIpIlIitFZNxRrpvia5OVn58fYOmmSRRuh+euhJJCuPV16DnU7YqMMScpkKBvaNcIPUrbicBCVfX4HeupqpnATcCfROS0I26mOkdVM1U1MyUlJYCSTJPYvxWeGwPlB+G21yHtHLcrMsYEQSBBnwv08HueBuQdpe1E6g3bqGqe7zEbWEbd8XsTKvI3OyFfVQa3/wu6238mY8JFIEG/CuglIhkiEoMT5kfMnhGR3kAisMLvWKKIxPo+TwaGARvqX2tctnejM7tGPXDbv6DrALcrMsYE0XFn3ahqlYjcA7wNRAJzVXW9iMwCslS1OvRvBOarqv+wzpnA30TEi/NH5TH/2TomBOxe58yTj4h0Qj6lt9sVGWOCTOrmsvsyMzM1KyvL7TJah7w1zrIGUfFw2xuQfLrbFRljTpCIrPa9HnoEe2dsa1W00wn5mLZw2xJIOtXtiowxTSSgd8aaMOP1wuLvQ1UFTFpsIW9MmLMefWu0cjZsWw5XP27DNca0Atajb212r4X3Z0Gfq+DsW92uxhjTDCzoW5PKUlh0N8QnOr15aei9cMaYcGNDN63JezMh/2u4ZRG06eR2NcaYZmI9+tZiy3vw+VMw9Ptw+mVuV2OMaUYW9K3B4X2w+AeQciZcNtPtaowxzcyGbsKdKrxxH5QWOkM20fFuV2SMaWbWow93X7wIX/8LLv2FrWFjTCtlQR/O9m+FpdMhYzicN9XtaowxLrGgD1eeSnh1MkTGwLinIML+UxvTWtkYfbj66LewczV893no0OCGYMaYVsK6eeHo25Xw8e/hrJug37VuV2OMcZkFfbgpOwivToEOPeCK37hdjTEmBNjQTbj594NQlAN3/Bvi2rtdjTEmBFiPPpysfw2+fAUu/H/Q8zy3qzHGhAgL+nBRtBPeuB9Sz4GLfuJ2NcaYEGJBHw6qNxLxVML4pyEy2u2KjDEhxMbow4H/RiKdTnO7GmNMiLEefUtnG4kYY47Dgr4lqyyFRZNtIxFjzDEFFPQiMlpENonIFhGZ3sD5/xORNb6PzSJywO/cbSLyje/jtmAW3+q9NxPyN8K4v9pGIsaYozruGL2IRAKzgcuBXGCViCxR1Q3VbVR1ml/7e4HBvs+TgBlAJqDAat+1hUH9Lloj20jEGBOgQHr0Q4AtqpqtqhXAfGDsMdrfCMzzfT4KeFdVC3zh/i4w+mQKNsDh/baRiDEmYIEEfSqQ4/c813fsCCJyCpABfNCYa0VkiohkiUhWfn5+IHW3Xqrwxg+djUQmPG0biRhjjiuQoG/oFT49StuJwEJV9TTmWlWdo6qZqpqZkpISQEmtmG0kYoxppECCPhfo4fc8Dcg7StuJ1A7bNPZaczy2kYgx5gQEEvSrgF4ikiEiMThhvqR+IxHpDSQCK/wOvw2MFJFEEUkERvqOmcayjUSMMSfouLNuVLVKRO7BCehIYK6qrheRWUCWqlaH/o3AfFVVv2sLROQRnD8WALNUtSC430IrYRuJGGNOkPjlckjIzMzUrKwst8sILd+uhOeugIET4don3a7GGBOCRGS1qmY2dM7+/R/qbCMRY8xJskXNQp1tJGKMOUnWow9ltpGIMSYILOhDlW0kYowJEgv6UOT1wuL/sY1EjDFBYWP0oWjlbNj2kW0kYowJCuvRhxrbSMQYE2QW9KHENhIxxjQBG7oJJdUbidyyyDYSMcYEjfXoQ4VtJGKMaSIW9KHANhIxxjQhG7pxm/9GIrcsso1EjDFBZz16t9lGIsaYJmZB7ybbSMQY0wws6N1iG4kYY5qJjdG7xTYSMcY0E+tGuuHbz+Hj38NZN0G/a92uxhgT5qxH39zKDjpDNraRiDHNotLjpaTCQ0lFFYfLPZRWeCiv8uDxKh5VvF58j+p3zHn0eBWvKh4veL1Klf/5mnP+x6hzvf89PfWv8XLE9enJbXhwdJ+g/wws6JubbSRiTIOqPF5KKj2UlHs4XFFFaYWHw+VVvpB2jpWUV3G4wglr57nHd01V7TUVzvPqe1V4vM3+vUQIREYIESJERgiRIkRGOo8R1c8jhIgI6hyLiWqaQRYL+uZUvZHI8J/YRiLmhJVVetiaX8ym3YfYtOcQm3cfori8CkEQgQhxAiRCBBEhovqYgIgg1LZxzte2qb7+yDbV96jXJsLva8qRX7PS4+VwudOb9u9VV4dzzbEKDxVVgQdyhECbmCgSYiNJiIkiISaSNjFRdEyIITWx9lhCTBRtYiKJj4mkTWztsZioCKL8g9j386oTzvWCOiICoiIiasLZCWq/0K55dH5mocSCvrnYRiKmkTxe5duCEjbtPsim3cVs2nOQTbsPsX1/CR6vAhAdKZyW0pbEhBgUxatQ5fXi9TjDAl4FVWe4QJU6z71KzXGt99zr10brHGu4DfWf+0QItUEcG0V8dCRtYiPpGB9N9w5xThDH+oLYP5xj/cPa/1rnWGxURMiFaSizoG8OtpGIOQZVZe+hcqeH7uulb9p9iG/2HqKs0unlikDPpATO6NKOMQO6cUaXdvTp2o705DZER4benAr1jT1HRogFcggIKOhFZDTwZyASeEZVH2ugzfXATECBL1X1Jt9xD7DW1+xbVb0mCHW3LLaRiPE5WFbJ5t2H+Hr3ITbvqX08UFJZ0yalXSy9u7Tj5qGn0LtLO3p3bUevLm1JiGk5/TIRISrSAj5UHPc3R0QigdnA5UAusEpElqjqBr82vYCfAsNUtVBEOvvdolRVBwW57pZj52rbSKQVKq/ysGVvcW2Y+3rreUVlNW3axkZxRpe2XNG/qy/Q23NGl7Z0ahvrYuUmHAXSRRgCbFHVbAARmQ+MBTb4tZkMzFbVQgBV3RvsQlukkgJYcBu07QLX/MU2EglDtePoTpA7wX6wwXH0czOS6N21XU0vPbVjvA1rmGYRSNCnAjl+z3OBofXanAEgIp/iDO/MVNWlvnNxIpIFVAGPqeri+l9ARKYAUwB69uzZqG8gZHm98OoUKN4Ddy6FhCS3KzL1aJ05zNTOm25oLrXXeZEzt7C0ppfe0sfRTesRSNA31OXQes+jgF7ACCAN+FhE+qvqAaCnquaJyKnAByKyVlW31rmZ6hxgDkBmZmb9e7dMH/8etrwLV/7RmWljGlRW6WHngVJyCkrILSz1fZRQUuGp9+YS/zengMfrrXkTy5GhXP157RtSjnwDTN3ZIY0VDuPopvUI5LcyF+jh9zwNyGugzUpVrQS2icgmnOBfpap5AKqaLSLLgMHAVsLZlvfhw0dh4A2Qeafb1biqospL3oFScgpLakI8p8D3WFhK/qHyOu1jIiPo3jGOtnFRdd5IEhEhRE7W3lkAAA20SURBVEVEEBtVfYyauc5Hznf2e6z3hhT/uc+1x/Dd/1hzqJ12XdrH0btLOxtHNy1KIEG/CuglIhnATmAicFO9NouBG4HnRSQZZygnW0QSgRJVLfcdHwb8NmjVh6IDObDobuh8Jlz1f2E/Ll/p8bK7qMyvR+4EeK4v2HcfLEP9es6REUL3jnH0SEzg4t4p9EhMIC0p3nlMTKBzu1giIsL7Z2ZMcztu0KtqlYjcA7yNM/4+V1XXi8gsIEtVl/jOjRSRDYAH+LGq7heR7wB/ExEvzgJqj/nP1gk7VRXwz9ud+fLXvwQxbdyu6KR5vMrug2XkFtQGeHWPPLewlF1FpUe8QaZbh3jSEuP5zmnJpCXG0yMpoeaxS7tYomy82phmJaqhNSSemZmpWVlZbpdxYt76MfxnDlz/IvQd63Y1AfF6nTfrOD3xEnIL/IdZSsk7UEqVX5KLQJd2cXUDPNF5TEtMoFvHOHvh0RgXiMhqVc1s6Jy9chQsX/3TCfnz7wnZkD9YVsn6nQdZn1fE2p1FrM87yLf7S45Y9CmlXSxpifGc1aMjVw3sRlpiAj2SnCDv3jGO2KhIl74DY8yJsKAPhr1fOxt89zwfLpvpdjUAFJVUsi6viHU7nVBft7OI7ftLas53bR9H/9T2XNqnM2n1euZx0RbkxoQTC/qTVX4IFkxyxuOve86VdWwKD1c4Ye4X7DkFpTXnUzvG0z+1Pdedk0a/1A70796BlHY2a8SY1sKC/mSowpJ7Yf8WuHUJtO/W5F9yX3G5M+xS01M/yM4DtaHeIymeAakduHFIT/p370D/1A4ktYlp8rqMMaHLgv5kfP43Z435y2ZCxoVBv/3eg2U1YV49/LL7YO1aKemdEhjcsyO3nn8K/X099Q4JtjKmMaYuC/oT9e3n8M5D0HsMDLv/pG6l6kxhXJtbxLq8gzXDL9VvJhKBjOQ2DD01qaaX3i+1Pe3jLNSNMcdnQX8iivOd+fId0mDck416U5SqsvNAKev8eurr84rYV1wBOPPQT0tpy4WnJ9MvtQMDUjvQt3t72sbafypjzImx9GgsrwcW3QWlBXDXuxDf8ZjNqzxePtqcT9aOQl+4F1HoW3s8MkLo1bktI3p3pn/39gxI68CZ3drbeinGmKCyRGmsDx91NhG55gnoNvCozQ6XV/HPrBye/XQbOQWlREUIZ3Rpx8i+Xemf2p7+qU6o21RGY0xTs6BvjM1vO6tSDp4EZ09qsMneg2W8sGI7f1/5LUWllZxzSiIPjTmTEb07W6gbY1xhQR+owu3w6mToOgDG/O6I05v3HOLp5dm8viaPSq+X0f26cveFp3LOKYnNX6sxxvixoA9EZRksuNVZhf/6lyA6HnBeWF2xdT9zPs5m2aZ84qIjmDikB3cOyyA9ueUvaGaMCQ8W9IFY+iDs+hImzoOkDCo9Xt78ahdPf5zN+ryDJLeN4UeXn8Et551Cor05yRgTYizoj2fNK7D6ebhgGofSL2f+8mzmfrqNXUVlnJbShsfGD2Dc4FQbfzfGhCwL+mPZvRb+NY3ytGH8sXwCr/z6Aw6VV3HeqUn86tr+jDijs22SYYwJeRb0R1NWRPkrt1BGG0Zm38K+7BzGDOjG5AszGJh27LnzxhgTSizo61FVPtq0lzaLb2dw6bfcozO46juDuGNYOmmJCW6XZ4wxjWZB71Ne5WHJmjye+Xgbw/fN46Hoz/is1wM8MWEqHeJtTRljTMvV6oO+qKSSv3++g+c/207+oXKuS97BT2P+gbfPWL5z/S/CfnNvY0z4a7VBn1NQwrOfbGNBVg4lFR4u7JXMX67uxtB3piFJGTD2CQt5Y0xYaHVBvybnAE8vz+bf63YRGSFcfVZ3Jl94Kmd2ToAXr3F2jJq0GOLau12qMcYERasIeq9Xef/rvTy9PJv/bC+gXVwUU4afxu3fSadrhzin0Ts/hx2fwrVzoEtfdws2xpggCuugL6v08OoXO3nm42yy9x0mtWM8P7+qLzec26Pu+u4b34DPHofMu+CsG9wr2BhjmkBAQS8io4E/A5HAM6r6WANtrgdm4qwI86Wq3uQ7fhvwsK/ZL1X1hSDUfUz7i8t5aeUOXlqxg/2HKxiQ2oHHbxzMmP5diYqMqNd4Kyz+AXQ/G0b/uqlLM8aYZnfcoBeRSGA2cDmQC6wSkSWqusGvTS/gp8AwVS0Ukc6+40nADCAT5w/Aat+1hcH/ViA7v5hnP9nGwtW5lFd5ubRPZyYPP5WhGUlIQy+sVpQ4i5VFRML1L0BUbFOUZYwxrgqkRz8E2KKq2QAiMh8YC2zwazMZmF0d4Kq613d8FPCuqhb4rn0XGA3MC075tbbvO8ylf/yI6IgIxp+dyt0XZnB653ZHv0AV3vwR7FkPNy+Ejj2DXZIxxoSEQII+Fcjxe54LDK3X5gwAEfkUZ3hnpqouPcq1qfW/gIhMAaYA9Ox5YoGbntyGX40bwOV9u5DSLoCe+RcvwJevwEUPQq/LTuhrGmNMSxBI0Dc0mVwbuE8vYASQBnwsIv0DvBZVnQPMAcjMzDzifKBuGhrgH4m8/8JbP4HTLnGC3hhjwljE8ZuQC/Twe54G5DXQ5nVVrVTVbcAmnOAP5NrmVVLgjMu3SYHxzzjj88YYE8YCCfpVQC8RyRCRGGAisKRem8XAxQAikowzlJMNvA2MFJFEEUkERvqOucPrhde+Dwd3OS++tunkWinGGNNcjjt0o6pVInIPTkBHAnNVdb2IzAKyVHUJtYG+AfAAP1bV/QAi8gjOHwuAWdUvzLrikz/AN2/DmN9DWqZrZRhjTHMS1RMeEm8SmZmZmpWVFfwbZy+Dl66FfuNhwjO2jo0xJqyIyGpVbbAHG8jQTctXtBMW3gXJZ8DVf7aQN8a0KuEf9FUV8M/boaoMrn8JYtu6XZExxjSrsF7rBoB3fwG5/4HrnoOUM9yuxhhjml149+jXLYLPn4Sh/wP9x7tdjTHGuCJ8gz5/Myz5IaQNgctnuV2NMca4JjyDvrwYFkyCqDj47vMQFeN2RcYY45rwG6NXhTfug32bYdJr0OGIpXWMMaZVCb8e/apnYN1CuPhncOoIt6sxxhjXhVfQ56yCpT+FXqPggh+5XY0xxoSE8An6w/ud+fLtu8H4v0FE+HxrxhhzMsIrDbsOgOtfhPhEtysxxpiQET4vxrbpBDfNd7sKY4wJOeHVozfGGHMEC3pjjAlzFvTGGBPmLOiNMSbMWdAbY0yYs6A3xpgwZ0FvjDFhzoLeGGPCXMhtDi4i+cCOk7hFMrAvSOUEk9XVOFZX41hdjROOdZ2iqikNnQi5oD9ZIpJ1tJ3Q3WR1NY7V1ThWV+O0trps6MYYY8KcBb0xxoS5cAz6OW4XcBRWV+NYXY1jdTVOq6or7MbojTHG1BWOPXpjjDF+LOiNMSbMhU3Qi8hoEdkkIltEZLrb9VQTkbkisldE1rldSzUR6SEiH4rIRhFZLyL3uV0TgIjEich/RORLX13/63ZN/kQkUkT+KyL/crsWfyKyXUTWisgaEclyu55qItJRRBaKyNe+37XzQ6Cm3r6fU/XHQRG53+26AERkmu/3fp2IzBORuKDdOxzG6EUkEtgMXA7kAquAG1V1g6uFASIyHCgGXlTV/m7XAyAi3YBuqvqFiLQDVgPj3P55iYgAbVS1WESigU+A+1R1pZt1VRORB4BMoL2qXuV2PdVEZDuQqaoh9QYgEXkB+FhVnxGRGCBBVQ+4XVc1X27sBIaq6sm8STMYtaTi/L73VdVSEVkAvKWqzwfj/uHSox8CbFHVbFWtAOYDY12uCQBVXQ4UuF2HP1Xdpapf+D4/BGwEUt2tCtRR7Hsa7fsIiZ6IiKQBVwLPuF1LSyAi7YHhwLMAqloRSiHvcymw1e2Q9xMFxItIFJAA5AXrxuES9KlAjt/zXEIguFoCEUkHBgOfu1uJwzc8sgbYC7yrqiFRF/An4CeA1+1CGqDAOyKyWkSmuF2Mz6lAPvCcb7jrGRFp43ZR9UwE5rldBICq7gR+D3wL7AKKVPWdYN0/XIJeGjgWEj3BUCYibYFFwP2qetDtegBU1aOqg4A0YIiIuD7cJSJXAXtVdbXbtRzFMFU9G7gCmOobLnRbFHA28KSqDgYOA6H02lkMcA3wT7drARCRRJxRiAygO9BGRG4J1v3DJehzgR5+z9MI4j97wpFvDHwR8LKqvup2PfX5/pm/DBjtcikAw4BrfGPh84FLROTv7pZUS1XzfI97gddwhjLdlgvk+v2LbCFO8IeKK4AvVHWP24X4XAZsU9V8Va0EXgW+E6ybh0vQrwJ6iUiG7y/1RGCJyzWFLN+Lns8CG1X1j27XU01EUkSko+/zeJxf/q/drQpU9aeqmqaq6Ti/Wx+oatB6WydDRNr4XlDHNzQyEnB9hpeq7gZyRKS379ClgOuTI/zcSIgM2/h8C5wnIgm+/z8vxXntLCiignUjN6lqlYjcA7wNRAJzVXW9y2UBICLzgBFAsojkAjNU9Vl3q2IYMAlY6xsPB/iZqr7lYk0A3YAXfLMhIoAFqhpSUxlDUBfgNScbiAJeUdWl7pZU417gZV/nKxu4w+V6ABCRBJwZet9zu5Zqqvq5iCwEvgCqgP8SxOUQwmJ6pTHGmKMLl6EbY4wxR2FBb4wxYc6C3hhjwpwFvTHGhDkLemOMCXMW9MYYE+Ys6I0xJsz9fyVAHstooMoyAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXoAAAD4CAYAAADiry33AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAgAElEQVR4nO3deXjU1dn/8fdNWEIgkgBRlgAJlH0RS4ooYqGKjaiAihaxFNRHHi34KMUiVKtoXahga7XYSi0Fn0cFRJAg/ERRFBVQJhiDBJEQgcSwhH2RQJb798cMOsZgJjDJmczcr+vKxXzX+UyM95w53++cI6qKMcaY8FXLdQBjjDFVywq9McaEOSv0xhgT5qzQG2NMmLNCb4wxYa626wBlNW3aVJOSklzHMMaYGiU9PX2vqiaUty3kCn1SUhIej8d1DGOMqVFEZPvptlnXjTHGhDkr9MYYE+YCKvQikioim0UkW0QmlbO9jYi8IyKZIvKeiCT6bRslIlt8P6OCGd4YY0zFKuyjF5EoYAYwEMgD1olImqpm+e02HXhRVeeIyC+AJ4CRItIYeAhIARRI9x17oDIhi4qKyMvLo7CwsDKHRbzo6GgSExOpU6eO6yjGGIcCuRjbG8hW1RwAEZkLDAH8C30XYLzv8Urgdd/jXwJvq+p+37FvA6nAK5UJmZeXR2xsLElJSYhIZQ6NWKrKvn37yMvLIzk52XUcY4xDgXTdtARy/ZbzfOv8fQZc73t8LRArIk0CPBYRGSMiHhHxFBQU/CBAYWEhTZo0sSJfCSJCkyZN7FOQMSagQl9edS075OW9wM9F5FPg58DXQHGAx6KqM1U1RVVTEhLKvQ3UivwZsN+ZMQYC67rJA1r5LScC+f47qGo+cB2AiDQErlfVQyKSB/Qvc+x7Z5HXhLvSUvjsFTiwzXUSY6rfOS0g5ZagnzaQQr8OaC8iyXhb6sOBEf47iEhTYL+qlgKTgVm+TcuBx0Uk3rd8hW+7MT/0zX5YdAdsWe5bYZ9ITIRJTHFT6FW1WETG4S3aUcAsVd0oIo8AHlVNw9tqf0JEFFgFjPUdu19E/oT3zQLgkVMXZsNdw4YNOXr0aLnbtm3bxurVqxkxwvt+OXv2bDweD3//+9+rM2Jo+Xo9vDoKDu+EQdPhZ/8F1vVkTFAENASCqi4DlpVZ96Df4wXAgtMcO4vvWvgGb6F/+eWXvy30gSopKSEqKqqKUjmiCp5Z8OYkaHge3LocEnu5TmVMWAm5sW4q8vCSjWTlHw7qObu0OIeHrun6o/vcd999tGnTht/+9rcATJkyBRFh1apVHDhwgKKiIh599FGGDBlS4fNNmjSJTZs20bNnT0aNGkV8fDz5+fmkpqaydetWrr32Wp588knA+8ngd7/7HcuXL+epp57ikksuOfsXHCpOHoMl98CG+fCTgXDdTIhp7DqVMWHHhkAI0PDhw5k3b963y/Pnz+eWW25h0aJFrF+/npUrVzJhwgQCmYN36tSp9OvXj4yMDMaP9379ICMjg3nz5rFhwwbmzZtHbq73rtRjx47RrVs3Pv744/Aq8gVfwr9+ARtehQEPwIj5VuSNqSI1rkVfUcu7qlxwwQXs2bOH/Px8CgoKiI+Pp3nz5owfP55Vq1ZRq1Ytvv76a3bv3k2zZs0qff7LLruMRo0aAdClSxe2b99Oq1atiIqK4vrrr6/g6BpmwwJI+x+oUx9GLoJ2A1wnMias1bhC79KwYcNYsGABu3btYvjw4bz00ksUFBSQnp5OnTp1SEpKOuMvKNWrV+/bx1FRURQXFwPeYQzCpl+++CS8dT98MhNa9YEb/uO9ncwYU6Ws0FfC8OHDuf3229m7dy/vv/8+8+fP59xzz6VOnTqsXLmS7dtPOxz098TGxnLkyJEqThtiDubCq6Phaw9cNA4unwJRNgaPMdXBCn0ldO3alSNHjtCyZUuaN2/OzTffzDXXXENKSgo9e/akU6dOAZ2nR48e1K5dm/PPP5/Ro0cTHx9f8UE12ZYVsPC/oLQEbvxf6DLYdSJjIooEcvGwOqWkpGjZGaY2bdpE586dHSWq2Zz+7kpL4P0/w/tPwnld4cYXoUk7N1mMCXMikq6qKeVtsxa9qRrH9sJrt0HOe9Dz13DVdO/FV2NMtbNCX4U2bNjAyJEjv7euXr16fPzxx44SVZMdH3v744/vh8F/h5+OrPAQY0zVsUJfhbp3705GRobrGNVHFdY+B28/CI1awW1vQ/MerlMZE/Gs0JvgKDwMi8fCpjTodDUMmQH141ynMsZghd4Ew67PYf5vvEMLX/Go9/ZJG5DMmJBhhd6cnU9fgqW/g+g4GP0GtLnYdSJjTBlW6M2ZKToO/28irH8RkvrBsFnQ8FzXqYwx5bBBzQJ08OBBnnvuuUofN2jQIA4ePFjp42bPnk1+/ncTeSUlJbF3795Kn6dK7M+Bfw/0Fvl+98JvFluRNyaEWaEP0OkKfUlJyY8et2zZMuLiKn9RsmyhD8Sp8XGq1KY34Pn+3iENRsyHy/4ItcJkLB5jwlTN67r5f5Ng14bgnrNZd7hy6o/uMmnSJLZu3UrPnj2pU6cODRs2pHnz5mRkZJCVlcXQoUPJzc2lsLCQu+++mzFjxgDelrjH4+Ho0aNceeWVXHLJJaxevZqWLVuyePFi6tf/4ZeIFixYgMfj4eabb6Z+/fqsWbMGgGeffZYlS5ZQVFTEq6++SqdOnZgyZQr5+fls27aNpk2b8vLLLwf3d3NKSRG88zCsfhZaXAA3zIH4NlXzXMaYoLIWfYCmTp1Ku3btyMjIYNq0aXzyySc89thjZGVlATBr1izS09PxeDw888wz7Nu37wfn2LJlC2PHjmXjxo3ExcXx2muvlftcw4YNIyUlhZdeeomMjIxv3wyaNm3K+vXrufPOO5k+ffq3+6enp7N48eKqK/KHd8Kcwd4i/7P/8s4CZUXemBqj5rXoK2h5V5fevXuTnJz87fIzzzzDokWLAMjNzWXLli00adLke8ckJyfTs2dPAHr16sW2bdsq9ZzXXXfdt8cuXLjw2/WDBw8u95NBUHy1Chbc6p0N6roXoMcNVfM8xpgqU/MKfYho0KDBt4/fe+89VqxYwZo1a4iJiaF///7ljktfdsz548ePV+o5Tx3vP1592SxBU1oKH/4FVj4GTX4Co96AcwMbndMYE1oC6roRkVQR2Swi2SIyqZztrUVkpYh8KiKZIjLItz5JRI6LSIbv55/BfgHV5cfGkD906BDx8fHExMTwxRdfsHbt2ip9vir3zX54ZTi8+yfoei3cvtKKvDE1WIUtehGJAmYAA4E8YJ2IpKlqlt9uDwDzVfUfItIFWAYk+bZtVdWewY1d/Zo0aULfvn3p1q0b9evX57zzzvt2W2pqKv/85z/p0aMHHTt2pE+fPmf9fKNHj+aOO+743sXYavH1enh1lLdfftB0b5+8fcvVmBqtwvHoReQiYIqq/tK3PBlAVZ/w2+d5IEdV/+zb/ylVvVhEkoA3VLVboIFsPPrgCvh3pwqeWfDmJGh4nveumsReVR/QGBMUZzsefUsg1285D7iwzD5TgLdE5C6gAXC537ZkEfkUOAw8oKoflBNwDDAGoHXr1gFEMkF18hgsuQc2zIefDITrZkJMY9epjDFBEkihL+9ze9mPATcBs1X1KV+L/n9FpBuwE2itqvtEpBfwuoh0VdXD3zuZ6kxgJnhb9JV+FTXY2LFj+eijj7637u677+aWW26pngAFX8L8kVCwGQY8AP0mQC2769aYcBJIoc8DWvktJwJlv7J5G5AKoKprRCQaaKqqe4ATvvXpIrIV6AB4qCRVRcKwr3jGjBlVdu4Kp4n8fCEsHued+WnkImg3oMqyGGPcCaTptg5oLyLJIlIXGA6kldlnB3AZgIh0BqKBAhFJ8F3MRUTaAu2BnMqGjI6OZt++fRUXLvMtVWXfvn1ER0eXv8PX6d6p/pp1gzs+sCJvTBirsEWvqsUiMg5YDkQBs1R1o4g8AnhUNQ2YAPxLRMbj7dYZraoqIpcCj4hIMVAC3KGq+ysbMjExkby8PAoKCip7aESLjo4mMTHxhxuKT3pb8rHN4eYFEH1O9YczxlSbCu+6qW7l3XVjgmzlE/D+VBjxKnS4wnUaY0wQ/NhdN3bVLdLs3ggfTIcev7Iib0yEsEIfSUqKvfO61o+H1NAYM8gYU/VsrJtIsnYG5H8KN8y2++SNiSDWoo8Ue7Nh5ePQ6WroMtR1GmNMNbJCHwlKSyHtLqhdD656ysauMSbCWNdNJPD8G3ashiHPQWwz12mMMdXMWvTh7uAOWDEF2l0GPUe4TmOMccAKfThT9Q5WBnDN09ZlY0yEsq6bcPbZK7D1He+48nE2Kqgxkcpa9OHqyG54czK0vghSbnOdxhjjkBX6cLVsAhQdh8HP2rDDxkQ4qwDhaOPrsGkJDJgMTdu7TmOMccwKfbj5Zj8suxea94SL7nKdxhgTAuxibLh5czIcP+CdSCTK/vMaY6xFH16+fAsy58Ilv4Nm3V2nMcaECCv04aLwMLxxDyR0gkvvdZ3GGBNC7LN9uFjxEBzZCTe+6B3TxhhjfKxFHw6++gA8s6DPbyGx3AlmjDERzAp9TXfyG+/IlPHJMOB+12mMMSHIum5qupWPwYGvYNQbUDfGdRpjTAgKqEUvIqkisllEskVkUjnbW4vIShH5VEQyRWSQ37bJvuM2i8gvgxk+4uWlw9rnoNctkNzPdRpjTIiqsEUvIlHADGAgkAesE5E0Vc3y2+0BYL6q/kNEugDLgCTf4+FAV6AFsEJEOqhqSbBfSMQpPuGd/zW2OQx8xHUaY0wIC6RF3xvIVtUcVT0JzAWGlNlHgXN8jxsB+b7HQ4C5qnpCVb8Csn3nM2frg6egYBNc/TREn1Px/saYiBVIoW8J5Pot5/nW+ZsC/FpE8vC25k999z6QYxGRMSLiERFPQUFBgNEj2K7PvYW+x6+gwxWu0xhjQlwghb682Sq0zPJNwGxVTQQGAf8rIrUCPBZVnamqKaqakpCQEECkCFZS7O2yqR8PqVNdpzHG1ACB3HWTB7TyW07ku66ZU24DUgFUdY2IRANNAzzWVMaav8PODLhhNsQ0dp3GGFMDBNKiXwe0F5FkEamL9+JqWpl9dgCXAYhIZyAaKPDtN1xE6olIMtAe+CRY4SPO3mx47wnodDV0Geo6jTGmhqiwRa+qxSIyDlgORAGzVHWjiDwCeFQ1DZgA/EtExuPtmhmtqgpsFJH5QBZQDIy1O27OUGkppI3zDm9w1VM2/6sxJmABfWFKVZfhvcjqv+5Bv8dZQN/THPsY8NhZZDQAnn/DjjUw5DmIbeY6jTGmBrEhEGqCgztgxRRodxn0HOE6jTGmhrFCH+pUYcnd3sfXPG1dNsaYSrOxbkJdxsuw9V0YNB3iWrtOY4ypgaxFH8qO7ILlk6H1xZBym+s0xpgaygp9qFKFpRO8Y9oMfhZq2X8qY8yZseoRqrJehy/egP6ToelPXKcxxtRgVuhD0Tf7YdnvoXlPuGic6zTGmBrOLsaGojcnwfEDMHIRRNl/ImPM2bEWfaj58i3InAeX/A6adXedxhgTBqzQh5LCw/DGPZDQCS6913UaY0yYsH6BUPL2g3BkJ9z4ondMG2OMCQJr0YeKrz6A9P9An99CYorrNMaYMGKFPhSc/AbS7oL4ZBhwv+s0xpgwY103oWDlY3DgKxj1BtSNcZ3GGBNmrEXvWp4H1j4HvW6B5H6u0xhjwpAVepeKT3jnf41tDgMfcZ3GGBOmrOvGpVXToeALGPEqRJ/jOo0xJkxZi96VXRvgw79Aj19BhytcpzHGhDEr9C6UFMPicVA/HlKnuk5jjAlz1nXjwppnYWcG3DAbYho7i7E0cyfZe446e35jzPc1a1SPX/0s+BMMBVToRSQV+BsQBbygqlPLbP8rMMC3GAOcq6pxvm0lwAbfth2qOjgYwWusvVtg5RPQ6WroMtRZjHc27Wbsy+udPb8x5od6topzU+hFJAqYAQwE8oB1IpKmqlmn9lHV8X773wVc4HeK46raM3iRa7DSUu8Xo+pEw1VPOZv/9XBhEfcv+pxOzWJ5fWxf6kZZD54x4SyQFn1vIFtVcwBEZC4wBMg6zf43AQ8FJ16YWfcC7FgDQ56D2GbOYjyxbBN7jhTy/MheRNeJcpbDGFM9AmnKtQRy/ZbzfOt+QETaAMnAu36ro0XEIyJrRaTcvgoRGePbx1NQUBBg9BrmwHZYMQXaXQY9RziLsTp7L698ksvt/dpyfqs4ZzmMMdUnkEJfXv+Cnmbf4cACVS3xW9daVVOAEcDTItLuBydTnamqKaqakpCQEECkGkYVltzt7aq55mlnXTbfnCzmvoWZJDdtwPiBHZxkMMZUv0AKfR7Qym85Ecg/zb7DgVf8V6hqvu/fHOA9vt9/HxkyXoKclXD5FIgL/oWWQE1f/iW5+48z9bru1mVjTAQJpNCvA9qLSLKI1MVbzNPK7iQiHYF4YI3fungRqed73BToy+n79sPTkV2w/A/Q+mJIuc1ZjPTtB/jP6q8Y2acNF7Zt4iyHMab6VXgxVlWLRWQcsBzv7ZWzVHWjiDwCeFT1VNG/CZirqv7dOp2B50WkFO+bylT/u3XCniosneAd02bws1DLzd0tJ4pLuO+1TFo0qs99V3ZyksEY405A99Gr6jJgWZl1D5ZZnlLOcauByJ34dOMi+OINuPxhaPoTZzGefSeb7D1HmXNrbxrWs+/IGRNp7AbqqnJsHyz7PTTvCReNcxZjY/4h/vH+Vq7/aSI/7xCGF7qNMRWy5l1VeXMSFB6E3yyGKDe/5qKSUiYuyCQ+pi5/vLqzkwzGGPesRV8VvlwOG+ZDvwnQrJuzGDNX5bAx/zCPDu1KXExdZzmMMW5ZoQ+2wkOw5B5I6Az97nUWI3vPUf72zhYGdW9GarfmznIYY9yzrptge/tBOLoLfvV/UNtNK7qkVLnvtUxi6kbx8GB3nyiMMaHBWvTB9NUqSJ8NfX4Lib2cxXhxzTbStx/gwau7kBBbz1kOY0xosEIfLCePeUembNwWBtzvLEbu/m948s3N9O+YwLUXlDskkTEmwljXTbC8+xgc2Aajl0LdGCcRVJXJCzcQVUt4/NruiKMxdYwxocVa9MGQuw7WPgcpt0LSJc5izPfk8mH2XiZd2YkWcfWd5TDGhBYr9Ger+AQsHgvntPR+A9aR3YcLeXTpJi5MbsyI3u4GTjPGhB7rujlbq6bB3s1w8wKIPsdJBFXl/kWfU1RSyp+v70GtWtZlY4z5jrXoz8auDfDhX6HHcGg/0FmMJZk7WbFpNxMGdiSpaQNnOYwxockK/ZkqKYbF46B+PKQ+4SzGvqMnmJK2kfMTG3FL3yRnOYwxocu6bs7UmmdhZwbcMAdiGjuL8fCSLI4UFvHksD7Utkm+jTHlsMpwJvZugZVPQKerocsQZzFWZO0m7bN8xg74CR2bxTrLYYwJbVboK6u01PvFqDrRcNVTzuZ/PXS8iPtf30CnZrH8tr+7se6NMaHPum4qa90LsGMNDHkOYps5i/HEsk0UHDnBzJEp1K1t79fGmNOzClEZB7bDiinQ7jLoOcJZjI+y9zJ3XS6392vL+a3inOUwxtQMVugDpQpL7vZ21VzztLMum29OFjNpYSbJTRswfmAHJxmMMTWLdd0EKuMlyFkJg6ZDnLtvnk5bvpnc/ceZN6YP0XWinOUwxtQc1qIPxJFdsPwP0PpiSLnNWYz07fuZvXobI/u04cK2TZzlMMbULAEVehFJFZHNIpItIpPK2f5XEcnw/XwpIgf9to0SkS2+n1HBDF8tVGHpBO+YNoOfhVpu3hsLi0qYuCCTFo3qc9+VnZxkMMbUTBV23YhIFDADGAjkAetEJE1Vs07to6rj/fa/C7jA97gx8BCQAiiQ7jv2QFBfRVXauAi+eMM7YFlTd7cxPvvuFrYWHGPOrb1pWM963IwxgQukedobyFbVHFU9CcwFfuxbQjcBr/ge/xJ4W1X3+4r720Dq2QSuVsf2wbLfQ/OecNE4ZzE25h/in+/ncP1PE/l5hwRnOYwxNVMghb4lkOu3nOdb9wMi0gZIBt6tzLEiMkZEPCLiKSgoCCR39XhzEhQehCEzIMpNK7qopJSJCzKJj6nLH6/u7CSDMaZmC6TQl3cfoZ5m3+HAAlUtqcyxqjpTVVNUNSUhIURarF8uhw3zod8EaOZugu2Zq3LYmH+YR4d2JS7GzWTjxpiaLZBCnwe08ltOBPJPs+9wvuu2qeyxoaPwECy5BxI6Q797ncXI3nOUv72zhUHdm5HarbmzHMaYmi2QQr8OaC8iySJSF28xTyu7k4h0BOKBNX6rlwNXiEi8iMQDV/jWhba3H4Sju7xdNrXdtKJLSpX7Xsskpm4UDw9294nCGFPzVdjxrKrFIjIOb4GOAmap6kYReQTwqOqpon8TMFdV1e/Y/SLyJ7xvFgCPqOr+4L6EIPtqFaTP9l58TezlLMaLa7aRvv0Af7nxfBJi6znLYYyp+cSvLoeElJQU9Xg8bp785DH4x8UgteCOj6BujJMYufu/4Yq/ruLCto35z+ifIY6GWzDG1Bwikq6qKeVtsxuy/b37GBzYBqOXOivyqsrkhRuIqiU8fm13K/LGmLNmQyCckrsO1j4HKbdC0iXOYsz35PJh9l4mXdmJFnH1neUwxoQPK/TgHd5g8Vg4p6X3G7CO7D5cyKNLN3FhcmNG9HY3cJoxJrxY1w3AqmmwdzPcvACiz3ESQVW5f9HnFJWU8ufre1CrlnXZGGOCw1r0uzbAh3+FHsOh/UBnMZZk7mTFpt1MGNiRpKYNnOUwxoSfyC70JcXeLpv68ZD6hLMY+46eYEraRs5vFcetlyQ7y2GMCU+R3XWz+hnY+RncMAdiGjuL8fCSLI4UFjFtWA+irMvGGBNkkdui37sF3psKna+BrkOdxViRtZu0z/IZN6A9Hc6LdZbDGBO+IrPQl5bC4nFQpz4MespZjEPHi7j/9Q10ahbLnf3bOcthjAlvkdl1s+5fkLsWhv4DYs9zFuOJZZsoOHKCf/0mhbq1I/M91xhT9SKvuhzYDisehp9cDuff5CzGR9l7mbsul9svbUuPxDhnOYwx4S+yCr0qLPkfEIGrn/b+68A3J4uZtDCT5KYNGH95BycZjDGRI7K6bj79P8h5DwZNh7hWFe5eVaYt30zu/uPMG9OH6DpRznIYYyJD5LToD++E5fdDm76QcpuzGOnb9zN79TZG9mnDhW2bOMthjIkckVHoVWHpBCg5AYOfhVpuXnZhUQkTF2TSolF97ruyk5MMxpjIExmFfuNC2LwUBvwBmri7jfHZd7ewteAYj1/XnYb1IqvXzBjjTvgX+mP7YNlEaHEB9BnrLMbnXx/in+/ncP1PE/l5hxCZAN0YExHCv1n55n3eyb6HpEGUm5dbVFLKxAWZxMfU5Y9Xd3aSwRgTucK7Rb/5TdjwKvSbAOd1dRZj5qocsnYe5tGhXYmLcTPZuDEmcoVvoS88BG+Mh3O7eAu9I9l7jvC3FVsY1L0Zqd2aO8thjIlcARV6EUkVkc0iki0ik06zz40ikiUiG0XkZb/1JSKS4ftJC1bwCr31Rzi6C4b8HWq7aUWXlCoTF2QSUy+Khwd3c5LBGGMq7LQWkShgBjAQyAPWiUiaqmb57dMemAz0VdUDInKu3ymOq2rPIOf+cTnvw/o5cPFd0LJXtT61vzmrt7F+x0H+cuP5JMTWc5bDGBPZAmnR9wayVTVHVU8Cc4EhZfa5HZihqgcAVHVPcGNWwsljkHYXNG4L/f/gLMaOfd8wbflm+ndM4NoLWjrLYYwxgRT6lkCu33Keb52/DkAHEflIRNaKSKrftmgR8fjWlzvwu4iM8e3jKSgoqNQL+IF3H4WD22Hw36FuzNmd6wypKpMXZRJVS3j82u6IozF1jDEGAru9srwqpeWcpz3QH0gEPhCRbqp6EGitqvki0hZ4V0Q2qOrW751MdSYwEyAlJaXsuQOX+wms/Yd3iIOkvmd8mrM1b10uH2Xv49Gh3WgRV99ZDmOMgcBa9HmA/whgiUB+OfssVtUiVf0K2Iy38KOq+b5/c4D3gAvOMnP5ik94JxM5pyVcPqVKniIQuw4V8tjSTVyY3JgRvVs7y2GMMacEUujXAe1FJFlE6gLDgbJ3z7wODAAQkaZ4u3JyRCReROr5re8LZFEVjuwCqQXX/A2iz6mSp6iIqvLA6xsoKi3lz9f3oJbN/2qMCQEVdt2oarGIjAOWA1HALFXdKCKPAB5VTfNtu0JEsoAS4Pequk9ELgaeF5FSvG8qU/3v1gmq+DZwx4fOvv0KkPZZPis27eH+QZ1JatrAWQ5jjPEnqmfeJV4VUlJS1OPxuI5RafuOnmDgX1fRqnEMC++8mChrzRtjqpGIpKtqSnnbwvebsdVsypIsjhQWMW1YDyvyxpiQYoU+CN7O2s2Sz/IZN6A9Hc6LdR3HGGO+xwr9WTp0vIj7F22gU7NY7uzvbqx7Y4w5nfAfpriKPb50E3uPnuCFUSnUrW3vm8aY0GOV6Sx8uGUv8zy53H5pW3okxrmOY4wx5bJCf4aOnShm0sJMkps2YPzlHVzHMcaY07KumzM0bflm8g4cZ/5/X0R0nSjXcYwx5rSsRX8GPNv2M2fNNn5zURt6Jzd2HccYY36UFfpKKiwqYeJrmbRoVJ+JqZ1cxzHGmApZ100lPfPOFnIKjvHirb1pWM9+fcaY0Gct+kr4/OtDPL8qh2G9Erm0Q4LrOMYYExAr9AEqKill4oJMGjeoyx+v6uI6jjHGBMz6HgL0/Ptbydp5mH/+uheNYuq4jmOMMQGzFn0Atuw+wjPvZHNV9+akdmvmOo4xxlSKFfoKlJQqE1/LJKZeFFMGd3UdxxhjKs0KfQXmrN7GpzsO8tA1XUiIrec6jjHGVJoV+h+xY983TFu+mQEdExjas6XrOMYYc0as0J+GqjJ5USZRtYTHru2OiE0mYoypmazQn8a8dbl8lL2PyYM60SKuvus4xhhzxqzQl2PXoUIeW7qJPm0bc9PPWruOY4wxZyWgQi8iqSKyWUSyRWTSafa5UUSyRGSjiLzst36UiGzx/YwKVvCqoqo88P+K4jwAAAlASURBVPoGikpL+fP1Pahl878aY2q4Cr8wJSJRwAxgIJAHrBORNFXN8tunPTAZ6KuqB0TkXN/6xsBDQAqgQLrv2APBfynBkfZZPis27eGBqzrTpkkD13GMMeasBdKi7w1kq2qOqp4E5gJDyuxzOzDjVAFX1T2+9b8E3lbV/b5tbwOpwYkefPuOnuDhJVn0bBXHLX2TXccxxpigCKTQtwRy/ZbzfOv8dQA6iMhHIrJWRFIrcSwiMkZEPCLiKSgoCDx9kE1ZksWRwiKeHNaDKOuyMcaEiUAKfXkVT8ss1wbaA/2Bm4AXRCQuwGNR1ZmqmqKqKQkJbkaFfDtrN0s+y+euX7Snw3mxTjIYY0xVCKTQ5wGt/JYTgfxy9lmsqkWq+hWwGW/hD+RY5w4dL+L+RRvo1CyWO/u3cx3HGGOCKpBCvw5oLyLJIlIXGA6kldnndWAAgIg0xduVkwMsB64QkXgRiQeu8K0LKY8v3cS+YyeZNux86kTZHafGmPBS4V03qlosIuPwFugoYJaqbhSRRwCPqqbxXUHPAkqA36vqPgAR+RPeNwuAR1R1f1W8kDP14Za9zPPkcsfP29E9sZHrOMYYE3Si+oMuc6dSUlLU4/FUy3MdO1HML59eRd2oWiy7ux/RdaKq5XmNMSbYRCRdVVPK2xbRE49MW76Zrw8eZ/5/X2RF3hgTtiK2Q9qzbT9z1mzjN33a8LOkxq7jGGNMlYnIQl9YVMLE1zJp0ag+E1M7uY5jjDFVKiK7bp55Zws5Bcd48dbeNKgXkb8CY0wEibgW/edfH+L5VTnc0CuRSzu4+XKWMcZUp4gq9EUlpUxckEnjBnV54KouruMYY0y1iKh+i+ff30rWzsM8P7IXjWLquI5jjDHVImJa9Ft2H+GZd7K5qkdzftm1mes4xhhTbSKi0JeUKhNfyySmXhQPD+7qOo4xxlSriCj0s1dv49MdB3nomi40bVjPdRxjjKlWYV/od+z7hunLNzOgYwJDe/5gKHxjjAl7YV3oVZVJCzOJqiU8dm13RGwyEWNM5AnrQj93XS6rt+5j8qBOtIir7zqOMcY4EbaFfueh4zy+dBN92jbmpp+1dh3HGGOcCctCr6o8sOhzikpL+fP1Pahl878aYyJYWBb6tM/yeeeLPdx7RUfaNGngOo4xxjgVdoV+79ETTEnbSM9WcdzSN9l1HGOMcS7sCv2UtI0cPVHMk8N6EGVdNsYYE16F/q2Nu3gjcyd3/aI9Hc6LdR3HGGNCQtgU+kPHi3jg9c/p1CyWO/u3cx3HGGNCRkCFXkRSRWSziGSLyKRyto8WkQIRyfD9/JffthK/9WnBDO/vZHEp57eKY9qw86kTFTbvX8YYc9YqHKZYRKKAGcBAIA9YJyJpqppVZtd5qjqunFMcV9WeZx/1xyXE1uNfvyl3AnRjjIlogTR9ewPZqpqjqieBucCQqo1ljDEmWAIp9C2BXL/lPN+6sq4XkUwRWSAirfzWR4uIR0TWisjQ8p5ARMb49vEUFBQEnt4YY0yFAin05d2jqGWWlwBJqtoDWAHM8dvWWlVTgBHA0yLygyulqjpTVVNUNSUhweZxNcaYYAqk0OcB/i30RCDffwdV3aeqJ3yL/wJ6+W3L9/2bA7wHXHAWeY0xxlRSIIV+HdBeRJJFpC4wHPje3TMi0txvcTCwybc+XkTq+R43BfoCZS/iGmOMqUIV3nWjqsUiMg5YDkQBs1R1o4g8AnhUNQ34HxEZDBQD+4HRvsM7A8+LSCneN5Wp5dytY4wxpgqJatnudrdSUlLU4/G4jmGMMTWKiKT7rof+gH2zyBhjwlzItehFpADYfhanaArsDVKcYLJclWO5KsdyVU445mqjquXethhyhf5siYjndB9fXLJclWO5KsdyVU6k5bKuG2OMCXNW6I0xJsyFY6Gf6TrAaViuyrFclWO5KieicoVdH70xxpjvC8cWvTHGGD9W6I0xJsyFTaGvaBYsV0RklojsEZHPXWc5RURaichKEdkkIhtF5G7XmQBEJFpEPhGRz3y5HnadyZ+IRInIpyLyhuss/kRkm4hs8M3iFjJfKxeRON+w5V/4/tYuCoFMHf1mvMsQkcMico/rXAAiMt73d/+5iLwiItFBO3c49NH7ZsH6Er9ZsICbQmFcHRG5FDgKvKiq3VzngW8HoWuuqutFJBZIB4a6/n2JiAANVPWoiNQBPgTuVtW1LnOdIiK/A1KAc1T1atd5ThGRbUCKqobUF4BEZA7wgaq+4BsQMUZVD7rOdYqvbnwNXKiqZ/MlzWBkaYn3772Lqh4XkfnAMlWdHYzzh0uLPmRnwVLVVXgHegsZqrpTVdf7Hh/BO9poeZPJVCv1OupbrOP7CYmWiIgkAlcBL7jOUhOIyDnApcC/AVT1ZCgVeZ/LgK2ui7yf2kB9EakNxFBmOPizES6FPtBZsEwZIpKEd46Aj90m8fJ1j2QAe4C3VTUkcgFPAxOBUtdByqHAWyKSLiJjXIfxaQsUAP/xdXe9ICINXIcqYzjwiusQAKr6NTAd2AHsBA6p6lvBOn+4FPpAZsEyZYhIQ+A14B5VPew6D4Cqlvgmk08EeouI8+4uEbka2KOq6a6znEZfVf0pcCUw1tdd6Fpt4KfAP1T1AuAYEErXzurinTvjVddZwDt3B95eiGSgBdBARH4drPOHS6GvcBYs832+PvDXgJdUdaHrPGX5Pua/B6Q6jgLeCXMG+/rC5wK/EJH/cxvpO36zuO0BFuHtynQtD8jz+0S2AG/hDxVXAutVdbfrID6XA1+paoGqFgELgYuDdfJwKfQVzoJlvuO76PlvYJOq/sV1nlNEJEFE4nyP6+P94//CbSpQ1cmqmqiqSXj/tt5V1aC1ts6GiDTwXVDH1zVyBeD8Di9V3QXkikhH36rLCK3Z5W4iRLptfHYAfUQkxvf/52X4ZuoLhgpnmKoJTjcLluNYAIjIK0B/oKmI5AEPqeq/3aaiLzAS2ODrDwf4g6ouc5gJoDkwx3c3RC1gvqqG1K2MIeg8YJG3NlAbeFlV33Qb6Vt3AS/5Gl85wC2O8wAgIjF479D7b9dZTlHVj0VkAbAe70x9nxLE4RDC4vZKY4wxpxcuXTfGGGNOwwq9McaEOSv0xhgT5qzQG2NMmLNCb4wxYc4KvTHGhDkr9MYYE+b+P3hfIN4jwDGjAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plot_hist(thr_score_hist)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "torch.save(model.text_embedor, 'data/text_models/bert_final_10ep_text_emb.pth')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "torch.save(model.state_dict(), 'data/text_models/bert_final_10ep_sd.pth')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "torch.save(tokenizer, 'data/text_models/bert_tokenizer.pth')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=536.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    }
   ],
   "source": [
    "with torch.no_grad():\n",
    "    pbar = tqdm(full_dl, leave=False)\n",
    "    text_embeddings = []\n",
    "    for txts, _ in pbar:\n",
    "        txts = text_to_device(txts, device)\n",
    "        feature =  model(txts)\n",
    "        text_embeddings.append(feature)\n",
    "    text_embeddings = F.normalize(torch.cat(text_embeddings, 0))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([34250, 768])"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "text_embeddings.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=34250.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=34250.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r",
      "f1 score : 0.5241056886066721 | precision : 0.7149890345780057 | recall : 0.5488528699864234\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(0.5241056886066721, 0.7149890345780057, 0.5488528699864234)"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "compute_f1(text_embeddings, df['label_group'], 0.5)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### train with freeze full data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "from utils import compute_f1, load_data\n",
    "from text_train.imports import *\n",
    "from text_train.data import TextDS, text_to_device\n",
    "from arcface import ArcMarginProduct\n",
    "from text_train.model import EMBBert\n",
    "from text_train.train import *\n",
    "np.random.seed(1337)\n",
    "device = torch.device('cuda')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from imports import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def plot_hist(history) : \n",
    "    train_scores = [info['train']['f1'] for info in history]\n",
    "    train_thr = [info['train']['thr'] for info in history]\n",
    "    val_scores = [info['val']['f1'] for info in history]\n",
    "    val_thr = [info['val']['thr'] for info in history]\n",
    "    x = range(len(train_scores))\n",
    "    plt.plot(x, val_scores, label='val_score')\n",
    "    plt.plot(x, train_scores, label='train_score')\n",
    "    plt.legend()\n",
    "    plt.show()\n",
    "    plt.plot(x, val_thr, label='val_thr')\n",
    "    plt.plot(x, train_thr, label='train_thr')\n",
    "    plt.legend()\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "df, train_df, val_df, train_labels, val_labels = load_data(train_perc=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "bs = 16\n",
    "language_model = 'bert-base-multilingual-uncased'\n",
    "tokenizer = BertTokenizer.from_pretrained(language_model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "tr_ds = TextDS(train_df, tokenizer)\n",
    "tr_test_dl = DataLoader(tr_ds, batch_size = bs, shuffle = False, pin_memory = True)\n",
    "\n",
    "tr_ds = TextDS(train_df, tokenizer)\n",
    "tr_dl = DataLoader(tr_ds, batch_size = bs, shuffle = True, pin_memory = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "torch.save(tr_test_dl, 'data/text_models/tr_test_dl_full.pth')\n",
    "torch.save(tr_dl, 'data/text_models/tr_dl_full.pth')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "tr_dl = torch.load('data/text_models/tr_dl_full.pth')\n",
    "tr_test_dl = torch.load('data/text_models/tr_test_dl_full.pth')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# model\n",
    "model = EMBBert(language_model).to(device)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "centers = compute_centers(tr_test_dl, model, train_df)\n",
    "torch.save(centers, 'data/text_models/centers_full.pth')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "centers = torch.load('data/text_models/centers_full.pth')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Using center as wieghts\n"
     ]
    }
   ],
   "source": [
    "metric_fc = ArcMarginProduct(512, train_df['label_group'].nunique(), s=30, m=0.5,\n",
    "                             easy_margin=False, centers=centers, device=device).to(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Unfroze 5 parameters\n"
     ]
    }
   ],
   "source": [
    "n_epochs, lf, params, optimizer, sched = get_hparams(tr_dl, model, metric_fc, lr=5e-4, n_epochs=5)\n",
    "freeze_bert_embeddings(model)\n",
    "freeze(model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Unfroze 32 parameters\n"
     ]
    }
   ],
   "source": [
    "unfreeze_bert_to(model, 10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "loss_hist = []\n",
    "best_thr_score={'train': {'thr': None, 'f1': None}}\n",
    "thr_score_hist=[]\n",
    "ep_start = 0\n",
    "save_path = 'data/text_models/2april'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=5.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=1071.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Thresholds', max=10.0, style=ProgressStyle(description_wi…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Checkpoint : saved model to data/text_models/2april_ep_0.pth\n",
      "Ep 0: Loss 11.4460 | F score 0.5088 with thresh 0.64\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=1071.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Thresholds', max=5.0, style=ProgressStyle(description_wid…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Ep 1: Loss 10.1777 | F score 0.5531 with thresh 0.74\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=1071.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Thresholds', max=5.0, style=ProgressStyle(description_wid…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Checkpoint : saved model to data/text_models/2april_ep_2.pth\n",
      "Ep 2: Loss 7.6275 | F score 0.6458 with thresh 0.84\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=1071.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Thresholds', max=5.0, style=ProgressStyle(description_wid…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Ep 3: Loss 5.2639 | F score 0.7408 with thresh 0.94\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=1071.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Thresholds', max=5.0, style=ProgressStyle(description_wid…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Checkpoint : saved model to data/text_models/2april_ep_4.pth\n",
      "Ep 4: Loss 3.8014 | F score 0.7810 with thresh 0.94\n",
      "\r"
     ]
    }
   ],
   "source": [
    "best_thr_score, thr_score_hist, losses = train_full_data(model, optimizer, lf, sched, metric_fc, tr_dl,\n",
    "                                               n_epochs, train_df, \n",
    "                                               save_path=save_path, \n",
    "                                               prev_best_info=best_thr_score, info_history=thr_score_hist,\n",
    "                                               ep_start=ep_start)\n",
    "loss_hist.append(losses)\n",
    "ep_start += n_epochs"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "torch.save(metric_fc.state_dict(), save_path+'_met_1fr.pth')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<All keys matched successfully>"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "metric_fc.load_state_dict(torch.load(save_path+'_met_1fr.pth'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<All keys matched successfully>"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.load_state_dict(torch.load('data/text_models/2april_ep_4.pth'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Unfroze 5 parameters\n"
     ]
    }
   ],
   "source": [
    "n_epochs, lf, params, optimizer, sched = get_hparams(tr_dl, model, metric_fc, lr=1e-4, n_epochs=2)\n",
    "freeze_bert_embeddings(model)\n",
    "freeze(model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Unfroze 96 parameters\n"
     ]
    }
   ],
   "source": [
    "freeze(model)\n",
    "unfreeze_bert_to(model, 6)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=2.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=1071.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Thresholds', max=10.0, style=ProgressStyle(description_wi…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Checkpoint : saved model to data/text_models/2april_ep_0.pth\n",
      "Ep 0: Loss 3.9497 | F score 0.7760 with thresh 0.91\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=1071.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Thresholds', max=5.0, style=ProgressStyle(description_wid…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Ep 1: Loss 3.1694 | F score 0.8175 with thresh 0.96\n",
      "\r"
     ]
    }
   ],
   "source": [
    "best_thr_score, thr_score_hist, losses = train_full_data(model, optimizer, lf, sched, metric_fc, tr_dl,\n",
    "                                               n_epochs, train_df, \n",
    "                                               save_path=save_path, \n",
    "                                               prev_best_info=best_thr_score, info_history=thr_score_hist,\n",
    "                                               ep_start=ep_start)\n",
    "loss_hist.append(losses)\n",
    "ep_start += n_epochs"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "torch.save(metric_fc.state_dict(), save_path+'_met_3fr.pth')\n",
    "torch.save(model.state_dict(), save_path+'_mod_3fr.pth')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<All keys matched successfully>"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.load_state_dict(torch.load(save_path+'_mod_3fr.pth'))\n",
    "metric_fc.load_state_dict(torch.load(save_path+'_met_3fr.pth'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Unfroze 5 parameters\n"
     ]
    }
   ],
   "source": [
    "n_epochs, lf, params, optimizer, sched = get_hparams(tr_dl, model, metric_fc, lr=1e-5, n_epochs=2)\n",
    "freeze_bert_embeddings(model)\n",
    "freeze(model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "unfreeze(model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=2.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=2141.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Thresholds', max=10.0, style=ProgressStyle(description_wi…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Checkpoint : saved model to data/text_models/2april_ep_0.pth\n",
      "Ep 0: Loss 2.6738 | F score 0.8324 with thresh 0.91\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=2141.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Thresholds', max=5.0, style=ProgressStyle(description_wid…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Ep 1: Loss 2.3517 | F score 0.8514 with thresh 0.96\n",
      "\r"
     ]
    }
   ],
   "source": [
    "best_thr_score, thr_score_hist, losses = train_full_data(model, optimizer, lf, sched, metric_fc, tr_dl,\n",
    "                                               n_epochs, train_df, \n",
    "                                               save_path=save_path, \n",
    "                                               prev_best_info=best_thr_score, info_history=thr_score_hist,\n",
    "                                               ep_start=ep_start)\n",
    "loss_hist.append(losses)\n",
    "ep_start += n_epochs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "torch.save(model.state_dict(), 'data/text_models/4ap_bert_8ep.pth')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXoAAAD4CAYAAADiry33AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAgAElEQVR4nO3deXxU1f3/8dcne8KakLAlYKIiyCZoBC2KuAHiAoJVXHCH9lu0iv210moLX2ytXb+tlWpRcatCKShitbgjLmAJFmUThAAmhCWQEAjZZz6/P+4kmYQAE5jkTiaf5+ORx2TuPffmkxjfOZw5c46oKsYYY8JXhNsFGGOMaVoW9MYYE+Ys6I0xJsxZ0BtjTJizoDfGmDAX5XYB9SUnJ2t6errbZRhjTIuyevXqfaqa0tC5gIJeREYDfwYigWdU9bF6508B5gIpQAFwi6rm+s7dBjzsa/pLVX3hWF8rPT2drKysQMoyxhjjIyI7jnbuuEM3IhIJzAauAPoCN4pI33rNfg+8qKoDgVnAr33XJgEzgKHAEGCGiCSeyDdhjDHmxAQyRj8E2KKq2apaAcwHxtZr0xd43/f5h37nRwHvqmqBqhYC7wKjT75sY4wxgQok6FOBHL/nub5j/r4EJvg+vxZoJyKdArwWEZkiIlkikpWfnx9o7cYYYwIQyBi9NHCs/roJ/w94QkRuB5YDO4GqAK9FVecAcwAyMzOPOF9ZWUlubi5lZWUBlGtORlxcHGlpaURHR7tdijEmSAIJ+lygh9/zNCDPv4Gq5gHjAUSkLTBBVYtEJBcYUe/aZY0tMjc3l3bt2pGeno5IQ387TDCoKvv37yc3N5eMjAy3yzHGBEkgQzergF4ikiEiMcBEYIl/AxFJFpHqe/0UZwYOwNvASBFJ9L0IO9J3rFHKysro1KmThXwTExE6depk/3IyJswcN+hVtQq4ByegNwILVHW9iMwSkWt8zUYAm0RkM9AF+JXv2gLgEZw/FquAWb5jjWYh3zzs52xM+AloHr2qvgW8Ve/YL/w+XwgsPMq1c6nt4RtjjPF3eB/s3QB7NkBULGTeEfQvEXLvjDXGmLBUcRj2fu2E+t4NsGc97N0Ih/fWtkkbYkHfkrRt25bi4mK3yzDGNDdPJezfCnt9Qb5ng/N54Q5qJh1GxUPnPtDrcujcFzqfCV36QdsuTVKSBX0YqKqqIirK/lMa06xUoSi3bu987wbYtxk8FU4biYBOp0O3QXDWTdClrxPsiekQEdlspba4dPjfN9azIe9gUO/Zt3t7Zlzd75htHnzwQU455RR+8IMfADBz5kxEhOXLl1NYWEhlZSW//OUvGTu2/puGj7Rr1y5uuOEGDh48SFVVFU8++SQXXnghS5cu5Wc/+xkej4fk5GTef/99CgoKuPPOO8nOziYhIYE5c+YwcOBAZs6cSV5eHtu3byc5OZmXXnqJ6dOns2zZMsrLy5k6dSrf+973gvLzMabVKymoHUev7qnv3QjlflnUPs3pmZ9+qa+X3heSz4DoOPfq9mlxQe+WiRMncv/999cE/YIFC1i6dCnTpk2jffv27Nu3j/POO49rrrnmuDNXXnnlFUaNGsVDDz2Ex+OhpKSE/Px8Jk+ezPLly8nIyKCgwJmcNGPGDAYPHszixYv54IMPuPXWW1mzZg0Aq1ev5pNPPiE+Pp45c+bQoUMHVq1aRXl5OcOGDWPkyJE2H96Yxqgogfyva3vn1eFevLu2TVxHZ5hl4PW1gd75TIjv6F7dx9Higv54Pe+mMnjwYPbu3UteXh75+fkkJibSrVs3pk2bxvLly4mIiGDnzp3s2bOHrl27HvNe5557LnfeeSeVlZWMGzeOQYMGsWzZMoYPH14TzElJSQB88sknLFq0CIBLLrmE/fv3U1RUBMA111xDfHw8AO+88w5fffUVCxc6k5+Kior45ptvLOiNaYinCgqyj3xhtCCb2nH0OEjpDadd4htD7wud+0G7rtDCpiG3uKB303XXXcfChQvZvXs3EydO5OWXXyY/P5/Vq1cTHR1Nenp6QG82Gj58OMuXL+fNN99k0qRJ/PjHP6Zjx44N/ktA9YgVIWratWnTpk67v/zlL4waNeokvkNjwoynEgq2Ob30fZsgf5Pzef5m8JQ7bSQCkk47speelNGs4+hNyYK+ESZOnMjkyZPZt28fH330EQsWLKBz585ER0fz4YcfsmPHUZeDrmPHjh2kpqYyefJkDh8+zBdffMFDDz3E1KlT2bZtW83QTVJSEsOHD+fll1/m5z//OcuWLSM5OZn27dsfcc9Ro0bx5JNPcskllxAdHc3mzZtJTU2t88fAmLBVVe7MdMn/2i/MN8H+LeCtrG3XsSek9IFTRzi98y7V4+jxblXeLCzoG6Ffv34cOnSI1NRUunXrxs0338zVV19NZmYmgwYNok+fPgHdZ9myZfzud78jOjqatm3b8uKLL5KSksKcOXMYP348Xq+Xzp078+677zJz5kzuuOMOBg4cSEJCAi+80PC+LXfffTfbt2/n7LPPRlVJSUlh8eLFwfz2jXFfRQns/8bpked/XRvoBdmgHqeNRDizWlL6QO/RzmNKbyfQY1pnx0caGhpwU2ZmptbfYWrjxo2ceeaZLlXU+tjP27iuvLjeUIvv0X8uekSUM+SS0tv34Qv0TqeHfQ+9ISKyWlUzGzpnPXpjjHtKDzghXj/Ui/y2sYiMgU69oPvZzlz06lBPOhWiYtyrvQWxoG9Ca9euZdKkSXWOxcbG8vnnn7tUkTEuOby/7lBL9aP/tMWoOGd4pef5kHK7r4fexxmGibSoOhn202tCAwYMqJnzbkyrcnAXZD0LO1Y4oV6yr/ZcTNvaaYv+Qy4de4bNLJdQY0FvjAme3WthxWxYuxC8VZCWCX3G1IZ5Sh9on9ri5qG3dBb0xpiTowpb3oPP/gLbPoLoNpB5J5z3fWcc3bjOgt4Yc2Iqy+Crf8DKvzrDM+26waUznGV24xPdrs74saA3xjTO4X2w6llY9TQczocuA+Dav0G/8TYLJkQFsmesAQ4cOMBf//rXRl83ZswYDhw40AQVGdPM9n0Db9wP/9cPlj0K3QfDra/D9z+GsyZayIcw69EHqDroq1evrObxeIiMPPpMgbfeeuuo55rT8eo0pkGqsP0TWPEEbF4KkbFw1g1w3lRn4wzTIrS8oP/3dOeV/WDqOgCueOyYTaZPn87WrVsZNGhQzdIF3bp1Y82aNWzYsIFx48aRk5NDWVkZ9913H1OmTAEgPT2drKwsiouLueKKK7jgggv47LPPSE1N5fXXX69ZfbK+xx9/nKeeeoqoqCj69u3L/PnzKS4u5t577yUrKwsRYcaMGUyYMIF58+bx6KOPoqpceeWV/OY3vwGcXa4eeOAB3n77bf7whz8QHx/PAw88QHFxMcnJyTz//PN069YtuD9LEx48lbD+NSfgd30JCZ3gogfh3LuhbWe3qzON1PKC3iWPPfYY69atY82aNSxbtowrr7ySdevW1SwDPHfuXJKSkigtLeXcc89lwoQJdOrUqc49vvnmG+bNm8fTTz/N9ddfz6JFi7jllluO+vW2bdtGbGxszdDPI488QocOHVi71vlDV1hYSF5eHg8++CCrV68mMTGRkSNHsnjxYsaNG8fhw4fp378/s2bNorKykosuuojXX3+dlJQU/vGPf/DQQw8xd67t2278lB6AL16Az/8GB3c670i96k/O0EwrXFYgXLS8oD9Oz7u5DBkypM5a748//jivvfYaADk5OXzzzTdHBH1GRgaDBg0C4JxzzmH79u1Hvf/AgQO5+eabGTduHOPGjQPgvffeY/78+TVtEhMTWb58OSNGjCAlJQWAm2++meXLlzNu3DgiIyOZMGECAJs2bWLdunVcfvnlgDOUY715U6NwB3z+FHzxIlQUQ/qFcOUfoddIiLCX8lq6gIJeREYDfwYigWdU9bF653sCLwAdfW2mq+pbIpIObAQ2+ZquVNXvB6d0d/kv/7ts2TLee+89VqxYQUJCAiNGjGhwXfrY2NiazyMjIyktLT3q/d98802WL1/OkiVLeOSRR1i/fj2qesSa9cdalC4uLq5mXF5V6devHytWrAj4ezStQG6WM/994xJn1cd+4+H8qdB9kNuVmSA67p9qEYkEZgNXAH2BG0Wkb71mDwMLVHUwMBHwn56yVVUH+T5abMi3a9eOQ4cONXiuqKiIxMREEhIS+Prrr1m5cuVJfS2v10tOTg4XX3wxv/3tbzlw4ADFxcWMHDmSJ554oqZdYWEhQ4cO5aOPPmLfvn14PB7mzZvHRRdddMQ9e/fuTX5+fk3QV1ZWsn79+pOq07RQXg9sWALPjoJnLoWtH8L598B9X8GEpy3kw1AgPfohwBZVzQYQkfnAWGCDXxsFqnfD6ADkBbPIUNCpUyeGDRtG//79iY+Pp0uXLjXnRo8ezVNPPcXAgQPp3bs355133kl9LY/Hwy233EJRURGqyrRp0+jYsSMPP/wwU6dOpX///kRGRjJjxgzGjx/Pr3/9ay6++GJUlTFjxjS4QXlMTAwLFy7khz/8IUVFRVRVVXH//ffTr587WzMaF1Qchv++7LzBqXCbs7bM6Mdg8C0Q287t6kwTOu569CJyHTBaVe/2PZ8EDFXVe/zadAPeARKBNsBlqrraN3SzHtgMHAQeVtWPj/X1bD1699nPO8wc3AX/mQNZc6HsAKRmwnfugT5X26qQYeRk16NvaPWh+n8dbgSeV9U/iMj5wEsi0h/YBfRU1f0icg6wWET6qerBegVOAaYA9OzZM4CSjDHHVX+BsTOvgvPvhZ5D3a7MNLNAgj4X6OH3PI0jh2buAkYDqOoKEYkDklV1L1DuO75aRLYCZwB1uuyqOgeYA06P/gS+jxZr6tSpfPrpp3WO3Xfffdxxxx0uVWRatOoFxlY8AdnLbIExAwQW9KuAXiKSAezEebH1pnptvgUuBZ4XkTOBOCBfRFKAAlX1iMipQC8g+0QKbWjGSTiYPXu22yXUEWpbS5oAVZbB2gVOD94WGDP1HDfoVbVKRO4B3saZOjlXVdeLyCwgS1WXAD8CnhaRaTjDOrerqorIcGCWiFQBHuD7qlrQ2CLj4uLYv38/nTp1CsuwDxWqyv79+4mLi3O7FHMslWXOVnuF252P/Vth3UJbYMwcVYvYHLyyspLc3NwG56ab4IqLiyMtLY3o6Gi3S2m9VKF4jy/Id9QGeuF2OLADDuZR52WyqDjIGO7Mf8+4yDb1aKVa/Obg0dHRdd6FakyLV17shHZDQV64A6rqvZmuXXdn79SMi5zHxFN8j+nQtouFuzmmFhH0xrQ4Xo/T8z4ixH2fH86v2z6mnRPanU6H0y+rDfHEdOjQA6JtOM2cOAt6Y05U6YGjB/mBHPBW1raVSOiQ5gR37yvqBnnHdEhIsl65aTIW9MYEorIUVj0Duatqw7ysqG6b+CRnSKXbWdB3rC/EfUMsHdIg0l73MO6woDfmWFTh63/B2z+DA99C0mmQlAFp59aGePWYeVwHt6s1pkEW9MYczd6vYemDzhuPOveF295wZrcY08JY0BtTX+kB+Og3zuYbsW3hit857y61dWFMC2W/ucZU83rgv3+H92dByX4453a45OfQptNxLzUmlFnQGwOQ8x9468ewaw30PB+ueNV5UdWYMGBBb1q3Q7vh3Rnw1XznTUkTnoX+E2yqowkrFvSmdaoqh5VPwvLfgacCLvwRXPCAMyZvTJixoDetz+Z3YOl0KNgKvcfAqF/ZEr4mrFnQm9Zj/1ZY+lP45m3o1AtuXgS9LnO7KmOanAW9CX/lh2D575212qPiYOQvYcj3bBlf02pY0JvwpQpfLYB3fwHFu2HQzc5mHO26HP9aY8KIBb0JT3n/hX8/CDmfQ/ezYeLLkNbgUt3GhD0LehNeDu9z3vD0xYvQJhnGzoazboKICLcrM8Y1FvQmPHgqYdWz8OGjUHnY2W3pop/YQmPGYEFvwkH2Mvj3dMjfCKddAqMfg5TebldlTMiwoDctV+EOeOdh2LjEWSp44ivOvHh7V6sxdVjQm5anogQ+/TN8+ieQCGfhsfPvse32jDkKC3rTcqjChtedXnxRDvS/Di6fBR1S3a7MmJBmQW9ahj3rnemS2z+GLgNg/Bw45TtuV2VMixDQnDMRGS0im0Rki4hMb+B8TxH5UET+KyJficgYv3M/9V23SURGBbN40wqUFDjLBz91AexZB1f+Eb73kYW8MY1w3B69iEQCs4HLgVxglYgsUdUNfs0eBhao6pMi0hd4C0j3fT4R6Ad0B94TkTNU1RPsb8SEGa8HvngB3n8Eyg5A5l1w8c8gIcntyoxpcQIZuhkCbFHVbAARmQ+MBfyDXoH2vs87AHm+z8cC81W1HNgmIlt891sRhNpNuPr2c3jrR7B7LZxyAVzxG+ja3+2qjGmxAgn6VCDH73kuMLRem5nAOyJyL9AGqF4SMBVYWe/aI145E5EpwBSAnj17BlK3CVfrX4NFd0PbrnDdc9DvWpsuacxJCmSMvqH/y7Te8xuB51U1DRgDvCQiEQFei6rOUdVMVc1MSUkJoCQTltYuhIV3Qdq58IMV0H+8hbwxQRBIjz4X6OH3PI3aoZlqdwGjAVR1hYjEAckBXmsMrJkHr/8AThkGN863nZ6MCaJAevSrgF4ikiEiMTgvri6p1+Zb4FIAETkTiAPyfe0mikisiGQAvYD/BKt4Eya+eBEW/w9kDIebFljIGxNkx+3Rq2qViNwDvA1EAnNVdb2IzAKyVHUJ8CPgaRGZhjM0c7uqKrBeRBbgvHBbBUy1GTemjlXPwpsPwOmXwQ1/h+h4tysyJuyIk8ehIzMzU7OystwuwzSHz/8G//4JnHEFXP8CRMW6XZExLZaIrFbVBjddsEW6jTs+e8IJ+T5XwfUvWsgb04Qs6E3z+/iP8M5D0HccfPd527vVmCZma92Y5rXsN7DsURjwXRj3FETar6AxTc3+LzPNQxU+/BUs/52ztd/YJyAi0u2qjGkVLOhN01OF92Y668effStc9Wfbw9WYZmRBb5qWKrz9EKyc7SxMNub3FvLGNDMLetN0VJ2ZNf+ZA0O/7+zlaksaGNPsLOhN0/B6nTdCrX7O2eZv5C8t5I1xiQW9CT6vB5b8ENb8HS54AC79hYW8MS6yoDfB5alyFif76h9w0XQYMd1C3hiXWdCb4PFUwWtTYN0iuPhhuOjHbldkjMGC3gSLpxIW3gkbl8Bl/wsX3O92RcYYHwt6c/KqyuGfd8CmN2HUr+H8H7hdkTHGjwW9OTmVZbBgEnzzjjNHfshktysyxtRjQW9OXGUpzL8Jtn4AV/0JMu9wuyJjTAMs6M2JqTgM8ybCto9h7GwYfIvbFRljjsKC3jRe+SF4+XrIWQnX/g3OusHtiowxx2BBbxqn7CC8fB3kZsH4p2HAdW5XZIw5Dgt6E7jSA/D38bDrS/juc9B3rNsVGWMCYEFvAlNSAC+Ngz0bnK3/+lzpdkXGmABZ0JvjO7wPXhwH+zbDxFfgjJFuV2SMaQQLenNsxXvhxbFQkA03zoPTL3W7ImNMIwW0A4SIjBaRTSKyRUSmN3D+/0Rkje9js4gc8Dvn8Tu3JJjFmyZ2aDc8fyUUboebFljIG9NCHbdHLyKRwGzgciAXWCUiS1R1Q3UbVZ3m1/5eYLDfLUpVdVDwSjbNomgnvHC1E/Y3L4T0YW5XZIw5QYH06IcAW1Q1W1UrgPnAsaZb3AjMC0ZxxiUHvoXnxzjDNpNes5A3poULJOhTgRy/57m+Y0cQkVOADOADv8NxIpIlIitFZNxRrpvia5OVn58fYOmmSRRuh+euhJJCuPV16DnU7YqMMScpkKBvaNcIPUrbicBCVfX4HeupqpnATcCfROS0I26mOkdVM1U1MyUlJYCSTJPYvxWeGwPlB+G21yHtHLcrMsYEQSBBnwv08HueBuQdpe1E6g3bqGqe7zEbWEbd8XsTKvI3OyFfVQa3/wu6238mY8JFIEG/CuglIhkiEoMT5kfMnhGR3kAisMLvWKKIxPo+TwaGARvqX2tctnejM7tGPXDbv6DrALcrMsYE0XFn3ahqlYjcA7wNRAJzVXW9iMwCslS1OvRvBOarqv+wzpnA30TEi/NH5TH/2TomBOxe58yTj4h0Qj6lt9sVGWOCTOrmsvsyMzM1KyvL7TJah7w1zrIGUfFw2xuQfLrbFRljTpCIrPa9HnoEe2dsa1W00wn5mLZw2xJIOtXtiowxTSSgd8aaMOP1wuLvQ1UFTFpsIW9MmLMefWu0cjZsWw5XP27DNca0Atajb212r4X3Z0Gfq+DsW92uxhjTDCzoW5PKUlh0N8QnOr15aei9cMaYcGNDN63JezMh/2u4ZRG06eR2NcaYZmI9+tZiy3vw+VMw9Ptw+mVuV2OMaUYW9K3B4X2w+AeQciZcNtPtaowxzcyGbsKdKrxxH5QWOkM20fFuV2SMaWbWow93X7wIX/8LLv2FrWFjTCtlQR/O9m+FpdMhYzicN9XtaowxLrGgD1eeSnh1MkTGwLinIML+UxvTWtkYfbj66LewczV893no0OCGYMaYVsK6eeHo25Xw8e/hrJug37VuV2OMcZkFfbgpOwivToEOPeCK37hdjTEmBNjQTbj594NQlAN3/Bvi2rtdjTEmBFiPPpysfw2+fAUu/H/Q8zy3qzHGhAgL+nBRtBPeuB9Sz4GLfuJ2NcaYEGJBHw6qNxLxVML4pyEy2u2KjDEhxMbow4H/RiKdTnO7GmNMiLEefUtnG4kYY47Dgr4lqyyFRZNtIxFjzDEFFPQiMlpENonIFhGZ3sD5/xORNb6PzSJywO/cbSLyje/jtmAW3+q9NxPyN8K4v9pGIsaYozruGL2IRAKzgcuBXGCViCxR1Q3VbVR1ml/7e4HBvs+TgBlAJqDAat+1hUH9Lloj20jEGBOgQHr0Q4AtqpqtqhXAfGDsMdrfCMzzfT4KeFdVC3zh/i4w+mQKNsDh/baRiDEmYIEEfSqQ4/c813fsCCJyCpABfNCYa0VkiohkiUhWfn5+IHW3Xqrwxg+djUQmPG0biRhjjiuQoG/oFT49StuJwEJV9TTmWlWdo6qZqpqZkpISQEmtmG0kYoxppECCPhfo4fc8Dcg7StuJ1A7bNPZaczy2kYgx5gQEEvSrgF4ikiEiMThhvqR+IxHpDSQCK/wOvw2MFJFEEUkERvqOmcayjUSMMSfouLNuVLVKRO7BCehIYK6qrheRWUCWqlaH/o3AfFVVv2sLROQRnD8WALNUtSC430IrYRuJGGNOkPjlckjIzMzUrKwst8sILd+uhOeugIET4don3a7GGBOCRGS1qmY2dM7+/R/qbCMRY8xJskXNQp1tJGKMOUnWow9ltpGIMSYILOhDlW0kYowJEgv6UOT1wuL/sY1EjDFBYWP0oWjlbNj2kW0kYowJCuvRhxrbSMQYE2QW9KHENhIxxjQBG7oJJdUbidyyyDYSMcYEjfXoQ4VtJGKMaSIW9KHANhIxxjQhG7pxm/9GIrcsso1EjDFBZz16t9lGIsaYJmZB7ybbSMQY0wws6N1iG4kYY5qJjdG7xTYSMcY0E+tGuuHbz+Hj38NZN0G/a92uxhgT5qxH39zKDjpDNraRiDHNotLjpaTCQ0lFFYfLPZRWeCiv8uDxKh5VvF58j+p3zHn0eBWvKh4veL1Klf/5mnP+x6hzvf89PfWv8XLE9enJbXhwdJ+g/wws6JubbSRiTIOqPF5KKj2UlHs4XFFFaYWHw+VVvpB2jpWUV3G4wglr57nHd01V7TUVzvPqe1V4vM3+vUQIREYIESJERgiRIkRGOo8R1c8jhIgI6hyLiWqaQRYL+uZUvZHI8J/YRiLmhJVVetiaX8ym3YfYtOcQm3cfori8CkEQgQhxAiRCBBEhovqYgIgg1LZxzte2qb7+yDbV96jXJsLva8qRX7PS4+VwudOb9u9VV4dzzbEKDxVVgQdyhECbmCgSYiNJiIkiISaSNjFRdEyIITWx9lhCTBRtYiKJj4mkTWztsZioCKL8g9j386oTzvWCOiICoiIiasLZCWq/0K55dH5mocSCvrnYRiKmkTxe5duCEjbtPsim3cVs2nOQTbsPsX1/CR6vAhAdKZyW0pbEhBgUxatQ5fXi9TjDAl4FVWe4QJU6z71KzXGt99zr10brHGu4DfWf+0QItUEcG0V8dCRtYiPpGB9N9w5xThDH+oLYP5xj/cPa/1rnWGxURMiFaSizoG8OtpGIOQZVZe+hcqeH7uulb9p9iG/2HqKs0unlikDPpATO6NKOMQO6cUaXdvTp2o705DZER4benAr1jT1HRogFcggIKOhFZDTwZyASeEZVH2ugzfXATECBL1X1Jt9xD7DW1+xbVb0mCHW3LLaRiPE5WFbJ5t2H+Hr3ITbvqX08UFJZ0yalXSy9u7Tj5qGn0LtLO3p3bUevLm1JiGk5/TIRISrSAj5UHPc3R0QigdnA5UAusEpElqjqBr82vYCfAsNUtVBEOvvdolRVBwW57pZj52rbSKQVKq/ysGVvcW2Y+3rreUVlNW3axkZxRpe2XNG/qy/Q23NGl7Z0ahvrYuUmHAXSRRgCbFHVbAARmQ+MBTb4tZkMzFbVQgBV3RvsQlukkgJYcBu07QLX/MU2EglDtePoTpA7wX6wwXH0czOS6N21XU0vPbVjvA1rmGYRSNCnAjl+z3OBofXanAEgIp/iDO/MVNWlvnNxIpIFVAGPqeri+l9ARKYAUwB69uzZqG8gZHm98OoUKN4Ddy6FhCS3KzL1aJ05zNTOm25oLrXXeZEzt7C0ppfe0sfRTesRSNA31OXQes+jgF7ACCAN+FhE+qvqAaCnquaJyKnAByKyVlW31rmZ6hxgDkBmZmb9e7dMH/8etrwLV/7RmWljGlRW6WHngVJyCkrILSz1fZRQUuGp9+YS/zengMfrrXkTy5GhXP157RtSjnwDTN3ZIY0VDuPopvUI5LcyF+jh9zwNyGugzUpVrQS2icgmnOBfpap5AKqaLSLLgMHAVsLZlvfhw0dh4A2Qeafb1biqospL3oFScgpLakI8p8D3WFhK/qHyOu1jIiPo3jGOtnFRdd5IEhEhRE7W3lkAAA20SURBVEVEEBtVfYyauc5Hznf2e6z3hhT/uc+1x/Dd/1hzqJ12XdrH0btLOxtHNy1KIEG/CuglIhnATmAicFO9NouBG4HnRSQZZygnW0QSgRJVLfcdHwb8NmjVh6IDObDobuh8Jlz1f2E/Ll/p8bK7qMyvR+4EeK4v2HcfLEP9es6REUL3jnH0SEzg4t4p9EhMIC0p3nlMTKBzu1giIsL7Z2ZMcztu0KtqlYjcA7yNM/4+V1XXi8gsIEtVl/jOjRSRDYAH+LGq7heR7wB/ExEvzgJqj/nP1gk7VRXwz9ud+fLXvwQxbdyu6KR5vMrug2XkFtQGeHWPPLewlF1FpUe8QaZbh3jSEuP5zmnJpCXG0yMpoeaxS7tYomy82phmJaqhNSSemZmpWVlZbpdxYt76MfxnDlz/IvQd63Y1AfF6nTfrOD3xEnIL/IdZSsk7UEqVX5KLQJd2cXUDPNF5TEtMoFvHOHvh0RgXiMhqVc1s6Jy9chQsX/3TCfnz7wnZkD9YVsn6nQdZn1fE2p1FrM87yLf7S45Y9CmlXSxpifGc1aMjVw3sRlpiAj2SnCDv3jGO2KhIl74DY8yJsKAPhr1fOxt89zwfLpvpdjUAFJVUsi6viHU7nVBft7OI7ftLas53bR9H/9T2XNqnM2n1euZx0RbkxoQTC/qTVX4IFkxyxuOve86VdWwKD1c4Ye4X7DkFpTXnUzvG0z+1Pdedk0a/1A70796BlHY2a8SY1sKC/mSowpJ7Yf8WuHUJtO/W5F9yX3G5M+xS01M/yM4DtaHeIymeAakduHFIT/p370D/1A4ktYlp8rqMMaHLgv5kfP43Z435y2ZCxoVBv/3eg2U1YV49/LL7YO1aKemdEhjcsyO3nn8K/X099Q4JtjKmMaYuC/oT9e3n8M5D0HsMDLv/pG6l6kxhXJtbxLq8gzXDL9VvJhKBjOQ2DD01qaaX3i+1Pe3jLNSNMcdnQX8iivOd+fId0mDck416U5SqsvNAKev8eurr84rYV1wBOPPQT0tpy4WnJ9MvtQMDUjvQt3t72sbafypjzImx9GgsrwcW3QWlBXDXuxDf8ZjNqzxePtqcT9aOQl+4F1HoW3s8MkLo1bktI3p3pn/39gxI68CZ3drbeinGmKCyRGmsDx91NhG55gnoNvCozQ6XV/HPrBye/XQbOQWlREUIZ3Rpx8i+Xemf2p7+qU6o21RGY0xTs6BvjM1vO6tSDp4EZ09qsMneg2W8sGI7f1/5LUWllZxzSiIPjTmTEb07W6gbY1xhQR+owu3w6mToOgDG/O6I05v3HOLp5dm8viaPSq+X0f26cveFp3LOKYnNX6sxxvixoA9EZRksuNVZhf/6lyA6HnBeWF2xdT9zPs5m2aZ84qIjmDikB3cOyyA9ueUvaGaMCQ8W9IFY+iDs+hImzoOkDCo9Xt78ahdPf5zN+ryDJLeN4UeXn8Et551Cor05yRgTYizoj2fNK7D6ebhgGofSL2f+8mzmfrqNXUVlnJbShsfGD2Dc4FQbfzfGhCwL+mPZvRb+NY3ytGH8sXwCr/z6Aw6VV3HeqUn86tr+jDijs22SYYwJeRb0R1NWRPkrt1BGG0Zm38K+7BzGDOjG5AszGJh27LnzxhgTSizo61FVPtq0lzaLb2dw6bfcozO46juDuGNYOmmJCW6XZ4wxjWZB71Ne5WHJmjye+Xgbw/fN46Hoz/is1wM8MWEqHeJtTRljTMvV6oO+qKSSv3++g+c/207+oXKuS97BT2P+gbfPWL5z/S/CfnNvY0z4a7VBn1NQwrOfbGNBVg4lFR4u7JXMX67uxtB3piFJGTD2CQt5Y0xYaHVBvybnAE8vz+bf63YRGSFcfVZ3Jl94Kmd2ToAXr3F2jJq0GOLau12qMcYERasIeq9Xef/rvTy9PJv/bC+gXVwUU4afxu3fSadrhzin0Ts/hx2fwrVzoEtfdws2xpggCuugL6v08OoXO3nm42yy9x0mtWM8P7+qLzec26Pu+u4b34DPHofMu+CsG9wr2BhjmkBAQS8io4E/A5HAM6r6WANtrgdm4qwI86Wq3uQ7fhvwsK/ZL1X1hSDUfUz7i8t5aeUOXlqxg/2HKxiQ2oHHbxzMmP5diYqMqNd4Kyz+AXQ/G0b/uqlLM8aYZnfcoBeRSGA2cDmQC6wSkSWqusGvTS/gp8AwVS0Ukc6+40nADCAT5w/Aat+1hcH/ViA7v5hnP9nGwtW5lFd5ubRPZyYPP5WhGUlIQy+sVpQ4i5VFRML1L0BUbFOUZYwxrgqkRz8E2KKq2QAiMh8YC2zwazMZmF0d4Kq613d8FPCuqhb4rn0XGA3MC075tbbvO8ylf/yI6IgIxp+dyt0XZnB653ZHv0AV3vwR7FkPNy+Ejj2DXZIxxoSEQII+Fcjxe54LDK3X5gwAEfkUZ3hnpqouPcq1qfW/gIhMAaYA9Ox5YoGbntyGX40bwOV9u5DSLoCe+RcvwJevwEUPQq/LTuhrGmNMSxBI0Dc0mVwbuE8vYASQBnwsIv0DvBZVnQPMAcjMzDzifKBuGhrgH4m8/8JbP4HTLnGC3hhjwljE8ZuQC/Twe54G5DXQ5nVVrVTVbcAmnOAP5NrmVVLgjMu3SYHxzzjj88YYE8YCCfpVQC8RyRCRGGAisKRem8XAxQAikowzlJMNvA2MFJFEEUkERvqOucPrhde+Dwd3OS++tunkWinGGNNcjjt0o6pVInIPTkBHAnNVdb2IzAKyVHUJtYG+AfAAP1bV/QAi8gjOHwuAWdUvzLrikz/AN2/DmN9DWqZrZRhjTHMS1RMeEm8SmZmZmpWVFfwbZy+Dl66FfuNhwjO2jo0xJqyIyGpVbbAHG8jQTctXtBMW3gXJZ8DVf7aQN8a0KuEf9FUV8M/boaoMrn8JYtu6XZExxjSrsF7rBoB3fwG5/4HrnoOUM9yuxhhjml149+jXLYLPn4Sh/wP9x7tdjTHGuCJ8gz5/Myz5IaQNgctnuV2NMca4JjyDvrwYFkyCqDj47vMQFeN2RcYY45rwG6NXhTfug32bYdJr0OGIpXWMMaZVCb8e/apnYN1CuPhncOoIt6sxxhjXhVfQ56yCpT+FXqPggh+5XY0xxoSE8An6w/ud+fLtu8H4v0FE+HxrxhhzMsIrDbsOgOtfhPhEtysxxpiQET4vxrbpBDfNd7sKY4wJOeHVozfGGHMEC3pjjAlzFvTGGBPmLOiNMSbMWdAbY0yYs6A3xpgwZ0FvjDFhzoLeGGPCXMhtDi4i+cCOk7hFMrAvSOUEk9XVOFZX41hdjROOdZ2iqikNnQi5oD9ZIpJ1tJ3Q3WR1NY7V1ThWV+O0trps6MYYY8KcBb0xxoS5cAz6OW4XcBRWV+NYXY1jdTVOq6or7MbojTHG1BWOPXpjjDF+LOiNMSbMhU3Qi8hoEdkkIltEZLrb9VQTkbkisldE1rldSzUR6SEiH4rIRhFZLyL3uV0TgIjEich/RORLX13/63ZN/kQkUkT+KyL/crsWfyKyXUTWisgaEclyu55qItJRRBaKyNe+37XzQ6Cm3r6fU/XHQRG53+26AERkmu/3fp2IzBORuKDdOxzG6EUkEtgMXA7kAquAG1V1g6uFASIyHCgGXlTV/m7XAyAi3YBuqvqFiLQDVgPj3P55iYgAbVS1WESigU+A+1R1pZt1VRORB4BMoL2qXuV2PdVEZDuQqaoh9QYgEXkB+FhVnxGRGCBBVQ+4XVc1X27sBIaq6sm8STMYtaTi/L73VdVSEVkAvKWqzwfj/uHSox8CbFHVbFWtAOYDY12uCQBVXQ4UuF2HP1Xdpapf+D4/BGwEUt2tCtRR7Hsa7fsIiZ6IiKQBVwLPuF1LSyAi7YHhwLMAqloRSiHvcymw1e2Q9xMFxItIFJAA5AXrxuES9KlAjt/zXEIguFoCEUkHBgOfu1uJwzc8sgbYC7yrqiFRF/An4CeA1+1CGqDAOyKyWkSmuF2Mz6lAPvCcb7jrGRFp43ZR9UwE5rldBICq7gR+D3wL7AKKVPWdYN0/XIJeGjgWEj3BUCYibYFFwP2qetDtegBU1aOqg4A0YIiIuD7cJSJXAXtVdbXbtRzFMFU9G7gCmOobLnRbFHA28KSqDgYOA6H02lkMcA3wT7drARCRRJxRiAygO9BGRG4J1v3DJehzgR5+z9MI4j97wpFvDHwR8LKqvup2PfX5/pm/DBjtcikAw4BrfGPh84FLROTv7pZUS1XzfI97gddwhjLdlgvk+v2LbCFO8IeKK4AvVHWP24X4XAZsU9V8Va0EXgW+E6ybh0vQrwJ6iUiG7y/1RGCJyzWFLN+Lns8CG1X1j27XU01EUkSko+/zeJxf/q/drQpU9aeqmqaq6Ti/Wx+oatB6WydDRNr4XlDHNzQyEnB9hpeq7gZyRKS379ClgOuTI/zcSIgM2/h8C5wnIgm+/z8vxXntLCiignUjN6lqlYjcA7wNRAJzVXW9y2UBICLzgBFAsojkAjNU9Vl3q2IYMAlY6xsPB/iZqr7lYk0A3YAXfLMhIoAFqhpSUxlDUBfgNScbiAJeUdWl7pZU417gZV/nKxu4w+V6ABCRBJwZet9zu5Zqqvq5iCwEvgCqgP8SxOUQwmJ6pTHGmKMLl6EbY4wxR2FBb4wxYc6C3hhjwpwFvTHGhDkLemOMCXMW9MYYE+Ys6I0xJsz9fyVAHstooMoyAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXoAAAD4CAYAAADiry33AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAgAElEQVR4nO3deXjU1dn/8fdNWEIgkgBRlgAJlH0RS4ooYqGKjaiAihaxFNRHHi34KMUiVKtoXahga7XYSi0Fn0cFRJAg/ERRFBVQJhiDBJEQgcSwhH2RQJb798cMOsZgJjDJmczcr+vKxXzX+UyM95w53++cI6qKMcaY8FXLdQBjjDFVywq9McaEOSv0xhgT5qzQG2NMmLNCb4wxYa626wBlNW3aVJOSklzHMMaYGiU9PX2vqiaUty3kCn1SUhIej8d1DGOMqVFEZPvptlnXjTHGhDkr9MYYE+YCKvQikioim0UkW0QmlbO9jYi8IyKZIvKeiCT6bRslIlt8P6OCGd4YY0zFKuyjF5EoYAYwEMgD1olImqpm+e02HXhRVeeIyC+AJ4CRItIYeAhIARRI9x17oDIhi4qKyMvLo7CwsDKHRbzo6GgSExOpU6eO6yjGGIcCuRjbG8hW1RwAEZkLDAH8C30XYLzv8Urgdd/jXwJvq+p+37FvA6nAK5UJmZeXR2xsLElJSYhIZQ6NWKrKvn37yMvLIzk52XUcY4xDgXTdtARy/ZbzfOv8fQZc73t8LRArIk0CPBYRGSMiHhHxFBQU/CBAYWEhTZo0sSJfCSJCkyZN7FOQMSagQl9edS075OW9wM9F5FPg58DXQHGAx6KqM1U1RVVTEhLKvQ3UivwZsN+ZMQYC67rJA1r5LScC+f47qGo+cB2AiDQErlfVQyKSB/Qvc+x7Z5HXhLvSUvjsFTiwzXUSY6rfOS0g5ZagnzaQQr8OaC8iyXhb6sOBEf47iEhTYL+qlgKTgVm+TcuBx0Uk3rd8hW+7MT/0zX5YdAdsWe5bYZ9ITIRJTHFT6FW1WETG4S3aUcAsVd0oIo8AHlVNw9tqf0JEFFgFjPUdu19E/oT3zQLgkVMXZsNdw4YNOXr0aLnbtm3bxurVqxkxwvt+OXv2bDweD3//+9+rM2Jo+Xo9vDoKDu+EQdPhZ/8F1vVkTFAENASCqi4DlpVZ96Df4wXAgtMcO4vvWvgGb6F/+eWXvy30gSopKSEqKqqKUjmiCp5Z8OYkaHge3LocEnu5TmVMWAm5sW4q8vCSjWTlHw7qObu0OIeHrun6o/vcd999tGnTht/+9rcATJkyBRFh1apVHDhwgKKiIh599FGGDBlS4fNNmjSJTZs20bNnT0aNGkV8fDz5+fmkpqaydetWrr32Wp588knA+8ngd7/7HcuXL+epp57ikksuOfsXHCpOHoMl98CG+fCTgXDdTIhp7DqVMWHHhkAI0PDhw5k3b963y/Pnz+eWW25h0aJFrF+/npUrVzJhwgQCmYN36tSp9OvXj4yMDMaP9379ICMjg3nz5rFhwwbmzZtHbq73rtRjx47RrVs3Pv744/Aq8gVfwr9+ARtehQEPwIj5VuSNqSI1rkVfUcu7qlxwwQXs2bOH/Px8CgoKiI+Pp3nz5owfP55Vq1ZRq1Ytvv76a3bv3k2zZs0qff7LLruMRo0aAdClSxe2b99Oq1atiIqK4vrrr6/g6BpmwwJI+x+oUx9GLoJ2A1wnMias1bhC79KwYcNYsGABu3btYvjw4bz00ksUFBSQnp5OnTp1SEpKOuMvKNWrV+/bx1FRURQXFwPeYQzCpl+++CS8dT98MhNa9YEb/uO9ncwYU6Ws0FfC8OHDuf3229m7dy/vv/8+8+fP59xzz6VOnTqsXLmS7dtPOxz098TGxnLkyJEqThtiDubCq6Phaw9cNA4unwJRNgaPMdXBCn0ldO3alSNHjtCyZUuaN2/OzTffzDXXXENKSgo9e/akU6dOAZ2nR48e1K5dm/PPP5/Ro0cTHx9f8UE12ZYVsPC/oLQEbvxf6DLYdSJjIooEcvGwOqWkpGjZGaY2bdpE586dHSWq2Zz+7kpL4P0/w/tPwnld4cYXoUk7N1mMCXMikq6qKeVtsxa9qRrH9sJrt0HOe9Dz13DVdO/FV2NMtbNCX4U2bNjAyJEjv7euXr16fPzxx44SVZMdH3v744/vh8F/h5+OrPAQY0zVsUJfhbp3705GRobrGNVHFdY+B28/CI1awW1vQ/MerlMZE/Gs0JvgKDwMi8fCpjTodDUMmQH141ynMsZghd4Ew67PYf5vvEMLX/Go9/ZJG5DMmJBhhd6cnU9fgqW/g+g4GP0GtLnYdSJjTBlW6M2ZKToO/28irH8RkvrBsFnQ8FzXqYwx5bBBzQJ08OBBnnvuuUofN2jQIA4ePFjp42bPnk1+/ncTeSUlJbF3795Kn6dK7M+Bfw/0Fvl+98JvFluRNyaEWaEP0OkKfUlJyY8et2zZMuLiKn9RsmyhD8Sp8XGq1KY34Pn+3iENRsyHy/4ItcJkLB5jwlTN67r5f5Ng14bgnrNZd7hy6o/uMmnSJLZu3UrPnj2pU6cODRs2pHnz5mRkZJCVlcXQoUPJzc2lsLCQu+++mzFjxgDelrjH4+Ho0aNceeWVXHLJJaxevZqWLVuyePFi6tf/4ZeIFixYgMfj4eabb6Z+/fqsWbMGgGeffZYlS5ZQVFTEq6++SqdOnZgyZQr5+fls27aNpk2b8vLLLwf3d3NKSRG88zCsfhZaXAA3zIH4NlXzXMaYoLIWfYCmTp1Ku3btyMjIYNq0aXzyySc89thjZGVlATBr1izS09PxeDw888wz7Nu37wfn2LJlC2PHjmXjxo3ExcXx2muvlftcw4YNIyUlhZdeeomMjIxv3wyaNm3K+vXrufPOO5k+ffq3+6enp7N48eKqK/KHd8Kcwd4i/7P/8s4CZUXemBqj5rXoK2h5V5fevXuTnJz87fIzzzzDokWLAMjNzWXLli00adLke8ckJyfTs2dPAHr16sW2bdsq9ZzXXXfdt8cuXLjw2/WDBw8u95NBUHy1Chbc6p0N6roXoMcNVfM8xpgqU/MKfYho0KDBt4/fe+89VqxYwZo1a4iJiaF///7ljktfdsz548ePV+o5Tx3vP1592SxBU1oKH/4FVj4GTX4Co96AcwMbndMYE1oC6roRkVQR2Swi2SIyqZztrUVkpYh8KiKZIjLItz5JRI6LSIbv55/BfgHV5cfGkD906BDx8fHExMTwxRdfsHbt2ip9vir3zX54ZTi8+yfoei3cvtKKvDE1WIUtehGJAmYAA4E8YJ2IpKlqlt9uDwDzVfUfItIFWAYk+bZtVdWewY1d/Zo0aULfvn3p1q0b9evX57zzzvt2W2pqKv/85z/p0aMHHTt2pE+fPmf9fKNHj+aOO+743sXYavH1enh1lLdfftB0b5+8fcvVmBqtwvHoReQiYIqq/tK3PBlAVZ/w2+d5IEdV/+zb/ylVvVhEkoA3VLVboIFsPPrgCvh3pwqeWfDmJGh4nveumsReVR/QGBMUZzsefUsg1285D7iwzD5TgLdE5C6gAXC537ZkEfkUOAw8oKoflBNwDDAGoHXr1gFEMkF18hgsuQc2zIefDITrZkJMY9epjDFBEkihL+9ze9mPATcBs1X1KV+L/n9FpBuwE2itqvtEpBfwuoh0VdXD3zuZ6kxgJnhb9JV+FTXY2LFj+eijj7637u677+aWW26pngAFX8L8kVCwGQY8AP0mQC2769aYcBJIoc8DWvktJwJlv7J5G5AKoKprRCQaaKqqe4ATvvXpIrIV6AB4qCRVRcKwr3jGjBlVdu4Kp4n8fCEsHued+WnkImg3oMqyGGPcCaTptg5oLyLJIlIXGA6kldlnB3AZgIh0BqKBAhFJ8F3MRUTaAu2BnMqGjI6OZt++fRUXLvMtVWXfvn1ER0eXv8PX6d6p/pp1gzs+sCJvTBirsEWvqsUiMg5YDkQBs1R1o4g8AnhUNQ2YAPxLRMbj7dYZraoqIpcCj4hIMVAC3KGq+ysbMjExkby8PAoKCip7aESLjo4mMTHxhxuKT3pb8rHN4eYFEH1O9YczxlSbCu+6qW7l3XVjgmzlE/D+VBjxKnS4wnUaY0wQ/NhdN3bVLdLs3ggfTIcev7Iib0yEsEIfSUqKvfO61o+H1NAYM8gYU/VsrJtIsnYG5H8KN8y2++SNiSDWoo8Ue7Nh5ePQ6WroMtR1GmNMNbJCHwlKSyHtLqhdD656ysauMSbCWNdNJPD8G3ashiHPQWwz12mMMdXMWvTh7uAOWDEF2l0GPUe4TmOMccAKfThT9Q5WBnDN09ZlY0yEsq6bcPbZK7D1He+48nE2Kqgxkcpa9OHqyG54czK0vghSbnOdxhjjkBX6cLVsAhQdh8HP2rDDxkQ4qwDhaOPrsGkJDJgMTdu7TmOMccwKfbj5Zj8suxea94SL7nKdxhgTAuxibLh5czIcP+CdSCTK/vMaY6xFH16+fAsy58Ilv4Nm3V2nMcaECCv04aLwMLxxDyR0gkvvdZ3GGBNC7LN9uFjxEBzZCTe+6B3TxhhjfKxFHw6++gA8s6DPbyGx3AlmjDERzAp9TXfyG+/IlPHJMOB+12mMMSHIum5qupWPwYGvYNQbUDfGdRpjTAgKqEUvIqkisllEskVkUjnbW4vIShH5VEQyRWSQ37bJvuM2i8gvgxk+4uWlw9rnoNctkNzPdRpjTIiqsEUvIlHADGAgkAesE5E0Vc3y2+0BYL6q/kNEugDLgCTf4+FAV6AFsEJEOqhqSbBfSMQpPuGd/zW2OQx8xHUaY0wIC6RF3xvIVtUcVT0JzAWGlNlHgXN8jxsB+b7HQ4C5qnpCVb8Csn3nM2frg6egYBNc/TREn1Px/saYiBVIoW8J5Pot5/nW+ZsC/FpE8vC25k999z6QYxGRMSLiERFPQUFBgNEj2K7PvYW+x6+gwxWu0xhjQlwghb682Sq0zPJNwGxVTQQGAf8rIrUCPBZVnamqKaqakpCQEECkCFZS7O2yqR8PqVNdpzHG1ACB3HWTB7TyW07ku66ZU24DUgFUdY2IRANNAzzWVMaav8PODLhhNsQ0dp3GGFMDBNKiXwe0F5FkEamL9+JqWpl9dgCXAYhIZyAaKPDtN1xE6olIMtAe+CRY4SPO3mx47wnodDV0Geo6jTGmhqiwRa+qxSIyDlgORAGzVHWjiDwCeFQ1DZgA/EtExuPtmhmtqgpsFJH5QBZQDIy1O27OUGkppI3zDm9w1VM2/6sxJmABfWFKVZfhvcjqv+5Bv8dZQN/THPsY8NhZZDQAnn/DjjUw5DmIbeY6jTGmBrEhEGqCgztgxRRodxn0HOE6jTGmhrFCH+pUYcnd3sfXPG1dNsaYSrOxbkJdxsuw9V0YNB3iWrtOY4ypgaxFH8qO7ILlk6H1xZBym+s0xpgaygp9qFKFpRO8Y9oMfhZq2X8qY8yZseoRqrJehy/egP6ToelPXKcxxtRgVuhD0Tf7YdnvoXlPuGic6zTGmBrOLsaGojcnwfEDMHIRRNl/ImPM2bEWfaj58i3InAeX/A6adXedxhgTBqzQh5LCw/DGPZDQCS6913UaY0yYsH6BUPL2g3BkJ9z4ondMG2OMCQJr0YeKrz6A9P9An99CYorrNMaYMGKFPhSc/AbS7oL4ZBhwv+s0xpgwY103oWDlY3DgKxj1BtSNcZ3GGBNmrEXvWp4H1j4HvW6B5H6u0xhjwpAVepeKT3jnf41tDgMfcZ3GGBOmrOvGpVXToeALGPEqRJ/jOo0xJkxZi96VXRvgw79Aj19BhytcpzHGhDEr9C6UFMPicVA/HlKnuk5jjAlz1nXjwppnYWcG3DAbYho7i7E0cyfZe446e35jzPc1a1SPX/0s+BMMBVToRSQV+BsQBbygqlPLbP8rMMC3GAOcq6pxvm0lwAbfth2qOjgYwWusvVtg5RPQ6WroMtRZjHc27Wbsy+udPb8x5od6topzU+hFJAqYAQwE8oB1IpKmqlmn9lHV8X773wVc4HeK46raM3iRa7DSUu8Xo+pEw1VPOZv/9XBhEfcv+pxOzWJ5fWxf6kZZD54x4SyQFn1vIFtVcwBEZC4wBMg6zf43AQ8FJ16YWfcC7FgDQ56D2GbOYjyxbBN7jhTy/MheRNeJcpbDGFM9AmnKtQRy/ZbzfOt+QETaAMnAu36ro0XEIyJrRaTcvgoRGePbx1NQUBBg9BrmwHZYMQXaXQY9RziLsTp7L698ksvt/dpyfqs4ZzmMMdUnkEJfXv+Cnmbf4cACVS3xW9daVVOAEcDTItLuBydTnamqKaqakpCQEECkGkYVltzt7aq55mlnXTbfnCzmvoWZJDdtwPiBHZxkMMZUv0AKfR7Qym85Ecg/zb7DgVf8V6hqvu/fHOA9vt9/HxkyXoKclXD5FIgL/oWWQE1f/iW5+48z9bru1mVjTAQJpNCvA9qLSLKI1MVbzNPK7iQiHYF4YI3fungRqed73BToy+n79sPTkV2w/A/Q+mJIuc1ZjPTtB/jP6q8Y2acNF7Zt4iyHMab6VXgxVlWLRWQcsBzv7ZWzVHWjiDwCeFT1VNG/CZirqv7dOp2B50WkFO+bylT/u3XCniosneAd02bws1DLzd0tJ4pLuO+1TFo0qs99V3ZyksEY405A99Gr6jJgWZl1D5ZZnlLOcauByJ34dOMi+OINuPxhaPoTZzGefSeb7D1HmXNrbxrWs+/IGRNp7AbqqnJsHyz7PTTvCReNcxZjY/4h/vH+Vq7/aSI/7xCGF7qNMRWy5l1VeXMSFB6E3yyGKDe/5qKSUiYuyCQ+pi5/vLqzkwzGGPesRV8VvlwOG+ZDvwnQrJuzGDNX5bAx/zCPDu1KXExdZzmMMW5ZoQ+2wkOw5B5I6Az97nUWI3vPUf72zhYGdW9GarfmznIYY9yzrptge/tBOLoLfvV/UNtNK7qkVLnvtUxi6kbx8GB3nyiMMaHBWvTB9NUqSJ8NfX4Lib2cxXhxzTbStx/gwau7kBBbz1kOY0xosEIfLCePeUembNwWBtzvLEbu/m948s3N9O+YwLUXlDskkTEmwljXTbC8+xgc2Aajl0LdGCcRVJXJCzcQVUt4/NruiKMxdYwxocVa9MGQuw7WPgcpt0LSJc5izPfk8mH2XiZd2YkWcfWd5TDGhBYr9Ger+AQsHgvntPR+A9aR3YcLeXTpJi5MbsyI3u4GTjPGhB7rujlbq6bB3s1w8wKIPsdJBFXl/kWfU1RSyp+v70GtWtZlY4z5jrXoz8auDfDhX6HHcGg/0FmMJZk7WbFpNxMGdiSpaQNnOYwxockK/ZkqKYbF46B+PKQ+4SzGvqMnmJK2kfMTG3FL3yRnOYwxocu6bs7UmmdhZwbcMAdiGjuL8fCSLI4UFvHksD7Utkm+jTHlsMpwJvZugZVPQKerocsQZzFWZO0m7bN8xg74CR2bxTrLYYwJbVboK6u01PvFqDrRcNVTzuZ/PXS8iPtf30CnZrH8tr+7se6NMaHPum4qa90LsGMNDHkOYps5i/HEsk0UHDnBzJEp1K1t79fGmNOzClEZB7bDiinQ7jLoOcJZjI+y9zJ3XS6392vL+a3inOUwxtQMVugDpQpL7vZ21VzztLMum29OFjNpYSbJTRswfmAHJxmMMTWLdd0EKuMlyFkJg6ZDnLtvnk5bvpnc/ceZN6YP0XWinOUwxtQc1qIPxJFdsPwP0PpiSLnNWYz07fuZvXobI/u04cK2TZzlMMbULAEVehFJFZHNIpItIpPK2f5XEcnw/XwpIgf9to0SkS2+n1HBDF8tVGHpBO+YNoOfhVpu3hsLi0qYuCCTFo3qc9+VnZxkMMbUTBV23YhIFDADGAjkAetEJE1Vs07to6rj/fa/C7jA97gx8BCQAiiQ7jv2QFBfRVXauAi+eMM7YFlTd7cxPvvuFrYWHGPOrb1pWM963IwxgQukedobyFbVHFU9CcwFfuxbQjcBr/ge/xJ4W1X3+4r720Dq2QSuVsf2wbLfQ/OecNE4ZzE25h/in+/ncP1PE/l5hwRnOYwxNVMghb4lkOu3nOdb9wMi0gZIBt6tzLEiMkZEPCLiKSgoCCR39XhzEhQehCEzIMpNK7qopJSJCzKJj6nLH6/u7CSDMaZmC6TQl3cfoZ5m3+HAAlUtqcyxqjpTVVNUNSUhIURarF8uhw3zod8EaOZugu2Zq3LYmH+YR4d2JS7GzWTjxpiaLZBCnwe08ltOBPJPs+9wvuu2qeyxoaPwECy5BxI6Q797ncXI3nOUv72zhUHdm5HarbmzHMaYmi2QQr8OaC8iySJSF28xTyu7k4h0BOKBNX6rlwNXiEi8iMQDV/jWhba3H4Sju7xdNrXdtKJLSpX7Xsskpm4UDw9294nCGFPzVdjxrKrFIjIOb4GOAmap6kYReQTwqOqpon8TMFdV1e/Y/SLyJ7xvFgCPqOr+4L6EIPtqFaTP9l58TezlLMaLa7aRvv0Af7nxfBJi6znLYYyp+cSvLoeElJQU9Xg8bp785DH4x8UgteCOj6BujJMYufu/4Yq/ruLCto35z+ifIY6GWzDG1Bwikq6qKeVtsxuy/b37GBzYBqOXOivyqsrkhRuIqiU8fm13K/LGmLNmQyCckrsO1j4HKbdC0iXOYsz35PJh9l4mXdmJFnH1neUwxoQPK/TgHd5g8Vg4p6X3G7CO7D5cyKNLN3FhcmNG9HY3cJoxJrxY1w3AqmmwdzPcvACiz3ESQVW5f9HnFJWU8ufre1CrlnXZGGOCw1r0uzbAh3+FHsOh/UBnMZZk7mTFpt1MGNiRpKYNnOUwxoSfyC70JcXeLpv68ZD6hLMY+46eYEraRs5vFcetlyQ7y2GMCU+R3XWz+hnY+RncMAdiGjuL8fCSLI4UFjFtWA+irMvGGBNkkdui37sF3psKna+BrkOdxViRtZu0z/IZN6A9Hc6LdZbDGBO+IrPQl5bC4nFQpz4MespZjEPHi7j/9Q10ahbLnf3bOcthjAlvkdl1s+5fkLsWhv4DYs9zFuOJZZsoOHKCf/0mhbq1I/M91xhT9SKvuhzYDisehp9cDuff5CzGR9l7mbsul9svbUuPxDhnOYwx4S+yCr0qLPkfEIGrn/b+68A3J4uZtDCT5KYNGH95BycZjDGRI7K6bj79P8h5DwZNh7hWFe5eVaYt30zu/uPMG9OH6DpRznIYYyJD5LToD++E5fdDm76QcpuzGOnb9zN79TZG9mnDhW2bOMthjIkckVHoVWHpBCg5AYOfhVpuXnZhUQkTF2TSolF97ruyk5MMxpjIExmFfuNC2LwUBvwBmri7jfHZd7ewteAYj1/XnYb1IqvXzBjjTvgX+mP7YNlEaHEB9BnrLMbnXx/in+/ncP1PE/l5hxCZAN0YExHCv1n55n3eyb6HpEGUm5dbVFLKxAWZxMfU5Y9Xd3aSwRgTucK7Rb/5TdjwKvSbAOd1dRZj5qocsnYe5tGhXYmLcTPZuDEmcoVvoS88BG+Mh3O7eAu9I9l7jvC3FVsY1L0Zqd2aO8thjIlcARV6EUkVkc0iki0ik06zz40ikiUiG0XkZb/1JSKS4ftJC1bwCr31Rzi6C4b8HWq7aUWXlCoTF2QSUy+Khwd3c5LBGGMq7LQWkShgBjAQyAPWiUiaqmb57dMemAz0VdUDInKu3ymOq2rPIOf+cTnvw/o5cPFd0LJXtT61vzmrt7F+x0H+cuP5JMTWc5bDGBPZAmnR9wayVTVHVU8Cc4EhZfa5HZihqgcAVHVPcGNWwsljkHYXNG4L/f/gLMaOfd8wbflm+ndM4NoLWjrLYYwxgRT6lkCu33Keb52/DkAHEflIRNaKSKrftmgR8fjWlzvwu4iM8e3jKSgoqNQL+IF3H4WD22Hw36FuzNmd6wypKpMXZRJVS3j82u6IozF1jDEGAru9srwqpeWcpz3QH0gEPhCRbqp6EGitqvki0hZ4V0Q2qOrW751MdSYwEyAlJaXsuQOX+wms/Yd3iIOkvmd8mrM1b10uH2Xv49Gh3WgRV99ZDmOMgcBa9HmA/whgiUB+OfssVtUiVf0K2Iy38KOq+b5/c4D3gAvOMnP5ik94JxM5pyVcPqVKniIQuw4V8tjSTVyY3JgRvVs7y2GMMacEUujXAe1FJFlE6gLDgbJ3z7wODAAQkaZ4u3JyRCReROr5re8LZFEVjuwCqQXX/A2iz6mSp6iIqvLA6xsoKi3lz9f3oJbN/2qMCQEVdt2oarGIjAOWA1HALFXdKCKPAB5VTfNtu0JEsoAS4Pequk9ELgaeF5FSvG8qU/3v1gmq+DZwx4fOvv0KkPZZPis27eH+QZ1JatrAWQ5jjPEnqmfeJV4VUlJS1OPxuI5RafuOnmDgX1fRqnEMC++8mChrzRtjqpGIpKtqSnnbwvebsdVsypIsjhQWMW1YDyvyxpiQYoU+CN7O2s2Sz/IZN6A9Hc6LdR3HGGO+xwr9WTp0vIj7F22gU7NY7uzvbqx7Y4w5nfAfpriKPb50E3uPnuCFUSnUrW3vm8aY0GOV6Sx8uGUv8zy53H5pW3okxrmOY4wx5bJCf4aOnShm0sJMkps2YPzlHVzHMcaY07KumzM0bflm8g4cZ/5/X0R0nSjXcYwx5rSsRX8GPNv2M2fNNn5zURt6Jzd2HccYY36UFfpKKiwqYeJrmbRoVJ+JqZ1cxzHGmApZ100lPfPOFnIKjvHirb1pWM9+fcaY0Gct+kr4/OtDPL8qh2G9Erm0Q4LrOMYYExAr9AEqKill4oJMGjeoyx+v6uI6jjHGBMz6HgL0/Ptbydp5mH/+uheNYuq4jmOMMQGzFn0Atuw+wjPvZHNV9+akdmvmOo4xxlSKFfoKlJQqE1/LJKZeFFMGd3UdxxhjKs0KfQXmrN7GpzsO8tA1XUiIrec6jjHGVJoV+h+xY983TFu+mQEdExjas6XrOMYYc0as0J+GqjJ5USZRtYTHru2OiE0mYoypmazQn8a8dbl8lL2PyYM60SKuvus4xhhzxqzQl2PXoUIeW7qJPm0bc9PPWruOY4wxZyWgQi8iqSKyWUSyRWTSafa5UUSyRGSjiLzst36UiGzx/YwKVvCqoqo88P+K4jwAAAlASURBVPoGikpL+fP1Pahl878aY2q4Cr8wJSJRwAxgIJAHrBORNFXN8tunPTAZ6KuqB0TkXN/6xsBDQAqgQLrv2APBfynBkfZZPis27eGBqzrTpkkD13GMMeasBdKi7w1kq2qOqp4E5gJDyuxzOzDjVAFX1T2+9b8E3lbV/b5tbwOpwYkefPuOnuDhJVn0bBXHLX2TXccxxpigCKTQtwRy/ZbzfOv8dQA6iMhHIrJWRFIrcSwiMkZEPCLiKSgoCDx9kE1ZksWRwiKeHNaDKOuyMcaEiUAKfXkVT8ss1wbaA/2Bm4AXRCQuwGNR1ZmqmqKqKQkJbkaFfDtrN0s+y+euX7Snw3mxTjIYY0xVCKTQ5wGt/JYTgfxy9lmsqkWq+hWwGW/hD+RY5w4dL+L+RRvo1CyWO/u3cx3HGGOCKpBCvw5oLyLJIlIXGA6kldnndWAAgIg0xduVkwMsB64QkXgRiQeu8K0LKY8v3cS+YyeZNux86kTZHafGmPBS4V03qlosIuPwFugoYJaqbhSRRwCPqqbxXUHPAkqA36vqPgAR+RPeNwuAR1R1f1W8kDP14Za9zPPkcsfP29E9sZHrOMYYE3Si+oMuc6dSUlLU4/FUy3MdO1HML59eRd2oWiy7ux/RdaKq5XmNMSbYRCRdVVPK2xbRE49MW76Zrw8eZ/5/X2RF3hgTtiK2Q9qzbT9z1mzjN33a8LOkxq7jGGNMlYnIQl9YVMLE1zJp0ag+E1M7uY5jjDFVKiK7bp55Zws5Bcd48dbeNKgXkb8CY0wEibgW/edfH+L5VTnc0CuRSzu4+XKWMcZUp4gq9EUlpUxckEnjBnV54KouruMYY0y1iKh+i+ff30rWzsM8P7IXjWLquI5jjDHVImJa9Ft2H+GZd7K5qkdzftm1mes4xhhTbSKi0JeUKhNfyySmXhQPD+7qOo4xxlSriCj0s1dv49MdB3nomi40bVjPdRxjjKlWYV/od+z7hunLNzOgYwJDe/5gKHxjjAl7YV3oVZVJCzOJqiU8dm13RGwyEWNM5AnrQj93XS6rt+5j8qBOtIir7zqOMcY4EbaFfueh4zy+dBN92jbmpp+1dh3HGGOcCctCr6o8sOhzikpL+fP1Pahl878aYyJYWBb6tM/yeeeLPdx7RUfaNGngOo4xxjgVdoV+79ETTEnbSM9WcdzSN9l1HGOMcS7sCv2UtI0cPVHMk8N6EGVdNsYYE16F/q2Nu3gjcyd3/aI9Hc6LdR3HGGNCQtgU+kPHi3jg9c/p1CyWO/u3cx3HGGNCRkCFXkRSRWSziGSLyKRyto8WkQIRyfD9/JffthK/9WnBDO/vZHEp57eKY9qw86kTFTbvX8YYc9YqHKZYRKKAGcBAIA9YJyJpqppVZtd5qjqunFMcV9WeZx/1xyXE1uNfvyl3AnRjjIlogTR9ewPZqpqjqieBucCQqo1ljDEmWAIp9C2BXL/lPN+6sq4XkUwRWSAirfzWR4uIR0TWisjQ8p5ARMb49vEUFBQEnt4YY0yFAin05d2jqGWWlwBJqtoDWAHM8dvWWlVTgBHA0yLygyulqjpTVVNUNSUhweZxNcaYYAqk0OcB/i30RCDffwdV3aeqJ3yL/wJ6+W3L9/2bA7wHXHAWeY0xxlRSIIV+HdBeRJJFpC4wHPje3TMi0txvcTCwybc+XkTq+R43BfoCZS/iGmOMqUIV3nWjqsUiMg5YDkQBs1R1o4g8AnhUNQ34HxEZDBQD+4HRvsM7A8+LSCneN5Wp5dytY4wxpgqJatnudrdSUlLU4/G4jmGMMTWKiKT7rof+gH2zyBhjwlzItehFpADYfhanaArsDVKcYLJclWO5KsdyVU445mqjquXethhyhf5siYjndB9fXLJclWO5KsdyVU6k5bKuG2OMCXNW6I0xJsyFY6Gf6TrAaViuyrFclWO5KieicoVdH70xxpjvC8cWvTHGGD9W6I0xJsyFTaGvaBYsV0RklojsEZHPXWc5RURaichKEdkkIhtF5G7XmQBEJFpEPhGRz3y5HnadyZ+IRInIpyLyhuss/kRkm4hs8M3iFjJfKxeRON+w5V/4/tYuCoFMHf1mvMsQkcMico/rXAAiMt73d/+5iLwiItFBO3c49NH7ZsH6Er9ZsICbQmFcHRG5FDgKvKiq3VzngW8HoWuuqutFJBZIB4a6/n2JiAANVPWoiNQBPgTuVtW1LnOdIiK/A1KAc1T1atd5ThGRbUCKqobUF4BEZA7wgaq+4BsQMUZVD7rOdYqvbnwNXKiqZ/MlzWBkaYn3772Lqh4XkfnAMlWdHYzzh0uLPmRnwVLVVXgHegsZqrpTVdf7Hh/BO9poeZPJVCv1OupbrOP7CYmWiIgkAlcBL7jOUhOIyDnApcC/AVT1ZCgVeZ/LgK2ui7yf2kB9EakNxFBmOPizES6FPtBZsEwZIpKEd46Aj90m8fJ1j2QAe4C3VTUkcgFPAxOBUtdByqHAWyKSLiJjXIfxaQsUAP/xdXe9ICINXIcqYzjwiusQAKr6NTAd2AHsBA6p6lvBOn+4FPpAZsEyZYhIQ+A14B5VPew6D4Cqlvgmk08EeouI8+4uEbka2KOq6a6znEZfVf0pcCUw1tdd6Fpt4KfAP1T1AuAYEErXzurinTvjVddZwDt3B95eiGSgBdBARH4drPOHS6GvcBYs832+PvDXgJdUdaHrPGX5Pua/B6Q6jgLeCXMG+/rC5wK/EJH/cxvpO36zuO0BFuHtynQtD8jz+0S2AG/hDxVXAutVdbfrID6XA1+paoGqFgELgYuDdfJwKfQVzoJlvuO76PlvYJOq/sV1nlNEJEFE4nyP6+P94//CbSpQ1cmqmqiqSXj/tt5V1aC1ts6GiDTwXVDH1zVyBeD8Di9V3QXkikhH36rLCK3Z5W4iRLptfHYAfUQkxvf/52X4ZuoLhgpnmKoJTjcLluNYAIjIK0B/oKmI5AEPqeq/3aaiLzAS2ODrDwf4g6ouc5gJoDkwx3c3RC1gvqqG1K2MIeg8YJG3NlAbeFlV33Qb6Vt3AS/5Gl85wC2O8wAgIjF479D7b9dZTlHVj0VkAbAe70x9nxLE4RDC4vZKY4wxpxcuXTfGGGNOwwq9McaEOSv0xhgT5qzQG2NMmLNCb4wxYc4KvTHGhDkr9MYYE+b+P3hfIN4jwDGjAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plot_hist(thr_score_hist)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### No train test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<All keys matched successfully>"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.load_state_dict(torch.load('data/text_models/4ap_bert_8ep.pth'))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "model = BertModel.from_pretrained('bert-base-uncased').to(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=2141.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    }
   ],
   "source": [
    "model.eval()\n",
    "with torch.no_grad():\n",
    "    pbar = tqdm(tr_test_dl, leave=False)\n",
    "    embs = []\n",
    "    for txts, _ in pbar:\n",
    "        txts = text_to_device(txts, device)\n",
    "        feature =  model(txts)\n",
    "        embs.append(feature)\n",
    "    embs = torch.cat(embs, 0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "torch.save(embs, 'data/text_embs/bertmulti_4ap.pth')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([ 0.2382, -0.1216,  0.3433,  0.4693,  0.0672, -0.8577,  0.2648,  0.0217,\n",
       "         0.2051, -0.0642, -0.0443, -0.3666,  0.4171,  0.5944, -0.2593, -0.3102,\n",
       "         0.1456, -0.1105, -0.4717,  0.2841, -0.0559, -0.3004, -0.1875, -0.4790,\n",
       "        -0.1663, -0.2223, -0.0750,  0.3664,  0.1350,  0.0313, -0.0846, -0.1551,\n",
       "        -0.5078,  0.0983, -0.0549,  0.1436, -0.4476,  0.5313,  0.0747, -0.5308,\n",
       "        -0.3245,  0.2828,  0.4038,  0.0660, -0.5474,  0.0403, -0.1574,  0.0808,\n",
       "        -0.4188,  0.0085, -0.2077, -0.5169,  0.0304, -0.2926, -0.3524, -0.0917,\n",
       "        -0.4470, -0.0627, -0.3675, -0.1950, -0.2660,  0.2865, -0.1969,  0.2587,\n",
       "         0.2515, -0.3044, -0.0143, -0.0617, -0.0580, -0.0831,  0.2408,  0.2549,\n",
       "         0.0910, -0.1716, -0.2711, -0.5112,  0.0345, -0.4528, -0.3019, -0.2169,\n",
       "         0.2535,  0.4300,  0.2293, -0.1192,  0.0551, -0.5761, -0.1208,  0.2737,\n",
       "        -0.3509,  0.4892,  0.1303, -0.1677, -0.1343,  0.1137, -0.0112,  0.2284,\n",
       "        -0.3209,  0.2821,  0.1479,  0.0384, -0.2910, -0.2234,  0.4195, -0.1028,\n",
       "        -0.2709,  0.1456,  0.5408, -0.3124,  0.1663,  0.2240, -0.5002, -0.1209,\n",
       "        -0.6723, -0.2292, -0.2946,  0.1222,  0.4109, -0.1466,  0.0461, -0.3392,\n",
       "        -0.1435, -0.5404, -0.3914, -0.1293,  0.1090,  0.4065, -0.1480,  0.4039,\n",
       "        -0.3162, -0.2732,  0.0809, -0.0931,  0.3015, -0.5345,  0.1729,  0.2377,\n",
       "        -0.1014, -0.0896, -0.1280,  0.1069,  0.0172,  0.1334, -0.0999, -0.1799,\n",
       "         0.1536,  0.2245, -0.3562,  0.6414, -0.6757, -0.1457, -0.5271,  0.3000,\n",
       "        -0.0533,  0.1462,  0.1776, -0.6251, -0.0866, -0.2039,  0.0862, -0.0442,\n",
       "         0.2434,  0.5358, -0.1696, -0.1354,  0.0204, -0.2508,  0.3390,  0.0386,\n",
       "        -0.1809, -0.2808,  0.1491,  0.2016, -0.0945, -0.0835,  0.0517,  0.0755,\n",
       "        -0.2989, -0.0322, -0.1801,  0.2262,  0.1557, -0.0199,  0.2371, -0.0076,\n",
       "        -0.1531,  0.1175,  0.1233,  0.1768,  0.5339,  0.0840,  0.4150,  0.1912,\n",
       "         0.3224,  0.2225,  0.6446,  0.7755, -0.5113, -0.8299,  0.0258, -0.5639,\n",
       "         0.1722,  0.3823,  0.1309,  0.3384,  0.0961,  0.2690, -0.0495,  0.1123,\n",
       "        -0.3525,  0.3000,  0.0278,  0.1832,  0.2347, -0.6205,  0.4380, -0.0749,\n",
       "         0.5082, -0.0129, -0.1051,  0.0058, -0.2884, -0.1886,  0.2713, -0.2003,\n",
       "         0.0483, -0.1207, -0.9353, -0.0985, -0.5687,  0.2084, -0.5250,  0.4457,\n",
       "        -0.3381,  0.0290, -0.2337, -0.0833,  0.7115,  0.3840,  0.1510,  0.1588,\n",
       "         0.0721, -0.0910, -0.0688,  0.0144, -0.0569,  0.1554, -0.2314, -0.0221,\n",
       "        -0.2283, -0.1904,  0.0309, -0.3718,  0.0374, -0.3626, -0.0633,  0.0846,\n",
       "         0.1741, -0.3476, -0.2961, -0.1497, -0.0251, -0.7364, -0.3160, -0.3272,\n",
       "         0.2436,  0.4512, -0.5290,  0.3473, -0.5847, -0.1606, -0.1030,  0.0075,\n",
       "        -0.0030,  0.3145, -0.1280,  0.1235,  1.0063,  0.5420,  0.1690, -0.2496,\n",
       "         0.0292,  0.0434, -0.0886,  0.3088, -0.0096,  0.2059,  0.1360, -0.5939,\n",
       "         0.2519,  0.1529, -0.3511,  0.2505,  0.1948,  0.0836, -0.4298, -0.1535,\n",
       "        -0.3398, -0.7915,  0.6781, -0.1984,  0.4666, -0.0294, -0.3931,  0.3776,\n",
       "        -0.2799, -0.1389, -0.0750,  0.1660,  0.3012,  0.0918,  0.4078,  0.1479,\n",
       "         0.5984,  0.3673, -0.2710, -0.0456,  0.0958, -0.2466, -0.4615, -0.2659,\n",
       "        -0.4809,  0.1063, -0.0568,  0.4142,  0.3616,  0.2894,  0.8392, -0.6743,\n",
       "         0.2870, -0.3898, -0.1259,  0.6792, -0.1560,  0.0734, -0.0759,  0.1050,\n",
       "        -0.0811,  0.1762, -0.1835,  0.0945, -0.3004,  0.0845,  0.0698,  0.1182,\n",
       "         0.2706, -0.2392,  0.2848,  0.1564,  0.0657,  0.2345,  0.3493, -0.2013,\n",
       "        -0.1402, -0.5805,  0.5980,  0.3712,  0.2669, -0.0990,  0.0085,  0.0551,\n",
       "        -0.1122, -0.8489, -0.5546,  0.9048,  0.1528, -0.4788,  0.2614, -0.2486,\n",
       "         0.0410, -0.3133, -0.0068, -0.0633, -0.1458,  0.1177, -0.3326,  0.1564,\n",
       "         0.3294,  0.5608, -0.4571,  0.2158, -0.2804,  0.1789, -0.3099,  0.2774,\n",
       "         0.5527, -0.2267,  0.2170,  0.6825, -0.2312,  0.0367,  0.2743,  0.8161,\n",
       "         0.0329,  0.0792, -0.6317,  0.1718,  0.5090,  0.2790,  0.1252, -0.0408,\n",
       "         0.4949,  0.3041, -0.2178,  0.0886, -0.2471,  0.5322,  1.0519,  0.3937,\n",
       "         0.0648, -0.2677, -0.6676, -0.1983,  0.4012, -0.2307, -0.1883,  0.1018,\n",
       "         0.1294,  0.0847,  0.0482, -0.0912,  0.2705, -0.1664,  0.3650,  0.3055,\n",
       "         0.1834,  0.0330, -0.1684,  0.1150,  0.0926, -0.0193,  0.1894,  0.3215,\n",
       "         0.1128,  0.1515, -0.2601,  0.0332,  0.5764, -0.0488, -0.4290,  0.5728,\n",
       "        -0.2592,  0.1047, -0.1173,  0.2900, -0.2772, -0.4035,  0.1829,  0.0599,\n",
       "        -0.4778,  0.6035, -0.1587,  0.1694, -0.4781, -0.0129,  0.1550,  0.0984,\n",
       "        -0.3891, -0.3741,  0.0301, -0.1386, -0.3416, -0.3427, -0.0831,  0.2768,\n",
       "        -0.1529, -0.3796, -0.0119,  0.0369,  0.2143,  0.2553,  0.7650, -0.1473,\n",
       "         0.6256,  0.4126,  0.5174,  0.1352, -0.3212, -0.1921, -0.2896,  0.1690,\n",
       "        -0.4611,  0.4652,  0.3748, -0.2579, -0.3572, -0.2011,  0.2524, -0.0390,\n",
       "         0.1758,  0.2300, -0.2405,  0.4078,  0.0954,  0.1936,  0.0618,  0.0779,\n",
       "        -0.2207,  0.3070,  0.4917,  0.6060,  0.2394, -0.1223,  0.0614, -0.0235,\n",
       "         0.0686, -0.3675, -0.2526,  0.1599,  0.5522,  0.2439, -0.2891, -0.3304,\n",
       "        -0.0898,  0.0031, -0.2623, -0.3265, -0.4035,  0.2726, -0.5537, -0.2990,\n",
       "         0.3721,  0.1883,  0.0881,  0.1399, -0.1942,  0.3763,  0.2262,  0.1406,\n",
       "         0.6282, -0.4878,  0.4733,  0.1873,  0.3989,  0.3042, -0.4522,  0.0547,\n",
       "        -0.1386, -0.3214, -0.1740, -0.3421, -0.2830, -0.0872,  0.2186,  0.1046,\n",
       "        -0.2692, -0.0702, -0.2014, -0.2683, -0.1816, -0.0834, -0.3810,  0.2589,\n",
       "         0.5926, -0.3825,  0.4202,  0.4111,  0.4888,  0.2230, -0.4824,  0.0531,\n",
       "        -0.2386,  0.4171,  0.2460,  0.2190,  0.2085,  0.4474,  0.0424,  0.2396,\n",
       "         0.3170,  0.2706,  0.1183, -0.0360,  0.1424,  0.1123,  0.0261, -0.0860,\n",
       "        -0.3260, -0.0884, -0.7425, -0.3972, -0.2626, -0.6666,  0.3088,  0.0922,\n",
       "        -0.0484,  0.4796,  0.2522,  0.5742, -0.0846, -0.0100, -0.1780, -0.1389,\n",
       "        -0.4889, -0.2262,  0.0312, -0.3230, -0.0978,  0.5538, -0.1739,  0.4065,\n",
       "        -0.4279,  0.2702,  0.1650, -0.4476, -0.0169,  0.4679,  0.1321, -0.5049,\n",
       "         0.0142, -0.3640, -0.2578, -0.0399,  0.1235, -0.0682, -0.1702, -0.1968,\n",
       "        -0.2131, -0.2495,  0.2736, -0.2528, -0.5722,  0.1503, -0.5358, -0.1368,\n",
       "         0.3255,  0.3088,  0.0472, -0.0806, -0.3716, -0.1451, -0.1827,  0.2134,\n",
       "        -0.3885, -0.0830,  0.1435, -0.1601,  0.5034, -0.2045,  0.0383, -0.0070,\n",
       "        -0.4497,  0.1821,  0.4589,  0.0568, -0.1151,  0.1443,  0.0321, -0.1880,\n",
       "        -0.4229, -0.1102, -0.3756,  0.2317,  0.1373, -0.1043,  0.5817,  0.2730,\n",
       "        -0.1426, -0.2062, -0.1120,  0.0476,  0.2441,  0.2838, -0.3628,  0.2570,\n",
       "         0.3030,  0.0623,  0.5598,  0.6737, -0.3254,  0.1520,  0.1323,  0.1570,\n",
       "         0.4395,  0.4651,  0.2660,  0.2174,  0.5913,  0.0261, -0.3583, -0.0818,\n",
       "        -0.0503,  0.0893, -0.3443, -0.1273, -0.3888, -0.3827,  0.5579, -0.1131,\n",
       "         0.0349,  0.2012,  0.1922, -0.3077,  0.0993,  0.2785,  0.0893,  0.1805,\n",
       "        -0.1459,  0.2621,  0.5925,  0.3581,  0.3011, -0.1985,  0.1313,  0.1233,\n",
       "        -0.5923,  0.0919, -0.2354, -0.0330, -0.2652, -0.1618,  0.5102, -0.4413,\n",
       "        -0.1036,  0.0349, -0.4032,  0.1254, -0.2723, -0.0197,  0.3724, -0.2214,\n",
       "         0.1961,  0.4584, -0.0322,  0.7325, -0.5836,  0.2203,  0.0024, -0.2560,\n",
       "         0.2212,  0.4572,  0.1936,  0.1048, -0.1108,  0.7716,  0.1369,  0.5560,\n",
       "         0.4856, -0.1332, -0.2761, -0.1849,  0.0839,  0.1471,  0.0066, -0.7113,\n",
       "        -0.0448, -0.2437,  0.3349,  0.2588, -0.5550,  0.1969, -0.2208, -0.3148,\n",
       "         0.5977, -0.1410,  0.0230, -0.1586,  0.2859, -0.0721,  0.1241, -0.0170,\n",
       "        -0.2422, -0.0666, -0.5514,  0.2470, -0.4943, -0.4702, -0.1033, -0.1329],\n",
       "       device='cuda:0')"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "embs[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([34250, 768])"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "embs.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "embs = F.normalize(embs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Thresholds', max=1.0, style=ProgressStyle(description_wid…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "text/plain": [
       "({0.9: 0.8814814250072337}, 0.9, 0.8814814250072337)"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "compute_f1(embs, df['label_group'], [0.9])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "??compute_f1"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Effnet image"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import random\n",
    "import functools\n",
    "from functools import partial\n",
    "import PIL\n",
    "\n",
    "import numpy as np \n",
    "import pandas as pd\n",
    "\n",
    "from tqdm.notebook import tqdm\n",
    "import math\n",
    "\n",
    "import torch\n",
    "from torch.nn.parameter import Parameter\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "import torchvision\n",
    "from torchvision import transforms\n",
    "from torch.utils.data import Dataset, DataLoader\n",
    "\n",
    "from transformers import BertTokenizer\n",
    "from transformers import BertModel\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "import timm\n",
    "np.random.seed(1337)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# load in data\n",
    "\n",
    "df = pd.read_csv('data/train.csv')\n",
    "small_images_dir = 'data/small_train_images/'\n",
    "n_classes = df['label_group'].nunique()\n",
    "\n",
    "labels = np.random.permutation(df['label_group'].unique())\n",
    "\n",
    "train_perc = 0.7\n",
    "train_idx = int(train_perc * len(labels))\n",
    "\n",
    "train_labels = labels[:train_idx]\n",
    "val_labels = labels[train_idx:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_df = df[df['label_group'].isin(train_labels)]\n",
    "val_df = df[df['label_group'].isin(val_labels)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>posting_id</th>\n",
       "      <th>image</th>\n",
       "      <th>image_phash</th>\n",
       "      <th>title</th>\n",
       "      <th>label_group</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>train_129225211</td>\n",
       "      <td>0000a68812bc7e98c42888dfb1c07da0.jpg</td>\n",
       "      <td>94974f937d4c2433</td>\n",
       "      <td>Paper Bag Victoria Secret</td>\n",
       "      <td>249114794</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>train_2288590299</td>\n",
       "      <td>000a190fdd715a2a36faed16e2c65df7.jpg</td>\n",
       "      <td>b94cb00ed3e50f78</td>\n",
       "      <td>Maling TTS Canned Pork Luncheon Meat 397 gr</td>\n",
       "      <td>2395904891</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>train_2406599165</td>\n",
       "      <td>00117e4fc239b1b641ff08340b429633.jpg</td>\n",
       "      <td>8514fc58eafea283</td>\n",
       "      <td>Daster Batik Lengan pendek - Motif Acak / Camp...</td>\n",
       "      <td>4093212188</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>train_1802986387</td>\n",
       "      <td>00144a49c56599d45354a1c28104c039.jpg</td>\n",
       "      <td>f815c9bb833ab4c8</td>\n",
       "      <td>Jubah anak size 1-12 thn</td>\n",
       "      <td>1835033137</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>train_86570404</td>\n",
       "      <td>0019a3c6755a194cb2e2c12bfc63972e.jpg</td>\n",
       "      <td>ea9af4f483249972</td>\n",
       "      <td>[LOGU] Tempelan kulkas magnet angka, tempelan ...</td>\n",
       "      <td>2359912463</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         posting_id                                 image       image_phash  \\\n",
       "0   train_129225211  0000a68812bc7e98c42888dfb1c07da0.jpg  94974f937d4c2433   \n",
       "2  train_2288590299  000a190fdd715a2a36faed16e2c65df7.jpg  b94cb00ed3e50f78   \n",
       "3  train_2406599165  00117e4fc239b1b641ff08340b429633.jpg  8514fc58eafea283   \n",
       "6  train_1802986387  00144a49c56599d45354a1c28104c039.jpg  f815c9bb833ab4c8   \n",
       "8    train_86570404  0019a3c6755a194cb2e2c12bfc63972e.jpg  ea9af4f483249972   \n",
       "\n",
       "                                               title  label_group  \n",
       "0                          Paper Bag Victoria Secret    249114794  \n",
       "2        Maling TTS Canned Pork Luncheon Meat 397 gr   2395904891  \n",
       "3  Daster Batik Lengan pendek - Motif Acak / Camp...   4093212188  \n",
       "6                           Jubah anak size 1-12 thn   1835033137  \n",
       "8  [LOGU] Tempelan kulkas magnet angka, tempelan ...   2359912463  "
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "val_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "class SoftMaxDS(Dataset):\n",
    "    def __init__(self, data, images_path, return_triplet = True):\n",
    "        super().__init__()\n",
    "        self.imgs = data['image'].tolist()\n",
    "        self.unique_labels = data['label_group'].unique().tolist()\n",
    "        self.labels = data['label_group'].astype('category')\n",
    "        self.label_codes = self.labels.cat.codes\n",
    "        \n",
    "        self.images_path = images_path\n",
    "        \n",
    "    def __getitem__(self, idx):\n",
    "        \n",
    "        img = self._get_item(idx)\n",
    "        label = self.label_codes.iloc[idx]\n",
    "        return img, label\n",
    "    def __len__(self):\n",
    "        return len(self.imgs)\n",
    "    \n",
    "    def _get_item(self, idx):\n",
    "        im = PIL.Image.open(os.path.join(self.images_path, self.imgs[idx]))\n",
    "        im = torch.tensor(np.array(im) / 255.0, dtype = torch.float).permute(2,0,1)\n",
    "        return im"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# creating dataloaders\n",
    "\n",
    "vision_model = 'resnet50'\n",
    "\n",
    "bs = 64\n",
    "tr_ds = SoftMaxDS(train_df, small_images_dir)\n",
    "tr_dl = DataLoader(tr_ds, batch_size = bs, shuffle = True, pin_memory = True)\n",
    "tr_test_dl = DataLoader(tr_ds, batch_size = bs, shuffle = False, pin_memory = True)\n",
    "\n",
    "val_ds = SoftMaxDS(val_df, small_images_dir)\n",
    "val_dl = DataLoader(val_ds, batch_size = bs, shuffle = False, pin_memory = True)\n",
    "\n",
    "full_ds = SoftMaxDS(df, small_images_dir)\n",
    "full_dl = DataLoader(full_ds, batch_size = bs, shuffle = False, pin_memory = True)\n",
    "\n",
    "device = torch.device('cuda')\n",
    "\n",
    "val_label_count = val_df['label_group'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "class EMBRes(nn.Module) :\n",
    "    def __init__(self, pretrained_image_embedor='resnet50',\n",
    "                output_dim=512) :\n",
    "        super(EMBRes, self).__init__()\n",
    "        self.image_embedor = timm.create_model(pretrained_image_embedor, pretrained=True, n_classes=0)\n",
    "    \n",
    "    def _get_embs(self, x) :\n",
    "        images = x\n",
    "        out_images = self.image_embedor.forward_features(images)\n",
    "        out_images = self.image_pool(out_images).squeeze()\n",
    "        #return F.normalize(out_images, dim=-1)\n",
    "        return out_images\n",
    "    \n",
    "    def forward(self, x) :\n",
    "        out_images = self._get_embs(x)\n",
    "        \n",
    "        return self.head(out_images)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# https://github.com/ronghuaiyang/arcface-pytorch\n",
    "\n",
    "class ArcMarginProduct(nn.Module):\n",
    "    r\"\"\"Implement of large margin arc distance: :\n",
    "        Args:\n",
    "            in_features: size of each input sample\n",
    "            out_features: size of each output sample\n",
    "            s: norm of input feature\n",
    "            m: margin\n",
    "            cos(theta + m)\n",
    "        \"\"\"\n",
    "    def __init__(self, in_features, out_features, s=30.0, m=0.50, easy_margin=False):\n",
    "        super(ArcMarginProduct, self).__init__()\n",
    "        self.in_features = in_features\n",
    "        self.out_features = out_features\n",
    "        self.s = s\n",
    "        self.m = m\n",
    "        self.weight = Parameter(torch.FloatTensor(out_features, in_features))\n",
    "        nn.init.xavier_uniform_(self.weight)\n",
    "\n",
    "        self.easy_margin = easy_margin\n",
    "        self.cos_m = math.cos(m)\n",
    "        self.sin_m = math.sin(m)\n",
    "        self.th = math.cos(math.pi - m)\n",
    "        self.mm = math.sin(math.pi - m) * m\n",
    "\n",
    "    def forward(self, input, label):\n",
    "        # --------------------------- cos(theta) & phi(theta) ---------------------------\n",
    "        cosine = F.linear(F.normalize(input), F.normalize(self.weight))\n",
    "        sine = torch.sqrt((1.0 - torch.pow(cosine, 2)).clamp(0, 1))\n",
    "        phi = cosine * self.cos_m - sine * self.sin_m\n",
    "        if self.easy_margin:\n",
    "            phi = torch.where(cosine > 0, phi, cosine)\n",
    "        else:\n",
    "            phi = torch.where(cosine > self.th, phi, cosine - self.mm)\n",
    "        # --------------------------- convert label to one-hot ---------------------------\n",
    "        # one_hot = torch.zeros(cosine.size(), requires_grad=True, device='cuda')\n",
    "        one_hot = torch.zeros(cosine.size(), device='cuda')\n",
    "        one_hot.scatter_(1, label.view(-1, 1).long(), 1)\n",
    "        # -------------torch.where(out_i = {x_i if condition_i else y_i) -------------\n",
    "        output = (one_hot * phi) + ((1.0 - one_hot) * cosine)  # you can use torch.where if your torch.__version__ is 0.4\n",
    "        output *= self.s\n",
    "        # print(output)\n",
    "\n",
    "        return output\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "def compute_f1(embeddings, ls, threshold) :\n",
    "    dists = torch.cdist(embeddings, embeddings)\n",
    "    distances, indices = torch.topk(dists, 50, dim=1, largest=False)\n",
    "    \n",
    "    THRESHOLD = threshold\n",
    "    preds = [[] for _ in range(embeddings.shape[0])]\n",
    "    for i in tqdm(range(distances.shape[0]), leave=False) :\n",
    "        IDX = torch.where(distances[i,]<THRESHOLD)[0]\n",
    "        IDS = indices[i,IDX]\n",
    "        preds[i] = IDS.cpu().numpy()\n",
    "            \n",
    "    label_counts = ls.value_counts()\n",
    "    f_score = 0 \n",
    "    precision = 0\n",
    "    recall = 0\n",
    "    for i in tqdm(range(embeddings.shape[0]), leave=False) :\n",
    "        tp = 0\n",
    "        fp = 0\n",
    "        true_label = ls.iloc[i]\n",
    "        for index in preds[i] :\n",
    "            if ls.iloc[index] == true_label :\n",
    "                tp += 1\n",
    "            else :\n",
    "                fp += 1\n",
    "        fn = label_counts[true_label] - tp\n",
    "        #print(label_counts[true_label]-1, tp)\n",
    "        f_score += 2*tp / (label_counts[true_label] + len(preds[i]))\n",
    "        precision += tp / len(preds[i])\n",
    "        recall += tp/ label_counts[true_label]\n",
    "    f_score = f_score/embeddings.shape[0]\n",
    "    precision = precision/embeddings.shape[0]\n",
    "    recall = recall/embeddings.shape[0]\n",
    "    \n",
    "    print('f1 score : {} | precision : {} | recall : {}'.format(f_score, precision, recall))\n",
    "    return f_score, precision, recall"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "***Embeddings normalization is not done in the model but in the arcface metric***"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "#model = EMBRes('efficientnet_b0').to(device)\n",
    "#metric_fc = ArcMarginProduct(512, df['label_group'].nunique(), s=30, m=0.5, easy_margin=False).to(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'model' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-11-03b61e93ef5a>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m     21\u001b[0m             \"BatchNorm1d.weight\", \"BatchNorm1d.bias\"]\n\u001b[1;32m     22\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 23\u001b[0;31m \u001b[0mparams\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mlist\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmodel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnamed_parameters\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0mlist\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmetric_fc\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnamed_parameters\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     24\u001b[0m optimizer_grouped_parameters = [\n\u001b[1;32m     25\u001b[0m     {\n",
      "\u001b[0;31mNameError\u001b[0m: name 'model' is not defined"
     ]
    }
   ],
   "source": [
    "\n",
    "normalize = transforms.Normalize(mean=(0.485, 0.456, 0.406),\n",
    "                                 std=(0.229, 0.224, 0.225))\n",
    "\n",
    "train_transforms = transforms.Compose([transforms.ColorJitter(.3,.3,.3),\n",
    "                                       transforms.RandomRotation(5),\n",
    "                                       transforms.RandomCrop(224),\n",
    "                                       transforms.RandomHorizontalFlip(),\n",
    "                                       normalize\n",
    "                                       ])\n",
    "\n",
    "val_transforms = transforms.Compose([normalize\n",
    "                                     ])\n",
    "\n",
    "n_epochs = 30\n",
    "\n",
    "lf = nn.CrossEntropyLoss()\n",
    "\n",
    "lr = 1e-2\n",
    "wd = 0\n",
    "no_decay = [\"bias\", \"BatchNorm2d.weight\", \"BatchNorm2d.bias\", \"LayerNorm.weight\", 'LayerNorm.bias',\n",
    "            \"BatchNorm1d.weight\", \"BatchNorm1d.bias\"]\n",
    "\n",
    "params = list(model.named_parameters()) + list(metric_fc.named_parameters())\n",
    "optimizer_grouped_parameters = [\n",
    "    {\n",
    "        \"params\": [p for n, p in params if not any(nd in n for nd in no_decay)],\n",
    "        \"weight_decay\": wd,\n",
    "    },\n",
    "    {\n",
    "        \"params\": [p for n, p in  model.named_parameters() if any(nd in n for nd in no_decay)],\n",
    "        \"weight_decay\": 0.0,\n",
    "    },\n",
    "]\n",
    "optimizer = torch.optim.AdamW(optimizer_grouped_parameters, lr=lr)\n",
    "\n",
    "# learning rate scheduler\n",
    "sched = torch.optim.lr_scheduler.OneCycleLR(optimizer, max_lr =lr, pct_start = 0.3, #anneal_strategy = 'linear',\n",
    "                                            total_steps = int(n_epochs * len(tr_dl)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### b0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "small_images_dir = 'data/small_train_images_600/'\n",
    "full_ds = SoftMaxDS(df, small_images_dir)\n",
    "full_dl = DataLoader(full_ds, batch_size = bs, shuffle = False, pin_memory = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = timm.create_model('efficientnet_b0', pretrained=True, num_classes=0).to(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=536.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    }
   ],
   "source": [
    "with torch.no_grad():\n",
    "    pbar = tqdm(full_dl, leave=False)\n",
    "    embs = []\n",
    "    for imgs, _ in pbar:\n",
    "        imgs = val_transforms(imgs.to(device))\n",
    "        feature = model(imgs)\n",
    "        embs.append(feature)\n",
    "    embs = torch.cat(embs, 0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "torch.save(embs, 'data/effnet_embs/embs_b0_notrain_600.pth')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=34250.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=34250.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r",
      "f1 score : 0.6363438917214435 | precision : 0.9273578644382607 | recall : 0.558110326058612\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(0.6363438917214435, 0.9273578644382607, 0.558110326058612)"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "compute_f1(embs, df['label_group'], 6.6)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### TF b3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "embs = torch.load('data/effnet_embs/image_embeddings.pth')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "embs = torch.from_numpy(embs).float()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=34250.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=34250.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r",
      "f1 score : 0.6409645357216606 | precision : 0.9576403504043125 | recall : 0.5480520553212448\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(0.6409645357216606, 0.9576403504043125, 0.5480520553212448)"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "compute_f1(embs, df['label_group'], 5)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true
   },
   "source": [
    "### TF b0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "embs = torch.load('data/effnet_embs/tf_embs_b0_256.pth')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "hidden": true
   },
   "outputs": [
    {
     "ename": "TypeError",
     "evalue": "expected np.ndarray (got Tensor)",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mTypeError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-13-6a788cef8524>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0membs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfrom_numpy\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0membs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfloat\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mto\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdevice\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;31mTypeError\u001b[0m: expected np.ndarray (got Tensor)"
     ]
    }
   ],
   "source": [
    "embs = torch.from_numpy(embs).float().to(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "hidden": true
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=34250.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=34250.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r",
      "f1 score : 0.642667609875153 | precision : 0.9084886567491639 | recall : 0.5867818006410631\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(0.642667609875153, 0.9084886567491639, 0.5867818006410631)"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "compute_f1(embs, df['label_group'], 7.5)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### b2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = timm.create_model('efficientnet_b2', pretrained=True, num_classes=0).to(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=536.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    }
   ],
   "source": [
    "with torch.no_grad():\n",
    "    pbar = tqdm(full_dl, leave=False)\n",
    "    embs = []\n",
    "    for imgs, _ in pbar:\n",
    "        imgs = val_transforms(imgs.to(device))\n",
    "        feature = model(imgs)\n",
    "        embs.append(feature)\n",
    "    embs = torch.cat(embs, 0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=34250.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=34250.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r",
      "f1 score : 0.6196357491479187 | precision : 0.9479240435254366 | recall : 0.5289644536998594\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(0.6196357491479187, 0.9479240435254366, 0.5289644536998594)"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "compute_f1(embs, df['label_group'], 10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "#torch.save(embs, 'data/effnet_embs/embs_b2_notrain.pth')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### b3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "small_images_dir = 'data/small_train_images_300/'\n",
    "full_ds = SoftMaxDS(df, small_images_dir)\n",
    "full_dl = DataLoader(full_ds, batch_size = bs, shuffle = False, pin_memory = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "model = timm.create_model('efficientnet_b3', pretrained=True, num_classes=0).to(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=536.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    }
   ],
   "source": [
    "with torch.no_grad():\n",
    "    pbar = tqdm(full_dl, leave=False)\n",
    "    embs = []\n",
    "    for imgs, _ in pbar:\n",
    "        imgs = val_transforms(imgs.to(device))\n",
    "        feature = model(imgs)\n",
    "        embs.append(feature)\n",
    "    embs = torch.cat(embs, 0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=34250.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=34250.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r",
      "f1 score : 0.60674607145152 | precision : 0.9222303967400572 | recall : 0.5332024273267191\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(0.60674607145152, 0.9222303967400572, 0.5332024273267191)"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "compute_f1(embs, df['label_group'], 8.1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "torch.save(embs, 'data/effnet_embs/embs_b3_notrain.pth')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "embs = torch.load('data/effnet_embs/embs_b3_notrain.pth')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": true,
   "toc_position": {
    "height": "calc(100% - 180px)",
    "left": "10px",
    "top": "150px",
    "width": "304.716px"
   },
   "toc_section_display": true,
   "toc_window_display": true
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}