clip.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "toc": true
   },
   "source": [
    "<h1>Table of Contents<span class=\"tocSkip\"></span></h1>\n",
    "<div class=\"toc\"><ul class=\"toc-item\"><li><span><a href=\"#Only-image\" data-toc-modified-id=\"Only-image-1\"><span class=\"toc-item-num\">1&nbsp;&nbsp;</span>Only image</a></span><ul class=\"toc-item\"><li><span><a href=\"#Train-test-vit\" data-toc-modified-id=\"Train-test-vit-1.1\"><span class=\"toc-item-num\">1.1&nbsp;&nbsp;</span>Train test vit</a></span></li><li><span><a href=\"#Train-test-resnet50\" data-toc-modified-id=\"Train-test-resnet50-1.2\"><span class=\"toc-item-num\">1.2&nbsp;&nbsp;</span>Train test resnet50</a></span></li><li><span><a href=\"#Effnetb0-(noclip)\" data-toc-modified-id=\"Effnetb0-(noclip)-1.3\"><span class=\"toc-item-num\">1.3&nbsp;&nbsp;</span>Effnetb0 (noclip)</a></span></li></ul></li><li><span><a href=\"#Only-text\" data-toc-modified-id=\"Only-text-2\"><span class=\"toc-item-num\">2&nbsp;&nbsp;</span>Only text</a></span><ul class=\"toc-item\"><li><span><a href=\"#Traduction-indonesian-english\" data-toc-modified-id=\"Traduction-indonesian-english-2.1\"><span class=\"toc-item-num\">2.1&nbsp;&nbsp;</span>Traduction indonesian english</a></span></li><li><span><a href=\"#Split\" data-toc-modified-id=\"Split-2.2\"><span class=\"toc-item-num\">2.2&nbsp;&nbsp;</span>Split</a></span></li><li><span><a href=\"#Split-no-translation\" data-toc-modified-id=\"Split-no-translation-2.3\"><span class=\"toc-item-num\">2.3&nbsp;&nbsp;</span>Split no translation</a></span></li></ul></li><li><span><a href=\"#Multimodal-training\" data-toc-modified-id=\"Multimodal-training-3\"><span class=\"toc-item-num\">3&nbsp;&nbsp;</span>Multimodal training</a></span><ul class=\"toc-item\"><li><span><a href=\"#Testing-zero-shot\" data-toc-modified-id=\"Testing-zero-shot-3.1\"><span class=\"toc-item-num\">3.1&nbsp;&nbsp;</span>Testing zero shot</a></span><ul class=\"toc-item\"><li><span><a href=\"#Translated-titles\" data-toc-modified-id=\"Translated-titles-3.1.1\"><span class=\"toc-item-num\">3.1.1&nbsp;&nbsp;</span>Translated titles</a></span></li><li><span><a href=\"#No-translation\" data-toc-modified-id=\"No-translation-3.1.2\"><span class=\"toc-item-num\">3.1.2&nbsp;&nbsp;</span>No translation</a></span></li></ul></li><li><span><a href=\"#Fine-tuning-clip\" data-toc-modified-id=\"Fine-tuning-clip-3.2\"><span class=\"toc-item-num\">3.2&nbsp;&nbsp;</span>Fine tuning clip</a></span><ul class=\"toc-item\"><li><span><a href=\"#Zero-shot-without-fine-tuning-on-split\" data-toc-modified-id=\"Zero-shot-without-fine-tuning-on-split-3.2.1\"><span class=\"toc-item-num\">3.2.1&nbsp;&nbsp;</span>Zero shot without fine tuning on split</a></span></li></ul></li></ul></li></ul></div>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "import torch\n",
    "import clip\n",
    "from PIL import Image"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Only image"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['RN50', 'RN101', 'RN50x4', 'ViT-B/32']"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "clip.available_models()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Train test vit"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from imports import *\n",
    "from utils import load_data\n",
    "from image_train.data import create_dl, ImageDS\n",
    "from image_train.model import EMBRes\n",
    "from arcface import ArcMarginProduct, compute_centers\n",
    "from clip_.train_image import *\n",
    "import matplotlib.pyplot as plt\n",
    "import clip\n",
    "np.random.seed(1337)\n",
    "device = torch.device('cuda')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "df, train_df, val_df, train_labels, val_labels = load_data(train_perc=0.3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# creating dataloaders\n",
    "small_images_dir_train = 'data/small_train_images_250/'\n",
    "small_images_dir_val = 'data/small_train_images_224/'\n",
    "bs = 64\n",
    "\n",
    "tr_dl = create_dl(train_df, small_images_dir_train, batch_size=bs)\n",
    "tr_test_dl = create_dl(train_df, small_images_dir_val, shuffle=False, batch_size=bs)\n",
    "val_dl = create_dl(val_df, small_images_dir_val, shuffle=False, batch_size=bs)\n",
    "#full_dl = create_dl(df, small_images_dir, shuffle=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "class CLIPImg(torch.nn.Module) :\n",
    "    def __init__(self, model_name=\"ViT-B/32\", device='cuda') :\n",
    "        super().__init__()\n",
    "        self.model = clip.load(model_name, device=device, jit=False)[0]\n",
    "    def forward(self, imgs) :\n",
    "        return self.model.encode_image(imgs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "model = CLIPImg()\n",
    "train_tfms, val_tfms = get_tfms(crop=224)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6a0aa8035325470bae0cf485619e26c7",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=164.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "centers = compute_centers(tr_test_dl, model, val_tfms, train_df)\n",
    "torch.save(centers, 'data/clip/centers_im_0.3_vit.pth')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "centers = torch.load('data/clip/centers_im_0.3_vit.pth')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Using center as wieghts\n"
     ]
    }
   ],
   "source": [
    "metric_fc = ArcMarginProduct(512, train_df['label_group'].nunique(), \n",
    "                             s=30, m=0.5, easy_margin=False, centers=centers, half=True).to(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "n_epochs, lf, params, optimizer, sched = get_hparams(tr_dl, model, metric_fc, lr=1e-5, n_epochs=8)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "for n, _ in model.named_parameters() :\n",
    "    print(n)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "mnp = list(model.named_parameters())\n",
    "\n",
    "start = ['model.visual.class_embedding', 'model.visual.positional_embedding', 'model.visual.conv1.weight',\n",
    "        'model.visual.ln_pre.weight', 'model.visual.ln_pre.bias']\n",
    "param_groups = [{'params' : [p for n,p in mnp if n in start]}]\n",
    "params_names = [n for n,p in mnp if n in start]\n",
    "\n",
    "n_blocks = 12\n",
    "for i in range(n_blocks):\n",
    "    ith_block = [p for n, p in mnp if f'resblocks.{i}.' in n]\n",
    "    ith_block_names = [n for n, p in mnp if f'resblocks.{i}.' in n] \n",
    "    param_groups.append({'params' : ith_block})\n",
    "    params_names += ith_block_names\n",
    "    \n",
    "end = ['model.visual.proj', 'model.visual.ln_post.weight', 'model.visual.ln_post.bias']\n",
    "param_groups.append({'params' : [p for n,p in mnp if n in end]})\n",
    "params_names += [n for n,p in mnp if n in end]\n",
    "\n",
    "param_groups.append({'params' : metric_fc.parameters()})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "model.positional_embedding\n",
      "model.text_projection\n",
      "model.logit_scale\n",
      "model.token_embedding.weight\n",
      "model.ln_final.weight\n",
      "model.ln_final.bias\n"
     ]
    }
   ],
   "source": [
    "for n, p in model.named_parameters() :\n",
    "    if n not in params_names :\n",
    "        print(n)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "lf = nn.CrossEntropyLoss()\n",
    "optimizer = torch.optim.AdamW(param_groups, weight_decay = 1e-5)\n",
    "\n",
    "lr = list(np.linspace(5e-6,3e-4,len(param_groups)))\n",
    "\n",
    "n_epochs = 10\n",
    "sched = torch.optim.lr_scheduler.OneCycleLR(optimizer, max_lr=lr, pct_start=0.3,\n",
    "                                            total_steps=int(n_epochs * len(tr_dl)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "loss_hist = []\n",
    "best_thr_score={'val': {'thr': None, 'f1': None}, 'train': {'thr': None, 'f1': None}}\n",
    "thr_score_hist=[]\n",
    "ep_start = 0\n",
    "save_path = 'data/clip/test_20ap_im'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=10.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=164.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Thresholds', max=10.0, style=ProgressStyle(description_wi…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Checkpoint : saved model to data/clip/test_20ap_im_ep_0.pth\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=373.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Thresholds', max=10.0, style=ProgressStyle(description_wi…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Saved best model ep 0 with f score : 0.6560567367188737\n",
      "Ep 0: Train loss 9.6472 | Val f score 0.6561 with thresh 0.47, train f score 0.6313 with thresh 0.64\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=164.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Thresholds', max=5.0, style=ProgressStyle(description_wid…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=373.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Thresholds', max=5.0, style=ProgressStyle(description_wid…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Saved best model ep 1 with f score : 0.658220134229311\n",
      "Ep 1: Train loss 8.9498 | Val f score 0.6582 with thresh 0.47, train f score 0.6663 with thresh 0.64\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=164.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Thresholds', max=5.0, style=ProgressStyle(description_wid…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Checkpoint : saved model to data/clip/test_20ap_im_ep_2.pth\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=373.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Thresholds', max=5.0, style=ProgressStyle(description_wid…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Saved best model ep 2 with f score : 0.6675442911513282\n",
      "Ep 2: Train loss 8.3158 | Val f score 0.6675 with thresh 0.47, train f score 0.6928 with thresh 0.69\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=164.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Thresholds', max=5.0, style=ProgressStyle(description_wid…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=373.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Thresholds', max=5.0, style=ProgressStyle(description_wid…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Saved best model ep 3 with f score : 0.6737946209459528\n",
      "Ep 3: Train loss 6.8196 | Val f score 0.6738 with thresh 0.52, train f score 0.7576 with thresh 0.69\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=164.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Thresholds', max=5.0, style=ProgressStyle(description_wid…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Checkpoint : saved model to data/clip/test_20ap_im_ep_4.pth\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=373.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Thresholds', max=5.0, style=ProgressStyle(description_wid…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Ep 4: Train loss 4.8818 | Val f score 0.6716 with thresh 0.57, train f score 0.8440 with thresh 0.74\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=164.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Thresholds', max=5.0, style=ProgressStyle(description_wid…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=373.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Thresholds', max=5.0, style=ProgressStyle(description_wid…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Saved best model ep 5 with f score : 0.6813593513285545\n",
      "Ep 5: Train loss 3.1572 | Val f score 0.6814 with thresh 0.62, train f score 0.9083 with thresh 0.74\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=164.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Thresholds', max=5.0, style=ProgressStyle(description_wid…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Checkpoint : saved model to data/clip/test_20ap_im_ep_6.pth\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=373.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Thresholds', max=5.0, style=ProgressStyle(description_wid…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Saved best model ep 6 with f score : 0.6935228412147199\n",
      "Ep 6: Train loss 1.8496 | Val f score 0.6935 with thresh 0.62, train f score 0.9537 with thresh 0.74\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=164.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Thresholds', max=5.0, style=ProgressStyle(description_wid…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=373.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Thresholds', max=5.0, style=ProgressStyle(description_wid…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Ep 7: Train loss 1.0444 | Val f score 0.6927 with thresh 0.62, train f score 0.9763 with thresh 0.74\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=164.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Thresholds', max=5.0, style=ProgressStyle(description_wid…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Checkpoint : saved model to data/clip/test_20ap_im_ep_8.pth\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=373.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Thresholds', max=5.0, style=ProgressStyle(description_wid…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Saved best model ep 8 with f score : 0.6938382306243026\n",
      "Ep 8: Train loss 0.6298 | Val f score 0.6938 with thresh 0.62, train f score 0.9877 with thresh 0.69\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=164.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Thresholds', max=5.0, style=ProgressStyle(description_wid…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=373.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Thresholds', max=5.0, style=ProgressStyle(description_wid…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Saved best model ep 9 with f score : 0.6943899229626291\n",
      "Ep 9: Train loss 0.5154 | Val f score 0.6944 with thresh 0.62, train f score 0.9899 with thresh 0.69\n",
      "\r"
     ]
    }
   ],
   "source": [
    "best_thr_score, thr_score_hist, losses = train(model, optimizer, lf, sched, metric_fc, tr_dl, val_dl,\n",
    "                                               n_epochs, train_df, val_df, train_tfms, val_tfms, \n",
    "                                               save_path=save_path, \n",
    "                                               prev_best_info=best_thr_score, info_history=thr_score_hist,\n",
    "                                               ep_start=ep_start, half_precision=True)\n",
    "loss_hist.append(losses)\n",
    "ep_start += n_epochs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXoAAAD4CAYAAADiry33AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+j8jraAAAgAElEQVR4nO3deXyU1dn/8c+VyQaEhABhMWEJisoq1AC2KGJVwKWAqAXF3UJb0aptfR6qtli0LrW/VvGhUrS0almkqIiVSkFZXKoSFGWTLQoEBEICJIFsM3P9/riHMAmRDDLhnplc79crr8zcy+TKEL45Ofe5zxFVxRhjTOyKc7sAY4wxDcuC3hhjYpwFvTHGxDgLemOMiXEW9MYYE+Pi3S6gttatW2vnzp3dLsMYY6LKqlWr9qlqRl376g16EZkBXAnsVdWedewX4GngcuAwcIuqfhLYdzPwYODQR1T1hfq+XufOncnNza3vMGOMMUFEZNs37Qul6+bvwLDj7L8M6Br4GA88G/iiLYFJwACgPzBJRNJDK9kYY0y41Bv0qroCKDrOISOAF9XxIdBCRNoDQ4HFqlqkqvuBxRz/F4YxxpgGEI6LsZnAjqDn+YFt37TdGGPMKRQRF2NFZDxOtw8dO3Y8Zn9VVRX5+fmUl5ef6tIaneTkZLKyskhISHC7FGNMmIQj6HcCHYKeZwW27QQG19q+rK4XUNXpwHSAnJycYybfyc/Pp3nz5nTu3Bnn2q9pCKpKYWEh+fn5ZGdnu12OMSZMwtF1swC4SRznAQdV9WtgETBERNIDF2GHBLadsPLyclq1amUh38BEhFatWtlfTsbEmFCGV87GaZm3FpF8nJE0CQCqOg1YiDO0cgvO8MpbA/uKRORhYGXgpSar6vEu6tZXx7c91ZwAe5+NiT31Br2qXlfPfgUmfMO+GcCMb1eaMcZEIb8fvOXOR1VZzc/ecqgqB29Z3Z9T2kDOrWEvKSIuxhpjjCsqD8PeDXDgKyeMq8rAWxEUwHWEdX3h7av49vVk9bOgjyYpKSmUlpa6XYYx5ojSvbD7c9i9FnavcT4KN4P66z5e4iC+CSQk1/E52Wl9xycd55igY+OTIaHJ8T/HJ4OnYSLZgj4GeL1e4uPtn9IYAPw+KMoLhPqaox+le44ek9YR2vWCHiOdz63OgISmtUI3AWLkmlXUpcNv31jH+l3FYX3N7qelMukHPY57zMSJE+nQoQMTJjiXIx566CHi4+NZunQp+/fvp6qqikceeYQRI0bU+/W+/vprRo8eTXFxMV6vl2effZYLLriAt956i/vvvx+fz0fr1q15++23KSoq4rbbbiMvL4+mTZsyffp0evfuzUMPPcTWrVvJy8ujY8eOTJkyhZ/85Cds374dgKeeeoqBAwee/JtjTCSrPAR71sOeoEDfsw6qDjv74xIg42w4/WIn0Nv1gnY9oUnjmo0l6oLeLaNHj+aee+6pDvq5c+eyaNEifvazn5Gamsq+ffs477zzGD58eL0jV2bNmsXQoUN54IEH8Pl8HD58mIKCAsaNG8eKFSvIzs6mqMgZoDRp0iT69u3L/Pnzeeedd7jppptYvXo1AOvXr+e9996jSZMmXH/99dx7772cf/75bN++naFDh7Jhw4aGfVOMOZVK9gTCPKilXrgFCNx6k5wG7XrDubccDfXWZ0F8optVR4SoC/r6Wt4NpW/fvuzdu5ddu3ZRUFBAeno67dq1495772XFihXExcWxc+dO9uzZQ7t27Y77Wv369eO2226jqqqKkSNH0qdPH5YtW8agQYOqb1Rq2bIlAO+99x6vvPIKAN///vcpLCykuNj5i2b48OE0adIEgCVLlrB+/frqr1FcXExpaSkpKSlhfy+MaVB+nxPgwd0uu9fAob1Hj2nRyQnyXtccDfW0DjHT1RJuURf0brr22muZN28eu3fvZvTo0cycOZOCggJWrVpFQkICnTt3Dulmo0GDBrFixQrefPNNbrnlFn7+85+Tnn7if0o2a9as+rHf7+fDDz8kOTn5hF/HGNdUlMLe9TVb6XvWOyNYwOl6adMNug45Guhte0CTFu7WHWUs6E/A6NGjGTduHPv27WP58uXMnTuXNm3akJCQwNKlS9m27Rung65h27ZtZGVlMW7cOCoqKvjkk0944IEHuOOOO/jyyy+ru25atmzJBRdcwMyZM/n1r3/NsmXLaN26Nampqce85pAhQ3jmmWe47777AFi9ejV9+vQJ6/dvTFhUHoIN/4LPZsOXy4+OekluAe17Q7/boW3PQNfLmdb1EgYW9CegR48elJSUkJmZSfv27Rk7diw/+MEP6NWrFzk5OZx99tkhvc6yZct48sknSUhIICUlhRdffJGMjAymT5/OqFGj8Pv9tGnThsWLF/PQQw9x22230bt3b5o2bcoLL9S9dsuUKVOYMGECvXv3xuv1MmjQIKZNmxbOb9+Yb8/vh6/ehc/mwPrXoeoQtOgIA++BDv2dUE/NtK6XBiLOja2RIycnR2uvMLVhwwa6devmUkWNj73fJmwKNjrh/vlcKM6HpFRnSOM510GH8yDOlq0OFxFZpao5de2zFr0xJrwOFcLaV5yumV2fgHjgjIthyGQ463JnrLo5pSzoG9CaNWu48cYba2xLSkrio48+cqkiYxqItwI2LXJa75sXgd/rdMcMfRR6XgPN27pdYaNmQd+AevXqVT3m3ZiYowr5uU7Lfe0rUH4AUtrCeT+F3mOcG5NMRLCgN8acmP3bnD73z2ZD0VZnTpduV8I5YyB7cIPN12K+PfsXMcbUr7zYGS3z2RzY9p6zrfMFcMHPodtwSD52yK+JHBb0xpi6+byQt8xpuX/xL2c63lZnwPcfhN6jneGRJipY0Btjatq91gn3Nf90Znxskg59b3CGRGaea2Pdo5AFfYgOHDjArFmzuOOOO07ovMsvv5xZs2bRooXdsm0iWMkeJ9g/m+PMBBmXAGcOdfrduw5x5l03UcuCPkQHDhzgz3/+8zFBX99c8AsXLmzo0kLi8/nweDxul2EiSeVh2LjQCfetbztTEWSeC5f/AXqMgmat3K7QhElIQS8iw4CnAQ/wvKo+Xmt/J5y1YTOAIuAGVc0P7PMBawKHblfV4SdV8b8nOhMfhVO7XnDZ48c9ZOLEiWzdupU+ffqQkJBAcnIy6enpfPHFF2zatImRI0eyY8cOysvLufvuuxk/fjwAnTt3Jjc3l9LSUi677DLOP/98PvjgAzIzM3n99derZ5+sbcqUKUybNo34+Hi6d+/OnDlzKC0t5a677iI3NxcRYdKkSVx99dXMnj2bRx99FFXliiuu4IknngCcVa5+/OMfs2TJEqZOncpXX33FlClTqKysZMCAAfz5z3+28G+MDu2DtyfD2lehssSZ9fH8nzut99Zd3a7ONIB67z8WEQ8wFbgM6A5cJyLdax32B+BFVe0NTAYeC9pXpqp9Ah8nF/Iuevzxxzn99NNZvXo1Tz75JJ988glPP/00mzZtAmDGjBmsWrWK3NxcpkyZQmFh4TGvsXnzZiZMmMC6deto0aJF9fTD3/T1Pv30Uz7//PPqOWsefvhh0tLSWLNmDZ9//jnf//732bVrF//7v//LO++8w+rVq1m5ciXz588H4NChQwwYMIDPPvuMVq1a8fLLL/P++++zevVqPB4PM2fObIB3ykS0L9+FZwc6rfjuw+HmN+Duz+HiX1vIx7BQWvT9gS2qmgcgInOAEcD6oGO6Az8PPF4KzA9nkTXU0/I+Vfr37189dzw4LfDXXnsNgB07drB582Zatar5p292dnb1jJLnnnsuX3311Te+fu/evRk7diwjR45k5MiRgDPn/Jw5c6qPSU9PZ8WKFQwePJiMjAwAxo4dy4oVKxg5ciQej4err74agLfffptVq1bRr18/AMrKymjTps1Jvgsmavh9sOJJWP4EtOwCN8xz/pI1jUIoQZ8J7Ah6ng8MqHXMZ8AonO6dq4DmItJKVQuBZBHJBbzA46p6zC8BERkPjAfo2DE6hmwFzwW/bNkylixZwn//+1+aNm3K4MGD65yXPinp6AUtj8dDWVnZN77+m2++yYoVK3jjjTf43e9+x5o1J95dlZycXN01o6rcfPPNPPbYY/WcZWJO8dfw6jhn9sjeY+CK/wdJtiBNYxKuqeN+CVwoIp8CFwI7AV9gX6fAjGrXA0+JyOm1T1bV6aqao6o5R1qmkaZ58+aUlJTUue/gwYOkp6fTtGlTvvjiCz788MOT+lp+v58dO3Zw0UUX8cQTT3Dw4EFKS0u59NJLmTp1avVx+/fvp3///ixfvpx9+/bh8/mYPXs2F1544TGvefHFFzNv3jz27nVW6SkqKgp5/nwTxbYsgWnnw85VMPJZGPUXC/lGKJQW/U6gQ9DzrMC2aqq6C6dFj4ikAFer6oHAvp2Bz3kisgzoC2w96cpPsVatWjFw4EB69uxJkyZNaNv26CRNw4YNY9q0aXTr1o2zzjqL884776S+ls/n44YbbuDgwYOoKj/72c9o0aIFDz74IBMmTKBnz554PB4mTZrEqFGjePzxx7nooouqL8bWtUB59+7deeSRRxgyZAh+v5+EhASmTp1Kp06dTqpWE6F8VfDOI/D+U9CmO1z7d8g4y+2qjEvqnY9eROKBTcDFOAG/ErheVdcFHdMaKFJVv4j8DvCp6m9EJB04rKoVgWP+C4xQ1fXHfiWHzUfvPnu/o9yB7TDvdsj/2Fkoe9jjNjVwI3BS89GrqldE7gQW4QyvnKGq60RkMpCrqguAwcBjIqLACmBC4PRuwF9ExI/TTfT48ULeGHOSvngT5t/hXHy9Zgb0vNrtikwECGkcvaouBBbW2vaboMfzgHl1nPcBYJf2j2PChAm8//77Nbbdfffd3HrrrS5VZKKStwIW/wY+mgbt+zgh3+qYy2GmkYqaO2NVFYnBOTaCL65GgkhbWtKEoHArzLsVvv4MBvwULv2tTVlgaoiKoE9OTqawsJBWrVrFZNhHClWlsLCQ5ORkt0sxoVozD964B+I8MGYWnH2F2xWZCBQVQZ+VlUV+fj4FBQVulxLzkpOTycrKcrsMU5/Kw/DWRPjkBegwAK7+K7ToUP95plGKiqBPSEiocReqMY3a3i+crpq96+H8e+GiB8CT4HZVJoJFRdAbY3DWaF09ExbeBwlN4YZX4IxL3K7KRAELemOiQUUJvPkL+PxlZwm/Uc9Banu3qzJRwoLemEj39edOV01RHgy+Hwb90rn4akyILOiNiVSqsPJ5WPSAs5zfTQsg+wK3qzJRyILemEhUdgAW3AUbFsAZl8JV06BZa7erMlHKgt6YSJO/CubdAsW74NLJ8N27IC5cE82axsiC3phI4ffDh1NhyUPQvD3c+m/o0N/tqkwMsKA3JhIcKoT5P4XNi+DsK2HE/zn98saEgQW9MW7b9oEzrfDhfXDZk9B/HNhUHyaMLOiNcYvfB+/+EZY9Cumd4fbFcFoft6syMciC3hg3lOxx1nH9cjn0vAZ+8BQkNXe7KhOjLOiNOdW2vgOvjoeKUhj+DPS90bpqTIOyoDfmVFGFpY/Ciied9VtvWgBtu7tdlWkELOiNORVUYfGv4YNnoM9YuPwPkNjU7apMI2FBb8ypsPwJJ+T7/cgJeeuqMadQSLfbicgwEdkoIltEZGId+zuJyNsi8rmILBORrKB9N4vI5sDHzeEs3pio8P4UWPaY05K/7EkLeXPK1Rv0IuIBpgKXAd2B60SkdsfiH4AXVbU3MBl4LHBuS2ASMADoD0wSEbsLxDQeK593umx6XOVceLWpDIwLQvmp6w9sUdU8Va0E5gAjah3THXgn8Hhp0P6hwGJVLVLV/cBiYNjJl21MFFg9y5lD/sxhcNV0m1rYuCaUoM8EdgQ9zw9sC/YZMCrw+CqguYi0CvFcRGS8iOSKSK6tC2tiwrr58PoEyL4Qrn0B4hPdrsg0YuH6O/KXwIUi8ilwIbAT8IV6sqpOV9UcVc3JyMgIU0nGuGTTInjldsjqD9fNhoRktysyjVwoo252AsHLy2cFtlVT1V0EWvQikgJcraoHRGQnMLjWuctOol5jIlvecnj5RmjbE8bOhcRmbldkTEgt+pVAVxHJFpFEYAywIPgAEWktIkde61fAjMDjRcAQEUkPXIQdEthmTOzZ/hHMvg5anQ43vgbJaW5XZAwQQtCrqhe4EyegNwBzVXWdiEwWkeGBwwYDG0VkE9AW+F3g3CLgYZxfFiuByYFtxsSWXath5jXQvB3cOB+atnS7ImOqiaq6XUMNOTk5mpub63YZxoRu7wb42+WQmAK3/RvSsuo/x5gwE5FVqppT1z4b1GvMySjcCi+OAE8i3DTfQt5EJJsCwZhv68B2eGE4+L1wy0Knb96YCGRBb8y3UbLbaclXlMAtb0Cbs92uyJhvZEFvzIk6VOiEfMkep7um/TluV2TMcVnQG3Miyg7ASyNh/1cwdh506O92RcbUy4LemFBVlMKsHzqjbMbMguwL3K7ImJBY0BsTiqoymHMd5K+Ea/8OZw5xuyJjQmbDK42pj7cS5t4MX74LI6dB99qTtxoT2axFb8zx+Lzw6o9g8yK48k9wzmi3KzLmhFmL3phv4vfDgjth/esw9FHIuc3tioz5VizojamLKiz8JXw2Gy56AL47we2KjPnWLOiNqU3VWf4v968w8G4YdJ/bFRlzUizojalt+RPwwTPQbxxc8ltbzNtEPQt6Y4K9PwWWPQZ9xsJlv7eQNzHBgt6YI1Y+73TZ9BgFw5+BOPvvYWKD/SQbA7B6Frz5CzjzMhg1HeI8bldkTNhY0Buzbj68PgG6DHbuevUkuFyQMeFlQW8at02L4JXbIau/M39NQrLbFRkTdiEFvYgME5GNIrJFRCbWsb+jiCwVkU9F5HMRuTywvbOIlInI6sDHtHB/A8Z8a3nL4eUboV0vGDsXEpu5XZExDaLeKRBExANMBS4F8oGVIrJAVdcHHfYgzqLhz4pId2Ah0Dmwb6uq9glv2cacpO0fwezrnFWhbngVktPcrsiYBhNKi74/sEVV81S1EpgD1J7VSYHUwOM0YFf4SjQmzHathpnXQPN2cON8aNrS7YqMaVChBH0msCPoeX5gW7CHgBtEJB+nNX9X0L7sQJfOchGpcwJvERkvIrkikltQUBB69cacqL0b4KWrILkF3LwAmrd1uyJjGly4LsZeB/xdVbOAy4GXRCQO+BroqKp9gZ8Ds0QktfbJqjpdVXNUNScjIyNMJRlTS+FWZwlATyLc/DqkZbldkTGnRChBvxPoEPQ8K7At2O3AXABV/S+QDLRW1QpVLQxsXwVsBc482aKNOWEHtsMLw8HvhZteh5Zd3K7ImFMmlKBfCXQVkWwRSQTGAAtqHbMduBhARLrhBH2BiGQELuYiIl2ArkBeuIo3JiSVh2HmtVBZ4vTJtznb7YqMOaXqHXWjql4RuRNYBHiAGaq6TkQmA7mqugD4BfCciNyLc2H2FlVVERkETBaRKsAP/ERVixrsuzGmLot/DQVfwI2vQfvebldjzCknqup2DTXk5ORobm6u22WYWLHxLZg9Gr57Jwz9ndvVGNNgRGSVqubUtc/ujDWxq3SvM7VB215w8W/crsYY11jQm9ikCvPvgMpSuPp5iE9yuyJjXGOLg5vY9PFzsGUxXP4Hu/hqGj1r0ZvYs3cD/OdB6DoE+v3I7WqMcZ0FvYktVeXwyo8gORVGTLUVoozBum5MrHl7MuxZC9f/E1LauF2NMRHBWvQmdmx9Bz6cCv3Hw5lD3K7GmIhhQW9iw6FCeO2nkHE2XDrZ7WqMiSjWdWOinyosuAvKiuCGeZDQxO2KjIkoFvQm+n3yAmx8E4b8zlktyhhTg3XdmOi2bwu89StnYe/z7nC7GmMikgW9iV7eSmdh7/gkGDkN4uzH2Zi6WNeNiV7LHoOvV8Pof0Bqe7erMSZiWRPIRKev3oP3/gTfuQm6/cDtaoyJaBb0JvqU7YdXf+ysEjX0MberMSbiWdeNiS6q8K97oXQ33P4fSEpxuyJjIp616E10+WwOrHsNLrofMs91uxpjooIFvYkeRV/Cwvug00AYeI/b1RgTNSzoTXTweeHV8SBxcNVfIM7jdkXGRI2Qgl5EhonIRhHZIiIT69jfUUSWisinIvK5iFwetO9XgfM2isjQcBZvGpF3/wD5H8OVf4QWHdyuxpioUu/FWBHxAFOBS4F8YKWILFDV9UGHPQjMVdVnRaQ7sBDoHHg8BugBnAYsEZEzVdUX7m/ExLDtH8HyJ6D3GOh1jdvVGBN1QmnR9we2qGqeqlYCc4ARtY5RIDXwOA3YFXg8ApijqhWq+iWwJfB6xoSmvBheHQdpHeDyJ92uxpioFErQZwI7gp7nB7YFewi4QUTycVrzd53AuYjIeBHJFZHcgoKCEEs3jcK//xcO7oBRzzmrRhljTli4LsZeB/xdVbOAy4GXRCTk11bV6aqao6o5GRkZYSrJRL21r8Bns2DQ/0DHAW5XY0zUCuWGqZ1A8NWvrMC2YLcDwwBU9b8ikgy0DvFcY451YIdzY1RWPxh0n9vVGBPVQml1rwS6iki2iCTiXFxdUOuY7cDFACLSDUgGCgLHjRGRJBHJBroCH4ereBOj/D547SfO51HTwWM3cBtzMur9H6SqXhG5E1gEeIAZqrpORCYDuaq6APgF8JyI3ItzYfYWVVVgnYjMBdYDXmCCjbgx9Xr/adj2Hox81pnPxhhzUsTJ48iRk5Ojubm5bpdh3LLrU3j+EmdGymv+BiJuV2RMVBCRVaqaU9c+uzPWRI7KQ/DKjyClLVz5Jwt5Y8LEOj9N5Fh0PxRuhZvfgCbpbldjTMywFr2JDBv+Bav+DgPvhuwL3K7GmJhiQW/cV/w1LLgL2p8DFz3gdjXGxBwLeuMuvx/m/xSqyuDqv0J8otsVGRNzrI/euOujaZC3FK58Clp3dbsaY2KSteiNe3avhSWT4Kwr4Nxb3K7GmJhlQW/cUVXmDKVskg7Dn7GhlMY0IOu6Me5YPAkKNsANr0CzVm5XY0xMsxa9OfU2L4aP/wLn3QFnXOJ2NcbEPAt6c2qVFsD8O6BND7h4ktvVGNMoWNeNOXVUYcGdUH4QbnodEpLdrsiYRsGC3pw6uX+FTW/BsCegbXe3qzGm0bCuG3NqFGyERQ84ffIDfux2NcY0Khb0puF5K+CV2yGxGYz4sw2lNOYUs64b0/DeeRh2r4Hr5kDztm5XY0yjYy1607C2LoUP/g9yboezLnO7GmMaJQt603B2rYa5N0HGWTDkEberMabRCinoRWSYiGwUkS0iMrGO/X8SkdWBj00iciBony9oX+1FxU2sKtgI/xgFyWnO3a+JTd2uyJhGq94+ehHxAFOBS4F8YKWILFDV9UeOUdV7g46/C+gb9BJlqtonfCWbiLf/K3hxBMTFO+Pl07LcrsiYRi2UFn1/YIuq5qlqJTAHGHGc468DZoejOBOFinfBC8PBWw43zodWp7tdkTGNXihBnwnsCHqeH9h2DBHpBGQD7wRtThaRXBH5UERGfsN54wPH5BYUFIRYuok4h/bBiyPhcJHTXWM3RRkTEcJ9MXYMME9VfUHbOqlqDnA98JSIHNPEU9XpqpqjqjkZGRlhLsmcEuUH4aWr4MA2uP5lyDzX7YqMMQGhBP1OoEPQ86zAtrqMoVa3jaruDHzOA5ZRs//exILKQzDzh7B3A4yeCZ0Hul2RMSZIKEG/EugqItkikogT5seMnhGRs4F04L9B29JFJCnwuDUwEFhf+1wTxarKYc5YyP8YrvkrdLVph42JNPWOulFVr4jcCSwCPMAMVV0nIpOBXFU9EvpjgDmqqkGndwP+IiJ+nF8qjweP1jFRzlcF825z1nwd+Sx0P941emOMW6RmLrsvJydHc3Nz3S7D1Mfvh9d+DGvmwmVPwoDxbldkTKMmIqsC10OPYXfGmhOnCgt/4YT8xb+xkDcmwlnQmxOjCot/A7kz4Px74YJfuF2RMaYeFvTmxKz4A3wwBfqNs6UAjYkSFvQmdB8+C0sfgXOug8t+b/PKGxMlLOhNaD55Cd6aCN2Gw/D/gzj70TEmWtj/VlO/ta/AgrucZQCvfh48tl6NMdHEgt4c36ZF8Op46Phd+OFLEJ/kdkXGmBNkQW++2Zcr4OUboV0vZ/4am1PemKhkQW/qlp8Ls6+Dll3ghlchOdXtiowx35IFvTnW7rXO6lDNMuCm+dC0pdsVGWNOggW9qWnfFnhpJCSmOKtDNW/ndkXGmJNkQW+OOrDdWQIQnJBP7+RuPcaYsLBxcsZRstsJ+coSuOVNaN3V7YqMMWFiQW+cpf9eugpK9jgt+Xa93K7IGBNGFvSNXXkx/ONqKNwKY/8JHfq5XZExJsws6BuzysMwewzs/txZArDLhW5XZIxpABb0jZW3EubeBNs+cJYAPGuY2xUZYxqIBX1j5PPCK7fDlsUw/BnoebXbFRljGpANr2xs/H5442ewYQEMfQy+c5PbFRljGlhIQS8iw0Rko4hsEZGJdez/k4isDnxsEpEDQftuFpHNgY+bw1m8OUGqzlTDq2fCRQ/Ad+9wuyJjzClQb9eNiHiAqcClQD6wUkQWqOr6I8eo6r1Bx98F9A08bglMAnIABVYFzt0f1u/ChOadh+Hjv8D37oJB97ldjTHmFAmlRd8f2KKqeapaCcwBRhzn+OuA2YHHQ4HFqloUCPfFgF31c8O7f4R3/x+cewtc+rCtDmVMIxJK0GcCO4Ke5we2HUNEOgHZwDsncq6IjBeRXBHJLSgoCKVucyI+fg7e/i30uhau+KOFvDGNTLgvxo4B5qmq70ROUtXpqpqjqjkZGRlhLqmRWz0bFv4SzroCRj4LcR63KzLGnGKhDK/cCXQIep4V2FaXMcCEWucOrnXustDLMydl/QJ4/Q7oMhiumQGeBLcrMhHA51d27i/D6/cTHxeHxyN4RPDE1fyIjxPiJPA5LjL/ClRVfH7F61eqfH68PqXK73e2+QLbgvZ5/Yr3mG1+fH7wqaKq+FXx+cEfeB782K/O++c8d86pfhzY7lfwBz9WDTwPPNZvOM6vdGzVlHsuOTPs71MoQb8S6Coi2TjBPQa4vvZBInI2kA78N2jzIgyQEjcAAA4GSURBVOBREUkPPB8C/OqkKjb1O1wEq/4GSx+DrH4wZhYkJLtdlXGBz69sLShlTf5B1u46yNqdB1m3q5jDlSf0RzdAdeDHxwV+MXzDLwhPXM3t8XXtj4vDIxAn4oSv30+V72gIHwngI8FdI6RrbYtEnjghTkDEeS/iAt+ryJF94uyLc7Yf2fdt/l1CUW/Qq6pXRO7ECW0PMENV14nIZCBXVRcEDh0DzFFVDTq3SEQexvllATBZVYvC+y2YanvWwUd/gc/ngrcssJj3XyGxmduVmVPA6/Oz5Uio7zzI2l3FrN9VTFmVEx5NEjx0Py2VH+Z0oFv75iTFe/D5nRaxT53Q9PtrfVYndH1+f/3HBFrXx3wEbff6laoqPz6/r7oFHB8nxHviiI8TmibGE+8R4uPiSPA4vxQSAvviPc62+Li4wDGBbTX2CZ5jttV9fLxHSAjsOxK+RwL62FAmENiB53FHnwcfdyTcI40E5XJEyMnJ0dzcXLfLiB5+H2x6Cz6a5qzxGt8Eev8QBvwE2nZ3uzrTQKp8fjbvKWXtzoOsCXxs+LqYCq8fgGaJHnqclkaPzFR6ZabRKzONLhkpeCK0C8acPBFZpao5de2zKRCiVdkB+PQf8PF0OLANUrPgkt86d7ra0n8xpdLrZ9OeEtbsDLTUdx5kw+4SKgOhnpIUT4/TUrnxvE70zEyjZ2Ya2a2bWaibahb00aZgk3PT0+rZUHUIOn4PLp0MZ18JHvvnjHYVXh8bdweHejEbd5dQ6XNCvXlyPD1PS+OW73V2Qv20VDq3ahaxF0tNZLBkiAZ+P2xZ4nTPbH0bPInOmPgBP4b257hdnfmWyqt8fHEk1POd7pdNe0qqLzCmNUmgZ2Yqt57fubr7pWPLphHZB2wimwV9JKsogdWznAusRVshpR1c9KBzd2uK3W8QLVSVPcUVbNlbypa9JazbVcyanQfZvLcUXyDU05sm0DMzjXFndakO9az0JhbqJiws6CNR4VbnbtZP/+Gs4ZrVDy66H7oNh/hEt6sz38Dr87O96DBb9payteCQE+wFpeTtLaWkwlt9XKtmifTMTOOSbm3pmZlKz8w0MltYqJuGY0EfKVQhb6nTet+0COLiocdVzuiZrHPdrs4EKav0kbev1An0QJhv2VvKV/sOV/elA7RNTeKMNilc9Z1MzmiTwhkZKZzRJoWM5kkW6uaUsqB3W+Uh+GyOE/D7NkKzDLjwfyDnNmjezu3qGrUDhysD3S2l1a3zLXtL2XmgjCOjkuMEOrVqxukZzbjo7DbVYX56mxRSk+1OZBMZLOjdsn8brHwOPnkRyg86F1VHToOeoyA+ye3qTlqF10dBSQV7iis4XOklOcFDcryH5IQ453GC87hJgod4j3vr36gqXx8sPybM8wpK2VdaWX1cUnwcXTJS6NsxnWvP7eC00Nuk0Ll1U5Libf4gE9ks6E8lVfjqPWf0zMaFgED34TDgp9Chf1TMKun1+dlXWsme4nLno6SCvUceF1ewp7icvSUVFB2qrP/FAuLjpEb4B/8SSE7wkBTvoUmih+R4Z9+Rx0mB/U2CzmuS4CEp6HH1a8Z7KDxUGeg/P9rlsnVvKYeCbjtPa5LAGW1SuPjsttVhfkabFE5r0cTGpZuoZUF/KlSVwZp/Ot0ze9ZCk5Yw8B7odzukZbldHeBMqFR02AnwvYHA3lNcwZ6S8kCQO9v2lVZQe3qROIGM5km0TU0mK70p53ZKp21qMm1TnW3NkuKpqPJTVuWjvMaHn/IqX2C7n3LvsftKK7zsK62kovq4o8eezE3d7dOSOaNNCtfmdOD0oP7z1imJ1n9uYo4FfUM6uBNWPg+r/g5lRdC2p7MYd69rIaHJKSlBVSku87KnpGare09QK3xvoBVe1wRRrZolVod2j9NSaXMkwJsnV29vlZJ0ylu7qkqF109FIPTLKn2BXxT+6scVgV8KR35BpCYnVPefpyTZj75pPOynvSFs/wg+etaZJhiFsy53Rs90Pr9Bumf8fmXXwTLyCg6RV1DKl/sOkbfvENsKD7OnuLx6/pNgaU0Sqlvcp2e0rn7cNjUpEObJZKQkkRgfmevHixzt7knDLnoaczwW9OG0bzMsegA2L4LkNGfx7X7jIL1TWF6+pLzKCfN9pYFQP8TWglK+KjxEedXRME9JiqdLRjPO6dCC9mnJtGl+JMSTaZeaTJvUJJIT7AKiMY2FBX04HC6C5b93RtEkNHXmnun3o281PbDX52fH/jLyCkqrQ31rwSG+3HeIgpKK6uM8cUKH9CZ0yUjh/DNa0yUjhezWzjA/G6dtjAlmQX8yfN7AAh+/c4ZIfudmuOiBkKYnKDpUWR3mW6tb6KVsLzpMle9oX3nLZolkt27G4DMz6JKRQpcMJ8w7tmwWsd0qxpjIYkH/bW1Z4nTTFHwB2YNg6GPQrmeNQyq8PrYVHiavwGmVH2mhf7nvEAcOV1Ufl+iJo1OrppzRJoUhPdrRpXUzumSkcHpGM1o0tSkPjDEnx4KeY0dwBA/vK6/yUe49+jhx/xZ6rXuSrH3vciA5i7fP/D1rUwZS/oFSUbWacq+PknIv2woPk7//cI2hiG1Tk+jSOoUrerUPdLM4LfSs9KY2RtsY02BiJuhLK7w8tyKv5jjs2qHt9QeG3NXcX+H11zsmO41S7o5/lRs9iykjkUe8Y3mxfAhakkhy/M7AzTvOjTrNEj30zkpjZN9MTs9oRpfWKWRnNLMhfcYYV4SUPCIyDHgaZ83Y51X18TqO+SHwEKDAZ6p6fWC7D1gTOGy7qg4PQ93HqPT6efrtzSTGx1XfQVnjTst4D2lNEkhunlTrDsyad1keuYuy+u5Mj5+srXM4bfVTeCqLOdRjLFUXTOTeFu2YGB/n6u37xhgTinqDXkQ8wFTgUiAfWCkiC1R1fdAxXYFfAQNVdb+ItAl6iTJV7RPmuo+R3jSBvEcvD+9KO1uWwFv3O5ONBfrhU9r1JCV8X8EYYxpcKC36/sAWVc0DEJE5wAhgfdAx44CpqrofQFX3hrvQ+khgNfawKNgE/3kANv8H0rNhzCznpicbsmiMiUKhBH0msCPoeT4woNYxZwKIyPs43TsPqepbgX3JIpILeIHHVXX+yZXcgA4XwfInnGkLEprCkEeg//iYmE3SGNN4hevqYDzQFRgMZAErRKSXqh4AOqnqThHpArwjImtUdWvwySIyHhgP0LFjxzCVdAJ8VZD7N1j26AmPhzfGmEgXStDvBDoEPc8KbAuWD3ykqlXAlyKyCSf4V6rqTgBVzRORZUBfoEbQq+p0YDpATk7OScxJ+C1sXgKLavbD1x4Pb4wx0SyUISMrga4iki0iicAYYEGtY+bjtOYRkdY4XTl5IpIuIklB2wdSs2/fPQWb4B/XwMyrwVfp9MPftMBC3hgTc+pt0auqV0TuBBbh9L/PUNV1IjIZyFXVBYF9Q0RkPeAD7lPVQhH5HvAXEfHj/FJ5PHi0jiuO9MN//JwzF431wxtjYpzoyaze0ABycnI0Nzc3/C/sq4LcGbD0Uagotn54Y0xMEZFVqppT177Gcaum9cMbYxqx2A76go3OxGNbFkPLLjBmNpx1mY2HN8Y0KrEZ9IeLYNnjznj46n74H0O8zQRpjGl8Yivoa/fDn3uL0w/frLXblRljjGtiJ+j3fwUzr4V9myD7Qhj2GLTt4XZVxhjjutgJ+tRMZ16aS35r/fDGGBMkdoLekwBj57pdhTHGRBybTN0YY2KcBb0xxsQ4C3pjjIlxFvTGGBPjLOiNMSbGWdAbY0yMs6A3xpgYZ0FvjDExLuLmoxeRAmDbSbxEa2BfmMqJdvZe1GTvR032fhwVC+9FJ1Wtc4GNiAv6kyUiud80+X5jY+9FTfZ+1GTvx1Gx/l5Y140xxsQ4C3pjjIlxsRj0090uIILYe1GTvR812ftxVEy/FzHXR2+MMaamWGzRG2OMCWJBb4wxMS5mgl5EhonIRhHZIiIT3a7HTSLSQUSWish6EVknIne7XZPbRMQjIp+KyL/crsVtItJCROaJyBciskFEvut2TW4SkXsD/0/WishsEUl2u6Zwi4mgFxEPMBW4DOgOXCci3d2tylVe4Beq2h04D5jQyN8PgLuBDW4XESGeBt5S1bOBc2jE74uIZAI/A3JUtSfgAca4W1X4xUTQA/2BLaqap6qVwBxghMs1uUZVv1bVTwKPS3D+I2e6W5V7RCQLuAJ43u1a3CYiacAg4K8Aqlqpqgfcrcp18UATEYkHmgK7XK4n7GIl6DOBHUHP82nEwRZMRDoDfYGP3K3EVU8B/wP43S4kAmQDBcDfAl1Zz4tIM7eLcouq7gT+AGwHvgYOqup/3K0q/GIl6E0dRCQFeAW4R1WL3a7HDSJyJbBXVVe5XUuEiAe+Azyrqn2BQ0CjvaYlIuk4f/1nA6cBzUTkBnerCr9YCfqdQIeg51mBbY2WiCTghPxMVX3V7XpcNBAYLiJf4XTpfV9E/uFuSa7KB/JV9chfePNwgr+xugT4UlULVLUKeBX4nss1hV2sBP1KoKuIZItIIs7FlAUu1+QaERGcPtgNqvpHt+txk6r+SlWzVLUzzs/FO6oacy22UKnqbmCHiJwV2HQxsN7Fkty2HThPRJoG/t9cTAxenI53u4BwUFWviNwJLMK5aj5DVde5XJabBgI3AmtEZHVg2/2qutDFmkzkuAuYGWgU5QG3ulyPa1T1IxGZB3yCM1rtU2JwOgSbAsEYY2JcrHTdGGOM+QYW9MYYE+Ms6I0xJsZZ0BtjTIyzoDfGmBhnQW+MMTHOgt4YY2Lc/wdo2q1pFp9pMgAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXoAAAD4CAYAAADiry33AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+j8jraAAAgAElEQVR4nO3deXxU9b3/8deHEAgIQmQR2RERUFGQNGjRauuGtgVXQHYV0Vat13q9xdv2atXe8rO2ViveapFFdkRUsFYrV6y3isCAKLIvggQEQtj3LJ/fH2fUAEESmeTMnLyfj0ceZM6cM3lnSN45c86Z79fcHRERia4qYQcQEZHypaIXEYk4Fb2ISMSp6EVEIk5FLyIScVXDDnCk+vXre8uWLcOOISKSUubPn7/V3RuUdF/SFX3Lli2JxWJhxxARSSlmtu5Y9+nQjYhIxJWq6M2sm5ktN7NVZja0hPufNLOF8Y8VZraj2H2Fxe6bnsjwIiJyfMc9dGNmacBw4AogB5hnZtPdfcmX67j7fcXWvwfoVOwh9rt7x8RFFhGRsijNMfpsYJW7rwEws0lAD2DJMda/GXgoMfEC+fn55OTkcODAgUQ+bORlZGTQtGlT0tPTw44iIiEqTdE3AdYXu50DdClpRTNrAbQC3im2OMPMYkABMMzdXy1huyHAEIDmzZsf9bg5OTnUrl2bli1bYmaliCzuTl5eHjk5ObRq1SrsOCISokSfjO0NTHX3wmLLWrh7FtAH+JOZtT5yI3d/3t2z3D2rQYOjrw46cOAA9erVU8mXgZlRr149vQoSkVIV/QagWbHbTePLStIbmFh8gbtviP+7BniXw4/fl5pKvuz0nIkIlO7QzTygjZm1Iij43gR754cxs3ZAJjC72LJMYJ+7HzSz+kBX4PFEBBcpk5UzYf2csFPIkWrWg6xboWq1sJNE2nGL3t0LzOxu4C0gDRjp7ovN7BEg5u5fXjLZG5jkhw9w3x54zsyKCF49DCt+tY5IhfjkJZh2O+CAXuUkF4f1H8INL0CVtLDDRFap3hnr7m8Abxyx7L+OuP1wCdt9AHQ4gXwpq1atWuzZs6fE+9auXcsHH3xAnz7BC6PRo0cTi8V45plnKjJi5bDsb/DKHdDyIuj7EqTXCDuRFPfBM/CPX0J6Tej+DFTRezjLg57VEKxdu5YJEyaUebvCwsLjryRfW/0OvDQIGneCmyeq5JPRd++GSx+EhePhzV+AZrwrF0k31s3x/GbGYpZs3JXQxzyr8ck89OOzv3GdoUOH0qxZM+666y4AHn74YapWrcqsWbPYvn07+fn5PPbYY/To0eO4X2/o0KEsXbqUjh07MnDgQDIzM9m4cSPdunVj9erVXHfddTz+eHAqo1atWtxxxx3MnDmT4cOHc9FFF534N1wZrJsNE/tA/bbQbypUrx12IjmWS34BB3fD7GegWi24PKFvwxG0R19qvXr1YsqUKV/dnjJlCgMHDuSVV15hwYIFzJo1i/vvv5/SzME7bNgwLr74YhYuXMh99wVvKl64cCGTJ09m0aJFTJ48mfXrg7cu7N27ly5duvDxxx+r5EtrwwKY0BPqNIX+r0CNzLATyTcxgysfC07K/uuP8N4TYSeKnJTboz/ennd56dSpE1u2bGHjxo3k5uaSmZlJo0aNuO+++3jvvfeoUqUKGzZsYPPmzTRq1KjMj3/ZZZdRp04dAM466yzWrVtHs2bNSEtL44Ybbkj0txNdm5fAuOuhRl0Y8BrUKnHUVkk2ZnDNH+DQXnjn0WDP/oI7w04VGSlX9GG66aabmDp1Kps2baJXr16MHz+e3Nxc5s+fT3p6Oi1btvzWb1CqXr36V5+npaVRUFAABMMYpKXpaoRSyVsNY6+FqhkwYDrUaRJ2IimLKlWgx7NB2b/5C6h2EpzfP+xUkaBDN2XQq1cvJk2axNSpU7npppvYuXMnDRs2JD09nVmzZrFu3TGHgz5M7dq12b17dzmnrWR2rIcXe0BRQbAnf4qGfUhJaVXhxpHQ+jKY8TP49OWwE0WCir4Mzj77bHbv3k2TJk047bTT6Nu3L7FYjA4dOvDiiy/Srl27Uj3OueeeS1paGueddx5PPvlkOaeuBHZvhhe7w4FdwTH5Bm3DTiQnomp16DUOml8I04bA8r+HnSjlWWlOHlakrKwsP3KGqaVLl9K+ffuQEqW2yD93+7bB6B/C9nUw4FVolh12IkmUA7uCV2mbF0PfKXD6pWEnSmpmNj8+rthRtEcvqevAruDEa97q4Dp5lXy0ZJwM/V6GemcEl8p+riEsvi0VfTlatGgRHTt2POyjS5cSR3iWsjq0Dyb0gk2LoOeLcPolYSeS8lDzlOCVWu1GMP4m2Lgw7EQpSVfdlKMOHTqwcKF+MBOu4CBM7vv1GCltu4WdSMpTrYbBCfZRVwev4Aa9AQ1Ldz5MAtqjl9RSmA9Tbw2GN+j+Zzjn+rATSUWo2ywo+ypVg+P229aEnSilqOgldRQVwas/hWWvw9WPQ6d+YSeSilSvdVD2hYdgTA/YeaxpMeRIKnpJDe7wt5/Doilw2X9BlzvCTiRhaNge+k+DAzuCPfs9uWEnSgkqekl+7vCPX8H8UXDRz+Hi+8NOJGFq3An6TIFdG4J3Qu/bFnaipKeiL6UdO3bw7LPPlnm7a665hh07dpR5u9GjR7Nx48avbrds2ZKtW7eW+XEi4d1hwciG2XcEe/MiLS6E3hNg6woYf2Mw+qUck4q+lI5V9F+OSXMsb7zxBnXr1i3z1zuy6EvjeFlS0gd/hn8Og479oNuwYPArEYDW34ebRgeXXE68GfL3h50oaaXe5ZV/HxpcO51IjTrA1cO+cZWhQ4eyevVqOnbsSHp6OhkZGWRmZrJs2TJWrFjBtddey/r16zlw4AD33nsvQ4YMAYI98Vgsxp49e7j66qu56KKL+OCDD2jSpAmvvfYaNWocPRnG1KlTicVi9O3blxo1ajB7djAN75///GdmzJhBfn4+L730Eu3atePhhx9m9erVrFmzhubNmzNx4sSjHi9lzXshOGRz9nXQ/WnNPiRHa/dDuP55eHkwTO4f7OVr/tmj6DenlIYNG0br1q1ZuHAhv//971mwYAFPPfUUK1asAGDkyJHMnz+fWCzG008/TV5e3lGPsXLlSu666y4WL15M3bp1efnlkgdsuvHGG8nKymL8+PEsXLjwqz8G9evXZ8GCBfzkJz/hiSe+HrN7yZIlzJw5M1ol//Fk+Nv90OYquO55zScqx9bhRvjxU7DqbXj5NiiM4CvbE5R6e/TH2fOuKNnZ2bRq9fUIiU8//TSvvPIKAOvXr2flypXUq1fvsG1atWpFx44dAejcuTNr164t09e8/vrrv9p22rRpXy3v3r17ia8MUtbSGfDqT6DVxdBzjPbQ5Pg6DwyGN37rQZh+dzDcsV4BfiX1ij5JnHTSSV99/u677zJz5kxmz55NzZo1ufTSS0scl/7IMef37y/bMcUvty8+Xv2RWVLeypnw0i3QpDP01jyvUgYX/jQo+1mPBWPZX/OEzunEqehL6ZvGkN+5cyeZmZnUrFmTZcuW8eGHH5br14uste8HQxs0bAd9X4LqtcJOJKnme/8Oh3bD+08FZX/5b1T2qOhLrV69enTt2pVzzjmHGjVqcOqpp351X7du3fjLX/5C+/btadu2LRdccMEJf71BgwZx5513HnYyNtI2zA8GKavbAvq/GkwFKFJWZkG5H9obL/vacMkDYacKncajj7iUeO42fRqMKZ9RB259E05uHHYiSXVFRfDaT+HjiXDV74LDOhH3TePRa49ewrV1VfDuxvSaMHC6Sl4So0oV6P7M1ydoq50UnLCtpHRaOmR33XXXUWPWjxo1KuxYFWPH58F4Je7BYFWZLcNOJFGSVjUYxvqMK2DGvbBoatiJQpMye/TujkXwpMrw4cPL7bGT7bDcYXZvgjHdgxNnA1+HBmeGnUiiqGo16DU2mLRk2pDglWO7a8JOVeFKtUdvZt3MbLmZrTKzoSXc/6SZLYx/rDCzHcXuG2hmK+Mf3+q1U0ZGBnl5ecldXEnG3cnLyyMjIyPsKEfbmwcvXgt7tkDfl+G0c8NOJFGWXiOYarJxJ3hpIKyeFXaiCnfck7FmlgasAK4AcoB5wM3uvuQY698DdHL3W83sFCAGZAEOzAc6u/v2Y329kk7G5ufnk5OTU+K16XJsGRkZNG3alPT09LCjfO3AThjzY8hdHlxC2ep7YSeSymL/dhj9o2DSkn7TgoHRIuRET8ZmA6vcfU38wSYBPYASix64GXgo/vlVwNvuvi2+7dtAN6BM79VPT08/7F2okqIO7YXxPWHzkmBMEpW8VKQamdD/FRh1DUzoGZz8b9wp7FQVojSHbpoA64vdzokvO4qZtQBaAe+UZVszG2JmMTOL5eZqIoFIyj8Ak/pCzly4YQSceWXYiaQy+nL+2Rp1Yez1sGVp2IkqRKKvuukNTHX3wrJs5O7Pu3uWu2c1aNAgwZEkdIX5MPUWWDMLegyHs68NO5FUZnWaBGWfVi246itvddiJyl1pin4D0KzY7abxZSXpzeGHZcqyrURRUSG8cicsfyMYe6Rjn7ATicAppwdlX1QQlP3OnLATlavSHKOfB7Qxs1YEJd0bOOq31czaAZlA8ffrvwX8t5llxm9fCTx4QomldOa9EMzMVJQfbo6iQji4Cy5/GLJvDzeLSHEN2wUnZcd0hz93To4B9E7rCANeTfjDHrfo3b3AzO4mKO00YKS7LzazR4CYu0+Pr9obmOTFLuNx921m9ijBHwuAR748MSvlaMHYYCLtFl3h1LPDThP88HbqG3YKkaM17giDZsDCCeBFYaeBus3L5WFTYqwbKYNPpwWTL5z+/eDa4arVj7+NiKS8b7q8UkMgRMnyN2Ha7dDsAug1TiUvIoCKPjrWvAtTBkCjc6HPZKhWM+xEIpIkVPRRsH4uTOwD9VpDv5ch4+SwE4lIElHRp7ovPoZxN0LtRsGEHTVPCTuRiCQZFX0q27IMxl4X7MEPeA1qn3r8bUSk0lHRp6ptnwUTdlSpGpR83WbH30ZEKqWUGY9eitm5AV7sDgUHYNAbwbF5EZFjUNGnmj25wVu29+8IRt879aywE4lIklPRp5L924Nj8jtzguFWK8kQqyJyYlT0qeLg7uDqmq3Lg+vkIzZpgoiUHxV9KsjfDxNvho0fBfNftv5B2IlEJIWo6JNdwSGY3B/W/iuYsKPdD8NOJCIpRkWfzAoLYNpgWPU2/Pgp6HBj2IlEJAXpOvpkVVQE0++GJa/BVb+DzoPCTiQiKUpFn4zc4e8PwMcT4fu/hAt/GnYiEUlhKvpk4w4zH4J5I+C7P4PvPRB2IhFJcSr6ZPPeE/D+U5B1G1zxCJiFnUhEUpyKPpnMfhZmPQbn9g4m0lbJi0gCqOiTxfwx8NaD0L479BgOVfRfIyKJoTZJBoumwox74YzL4YYXIE1XvYpI4qjow7bsDZg2BFp0hZ5joWq1sBOJSMSo6MO0eha8NBAad4Q+kzTPq4iUCxV9WNbNhkl9oP6Z0HcqVK8ddiIRiSgVfRg2fgQTesLJjYPhhjXPq4iUIxV9RduyFMZeDxl1gykAazUMO5GIRJyKviLlrQ5mh0qrBgNfgzpNw04kIpVAqYrezLqZ2XIzW2VmQ4+xTk8zW2Jmi81sQrHlhWa2MP4xPVHBU87OnKDkC/ODPflTTg87kYhUEse9YNvM0oDhwBVADjDPzKa7+5Ji67QBHgS6uvt2Myt+PGK/u3dMcO7UsnszjOkOB3bCwBnQsF3YiUSkEinNHn02sMrd17j7IWAS0OOIdW4Hhrv7dgB335LYmCls37ZgntfdXwRX1zSu3H/zRKTilabomwDri93OiS8r7kzgTDN738w+NLNuxe7LMLNYfPm1JX0BMxsSXyeWm5tbpm8gqR3YBeNugLyV0HsCNO8SdiIRqYQS9V77qkAb4FKgKfCemXVw9x1AC3ffYGanA++Y2SJ3X118Y3d/HngeICsryxOUKVyH9sHE3rDpE+g1Dlp/P+xEIlJJlWaPfgPQrNjtpvFlxeUA0909390/A1YQFD/uviH+7xrgXaDTCWZOfgUHYUp/WPcBXPcctL067EQiUomVZo9+HtDGzFoRFHxvoM8R67wK3AyMMrP6BIdy1phZJrDP3Q/Gl3cFHk9Y+uLyD8Cy18vlocvs05dh1Uzo/mfN8yoioTtu0bt7gZndDbwFpAEj3X2xmT0CxNx9evy+K81sCVAIPODueWb2XeA5MysiePUwrPjVOgl1aA+8fFu5PPS30m0YnD8g7BQiIph7ch0Sz8rK8lgsVvYNCwtg25rEB/o2qtcKhjcQEakgZjbf3bNKui86A5+nVYUGZ4adQkQk6WgIBBGRiFPRi4hEnIpeRCTiVPQiIhGnohcRiTgVvYhIxKnoRUQiTkUvIhJxKnoRkYhT0YuIRJyKXkQk4lT0IiIRp6IXEYk4Fb2ISMSp6EVEIk5FLyIScdGZeEQkyRUVOSPf/4zPt+0LO4okqaaZNRjyvdYJf1wVvUgFcHd+/dqnjJ/zOXVrpmNhB5KkdE6TOip6kVTk7vzu78sYP+dz7rykNb/o1hYzVb1UHB2jFylnT//vKp5/bw0DLmyhkpdQqOhFytGI/1vDkzNXcMP5TXn4x2er5CUUKnqRcjJhzuc89relXNOhEf/vhg5UqaKSl3Co6EXKwasfbeCXry7i+20b8Kdenaiapl81CY9++kQS7K3Fm7j/pY/p0uoU/qdfZ6pV1a+ZhEs/gSIJ9N6KXO6Z8BEdmtRhxMDvkJGeFnYkERW9SKLM/WwbQ8bGaN2wFmNuyaZWdV29LMmhVEVvZt3MbLmZrTKzocdYp6eZLTGzxWY2odjygWa2Mv4xMFHBRZLJJzk7uHX0PBrXrcHY27KpUzM97EgiXznuLoeZpQHDgSuAHGCemU139yXF1mkDPAh0dfftZtYwvvwU4CEgC3Bgfnzb7Yn/VkTCsXzTbgaMnEvdmumMH9yF+rWqhx1J5DCl2aPPBla5+xp3PwRMAnocsc7twPAvC9zdt8SXXwW87e7b4ve9DXRLTHSR8H22dS/9XphD9apVmDD4Ak6rUyPsSCJHKU3RNwHWF7udE19W3JnAmWb2vpl9aGbdyrAtZjbEzGJmFsvNzS19epEQbdixn75//ZDCImf84C40r1cz7EgiJUrUydiqQBvgUuBm4K9mVre0G7v78+6e5e5ZDRo0SFAkkfKzZfcB+v71Q3YfLODFW7M5o2HtsCOJHFNpin4D0KzY7abxZcXlANPdPd/dPwNWEBR/abYVSSnb9x6i/4i5bNl9kNG3ZHNOkzphRxL5RqUp+nlAGzNrZWbVgN7A9CPWeZVgbx4zq09wKGcN8BZwpZllmlkmcGV8mUhK2n0gn4Gj5vJZ3l5GDMiic4vMsCOJHNdxr7px9wIzu5ugoNOAke6+2MweAWLuPp2vC30JUAg84O55AGb2KMEfC4BH3H1beXwjIuVt/6FCbhsdY8nGXTzXvzPfPaN+2JFESsXcPewMh8nKyvJYLBZ2DJHDHCwoZPCYGO+v2srTN3fiR+c2DjuSyGHMbL67Z5V0n966J3Ic+YVF3DPhI/5v5VYev/FclbykHA2BIPINioqcB176mH8s2czDPz6LnlnNjr+RSJJR0Yscg7vzq9c+5dWFG3ngqrYM6toq7Egi34qKXqQE7s5v/7aUCXM+56eXtuau758RdiSRb01FL1KCP81cyYh/fcag77bkgavahh1H5ISo6EWO8Nf31vDU/67kps5N+a8fnaV5XiXlqehFihn34Tp++8ZSfnjuaQy74VzN8yqRoKIXiXvloxx+/dqnXNauIU/27EiaSl4iQkUvArz56Sb+/aVPuPD0egzve77meZVI0U+zVHr/XJHLPRMXcF7TOvx1QJbmeZXIUdFLpTZnTR53jI3RpmFtRt2SzUma51UiSEUvldbH63dw25gYTTNrBvO81tA8rxJNKnqplJZt2sWAkXPJPCmdcbd1oZ7meZUIU9FLpbMmdw/9RsylRnoaEwZfQKM6GWFHEilXKnqpVHK276PfiDm4O+MGd6HZKZrnVaJPRS+VxpZdB+g7Yg57DhYw9rYunNGwVtiRRCqEil4qhW17D9HvhTls3X2Q0bdmc1bjk8OOJFJhdC2ZRN6uA/kMHDmXdXn7GH1LNuc31zyvUrloj14ibd+hAm4dNY9lm3bxl36dubB1vbAjiVQ4Fb1E1oH8Qu4YO58Fn2/nqd6d+H67hmFHEgmFDt1IJOUXFnHPxGCe1yduOo9rOpwWdiSR0GiPXiKnsMi5f8rHvL1kM4/2OJsbOzcNO5JIqFT0Einuzi9fWcT0jzfyi27t6H9hy7AjiYRORS+R4e48+vpSJs1bzz0/OIOfXNo67EgiSUFFL5Hx5NsrGPn+Z9zStSU/v+LMsOOIJA0VvUTCc/9czdPvrKJXVjPN8ypyhFIVvZl1M7PlZrbKzIaWcP8gM8s1s4Xxj8HF7isstnx6IsOLAIydvZbf/X0ZPz6vMf99fQeVvMgRjnt5pZmlAcOBK4AcYJ6ZTXf3JUesOtnd7y7hIfa7e8cTjypytJfn5/Dr1xZzeftT+WPP8zTPq0gJSrNHnw2scvc17n4ImAT0KN9YIsf390Vf8MDUj+l6Rj2e6dOJ9DQdiRQpSWl+M5oA64vdzokvO9INZvaJmU01s2bFlmeYWczMPjSza08krMiXZi3fws8mfUSn5pma51XkOBK1CzQDaOnu5wJvA2OK3dfC3bOAPsCfzOyoa97MbEj8j0EsNzc3QZEkqmavzuPOsfNp26g2Iwd9h5rV9AZvkW9SmqLfABTfQ28aX/YVd89z94PxmyOAzsXu2xD/dw3wLtDpyC/g7s+7e5a7ZzVo0KBM34BULh99vp3BY+bR/JSavHhrF83zKlIKpSn6eUAbM2tlZtWA3sBhV8+YWfGBRLoDS+PLM82sevzz+kBX4MiTuCKlsmTjLgaOnEv92tUZN7gLp5xULexIIinhuK953b3AzO4G3gLSgJHuvtjMHgFi7j4d+JmZdQcKgG3AoPjm7YHnzKyI4I/KsBKu1hE5rtW5e+j/whxOql6Vcbd14dSTNc+rSGmZu4ed4TBZWVkei8XCjiFJZP22ffR8bjb5hUVMueNCTm+gKQBFjmRm8+PnQ4+is1iS1DbH53ndd6iQSUMuUMmLfAu68FiSVt6eg/QdMYe8PQcZc2s27U/TPK8i34b26CUp7dyfz4CRc1m/bR9jbs2mY7O6YUcSSVnao5eks/dgAbeOnseKzbt5rn9nLjhd87yKnAgVvSSVA/mF3P5ijI8+387TvTtxaVvN8ypyonToRpJGfmERd09YwAer8/hjz/O4WvO8iiSE9uglKRQWOfdNXsjMpVt49NpzuP58zfMqkigqegldUZHz4LRPeP2TL3jw6nb0v6BF2JFEIkVFL6Fydx55fQlTYjn87LI23HGJ5nkVSTQVvYTqD/9YwegP1nLbRa247/I2YccRiSQVvYTm2XdX8cysVdyc3Yxf/bC9pgAUKScqegnFmA/W8viby+nRsTGPXat5XkXKk4peKtxLsfU8NH0xV5x1Kk/cpHleRcqbil4q1N8++YJfvPwJF7epr3leRSqIfsukwryzbDP3TvqIzi0yea5/Z6pX1TyvIhVBRS8V4oNVW7lz3ALan3YyL2ieV5EKpaKXcjd/3XYGvxijZb2avHhrNidnaJ5XkYqkopdytXjjTm4ZNZeGtasz7rYuZGqeV5EKp6KXcrNqy24GvDCXWtWrMm5wFxpqnleRUKjopVx8nrePviPmYGaMv/0CmmbWDDuSSKWlopeE27TzAH1f+JCDBUWMG5xNq/onhR1JpFJT0UtCbd1zkL4jPmT73nzG3JJNu0aa51UkbCp6SZid+/IZ8MJcNuzYz8hB3+E8zfMqkhRU9JIQew4WMGj0XFZt2cNz/bPIbnVK2JFEJE7vWpETdiC/kNvHxPgkZyfD+5zPJWc2CDuSiBSjPXo5IYcKivjp+AV8+Fkef7jpPLqd0yjsSCJyBBW9fGsFhUXcN3kh7yzbwm+v7cC1nZqEHUlESlCqojezbma23MxWmdnQEu4fZGa5ZrYw/jG42H0DzWxl/GNgIsNLeIqKnKHTFvG3RV/wqx+2p0+X5mFHEpFjOO4xejNLA4YDVwA5wDwzm+7uS45YdbK7333EtqcADwFZgAPz49tuT0h6CYW785sZi5k6P4d/u7wNgy8+PexIIvINSrNHnw2scvc17n4ImAT0KOXjXwW87e7b4uX+NtDt20WVZPH7t5YzZvY6br+4FfdepnleRZJdaYq+CbC+2O2c+LIj3WBmn5jZVDNrVpZtzWyImcXMLJabm1vK6BKG4bNW8ey7q+nbpTn/eY3meRVJBYk6GTsDaOnu5xLstY8py8bu/ry7Z7l7VoMGujQvWY16/zN+/9ZyruvUhEd7nKOSF0kRpSn6DUCzYrebxpd9xd3z3P1g/OYIoHNpt5XUMGXeen4zYwlXnX0qv7/xXKponleRlFGaop8HtDGzVmZWDegNTC++gpmdVuxmd2Bp/PO3gCvNLNPMMoEr48skhcz4eCO/mPYJ3zuzAU/f3ImqmudVJKUc96obdy8ws7sJCjoNGOnui83sESDm7tOBn5lZd6AA2AYMim+7zcweJfhjAfCIu28rh+9DysnMJZu5b/JCvtPiFJ7rp3leRVKRuXvYGQ6TlZXlsVgs7BgCvL9qK7eMnkf7RrUZN7gLtTUFoEjSMrP57p5V0n16DS4lmr9uG4PHxGhV7yRG35KtkhdJYSp6OcqnG3YyaNQ8GtXJYOzgbM3zKpLiVPRymJWbdzNg5FxOzkgP5nmtrXleRVKdil6+si5vL31HzCGtijF+cBea1K0RdiQRSQAVvQDwxc799PnrHPILixh3Wxdaap5XkchQ0Ut8ntc57Nqfz4u3dqFto9phRxKRBNIMU5Xcjn2H6DdiDht37GfsbV3o0LRO2JFEJMG0R1+J7TlYwMBR81iTu5e/DsjiOy01z6tIFGmPvpI6kF/I4DHz+HTDTv6n7/lc3EaDyYlElfboK6FDBUXcOW4+cz7bxlfzbzMAAAQZSURBVB97nseVZ2ueV5EoU9FXMgWFRdw76SPeXZ7L767rQI+OmudVJOpU9JVIUZHzHy9/wt8/3cSvf3QWvbM1z6tIZaCiryTcnYemL2bagg38/Iozue2iVmFHEpEKoqKvBNydYW8uY+yH67jjktO55wdnhB1JRCqQir4SeOadVTz3zzX0v6AFQ7u10xSAIpWMij7iXvjXZ/zh7RVcf34TftP9bJW8SCWkoo+wSXM/59HXl3D1OY14/AbN8ypSWUXmDVM79h3ipr/MDjtG0nBgde4eLm3bgKd6a55XkcosMkVfpYrR5tRaYcdIKt9r04D/6NaWalVV8iKVWWSK/uSMdJ7t2znsGCIiSUe7eiIiEaeiFxGJOBW9iEjEqehFRCJORS8iEnEqehGRiFPRi4hEnIpeRCTizN3DznAYM8sF1p3AQ9QHtiYoTqrTc3E4PR+H0/PxtSg8Fy3cvcTJn5Ou6E+UmcXcPSvsHMlAz8Xh9HwcTs/H16L+XOjQjYhIxKnoRUQiLopF/3zYAZKInovD6fk4nJ6Pr0X6uYjcMXoRETlcFPfoRUSkGBW9iEjERabozaybmS03s1VmNjTsPGEys2ZmNsvMlpjZYjO7N+xMYTOzNDP7yMxeDztL2MysrplNNbNlZrbUzC4MO1OYzOy++O/Jp2Y20cwyws6UaJEoejNLA4YDVwNnATeb2VnhpgpVAXC/u58FXADcVcmfD4B7gaVhh0gSTwFvuns74Dwq8fNiZk2AnwFZ7n4OkAb0DjdV4kWi6IFsYJW7r3H3Q8AkoEfImULj7l+4+4L457sJfpGbhJsqPGbWFPghMCLsLGEzszrA94AXANz9kLvvCDdV6KoCNcysKlAT2BhynoSLStE3AdYXu51DJS624sysJdAJmBNuklD9CfgPoCjsIEmgFZALjIofyhphZieFHSos7r4BeAL4HPgC2Onu/wg3VeJFpeilBGZWC3gZ+Dd33xV2njCY2Y+ALe4+P+wsSaIqcD7wP+7eCdgLVNpzWmaWSfDqvxXQGDjJzPqFmyrxolL0G4BmxW43jS+rtMwsnaDkx7v7tLDzhKgr0N3M1hIc0vuBmY0LN1KocoAcd//yFd5UguKvrC4HPnP3XHfPB6YB3w05U8JFpejnAW3MrJWZVSM4mTI95EyhMTMjOAa71N3/GHaeMLn7g+7e1N1bEvxcvOPukdtjKy133wSsN7O28UWXAUtCjBS2z4ELzKxm/PfmMiJ4crpq2AESwd0LzOxu4C2Cs+Yj3X1xyLHC1BXoDywys4XxZf/p7m+EmEmSxz3A+PhO0RrglpDzhMbd55jZVGABwdVqHxHB4RA0BIKISMRF5dCNiIgcg4peRCTiVPQiIhGnohcRiTgVvYhIxKnoRUQiTkUvIhJx/x9L6EjdOmvP8QAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plot_hist(thr_score_hist)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Train test resnet50"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from utils import load_data\n",
    "from image_train.data import create_dl, ImageDS\n",
    "from image_train.model import EMBRes\n",
    "from arcface import ArcMarginProduct, compute_centers\n",
    "from clip_.train import *\n",
    "import matplotlib.pyplot as plt\n",
    "import clip\n",
    "np.random.seed(1337)\n",
    "device = torch.device('cuda')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "df, train_df, val_df, train_labels, val_labels = load_data(train_perc=0.3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "# creating dataloaders\n",
    "small_images_dir_train = 'data/small_train_images_250/'\n",
    "small_images_dir_val = 'data/small_train_images_224/'\n",
    "bs = 32\n",
    "\n",
    "tr_dl = create_dl(train_df, small_images_dir_train, batch_size=bs)\n",
    "tr_test_dl = create_dl(train_df, small_images_dir_val, shuffle=False, batch_size=bs)\n",
    "val_dl = create_dl(val_df, small_images_dir_val, shuffle=False, batch_size=bs)\n",
    "#full_dl = create_dl(df, small_images_dir, shuffle=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "class CLIPImg(torch.nn.Module) :\n",
    "    def __init__(self, model_name=\"RN50\", device='cuda') :\n",
    "        super().__init__()\n",
    "        self.model = clip.load(model_name, device=device, jit=False)[0]\n",
    "    def forward(self, imgs) :\n",
    "        return self.model.encode_image(imgs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = CLIPImg()\n",
    "train_tfms, val_tfms = get_tfms(crop=224)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "scrolled": true
   },
   "source": [
    "centers = compute_centers(tr_test_dl, model, val_tfms, train_df)\n",
    "torch.save(centers, 'data/clip/centers_im_0.3_res50.pth')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "centers = torch.load('data/clip/centers_im_0.3_res50.pth')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Using center as wieghts\n"
     ]
    }
   ],
   "source": [
    "metric_fc = ArcMarginProduct(512, train_df['label_group'].nunique(), \n",
    "                             s=30, m=0.5, easy_margin=False, centers=centers, half=True).to(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "n_epochs, lf, params, optimizer, sched = get_hparams(tr_dl, model, metric_fc, lr=1e-5, n_epochs=10)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "for n, _ in model.named_parameters() :\n",
    "    print(n)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "mnp = list(model.named_parameters())\n",
    "\n",
    "param_groups = [{'params' : [p for n,p in mnp if 'embeddings' in n]}]\n",
    "\n",
    "params_names = []\n",
    "n_blocks = 5\n",
    "for i in range(n_blocks):\n",
    "    ith_block = [p for n, p in mnp if f'layer{i}.' in n]\n",
    "    ith_block_name = [n for n, p in mnp if f'layer{i}.' in n]\n",
    "    params_names += ith_block_name\n",
    "    param_groups.append({'params' : ith_block})\n",
    "    \n",
    "param_groups.append({'params' : [p for n,p in mnp if 'attnpool'in n]})\n",
    "params_names += [n for n,p in mnp if 'attnpool' in n]\n",
    "\n",
    "param_groups.append({'params' : metric_fc.parameters()})\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "for n, p in model.named_parameters() :\n",
    "    if n not in params_names :\n",
    "        print(n)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "lf = nn.CrossEntropyLoss()\n",
    "optimizer = torch.optim.AdamW(param_groups, weight_decay = 1e-5)\n",
    "\n",
    "lr = list(np.linspace(1e-5,5e-4,len(param_groups)))\n",
    "\n",
    "n_epochs = 10\n",
    "sched = torch.optim.lr_scheduler.OneCycleLR(optimizer, max_lr=lr, pct_start=0.3,\n",
    "                                            total_steps=int(n_epochs * len(tr_dl)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "loss_hist = []\n",
    "best_thr_score={'val': {'thr': None, 'f1': None}, 'train': {'thr': None, 'f1': None}}\n",
    "thr_score_hist=[]\n",
    "ep_start = 0\n",
    "save_path = 'data/clip/test_20ap_im_res50'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=10.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=327.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Thresholds', max=10.0, style=ProgressStyle(description_wi…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Checkpoint : saved model to data/clip/test_20ap_im_res50_ep_0.pth\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=745.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Thresholds', max=10.0, style=ProgressStyle(description_wi…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Saved best model ep 0 with f score : 0.5923946402416435\n",
      "Ep 0: Train loss 14.7105 | Val f score 0.5924 with thresh 0.29, train f score 0.4926 with thresh 0.56\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=327.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Thresholds', max=5.0, style=ProgressStyle(description_wid…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=745.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Thresholds', max=5.0, style=ProgressStyle(description_wid…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Saved best model ep 1 with f score : 0.6219290318389071\n",
      "Ep 1: Train loss 13.0619 | Val f score 0.6219 with thresh 0.34, train f score 0.4755 with thresh 0.56\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=327.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Thresholds', max=5.0, style=ProgressStyle(description_wid…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Checkpoint : saved model to data/clip/test_20ap_im_res50_ep_2.pth\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=745.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Thresholds', max=5.0, style=ProgressStyle(description_wid…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Saved best model ep 2 with f score : 0.6336896602294358\n",
      "Ep 2: Train loss 11.6626 | Val f score 0.6337 with thresh 0.44, train f score 0.5413 with thresh 0.66\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=327.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Thresholds', max=5.0, style=ProgressStyle(description_wid…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=745.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Thresholds', max=5.0, style=ProgressStyle(description_wid…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Saved best model ep 3 with f score : 0.6588727139419738\n",
      "Ep 3: Train loss 9.4484 | Val f score 0.6589 with thresh 0.49, train f score 0.6547 with thresh 0.76\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=327.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Thresholds', max=5.0, style=ProgressStyle(description_wid…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Checkpoint : saved model to data/clip/test_20ap_im_res50_ep_4.pth\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=745.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Thresholds', max=5.0, style=ProgressStyle(description_wid…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Saved best model ep 4 with f score : 0.6766706201743063\n",
      "Ep 4: Train loss 7.0888 | Val f score 0.6767 with thresh 0.59, train f score 0.7428 with thresh 0.76\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=327.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Thresholds', max=5.0, style=ProgressStyle(description_wid…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=745.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Thresholds', max=5.0, style=ProgressStyle(description_wid…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Saved best model ep 5 with f score : 0.6884154413996265\n",
      "Ep 5: Train loss 5.1298 | Val f score 0.6884 with thresh 0.59, train f score 0.8078 with thresh 0.81\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=327.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Thresholds', max=5.0, style=ProgressStyle(description_wid…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Checkpoint : saved model to data/clip/test_20ap_im_res50_ep_6.pth\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=745.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Thresholds', max=5.0, style=ProgressStyle(description_wid…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Saved best model ep 6 with f score : 0.6937082190002374\n",
      "Ep 6: Train loss 3.3783 | Val f score 0.6937 with thresh 0.64, train f score 0.8636 with thresh 0.81\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=327.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Thresholds', max=5.0, style=ProgressStyle(description_wid…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=745.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Thresholds', max=5.0, style=ProgressStyle(description_wid…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Ep 7: Train loss 2.1293 | Val f score 0.6777 with thresh 0.64, train f score 0.9101 with thresh 0.86\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=327.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Thresholds', max=5.0, style=ProgressStyle(description_wid…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Checkpoint : saved model to data/clip/test_20ap_im_res50_ep_8.pth\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=745.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Thresholds', max=5.0, style=ProgressStyle(description_wid…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Saved best model ep 8 with f score : 0.7024859165653851\n",
      "Ep 8: Train loss 1.3773 | Val f score 0.7025 with thresh 0.69, train f score 0.9408 with thresh 0.81\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=327.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Thresholds', max=5.0, style=ProgressStyle(description_wid…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=745.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Thresholds', max=5.0, style=ProgressStyle(description_wid…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Ep 9: Train loss 1.0794 | Val f score 0.7024 with thresh 0.69, train f score 0.9532 with thresh 0.86\n",
      "\r"
     ]
    }
   ],
   "source": [
    "best_thr_score, thr_score_hist, losses = train(model, optimizer, lf, sched, metric_fc, tr_dl, val_dl,\n",
    "                                               n_epochs, train_df, val_df, train_tfms, val_tfms, \n",
    "                                               save_path=save_path, \n",
    "                                               prev_best_info=best_thr_score, info_history=thr_score_hist,\n",
    "                                               ep_start=ep_start, half_precision=True)\n",
    "loss_hist.append(losses)\n",
    "ep_start += n_epochs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAD4CAYAAAD8Zh1EAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+j8jraAAAgAElEQVR4nO3dd3xUVd7H8c9JI410EiAhJCAtFImGovQO6gKCSlNUFHTFum7B1UdZ17Wsu/u4uOwiq/hYUESUIiC9WRAJJIIk9JYCIYUkJIGUmfP8cUMIGCCBmdyZye/9esVk7tzM/WWELye/e+85SmuNEEII5+dmdgFCCCFsQwJdCCFchAS6EEK4CAl0IYRwERLoQgjhIjzMOnBYWJiOiYkx6/BCCOGUduzYkaO1blLTc6YFekxMDImJiWYdXgghnJJS6tjlnpOWixBCuAgJdCGEcBES6EII4SJM66HXpLy8nPT0dM6dO2d2KS7N29ubqKgoPD09zS5FCGFDDhXo6enpNG7cmJiYGJRSZpfjkrTW5Obmkp6eTmxsrNnlCCFsyKFaLufOnSM0NFTC3I6UUoSGhspvQUK4IIcKdEDCvB7IeyyEa3KolosQQriksmLIPQg5ByD3ELQdBs272vwwEuhCCGELVgvkHzeCuyq8D0DOQTiTWW1HBX5hEuiOxt/fn6KiIrPLEELUp5K8SwK7ctSddxgspRf28w6E0DbQqh+E3mB8hLWBkFbg6WOX0iTQnURFRQUeHvK/S4h6UVEKeUcuDuzzX5/Nu7CfmyeExBph3WaIEdihbYzPvqFQz+erHDYh/vTVHlIyC236mnHNA3jpVx0v+/yMGTNo0aIF06dPB2DmzJl4eHiwceNGTp8+TXl5Oa+88gqjRo266rFOnDjBuHHjKCwspKKigv/85z/06dOHVatW8cc//hGLxUJYWBjr168nLy+PKVOmcPjwYXx9fZk7dy5dunRh5syZHDp0iMOHDxMdHc2sWbN49NFHOX78OABvvfUWvXr1ss2bI0RDozWcOVEZ2Je0SfKPg7Ze2Nc/wgjquJHG5/Oj7aCW4O44Meo4lTiAcePG8fTTT1cF+sKFC1m9ejVPPvkkAQEB5OTk0LNnT0aOHHnVK0U++eQThg0bxvPPP4/FYqGkpITs7GymTp3Kli1biI2NJS/P+Jf+pZdeIj4+niVLlrBhwwYmT55McnIyACkpKXz77bf4+PgwceJEnnnmGXr37s3x48cZNmwYqamp9n1ThHAFFWVwaD1kJhuBnXvQGHWXVWuZevpCaGtoHg+d76kcbd9gbPMONK/2OnDYQL/SSNpe4uPjOXXqFJmZmWRnZxMcHEzTpk155pln2LJlC25ubmRkZJCVlUXTpk2v+FrdunVjypQplJeXM3r0aLp27cqmTZvo27dv1Q09ISEhAHz77bd88cUXAAwcOJDc3FwKC43fTkaOHImPj9FvW7duHSkpKVXHKCwspKioCH9/f5u/F0K4hBO7IPkT2L0QSnIBBUEtjFF29C0X97YbNwc3h7uSu04cNtDNcvfdd7No0SJOnjzJuHHjmD9/PtnZ2ezYsQNPT09iYmJqdVNO37592bJlCytWrOCBBx7gN7/5DcHBwXWux8/Pr+prq9XKDz/8gLe3d51fR4gGozgHdi00gjxrN7h7QbvboOskiO1jtxOSjsC5/zmyg3HjxrFgwQIWLVrE3XffTUFBAeHh4Xh6erJx40aOHbvsVMQXOXbsGBEREUydOpWHH36YnTt30rNnT7Zs2cKRI0cAqlouffr0Yf78+QBs2rSJsLAwAgICfvGaQ4cO5e233656fL4tI0SDZymHvStgwST4eztY/ZzR277tb/DsPrjnA2g71KXDHGSE/gsdO3bkzJkzREZG0qxZMyZNmsSvfvUrOnfuTEJCAu3bt6/V62zatIk333wTT09P/P39+fDDD2nSpAlz585lzJgxWK1WwsPDWbt2LTNnzmTKlCl06dIFX19fPvjggxpfc9asWUyfPp0uXbpQUVFB3759mTNnji1/fCGcy8ndxkh810IoyQG/cOj5a7hxIkTEmV1dvVNaa1MOnJCQoC9dsSg1NZUOHTqYUk9DI++1cFrFuUZPPHm+EehuntC+sqXSepBDXXViD0qpHVrrhJqec+2fXAjhGizlcGCtEeL7V4O1HJp1hRFvQue7wDfE7AodggT6ddq9ezf33XffRdsaNWrEtm3bTKpICBeStQeS5sOuzypbKk2gxyPQdSJE1P+VcI5OAv06de7cWU5OCmFLxbnw8yJjNH7iJ6Ol0m44dL0XbhgE7rIwy+VIoAshzGcph4PrjBDft6qypXIjjPgrdLoL/ELNrtApSKALIcyTlWKE+K6FUHzqQkvlxgnQtJPZ1TkdCXQhRP0qyYPd51sqyeDmAW2HG1eptBkiLZXrIIEuhLA/S4Uxl0ryfNj3NVjKoGkXGP6GcZWKX5jZFboEuVO0mvz8fP7973/X+ftuu+028vPz7VCREE6urBg2vQ7/6ACf3ANHv4VuD8Oj38Kj30DPRyXMbUhG6NWcD/THHnvsou1Xm4t85cqV9i6tViwWC+7u7maXIQRYrbBrAax/2Ziitu1wuGky3DAEPLzMrs5l1SrQlVLDgX8C7sC7WuvXL3m+JTAPaALkAfdqrdOvq7KvZxh3gdlS084w4vXLPj1jxgwOHTpE165d8fT0xNvbm+DgYPbu3cv+/fsZPXo0aWlpnDt3jqeeeopp06YBEBMTQ2JiIkVFRYwYMYLevXvz/fffExkZydKlS6tmS7zUrFmzmDNnDh4eHsTFxbFgwQKKiop44oknSExMRCnFSy+9xNixY/n000959dVX0Vpz++2388YbbwDGqkmPPPII69atY/bs2Rw9epRZs2ZRVlZGjx49+Pe//y0hL+rXsa3GXCqZSRB5M9z9AUT3MLuqBuGqLRellDswGxgBxAETlFKXTpLwN+BDrXUX4GXgNVsXWh9ef/11WrduTXJyMm+++SY7d+7kn//8J/v37wdg3rx57Nixg8TERGbNmkVubu4vXuPAgQNMnz6dPXv2EBQUVDUt7uWOl5SUxK5du6rmZPnzn/9MYGAgu3fvZteuXQwcOJDMzEz+8Ic/sGHDBpKTk9m+fTtLliwBoLi4mB49evDTTz8RGhrKZ599xnfffUdycjLu7u5Vk34JYXenj8LC++H94XAmC+6cCw+tkzCvR7UZoXcHDmqtDwMopRYAo4CUavvEAb+p/HojsOS6K7vCSLq+dO/evWrucjBG1IsXLwYgLS2NAwcOEBp68fWxsbGxdO1qLP568803c/To0cu+fpcuXZg0aRKjR49m9OjRgDHn+YIFC6r2CQ4OZsuWLfTv358mTZoAMGnSJLZs2cLo0aNxd3dn7NixAKxfv54dO3bQrVs3AM6ePUt4ePh1vgtCXEXpGfjmH7B1Nri5Q//n4NYnwMvv6t8rbKo2gR4JpFV7nA5c+k/uT8AYjLbMnUBjpVSo1vqiIaxSahowDSA6Ovpaa6431eci37RpE+vWrWPr1q34+vrSv3//GudFb9SoUdXX7u7unD179rKvv2LFCrZs2cJXX33FX/7yF3bvrnuLydvbu6qlorXm/vvv57XXnPIXJOFsrBZI+hg2vGJcQ95lPAx6EQIjza6swbLVVS6/BfoppZKAfkAGYLl0J631XK11gtY64fxo05E0btyYM2fO1PhcQUEBwcHB+Pr6snfvXn744YfrOpbVaiUtLY0BAwbwxhtvUFBQQFFREUOGDGH27NlV+50+fZru3buzefNmcnJysFgsfPrpp/Tr1+8Xrzlo0CAWLVrEqVOnAGO+9drO3y5EnRzZAu/0g6+eNBZJfngDjHlHwtxktRmhZwAtqj2OqtxWRWudiTFCRynlD4zVWjvddXyhoaH06tWLTp064ePjQ0RERNVzw4cPZ86cOXTo0IF27drRs2fP6zqWxWLh3nvvpaCgAK01Tz75JEFBQbzwwgtMnz6dTp064e7uzksvvcSYMWN4/fXXGTBgQNVJ0ZoWqo6Li+OVV15h6NChWK1WPD09mT17Ni1btryuWoWoknsI1r4Ie5dDYDTcNQ86jqn31e1Fza46H7pSygPYDwzCCPLtwESt9Z5q+4QBeVprq1LqL4BFa/3ilV5X5kM3l7zXok7OFcCWN+GHOcaSbn1+A7dMd/kVgBzRdc2HrrWuUEo9DqzGuGxxntZ6j1LqZSBRa70M6A+8ppTSwBZgus2qF0KYx1IBO/8PNr5q3LIfPwkG/g80vvIi6cIctboOXWu9Elh5ybYXq329CFhk29Jcx/Tp0/nuu+8u2vbUU0/x4IMPmlSRELVwcD2sfh6yU6Flbxj+qjEDonBYDnenqNYa5WL9uOonOR2BWcsOCieRvR/WvAAHVkNwDNzzEXT4lfTJnYBDBbq3tze5ubmEhoa6XKg7Cq01ubm5eHt7m12KcDQlebD5Ddj+Lnj4wJCXocej4NHo6t8rHIJDBXpUVBTp6elkZ2ebXYpL8/b2JioqyuwyhKOwlMP292DTa1BaCDfdDwOeB3/Hu7RYXJlDBbqnp+dFd2YKIexIaziwxuiT5x6AVv1h2KuyVqcTc6hAF0LUk1OpsPqPcGgDhN4AEz6DtsOkT+7kJNCFaEiKc4xLEHe8D40aw7DXjPnJZUpblyCBLkRDUFEGP74Dm9+EsiIjxPs/B74hZlcmbEgCXQhXpjXsXQFr/wfyDhsLTAx9BcLbm12ZsAMJdCFc1alUWPk7OPoNhLWDSV9Am8FmVyXsSAJdCFdjtcDWfxnT2nr5wW1/g5sfBHf56+7q5P+wEK4k7zAseQyOb4X2d8Adb8n15A2IBLoQrkBrSHwP1vwPuHnCne9Al3FyGWIDI4EuhLMryIBljxvXlLcaAKNmy0ITDZQEuhDOSmvYtdA48Wkth9v/DgkPyai8AZNAF8IZFefA8qch9Sto0QNG/wdCW5tdlTCZBLoQzmbvCvjqKWMVocF/glufADd3s6sSDkACXQhncTYfVs2Anz6Fpp1h8lKZSEtcRAJdCGdwaCMsnQ5nTkLf30Pf38n8K+IXJNCFcGRlxbD2Jdj+XwhrCw+thaibza5KOCgJdCEc1fFtsORR42ahno/BoBfB08fsqoQDk0AXwtFUlBpT3H4/CwKj4P7lENvH7KqEE5BAF8KRnNgFix+FU3vgpsnGCkKNGptdlXASEuhCOAJLBXz3v7DpDWOO8okLjRWEhKgDCXQhzJZzABY/Ahk7oOMY445PWXhCXAMJdCHMYrUaqwitm2mc7LxrHnQaa3ZVwolJoAthhvzjxjS3R7+BNsNg5Cxo3NTsqoSTk0AXoj5pDUkfwao/AhpGvg3x98mEWsImJNCFqC9nThpzsOxfBTF9jGlug1uaXZVwIRLoQtSHn7+EFb+B8rMw/HXo/gi4uZldlXAxEuhC2FNJHqz8Lfz8BUTeDKPnQJO2ZlclXJQEuhD2sn8NLHsCSnJg4AvQ6xlZqFnYlfzpEsLWSotg9R9h5wcQHgeTPodmXcyuSjQAtWriKaWGK6X2KaUOKqVm1PB8tFJqo1IqSSm1Syl1m+1LFcIJ5B6CdwcbV7L0ehqmbZIwF/XmqiN0pZQ7MBsYAqQD25VSy7TWKdV2ewFYqLX+j1IqDlgJxNihXiEc1/418MXDxsnOe7+E1gPMrkg0MLUZoXcHDmqtD2uty4AFwKhL9tFAQOXXgUCm7UoUwsFZrbD5TfjkHgiOhmmbJcyFKWrTQ48E0qo9Tgd6XLLPTGCNUuoJwA8YbJPqhHB05wphya9h73LoMg7ueAu8fM2uSjRQtroQdgLwf1rrKOA24COl1C9eWyk1TSmVqJRKzM7OttGhhTBJ9n54dxDs+xqGvwF3viNhLkxVm0DPAFpUexxVua26h4CFAFrrrYA3EHbpC2mt52qtE7TWCU2aNLm2ioVwBHtXwH8HGteZT14KPR+V2/eF6WoT6NuBNkqpWKWUFzAeWHbJPseBQQBKqQ4YgS5DcOF6rFbY8BdYMBHCboBHNstqQsJhXLWHrrWuUEo9DqwG3IF5Wus9SqmXgUSt9TLgWeC/SqlnME6QPqC11vYsXIh6dzYfvpwKB9ZA13uNecs9vc2uSogqtbqxSGu9EuNSxOrbXqz2dQrQy7alCeFAslLgs0mQn2YEecJD0mIRDkfuFBXiavYshiXToZE/PLAconuaXZEQNZJAF+JyrBZY/zJ89xZEdYd7PoSAZmZXJcRlSaALUZOSPPjiITi0ARKmGJcleniZXZUQVySBLsSlTuwy+uVnThorCt002eyKhKgVCXQhqtv1uTHlrU8wPPg1RCWYXZEQtSaBLgSApQLWvgg/zIboW+GeD8A/3OyqhKgTCXQhirJh0YNw9Bvo8SgMfQXcPc2uSog6k0AXDVvGTvjsPmNVoTvfgRvHm12RENdMAl00XEnzYfkzRmtlympo3tXsioS4LhLoouGpKDOWiNv+X4jtB3e9D36hZlclxHWTQBcNy5ks+Px+OL4Vbn0CBs2UhZuFy5A/yaLhSNsOC+8zJtka+x50vsvsioSwKQl00TAkvg8rfweBkfDwOmjayeyKhLA5CXTh2ipKjSDf+QG0HgRj3wXfELOrEsIuJNCF6yrMNC5JzEiEPs/CgOfBzd3sqoSwGwl04ZqOfQ8L74fyErjnI4gbaXZFQtidBLpwLVrD9ndh1QwIagn3fwXh7c2uSoh6IYEuXEdFGSx/GpLnQ9vhMGYueAeaXZUQ9UYCXbgGqwUWPwJ7voR+f4B+M8CtNmugC+E6JNCF89MaVv7WCPPBf4LeT5tdkRCmkCGMcH4bXoHEedDrKQlz0aBJoAvn9v2/4Ju/GasKDf6T2dUIYSoJdOG8kj6GNc9D3Ci44y1QyuyKhDCVBLpwTqnLjaXiWg2AMf+VG4aEQAJdOKPDm40VhprfBOM+Bo9GZlckhEOQQBfOJWMHLJgIIa1h0ufQyN/sioRwGBLownlk74OP7zIm17pvsUyyJcQlJNCFc8g/Dh/dCW4ecN8SCGhmdkVCOBy5sUg4vqJs+HA0lBbBgyshtLXZFQnhkCTQhWM7VwAfjzGmwp28RBamEOIKJNCF4yo/C59OgFMpMGEBRPc0uyIhHJoEunBMlnL4/AFjXvOx70KbIWZXJITDk0AXjsdqhaXTYf8quP3vspizELVUq6tclFLDlVL7lFIHlVIzanj+f5VSyZUf+5VS+bYvVTQIWhuLU+z6DAa+AN0eNrsiIZzGVUfoSil3YDYwBEgHtiullmmtU87vo7V+ptr+TwDxdqhVNASb34Af34Ge06HPb82uRriArMJzrE89xcZ9pzhdXPaL52uaAkhR48babKrV6z3UO5bBcRGXqfja1abl0h04qLU+DKCUWgCMAlIus/8E4CXblCcalG3vwKbX4MaJMPQVmWxLXBOtNSknClmfeop1qVnsSi8AICrYh5ahvpfsW9P317CNX26seb8atuuq/1Sx1PTNNlCbQI8E0qo9Tgd61LSjUqolEAtsuMzz04BpANHR0XUqVLi4XQvh699Du9th5Nuy2pCok9IKCz8czmNdShbrU7PILDiHUtC1RRC/G9aOwR0iaBvhj3LxQYKtT4qOBxZprS01Pam1ngvMBUhISLDPP1HC+exbBYsfhZg+cNc8cJdz9derrMJKwdly3N0UIX5eZpdjF3nFZWzYe4r1qVls2Z9NcZkFH093ercJ4+nBbRnQPpwmjRvWxG21+ZuTAbSo9jiqcltNxgPTr7co0YAc/Q4+vx+adobxn4Cnt9kVOQStNUWlFRSeq6CgpJyCs+UUnqv8fP7jXAUFZy9sq77PuXJr1WtFBDSiU/NAOkYG0ql5AJ0iA2kW6O10o1WtNYeyi1iXeop1KVnsPH4aqzZ+vlHxkQzuEM6trcPw9my4UynXJtC3A22UUrEYQT4emHjpTkqp9kAwsNWmFQrXdeIn+HQ8BEXDvV+Ad4DZFdlUucVaLWgrLgre8+FrhHPFRdvO72e9wu+wSkGAtycBPh4E+ngS4O3JDeH+xtc+npXbPCitsJKSWcjPmQVs3Heq6jVD/LzoWBnunZoH0ikygOgQX4cL+QqLle1HT7Mu1WilHM0tAaBj8wAeH9iGIR0i6BQZ4HB1m+Wqga61rlBKPQ6sBtyBeVrrPUqpl4FErfWyyl3HAwu0tlO3X7iWnIPw0RhoFGDMnOgXZnZFdWa1ak6dKeVITjFHc4s5mlPMkZxijuWWkH66hOKyGjuPVbw83KqCN9DHkzB/L1o18asK6MDzwezjQUD1bb6e+Ht54OZWtxA7W2Yh9WQhezIK+DnDCPl3vzlMucX4K9vY28MI+eaBRtBHBhAb5o97HY9zvQrPlbN5XzbrUrPYtC+bgrPleLm7cUvrUB7q04pB7cNpHuRTrzU5C2VW/iYkJOjExERTji1MVpAB84YZt/ZPWQVhbcyu6LK0rhbaOcUczS2p/Gx8VG9teLm7ER3qS0yoHy1CfAj29boolKsHdYCPp0O0BkorLBzIKuLnjAJ+zjSCPvVEIaUVxs/l4+lOXPMAOjUPqGzZBNImwh9Pd9uetD6eW2KMwvdmse1wHhVWTYifFwPahTMkLpzebZrg30jOrQAopXZorRNqfE4CXdSr4lx4f4Qx2dYDy6F5V7MrQmtNdlEpR3OMsD6SeyG8j+UWU1JtpO3prmgR4ktsqB8xYcZHbKgfLUN9aR7kU++jWXuosFg5lF1cFfJ7MgrZk1lQ9RuHl4cb7Zs2pmNlq6ZzZCBtIxrX6R8oi1WTnJbP+tQs1qVmsT+rCIAbwv0Z3CGCwR3CiY8Odon309Yk0IVjKD0DH/wKTqUaPfOY3vV2aK01OUVlVa0R43NJZYuk+KL2iIebIjrE1wjsUD9iwoxRd2yYn8uEdl1ZrZqjucX8nFnZsqkczRecLQeM96xNROOqk64dmwfQoVkAftVG1SVlFXxzIId1KVls3HeKnKIy3N0U3WKCK0M8gpgwP7N+RKchgS7MV34OPrnbuKpl/HxoN8Iuh8krLuNIThFHqo22j1WGd1FpRdV+7m6KFsE+VaEdWznajgn1JTLIBw8btxRckdaa9NNnL2rX/JxRQG7l3ZhKQaswPzpFBlJ4tpzvDuVSVmGlsbcH/duFM7hDOP3bhhPo62nyT+JcrhTo0pQS9mepgC8egiNb4M53bBbmpRUWUjILSTqeT1JaPknHT5N++mzV8+5uiqhgH2JC/UhoGULLUN+qFklksI/N+8ANjVJG+6lFiC8jOhsrSGmtySosvSjkfzySh5eHG5N6RDOkQwTdYkPkvbcTCXRhX1rDV0/B3uUw/A24cfw1vowxGjwf3EnH80nJLKTMYpy8ax7oTXx0MJNvaUmb8Ma0DDWCRoKjfimlaBroTdNAb7vMVSKuTAJd2I/WsOYFSP4Y+s2Ano/W+luLSivYlXZ+5J1PctppcoqMX+W9Pd3oEhXEg71jiG8RRNcWwTQNlBuShJBAF/bz7T9g67+g+zTo/4tZl6tYrZqD2UUkH88nKc0Yfe/POlN1E0yrJn70axtOfHQQXVsE0b5pY+lxC1EDCXRhH9vfg/UvQ+d7jFZLtTv5cotKSU7LJ7ly9P1TWj5nKk9YBvp40rVFEMM7NSU+OpiuUUFy0kyIWpJAF7b38xew4lloM4yyO/5FakYhScdPGwGels+xytu33d0U7Zs2ZlR8c+JbBBMfHURsmJ/cxi3ENZJAFzajtSb3p5WELH2ENP8bmVHwa3b8eQNllXcdRgQ04qboYCZ2jyY+OpjOkYH4eJl/t6QQrkICXVyX0+enMN2bRenhrfyr4k+k6uZMzn+SVv6NeODWpnRtEUR8dBDNAmX+DSHsSQJd1Nmx3GLWpmSxJiWLxKN5WDX09j/JXP0a5X5NcRv9JT+0biWXDApRzyTQxVVZrZqf0vNZl5rF2pQL8260b9qYxwfcwPBYDzos/Q1KBcBDKwgIktWohDCDBLqo0blyC1sP5bKmckmvU2dKcXdTdI8J4cU7ohkSF0GLEF/jWvP5d8PZ0zB1gzG3uRDCFBLookp+idEPX5uSxeb92ZSUWfDzcqdfuyYMiYtgQLtwgnwvWc5s2xw4uBZu+xs07WRO4UIIQAK9wTueW8KalJOsTcki8dhpLFZNREAj7oyPZEhcBLe0DqWRx2WuRDmxC9a+CG1HQLeH67dwIcQvSKA3MFarZndGAWtTjH74vqwzALSLaMyv+7VmSFwEnSMDr74aTlmJMeGWTwiMmn3RjUNCCHNIoDcApRUWvj+Uy9rKfnhWYSluCrrFhPDC7R0YGteU6FDfur3o6ucg5wBMXgJ+ofYpXAhRJxLoLiq/pIyN+yr74fuyKS6z4OvlTr+2F/rhwX5eV3+hmqQsgx3/B72eglb9bVi1EOJ6SKC7kLS8EtakZLEuJYsfj+ZhsWqaNG7EyK6RDK3sh1/3OpYF6bDsCWgeDwNesE3hQgibkEB3UucXLz6cXcz3h3JYm5LF3pNGP7xNuD+P9G3FkLgIbowKqvPq8JdltcCXj4ClHMa+Bx7XOMIXQtiFBLoDu9zixcY6mCWcLTfWwXRTkBATwvO3dWBInB3XZfz2H3DsWxj9HwhtbZ9jCCGumQS6yc4vXnws1wjq2i5efGvrsKrFiztFBhJyrf3w2krbDhtfg05j4cYJ9j2WEOKaSKDXA601ecVlHM0tqVpx/nx4H8spqZoLHC5evLh7bAixYX60DPUlNszPvMWLzxUYlygGRsId/yuXKArhoCTQbeh0cZkxws4trlp1/nx4nzl3IbTdFEQFGyPtm6ODaVlt1fkoR1u8WGtjbvOCdHjwa/AONLsiIcRlSKBfA4tVs+1wLj8ezavsbRvhXXC2vGofpSAyyIfYMD9GdW1OTLXQbhHsi5eHA4X2lez6DHZ/DgOeh+geZlcjhLgCCfQ62HuykMVJGSxNyuRk4TmUguaBPrQM9eX2Ls2IDTUCOzbMWHH+srfMO4vcQ8boPPpW6POs2dUIIa5CAv0qsgrPsSw5ky+TMkg9UYi7m6Jf2yY8f3sHBnUIx9fLRd9CSzl88TC4ucOYucZnIYRDc9E0uj7FpRWs+vkkS5Iz+O5gDlYNN1CCAQMAAAzOSURBVLYI4k8jO3JHl2aE+jcyu0T72/gXyNwJd38AQS3MrkYIUQsS6JUqLFa+PZjD4qQM1uzJ4my5hRYhPjw+4AZGx0fSqom/2SXWn8Ob4du34KbJ0HG02dUIIWqpQQe61pqfM4y++LKfMskpKiXQx5M7b4pkTHwkN7cMbngr0BfnwuJHIPQGGP662dUIIeqgQQZ6+ukSliZnsjgpg4OnivByd2Ng+3BGx0cyoH0T5z+Zea20NuZpKcmFiZ+Bl53uOBVC2EWtAl0pNRz4J+AOvKu1/sXQTSl1DzAT0MBPWuuJNqzzuhWcLefr3Sf4MimDH4/kAdAtJphX7+zM7Z2bEejraXKFDiDxPdi3Aoa9Cs1uNLsaIUQdXTXQlVLuwGxgCJAObFdKLdNap1Tbpw3wHNBLa31aKRVur4LroqzCyqZ9p1iSnMG61FOUVVhpFebHs0PaMjo+0lgTUxiyUmD189B6EPT4tdnVCCGuQW1G6N2Bg1rrwwBKqQXAKCCl2j5Tgdla69MAWutTti60trTW7Dyez+KkdJbvOkF+STmhfl5M7B7NnfGRdIkKbHh98aspP2vc2t+oMdw5B9yc5KYnIcRFahPokUBatcfpwKW3DLYFUEp9h9GWmam1XnXpCymlpgHTAKKjbbs6/NGcYhYnZbAkOYNjuSU08nBjaMemjImPpHebMMe6nd7RrH0RTqXApEXg7xC/XAkhroGtTop6AG2A/kAUsEUp1VlrnV99J631XGAuQEJCgr7eg+YVl7F8l3FyM+l4PkrBra1DeXzADQzv1JTG3tIXv6p9q+DHudDzMWgzxOxqhBDXoTaBngFUv7MkqnJbdenANq11OXBEKbUfI+C326TKas6VW1ifeorFSels2pdNhVXTvmljnhvRnpFdm9Ms0MfWh3RdZ07C0scgojMMnml2NUKI61SbQN8OtFFKxWIE+Xjg0itYlgATgPeVUmEYLZjDtiz0vH9vPMisDQeJCGjElN6x3BkfSYdmAfY4lGuzWo3rzctK4K73wKMB3P0qhIu7aqBrrSuUUo8DqzH64/O01nuUUi8DiVrrZZXPDVVKpQAW4Hda61x7FHxPtxZ0jw3lltahuNtqabWGaOvbcHgT3PEWNGlndjVCCBtQWl93K/uaJCQk6MTERFOO3eBl7IT3hkC7EXDPR7JghRBORCm1Q2udUNNzculHQ1NaZMyi6B8Bv5olYS6EC2mQt/43aF//HvIOwwPLwTfE7GqEEDYkI/SGZPciSJ4PfX8LMb3NrkYIYWMS6A3F6WOw/BmI6gb9/mB2NUIIO5BAbwgsFfDlVGM2xbHvgrvccCWEK5IeekOw5a+Qtg3GvAvBMWZXI4SwExmhu7pj38OWN+HGCdDlbrOrEULYkQS6Kzt7Gr6YCkEt4bY3za5GCGFn0nJxVVrDV09B0UmYssaYGlcI4dJkhO6qkj6ClKUw4HmIutnsaoQQ9UAC3RVl74ev/wCxfaHX02ZXI4SoJxLorqai1Fh9yMMb7nxHVh8SogGRHrqrWf8ynNwF4z+BgOZmVyOEqEcyfHMlB9fB1n9BwkPQ/nazqxFC1DMJdFdRlA2Lfw1N2sOwv5hdjRDCBNJycQVaw5Jfw7kCmLwEPGUZPiEaIgl0V7BtDhxcCyPehIiOZlcjhDCJtFyc3cndsPZFaDscuk81uxohhIkk0J3ZiV3w6QTwCYFRs2X1ISEaOAl0Z7V7Ebw3FKwWmLgA/MLMrkgIYTLpoTsbqwXW/wm++ye06An3fAiNI8yuSgjhACTQncnZ07DoITi0HhKmwPA3wMPL7KqEEA5CAt1ZnEo1+uUF6XDHW5DwoNkVCSEcjAS6M0j9ChY/Cp6+8MByiO5pdkVCCAckge7IrFbY/DpsfgMib4ZxH8v8LEKIy5JAd1TnCmHxI7BvJXSdBLf/Azy9za5KCOHAJNAdUc5BWDABcg8Zd392nyrXmAshrkoC3dHsXw1fPAzunjB5KcT2MbsiIYSTkEB3FFrDN3+HDa9A084wfj4ERZtdlRDCiUigO4LSIlj6mLEGaKe7YOTb4OVrdlVCCCcjgW62vCOwYBJkp8LQV+CWx6VfLoS4JhLoZjq0AT6vvEFo0iK4YZC59QghnJpMzmUGreH7t+HjscZ15dM2SpgLIa5brQJdKTVcKbVPKXVQKTWjhucfUEplK6WSKz8etn2pLqL8LHw5Dda8YKz7+dBaCGlldlVCCBdw1ZaLUsodmA0MAdKB7UqpZVrrlEt2/Uxr/bgdanQd+Wnw2SRjHvOBL0Cf30q/XAhhM7XpoXcHDmqtDwMopRYAo4BLA11cydFvYeH9YCmDCQug3XCzKxJCuJjatFwigbRqj9Mrt11qrFJql1JqkVKqRU0vpJSappRKVEolZmdnX0O5Tkhr2DYXPhwFPsEwdYOEuRDCLmx1UvQrIEZr3QVYC3xQ005a67la6wStdUKTJk2u7UhW6zUXWe8qSmHZ4/D17+CGITB1PYS1MbsqIYSLqk3LJQOoPuKOqtxWRWudW+3hu8Bfr7+0y9jxvnFHZfN4YwbCyJuMr70D7XbIa1J4Aj67FzISoe/vof9z4CYXFQkh7Kc2gb4daKOUisUI8vHAxOo7KKWaaa1PVD4cCaTatMrqQlpBy16QsQP2Lr+wPawtNL/pQshHdDJvdsK0H40wLy2Cez6CuJHm1CGEaFCuGuha6wql1OPAasAdmKe13qOUehlI1FovA55USo0EKoA84AG7Vdx6gPEBxpJsmUlGuGfshMMbYdcC4zk3T4joWBnwlSEf1hbc3O1WGgA7PoAVz0JgFNy3BCLi7Hs8IYSopLTWphw4ISFBJyYm2vZFtYbCTCPgM3dWfk6G0kLjeS9/aNbVCPfzIR/YwjaXDlaUwaoZkPgetB4Id80zToIKIYQNKaV2aK0TanrOtW79VwoCI42P820OqxVyD14c8tvmGJcPAvg1ubhV0/wm8Aut23GLThmXJB7/Hno9BYNesv9vAkIIcQnXCvSauLlBk7bGR9cJxraKUsjac6FVk7kTDqwBKn9bCY65OOSb3QhefjW/fsZOo19ekgdj34POd9XHTyWEEL/g+oFeE49GlW2Xmy5sO1cIJ36qDPkdkL4d9nxpPKfcIDzu4itrwuNg9yL46inwj4CH1kCzLub8PEIIQUMN9Jp4BxirA1VfIehMVmWbZueFq2qSPjKec28EllKI6QN3f1D3No0QQtiYBPqVNI6AdiOMDzBOup4+ciHg/cLg1ieN5eKEEMJkEuh1oZRxHXxIK+mVCyEcjty6KIQQLkICXQghXIQEuhBCuAgJdCGEcBES6EII4SIk0IUQwkVIoAshhIuQQBdCCBdh2vS5Sqls4Ng1fnsYkGPDcpydvB8Xk/fjAnkvLuYK70dLrXWNa3iaFujXQymVeLn5gBsieT8uJu/HBfJeXMzV3w9puQghhIuQQBdCCBfhrIE+1+wCHIy8HxeT9+MCeS8u5tLvh1P20IUQQvySs47QhRBCXEICXQghXITTBbpSarhSap9S6qBSaobZ9ZhFKdVCKbVRKZWilNqjlHrK7JocgVLKXSmVpJRabnYtZlNKBSmlFiml9iqlUpVSt5hdk1mUUs9U/j35WSn1qVLK2+ya7MGpAl0p5Q7MBkYAccAEpVScuVWZpgJ4VmsdB/QEpjfg96K6p4BUs4twEP8EVmmt2wM30kDfF6VUJPAkkKC17gS4A+PNrco+nCrQge7AQa31Ya11GbAAGGVyTabQWp/QWu+s/PoMxl/WSHOrMpdSKgq4HXjX7FrMppQKBPoC7wForcu01vnmVmUqD8BHKeUB+AKZJtdjF84W6JFAWrXH6TTwEANQSsUA8cA2cysx3VvA7wGr2YU4gFggG3i/sgX1rlLKz+yizKC1zgD+BhwHTgAFWus15lZlH84W6OISSil/4Avgaa11odn1mEUpdQdwSmu9w+xaHIQHcBPwH611PFAMNMhzTkqpYIzf5GOB5oCfUupec6uyD2cL9AygRbXHUZXbGiSllCdGmM/XWn9pdj0m6wWMVEodxWjFDVRKfWxuSaZKB9K11ud/a1uEEfAN0WDgiNY6W2tdDnwJ3GpyTXbhbIG+HWijlIpVSnlhnNhYZnJNplBKKYz+aKrW+h9m12M2rfVzWusorXUMxp+LDVprlxyF1YbW+iSQppRqV7lpEJBiYklmOg70VEr5Vv69GYSLniD2MLuAutBaVyilHgdWY5ypnqe13mNyWWbpBdwH7FZKJVdu+6PWeqWJNQnH8gQwv3Lwcxh40OR6TKG13qaUWgTsxLg6LAkXnQJAbv0XQggX4WwtFyGEEJchgS6EEC5CAl0IIVyEBLoQQrgICXQhhHAREuhCCOEiJNCFEMJF/D9SMivxuriPcAAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAD4CAYAAAD8Zh1EAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+j8jraAAAgAElEQVR4nO3dd3hUVf7H8fchhSTUmACBJJAAoXciRapSRF2agDRpK2ABRXQL7u5PXdd1LaDSLICIdBAs4LIKSFN6EumQSkmCQAglEAhp5/fHjRgQJCEzOTOT7+t5eB5n5mbuhyH5eHPvuecorTVCCCGcXynTAYQQQtiGFLoQQrgIKXQhhHARUuhCCOEipNCFEMJFuJvasb+/vw4JCTG1eyGEcEqRkZFntdaVbvWasUIPCQkhIiLC1O6FEMIpKaWO3+41OeUihBAuQgpdCCFchBS6EEK4CGPn0G8lKyuLpKQkMjIyTEdxKl5eXgQFBeHh4WE6ihDCIIcq9KSkJMqVK0dISAhKKdNxnILWmtTUVJKSkggNDTUdRwhhkEOdcsnIyMDPz0/KvBCUUvj5+clvNUIIxyp0QMr8LshnJoQAByx0IYQLyc2FI/+FQ6sgN8d0GvOunIPvX4PUeLu8vUOdQxdCuAitIeZb2PBvOL3feq5Sfbj/b1C/J5S03yoz0mDnR7BtOly7BOWrgV8tm+9GjtCLoGzZsrd97dixYyxevPj643nz5jF+/PjiiCWEOVpD3PcwpwssGQRZ6fDoHOj/KegcWD4MZnWCmLXWtq4u8wpsnQpTm8LGf0NoR3h6K9w72i67kyN0O/ml0IcMGVKor8vJycHNzc1OqYSwo2NbYcPrcGIbVAiGXjOg6WBwy6uZBr1h/+ew6T+weAAEtYIH/gE1O5nNbQ/Z1yByHmyZDOlnoHZXuP/vENjCrrt12EL/5+qDHDqZZtP3bFCtPK/0bHjb1ydNmkRwcDDjxo0D4NVXX8Xd3Z2NGzdy/vx5srKyeP311+ndu/cd9zVp0iQOHz5Ms2bNGDFiBL6+vpw8eZIePXoQHx9P3759efvttwHrSP/JJ59k/fr1zJw5k/bt29vmLyxEcUiKhI2vQ/wGKBsAD0+GFsPBvfSN25Vyg6aDoFE/2LMINr8N83tBSAer2Ku3MZPflnKy8v5u70BaEtRoD4/Nhxpti2X3csoln4EDB7J8+fLrj5cvX86IESP48ssviYqKYuPGjbz44osUZB3WN998kw4dOrBnzx4mTpwIwJ49e1i2bBn79+9n2bJlJCYmApCenk7r1q3Zu3evlLlwHqf2w+JBMOcB+HkvdP83TNgDrcb8tszzc/OAliPh2Sjo8RakRMPcB2Fhfzj5U7HFt6ncHNi7FGaEw+oJUL4qDP8aRn5TbGUODnyE/ntH0vbSvHlzzpw5w8mTJ0lJScHX15eAgAAmTpzIli1bKFWqFMnJyZw+fZqAgIBCv3+XLl2oUKECAA0aNOD48eMEBwfj5uZGv379bP3XEcI+UqJh4xtw6CvwqgAP/B+0fhJKlyvc+3h4QZunoMUw2DUbtr4PszpDvT9YF0+rFH8HFFpuLhz+2vo8zsZAQBMYshzCuhu58OuwhW7KgAEDWLFiBadOnWLgwIEsWrSIlJQUIiMj8fDwICQk5K5v4ild+tejFjc3N7KzswHr1n05by4c3rkE2PQW7F8OHj7Q8S/Qdhx4Vyza+3qWgfbPQ/gfYceHsH2GNdSxUT/o/BL417ZNflu6eRRPpXrWqZV6PaGUuRMfUug3GThwIGPGjOHs2bNs3ryZ5cuXU7lyZTw8PNi4cSPHj992KuIblCtXjkuXLtk5rRDF4GISbHkHfloIpTyg7Xho9zyU8bPtfrzKQ+e/Wqdstk23hvkd/AKaDoFOfwHfGrbd393QGhI2Whd/kyPBNxQenW39z6eU+YMyKfSbNGzYkEuXLhEYGEjVqlUZOnQoPXv2pHHjxoSHh1OvXr0CvU+TJk1wc3OjadOmjBw5El9fXzsnF8LGLp2GH9+FiLlWkYX/ETq8COUKf7qxUHzuga6vQJtn4Mf3YPcc2LfMutDa8U/WGG4Tjm21hh4e35o3imd63igex5kUTxXkAp89hIeH65tXLDp8+DD169c3ksfZyWcnbObKOet89s5ZkJMJzYdCxz9Dxepm8qSdtIb/Rc0HVcoaw91+IpS95SpstnfDKJ4q1mdxq1E8xUQpFam1Dr/Va3KELoSwXL0AOz6A7R9A5mVo8hh0+qtd7mgslPLV4A/vQrvnrOGAOz+EyE+h9VNw37PWEb09nNpvXeyMXgM+ftD9dQh/Ajx97LM/G5BCL6L9+/czbNiwG54rXbo0O3fuNJRIiEK6dhl2fQxbp0HGBesGoM5/g8oFO71YbHxDoM9M6wLqpjd/PR3Tdjy0edo6B28LKdHWzU8Hv4TSFawx8q2fKvwoHgOk0IuocePG7Nmzx3QMIQov66p1fvyHd+HKWajTwxouWLWp6WS/zz8M+n8CHV6wjqA3vWEdtbd73rqg6lnm7t733FHY/JZ1vt7Dxzq10nYceDvP9S8pdCFKmuxM+Gm+dV760s9QszPc/w8Ivtd0ssKp0hAGLYLkKKvY178C22daF25bjrTGuRfEDaN43K0Sb/c8lPG3a3x7kEIXoqTIyYZ9S62x5BdPQHAba8hdaAfTyYomsAU8vgJO7LCGE377V9g2zTrCbv747UehmBrFY0dS6EK4utxcazz3xjfgXDxUaw4934NaXVxrGtvqbaxb7RM2w4Z/wTfPW6N1Ok2yLvD+Mk7c0Ubx2FCBCl0p1QOYCrgBc7TWb970enXgM6Bi3jaTtNZrbJxVCOdx7RLkZptOAcd+tIr8zCGo3BAGLYa6D7tWkd+sZidrmtrYdVaxf/WUdSTe8S+QGut4o3hs6I6FrpRyA2YC3YAkYLdSapXW+lC+zf4BLNdaf6iUagCsAULskNeuLly4wOLFi3nmmWcK9XUPP/wwixcvpmLFwt0CPW/ePLp37061ataNEiEhIURERODv73zn7kSek3usAo39znSSX/nVhv5zoUFfo7elFyuloE53COsGh1dbNwR9kTcHeYPe1pQClV3vvo2CHKG3AuK01gkASqmlQG8gf6Fr4JcxQxWAk7YMWVwuXLjABx988JtCz87Oxt399h/VmjV398vIvHnzaNSo0fVCL4g7ZRGGnDlsFfnhVeBVETr8yTEuqpULsOYXcSuh3zNKQYNeUO8Ra+GNcgFQtYnpVHZTkH/lQCAx3+MkoPVN27wKrFVKPQuUAboWOdn/JlkD+20poDE89OZtX540aRLx8fE0a9YMDw8PvLy88PX15ciRI8TExNCnTx8SExPJyMhgwoQJjB07Fvj1yPry5cs89NBDtG/fnm3bthEYGMjXX3+Nt7f3b/a1YsUKIiIiGDp0KN7e3mzfvh2A6dOns3r1arKysvj888+pV68er776KvHx8SQkJFC9enWWLFli289F3L3UeGtM9P7PwbOsdeTX5mlrFkLhOEq5WUfsLs5Wv38NBuZprYOAh4EFSqnfvLdSaqxSKkIpFZGSkmKjXdvOm2++Sa1atdizZw/vvPMOUVFRTJ06lZiYGADmzp1LZGQkERERTJs2jdTU1N+8R2xsLOPGjePgwYNUrFiRlStX3nJf/fv3Jzw8nEWLFrFnz57rpe/v709UVBRPP/00kydPvr79oUOHWL9+vZS5o7hwAr4eDzPuhSPfWDe7PL8POk+SMhfGFOQIPRkIzvc4KO+5/J4AegBorbcrpbwAf+BM/o201rOAWWDN5fK7e/2dI+ni0qpVK0JDQ68/njZtGl9++SUAiYmJxMbG4ud344xzoaGhNGvWDICWLVty7NixQu3z0Ucfvf61X3zxxfXne/XqdcsjfVHM0n6GH6ZYy4upUtY84O0nQtnKppMJUaBC3w2EKaVCsYp8EHDzQpkngC7APKVUfcALcLxD8EIqU+bXO842bdrE+vXr2b59Oz4+PnTu3PmW86LfPOf51atXC7XPX74+/3zpN2cRBqSf/fVW89xsa3KmDn+CCoGmkwlx3R0LXWudrZQaD3yHNSRxrtb6oFLqNSBCa70KeBGYrZSaiHWBdKQ2NY1jEfzeHOYXL17E19cXHx8fjhw5wo4dO+y6P+Egrp6HbTOshReyr0KTQdbc3PeE3vlrhShmBbr0nTemfM1Nz72c778PAe1sG634+fn50a5dOxo1aoS3tzdVqlS5/lqPHj346KOPqF+/PnXr1qVNm6IvaDty5EieeuqpGy6KCgdx7RLs+MhaaOHaRWsBg06ToFId08mEuC2ZD91FyGdnI5lXrNMqP74HV89B3UesCasCGplOJgQg86ELcWfZ1yDyM/hhMlw+bd0W/8DfIbCl6WRCFJgUejEYN24cW7duveG5CRMmMGrUKEOJxHU5WbBnMWx+G9KSoEY7GDAPatxnOpkQheZwha61RrnYPBMzZ8606/s74fVn83JzYP8KayGD80chMBx6z7CmknWx7z9RcjhUoXt5eZGamoqfn5/Llbq9aK1JTU3Fy6uAcz+XdLm51u35G9+As9FQpTEMXgZ1HpQiF07PoQo9KCiIpKQkHPEuUkfm5eVFUFCQ6RiOTWuIXWvNl31qH/jXhQGfQf1eJWfCKuHyHKrQPTw8brgzU4gi0xoSNlmz7SXtBt9Q6DsLGvf/dX5sIVyEQxW6EDZ1fLt1RH78RygfBD2nQbMht1/BRggnJ4UuXE9ylHVEHrceylaBh96BliPAvfSdv1YIJyaFLoouJ9uasOrCcdNJrEWP4zeA9z3Q7V9w72jw9DGdSohiIYUuiiY3B74cCwdWWqc1TI8UcfOA+/9uzUleupzZLEIUMyl0cfdyc605wQ+shK7/tOYEF0IYI+O1xN3RGv47EfYuhs5/kzIXwgFIoYvC0xq+nWQt8tD+BWs6WSGEcVLoonC0hnUvw86PoM046PKy+fPmQghACl0U1sY3YNs0a/TIg/+WMhfCgUihi4Lb8g5seRuaD7PGdkuZC+FQpNBFwWybbt112WQg9Jwq858I4YDkp1Lc2a7ZsPYf0KAP9P5A5kARwkFJoYvfFzkP1vzJWoqt3xxwk1sXhHBUUuji9vYsgdXPQ+1uMOBTmdRKCAcnhS5u7cBK+PoZCO0IAxfIxFZCOAEpdPFbh1fDyjEQ3AYGLwEPb9OJhBAFIIUubhSzFj4fBYEtYOhy8CxjOpEQooCk0MWv4jfAssehSkMYukJmKxTCyUihC8uxH2HJEPAPg2FfgndF04mEEIVUoEJXSvVQSkUrpeKUUpNu8fp7Sqk9eX9ilFIXbB9V2M2JnbDoMahYHYZ9BT73mE4khLgLdxxUrJRyA2YC3YAkYLdSapXW+tAv22itJ+bb/lmguR2yCntIjoJF/aFcFRixCspWMp1ICHGXCnKE3gqI01onaK0zgaVA79/ZfjCwxBbhhJ2d2g8L+oK3L4xYDeUCTCcSQhRBQQo9EEjM9zgp77nfUErVAEKBDUWPJuzqzGGY3xs8y1plXiHIdCIhRBHZ+qLoIGCF1jrnVi8qpcYqpSKUUhEpKSk23rUosLNx8FkvKOVhnWbxrWE6kRDCBgpS6MlAcL7HQXnP3cogfud0i9Z6ltY6XGsdXqmSnKs14txR+Kwn6FyrzP1qmU4khLCRgsy0tBsIU0qFYhX5IGDIzRsppeoBvsB2myYUtnMh0Toyz74KI76BSnVNJxKiWGit2ZFwjpMXrpqOAkCz6hWpVamszd/3joWutc5WSo0HvgPcgLla64NKqdeACK31qrxNBwFLtdba5ilF0aX9bB2ZZ1y0jswDGplOJESx2BZ3lslro4k64TijqV/v08hMoQNordcAa2567uWbHr9qu1jCpi6fgfm9ID0Fhn8N1ZqZTiSE3UUcO8eUtTFsT0ilagUv3ujbmHa1/VCYX2nLt4x9Zi6Vya1dXXqqNZrlYhI8vhKCwk0nEsKu9iddZMq6aDZFp+BftjSv9GzA4FbV8fJw/YVZpNBd2dXzsKAPnEuAIcuhxn2mEwlhN9GnLvHuumi+O3iaij4eTHqoHsPb1sDHs+TUXMn5m5Y0GWmwsB+kHIFBS6BmJ9OJhLCLhJTLvL8+ltX7TlLW052JXevwx/YhlPMqeQuySKG7omuXYdEA+HkvPLYAwrqaTiSEzSWeu8K072NZGZVEaXc3nu5Ui7Eda1LRx9N0NGOk0F1N5hVYMgiSdkH/T6Hew6YTCWFTpy5mMGNjLMt2J6KUYlS7UJ7qVItK5WRVLSl0V5KVAcuGWlPhPjoLGvYxnUgImzl7+RofbopnwY7j5OZqBrUKZvz9YQRU8DIdzWFIobuK7Ez4fKS1SEWvGdDkMdOJhLCJC1cymbUlgXnbjpGRlcOjLYKY0CWM4Ht8TEdzOFLoriAnG1Y+ATH/g0emQIthphMJUWSXMrL4dOsxZm9J4HJmNj2bVGNC1zC73JDjKqTQnV1uDnz1FBxeBQ/+B+4dbTqREEVyNTOH+duP8dHmeM5fyaJ7gyq80L0O9QLKm47m8KTQnVluLqx6FvZ/Dl1fhbbPmE4kxF27lp3Dkp0nmLExnrOXr9GpTiVe7F6HJkGyHGJBSaE7K63hvy/AnkXQ+SVoP/HOXyOEA8rKyWVFZBLTv4/l5MUMWofew4ePt+DeEFkKsbCk0J2R1vDtSxD5qVXknf5qOpEQhZaTq/l6TzLvr4/lxLkrNK9ekXcGNOW+Wn4oZX6+FWckhe5stIb1r8DOD6HNM9DlFZBvfuFEcnM1/ztwinfXRROfkk6DquWZOzKc++tWliIvIil0Z7PpP7B1KoQ/AQ++IWUunIbWmu8Pn2HKuhgO/5xGWOWyfDi0BQ82DKBUKfk+tgUpdGeyZTJsfguaPw4PT5YyF05Ba82PcWeZvDaGvYkXqOHnw/sDm9GzaTXcpMhtSgrdWWybARv+BY0fg57ToJStl4MVriTuzCXeWx/LtrizmF5xJjdXk5aRTWBFb97q15hHWwTh4Sbfv/Yghe4Mds2GtX+HBr2hz4dQyvXndRZ353hqOlO/j+Wrn5Lx9nDjkSZV8XaAecDrBJSjf8sgSrubz+LKpNAdXdR8WPMnqPsw9PsE3OSfTPzWyQtXmb4hjs8jEnErpRjdoSZPdqyJX1mZsKokkXZwZHuXwqrnoHZXGDAP3Ere/M7i9525lMEHG+NZvPMEGs3Q1tUZd39tKpeXCatKIil0R3XgC/jqaQjtAAMXgrscaYlfnU/P5KMt8Xy27RhZOZoBLYMY/0BtgnxlwqqSTArdER3+BlaOhuDWMHgpeHibTiQcRFpGFnN+OMrcH4+SnplNn2aBTOgSRoh/GdPRhAOQQnc0MWutaXCrNbfWAfWUH1QB6deymbftGLO2JHDxahYPNw7g+a51qFOlnOlowoFIoTuShE2w7HGo0gAeXwleMrtcSZeRlcPCHcf5cFM8qemZdKlXmYnd6tAosILpaMIBSaE7imNbYfEg8KsNw74Cb5lhriTLzM5lWUQiMzbEcjrtGu1r+/NC9zq0qO5rOppwYFLojiBxFyx+DCoGw/CvwUdmmSupsnNy+eKnZKZ9H0vS+auE1/Dl/YHNaVvLz3Q04QSk0E1LjoKF/aBsZRi+CspWMp1IGJCbq1m97yRT18eScDadxoEVeL1PIzrVqSQTVokCK1ChK6V6AFMBN2CO1vrNW2zzGPAqoIG9WushNszpmk7thwV9rdMrI1ZD+aqmE4liprXmu4OneW9dDNGnL1G3Sjk+HtaS7g2qSJGLQrtjoSul3ICZQDcgCditlFqltT6Ub5sw4CWgndb6vFKqsr0Cu4wzR2B+H2sUy4jVUCHIdCJRjLTWbI5JYcraGPYnX6SmfxmmDW7OHxpXlZkHxV0ryBF6KyBOa50AoJRaCvQGDuXbZgwwU2t9HkBrfcbWQV3K2TiY38uak2XEavANMZ1IFKPt8alMWRtNxPHzBPl6807/JvRtHoi7TFgliqgghR4IJOZ7nAS0vmmbOgBKqa1Yp2Ve1Vp/e/MbKaXGAmMBqlevfjd5nd+5o/BZT2tx55H/Bb9aphOJYhJ5/Dzvrotma1wqVcqX5vU+jXgsPBhPdylyYRu2uijqDoQBnYEgYItSqrHW+kL+jbTWs4BZAOHh4aZn9Sx+FxKtI/PsqzDiG6hcz3QiUQwOJF/k3XUxbDhyBr8ynvzfHxowtHV1vBxgFkThWgpS6MlAcL7HQXnP5ZcE7NRaZwFHlVIxWAW/2yYpXUHaz1aZX70AI1ZBQCPTiYSdxZy+xHvrYvjfgVNU8PbgLz3qMqJtCGVKy+AyYR8F+c7aDYQppUKxinwQcPMIlq+AwcCnSil/rFMwCbYM6tQup1hlfvkMDPvSuq1f2Ny17ByW7U7k4pUs01GIOXOZb/adpIynOxO6hPFEh1DKe8lsmcK+7ljoWutspdR44Dus8+NztdYHlVKvARFa61V5r3VXSh0CcoA/a61T7RncaVw5B/N7w8UkGLoCgluZTuSSMrNzGbfoJ9YfPm06CgA+nm482bEWT3asiW8ZT9NxRAmhtDZzKjs8PFxHREQY2XexuXrBugCaEg1Dl0PNzqYTuaTsnFyeW/oTa/af4rXeDRnSyvwF91JKyfBDYRdKqUitdfitXpOTefaSkWbdAXrmMAxeImVuJzm5mhc/38ua/af4xyP1Gd42xHQkIYyRQreHzHRrbpaf98Bj8yGsm+lELik3V/PSF/v4es9J/vxgXUZ3qGk6khBGSaHbWtZVWDIIEndaa4DWe8R0IpekteblVQdYHpHEc13CGHd/bdORhDBO7miwpexr1nzmR3+APh9Bo0dNJ3JJWmv+9c1hFu44wZOdajKxa5jpSEI4BDlCt5WcLGulobj10Gs6NB1oOpFL0lrz1rfRzN16lFHtQpjUo55MYiVEHjlCt4WcbGsN0Og18PBkaDHcdCKXNfX7WD7aHM/Q1tV5+Q8NpMyFyEcKvahyc+Crp+HQV/DgG9BqjOlELuuDTXG8vz6W/i2D+FfvRlLmQtxECr0ocnNh9XOwfzl0eRnajjOdyGV98uNR3v42ml5Nq/FWvyYyxluIW5BCv1taw5o/wU8LodNfocOLphO5rAU7jvOvbw7xUKMA3n2sKW5S5kLckvNdFI2cB1unmk5hXQS9mAjtnofOL5lO47KW707k/746QNf6lZk6qLnMGS7E73C+Qi8bAIEtTaewVHsG2jwNci7XLr76KZm/frGPjnUqMXNoC5k3XIg7cL5Cr9vD+iNc2n/3/cwLy/fQJtSPjx9vSWl3mTtciDuRQx7hcNYdOs2EpT/Rorovc0aE4+0pZS5EQUihC4eyKfoM4xZF0TCwAp+OulcWgxCiEKTQhcPYGneWJxdEElalLPNHtaKcLAghRKFIoQuHsOvoOUZ/FkGIXxkWPNGaCj5S5kIUlhS6MC7qxHlGfbqLqhW9WDi6NffICj9C3BUpdGHUgeSLjJi7C/9ypVk8ug2VypU2HUkIpyWFLow5ciqNxz/ZSXkvDxaPaUNABS/TkYRwalLowoi4M5cYOnsnXu5uLB7TmsCK3qYjCeH0pNBFsTt6Np0hs3eilGLRmNbU8CtjOpIQLkEKXRSrxHNXGDJ7B9m5msVjWlOrUlnTkYRwGVLooticvHCVIXN2cCUzh4VPtKZOlXKmIwnhUqTQRbE4k5bB0Dk7uZCexfw/tqJBtfKmIwnhcuS+amF3Zy9fY8icnZxOy2DBE61oGlzRdCQhXJIcoQu7Op+eyeNzdpJ0/gpzR95Lyxr3mI4khMsqUKErpXoopaKVUnFKqUm3eH2kUipFKbUn789o20cVzubi1SyGz91Fwtl0Zg8Pp01NP9ORhHBpdzzlopRyA2YC3YAkYLdSapXW+tBNmy7TWo+3Q0bhhC5fy2bkp7s4ciqNj4e1pENYJdORhHB5BTlCbwXEaa0TtNaZwFKgt31jCWd2JTObP366m31JF5k+uAUP1KtiOpIQJUJBCj0QSMz3OCnvuZv1U0rtU0qtUEoF3+qNlFJjlVIRSqmIlJSUu4grHF1GVg5j5kcQcfwc7w9sRo9GAaYjCVFi2Oqi6GogRGvdBFgHfHarjbTWs7TW4Vrr8EqV5FdwV3MtO4enFkayLT6Vd/o3pWfTaqYjCVGiFKTQk4H8R9xBec9dp7VO1Vpfy3s4B3CQVZxFccnKyWX84p/YFJ3CG30b069lkOlIQpQ4BSn03UCYUipUKeUJDAJW5d9AKVU138NewGHbRRSO7kxaBuMWRbHu0Gn+2ashg1tVNx1JiBLpjqNctNbZSqnxwHeAGzBXa31QKfUaEKG1XgU8p5TqBWQD54CRdswsHMS59Ew+3hzPZ9uPkZWj+ccj9RlxX4jpWEKUWEprbWTH4eHhOiIiwsi+RdFcvJrFJz8k8MmPR7mSlUPfZoE81yWMEH+ZNVEIe1NKRWqtw2/1mtz6Lwos/Vo287Yd4+PN8aRlZPNI46o83zWMMJlkSwiHIIUu7igjK4eFO47z4aZ4UtMz6Vq/MhO71aFhtQqmowkh8pFCF7eVmZ3Lst0nmLExjtNp1+gQ5s8L3erQvLqv6WhCiFuQQhe/kZ2TyxdRyUz9PpbkC1e5N8SXqYOay1wsQjg4KXRxXW6uZvW+k7y/PpajZ9NpElSBNx5tTMcwf5RSpuMJIe5ACl2gtea7g6d5b10M0acvUS+gHLOGtaRbgypS5EI4ESn0EkxrzaaYFKasjeZAcho1K5Vh+uDmPNK4KqVKSZEL4Wyk0EuobfFnmbI2hsjj5wny9WbygKb0aVYNdzdZ80QIZyWFXsJEHj/HlLUxbItPJaC8F//u24gBLYPxdJciF8LZSaGXEAeSLzJlbTQbo1PwL+vJy39owJDW1fHycDMdTQhhI1LoLi761CXeWxfDtwdPUcHbg7/2qMeI+2rg4yn/9EK4GvmpdlFHz6bz/voYVu09SRlPdyZ0CeOJDqGU9/IwHU0IYSdS6C4m8dwVpm+IZWVUMp5upXiyYy2e7FgT3zKepqMJIexMCt1FnE7LYMaGOJbuPoFCMbxtDZ7pXJtK5UqbjiaEKCZS6E4u9fI1Ptocz/ztx8nJ1Tx2bzDj769NtYrepqMJIeLp2fYAAAsMSURBVIqZFLoTO56azmMfbyfl0jX6Ng9iQpcwqvv5mI4lhDBECt1JJZ2/wpDZO8nMzmXV+PY0CpSpbIUo6eRuEif088WrDJm9k0sZWSx4orWUuRACkCN0p3PmUgZDZ+/kXHomC0dLmQshfiVH6E4k9fI1hs7eyam0DOaNupdmwRVNRxJCOBApdCdx4Uomwz7ZxYlzV5gzIpzwkHtMRxJCOBgpdCeQlpHFiLm7iDtzmVnDw7mvlr/pSEIIBySF7uAuX8tm1Ke7OXgyjQ+GtqBTnUqmIwkhHJRcFHVgVzNzeGLebvYkXmDG4OZ0bVDFdCQhhAOTI3QHlZGVw9gFEew6do53H2vKQ42rmo4khHBwUugOKDM7l2cWRfFD7Fne7teE3s0CTUcSQjiBAhW6UqqHUipaKRWnlJr0O9v1U0pppVS47SKWLFk5uTy7JIoNR85YqwmFB5uOJIRwEncsdKWUGzATeAhoAAxWSjW4xXblgAnATluHLClycjUvLN/LdwdP80rPBgxtXcN0JCGEEynIEXorIE5rnaC1zgSWAr1vsd2/gLeADBvmKzFyczV/WbGP1XtP8tJD9RjVLtR0JCGEkylIoQcCifkeJ+U9d51SqgUQrLX+7++9kVJqrFIqQikVkZKSUuiwrkprzd+/2s/KqCRe6FaHJzvVMh1JCOGEinxRVClVCngXePFO22qtZ2mtw7XW4ZUqyXhqsMr8n6sPsWRXIuPur8WzD9Q2HUkI4aQKUujJQP4rc0F5z/2iHNAI2KSUOga0AVbJhdE701rzn/8dYd62Y4xuH8qfutdFKWU6lhDCSRWk0HcDYUqpUKWUJzAIWPXLi1rri1prf611iNY6BNgB9NJaR9glsQt5b10Ms7YkMLxtDf7+SH0pcyFEkdyx0LXW2cB44DvgMLBca31QKfWaUqqXvQO6qhkbYpm2IY5B9wbzas+GUuZCiCIr0K3/Wus1wJqbnnv5Ntt2Lnos1zZrSzyT18bwaPNA/t23MaVKSZkLIYpO7hQtZp9tO8Yba47wSJOqvN2/CW5S5kIIG5FCL0ZLdp3glVUH6d6gCu8PbIa7m3z8QgjbkUYpJisjk/jbl/u5v24lpg9pjoeUuRDCxqRVisHqvSf584q9tKvlz4ePt6S0u5vpSEIIFySFbmffHjjF88v2EB5yD7OGt8TLQ8pcCGEfUuh2tOHIaZ5dEkWToArMHXkvPp6ynogQwn6k0O3kh9gUnloYRb2A8swb1YqypaXMhRD2JYVuBzsSUhkzP4Ka/mVY8EQrKnh7mI4khCgBpNBtLPL4Of44bzfBvj4sGt2aij6epiMJIUoIKXQb2pt4gZFzd1OlvBeLRrfGr2xp05GEECWIFLqNHDx5keFzd1GxjAeLx7Smcnkv05GEECWMFLoNxJy+xLBPdlHG043Fo9tQtYK36UhCiBJICr2IElIuM2T2TtxLKRaPaUPwPT6mIwkhSigp9CI4kXqFIbN3AprFY9oQ4l/GdCQhRAkmg6PvUuK5KwyevYOM7ByWjm1D7cplTUcSQpRwUuiFdC49k482xzN/+zE83EqxZEwb6gWUNx1LCCGk0Avq4tUs5vyQwNwfj3IlK4e+zQJ5vmsdqvvJOXMhhGOQQr+D9GvZzNt2jI83x5OWkc0jjavyfNcwwqqUMx1NCCFuIIV+GxlZOSzccZwPNsVzLj2TrvUrM7FbHRpWq2A6mhBC3JIU+k2uZeewfHci0zfEcebSNTqE+fNCtzo0r+5rOpoQQvwuKfQ82Tm5fBGVzNTvY0m+cJVWIfcwbXBz2tT0Mx1NCCEKpMQXek6u5pt9J3lvXQzHUq/QNKgC/3m0MR3C/FFKFnAWQjiPElvoWmu+O3iKd9fFEHP6MvUCyjF7eDhd61eWIhdCOKUSV+haazZFpzBlXTQHktOoWakMM4Y05+FGVSlVSopcCOG8SlShb4s7y+S10USduEDwPd5MGdCU3s2q4e4mMyAIIZxfiSj0iGPnmLI2hu0JqVSt4MUbfRszIDwIDylyIYQLKVChK6V6AFMBN2CO1vrNm15/ChgH5ACXgbFa60M2zlpo+5MuMmVdNJuiU/AvW5pXejZgcKvqeHm4mY4mhBA2d8dCV0q5ATOBbkASsFspteqmwl6stf4ob/tewLtADzvkLZDoU5d4d1003x08TUUfDyY9VI/hbWvg41kifiERQpRQBWm4VkCc1joBQCm1FOgNXC90rXVavu3LANqWIQsqIeUy76+PZfW+k5T1dGdi1zr8sX0I5bxkkWYhhOsrSKEHAon5HicBrW/eSCk1DngB8AQeuNUbKaXGAmMBqlevXtist5V47grTvo9lZVQSpd3deLpTLcZ2rCkLNAshShSbnYPQWs8EZiqlhgD/AEbcYptZwCyA8PDwIh/Fn7qYwYyNsSzbnYhSilHtQnm6cy38ZXFmIUQJVJBCTwaC8z0OynvudpYCHxYl1J2cvXyNDzfFs2DHcXJzNYNaBTP+/jACKsjCzEKIkqsghb4bCFNKhWIV+SBgSP4NlFJhWuvYvIePALHYybLdJ/jn6kNkZOXQr0UQz3UJk3U8hRCCAhS61jpbKTUe+A5r2OJcrfVBpdRrQITWehUwXinVFcgCznOL0y22EnyPD13rV2FC1zBqVZJl34QQ4hdKayMDUggPD9cRERFG9i2EEM5KKRWptQ6/1Wtyq6QQQrgIKXQhhHARUuhCCOEipNCFEMJFSKELIYSLkEIXQggXIYUuhBAuQgpdCCFchLEbi5RSKcDxu/xyf+CsDeM4O/k8biSfx6/ks7iRK3weNbTWlW71grFCLwqlVMTt7pQqieTzuJF8Hr+Sz+JGrv55yCkXIYRwEVLoQgjhIpy10GeZDuBg5PO4kXwev5LP4kYu/Xk45Tl0IYQQv+WsR+hCCCFuIoUuhBAuwukKXSnVQykVrZSKU0pNMp3HFKVUsFJqo1LqkFLqoFJqgulMjkAp5aaU+kkp9Y3pLKYppSoqpVYopY4opQ4rpdqazmSKUmpi3s/JAaXUEqWUSy5A7FSFrpRyA2YCDwENgMFKqQZmUxmTDbyotW4AtAHGleDPIr8JwGHTIRzEVOBbrXU9oCkl9HNRSgUCzwHhWutGWEtpDjKbyj6cqtCBVkCc1jpBa50JLAV6G85khNb6Z611VN5/X8L6YQ00m8ospVQQ1iLlc0xnMU0pVQHoCHwCoLXO1FpfMJvKKHfAWynlDvgAJw3nsQtnK/RAIDHf4yRKeIkBKKVCgObATrNJjHsf+AuQazqIAwgFUoBP805BzVFKlTEdygStdTIwGTgB/Axc1FqvNZvKPpyt0MVNlFJlgZXA81rrNNN5TFFK/QE4o7WONJ3FQbgDLYAPtdbNgXSgRF5zUkr5Yv0mHwpUA8oopR43m8o+nK3Qk4HgfI+D8p4rkZRSHlhlvkhr/YXpPIa1A3oppY5hnYp7QCm10Gwko5KAJK31L7+1rcAq+JKoK3BUa52itc4CvgDuM5zJLpyt0HcDYUqpUKWUJ9aFjVWGMxmhlFJY50cPa63fNZ3HNK31S1rrIK11CNb3xQattUsehRWE1voUkKiUqpv3VBfgkMFIJp0A2iilfPJ+brrgoheI3U0HKAytdbZSajzwHdaV6rla64OGY5nSDhgG7FdK7cl77m9a6zUGMwnH8iywKO/gJwEYZTiPEVrrnUqpFUAU1uiwn3DRKQDk1n8hhHARznbKRQghxG1IoQshhIuQQhdCCBchhS6EEC5CCl0IIVyEFLoQQrgIKXQhhHAR/w+ZFZlyKmOxWAAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plot_hist(thr_score_hist)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Effnetb0 (noclip)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "from imports import *\n",
    "from utils import load_data\n",
    "from image_train.data import create_dl, ImageDS\n",
    "from image_train.model import EMBRes\n",
    "from arcface import ArcMarginProduct, compute_centers\n",
    "from image_train.train import *\n",
    "import matplotlib.pyplot as plt\n",
    "np.random.seed(1337)\n",
    "device = torch.device('cuda')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "df, train_df, val_df, train_labels, val_labels = load_data(train_perc=0.3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "# creating dataloaders\n",
    "small_images_dir_train = 'data/small_train_images_250/'\n",
    "small_images_dir_val = 'data/small_train_images_224/'\n",
    "bs = 32\n",
    "\n",
    "tr_dl = create_dl(train_df, small_images_dir_train, batch_size=bs)\n",
    "tr_test_dl = create_dl(train_df, small_images_dir_val, shuffle=False, batch_size=bs)\n",
    "val_dl = create_dl(val_df, small_images_dir_val, shuffle=False, batch_size=bs)\n",
    "#full_dl = create_dl(df, small_images_dir, shuffle=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "***Embeddings normalization is not done in the model but in the arcface metric***"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "vision_model = 'efficientnet_b0'\n",
    "model = timm.create_model(vision_model, pretrained=True, num_classes=0).to('cuda')\n",
    "train_tfms, val_tfms = get_tfms(crop=224)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "2452fba9bf36495ba3057c625a622016",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=327.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "centers = compute_centers(tr_test_dl, model, val_tfms, train_df)\n",
    "torch.save(centers, 'data/clip/centers_im_0.3_effb0.pth')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "centers = torch.load('data/clip/centers_im_0.3_effb0.pth')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Using center as wieghts\n"
     ]
    }
   ],
   "source": [
    "metric_fc = ArcMarginProduct(2048, train_df['label_group'].nunique(), \n",
    "                             s=30, m=0.5, easy_margin=False, centers=centers).to(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "n_epochs, lf, params, optimizer, sched = get_hparams(tr_dl, model, metric_fc, lr=5e-4, n_epochs=15)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "for n, _ in model.named_parameters() :\n",
    "    print(n)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "mnp = list(model.named_parameters())\n",
    "\n",
    "start = ['bn1.weight', 'bn1.bias', 'conv_stem.weight']\n",
    "param_groups = [{'params' : [p for n,p in mnp if n in start]}]\n",
    "params_names = [n for n,p in mnp if n in start]\n",
    "\n",
    "n_blocks = 7\n",
    "for i in range(n_blocks):\n",
    "    ith_block = [p for n, p in mnp if f'blocks.{i}.' in n]\n",
    "    ith_block_names = [n for n, p in mnp if f'blocks.{i}.' in n] \n",
    "    param_groups.append({'params' : ith_block})\n",
    "    params_names += ith_block_names\n",
    "    \n",
    "end = ['bn2.weight', 'bn2.bias', 'conv_head.weight']\n",
    "param_groups.append({'params' : [p for n,p in mnp if n in end]})\n",
    "params_names += [n for n,p in mnp if n in end]\n",
    "\n",
    "param_groups.append({'params' : metric_fc.parameters()})\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "for n, p in model.named_parameters() :\n",
    "    if n not in params_names :\n",
    "        print(n)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "lf = nn.CrossEntropyLoss()\n",
    "optimizer = torch.optim.AdamW(param_groups, weight_decay = 1e-5)\n",
    "\n",
    "lr = list(np.linspace(1e-5,5e-4,len(param_groups)))\n",
    "\n",
    "n_epochs = 10\n",
    "sched = torch.optim.lr_scheduler.OneCycleLR(optimizer, max_lr=lr, pct_start=0.3,\n",
    "                                            total_steps=int(n_epochs * len(tr_dl)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "loss_hist = []\n",
    "best_thr_score={'val': {'thr': None, 'f1': None}, 'train': {'thr': None, 'f1': None}}\n",
    "thr_score_hist=[]\n",
    "ep_start = 0\n",
    "save_path = 'data/clip/test_20ap_im_effb0'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=10.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=327.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Thresholds', max=10.0, style=ProgressStyle(description_wi…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Checkpoint : saved model to data/clip/test_20ap_im_effb0_ep_0.pth\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=745.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Thresholds', max=10.0, style=ProgressStyle(description_wi…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Saved best model ep 0 with f score : 0.6848369335744343\n",
      "Ep 0: Train loss 9.0092 | Val f score 0.6848 with thresh 0.91, train f score 0.6783 with thresh 1.00\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=327.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Thresholds', max=5.0, style=ProgressStyle(description_wid…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=745.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Thresholds', max=5.0, style=ProgressStyle(description_wid…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Saved best model ep 1 with f score : 0.6926233519450762\n",
      "Ep 1: Train loss 7.2866 | Val f score 0.6926 with thresh 0.91, train f score 0.7174 with thresh 1.05\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=327.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Thresholds', max=5.0, style=ProgressStyle(description_wid…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Checkpoint : saved model to data/clip/test_20ap_im_effb0_ep_2.pth\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=745.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Thresholds', max=5.0, style=ProgressStyle(description_wid…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Saved best model ep 2 with f score : 0.6998781675319034\n",
      "Ep 2: Train loss 5.7266 | Val f score 0.6999 with thresh 0.96, train f score 0.7700 with thresh 1.05\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=327.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Thresholds', max=5.0, style=ProgressStyle(description_wid…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=745.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Thresholds', max=5.0, style=ProgressStyle(description_wid…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Saved best model ep 3 with f score : 0.7069573718869007\n",
      "Ep 3: Train loss 4.2515 | Val f score 0.7070 with thresh 0.96, train f score 0.8153 with thresh 1.10\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=327.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Thresholds', max=5.0, style=ProgressStyle(description_wid…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Checkpoint : saved model to data/clip/test_20ap_im_effb0_ep_4.pth\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=745.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Thresholds', max=5.0, style=ProgressStyle(description_wid…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Saved best model ep 4 with f score : 0.7090681065426806\n",
      "Ep 4: Train loss 3.0850 | Val f score 0.7091 with thresh 0.96, train f score 0.8529 with thresh 1.10\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=327.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Thresholds', max=5.0, style=ProgressStyle(description_wid…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=745.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Thresholds', max=5.0, style=ProgressStyle(description_wid…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Saved best model ep 5 with f score : 0.7098578800574555\n",
      "Ep 5: Train loss 2.1921 | Val f score 0.7099 with thresh 0.96, train f score 0.8826 with thresh 1.10\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=327.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Thresholds', max=5.0, style=ProgressStyle(description_wid…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Checkpoint : saved model to data/clip/test_20ap_im_effb0_ep_6.pth\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=745.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Thresholds', max=5.0, style=ProgressStyle(description_wid…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Saved best model ep 6 with f score : 0.7105623222975299\n",
      "Ep 6: Train loss 1.5107 | Val f score 0.7106 with thresh 0.96, train f score 0.9088 with thresh 1.10\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=327.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Thresholds', max=5.0, style=ProgressStyle(description_wid…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=745.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Thresholds', max=5.0, style=ProgressStyle(description_wid…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Saved best model ep 7 with f score : 0.7110321206713394\n",
      "Ep 7: Train loss 1.0644 | Val f score 0.7110 with thresh 0.96, train f score 0.9277 with thresh 1.10\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=327.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Thresholds', max=5.0, style=ProgressStyle(description_wid…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Checkpoint : saved model to data/clip/test_20ap_im_effb0_ep_8.pth\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=745.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Thresholds', max=5.0, style=ProgressStyle(description_wid…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Ep 8: Train loss 0.8136 | Val f score 0.7108 with thresh 0.96, train f score 0.9385 with thresh 1.10\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=327.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Thresholds', max=5.0, style=ProgressStyle(description_wid…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=745.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Thresholds', max=5.0, style=ProgressStyle(description_wid…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Saved best model ep 9 with f score : 0.7112635714411719\n",
      "Ep 9: Train loss 0.7459 | Val f score 0.7113 with thresh 0.96, train f score 0.9450 with thresh 1.10\n",
      "\r"
     ]
    }
   ],
   "source": [
    "best_thr_score, thr_score_hist, losses = train(model, optimizer, lf, sched, metric_fc, tr_dl, val_dl,\n",
    "                                               n_epochs, train_df, val_df, train_tfms, val_tfms, \n",
    "                                               save_path=save_path, \n",
    "                                               prev_best_info=best_thr_score, info_history=thr_score_hist,\n",
    "                                               ep_start=ep_start)\n",
    "loss_hist.append(losses)\n",
    "ep_start += n_epochs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "for n, _ in model.named_parameters() :\n",
    "    print(n)\n",
    "\n",
    "mnp = list(model.named_parameters())\n",
    "\n",
    "param_groups = [{'params' : [p for n,p in mnp if 'embeddings' in n]}]\n",
    "\n",
    "params_names = []\n",
    "n_blocks = 5\n",
    "for i in range(n_blocks):\n",
    "    ith_block = [p for n, p in mnp if f'layer{i}.' in n]\n",
    "    ith_block_name = [n for n, p in mnp if f'layer{i}.' in n]\n",
    "    params_names += ith_block_name\n",
    "    param_groups.append({'params' : ith_block})\n",
    "    \n",
    "param_groups.append({'params' : [p for n,p in mnp if 'attnpool'in n]})\n",
    "params_names += [n for n,p in mnp if 'attnpool' in n]\n",
    "\n",
    "param_groups.append({'params' : metric_fc.parameters()})\n",
    "\n",
    "\n",
    "for n, p in model.named_parameters() :\n",
    "    if n not in params_names :\n",
    "        print(n)\n",
    "\n",
    "lf = nn.CrossEntropyLoss()\n",
    "optimizer = torch.optim.AdamW(param_groups, weight_decay = 1e-5)\n",
    "\n",
    "lr = list(np.linspace(1e-5,5e-4,len(param_groups)))\n",
    "\n",
    "n_epochs = 10\n",
    "sched = torch.optim.lr_scheduler.OneCycleLR(optimizer, max_lr=lr, pct_start=0.3,\n",
    "                                            total_steps=int(n_epochs * len(tr_dl)))\n",
    "\n",
    "loss_hist = []\n",
    "best_thr_score={'val': {'thr': None, 'f1': None}, 'train': {'thr': None, 'f1': None}}\n",
    "thr_score_hist=[]\n",
    "ep_start = 0\n",
    "save_path = 'data/clip/test_20ap_im_res50'\n",
    "\n",
    "best_thr_score, thr_score_hist, losses = train(model, optimizer, lf, sched, metric_fc, tr_dl, val_dl,\n",
    "                                               n_epochs, train_df, val_df, train_tfms, val_tfms, \n",
    "                                               save_path=save_path, \n",
    "                                               prev_best_info=best_thr_score, info_history=thr_score_hist,\n",
    "                                               ep_start=ep_start, half_precision=True)\n",
    "loss_hist.append(losses)\n",
    "ep_start += n_epochsloss_hist = []\n",
    "best_thr_score={'val': {'thr': None, 'f1': None}, 'train': {'thr': None, 'f1': None}}\n",
    "thr_score_hist=[]\n",
    "ep_start = 0\n",
    "save_path = 'data/test_2ap'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "fcc10b7dcf7f4f4fa28338be5366f6ba",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value=''), FloatProgress(value=0.0, max=15.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value=''), FloatProgress(value=0.0, max=160.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Thresholds'), FloatProgress(value=0.0, max=10.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Checkpoint : saved model to data/test_2ap_ep_0.pth\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value=''), FloatProgress(value=0.0, max=377.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Thresholds'), FloatProgress(value=0.0, max=10.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Saved best model ep 0 with f score : 0.6469142561779476\n",
      "Ep 0: Train loss 7.8184 | Val f score 0.6469 with thresh 0.56, train f score 0.6472 with thresh 0.73\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value=''), FloatProgress(value=0.0, max=160.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Thresholds'), FloatProgress(value=0.0, max=5.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value=''), FloatProgress(value=0.0, max=377.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Thresholds'), FloatProgress(value=0.0, max=5.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Saved best model ep 1 with f score : 0.6575995952749828\n",
      "Ep 1: Train loss 6.7015 | Val f score 0.6576 with thresh 0.61, train f score 0.7029 with thresh 0.78\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c91d745e3da24129b650e35d2d4148e2",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value=''), FloatProgress(value=0.0, max=160.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-10-fcde6b83309c>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[1;32m----> 1\u001b[1;33m best_thr_score, thr_score_hist, losses = train(model, optimizer, lf, sched, metric_fc, tr_dl, val_dl,\n\u001b[0m\u001b[0;32m      2\u001b[0m                                                \u001b[0mn_epochs\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mtrain_df\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mval_df\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mtrain_tfms\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mval_tfms\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      3\u001b[0m                                                \u001b[0msave_path\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0msave_path\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      4\u001b[0m                                                \u001b[0mprev_best_info\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mbest_thr_score\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0minfo_history\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mthr_score_hist\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      5\u001b[0m                                                ep_start=ep_start)\n",
      "\u001b[1;32m~\\Documents\\shopee\\image_train\\train.py\u001b[0m in \u001b[0;36mtrain\u001b[1;34m(model, optimizer, loss_func, sched, metric_fc, train_dl, val_dl, n_epochs, train_df, val_df, train_transforms, val_transforms, save_path, val_first, prev_best_info, info_history, ep_start, half_precision)\u001b[0m\n\u001b[0;32m    184\u001b[0m             \u001b[0mloss\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mloss_func\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mout\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mlabels\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    185\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 186\u001b[1;33m             \u001b[0mloss\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mbackward\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    187\u001b[0m             \u001b[0moptimizer\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mstep\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    188\u001b[0m             \u001b[1;32mif\u001b[0m \u001b[0msched\u001b[0m \u001b[1;32mis\u001b[0m \u001b[1;32mnot\u001b[0m \u001b[1;32mNone\u001b[0m \u001b[1;33m:\u001b[0m \u001b[0msched\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mstep\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\anaconda3\\lib\\site-packages\\torch\\tensor.py\u001b[0m in \u001b[0;36mbackward\u001b[1;34m(self, gradient, retain_graph, create_graph, inputs)\u001b[0m\n\u001b[0;32m    243\u001b[0m                 \u001b[0mcreate_graph\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mcreate_graph\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    244\u001b[0m                 inputs=inputs)\n\u001b[1;32m--> 245\u001b[1;33m         \u001b[0mtorch\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mautograd\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mbackward\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mgradient\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mretain_graph\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mcreate_graph\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0minputs\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0minputs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    246\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    247\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0mregister_hook\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mhook\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\anaconda3\\lib\\site-packages\\torch\\autograd\\__init__.py\u001b[0m in \u001b[0;36mbackward\u001b[1;34m(tensors, grad_tensors, retain_graph, create_graph, grad_variables, inputs)\u001b[0m\n\u001b[0;32m    139\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    140\u001b[0m     \u001b[0mgrad_tensors_\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0m_tensor_or_tensors_to_tuple\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mgrad_tensors\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mlen\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mtensors\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 141\u001b[1;33m     \u001b[0mgrad_tensors_\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0m_make_grads\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mtensors\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mgrad_tensors_\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    142\u001b[0m     \u001b[1;32mif\u001b[0m \u001b[0mretain_graph\u001b[0m \u001b[1;32mis\u001b[0m \u001b[1;32mNone\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    143\u001b[0m         \u001b[0mretain_graph\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mcreate_graph\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\anaconda3\\lib\\site-packages\\torch\\autograd\\__init__.py\u001b[0m in \u001b[0;36m_make_grads\u001b[1;34m(outputs, grads)\u001b[0m\n\u001b[0;32m     49\u001b[0m                 \u001b[1;32mif\u001b[0m \u001b[0mout\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mnumel\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;33m!=\u001b[0m \u001b[1;36m1\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     50\u001b[0m                     \u001b[1;32mraise\u001b[0m \u001b[0mRuntimeError\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m\"grad can be implicitly created only for scalar outputs\"\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 51\u001b[1;33m                 \u001b[0mnew_grads\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mtorch\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mones_like\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mout\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mmemory_format\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mtorch\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mpreserve_format\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     52\u001b[0m             \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     53\u001b[0m                 \u001b[0mnew_grads\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;32mNone\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "best_thr_score, thr_score_hist, losses = train(model, optimizer, lf, sched, metric_fc, tr_dl, val_dl,\n",
    "                                               n_epochs, train_df, val_df, train_tfms, val_tfms, \n",
    "                                               save_path=save_path, \n",
    "                                               prev_best_info=best_thr_score, info_history=thr_score_hist,\n",
    "                                               ep_start=ep_start)\n",
    "loss_hist.append(losses)\n",
    "ep_start += n_epochs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXsAAAD4CAYAAAANbUbJAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAgAElEQVR4nO3deXxU9bnH8c+Tfd8DxIQlKoKACBpBiyJqBdRWUbyK+9Lq7S1axVuvWu2Fi63a3ra2Vqult7RqVaSutFp3EVFUAqLs+xZCJGRPyDp57h9nEoaQkAlMOJnJ83695jVzljnzHIZ85ze/Oed3RFUxxhgT2sLcLsAYY0z3s7A3xphewMLeGGN6AQt7Y4zpBSzsjTGmF4hwu4C2MjIydNCgQW6XYYwxQWXZsmV7VTWzo+U9LuwHDRpEfn6+22UYY0xQEZHth1pu3TjGGNMLWNgbY0wvYGFvjDG9QKd99iIyF/gOsEdVR7SzXIDfARcC+4AbVXW5d9kNwAPeVX+mqk8fTpGNjY0UFBRQV1d3OE83XRATE0NOTg6RkZFul2KMCSB/fqD9K/A48EwHyy8ABntvY4EngbEikgbMBPIABZaJyAJVLetqkQUFBSQmJjJo0CCczxbTHVSVkpISCgoKyM3NdbscY0wAddqNo6qLgNJDrHIJ8Iw6PgNSRCQLmAS8q6ql3oB/F5h8OEXW1dWRnp5uQd/NRIT09HT7BmVMCApEn302sNNnusA7r6P5BxGRW0UkX0Tyi4uL230RC/qjw/6djQlNgTjOvr100EPMP3im6hxgDkBeXp6NuWxMKGhuhvoKqC2DukpQjzO/9S/c50/dd6h1bfa5eQ6cbm4+xHL1rtPmOdpMayRJGIj3Hmnz2LusZTstzztgW9rB407qbHkd39dv75bQF0Zc1i1vRyDCvgDo7zOdAxR6509oM39hAF7PGOOWugoo3wkVO5376m+cMK8tg7py72PvfV0FHbTvTEey83p02C8AbhOReTg/0Fao6m4ReRt4SERSvetNBO4LwOsFhYSEBKqrq90uwxj/qEJDNewrgZq9+8O87X19xYHPk3CITYHYVIhJgbgMSD9+/3RsqrM8JhnCfOPG+8X/gG5D2X8n4Qe2eMPC22kJi7PeQcvk4Oc7O7m/Nd7aYlef+d55rdto0wKnk9b5QXWE799Oy79x228cbW+ttQaeP4devoDTQs8QkQKcI2windr1KeBNnMMuN+EcenmTd1mpiDwILPVuaraqHuqHXnOYmpqaiIjocSNfGDe1hHf1HudWswdqimFfqRPm+0pgn/e+psS599QfvJ3oZEjpD8n9YeC3nPuU/pA8wLmPz2wT2KZDLR9ChLvy8p0mhKpe1clyBaZ3sGwuMPfwSmvf//xjNWsKKwO5SYYdk8TM7w4/5Dr33HMPAwcO5Ic//CEAs2bNQkRYtGgRZWVlNDY28rOf/YxLLrmk09fbvXs3V155JZWVlTQ1NfHkk09y1lln8dZbb/GTn/wEj8dDRkYG77//PqWlpdx8881s2bKFuLg45syZw8iRI5k1axaFhYVs27aNjIwMnn32We69914WLlxIfX0906dP59///d8D8u9jeqCaEtizBvZugKoipzulpnh/sFcXQ1Nt+8+NToa4NIjPgKRs6HcyxKdDXLrTMo9Lh+QcJ8xjko/ufpluY81BP02bNo0777yzNeznz5/PW2+9xYwZM0hKSmLv3r2cfvrpXHzxxZ0e0fL8888zadIk7r//fjweD/v27aO4uJhbbrmFRYsWkZubS2mp8yVo5syZjB49mtdee40PPviA66+/nhUrVgCwbNkyFi9eTGxsLHPmzCE5OZmlS5dSX1/PuHHjmDhxoh0vH+zqKqB4vRPse9Z679c5gd5CwpyAju8DCZmQfpzT4k7os39efB9nXlw6RES5tz/GNUEX9p21wLvL6NGj2bNnD4WFhRQXF5OamkpWVhYzZsxg0aJFhIWFsWvXLr755hv69et3yG2ddtpp3HzzzTQ2NjJlyhRGjRrFwoULGT9+fGs4p6WlAbB48WJefvllAM4991xKSkqoqHD6TS+++GJiY2MBeOedd/j666956aWXAKioqGDjxo0W9sGksQ4KlsLWRVD4pRPulQX7l0fGQ5+hMHgi9DnRuWUOhcR+Tn+xMYcQdGHvpssvv5yXXnqJoqIipk2bxnPPPUdxcTHLli0jMjKSQYMG+XVC0vjx41m0aBFvvPEG1113HXfffTcpKSntfiNQPfhohpb14uPjD1jv97//PZMmTTqCPTRHVVMDFC6HrR/D1o9g5xdOv7mEQZ9hTh95nxOdx32GOv3kYTaclTk8FvZdMG3aNG655Rb27t3LRx99xPz58+nTpw+RkZF8+OGHbN9+yOGkW23fvp3s7GxuueUWampqWL58Offffz/Tp09n69atrd04aWlpjB8/nueee46f/vSnLFy4kIyMDJKSkg7a5qRJk3jyySc599xziYyMZMOGDWRnZx/wgWBc5mmCoq+clvvWj2HHZ9BY4yzrexKc9n3IPcsJeesrNwFmYd8Fw4cPp6qqiuzsbLKysrjmmmv47ne/S15eHqNGjWLo0KF+bWfhwoX87//+L5GRkSQkJPDMM8+QmZnJnDlzuOyyy2hubqZPnz68++67zJo1i5tuuomRI0cSFxfH00+3P5bc97//fbZt28Ypp5yCqpKZmclrr70WyN03h6PZA9s+hpV/h7X/8B57jtP9MupqyB0Pg850fjA1phtJe90EbsrLy9O2V6pau3YtJ554oksV9T72732EVGHXcifgV7/iHCkTlQgnfgeO/zYMOgsS+7pdpQkxIrJMVfM6Wm4te2MCpXg9rHzJCfmyrRAeBSdMghGXO/eRsW5XaHoxC/tutHLlSq677roD5kVHR/P555+7VJEJuPKdTut95d+haKXz42rueBj/Yxj6HefsUWN6AAv7bnTSSSe1HhNvQkjlbljzGqx6BQq+cOZl58HkX8DwS62LxvRIFvbG+KO62An41a/C9k8BdY6gOe+/nYBPO9btCo05JAt7YzqyrxTWLnBa8Ns+dgaqyhgCE+5zAj7zBLcrNMZvFvbGtFW+E978MWx6D5qbnFb7mXc5Q8/2GWYDf5mgZGFvjK+tH8PfbwBPI5wxHYZfBlknW8CboGfnXvupvLycP/zhD11+3oUXXkh5eXk3VGQCShU+nwPPXAKxaXDLB3D+bDhmlAW9CQkW9n7qKOw9Hs8hn/fmm2+SkuL+4Xed1dmrNdbB67fBv+52Bhm75X3IGOx2VcYEVPB14/zrXud45kDqdxJc8MghV7n33nvZvHkzo0aNah3mICsrixUrVrBmzRqmTJnCzp07qaur44477uDWW28FYNCgQeTn51NdXc0FF1zAmWeeyaeffkp2djavv/5666iVbT322GM89dRTREREMGzYMObNm0d1dTW33347+fn5iAgzZ85k6tSpvPDCCzz00EOoKhdddBG/+MUvAOdqWXfddRdvv/02v/71r4mNjeWuu+6iurqajIwM/vrXv5KVlRXYf8tgU1kIL14Lu5bB2ffA2ffaYGMmJAVf2LvkkUceYdWqVaxYsYKFCxdy0UUXsWrVqtYhhOfOnUtaWhq1tbWcdtppTJ06lfT09AO2sXHjRl544QX+9Kc/ccUVV/Dyyy9z7bXXdvh6W7duJTo6urUb6MEHHyQ5OZmVK50Pu7KyMgoLC7nnnntYtmwZqampTJw4kddee40pU6ZQU1PDiBEjmD17No2NjZx99tm8/vrrZGZm8uKLL3L//fczd25Ary0TXHZ8Bi9eB4374Mq/wYnfdbsiY7pN8IV9Jy3wo2XMmDEHjBX/2GOP8eqrrwKwc+dONm7ceFDY5+bmMmrUKABOPfVUtm3b1uH2R44cyTXXXMOUKVOYMmUKAO+99x7z5s1rXSc1NZVFixYxYcIEMjMzAbjmmmtYtGgRU6ZMITw8nKlTpwKwfv16Vq1axfnnnw843Tq9ulWf/xd4827nakw3LHCGEjYmhAVf2PcQvkMHL1y4kPfee48lS5YQFxfHhAkT2h3XPjo6uvVxeHg4tbUdXDYOeOONN1i0aBELFizgwQcfZPXq1ajqQWPeH2ogu5iYGMLDw1vXGz58OEuWLPF7H0NSUwP8679g2V+cQcmm/p9zUWxjQpxfnZMiMllE1ovIJhG5t53lA0XkfRH5WkQWikiOzzKPiKzw3hYEsvijKTExkaqqqnaXVVRUkJqaSlxcHOvWreOzzz47otdqbm5m586dnHPOOfzyl7+kvLyc6upqJk6cyOOPP966XllZGWPHjuWjjz5i7969eDweXnjhBc4+++yDtjlkyBCKi4tbw76xsZHVq1cfUZ1Bp7IQnv6uE/RnzoCr51vQm16j05a9iIQDTwDnAwXAUhFZoKprfFb7FfCMqj4tIucCDwMtI4DVquqoANd91KWnpzNu3DhGjBhBbGwsffvuH/9k8uTJPPXUU4wcOZIhQ4Zw+umnH9FreTwerr32WioqKlBVZsyYQUpKCg888ADTp09nxIgRhIeHM3PmTC677DIefvhhzjnnHFSVCy+8sN2LnkdFRfHSSy/xox/9iIqKCpqamrjzzjsZPtydyzweVUUr4bMnncHKwiLg8r84J0gZ04t0Op69iJwBzFLVSd7p+wBU9WGfdVYDk1S1QJx+hgpVTfIuq1bVBH8LsvHs3RcS/97NzbDhLfjsD85QB5FxMOoaOOOHNo6NCUmBGM8+G9jpM10AjG2zzlfAVOB3wKVAooikq2oJECMi+UAT8IiqHnT5JBG5FbgVYMCAAX6UZEwH6qthxXPw+VNQugWScpyTo0653rpsTK/mT9i3d/pg268DPwYeF5EbgUXALpxwBxigqoUicizwgYisVNXNB2xMdQ4wB5yWfRfqD3rTp0/nk08+OWDeHXfcwU033eRSRUGqfAd8MQeWPQP1FZBzGpz7U+dwyvBIt6szxnX+hH0B0N9nOgco9F1BVQuBywBEJAGYqqoVPstQ1S0ishAYDRwQ9v5o70iUUPDEE0+4XcIBetplKjtVVwH/nOEMPYzAsEvg9B9C/9PcrsyYHsWfsF8KDBaRXJwW+zTgat8VRCQDKFXVZuA+YK53fiqwT1XrveuMA37Z1SJjYmIoKSkhPT09JAO/p1BVSkpKiImJcbsU/9RVwLOXwu6v4YzbYMytznHzxpiDdBr2qtokIrcBbwPhwFxVXS0is4F8VV0ATAAeFhHF6caZ7n36icAfRaQZ5zDPR9ocxeOXnJwcCgoKKC4u7upTTRfFxMSQk5PT+Ypu8w36K56GoRe5XZExPVqnR+Mcbe0djWPMAeoq4NnLYPdXFvTGeAXiaBxjeo7WoF8BVzxjQW+Mn2x4PxM86irhb1OdoP83a9Eb0xUW9iY41FXC3y6Dwi+doD/xO25XZExQsbA3Pd8BQf9XC3pjDoOFvenZWrpuCr90xrSxMeeNOSwW9qbnag365U7QD7vY7YqMCVoW9qZnqquE5y53Lhd4+VwLemOOkB16aXqW5mZY/Qq8PxsqCuDf/uIMgWCMOSIW9qbn2PoxvPtTp3++70lw/euQe5bbVRkTEizsjfv2rIX3ZjnjzyflwJSnYOQVEBbudmXGhAwLe+Oeyt2w8CH48m8QlQDfngVjfwCRsW5XZkzIsbA3R199FXzyGCx5HDyNTsCf9WOIT3e7MmNCloW9OXpUnYt9f/gQ1BTD8MvgvJ/aZQKNOQos7M3R4WmEf9zhXDJw4Di46kXIOdXtqozpNSzsTfdrqIG/3wgb34EJ98HZ94BdhMaYo8rC3nSvmhJ4/grnLNjvPAp5N7tdkTG9koW96T5l253hDsp3wBXP2gBmxrjIwt50j6JVTtA31TonRw08w+2KjOnVbGwcE3hbP4a/XAASBje9ZUFvTA/gV9iLyGQRWS8im0Tk3naWDxSR90XkaxFZKCI5PstuEJGN3tsNgSze9ECrX3PGnk/Mgu+9A32HuV2RMQY/wl5EwoEngAuAYcBVItL2L/hXwDOqOhKYDTzsfW4aMBMYC4wBZopIauDKNz3KF39yjrrJGgU3vwUp/d2uyBjj5U/LfgywSVW3qGoDMA9oOwzhMOB97+MPfZZPAt5V1VJVLQPeBSYfedmmR1GF9x+EN38MJ0x2+ujj0tyuyhjjw5+wzwZ2+kwXeOf5+gqY6n18KZAoIul+PhcRuVVE8kUkv7i42N/aTU/xwc/g41/B6Ovgyr9BVJzbFRlj2vAn7Ns7+0XbTP8YOFtEvgTOBnYBTX4+F1Wdo6p5qpqXmZnpR0mmx9izFhY/CidfBRf/HsLtAC9jeiJ//jILAN/O1xyg0HcFVS0ELgMQkQRgqqpWiEgBMKHNcxceQb2mJ1GFN++G6ESY+HM7K9aYHsyflv1SYLCI5IpIFDANWOC7gohkiEjLtu4D5nofvw1MFJFU7w+zE73zTChY/Sps+9gZzMxGrDSmR+s07FW1CbgNJ6TXAvNVdbWIzBaRlguDTgDWi8gGoC/wc+9zS4EHcT4wlgKzvfNMsKuvhncegH4nwak3uV2NMaYTonpQF7qr8vLyND8/3+0yTGfem+X01d/8DgwY63Y1xvR6IrJMVfM6Wm5n0Jqu27sJPn0cTr7agt6YIGFhb7pGFf71X86lA8//H7erMcb4ycLedM26N2Dz+3DOTyChj9vVGGP8ZGFv/NdYC2/dB32GwWm3uF2NMaYL7AwY47/Fj0LFDrjxDTt5ypggYy1745/SrbD4tzDichh0ptvVGGO6yMLe+Oftn0B4JEz8mduVGGMOg4W96dyGd2D9m3D2f0FSltvVGGMOg4W9ObTGOudQy4wTYOx/uF2NMeYw2a9s5tCWPA5lW+G6VyEiyu1qjDGHyVr2pmPlO2HRr+DEi+G4c92uxhhzBCzsTcfeud+5n/SQu3UYY46Yhb05mCp89EtY8zqM/0+7lqwxIcD67M2BVOH92bD4N85AZ2fe5XZFxpgAsLA3+6k6x9N/9gdnjPqLfgNh9uXPmFBgYW8czc3wxl2w7C/OIZaTH7bLDBoTQizsDTR7YMHtsOI5OHMGnDfTgt6YEGNh39t5GuGVW2H1K3DO/TD+bgt6Y0KQhX1v1lQPL90M6/4J58+GcXe4XZExppv49eubiEwWkfUisklE7m1n+QAR+VBEvhSRr0XkQu/8QSJSKyIrvLenAr0D5jA11sK8a5ygv+B/LeiNCXGdtuxFJBx4AjgfKACWisgCVV3js9oDwHxVfVJEhgFvAoO8yzar6qjAlm2OSEMNvDANtn4M330MTr3B7YqMMd3Mn5b9GGCTqm5R1QZgHnBJm3UUSPI+TgYKA1eiCai6Snj2Mti2GC79owW9Mb2EP2GfDez0mS7wzvM1C7hWRApwWvW3+yzL9XbvfCQiZ7X3AiJyq4jki0h+cXGx/9Wbrmmogecuh135cPlcOPlKtysyxhwl/oR9e4dmaJvpq4C/qmoOcCHwrIiEAbuBAao6GrgLeF5Ekto8F1Wdo6p5qpqXmZnZtT0w/mmqhxevhYKlMPXPMPxStysyxhxF/oR9AeA7OEoOB3fTfA+YD6CqS4AYIENV61W1xDt/GbAZOOFIizZd5GmCl78Hmz+Ai38Pw6e4XZEx5ijzJ+yXAoNFJFdEooBpwII26+wAzgMQkRNxwr5YRDK9P/AiIscCg4EtgSre+KG5Gf7xI1j7D5j8CIy+1u2KjDEu6PRoHFVtEpHbgLeBcGCuqq4WkdlAvqouAP4T+JOIzMDp4rlRVVVExgOzRaQJ8AA/UNXSbtsbcyBVePs+58zYCffB6XalKWN6K1Ft2/3urry8PM3Pz3e7jNDw4UPw0S/g9Okw6ed2ZqwxIUxElqlqXkfLbUjDUPXp407Qj77Wgt4YY2EfkpY/41xlatglzklTFvTG9HoW9qFm9avwjzvg+G/DZf8HYeFuV2SM6QEs7EPJxnfh5Vug/1i44lmIiHK7ImNMD2FhHyq2fwovXgd9ToSrX4SoOLcrMsb0IBb2oWDnUnj+SufC4Ne9CjHJbldkjOlhLOyD3Y7P4NlLIS4drnsN4jPcrsgY0wNZ2AezbZ84I1gm9IGb3oTktuPTGWOMw8I+WG392BnBMjnbCfqkY9yuyBjTg1nYB6MtC+G5f4OUAXDjG5DYz+2KjDE9nIV9sNn0vvNjbNqxcMM/nS4cY4zphIV9MNnwDrxwFaQPhhv+AQk29r8xxj8W9sFi/b/gxWugz1C4YQHEp7tdkTEmiFjYB4O1/3ROmOo7HK5/HeLS3K7IGBNkLOx7ujWvw99vgKyTnePoY1PdrsgYE4Qs7Huy1a/C32+C7FOdM2NjU9yuyBgTpDq9UpVxyfp/wcvfh5zT4NqXIDrR7YqMMUHMWvY90eYPYf710O8kuObvFvTGmCNmYd/TbF8C8652Dq+89hWISXK7ImNMCPAr7EVksoisF5FNInJvO8sHiMiHIvKliHwtIhf6LLvP+7z1IjIpkMWHnF3LnDNjk7Lh+tfsqBtjTMB02mcvIuHAE8D5QAGwVEQWqOoan9UeAOar6pMiMgx4ExjkfTwNGA4cA7wnIieoqifQOxL0ilY5g5rFpTmHV9qZscaYAPKnZT8G2KSqW1S1AZgHXNJmHQVa+huSgULv40uAeapar6pbgU3e7RlfxRvg2SkQGeecMGWjVxpjAsyfsM8GdvpMF3jn+ZoFXCsiBTit+tu78FxE5FYRyReR/OLiYj9LDxFl2+CZS0DVadGnDnK7ImNMCPIn7KWdedpm+irgr6qaA1wIPCsiYX4+F1Wdo6p5qpqXmdmLxnup2AVPXwyN+5ygzzzB7YqMMSHKn+PsC4D+PtM57O+mafE9YDKAqi4RkRggw8/n9k7Ve5wW/b5SuOF16DfC7YqMMSHMn5b9UmCwiOSKSBTOD64L2qyzAzgPQEROBGKAYu9600QkWkRygcHAF4EqPmjtK4VnpkBFAVwz3zlD1hhjulGnLXtVbRKR24C3gXBgrqquFpHZQL6qLgD+E/iTiMzA6aa5UVUVWC0i84E1QBMwvdcfidNYB3+bCiUb4eoXYeC33K7IGNMLiJPJPUdeXp7m5+e7XUb3+fBh+OgRuOJZGHax29UYY0KEiCxT1byOltsZtEfT3o2w+Dcw4nILemPMUWVhf7Sowht3QUQsTHrI7WqMMb2MjXp5tHz9ImxdBBf9GhL7ul2NMaaXsZb90bCvFN6+H7Lz4NSb3a7GGNMLWcv+aHhvJtSWOYObhdnnqzHm6LPk6W47PoPlz8Dp/+GMT2+MMS6wsO9Onkb45wxIyoEJ97ldjTGmF7NunO605HHYswamvQDRCW5XY4zpxaxl313KtsHCX8CQi2DohZ2ubowx3cnCvjuowpt3g4TBhb90uxpjjLGw7xZrXoeN78A5P4HkHLerMcYYC/uAq6uEt+51jrwZ+wO3qzHGGMB+oA28D38OVUVw5XMQbv+8xpiewVr2gVT4JXwxB077HuTYGPXGmJ7Dwj5Qmj3wjzshPhPO+2+3qzHGmANYP0Og5M+F3Stg6p8hJtntaowx5gDWsg+Epnr4+NcwcByMmOp2NcYYcxAL+0D4ej5U7Yaz7gIRt6sxxpiDWNgfqeZm+OR3zqGWx53ndjXGGNMuv8JeRCaLyHoR2SQi97az/FERWeG9bRCRcp9lHp9lCwJZfI+w/g3n4uHj7rRWvTGmx+r0B1oRCQeeAM4HCoClIrJAVde0rKOqM3zWvx0Y7bOJWlUdFbiSexBVWPwopAyEYVPcrsYYYzrkT8t+DLBJVbeoagMwD7jkEOtfBbwQiOJ6vG2LYdcy+NbtdgKVMaZH8yfss4GdPtMF3nkHEZGBQC7wgc/sGBHJF5HPRKTd5q+I3OpdJ7+4uNjP0nuAT34LcRkw+lq3KzHGmEPyJ+zb64jWDtadBrykqh6feQNUNQ+4GvitiBx30MZU56hqnqrmZWZm+lFSD1C0Eja9B6f/ACJj3a7GGGMOyZ+wLwD6+0znAIUdrDuNNl04qlrovd8CLOTA/vzgtfi3EJUAp33f7UqMMaZT/oT9UmCwiOSKSBROoB90VI2IDAFSgSU+81JFJNr7OAMYB6xp+9ygU7YNVr8Cp94IsaluV2OMMZ3q9FdFVW0SkduAt4FwYK6qrhaR2UC+qrYE/1XAPFX17eI5EfijiDTjfLA84nsUT9D69HGQcDhjutuVGGOMX/w6hERV3wTebDPvv9tMz2rneZ8CJx1BfT1PdTF8+SycfCUkHeN2NcYY4xc7g7arvvijMxbOt+5wuxJjjPGbhX1X1Fc549UPvQgyT3C7GmOM8ZuFfVcsexrqKpyhEYwxJohY2PurqQGWPAEDz4T+p7ldjTHGdImFvb9WzoeqQjhzRufrGmNMD2Nh74+WYYz7ngTH2zDGxpjgY2Hvjw3/gr0bYNwdNoyxMSYoWdh3pnUY4wEw/FK3qzHGmMNiYd+Z7Z9CwVL41o9sGGNjTNCysO/M4kedYYxHXeN2JcYYc9gs7A+l8EvY9C6M/QFExbldjTHGHDYL+0P54GcQkwJjb3W7EmOMOSIW9h3ZvsS5OMmZd0JMstvVGGPMEbGwb48qfPAgxPeBMdaqN8YEPwv79mz+ALZ/AuN/DFHxbldjjDFHzMK+rZZWfXJ/50pUxhgTAizs21r3hnMUztn/BRHRbldjjDEBYWHvq9kDH/4c0o6Dk692uxpjjAkYOyXU16pXYM8amPpnO1vWGBNS/GrZi8hkEVkvIptE5N52lj8qIiu8tw0iUu6z7AYR2ei93RDI4gPK0wgLH4I+w2H4ZW5XY4wxAdVp81VEwoEngPOBAmCpiCxQ1TUt66jqDJ/1bwdGex+nATOBPECBZd7nlgV0LwJhxfNQugWmPQ9h1rtljAkt/qTaGGCTqm5R1QZgHnDJIda/CnjB+3gS8K6qlnoD/l1g8pEU3C2a6uGjX0L2qTDkQrerMcaYgPMn7LOBnT7TBd55BxGRgUAu8EFXnisit4pIvojkFxcX+1N3YOX/BSoL4NwHbLx6Y0xI8ifs20s/7WDdacBLqurpynNVdY6q5qlqXmZmph8lBVBDDXz8K+fasseec3Rf2xhjjhJ/wr4A6O8znZt2xdwAABAzSURBVAMUdrDuNPZ34XT1ue74/I9QUwzn/dRa9caYkOVP2C8FBotIrohE4QT6grYricgQIBVY4jP7bWCiiKSKSCow0TuvZ6gtd64te/z5MOB0t6sxxphu0+nROKraJCK34YR0ODBXVVeLyGwgX1Vbgv8qYJ6qqs9zS0XkQZwPDIDZqloa2F04AkuegLpyp6/eGGNCmPhkc4+Ql5en+fn53f9CNXvhdyfDcefClc92/+sZY0w3EpFlqprX0fLee5ro4kedH2fPud/tSkwvoKoUlNUSGxVOWlwUYWHB+fuQqrK3uoGte2vYUlzNvgYPKXGRpMRFkhwbRUpcJKlxUSTFRBAR3vXzVVoan2K/nwVc7wz7yt2w9P9g5JXQZ6jb1ZgQtbuilsUb9/LJpr18srmE4qp6ACLChIyEaDITo+mTGE2fpGgyE2Ocx4nR9EmKoX9qLOkJgRmIr6GpmfVFVeytqSc2Mty5Re2/j4sKJyYi/IAPoNoGD9tKathS7IT61r01bPYGfFVdk1+vmxgT4RP+kXialfomDw2eZhqafG6eZup9HqtCVHgYkeFCRHgYkeFhRIULkRHO44gwISrCuQfn8D5V72F+qj7T6txryzpKszrzWu4VDp7n/cDx3Q74bqdlD/WAbR/wHJ9pgMiW/QkLIyrCeRzp3Tffx4P7JPDAd4Ydztvcqd4Z9osfheYmmHDQyA8mwOoaPazYWc7nW0r5fGsJKwsqyEyKZkjfRIb0S2y9H5geT3iQtnZbVNQ28tmWEj7ZtJfFm/aypbgGgPT4KMYdn8GY3DSaPM3sqapvve0qr+WrgnJKahpo26PaNymaEcckM/yYJIZ573NSYw/Z6vU0K5uLq/lqZzlfF1TwdUE5a3dX0eBp7rT+6Igw4qLCCQ8LY291/QHLspJjODYznimjsjk2M57cjHiOy0wgMSaCitpGyvY1Ur6vwXlc00B5bSPl3nnltY1U1jYSHibERoWTHB5JVEQYURHhRIU74Rcd4dxHhYchAo0epdHTTJOnmQafx40epcHT7J1u+RaA914Q77S0mQYhPAwEISxs/7IwEUT23wvS+nxatyU+r+Fsq3W6vXU4+JuJp9nZhwbvPjR596HBozR6P+hqGjyU7Wvs9H06XL0v7PeVwpfPwklXQFqu29X0CA1Nzewqr2VH6T52lNQ496X7KCyvIy0+igFpcQxMj2NAWhwDvPdxUe3/16lt8LB8Rxmfbynhs62lrNhZTkNTMyIwtF8S3x11DCXV9awrquKt1UWtARcdEcbgvgmc0Hf/B8DxfRLolxRzWN0B4LSuiirrWFdUxbrdVawvqqS6vom+STFkJcfQLzmWfkkx9Et2puOjD/3noKrUNjp/kGU1DZTta6BsXyMbiqpYvGkvXxeU06wQFxXOmNw0rh4zgHHHZzCkb2Kn3TaNnmZKqhvYU1XHnsp6tpXUsGpXBasLK/lw/R6avf9OybGRDMtKYvgxSQzPTuK4zAS2lezja2+4ryqsYF+Dc5pLQnQEI7KTuHHcIEbmJHNMSix1DR5qG723hvbvG5qaOSYlltyM+NZg7+j9BkiJi2JgetfeG3P09b4faBf9yrk4yX98Cn2Hd9/r9DDNzU6f8ZrdlWwurmZHyb7WUN9dUdsaJuAE74C0OLJSYimtqWd7yb6DvrpnJETv/wBIi6PR08znW0v5uqCcRo8SJjD8mGTG5qYx9th0xgxKIzku8oBt1DZ42LSnmnVFlWz4pop1RVVs+KaKbyr3tyrDw4R+STFkp8aSkxpLTkosOalxrdNZybFERYRRXd/E+qIq1hdVsa6oknXexxW1+1tKWckxJMdG8k1lXbstqMSYCLKSY+ibFENmYjR1jR7Kahq9oe4Ee0PTwS3k8DBhVP8Uxh2fwZnHZzCqfwpREYEbX6m2wcO6okpWFzq3NYUVrCuqot6nlqiIMIZlJXFyTjIjc1I4uX8yx2YkBO1vA6brOvuBtneFfVM9PDoC+p0E173SPa8RIE2eZtbsruSbynrS4p1+z/T4aBJjIjr9A97X0NTaml27u5K1u53wq67fH9i+Yd3fG9gt05kJ0Qe8hqpSUdvI9pJ9bC/dx87SfWxv+QZQso/dlXWEiXBSdjJjj03j9Nx0Th2USlJMZHvldaqspoEN31SxZW8Nu8pq2VVey66yWgrK9lFUWXfAB5MIpMZFUVrT0DovITqCE/omMDQriaH9EhnaL4khfRMP+LCpa/TwTWUduyvqKKpoua+lqNKZLq6qJzYqnNS4KFLiolrfg5S4KFLjIlvv0+KjyEqJJaGTbwWB1uRpZnNxDZuLqxmQFseQfolEHuY3IBMaLOx9LX8WFtwG170Gx/WsoRHqGj18uaOcpdtKWbqtlOXby6hp8By0XniYtIZMalwUafHOLTEmkh2lNazdXcW2kprW7pHE6AiGZiVyYlZS621wn4ROuyy6WrsqxEaFB2ybHWn0NFNUUUeB90OgoGwf31TWk50Sw5B+Trh31q9tTCiyQy9bqMKSx6HvCDh2gtvVUFHbyLLtpXyxtYyl2/Z3fwAM7ZfIZafkcFpuGgPT4iivbaS0pp7SGqevuKSmgbKaBkprGti4p5rSGueHseyUWE7MSuSSUcdwYlYSw7I6/0EvEGIiuz/kW0SGh9Hf+23EGOO/3hP2m96D4nVw6R+P+hg4qsrWvTUs31HOlzvKWL6jnHVFlag6h+GdlJPMzeNyOW1QGnmDUkmJizqs17DWrDGmI70n7D99DBKPOSpXoaqqa+SrnRUs31HGlzvK+HJnOeXeHwQToiMY1T+FO84bzJhBaYwakHLIIx38ZUFvjDmU3hH2u7+CrYvg2/8DEV1vNR9Ky5mRn28tJX9bKV/uKGfDnqrWPvPBfRKYNKwfowekcMrAVI7LTAj648mNMcGnd4T9p49DVAKceuMRb0pV2VxcwxdbnZOEvthayu6KOsA5Bnr0gBQuPCmLUwamMDInheTYwzsixRhjAin0w76iAFa9DGN/ALEpXX56c7OyrqiKL7aW8MW2Ur7YWsreaucwv8zEaMbkpnF6bhpjctMZ3MeOazbG9EyhH/afPencn/4Dv1avrm/iq53lLN9exrIdZSzfXkal94Si7JRYxg/OZIz3RKFB6XHWV26MCQqhHfZ1FbDsaRg+BVIGHLS4pb992fay1tu6okqa1TlgZ3CfBC48KYsxuWmMyU0jJ9UO9zPGBKfQDvvlz0BDFZxxW+usbyrreGfNN3yycS/LdpS1jkQYHxXO6AGp3HbuYE4dmMqo/tbfbowJHaEb9p5Gpwtn0FlsjR7C2x9t5u3VRXy5oxyA/mmxnHl8BqcMTOXUAakM6ZdoR8kYY0JWSIa9qlLw8XP0r9zFT5tu5tlfLQTgpOxkfjzxBCYN78fxfRKsv90Y02uETNg3eZpZuq2Mt1cX8e7qIv5Y+yj1HMOm5DOYOe4YJg7vR3ZKrNtlGmOMK/wKexGZDPwO54Lj/6eqj7SzzhXALJyLs3ylqld753uAld7VdqjqxQGo+yBFlXVc9afPiIoI49acnYyo30bNpN/wwhnjuuPljDEmqHQa9iISDjwBnA8UAEtFZIGqrvFZZzBwHzBOVctEpI/PJmpVdVSA6z5ITmocT988hryBqcT/fRrEZxKfd013v6wxxgQFfwbAHgNsUtUtqtoAzAMuabPOLcATqloGoKp7Alumf84+IZP4io2w6V0YcytExrhRhjHG9Dj+hH02sNNnusA7z9cJwAki8omIfObt9mkRIyL53vlT2nsBEbnVu05+cXFxl3bgIEseh4hYyPvekW3HGGNCiD999u0dstL2iicRwGBgApADfCwiI1S1HBigqoUicizwgYisVNXNB2xMdQ4wB5yLl3RxH/arKoKv58Mp10O8XRTTGGNa+NOyLwD6+0znAIXtrPO6qjaq6lZgPU74o6qF3vstwEJg9BHW3LEv5jjH15/+w257CWOMCUb+hP1SYLCI5IpIFDANWNBmndeAcwBEJAOnW2eLiKSKSLTP/HHAGrpDQw0s/TMMvQjSj+uWlzDGmGDVaTeOqjaJyG3A2ziHXs5V1dUiMhvIV9UF3mUTRWQN4AHuVtUSEfkW8EcRacb5YHnE9yiegKqrdC43aK16Y4w5SO+64LgxxoSozi447k83jjHGmCBnYW+MMb2Ahb0xxvQCFvbGGNMLWNgbY0wvYGFvjDG9gIW9Mcb0Ahb2xhjTC/S4k6pEpBjYfgSbyAD2BqicniDU9gdCb59CbX8g9PYp1PYHDt6ngaqa2dHKPS7sj5SI5B/qLLJgE2r7A6G3T6G2PxB6+xRq+wNd3yfrxjHGmF7Awt4YY3qBUAz7OW4XEGChtj8QevsUavsDobdPobY/0MV9Crk+e2OMMQcLxZa9McaYNizsjTGmFwiZsBeRySKyXkQ2ici9btcTCCKyTURWisgKEQm6K7qIyFwR2SMiq3zmpYnIuyKy0Xuf6maNXdXBPs0SkV3e92mFiFzoZo1dISL9ReRDEVkrIqtF5A7v/KB8nw6xP8H8HsWIyBci8pV3n/7HOz9XRD73vkcvei8b2/F2QqHPXkTCgQ3A+TgXP18KXNVtl0A8SkRkG5CnqkF5MoiIjAeqgWdUdYR33i+BUlV9xPuhnKqq97hZZ1d0sE+zgGpV/ZWbtR0OEckCslR1uYgkAsuAKcCNBOH7dIj9uYLgfY8EiFfVahGJBBYDdwB3Aa+o6jwReQr4SlWf7Gg7odKyHwNsUtUtqtoAzAMucbmmXk9VFwGlbWZfAjztffw0zh9i0Ohgn4KWqu5W1eXex1XAWiCbIH2fDrE/QUsd1d7JSO9NgXOBl7zzO32PQiXss4GdPtMFBPkb7KXAOyKyTERudbuYAOmrqrvB+cME+rhcT6DcJiJfe7t5gqLLoy0RGQSMBj4nBN6nNvsDQfweiUi4iKwA9gDvApuBclVt8q7SaeaFSthLO/OCv38KxqnqKcAFwHRvF4LpeZ4EjgNGAbuBX7tbTteJSALwMnCnqla6Xc+Ramd/gvo9UlWPqo4CcnB6Mk5sb7VDbSNUwr4A6O8znQMUulRLwKhqofd+D/Aqzpsc7L7x9qu29K/ucbmeI6aq33j/GJuBPxFk75O3H/hl4DlVfcU7O2jfp/b2J9jfoxaqWg4sBE4HUkQkwruo08wLlbBfCgz2/jodBUwDFrhc0xERkXjvD0yISDwwEVh16GcFhQXADd7HNwCvu1hLQLSEotelBNH75P3x78/AWlX9jc+ioHyfOtqfIH+PMkUkxfs4Fvg2zm8RHwKXe1fr9D0KiaNxALyHUv0WCAfmqurPXS7piIjIsTiteYAI4Plg2ycReQGYgDMU6zfATOA1YD4wANgB/JuqBs0Pnh3s0wSc7gEFtgH/3tLf3dOJyJnAx8BKoNk7+yc4/dxB9z4dYn+uInjfo5E4P8CG4zTQ56vqbG9GzAPSgC+Ba1W1vsPthErYG2OM6ViodOMYY4w5BAt7Y4zpBSzsjTGmF7CwN8aYXsDC3hhjegELe2OM6QUs7I0xphf4f5NpmTphWmpDAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXsAAAD7CAYAAACL+TRnAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAgAElEQVR4nO3de3RV9Z338feXEAg3IZCoSLhEiojIPQNOddTWalGreMEOYDvi6gzTLunjOJfWPtNHLTOu8el0preHacdxAGtFtICCDq1Vq9NOq8AJd1AEEZIQLpFLALkl4fv8sXfiISTkJDnJzjnn81qLxdm3X77bI182v+/+/X7m7oiISHrrFHUAIiLS9pTsRUQygJK9iEgGULIXEckASvYiIhlAyV5EJAMklOzNbLKZbTWz7Wb2cAPHB5vZG2a2wczeMrOCuGM1ZrYu/LU8mcGLiEhirKn37M0sC3gfuBEoA1YD0919S9w5vwBecfenzeyzwP3u/uXw2DF379lWNyAiIk3rnMA5E4Ht7r4DwMwWAVOALXHnXAE8FH5+E3ippQHl5eX5kCFDWnq5iEhGKi4u/sjd8xs7nkiyHwCUxm2XAZPqnbMeuBv4IXAn0MvM+rn7ASDHzGJANfCEu5/3L4IhQ4YQi8USCEtERGqZ2a7zHU+kz94a2Fe/7+dvgevMbC1wHbCbILkDDHL3ImAG8AMzG9pAkLPMLGZmsYqKigRCEhGR5kgk2ZcBA+O2C4Dy+BPcvdzd73L3ccDfh/sqa4+Fv+8A3gLG1f8B7v6kuxe5e1F+fqP/ChERkRZKJNmvBoaZWaGZdQGmAWe9VWNmeWZW29a3gHnh/lwz61p7DnA1Z/f1i4hIO2gy2bt7NTAbeBV4F3jB3Teb2Rwzuz087Xpgq5m9D1wEPB7uHwHEzGw9QeH2ifi3eEREpH00+epleysqKnIVaEVEmsfMisP6aIM0glZEJAMo2YuIZIBE3rOXTHTmDKx6Eo4fiDoSyTSDJsGnPpe89nYXw+mPofDa5LVZshK2v5689mpdcAkU3Z/8dlGyl8Z88Ab86pvhRkNDLUTagkPX3vA370GX7q1v7swZWPwVOHEoaDO7W3LaXPoXcHgXSf+zUVCkZC/trHgB9MiHh7ZA5y5RRyOZYuf/wIJbYctLMHZG69v78L/h0IfB5y3LYMy01re5480g0d/9nzBqauvbayfqs5dzHdkDW38JY+9Vopf2NfhqyLsMYvOT017xAuiWC30vDT4npc350L0fjLgtOe21EyV7OdfaZ8BrYMJ9UUcimcYMJsyEslWwb3Pr2jq2H957JXhomXA/lLwN+99tXZtH98J7K8IHoa6ta6udKdnL2c7UQPHTcOn1wdOQSHsbMx2yurb+6X7tz+FMNYy/L+gSyurS+qf7ugehma1rJwJK9nK27W/AkbLgSUgkCt37whVTYMPzcPp4y9o4cwbWPA2Dr4H8y6BHXtDtsv45qDrR8jaLfxa81dPvnPkcOzwlezlb8XzocSFcfmvUkUgmK7ofTh2BzUtbdv2Hb8GhnWe/2TLhfjhZCZtbuNzGB7+BypKUfRBSspdPVO6G938F4+6FrOyoo5FMNuiPIW94y7tyYvOhW9+zi6hDroF+nwoeaFqieD50z4PLv9Cy6yOmZC+fWPtz8DNBH6dIlGoLtbtjsHdj8649ug+2rgj66eOLqLVtlq6Efc2cj7H2DbVxqfuGmpK9BM7UwJqfwdDPQt/CqKMRCd6Jz+ra/KLqurAw21B3y5gWFmrX/jwozKbwg5CSvQS2vRYWZmdGHYlIoHtfGHkHbHghmO4gEWfOBG+TDfkTyPvUucd79AuKv+sXJV78PVMTFHsLr0vJwmwtJXsJFC+AnhfB8FuijkTkExPCQu2mBAu1O34TjG4930PLhJlwqhI2v5hYmx/8BipL22wag/aiZC9QWQbbXoVxX1JhVjqWQVdB/uWJF1WLFzQ9urV2lG6iXTmx+cHUIcNT+w01JXuBNc+AO4z/s6gjETlbXaG2GPZsOP+5iY5ubc4o3SPlwRtqaTB1SELJ3swmm9lWM9tuZg83cHywmb1hZhvM7C0zK4g7dp+ZbQt/pW51I13VVAejAod+FnKHRB2NyLnGTIPOOU0/iTdndGuio3RrC7NpMHVIk8nezLKAucDNwBXAdDO7ot5p3wN+5u6jgTnAP4XX9gUeBSYBE4FHzSw3eeFLq21/DY7sTvn+SElj3XJh5J1BofbUsYbPOVPTvNGtZ43SbaT4Wzd1yGfSYuqQRJ7sJwLb3X2Hu58GFgFT6p1zBfBG+PnNuOOfB15z94Pufgh4DZjc+rAlaWLzoefFcJm+FunAJtwPp4/CpiUNH//gzeaPbq0bpdtIoXb768EbamnyIJRIsh8AlMZtl4X74q0H7g4/3wn0MrN+CV4rUTlcGjzZqzArHd3AiZA/ovGunOKwiNqc0a1NjdItXhBMHZImb6glkuwbWorF623/LXCdma0FrgN2A9UJXouZzTKzmJnFKioqEghJkmJtWJhNg/5ISXNmwRN2+RrYs/7sYy1df6G2zYZG6dZNHZI+D0KJJPsyYGDcdgFQHn+Cu5e7+13uPg74+3BfZSLXhuc+6e5F7l6Un5/fzFuQFqmpDkbMfupz0GdQ1NGING30nwaF2vpP4nWjW1vwNtnoP214lO7aZ4KpQ9LoQSiRZL8aGGZmhWbWBZgGLI8/wczyzKy2rW8B88LPrwI3mVluWJi9KdwnUdv2azi6RyNmJXV06wMj74KNv/ikUFs7uvXS61s2urV730+Kv7WF2vipQ9LoDbUmk727VwOzCZL0u8AL7r7ZzOaY2e3hadcDW83sfeAi4PHw2oPAPxD8hbEamBPuk6gVz4de/VWYldRSdD+cPgabFgfb298IRre2Ztrh2kJtbfF3W/iGWopOZdyYhBYcd/cVwIp6+x6J+7wYWNzItfP45ElfOoLDJcH/0Nf+HWRpzXlJIQV/BBeODLpyJsz8pDDbmiLqwEnBKN3Y/KArqHh+OHXIzUkLuyPQCNpMtOaZ4PfxX442DpHmqh39umddUJStLaK2ZnSrWfAUX74maHPbr9OqMFtLyT7T1I6YHXajCrOSmkZ/ETp3g6Wzkrf+wpiw+Lt0Vjh1SPoUZmvp3/DpYsdb8PFHTZ/30ftBYfbWf23zkETaRLc+cOXdwbz1yVp/oVtuUPxdvzB4Qy13cOvb7GCU7NPB7jXws/qDms+jzyAYdlPbxSPS1v7oK8Hi4RP/Molt/nkwfUIy2+xAlOzTQWweZHeHr/w6eGe4KT0vVGFWUtuA8fCND4In8mQpmJD8NjsQ/YlPdSfDV8auvBsuHhV1NCLtpy2ScpomelCBNvVtfAGqjqfNZE0i0jaU7FOZO8QWwMWj4ZLxUUcjIh2Ykn0q210M+zYG7x1bQ3POiYgElOxTWfF8yO4Bo+6JOhIR6eCU7FPVyUrYtBRGTYWcC6KORkQ6OCX7VLUhLMxq1koRSYCSfSpyDyZt6j8meN9YRKQJSvapqCwG+zen3RSsItJ2lOxTUfF86NIz6K8XEUmAkn2qOXH4k8Js115RRyMiKULJPtVseAGqT6gLR0SaRck+lbgHXTiXjINLxkYdjYikkISSvZlNNrOtZrbdzB5u4PggM3vTzNaa2QYzuyXcP8TMTpjZuvDXT5N9AxmldBXs36KnehFptiZnvTSzLGAucCNQBqw2s+XuviXutG8TLET+EzO7gmC92iHhsQ/cXY+hyVC8ALr0Cma4FBFphkSe7CcC2919h7ufBhYB9VfKcKB2GGdvoDx5IQoAJw7B5qUw+h7o2jPqaEQkxSSS7AcApXHbZeG+eI8BXzKzMoKn+q/HHSsMu3f+28z+pKEfYGazzCxmZrGKiorEo88k65+H6pPqwhGRFkkk2Tc0naLX254OLHD3AuAW4Bkz6wTsAQa5+zjgr4GFZnbORC7u/qS7F7l7UX5+fvPuIBPUFWbHQ//RUUcjIikokWRfBgyM2y7g3G6arwAvALj720AOkOfup9z9QLi/GPgAuKy1QWec0pVQ8Z4WKBGRFksk2a8GhplZoZl1AaYBy+udUwLcAGBmIwiSfYWZ5YcFXszsUmAYsCNZwWeM2HzoeoEKsyLSYk2+jePu1WY2G3gVyALmuftmM5sDxNx9OfA3wH+Y2UMEXTwz3d3N7FpgjplVAzXAV939YJvdTTo6fhA2vwjjvwxdekQdjYikqIQWHHf3FQSF1/h9j8R93gJc3cB1S4AlrYwxs61fBDWnNJWxiLSKRtB2ZO7Bu/UDiuDiUVFHIyIpLKEne0myyrJgQrOmVLwHH22FKXPbPiYRSWtK9u3twAcwdyKcqU7s/K69YeSdbRuTiKQ9Jfv2tubpoHvmrqegc9emz+/3KRVmRaTVlOzbU/VpWPssDL85mPZARKSdqEDbnt57GY5/pCkPRKTdKdm3p+IF0HsQDP1s1JGISIZRsm8vBz6AD38LE/4MOuk/u4i0L2Wd9lI8Hzp1hnFfjjoSEclASvbtofoUrFsYFGZ7XRx1NCKSgZTs28O7L8PxA5ryQEQio2TfHooXQJ/BcKkKsyISDSX7tvbRdtj5O5hwnwqzIhIZZZ+2VluYHfulqCMRkQymZN+Wqk6GhdlboNdFUUcjIhlMyb4tvfsynDio5QRFJHJK9m2peAHkDoHC6yMOREQyXULJ3swmm9lWM9tuZg83cHyQmb1pZmvNbIOZ3RJ37FvhdVvN7PPJDL5Dq3gfdv0PjFdhVkSi1+Ssl+GC4XOBG4EyYLWZLQ+XIqz1beAFd/+JmV1BsIThkPDzNGAkcAnwupld5u41yb6RDqd4QThiVoVZEYleIo+cE4Ht7r7D3U8Di4Ap9c5x4ILwc2+gPPw8BVjk7qfc/UNge9heeqs6CesXwuW3Qs8Lo45GRCShZD8AKI3bLgv3xXsM+JKZlRE81X+9GddiZrPMLGZmsYqKigRD78DeXQ4nDmkqYxHpMBJJ9tbAPq+3PR1Y4O4FwC3AM2bWKcFrcfcn3b3I3Yvy8/MTCKmDi82H3EIovC7qSEREgMSSfRkwMG67gE+6aWp9BXgBwN3fBnKAvASvTS8VW6HkD8E8OCrMikgHkUg2Wg0MM7NCM+tCUHBdXu+cEuAGADMbQZDsK8LzpplZVzMrBIYBq5IVfIdUvAA6ZcPYe6OORESkTpNv47h7tZnNBl4FsoB57r7ZzOYAMXdfDvwN8B9m9hBBN81Md3dgs5m9AGwBqoEH0vpNnKoTwYjZEV+AnmnQHSUiaSOhBcfdfQVB4TV+3yNxn7cAVzdy7ePA462IMXVsWQYnD6swKyIdjjqVk6l4AfS9FIb8SdSRiIicRck+Wfa/CyVvqzArIh2SslKyqDArIh2Ykn0yVJ2A9c/BiNugR17U0YiInEPJPhk2vwQnKzWVsYh0WEr2yVA8H/oOVWFWRDosJfvW2rcFSlcGhVlraHYIEZHoKdm3VvECyOqiwqyIdGhK9q1x+jisXwQjboce/aKORkSkUUr2rbHlJTilwqyIdHxK9q0Rmw/9hsHgBmeKEBHpMJTsW2rfZihbpcKsiKQEJfuWqivMzog6EhGRJinZt8Tp47D+ebhiCnTvG3U0IiJNUrJvic1Lg8KspjIWkRShZN8SxQsgbzgM/nTUkYiIJCShxUskzt5NULYaPv9PKsy2off2HuG1zfuiDkOkXV3UO4cvFg1s+sQWSCjZm9lk4IcEyxI+5e5P1Dv+feAz4WZ34EJ37xMeqwE2hsdK3P32ZAQemeL5kNUVxkyLOpK09s0lG1lfejjqMETa1diBfaJL9maWBcwFbgTKgNVmtjxcihAAd38o7vyvA+Pimjjh7mOTF3KETn8MG16AkXeoMNuGNpdXsr70MP/nC1dw3x8PjjockbSQyJP9RGC7u+8AMLNFwBSCRcQbMh14NDnhdTCblsKpIyrMtrGFK0vo2rkTU8cX0DlLZSWRZEjkT9IAoDRuuyzcdw4zGwwUAr+J251jZjEze8fM7mjkulnhObGKiooEQ49A8fygMDvoqqgjSVsfn6pm2bpybh3dn97ds6MORyRtJJLsG6pCeiPnTgMWu3tN3L5B7l4EzAB+YGZDz2nM/Ul3L3L3ovz8/ARCisCeDbC7OJgHR4XZNvPy+nKOnarm3kmDog5FJK0kkuzLgPiKQQFQ3si504Dn4ne4e3n4+w7gLc7uz08dxQugc44Ks21s4aoShl/Ui/GDcqMORSStJJLsVwPDzKzQzLoQJPTl9U8ys+FALvB23L5cM+safs4Drqbxvv6O69SxsDB7J3RTEmorm3ZXsqGskukTB2L615NIUjVZoHX3ajObDbxK8OrlPHffbGZzgJi71yb+6cAid4/v4hkB/LuZnSH4i+WJ+Ld4UsamJXD6aDDpmbSZhatKyMnuxJ3jC6IORSTtJPSevbuvAFbU2/dIve3HGrjuD8CoVsTXMRQvgPwRMHBS1JGkrWOnqlm2djdfGH0JvbupMCuSbHqvrSl71kP5GhVm29jydeV8fLqGGSrMirQJJfumxOYHhdnRX4w6krS2cNUuLr+4F+MG9ok6FJG0lD5z4xw/CE/dkPx2D5fCqHtUmG1DG8sq2bT7CHOmjFRhVqSNpE+y79QZBkxIfrsDJ8G1f5v8dqXOwlW76JadxR3jGhyrJyJJkD7JPucCuPupqKOQZjp6sopl68q5bUx/LshRYVakrajPXiK1fH05x0/XMH2iCrMibUnJXiLj7ixcWcKI/hcwVoVZkTalZC+R2VBWyebyI8yYNEiFWZE2pmQvkVm4soRu2VlMGXtJ1KGIpD0le4nE0ZNVLF9fzu1jLlFhVqQdKNlLJF5aV86JKo2YFWkvSvbS7moLsyMvuYDRBb2jDkckIyjZS7tbV3qYd/ccYfpEFWZF2ouSvbS751aV0L2LCrMi7UnJXtrVkZNVvLx+D1PGXkIvFWZF2o2SvbSrl9bu5kSVRsyKtLf0mRtHkm7ljgMcOn46qW0+8/YurhxwAaMLNGJWpD0llOzNbDLwQ4JlCZ9y9yfqHf8+8Jlwsztwobv3CY/dB3w7PPaP7v50MgKXtrW+9DB/+uQ7bdL2d+8e3Sbtikjjmkz2ZpYFzAVuBMqA1Wa2PH4tWXd/KO78rwPjws99gUeBIsCB4vDaQ0m9C0m6Z1fuonuXLBbNuorsrOT19mVnGUPzeyatPRFJTCJP9hOB7e6+A8DMFgFTgMYWDp9OkOABPg+85u4Hw2tfAyYDz7UmaGlb8UVUdbeIpIdEHtkGAKVx22XhvnOY2WCgEPhNc641s1lmFjOzWEVFRSJxSxtapiKqSNpJJNk3NOrFGzl3GrDY3Wuac627P+nuRe5elJ+fn0BI0lbcnWc1ulUk7SSS7MuAgXHbBUB5I+dO4+wumuZcKx3AutLDvLf3qKYdFkkziST71cAwMys0sy4ECX15/ZPMbDiQC7wdt/tV4CYzyzWzXOCmcJ90UAtXBqNbbx+j0a0i6aTJAq27V5vZbIIknQXMc/fNZjYHiLl7beKfDixyd4+79qCZ/QPBXxgAc2qLtdLxVJ6o4uUN5dw5boBGt4qkmYTes3f3FcCKevseqbf9WCPXzgPmtTA+aUfL1u3mZNUZZkwcHHUoIpJkmi5BgE+mHb5ywAWMUmFWJO0o2QsAa0rCwqye6kXSkpK9AMG0wz26ZHG7ph0WSUtK9kLliSpe2VDOlHED6NlVc+OJpCMle+HFNWVhYVYjZkXSlZJ9hnN3nltVyuiC3lw5QIVZkXSlZJ/h1pQcYuu+o3qqF0lzSvYZ7tmVJfTs2pnbNGJWJK0p2WewyuNV/NeGYCrjHirMiqQ1JfsMtnRtGaeqzzBjkrpwRNKdkn2Gqh0xO6agNyMvUWFWJN0p2Weo2K5DbNt/TE/1IhlCyT5DPafCrEhGUbLPQIePn+aVjXu4Y9wldO+iwqxIJlCyz0BL1uzmdLWmMhbJJEr2GSYYMVvC2IF9uOKSC6IOR0Taif4NnySnq89QcvDjqMNo0ta9x9i+/xjfvXt01KGISDtKKNmb2WTghwTLEj7l7k80cM4XgccAB9a7+4xwfw2wMTytxN1vT0LcHc6jyzfx3KrSqMNISK+cznxhTP+owxCRdtRksjezLGAucCNQBqw2s+XuviXunGHAt4Cr3f2QmV0Y18QJdx+b5Lg7lMoTVby4djefG3ERU1JgPvih+T1VmBXJMIn8iZ8IbHf3HQBmtgiYAmyJO+cvgLnufgjA3fcnO9CO7KW1wdqtf/W5YZo5UkQ6pEQKtAOA+P6JsnBfvMuAy8zs92b2TtjtUyvHzGLh/jsa+gFmNis8J1ZRUdGsG4ha7UhUTREsIh1ZIsneGtjn9bY7A8OA64HpwFNm1ic8Nsjdi4AZwA/MbOg5jbk/6e5F7l6Un5+fcPAdgaYIFpFUkEiyLwMGxm0XAOUNnLPM3avc/UNgK0Hyx93Lw993AG8B41oZc4eycGWpRqKKSIeXSLJfDQwzs0Iz6wJMA5bXO+cl4DMAZpZH0K2zw8xyzaxr3P6rObuvP6VVHg/XbtUUwSLSwTWZody92sxmA68SvHo5z903m9kcIObuy8NjN5nZFqAG+Dt3P2Bmnwb+3czOEPzF8kT8WzypTlMEi0iqMPf63e/RKioq8lgsFnUYTXJ3bvr+b+neJYtls6+JOhwRyXBmVhzWRxuk6RJaqFhTBItIClGyb6GFmiJYRFKIkn0LaIpgEUk1SvYtsFRTBItIilGybyZ3Z6GmCBaRFKNk30yrdx5i+/5jGjErIilFyb6ZnltVQq+umiJYRFKLkn0zHPr4NP+1cQ93jh+gwqyIpBQl+2ZYsqaM09VnmK4uHBFJMUr2Capdu3XcoD6M6K/CrIikFiX7BK368CAfVHyswqyIpCQl+wQtXFUSrN06WiNmRST1KNkn4NDHp/nlxr3cNW4A3bpkRR2OiEizKdknYMmaMk7XnGHGJI2YFZHUpPcHm1A7YnbC4FyGX9wr6nBE0kpVVRVlZWWcPHky6lBSRk5ODgUFBWRnZzfrOiX7Jqz88CA7Kj7me/d8KupQRNJOWVkZvXr1YsiQIZg1tNy1xHN3Dhw4QFlZGYWFhc26Vt04TVi4soQLcjrzhdEaMSuSbCdPnqRfv35K9AkyM/r169eifwkl9GRvZpOBHxIsS/iUuz/RwDlfBB4DHFjv7jPC/fcB3w5P+0d3f7rZUSbZj9/Yxr+89n7C58/89BByslWYFWkLSvTN09L/Xk0mezPLAuYCNwJlwGozWx6/lqyZDQO+BVzt7ofM7MJwf1/gUaCI4C+B4vDaQy2KNglOVdcw7/cfMqagN9cNv7DJ8zt3Mo2YFZGUl8iT/URgu7vvADCzRcAUIH7h8L8A5tYmcXffH+7/PPCaux8Mr30NmAw8l5zwm+9Xm/Zy6HgVP5w2nGsvy48qDBGRdpVIn/0AoDRuuyzcF+8y4DIz+72ZvRN2+yR6bbtauLKEQX27c82n8qIMQ0RSUM+ePRs9tnPnThYuXFi3vWDBAmbPnt0eYSUkkSf7hjqIvIF2hgHXAwXA78zsygSvxcxmAbMABg1quy6T7fuPsfLDg3xj8nA6dVI/oUhH8p2XN7Ol/EhS27zikgt49LaRSW2zMbXJfsaMGc26rqamhqystq8JJvJkXwYMjNsuAMobOGeZu1e5+4fAVoLkn8i1uPuT7l7k7kX5+W3XtbJoVQmdOxn3TBjY9Mkikva++c1v8m//9m9124899hjf+c53uOGGGxg/fjyjRo1i2bJlCbX18MMP87vf/Y6xY8fy/e9/H4Dy8nImT57MsGHD+MY3vlF3bs+ePXnkkUeYNGkSb7/9dnJvqjHuft5fBE/tO4BCoAuwHhhZ75zJwNPh5zyCrpt+QF/gQyA3/PUh0Pd8P2/ChAneFk6crvYx33nVv/bzWJu0LyLNt2XLlkh//po1a/zaa6+t2x4xYoTv2rXLKysr3d29oqLChw4d6mfOnHF39x49ejTa1ptvvum33npr3fb8+fO9sLDQDx8+7CdOnPBBgwZ5SUmJu7sD/vzzz7c47ob+uwExP09ubbIbx92rzWw28CrBq5fz3H2zmc0JG18eHrvJzLYANcDfufsBADP7B2B12NwcD4u17e1Xm/Zy+HiVFgkXkTrjxo1j//79lJeXU1FRQW5uLv379+ehhx7it7/9LZ06dWL37t3s27ePiy++uNnt33DDDfTu3RuAK664gl27djFw4ECysrK4++67k30755XQe/buvgJYUW/fI3GfHfjr8Ff9a+cB81oXZustXFXC4H7d+fTQflGHIiIdyNSpU1m8eDF79+5l2rRpPPvss1RUVFBcXEx2djZDhgxp8XQOXbt2rfuclZVFdXU1EEx50B799PEyYgTt9v1HWfXhQab90SAVZkXkLNOmTWPRokUsXryYqVOnUllZyYUXXkh2djZvvvkmu3btSqidXr16cfTo0TaOtuUyYm6chStLyc4y7ikqiDoUEelgRo4cydGjRxkwYAD9+/fn3nvv5bbbbqOoqIixY8dy+eWXJ9TO6NGj6dy5M2PGjGHmzJnk5ua2ceTNk/bJ/mRVDUvWlHHTyIvJ69m16QtEJONs3Lix7nNeXl6jb8gcO3as0Tays7N54403zto3c+bMus+vvPJKQu20lbTvxvnlpj1UnqjScoIiktHS/sl+4coShvTrzh9fqsKsiLTexo0b+fKXv3zWvq5du7Jy5cqIIkpMWif79/cdZfXOQ3zr5stVmBWRpBg1ahTr1q2LOoxmS+tunOdWlZCdZUydoMKsiGS2tE32J6tqWFJcxudHXkw/FWZFJMOlbbL/rw17OHKymhmTVJgVEUnbZL9wVQmFeT1UmBWRRh0+fPisidASdcstt3D48OFmX7dgwQLKyz+ZC3LIkCF89NFHzW6nJdIy2W/de5TiXYeYPnGgljwTkUY1luxramrOe92KFSvo06dPs39e/WSfiNopFlorLd/GeW5VCV2yOjFVUxmLpI5fPgx7NzZ9XnNcPApuPmfJ7DoPP/wwH3zwAWPHjiU7O5uePXvSv39/1q1bx5YtW7jjjjsoLS3l5MmTPPjgg8yaNQsInshjsRjHjh3j5ptv5r0vJc8AAAecSURBVJprruEPf/gDAwYMYNmyZXTr1u2cn7V48WJisRj33nsv3bp1qxu49eMf/5iXX36ZqqoqfvGLX3D55Zfz2GOPUV5ezs6dO8nLyztrUZSWSrsn+xOngxGzk6+8mL49ukQdjoh0YE888QRDhw5l3bp1/PM//zOrVq3i8ccfZ8uWYNXVefPmUVxcTCwW40c/+hEHDhw4p41t27bxwAMPsHnzZvr06cOSJUsa/FlTp06lqKiIZ599lnXr1tX9hZCXl8eaNWv42te+xve+972684uLi1m2bFlSEj2k4ZP9KxvKOarCrEjqOc8TeHuZOHEihYWFdds/+tGPePHFFwEoLS1l27Zt9Ot3dh2wsLCQsWPHAjBhwgR27tzZrJ9511131V27dOnSuv233357g/9CaKm0S/bPrSrh0vweTCrsG3UoIpJievToUff5rbfe4vXXX+ftt9+me/fuXH/99Q1OdVx/GuMTJ04062fWXh8/BXL9WJIhrbpx3tt7hDUlh5kxcZAKsyLSpPNNS1xZWUlubi7du3fnvffe45133mnTn9fW0urJfuHKoDB793iNmBWRpvXr14+rr76aK6+8km7dunHRRRfVHZs8eTI//elPGT16NMOHD+eqq65q9c+bOXMmX/3qV88q0LYXCxaZ6jiKioo8Fos1+7oTp2uY+PjrfHbEhfxw2rg2iExEku3dd99lxIgRUYeRchr672Zmxe5e1Ng1CXXjmNlkM9tqZtvN7OEGjs80swozWxf++vO4YzVx+5c3436a5cjJKq4bns+XrtIasyIi9TXZjWNmWcBc4EagDFhtZsvdfUu9U59399kNNHHC3ce2PtTzu+iCHP7fjPFt/WNERJr0wAMP8Pvf//6sfQ8++CD3339/RBEl1mc/Edju7jsAzGwRMAWon+xFRASYO3du1CGcI5FunAFAadx2WbivvrvNbIOZLTaz+KGrOWYWM7N3zOyOhn6Amc0Kz4lVVFQkHr2IpLyOVjfs6Fr63yuRZN/QO4z1f9rLwBB3Hw28Djwdd2xQWDSYAfzAzIae05j7k+5e5O5F+fn5CYYuIqkuJyeHAwcOKOEnyN05cOAAOTk5zb42kW6cMiD+Sb0AOGsmH3ePH0P8H8D/jTtWHv6+w8zeAsYBHzQ7UhFJOwUFBZSVlaF/0ScuJyeHgoLmv16eSLJfDQwzs0JgNzCN4Cm9jpn1d/c94ebtwLvh/lzguLufMrM84Grgu82OUkTSUnZ29lnTE0jbaTLZu3u1mc0GXgWygHnuvtnM5gAxd18O/C8zux2oBg4CM8PLRwD/bmZnCLqMnmjgLR4REWljaTOoSkQkkyVlUJWIiKS2Dvdkb2YVwK5WNJEHtM86X+0j3e4H0u+e0u1+IP3uKd3uB869p8Hu3ujrjB0u2beWmcXO90+ZVJNu9wPpd0/pdj+QfveUbvcDzb8ndeOIiGQAJXsRkQyQjsn+yagDSLJ0ux9Iv3tKt/uB9LundLsfaOY9pV2fvYiInCsdn+xFRKSetEn2TS2wkorMbKeZbQwXfkm5kWZmNs/M9pvZprh9fc3sNTPbFv6eG2WMzdXIPT1mZrvjFum5JcoYm8PMBprZm2b2rpltNrMHw/0p+T2d535S+TvKMbNVZrY+vKfvhPsLzWxl+B09b2ZdzttOOnTjhAusvE/cAivA9FSfmsHMdgJF7p6S7web2bXAMeBn7n5luO+7wEF3fyL8SznX3b8ZZZzN0cg9PQYcc/fvRRlbS5hZf6C/u68xs15AMXAHwZQnKfc9ned+vkjqfkcG9HD3Y2aWDfwP8CDw18BSd19kZj8F1rv7TxprJ12e7OsWWHH300DtAisSIXf/LcFcSfGm8MkU2E8T/EFMGY3cU8py9z3uvib8fJRgEsMBpOj3dJ77SVkeOBZuZoe/HPgssDjc3+R3lC7JPtEFVlKNA782s2IzmxV1MElyUe0MqeHvF0YcT7LMDhfvmZcqXR71mdkQginIV5IG31O9+4EU/o7MLMvM1gH7gdcIpok/7O7V4SlN5rx0SfaJLLCSiq529/HAzcADYReCdDw/AYYCY4E9wL9EG07zmVlPYAnwV+5+JOp4WquB+0np78jda8K1vAsIejJGNHTa+dpIl2Tf5AIrqShu4Zf9wIsEX3Kq2xf2q9b2r+6POJ5Wc/d94R/GMwSL96TU9xT2Ay8BnnX3peHulP2eGrqfVP+Oarn7YeAt4Cqgj5nVTlPfZM5Ll2Rft8BKWJGeBiyPOKZWMbMeYYEJM+sB3ARsOv9VKWE5cF/4+T5gWYSxJEVtUgzdSQp9T2Hx7z+Bd939X+MOpeT31Nj9pPh3lG9mfcLP3YDPEdQi3gSmhqc1+R2lxds4AOGrVD/gkwVWHo84pFYxs0sJnuYhWGRmYardk5k9B1xPMDvfPuBR4CXgBWAQUALc4+4pU/Bs5J6uJ+gecGAn8JdxK7d1aGZ2DfA7YCNwJtz9vwn6uVPuezrP/Uwndb+j0QQF2CyCB/QX3H1OmCMWAX2BtcCX3P1Uo+2kS7IXEZHGpUs3joiInIeSvYhIBlCyFxHJAEr2IiIZQMleRCQDKNmLiGQAJXsRkQygZC8ikgH+Pz7Rpbd+YcuVAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plot_hist(thr_score_hist)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "df, train_df, val_df, train_labels, val_labels = load_data(train_perc=0.3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "# creating dataloaders\n",
    "small_images_dir_train = 'data/small_train_images_250/'\n",
    "small_images_dir_val = 'data/small_train_images_224/'\n",
    "bs = 32\n",
    "\n",
    "tr_dl = create_dl(train_df, small_images_dir_train, batch_size=bs)\n",
    "tr_test_dl = create_dl(train_df, small_images_dir_val, shuffle=False, batch_size=bs)\n",
    "val_dl = create_dl(val_df, small_images_dir_val, shuffle=False, batch_size=bs)\n",
    "#full_dl = create_dl(df, small_images_dir, shuffle=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "class CLIPImg(torch.nn.Module) :\n",
    "    def __init__(self, model_name=\"RN50\", device='cuda') :\n",
    "        super().__init__()\n",
    "        self.model = clip.load(model_name, device=device, jit=False)[0]\n",
    "    def forward(self, imgs) :\n",
    "        return self.model.encode_image(imgs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = CLIPImg()\n",
    "train_tfms, val_tfms = get_tfms(crop=224)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d263d651f2c545c9908bd9202244af5d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=327.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "ename": "TypeError",
     "evalue": "unsupported operand type(s) for /: 'JpegImageFile' and 'float'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mTypeError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-29-e25b70683e56>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mcenters\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcompute_centers\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtr_test_dl\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmodel\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mval_tfms\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtrain_df\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      2\u001b[0m \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msave\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcenters\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'data/clip/centers_im_0.3_res50.pth'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/Documents/kaggle/shopee/arcface.py\u001b[0m in \u001b[0;36mcompute_centers\u001b[0;34m(dataloader, model, val_transforms, dataframe, batch_size)\u001b[0m\n\u001b[1;32m     67\u001b[0m     \u001b[0;32mwith\u001b[0m \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mno_grad\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     68\u001b[0m         \u001b[0membs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 69\u001b[0;31m         \u001b[0;32mfor\u001b[0m \u001b[0mimgs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0m_\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mtqdm\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdataloader\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     70\u001b[0m             \u001b[0mimgs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mval_transforms\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mimgs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mto\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'cuda'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     71\u001b[0m             \u001b[0mfeatures\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmodel\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mimgs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.7/site-packages/tqdm/notebook.py\u001b[0m in \u001b[0;36m__iter__\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m    215\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m__iter__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    216\u001b[0m         \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 217\u001b[0;31m             \u001b[0;32mfor\u001b[0m \u001b[0mobj\u001b[0m \u001b[0;32min\u001b[0m \u001b[0msuper\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtqdm_notebook\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__iter__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    218\u001b[0m                 \u001b[0;31m# return super(tqdm...) will not catch exception\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    219\u001b[0m                 \u001b[0;32myield\u001b[0m \u001b[0mobj\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.7/site-packages/tqdm/std.py\u001b[0m in \u001b[0;36m__iter__\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m   1105\u001b[0m                 fp_write=getattr(self.fp, 'write', sys.stderr.write))\n\u001b[1;32m   1106\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1107\u001b[0;31m         \u001b[0;32mfor\u001b[0m \u001b[0mobj\u001b[0m \u001b[0;32min\u001b[0m \u001b[0miterable\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1108\u001b[0m             \u001b[0;32myield\u001b[0m \u001b[0mobj\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1109\u001b[0m             \u001b[0;31m# Update and possibly print the progressbar.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.7/site-packages/torch/utils/data/dataloader.py\u001b[0m in \u001b[0;36m__next__\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    433\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_sampler_iter\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    434\u001b[0m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_reset\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 435\u001b[0;31m         \u001b[0mdata\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_next_data\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    436\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_num_yielded\u001b[0m \u001b[0;34m+=\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    437\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_dataset_kind\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0m_DatasetKind\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mIterable\u001b[0m \u001b[0;32mand\u001b[0m\u001b[0;31m \u001b[0m\u001b[0;31m\\\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.7/site-packages/torch/utils/data/dataloader.py\u001b[0m in \u001b[0;36m_next_data\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    473\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m_next_data\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    474\u001b[0m         \u001b[0mindex\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_next_index\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m  \u001b[0;31m# may raise StopIteration\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 475\u001b[0;31m         \u001b[0mdata\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_dataset_fetcher\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfetch\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mindex\u001b[0m\u001b[0;34m)\u001b[0m  \u001b[0;31m# may raise StopIteration\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    476\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_pin_memory\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    477\u001b[0m             \u001b[0mdata\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0m_utils\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpin_memory\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpin_memory\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.7/site-packages/torch/utils/data/_utils/fetch.py\u001b[0m in \u001b[0;36mfetch\u001b[0;34m(self, possibly_batched_index)\u001b[0m\n\u001b[1;32m     42\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mfetch\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpossibly_batched_index\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     43\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mauto_collation\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 44\u001b[0;31m             \u001b[0mdata\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdataset\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0midx\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0midx\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mpossibly_batched_index\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     45\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     46\u001b[0m             \u001b[0mdata\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdataset\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mpossibly_batched_index\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.7/site-packages/torch/utils/data/_utils/fetch.py\u001b[0m in \u001b[0;36m<listcomp>\u001b[0;34m(.0)\u001b[0m\n\u001b[1;32m     42\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mfetch\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpossibly_batched_index\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     43\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mauto_collation\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 44\u001b[0;31m             \u001b[0mdata\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdataset\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0midx\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0midx\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mpossibly_batched_index\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     45\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     46\u001b[0m             \u001b[0mdata\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdataset\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mpossibly_batched_index\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/Documents/kaggle/shopee/image_train/data.py\u001b[0m in \u001b[0;36m__getitem__\u001b[0;34m(self, idx)\u001b[0m\n\u001b[1;32m     13\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m__getitem__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0midx\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     14\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 15\u001b[0;31m         \u001b[0mimg\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_get_item\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0midx\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     16\u001b[0m         \u001b[0mlabel\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlabel_codes\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0miloc\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0midx\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     17\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mimg\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlabel\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/Documents/kaggle/shopee/image_train/data.py\u001b[0m in \u001b[0;36m_get_item\u001b[0;34m(self, idx)\u001b[0m\n\u001b[1;32m     21\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m_get_item\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0midx\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     22\u001b[0m         \u001b[0mim\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mPIL\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mImage\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mopen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mos\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpath\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mjoin\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mimages_path\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mimgs\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0midx\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 23\u001b[0;31m         \u001b[0mim\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtensor\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0marray\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mim\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m/\u001b[0m \u001b[0;36m255.0\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdtype\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfloat\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpermute\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m2\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     24\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mim\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     25\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mTypeError\u001b[0m: unsupported operand type(s) for /: 'JpegImageFile' and 'float'"
     ]
    }
   ],
   "source": [
    "centers = compute_centers(tr_test_dl, model, val_tfms, train_df)\n",
    "torch.save(centers, 'data/clip/centers_im_0.3_res50.pth')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "centers = torch.load('data/clip/centers_im_0.3_res50.pth')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Using center as wieghts\n"
     ]
    }
   ],
   "source": [
    "metric_fc = ArcMarginProduct(512, train_df['label_group'].nunique(), \n",
    "                             s=30, m=0.5, easy_margin=False, centers=centers, half=True).to(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "n_epochs, lf, params, optimizer, sched = get_hparams(tr_dl, model, metric_fc, lr=1e-5, n_epochs=10)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "for n, _ in model.named_parameters() :\n",
    "    print(n)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "mnp = list(model.named_parameters())\n",
    "\n",
    "param_groups = [{'params' : [p for n,p in mnp if 'embeddings' in n]}]\n",
    "\n",
    "params_names = []\n",
    "n_blocks = 5\n",
    "for i in range(n_blocks):\n",
    "    ith_block = [p for n, p in mnp if f'layer{i}.' in n]\n",
    "    ith_block_name = [n for n, p in mnp if f'layer{i}.' in n]\n",
    "    params_names += ith_block_name\n",
    "    param_groups.append({'params' : ith_block})\n",
    "    \n",
    "param_groups.append({'params' : [p for n,p in mnp if 'attnpool'in n]})\n",
    "params_names += [n for n,p in mnp if 'attnpool' in n]\n",
    "\n",
    "param_groups.append({'params' : metric_fc.parameters()})\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "for n, p in model.named_parameters() :\n",
    "    if n not in params_names :\n",
    "        print(n)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "lf = nn.CrossEntropyLoss()\n",
    "optimizer = torch.optim.AdamW(param_groups, weight_decay = 1e-5)\n",
    "\n",
    "lr = list(np.linspace(1e-5,5e-4,len(param_groups)))\n",
    "\n",
    "n_epochs = 10\n",
    "sched = torch.optim.lr_scheduler.OneCycleLR(optimizer, max_lr=lr, pct_start=0.3,\n",
    "                                            total_steps=int(n_epochs * len(tr_dl)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "loss_hist = []\n",
    "best_thr_score={'val': {'thr': None, 'f1': None}, 'train': {'thr': None, 'f1': None}}\n",
    "thr_score_hist=[]\n",
    "ep_start = 0\n",
    "save_path = 'data/clip/test_20ap_im_res50'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=10.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=327.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Thresholds', max=10.0, style=ProgressStyle(description_wi…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Checkpoint : saved model to data/clip/test_20ap_im_res50_ep_0.pth\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=745.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Thresholds', max=10.0, style=ProgressStyle(description_wi…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Saved best model ep 0 with f score : 0.5923946402416435\n",
      "Ep 0: Train loss 14.7105 | Val f score 0.5924 with thresh 0.29, train f score 0.4926 with thresh 0.56\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=327.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Thresholds', max=5.0, style=ProgressStyle(description_wid…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=745.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Thresholds', max=5.0, style=ProgressStyle(description_wid…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Saved best model ep 1 with f score : 0.6219290318389071\n",
      "Ep 1: Train loss 13.0619 | Val f score 0.6219 with thresh 0.34, train f score 0.4755 with thresh 0.56\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=327.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Thresholds', max=5.0, style=ProgressStyle(description_wid…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Checkpoint : saved model to data/clip/test_20ap_im_res50_ep_2.pth\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=745.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Thresholds', max=5.0, style=ProgressStyle(description_wid…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Saved best model ep 2 with f score : 0.6336896602294358\n",
      "Ep 2: Train loss 11.6626 | Val f score 0.6337 with thresh 0.44, train f score 0.5413 with thresh 0.66\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=327.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Thresholds', max=5.0, style=ProgressStyle(description_wid…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=745.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Thresholds', max=5.0, style=ProgressStyle(description_wid…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Saved best model ep 3 with f score : 0.6588727139419738\n",
      "Ep 3: Train loss 9.4484 | Val f score 0.6589 with thresh 0.49, train f score 0.6547 with thresh 0.76\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=327.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Thresholds', max=5.0, style=ProgressStyle(description_wid…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Checkpoint : saved model to data/clip/test_20ap_im_res50_ep_4.pth\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=745.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Thresholds', max=5.0, style=ProgressStyle(description_wid…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Saved best model ep 4 with f score : 0.6766706201743063\n",
      "Ep 4: Train loss 7.0888 | Val f score 0.6767 with thresh 0.59, train f score 0.7428 with thresh 0.76\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=327.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Thresholds', max=5.0, style=ProgressStyle(description_wid…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=745.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Thresholds', max=5.0, style=ProgressStyle(description_wid…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Saved best model ep 5 with f score : 0.6884154413996265\n",
      "Ep 5: Train loss 5.1298 | Val f score 0.6884 with thresh 0.59, train f score 0.8078 with thresh 0.81\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=327.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Thresholds', max=5.0, style=ProgressStyle(description_wid…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Checkpoint : saved model to data/clip/test_20ap_im_res50_ep_6.pth\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=745.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Thresholds', max=5.0, style=ProgressStyle(description_wid…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Saved best model ep 6 with f score : 0.6937082190002374\n",
      "Ep 6: Train loss 3.3783 | Val f score 0.6937 with thresh 0.64, train f score 0.8636 with thresh 0.81\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=327.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Thresholds', max=5.0, style=ProgressStyle(description_wid…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=745.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Thresholds', max=5.0, style=ProgressStyle(description_wid…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Ep 7: Train loss 2.1293 | Val f score 0.6777 with thresh 0.64, train f score 0.9101 with thresh 0.86\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=327.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Thresholds', max=5.0, style=ProgressStyle(description_wid…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Checkpoint : saved model to data/clip/test_20ap_im_res50_ep_8.pth\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=745.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Thresholds', max=5.0, style=ProgressStyle(description_wid…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Saved best model ep 8 with f score : 0.7024859165653851\n",
      "Ep 8: Train loss 1.3773 | Val f score 0.7025 with thresh 0.69, train f score 0.9408 with thresh 0.81\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=327.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Thresholds', max=5.0, style=ProgressStyle(description_wid…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=745.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Thresholds', max=5.0, style=ProgressStyle(description_wid…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Ep 9: Train loss 1.0794 | Val f score 0.7024 with thresh 0.69, train f score 0.9532 with thresh 0.86\n",
      "\r"
     ]
    }
   ],
   "source": [
    "best_thr_score, thr_score_hist, losses = train(model, optimizer, lf, sched, metric_fc, tr_dl, val_dl,\n",
    "                                               n_epochs, train_df, val_df, train_tfms, val_tfms, \n",
    "                                               save_path=save_path, \n",
    "                                               prev_best_info=best_thr_score, info_history=thr_score_hist,\n",
    "                                               ep_start=ep_start, half_precision=True)\n",
    "loss_hist.append(losses)\n",
    "ep_start += n_epochs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAD4CAYAAAD8Zh1EAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+j8jraAAAgAElEQVR4nO3dd3xUVd7H8c9JI410EiAhJCAtFImGovQO6gKCSlNUFHTFum7B1UdZ17Wsu/u4uOwiq/hYUESUIiC9WRAJJIIk9JYCIYUkJIGUmfP8cUMIGCCBmdyZye/9esVk7tzM/WWELye/e+85SmuNEEII5+dmdgFCCCFsQwJdCCFchAS6EEK4CAl0IYRwERLoQgjhIjzMOnBYWJiOiYkx6/BCCOGUduzYkaO1blLTc6YFekxMDImJiWYdXgghnJJS6tjlnpOWixBCuAgJdCGEcBES6EII4SJM66HXpLy8nPT0dM6dO2d2KS7N29ubqKgoPD09zS5FCGFDDhXo6enpNG7cmJiYGJRSZpfjkrTW5Obmkp6eTmxsrNnlCCFsyKFaLufOnSM0NFTC3I6UUoSGhspvQUK4IIcKdEDCvB7IeyyEa3KolosQQriksmLIPQg5ByD3ELQdBs272vwwEuhCCGELVgvkHzeCuyq8D0DOQTiTWW1HBX5hEuiOxt/fn6KiIrPLEELUp5K8SwK7ctSddxgspRf28w6E0DbQqh+E3mB8hLWBkFbg6WOX0iTQnURFRQUeHvK/S4h6UVEKeUcuDuzzX5/Nu7CfmyeExBph3WaIEdihbYzPvqFQz+erHDYh/vTVHlIyC236mnHNA3jpVx0v+/yMGTNo0aIF06dPB2DmzJl4eHiwceNGTp8+TXl5Oa+88gqjRo266rFOnDjBuHHjKCwspKKigv/85z/06dOHVatW8cc//hGLxUJYWBjr168nLy+PKVOmcPjwYXx9fZk7dy5dunRh5syZHDp0iMOHDxMdHc2sWbN49NFHOX78OABvvfUWvXr1ss2bI0RDozWcOVEZ2Je0SfKPg7Ze2Nc/wgjquJHG5/Oj7aCW4O44Meo4lTiAcePG8fTTT1cF+sKFC1m9ejVPPvkkAQEB5OTk0LNnT0aOHHnVK0U++eQThg0bxvPPP4/FYqGkpITs7GymTp3Kli1biI2NJS/P+Jf+pZdeIj4+niVLlrBhwwYmT55McnIyACkpKXz77bf4+PgwceJEnnnmGXr37s3x48cZNmwYqamp9n1ThHAFFWVwaD1kJhuBnXvQGHWXVWuZevpCaGtoHg+d76kcbd9gbPMONK/2OnDYQL/SSNpe4uPjOXXqFJmZmWRnZxMcHEzTpk155pln2LJlC25ubmRkZJCVlUXTpk2v+FrdunVjypQplJeXM3r0aLp27cqmTZvo27dv1Q09ISEhAHz77bd88cUXAAwcOJDc3FwKC43fTkaOHImPj9FvW7duHSkpKVXHKCwspKioCH9/f5u/F0K4hBO7IPkT2L0QSnIBBUEtjFF29C0X97YbNwc3h7uSu04cNtDNcvfdd7No0SJOnjzJuHHjmD9/PtnZ2ezYsQNPT09iYmJqdVNO37592bJlCytWrOCBBx7gN7/5DcHBwXWux8/Pr+prq9XKDz/8gLe3d51fR4gGozgHdi00gjxrN7h7QbvboOskiO1jtxOSjsC5/zmyg3HjxrFgwQIWLVrE3XffTUFBAeHh4Xh6erJx40aOHbvsVMQXOXbsGBEREUydOpWHH36YnTt30rNnT7Zs2cKRI0cAqlouffr0Yf78+QBs2rSJsLAwAgICfvGaQ4cO5e233656fL4tI0SDZymHvStgwST4eztY/ZzR277tb/DsPrjnA2g71KXDHGSE/gsdO3bkzJkzREZG0qxZMyZNmsSvfvUrOnfuTEJCAu3bt6/V62zatIk333wTT09P/P39+fDDD2nSpAlz585lzJgxWK1WwsPDWbt2LTNnzmTKlCl06dIFX19fPvjggxpfc9asWUyfPp0uXbpQUVFB3759mTNnji1/fCGcy8ndxkh810IoyQG/cOj5a7hxIkTEmV1dvVNaa1MOnJCQoC9dsSg1NZUOHTqYUk9DI++1cFrFuUZPPHm+EehuntC+sqXSepBDXXViD0qpHVrrhJqec+2fXAjhGizlcGCtEeL7V4O1HJp1hRFvQue7wDfE7AodggT6ddq9ezf33XffRdsaNWrEtm3bTKpICBeStQeS5sOuzypbKk2gxyPQdSJE1P+VcI5OAv06de7cWU5OCmFLxbnw8yJjNH7iJ6Ol0m44dL0XbhgE7rIwy+VIoAshzGcph4PrjBDft6qypXIjjPgrdLoL/ELNrtApSKALIcyTlWKE+K6FUHzqQkvlxgnQtJPZ1TkdCXQhRP0qyYPd51sqyeDmAW2HG1eptBkiLZXrIIEuhLA/S4Uxl0ryfNj3NVjKoGkXGP6GcZWKX5jZFboEuVO0mvz8fP7973/X+ftuu+028vPz7VCREE6urBg2vQ7/6ACf3ANHv4VuD8Oj38Kj30DPRyXMbUhG6NWcD/THHnvsou1Xm4t85cqV9i6tViwWC+7u7maXIQRYrbBrAax/2Ziitu1wuGky3DAEPLzMrs5l1SrQlVLDgX8C7sC7WuvXL3m+JTAPaALkAfdqrdOvq7KvZxh3gdlS084w4vXLPj1jxgwOHTpE165d8fT0xNvbm+DgYPbu3cv+/fsZPXo0aWlpnDt3jqeeeopp06YBEBMTQ2JiIkVFRYwYMYLevXvz/fffExkZydKlS6tmS7zUrFmzmDNnDh4eHsTFxbFgwQKKiop44oknSExMRCnFSy+9xNixY/n000959dVX0Vpz++2388YbbwDGqkmPPPII69atY/bs2Rw9epRZs2ZRVlZGjx49+Pe//y0hL+rXsa3GXCqZSRB5M9z9AUT3MLuqBuGqLRellDswGxgBxAETlFKXTpLwN+BDrXUX4GXgNVsXWh9ef/11WrduTXJyMm+++SY7d+7kn//8J/v37wdg3rx57Nixg8TERGbNmkVubu4vXuPAgQNMnz6dPXv2EBQUVDUt7uWOl5SUxK5du6rmZPnzn/9MYGAgu3fvZteuXQwcOJDMzEz+8Ic/sGHDBpKTk9m+fTtLliwBoLi4mB49evDTTz8RGhrKZ599xnfffUdycjLu7u5Vk34JYXenj8LC++H94XAmC+6cCw+tkzCvR7UZoXcHDmqtDwMopRYAo4CUavvEAb+p/HojsOS6K7vCSLq+dO/evWrucjBG1IsXLwYgLS2NAwcOEBp68fWxsbGxdO1qLP568803c/To0cu+fpcuXZg0aRKjR49m9OjRgDHn+YIFC6r2CQ4OZsuWLfTv358mTZoAMGnSJLZs2cLo0aNxd3dn7NixAKxfv54dO3bQrVs3AM6ePUt4ePh1vgtCXEXpGfjmH7B1Nri5Q//n4NYnwMvv6t8rbKo2gR4JpFV7nA5c+k/uT8AYjLbMnUBjpVSo1vqiIaxSahowDSA6Ovpaa6431eci37RpE+vWrWPr1q34+vrSv3//GudFb9SoUdXX7u7unD179rKvv2LFCrZs2cJXX33FX/7yF3bvrnuLydvbu6qlorXm/vvv57XXnPIXJOFsrBZI+hg2vGJcQ95lPAx6EQIjza6swbLVVS6/BfoppZKAfkAGYLl0J631XK11gtY64fxo05E0btyYM2fO1PhcQUEBwcHB+Pr6snfvXn744YfrOpbVaiUtLY0BAwbwxhtvUFBQQFFREUOGDGH27NlV+50+fZru3buzefNmcnJysFgsfPrpp/Tr1+8Xrzlo0CAWLVrEqVOnAGO+9drO3y5EnRzZAu/0g6+eNBZJfngDjHlHwtxktRmhZwAtqj2OqtxWRWudiTFCRynlD4zVWjvddXyhoaH06tWLTp064ePjQ0RERNVzw4cPZ86cOXTo0IF27drRs2fP6zqWxWLh3nvvpaCgAK01Tz75JEFBQbzwwgtMnz6dTp064e7uzksvvcSYMWN4/fXXGTBgQNVJ0ZoWqo6Li+OVV15h6NChWK1WPD09mT17Ni1btryuWoWoknsI1r4Ie5dDYDTcNQ86jqn31e1Fza46H7pSygPYDwzCCPLtwESt9Z5q+4QBeVprq1LqL4BFa/3ilV5X5kM3l7zXok7OFcCWN+GHOcaSbn1+A7dMd/kVgBzRdc2HrrWuUEo9DqzGuGxxntZ6j1LqZSBRa70M6A+8ppTSwBZgus2qF0KYx1IBO/8PNr5q3LIfPwkG/g80vvIi6cIctboOXWu9Elh5ybYXq329CFhk29Jcx/Tp0/nuu+8u2vbUU0/x4IMPmlSRELVwcD2sfh6yU6Flbxj+qjEDonBYDnenqNYa5WL9uOonOR2BWcsOCieRvR/WvAAHVkNwDNzzEXT4lfTJnYBDBbq3tze5ubmEhoa6XKg7Cq01ubm5eHt7m12KcDQlebD5Ddj+Lnj4wJCXocej4NHo6t8rHIJDBXpUVBTp6elkZ2ebXYpL8/b2JioqyuwyhKOwlMP292DTa1BaCDfdDwOeB3/Hu7RYXJlDBbqnp+dFd2YKIexIaziwxuiT5x6AVv1h2KuyVqcTc6hAF0LUk1OpsPqPcGgDhN4AEz6DtsOkT+7kJNCFaEiKc4xLEHe8D40aw7DXjPnJZUpblyCBLkRDUFEGP74Dm9+EsiIjxPs/B74hZlcmbEgCXQhXpjXsXQFr/wfyDhsLTAx9BcLbm12ZsAMJdCFc1alUWPk7OPoNhLWDSV9Am8FmVyXsSAJdCFdjtcDWfxnT2nr5wW1/g5sfBHf56+7q5P+wEK4k7zAseQyOb4X2d8Adb8n15A2IBLoQrkBrSHwP1vwPuHnCne9Al3FyGWIDI4EuhLMryIBljxvXlLcaAKNmy0ITDZQEuhDOSmvYtdA48Wkth9v/DgkPyai8AZNAF8IZFefA8qch9Sto0QNG/wdCW5tdlTCZBLoQzmbvCvjqKWMVocF/glufADd3s6sSDkACXQhncTYfVs2Anz6Fpp1h8lKZSEtcRAJdCGdwaCMsnQ5nTkLf30Pf38n8K+IXJNCFcGRlxbD2Jdj+XwhrCw+thaibza5KOCgJdCEc1fFtsORR42ahno/BoBfB08fsqoQDk0AXwtFUlBpT3H4/CwKj4P7lENvH7KqEE5BAF8KRnNgFix+FU3vgpsnGCkKNGptdlXASEuhCOAJLBXz3v7DpDWOO8okLjRWEhKgDCXQhzJZzABY/Ahk7oOMY445PWXhCXAMJdCHMYrUaqwitm2mc7LxrHnQaa3ZVwolJoAthhvzjxjS3R7+BNsNg5Cxo3NTsqoSTk0AXoj5pDUkfwao/AhpGvg3x98mEWsImJNCFqC9nThpzsOxfBTF9jGlug1uaXZVwIRLoQtSHn7+EFb+B8rMw/HXo/gi4uZldlXAxEuhC2FNJHqz8Lfz8BUTeDKPnQJO2ZlclXJQEuhD2sn8NLHsCSnJg4AvQ6xlZqFnYlfzpEsLWSotg9R9h5wcQHgeTPodmXcyuSjQAtWriKaWGK6X2KaUOKqVm1PB8tFJqo1IqSSm1Syl1m+1LFcIJ5B6CdwcbV7L0ehqmbZIwF/XmqiN0pZQ7MBsYAqQD25VSy7TWKdV2ewFYqLX+j1IqDlgJxNihXiEc1/418MXDxsnOe7+E1gPMrkg0MLUZoXcHDmqtD2uty4AFwKhL9tFAQOXXgUCm7UoUwsFZrbD5TfjkHgiOhmmbJcyFKWrTQ48E0qo9Tgd6XLLPTGCNUuoJwA8YbJPqhHB05wphya9h73LoMg7ueAu8fM2uSjRQtroQdgLwf1rrKOA24COl1C9eWyk1TSmVqJRKzM7OttGhhTBJ9n54dxDs+xqGvwF3viNhLkxVm0DPAFpUexxVua26h4CFAFrrrYA3EHbpC2mt52qtE7TWCU2aNLm2ioVwBHtXwH8HGteZT14KPR+V2/eF6WoT6NuBNkqpWKWUFzAeWHbJPseBQQBKqQ4YgS5DcOF6rFbY8BdYMBHCboBHNstqQsJhXLWHrrWuUEo9DqwG3IF5Wus9SqmXgUSt9TLgWeC/SqlnME6QPqC11vYsXIh6dzYfvpwKB9ZA13uNecs9vc2uSogqtbqxSGu9EuNSxOrbXqz2dQrQy7alCeFAslLgs0mQn2YEecJD0mIRDkfuFBXiavYshiXToZE/PLAconuaXZEQNZJAF+JyrBZY/zJ89xZEdYd7PoSAZmZXJcRlSaALUZOSPPjiITi0ARKmGJcleniZXZUQVySBLsSlTuwy+uVnThorCt002eyKhKgVCXQhqtv1uTHlrU8wPPg1RCWYXZEQtSaBLgSApQLWvgg/zIboW+GeD8A/3OyqhKgTCXQhirJh0YNw9Bvo8SgMfQXcPc2uSog6k0AXDVvGTvjsPmNVoTvfgRvHm12RENdMAl00XEnzYfkzRmtlympo3tXsioS4LhLoouGpKDOWiNv+X4jtB3e9D36hZlclxHWTQBcNy5ks+Px+OL4Vbn0CBs2UhZuFy5A/yaLhSNsOC+8zJtka+x50vsvsioSwKQl00TAkvg8rfweBkfDwOmjayeyKhLA5CXTh2ipKjSDf+QG0HgRj3wXfELOrEsIuJNCF6yrMNC5JzEiEPs/CgOfBzd3sqoSwGwl04ZqOfQ8L74fyErjnI4gbaXZFQtidBLpwLVrD9ndh1QwIagn3fwXh7c2uSoh6IYEuXEdFGSx/GpLnQ9vhMGYueAeaXZUQ9UYCXbgGqwUWPwJ7voR+f4B+M8CtNmugC+E6JNCF89MaVv7WCPPBf4LeT5tdkRCmkCGMcH4bXoHEedDrKQlz0aBJoAvn9v2/4Ju/GasKDf6T2dUIYSoJdOG8kj6GNc9D3Ci44y1QyuyKhDCVBLpwTqnLjaXiWg2AMf+VG4aEQAJdOKPDm40VhprfBOM+Bo9GZlckhEOQQBfOJWMHLJgIIa1h0ufQyN/sioRwGBLownlk74OP7zIm17pvsUyyJcQlJNCFc8g/Dh/dCW4ecN8SCGhmdkVCOBy5sUg4vqJs+HA0lBbBgyshtLXZFQnhkCTQhWM7VwAfjzGmwp28RBamEOIKJNCF4yo/C59OgFMpMGEBRPc0uyIhHJoEunBMlnL4/AFjXvOx70KbIWZXJITDk0AXjsdqhaXTYf8quP3vspizELVUq6tclFLDlVL7lFIHlVIzanj+f5VSyZUf+5VS+bYvVTQIWhuLU+z6DAa+AN0eNrsiIZzGVUfoSil3YDYwBEgHtiullmmtU87vo7V+ptr+TwDxdqhVNASb34Af34Ge06HPb82uRriArMJzrE89xcZ9pzhdXPaL52uaAkhR48babKrV6z3UO5bBcRGXqfja1abl0h04qLU+DKCUWgCMAlIus/8E4CXblCcalG3vwKbX4MaJMPQVmWxLXBOtNSknClmfeop1qVnsSi8AICrYh5ahvpfsW9P317CNX26seb8atuuq/1Sx1PTNNlCbQI8E0qo9Tgd61LSjUqolEAtsuMzz04BpANHR0XUqVLi4XQvh699Du9th5Nuy2pCok9IKCz8czmNdShbrU7PILDiHUtC1RRC/G9aOwR0iaBvhj3LxQYKtT4qOBxZprS01Pam1ngvMBUhISLDPP1HC+exbBYsfhZg+cNc8cJdz9derrMJKwdly3N0UIX5eZpdjF3nFZWzYe4r1qVls2Z9NcZkFH093ercJ4+nBbRnQPpwmjRvWxG21+ZuTAbSo9jiqcltNxgPTr7co0YAc/Q4+vx+adobxn4Cnt9kVOQStNUWlFRSeq6CgpJyCs+UUnqv8fP7jXAUFZy9sq77PuXJr1WtFBDSiU/NAOkYG0ql5AJ0iA2kW6O10o1WtNYeyi1iXeop1KVnsPH4aqzZ+vlHxkQzuEM6trcPw9my4UynXJtC3A22UUrEYQT4emHjpTkqp9kAwsNWmFQrXdeIn+HQ8BEXDvV+Ad4DZFdlUucVaLWgrLgre8+FrhHPFRdvO72e9wu+wSkGAtycBPh4E+ngS4O3JDeH+xtc+npXbPCitsJKSWcjPmQVs3Heq6jVD/LzoWBnunZoH0ikygOgQX4cL+QqLle1HT7Mu1WilHM0tAaBj8wAeH9iGIR0i6BQZ4HB1m+Wqga61rlBKPQ6sBtyBeVrrPUqpl4FErfWyyl3HAwu0tlO3X7iWnIPw0RhoFGDMnOgXZnZFdWa1ak6dKeVITjFHc4s5mlPMkZxijuWWkH66hOKyGjuPVbw83KqCN9DHkzB/L1o18asK6MDzwezjQUD1bb6e+Ht54OZWtxA7W2Yh9WQhezIK+DnDCPl3vzlMucX4K9vY28MI+eaBRtBHBhAb5o97HY9zvQrPlbN5XzbrUrPYtC+bgrPleLm7cUvrUB7q04pB7cNpHuRTrzU5C2VW/iYkJOjExERTji1MVpAB84YZt/ZPWQVhbcyu6LK0rhbaOcUczS2p/Gx8VG9teLm7ER3qS0yoHy1CfAj29boolKsHdYCPp0O0BkorLBzIKuLnjAJ+zjSCPvVEIaUVxs/l4+lOXPMAOjUPqGzZBNImwh9Pd9uetD6eW2KMwvdmse1wHhVWTYifFwPahTMkLpzebZrg30jOrQAopXZorRNqfE4CXdSr4lx4f4Qx2dYDy6F5V7MrQmtNdlEpR3OMsD6SeyG8j+UWU1JtpO3prmgR4ktsqB8xYcZHbKgfLUN9aR7kU++jWXuosFg5lF1cFfJ7MgrZk1lQ9RuHl4cb7Zs2pmNlq6ZzZCBtIxrX6R8oi1WTnJbP+tQs1qVmsT+rCIAbwv0Z3CGCwR3CiY8Odon309Yk0IVjKD0DH/wKTqUaPfOY3vV2aK01OUVlVa0R43NJZYuk+KL2iIebIjrE1wjsUD9iwoxRd2yYn8uEdl1ZrZqjucX8nFnZsqkczRecLQeM96xNROOqk64dmwfQoVkAftVG1SVlFXxzIId1KVls3HeKnKIy3N0U3WKCK0M8gpgwP7N+RKchgS7MV34OPrnbuKpl/HxoN8Iuh8krLuNIThFHqo22j1WGd1FpRdV+7m6KFsE+VaEdWznajgn1JTLIBw8btxRckdaa9NNnL2rX/JxRQG7l3ZhKQaswPzpFBlJ4tpzvDuVSVmGlsbcH/duFM7hDOP3bhhPo62nyT+JcrhTo0pQS9mepgC8egiNb4M53bBbmpRUWUjILSTqeT1JaPknHT5N++mzV8+5uiqhgH2JC/UhoGULLUN+qFklksI/N+8ANjVJG+6lFiC8jOhsrSGmtySosvSjkfzySh5eHG5N6RDOkQwTdYkPkvbcTCXRhX1rDV0/B3uUw/A24cfw1vowxGjwf3EnH80nJLKTMYpy8ax7oTXx0MJNvaUmb8Ma0DDWCRoKjfimlaBroTdNAb7vMVSKuTAJd2I/WsOYFSP4Y+s2Ano/W+luLSivYlXZ+5J1PctppcoqMX+W9Pd3oEhXEg71jiG8RRNcWwTQNlBuShJBAF/bz7T9g67+g+zTo/4tZl6tYrZqD2UUkH88nKc0Yfe/POlN1E0yrJn70axtOfHQQXVsE0b5pY+lxC1EDCXRhH9vfg/UvQ+d7jFZLtTv5cotKSU7LJ7ly9P1TWj5nKk9YBvp40rVFEMM7NSU+OpiuUUFy0kyIWpJAF7b38xew4lloM4yyO/5FakYhScdPGwGels+xytu33d0U7Zs2ZlR8c+JbBBMfHURsmJ/cxi3ENZJAFzajtSb3p5WELH2ENP8bmVHwa3b8eQNllXcdRgQ04qboYCZ2jyY+OpjOkYH4eJl/t6QQrkICXVyX0+enMN2bRenhrfyr4k+k6uZMzn+SVv6NeODWpnRtEUR8dBDNAmX+DSHsSQJd1Nmx3GLWpmSxJiWLxKN5WDX09j/JXP0a5X5NcRv9JT+0biWXDApRzyTQxVVZrZqf0vNZl5rF2pQL8260b9qYxwfcwPBYDzos/Q1KBcBDKwgIktWohDCDBLqo0blyC1sP5bKmckmvU2dKcXdTdI8J4cU7ohkSF0GLEF/jWvP5d8PZ0zB1gzG3uRDCFBLookp+idEPX5uSxeb92ZSUWfDzcqdfuyYMiYtgQLtwgnwvWc5s2xw4uBZu+xs07WRO4UIIQAK9wTueW8KalJOsTcki8dhpLFZNREAj7oyPZEhcBLe0DqWRx2WuRDmxC9a+CG1HQLeH67dwIcQvSKA3MFarZndGAWtTjH74vqwzALSLaMyv+7VmSFwEnSMDr74aTlmJMeGWTwiMmn3RjUNCCHNIoDcApRUWvj+Uy9rKfnhWYSluCrrFhPDC7R0YGteU6FDfur3o6ucg5wBMXgJ+ofYpXAhRJxLoLiq/pIyN+yr74fuyKS6z4OvlTr+2F/rhwX5eV3+hmqQsgx3/B72eglb9bVi1EOJ6SKC7kLS8EtakZLEuJYsfj+ZhsWqaNG7EyK6RDK3sh1/3OpYF6bDsCWgeDwNesE3hQgibkEB3UucXLz6cXcz3h3JYm5LF3pNGP7xNuD+P9G3FkLgIbowKqvPq8JdltcCXj4ClHMa+Bx7XOMIXQtiFBLoDu9zixcY6mCWcLTfWwXRTkBATwvO3dWBInB3XZfz2H3DsWxj9HwhtbZ9jCCGumQS6yc4vXnws1wjq2i5efGvrsKrFiztFBhJyrf3w2krbDhtfg05j4cYJ9j2WEOKaSKDXA601ecVlHM0tqVpx/nx4H8spqZoLHC5evLh7bAixYX60DPUlNszPvMWLzxUYlygGRsId/yuXKArhoCTQbeh0cZkxws4trlp1/nx4nzl3IbTdFEQFGyPtm6ODaVlt1fkoR1u8WGtjbvOCdHjwa/AONLsiIcRlSKBfA4tVs+1wLj8ezavsbRvhXXC2vGofpSAyyIfYMD9GdW1OTLXQbhHsi5eHA4X2lez6DHZ/DgOeh+geZlcjhLgCCfQ62HuykMVJGSxNyuRk4TmUguaBPrQM9eX2Ls2IDTUCOzbMWHH+srfMO4vcQ8boPPpW6POs2dUIIa5CAv0qsgrPsSw5ky+TMkg9UYi7m6Jf2yY8f3sHBnUIx9fLRd9CSzl88TC4ucOYucZnIYRDc9E0uj7FpRWs+vkkS5Iz+O5gDlYNN1CCAQMAAAzOSURBVLYI4k8jO3JHl2aE+jcyu0T72/gXyNwJd38AQS3MrkYIUQsS6JUqLFa+PZjD4qQM1uzJ4my5hRYhPjw+4AZGx0fSqom/2SXWn8Ob4du34KbJ0HG02dUIIWqpQQe61pqfM4y++LKfMskpKiXQx5M7b4pkTHwkN7cMbngr0BfnwuJHIPQGGP662dUIIeqgQQZ6+ukSliZnsjgpg4OnivByd2Ng+3BGx0cyoH0T5z+Zea20NuZpKcmFiZ+Bl53uOBVC2EWtAl0pNRz4J+AOvKu1/sXQTSl1DzAT0MBPWuuJNqzzuhWcLefr3Sf4MimDH4/kAdAtJphX7+zM7Z2bEejraXKFDiDxPdi3Aoa9Cs1uNLsaIUQdXTXQlVLuwGxgCJAObFdKLdNap1Tbpw3wHNBLa31aKRVur4LroqzCyqZ9p1iSnMG61FOUVVhpFebHs0PaMjo+0lgTUxiyUmD189B6EPT4tdnVCCGuQW1G6N2Bg1rrwwBKqQXAKCCl2j5Tgdla69MAWutTti60trTW7Dyez+KkdJbvOkF+STmhfl5M7B7NnfGRdIkKbHh98aspP2vc2t+oMdw5B9yc5KYnIcRFahPokUBatcfpwKW3DLYFUEp9h9GWmam1XnXpCymlpgHTAKKjbbs6/NGcYhYnZbAkOYNjuSU08nBjaMemjImPpHebMMe6nd7RrH0RTqXApEXg7xC/XAkhroGtTop6AG2A/kAUsEUp1VlrnV99J631XGAuQEJCgr7eg+YVl7F8l3FyM+l4PkrBra1DeXzADQzv1JTG3tIXv6p9q+DHudDzMWgzxOxqhBDXoTaBngFUv7MkqnJbdenANq11OXBEKbUfI+C326TKas6VW1ifeorFSels2pdNhVXTvmljnhvRnpFdm9Ms0MfWh3RdZ07C0scgojMMnml2NUKI61SbQN8OtFFKxWIE+Xjg0itYlgATgPeVUmEYLZjDtiz0vH9vPMisDQeJCGjElN6x3BkfSYdmAfY4lGuzWo3rzctK4K73wKMB3P0qhIu7aqBrrSuUUo8DqzH64/O01nuUUi8DiVrrZZXPDVVKpQAW4Hda61x7FHxPtxZ0jw3lltahuNtqabWGaOvbcHgT3PEWNGlndjVCCBtQWl93K/uaJCQk6MTERFOO3eBl7IT3hkC7EXDPR7JghRBORCm1Q2udUNNzculHQ1NaZMyi6B8Bv5olYS6EC2mQt/43aF//HvIOwwPLwTfE7GqEEDYkI/SGZPciSJ4PfX8LMb3NrkYIYWMS6A3F6WOw/BmI6gb9/mB2NUIIO5BAbwgsFfDlVGM2xbHvgrvccCWEK5IeekOw5a+Qtg3GvAvBMWZXI4SwExmhu7pj38OWN+HGCdDlbrOrEULYkQS6Kzt7Gr6YCkEt4bY3za5GCGFn0nJxVVrDV09B0UmYssaYGlcI4dJkhO6qkj6ClKUw4HmIutnsaoQQ9UAC3RVl74ev/wCxfaHX02ZXI4SoJxLorqai1Fh9yMMb7nxHVh8SogGRHrqrWf8ynNwF4z+BgOZmVyOEqEcyfHMlB9fB1n9BwkPQ/nazqxFC1DMJdFdRlA2Lfw1N2sOwv5hdjRDCBNJycQVaw5Jfw7kCmLwEPGUZPiEaIgl0V7BtDhxcCyPehIiOZlcjhDCJtFyc3cndsPZFaDscuk81uxohhIkk0J3ZiV3w6QTwCYFRs2X1ISEaOAl0Z7V7Ebw3FKwWmLgA/MLMrkgIYTLpoTsbqwXW/wm++ye06An3fAiNI8yuSgjhACTQncnZ07DoITi0HhKmwPA3wMPL7KqEEA5CAt1ZnEo1+uUF6XDHW5DwoNkVCSEcjAS6M0j9ChY/Cp6+8MByiO5pdkVCCAckge7IrFbY/DpsfgMib4ZxH8v8LEKIy5JAd1TnCmHxI7BvJXSdBLf/Azy9za5KCOHAJNAdUc5BWDABcg8Zd392nyrXmAshrkoC3dHsXw1fPAzunjB5KcT2MbsiIYSTkEB3FFrDN3+HDa9A084wfj4ERZtdlRDCiUigO4LSIlj6mLEGaKe7YOTb4OVrdlVCCCcjgW62vCOwYBJkp8LQV+CWx6VfLoS4JhLoZjq0AT6vvEFo0iK4YZC59QghnJpMzmUGreH7t+HjscZ15dM2SpgLIa5brQJdKTVcKbVPKXVQKTWjhucfUEplK6WSKz8etn2pLqL8LHw5Dda8YKz7+dBaCGlldlVCCBdw1ZaLUsodmA0MAdKB7UqpZVrrlEt2/Uxr/bgdanQd+Wnw2SRjHvOBL0Cf30q/XAhhM7XpoXcHDmqtDwMopRYAo4BLA11cydFvYeH9YCmDCQug3XCzKxJCuJjatFwigbRqj9Mrt11qrFJql1JqkVKqRU0vpJSappRKVEolZmdnX0O5Tkhr2DYXPhwFPsEwdYOEuRDCLmx1UvQrIEZr3QVYC3xQ005a67la6wStdUKTJk2u7UhW6zUXWe8qSmHZ4/D17+CGITB1PYS1MbsqIYSLqk3LJQOoPuKOqtxWRWudW+3hu8Bfr7+0y9jxvnFHZfN4YwbCyJuMr70D7XbIa1J4Aj67FzISoe/vof9z4CYXFQkh7Kc2gb4daKOUisUI8vHAxOo7KKWaaa1PVD4cCaTatMrqQlpBy16QsQP2Lr+wPawtNL/pQshHdDJvdsK0H40wLy2Cez6CuJHm1CGEaFCuGuha6wql1OPAasAdmKe13qOUehlI1FovA55USo0EKoA84AG7Vdx6gPEBxpJsmUlGuGfshMMbYdcC4zk3T4joWBnwlSEf1hbc3O1WGgA7PoAVz0JgFNy3BCLi7Hs8IYSopLTWphw4ISFBJyYm2vZFtYbCTCPgM3dWfk6G0kLjeS9/aNbVCPfzIR/YwjaXDlaUwaoZkPgetB4Id80zToIKIYQNKaV2aK0TanrOtW79VwoCI42P820OqxVyD14c8tvmGJcPAvg1ubhV0/wm8Aut23GLThmXJB7/Hno9BYNesv9vAkIIcQnXCvSauLlBk7bGR9cJxraKUsjac6FVk7kTDqwBKn9bCY65OOSb3QhefjW/fsZOo19ekgdj34POd9XHTyWEEL/g+oFeE49GlW2Xmy5sO1cIJ36qDPkdkL4d9nxpPKfcIDzu4itrwuNg9yL46inwj4CH1kCzLub8PEIIQUMN9Jp4BxirA1VfIehMVmWbZueFq2qSPjKec28EllKI6QN3f1D3No0QQtiYBPqVNI6AdiOMDzBOup4+ciHg/cLg1ieN5eKEEMJkEuh1oZRxHXxIK+mVCyEcjty6KIQQLkICXQghXIQEuhBCuAgJdCGEcBES6EII4SIk0IUQwkVIoAshhIuQQBdCCBdh2vS5Sqls4Ng1fnsYkGPDcpydvB8Xk/fjAnkvLuYK70dLrXWNa3iaFujXQymVeLn5gBsieT8uJu/HBfJeXMzV3w9puQghhIuQQBdCCBfhrIE+1+wCHIy8HxeT9+MCeS8u5tLvh1P20IUQQvySs47QhRBCXEICXQghXITTBbpSarhSap9S6qBSaobZ9ZhFKdVCKbVRKZWilNqjlHrK7JocgVLKXSmVpJRabnYtZlNKBSmlFiml9iqlUpVSt5hdk1mUUs9U/j35WSn1qVLK2+ya7MGpAl0p5Q7MBkYAccAEpVScuVWZpgJ4VmsdB/QEpjfg96K6p4BUs4twEP8EVmmt2wM30kDfF6VUJPAkkKC17gS4A+PNrco+nCrQge7AQa31Ya11GbAAGGVyTabQWp/QWu+s/PoMxl/WSHOrMpdSKgq4HXjX7FrMppQKBPoC7wForcu01vnmVmUqD8BHKeUB+AKZJtdjF84W6JFAWrXH6TTwEANQSsUA8cA2cysx3VvA7wGr2YU4gFggG3i/sgX1rlLKz+yizKC1zgD+BhwHTgAFWus15lZlH84W6OISSil/4Avgaa11odn1mEUpdQdwSmu9w+xaHIQHcBPwH611PFAMNMhzTkqpYIzf5GOB5oCfUupec6uyD2cL9AygRbXHUZXbGiSllCdGmM/XWn9pdj0m6wWMVEodxWjFDVRKfWxuSaZKB9K11ud/a1uEEfAN0WDgiNY6W2tdDnwJ3GpyTXbhbIG+HWijlIpVSnlhnNhYZnJNplBKKYz+aKrW+h9m12M2rfVzWusorXUMxp+LDVprlxyF1YbW+iSQppRqV7lpEJBiYklmOg70VEr5Vv69GYSLniD2MLuAutBaVyilHgdWY5ypnqe13mNyWWbpBdwH7FZKJVdu+6PWeqWJNQnH8gQwv3Lwcxh40OR6TKG13qaUWgTsxLg6LAkXnQJAbv0XQggX4WwtFyGEEJchgS6EEC5CAl0IIVyEBLoQQrgICXQhhHAREuhCCOEiJNCFEMJF/D9SMivxuriPcAAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAD4CAYAAAD8Zh1EAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+j8jraAAAgAElEQVR4nO3dd3hUVf7H8fchhSTUmACBJJAAoXciRapSRF2agDRpK2ABRXQL7u5PXdd1LaDSLICIdBAs4LIKSFN6EumQSkmCQAglEAhp5/fHjRgQJCEzOTOT7+t5eB5n5mbuhyH5eHPvuecorTVCCCGcXynTAYQQQtiGFLoQQrgIKXQhhHARUuhCCOEipNCFEMJFuJvasb+/vw4JCTG1eyGEcEqRkZFntdaVbvWasUIPCQkhIiLC1O6FEMIpKaWO3+41OeUihBAuQgpdCCFchBS6EEK4CGPn0G8lKyuLpKQkMjIyTEdxKl5eXgQFBeHh4WE6ihDCIIcq9KSkJMqVK0dISAhKKdNxnILWmtTUVJKSkggNDTUdRwhhkEOdcsnIyMDPz0/KvBCUUvj5+clvNUIIxyp0QMr8LshnJoQAByx0IYQLyc2FI/+FQ6sgN8d0GvOunIPvX4PUeLu8vUOdQxdCuAitIeZb2PBvOL3feq5Sfbj/b1C/J5S03yoz0mDnR7BtOly7BOWrgV8tm+9GjtCLoGzZsrd97dixYyxevPj643nz5jF+/PjiiCWEOVpD3PcwpwssGQRZ6fDoHOj/KegcWD4MZnWCmLXWtq4u8wpsnQpTm8LGf0NoR3h6K9w72i67kyN0O/ml0IcMGVKor8vJycHNzc1OqYSwo2NbYcPrcGIbVAiGXjOg6WBwy6uZBr1h/+ew6T+weAAEtYIH/gE1O5nNbQ/Z1yByHmyZDOlnoHZXuP/vENjCrrt12EL/5+qDHDqZZtP3bFCtPK/0bHjb1ydNmkRwcDDjxo0D4NVXX8Xd3Z2NGzdy/vx5srKyeP311+ndu/cd9zVp0iQOHz5Ms2bNGDFiBL6+vpw8eZIePXoQHx9P3759efvttwHrSP/JJ59k/fr1zJw5k/bt29vmLyxEcUiKhI2vQ/wGKBsAD0+GFsPBvfSN25Vyg6aDoFE/2LMINr8N83tBSAer2Ku3MZPflnKy8v5u70BaEtRoD4/Nhxpti2X3csoln4EDB7J8+fLrj5cvX86IESP48ssviYqKYuPGjbz44osUZB3WN998kw4dOrBnzx4mTpwIwJ49e1i2bBn79+9n2bJlJCYmApCenk7r1q3Zu3evlLlwHqf2w+JBMOcB+HkvdP83TNgDrcb8tszzc/OAliPh2Sjo8RakRMPcB2Fhfzj5U7HFt6ncHNi7FGaEw+oJUL4qDP8aRn5TbGUODnyE/ntH0vbSvHlzzpw5w8mTJ0lJScHX15eAgAAmTpzIli1bKFWqFMnJyZw+fZqAgIBCv3+XLl2oUKECAA0aNOD48eMEBwfj5uZGv379bP3XEcI+UqJh4xtw6CvwqgAP/B+0fhJKlyvc+3h4QZunoMUw2DUbtr4PszpDvT9YF0+rFH8HFFpuLhz+2vo8zsZAQBMYshzCuhu58OuwhW7KgAEDWLFiBadOnWLgwIEsWrSIlJQUIiMj8fDwICQk5K5v4ild+tejFjc3N7KzswHr1n05by4c3rkE2PQW7F8OHj7Q8S/Qdhx4Vyza+3qWgfbPQ/gfYceHsH2GNdSxUT/o/BL417ZNflu6eRRPpXrWqZV6PaGUuRMfUug3GThwIGPGjOHs2bNs3ryZ5cuXU7lyZTw8PNi4cSPHj992KuIblCtXjkuXLtk5rRDF4GISbHkHfloIpTyg7Xho9zyU8bPtfrzKQ+e/Wqdstk23hvkd/AKaDoFOfwHfGrbd393QGhI2Whd/kyPBNxQenW39z6eU+YMyKfSbNGzYkEuXLhEYGEjVqlUZOnQoPXv2pHHjxoSHh1OvXr0CvU+TJk1wc3OjadOmjBw5El9fXzsnF8LGLp2GH9+FiLlWkYX/ETq8COUKf7qxUHzuga6vQJtn4Mf3YPcc2LfMutDa8U/WGG4Tjm21hh4e35o3imd63igex5kUTxXkAp89hIeH65tXLDp8+DD169c3ksfZyWcnbObKOet89s5ZkJMJzYdCxz9Dxepm8qSdtIb/Rc0HVcoaw91+IpS95SpstnfDKJ4q1mdxq1E8xUQpFam1Dr/Va3KELoSwXL0AOz6A7R9A5mVo8hh0+qtd7mgslPLV4A/vQrvnrOGAOz+EyE+h9VNw37PWEb09nNpvXeyMXgM+ftD9dQh/Ajx97LM/G5BCL6L9+/czbNiwG54rXbo0O3fuNJRIiEK6dhl2fQxbp0HGBesGoM5/g8oFO71YbHxDoM9M6wLqpjd/PR3Tdjy0edo6B28LKdHWzU8Hv4TSFawx8q2fKvwoHgOk0IuocePG7Nmzx3QMIQov66p1fvyHd+HKWajTwxouWLWp6WS/zz8M+n8CHV6wjqA3vWEdtbd73rqg6lnm7t733FHY/JZ1vt7Dxzq10nYceDvP9S8pdCFKmuxM+Gm+dV760s9QszPc/w8Ivtd0ssKp0hAGLYLkKKvY178C22daF25bjrTGuRfEDaN43K0Sb/c8lPG3a3x7kEIXoqTIyYZ9S62x5BdPQHAba8hdaAfTyYomsAU8vgJO7LCGE377V9g2zTrCbv747UehmBrFY0dS6EK4utxcazz3xjfgXDxUaw4934NaXVxrGtvqbaxb7RM2w4Z/wTfPW6N1Ok2yLvD+Mk7c0Ubx2FCBCl0p1QOYCrgBc7TWb970enXgM6Bi3jaTtNZrbJxVCOdx7RLkZptOAcd+tIr8zCGo3BAGLYa6D7tWkd+sZidrmtrYdVaxf/WUdSTe8S+QGut4o3hs6I6FrpRyA2YC3YAkYLdSapXW+lC+zf4BLNdaf6iUagCsAULskNeuLly4wOLFi3nmmWcK9XUPP/wwixcvpmLFwt0CPW/ePLp37061ataNEiEhIURERODv73zn7kSek3usAo39znSSX/nVhv5zoUFfo7elFyuloE53COsGh1dbNwR9kTcHeYPe1pQClV3vvo2CHKG3AuK01gkASqmlQG8gf6Fr4JcxQxWAk7YMWVwuXLjABx988JtCz87Oxt399h/VmjV398vIvHnzaNSo0fVCL4g7ZRGGnDlsFfnhVeBVETr8yTEuqpULsOYXcSuh3zNKQYNeUO8Ra+GNcgFQtYnpVHZTkH/lQCAx3+MkoPVN27wKrFVKPQuUAboWOdn/JlkD+20poDE89OZtX540aRLx8fE0a9YMDw8PvLy88PX15ciRI8TExNCnTx8SExPJyMhgwoQJjB07Fvj1yPry5cs89NBDtG/fnm3bthEYGMjXX3+Nt7f3b/a1YsUKIiIiGDp0KN7e3mzfvh2A6dOns3r1arKysvj888+pV68er776KvHx8SQkJFC9enWWLFli289F3L3UeGtM9P7PwbOsdeTX5mlrFkLhOEq5WUfsLs5Wv38NBuZprYOAh4EFSqnfvLdSaqxSKkIpFZGSkmKjXdvOm2++Sa1atdizZw/vvPMOUVFRTJ06lZiYGADmzp1LZGQkERERTJs2jdTU1N+8R2xsLOPGjePgwYNUrFiRlStX3nJf/fv3Jzw8nEWLFrFnz57rpe/v709UVBRPP/00kydPvr79oUOHWL9+vZS5o7hwAr4eDzPuhSPfWDe7PL8POk+SMhfGFOQIPRkIzvc4KO+5/J4AegBorbcrpbwAf+BM/o201rOAWWDN5fK7e/2dI+ni0qpVK0JDQ68/njZtGl9++SUAiYmJxMbG4ud344xzoaGhNGvWDICWLVty7NixQu3z0Ucfvf61X3zxxfXne/XqdcsjfVHM0n6GH6ZYy4upUtY84O0nQtnKppMJUaBC3w2EKaVCsYp8EHDzQpkngC7APKVUfcALcLxD8EIqU+bXO842bdrE+vXr2b59Oz4+PnTu3PmW86LfPOf51atXC7XPX74+/3zpN2cRBqSf/fVW89xsa3KmDn+CCoGmkwlx3R0LXWudrZQaD3yHNSRxrtb6oFLqNSBCa70KeBGYrZSaiHWBdKQ2NY1jEfzeHOYXL17E19cXHx8fjhw5wo4dO+y6P+Egrp6HbTOshReyr0KTQdbc3PeE3vlrhShmBbr0nTemfM1Nz72c778PAe1sG634+fn50a5dOxo1aoS3tzdVqlS5/lqPHj346KOPqF+/PnXr1qVNm6IvaDty5EieeuqpGy6KCgdx7RLs+MhaaOHaRWsBg06ToFId08mEuC2ZD91FyGdnI5lXrNMqP74HV89B3UesCasCGplOJgQg86ELcWfZ1yDyM/hhMlw+bd0W/8DfIbCl6WRCFJgUejEYN24cW7duveG5CRMmMGrUKEOJxHU5WbBnMWx+G9KSoEY7GDAPatxnOpkQheZwha61RrnYPBMzZ8606/s74fVn83JzYP8KayGD80chMBx6z7CmknWx7z9RcjhUoXt5eZGamoqfn5/Llbq9aK1JTU3Fy6uAcz+XdLm51u35G9+As9FQpTEMXgZ1HpQiF07PoQo9KCiIpKQkHPEuUkfm5eVFUFCQ6RiOTWuIXWvNl31qH/jXhQGfQf1eJWfCKuHyHKrQPTw8brgzU4gi0xoSNlmz7SXtBt9Q6DsLGvf/dX5sIVyEQxW6EDZ1fLt1RH78RygfBD2nQbMht1/BRggnJ4UuXE9ylHVEHrceylaBh96BliPAvfSdv1YIJyaFLoouJ9uasOrCcdNJrEWP4zeA9z3Q7V9w72jw9DGdSohiIYUuiiY3B74cCwdWWqc1TI8UcfOA+/9uzUleupzZLEIUMyl0cfdyc605wQ+shK7/tOYEF0IYI+O1xN3RGv47EfYuhs5/kzIXwgFIoYvC0xq+nWQt8tD+BWs6WSGEcVLoonC0hnUvw86PoM046PKy+fPmQghACl0U1sY3YNs0a/TIg/+WMhfCgUihi4Lb8g5seRuaD7PGdkuZC+FQpNBFwWybbt112WQg9Jwq858I4YDkp1Lc2a7ZsPYf0KAP9P5A5kARwkFJoYvfFzkP1vzJWoqt3xxwk1sXhHBUUuji9vYsgdXPQ+1uMOBTmdRKCAcnhS5u7cBK+PoZCO0IAxfIxFZCOAEpdPFbh1fDyjEQ3AYGLwEPb9OJhBAFIIUubhSzFj4fBYEtYOhy8CxjOpEQooCk0MWv4jfAssehSkMYukJmKxTCyUihC8uxH2HJEPAPg2FfgndF04mEEIVUoEJXSvVQSkUrpeKUUpNu8fp7Sqk9eX9ilFIXbB9V2M2JnbDoMahYHYZ9BT73mE4khLgLdxxUrJRyA2YC3YAkYLdSapXW+tAv22itJ+bb/lmguR2yCntIjoJF/aFcFRixCspWMp1ICHGXCnKE3gqI01onaK0zgaVA79/ZfjCwxBbhhJ2d2g8L+oK3L4xYDeUCTCcSQhRBQQo9EEjM9zgp77nfUErVAEKBDUWPJuzqzGGY3xs8y1plXiHIdCIhRBHZ+qLoIGCF1jrnVi8qpcYqpSKUUhEpKSk23rUosLNx8FkvKOVhnWbxrWE6kRDCBgpS6MlAcL7HQXnP3cogfud0i9Z6ltY6XGsdXqmSnKs14txR+Kwn6FyrzP1qmU4khLCRgsy0tBsIU0qFYhX5IGDIzRsppeoBvsB2myYUtnMh0Toyz74KI76BSnVNJxKiWGit2ZFwjpMXrpqOAkCz6hWpVamszd/3joWutc5WSo0HvgPcgLla64NKqdeACK31qrxNBwFLtdba5ilF0aX9bB2ZZ1y0jswDGplOJESx2BZ3lslro4k64TijqV/v08hMoQNordcAa2567uWbHr9qu1jCpi6fgfm9ID0Fhn8N1ZqZTiSE3UUcO8eUtTFsT0ilagUv3ujbmHa1/VCYX2nLt4x9Zi6Vya1dXXqqNZrlYhI8vhKCwk0nEsKu9iddZMq6aDZFp+BftjSv9GzA4FbV8fJw/YVZpNBd2dXzsKAPnEuAIcuhxn2mEwlhN9GnLvHuumi+O3iaij4eTHqoHsPb1sDHs+TUXMn5m5Y0GWmwsB+kHIFBS6BmJ9OJhLCLhJTLvL8+ltX7TlLW052JXevwx/YhlPMqeQuySKG7omuXYdEA+HkvPLYAwrqaTiSEzSWeu8K072NZGZVEaXc3nu5Ui7Eda1LRx9N0NGOk0F1N5hVYMgiSdkH/T6Hew6YTCWFTpy5mMGNjLMt2J6KUYlS7UJ7qVItK5WRVLSl0V5KVAcuGWlPhPjoLGvYxnUgImzl7+RofbopnwY7j5OZqBrUKZvz9YQRU8DIdzWFIobuK7Ez4fKS1SEWvGdDkMdOJhLCJC1cymbUlgXnbjpGRlcOjLYKY0CWM4Ht8TEdzOFLoriAnG1Y+ATH/g0emQIthphMJUWSXMrL4dOsxZm9J4HJmNj2bVGNC1zC73JDjKqTQnV1uDnz1FBxeBQ/+B+4dbTqREEVyNTOH+duP8dHmeM5fyaJ7gyq80L0O9QLKm47m8KTQnVluLqx6FvZ/Dl1fhbbPmE4kxF27lp3Dkp0nmLExnrOXr9GpTiVe7F6HJkGyHGJBSaE7K63hvy/AnkXQ+SVoP/HOXyOEA8rKyWVFZBLTv4/l5MUMWofew4ePt+DeEFkKsbCk0J2R1vDtSxD5qVXknf5qOpEQhZaTq/l6TzLvr4/lxLkrNK9ekXcGNOW+Wn4oZX6+FWckhe5stIb1r8DOD6HNM9DlFZBvfuFEcnM1/ztwinfXRROfkk6DquWZOzKc++tWliIvIil0Z7PpP7B1KoQ/AQ++IWUunIbWmu8Pn2HKuhgO/5xGWOWyfDi0BQ82DKBUKfk+tgUpdGeyZTJsfguaPw4PT5YyF05Ba82PcWeZvDaGvYkXqOHnw/sDm9GzaTXcpMhtSgrdWWybARv+BY0fg57ToJStl4MVriTuzCXeWx/LtrizmF5xJjdXk5aRTWBFb97q15hHWwTh4Sbfv/Yghe4Mds2GtX+HBr2hz4dQyvXndRZ353hqOlO/j+Wrn5Lx9nDjkSZV8XaAecDrBJSjf8sgSrubz+LKpNAdXdR8WPMnqPsw9PsE3OSfTPzWyQtXmb4hjs8jEnErpRjdoSZPdqyJX1mZsKokkXZwZHuXwqrnoHZXGDAP3Ere/M7i9525lMEHG+NZvPMEGs3Q1tUZd39tKpeXCatKIil0R3XgC/jqaQjtAAMXgrscaYlfnU/P5KMt8Xy27RhZOZoBLYMY/0BtgnxlwqqSTArdER3+BlaOhuDWMHgpeHibTiQcRFpGFnN+OMrcH4+SnplNn2aBTOgSRoh/GdPRhAOQQnc0MWutaXCrNbfWAfWUH1QB6deymbftGLO2JHDxahYPNw7g+a51qFOlnOlowoFIoTuShE2w7HGo0gAeXwleMrtcSZeRlcPCHcf5cFM8qemZdKlXmYnd6tAosILpaMIBSaE7imNbYfEg8KsNw74Cb5lhriTLzM5lWUQiMzbEcjrtGu1r+/NC9zq0qO5rOppwYFLojiBxFyx+DCoGw/CvwUdmmSupsnNy+eKnZKZ9H0vS+auE1/Dl/YHNaVvLz3Q04QSk0E1LjoKF/aBsZRi+CspWMp1IGJCbq1m97yRT18eScDadxoEVeL1PIzrVqSQTVokCK1ChK6V6AFMBN2CO1vrNW2zzGPAqoIG9WushNszpmk7thwV9rdMrI1ZD+aqmE4liprXmu4OneW9dDNGnL1G3Sjk+HtaS7g2qSJGLQrtjoSul3ICZQDcgCditlFqltT6Ub5sw4CWgndb6vFKqsr0Cu4wzR2B+H2sUy4jVUCHIdCJRjLTWbI5JYcraGPYnX6SmfxmmDW7OHxpXlZkHxV0ryBF6KyBOa50AoJRaCvQGDuXbZgwwU2t9HkBrfcbWQV3K2TiY38uak2XEavANMZ1IFKPt8alMWRtNxPHzBPl6807/JvRtHoi7TFgliqgghR4IJOZ7nAS0vmmbOgBKqa1Yp2Ve1Vp/e/MbKaXGAmMBqlevfjd5nd+5o/BZT2tx55H/Bb9aphOJYhJ5/Dzvrotma1wqVcqX5vU+jXgsPBhPdylyYRu2uijqDoQBnYEgYItSqrHW+kL+jbTWs4BZAOHh4aZn9Sx+FxKtI/PsqzDiG6hcz3QiUQwOJF/k3XUxbDhyBr8ynvzfHxowtHV1vBxgFkThWgpS6MlAcL7HQXnP5ZcE7NRaZwFHlVIxWAW/2yYpXUHaz1aZX70AI1ZBQCPTiYSdxZy+xHvrYvjfgVNU8PbgLz3qMqJtCGVKy+AyYR8F+c7aDYQppUKxinwQcPMIlq+AwcCnSil/rFMwCbYM6tQup1hlfvkMDPvSuq1f2Ny17ByW7U7k4pUs01GIOXOZb/adpIynOxO6hPFEh1DKe8lsmcK+7ljoWutspdR44Dus8+NztdYHlVKvARFa61V5r3VXSh0CcoA/a61T7RncaVw5B/N7w8UkGLoCgluZTuSSMrNzGbfoJ9YfPm06CgA+nm482bEWT3asiW8ZT9NxRAmhtDZzKjs8PFxHREQY2XexuXrBugCaEg1Dl0PNzqYTuaTsnFyeW/oTa/af4rXeDRnSyvwF91JKyfBDYRdKqUitdfitXpOTefaSkWbdAXrmMAxeImVuJzm5mhc/38ua/af4xyP1Gd42xHQkIYyRQreHzHRrbpaf98Bj8yGsm+lELik3V/PSF/v4es9J/vxgXUZ3qGk6khBGSaHbWtZVWDIIEndaa4DWe8R0IpekteblVQdYHpHEc13CGHd/bdORhDBO7miwpexr1nzmR3+APh9Bo0dNJ3JJWmv+9c1hFu44wZOdajKxa5jpSEI4BDlCt5WcLGulobj10Gs6NB1oOpFL0lrz1rfRzN16lFHtQpjUo55MYiVEHjlCt4WcbGsN0Og18PBkaDHcdCKXNfX7WD7aHM/Q1tV5+Q8NpMyFyEcKvahyc+Crp+HQV/DgG9BqjOlELuuDTXG8vz6W/i2D+FfvRlLmQtxECr0ocnNh9XOwfzl0eRnajjOdyGV98uNR3v42ml5Nq/FWvyYyxluIW5BCv1taw5o/wU8LodNfocOLphO5rAU7jvOvbw7xUKMA3n2sKW5S5kLckvNdFI2cB1unmk5hXQS9mAjtnofOL5lO47KW707k/746QNf6lZk6qLnMGS7E73C+Qi8bAIEtTaewVHsG2jwNci7XLr76KZm/frGPjnUqMXNoC5k3XIg7cL5Cr9vD+iNc2n/3/cwLy/fQJtSPjx9vSWl3mTtciDuRQx7hcNYdOs2EpT/Rorovc0aE4+0pZS5EQUihC4eyKfoM4xZF0TCwAp+OulcWgxCiEKTQhcPYGneWJxdEElalLPNHtaKcLAghRKFIoQuHsOvoOUZ/FkGIXxkWPNGaCj5S5kIUlhS6MC7qxHlGfbqLqhW9WDi6NffICj9C3BUpdGHUgeSLjJi7C/9ypVk8ug2VypU2HUkIpyWFLow5ciqNxz/ZSXkvDxaPaUNABS/TkYRwalLowoi4M5cYOnsnXu5uLB7TmsCK3qYjCeH0pNBFsTt6Np0hs3eilGLRmNbU8CtjOpIQLkEKXRSrxHNXGDJ7B9m5msVjWlOrUlnTkYRwGVLooticvHCVIXN2cCUzh4VPtKZOlXKmIwnhUqTQRbE4k5bB0Dk7uZCexfw/tqJBtfKmIwnhcuS+amF3Zy9fY8icnZxOy2DBE61oGlzRdCQhXJIcoQu7Op+eyeNzdpJ0/gpzR95Lyxr3mI4khMsqUKErpXoopaKVUnFKqUm3eH2kUipFKbUn789o20cVzubi1SyGz91Fwtl0Zg8Pp01NP9ORhHBpdzzlopRyA2YC3YAkYLdSapXW+tBNmy7TWo+3Q0bhhC5fy2bkp7s4ciqNj4e1pENYJdORhHB5BTlCbwXEaa0TtNaZwFKgt31jCWd2JTObP366m31JF5k+uAUP1KtiOpIQJUJBCj0QSMz3OCnvuZv1U0rtU0qtUEoF3+qNlFJjlVIRSqmIlJSUu4grHF1GVg5j5kcQcfwc7w9sRo9GAaYjCVFi2Oqi6GogRGvdBFgHfHarjbTWs7TW4Vrr8EqV5FdwV3MtO4enFkayLT6Vd/o3pWfTaqYjCVGiFKTQk4H8R9xBec9dp7VO1Vpfy3s4B3CQVZxFccnKyWX84p/YFJ3CG30b069lkOlIQpQ4BSn03UCYUipUKeUJDAJW5d9AKVU138NewGHbRRSO7kxaBuMWRbHu0Gn+2ashg1tVNx1JiBLpjqNctNbZSqnxwHeAGzBXa31QKfUaEKG1XgU8p5TqBWQD54CRdswsHMS59Ew+3hzPZ9uPkZWj+ccj9RlxX4jpWEKUWEprbWTH4eHhOiIiwsi+RdFcvJrFJz8k8MmPR7mSlUPfZoE81yWMEH+ZNVEIe1NKRWqtw2/1mtz6Lwos/Vo287Yd4+PN8aRlZPNI46o83zWMMJlkSwiHIIUu7igjK4eFO47z4aZ4UtMz6Vq/MhO71aFhtQqmowkh8pFCF7eVmZ3Lst0nmLExjtNp1+gQ5s8L3erQvLqv6WhCiFuQQhe/kZ2TyxdRyUz9PpbkC1e5N8SXqYOay1wsQjg4KXRxXW6uZvW+k7y/PpajZ9NpElSBNx5tTMcwf5RSpuMJIe5ACl2gtea7g6d5b10M0acvUS+gHLOGtaRbgypS5EI4ESn0EkxrzaaYFKasjeZAcho1K5Vh+uDmPNK4KqVKSZEL4Wyk0EuobfFnmbI2hsjj5wny9WbygKb0aVYNdzdZ80QIZyWFXsJEHj/HlLUxbItPJaC8F//u24gBLYPxdJciF8LZSaGXEAeSLzJlbTQbo1PwL+vJy39owJDW1fHycDMdTQhhI1LoLi761CXeWxfDtwdPUcHbg7/2qMeI+2rg4yn/9EK4GvmpdlFHz6bz/voYVu09SRlPdyZ0CeOJDqGU9/IwHU0IYSdS6C4m8dwVpm+IZWVUMp5upXiyYy2e7FgT3zKepqMJIexMCt1FnE7LYMaGOJbuPoFCMbxtDZ7pXJtK5UqbjiaEKCZS6E4u9fI1Ptocz/ztx8nJ1Tx2bzDj769NtYrepqMJIeLp2fYAAAsMSURBVIqZFLoTO56azmMfbyfl0jX6Ng9iQpcwqvv5mI4lhDBECt1JJZ2/wpDZO8nMzmXV+PY0CpSpbIUo6eRuEif088WrDJm9k0sZWSx4orWUuRACkCN0p3PmUgZDZ+/kXHomC0dLmQshfiVH6E4k9fI1hs7eyam0DOaNupdmwRVNRxJCOBApdCdx4Uomwz7ZxYlzV5gzIpzwkHtMRxJCOBgpdCeQlpHFiLm7iDtzmVnDw7mvlr/pSEIIBySF7uAuX8tm1Ke7OXgyjQ+GtqBTnUqmIwkhHJRcFHVgVzNzeGLebvYkXmDG4OZ0bVDFdCQhhAOTI3QHlZGVw9gFEew6do53H2vKQ42rmo4khHBwUugOKDM7l2cWRfFD7Fne7teE3s0CTUcSQjiBAhW6UqqHUipaKRWnlJr0O9v1U0pppVS47SKWLFk5uTy7JIoNR85YqwmFB5uOJIRwEncsdKWUGzATeAhoAAxWSjW4xXblgAnATluHLClycjUvLN/LdwdP80rPBgxtXcN0JCGEEynIEXorIE5rnaC1zgSWAr1vsd2/gLeADBvmKzFyczV/WbGP1XtP8tJD9RjVLtR0JCGEkylIoQcCifkeJ+U9d51SqgUQrLX+7++9kVJqrFIqQikVkZKSUuiwrkprzd+/2s/KqCRe6FaHJzvVMh1JCOGEinxRVClVCngXePFO22qtZ2mtw7XW4ZUqyXhqsMr8n6sPsWRXIuPur8WzD9Q2HUkI4aQKUujJQP4rc0F5z/2iHNAI2KSUOga0AVbJhdE701rzn/8dYd62Y4xuH8qfutdFKWU6lhDCSRWk0HcDYUqpUKWUJzAIWPXLi1rri1prf611iNY6BNgB9NJaR9glsQt5b10Ms7YkMLxtDf7+SH0pcyFEkdyx0LXW2cB44DvgMLBca31QKfWaUqqXvQO6qhkbYpm2IY5B9wbzas+GUuZCiCIr0K3/Wus1wJqbnnv5Ntt2Lnos1zZrSzyT18bwaPNA/t23MaVKSZkLIYpO7hQtZp9tO8Yba47wSJOqvN2/CW5S5kIIG5FCL0ZLdp3glVUH6d6gCu8PbIa7m3z8QgjbkUYpJisjk/jbl/u5v24lpg9pjoeUuRDCxqRVisHqvSf584q9tKvlz4ePt6S0u5vpSEIIFySFbmffHjjF88v2EB5yD7OGt8TLQ8pcCGEfUuh2tOHIaZ5dEkWToArMHXkvPp6ynogQwn6k0O3kh9gUnloYRb2A8swb1YqypaXMhRD2JYVuBzsSUhkzP4Ka/mVY8EQrKnh7mI4khCgBpNBtLPL4Of44bzfBvj4sGt2aij6epiMJIUoIKXQb2pt4gZFzd1OlvBeLRrfGr2xp05GEECWIFLqNHDx5keFzd1GxjAeLx7Smcnkv05GEECWMFLoNxJy+xLBPdlHG043Fo9tQtYK36UhCiBJICr2IElIuM2T2TtxLKRaPaUPwPT6mIwkhSigp9CI4kXqFIbN3AprFY9oQ4l/GdCQhRAkmg6PvUuK5KwyevYOM7ByWjm1D7cplTUcSQpRwUuiFdC49k482xzN/+zE83EqxZEwb6gWUNx1LCCGk0Avq4tUs5vyQwNwfj3IlK4e+zQJ5vmsdqvvJOXMhhGOQQr+D9GvZzNt2jI83x5OWkc0jjavyfNcwwqqUMx1NCCFuIIV+GxlZOSzccZwPNsVzLj2TrvUrM7FbHRpWq2A6mhBC3JIU+k2uZeewfHci0zfEcebSNTqE+fNCtzo0r+5rOpoQQvwuKfQ82Tm5fBGVzNTvY0m+cJVWIfcwbXBz2tT0Mx1NCCEKpMQXek6u5pt9J3lvXQzHUq/QNKgC/3m0MR3C/FFKFnAWQjiPElvoWmu+O3iKd9fFEHP6MvUCyjF7eDhd61eWIhdCOKUSV+haazZFpzBlXTQHktOoWakMM4Y05+FGVSlVSopcCOG8SlShb4s7y+S10USduEDwPd5MGdCU3s2q4e4mMyAIIZxfiSj0iGPnmLI2hu0JqVSt4MUbfRszIDwIDylyIYQLKVChK6V6AFMBN2CO1vrNm15/ChgH5ACXgbFa60M2zlpo+5MuMmVdNJuiU/AvW5pXejZgcKvqeHm4mY4mhBA2d8dCV0q5ATOBbkASsFspteqmwl6stf4ob/tewLtADzvkLZDoU5d4d1003x08TUUfDyY9VI/hbWvg41kifiERQpRQBWm4VkCc1joBQCm1FOgNXC90rXVavu3LANqWIQsqIeUy76+PZfW+k5T1dGdi1zr8sX0I5bxkkWYhhOsrSKEHAon5HicBrW/eSCk1DngB8AQeuNUbKaXGAmMBqlevXtist5V47grTvo9lZVQSpd3deLpTLcZ2rCkLNAshShSbnYPQWs8EZiqlhgD/AEbcYptZwCyA8PDwIh/Fn7qYwYyNsSzbnYhSilHtQnm6cy38ZXFmIUQJVJBCTwaC8z0OynvudpYCHxYl1J2cvXyNDzfFs2DHcXJzNYNaBTP+/jACKsjCzEKIkqsghb4bCFNKhWIV+SBgSP4NlFJhWuvYvIePALHYybLdJ/jn6kNkZOXQr0UQz3UJk3U8hRCCAhS61jpbKTUe+A5r2OJcrfVBpdRrQITWehUwXinVFcgCznOL0y22EnyPD13rV2FC1zBqVZJl34QQ4hdKayMDUggPD9cRERFG9i2EEM5KKRWptQ6/1Wtyq6QQQrgIKXQhhHARUuhCCOEipNCFEMJFSKELIYSLkEIXQggXIYUuhBAuQgpdCCFchLEbi5RSKcDxu/xyf+CsDeM4O/k8biSfx6/ks7iRK3weNbTWlW71grFCLwqlVMTt7pQqieTzuJF8Hr+Sz+JGrv55yCkXIYRwEVLoQgjhIpy10GeZDuBg5PO4kXwev5LP4kYu/Xk45Tl0IYQQv+WsR+hCCCFuIoUuhBAuwukKXSnVQykVrZSKU0pNMp3HFKVUsFJqo1LqkFLqoFJqgulMjkAp5aaU+kkp9Y3pLKYppSoqpVYopY4opQ4rpdqazmSKUmpi3s/JAaXUEqWUSy5A7FSFrpRyA2YCDwENgMFKqQZmUxmTDbyotW4AtAHGleDPIr8JwGHTIRzEVOBbrXU9oCkl9HNRSgUCzwHhWutGWEtpDjKbyj6cqtCBVkCc1jpBa50JLAV6G85khNb6Z611VN5/X8L6YQ00m8ospVQQ1iLlc0xnMU0pVQHoCHwCoLXO1FpfMJvKKHfAWynlDvgAJw3nsQtnK/RAIDHf4yRKeIkBKKVCgObATrNJjHsf+AuQazqIAwgFUoBP805BzVFKlTEdygStdTIwGTgB/Axc1FqvNZvKPpyt0MVNlFJlgZXA81rrNNN5TFFK/QE4o7WONJ3FQbgDLYAPtdbNgXSgRF5zUkr5Yv0mHwpUA8oopR43m8o+nK3Qk4HgfI+D8p4rkZRSHlhlvkhr/YXpPIa1A3oppY5hnYp7QCm10Gwko5KAJK31L7+1rcAq+JKoK3BUa52itc4CvgDuM5zJLpyt0HcDYUqpUKWUJ9aFjVWGMxmhlFJY50cPa63fNZ3HNK31S1rrIK11CNb3xQattUsehRWE1voUkKiUqpv3VBfgkMFIJp0A2iilfPJ+brrgoheI3U0HKAytdbZSajzwHdaV6rla64OGY5nSDhgG7FdK7cl77m9a6zUGMwnH8iywKO/gJwEYZTiPEVrrnUqpFUAU1uiwn3DRKQDk1n8hhHARznbKRQghxG1IoQshhIuQQhdCCBchhS6EEC5CCl0IIVyEFLoQQrgIKXQhhHAR/w+ZFZlyKmOxWAAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plot_hist(thr_score_hist)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Only text"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true
   },
   "source": [
    "### Traduction indonesian english"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "from imports import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "def text_to_device(text, device):\n",
    "    return {'input_ids' : text['input_ids'].to(device),\n",
    "            'attention_mask' : text['attention_mask'].to(device)}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "df = pd.read_csv('data/train.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "hidden": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "MarianMTModel(\n",
       "  (model): MarianModel(\n",
       "    (shared): Embedding(54796, 512, padding_idx=54795)\n",
       "    (encoder): MarianEncoder(\n",
       "      (embed_tokens): Embedding(54796, 512, padding_idx=54795)\n",
       "      (embed_positions): MarianSinusoidalPositionalEmbedding(512, 512)\n",
       "      (layers): ModuleList(\n",
       "        (0): MarianEncoderLayer(\n",
       "          (self_attn): MarianAttention(\n",
       "            (k_proj): Linear(in_features=512, out_features=512, bias=True)\n",
       "            (v_proj): Linear(in_features=512, out_features=512, bias=True)\n",
       "            (q_proj): Linear(in_features=512, out_features=512, bias=True)\n",
       "            (out_proj): Linear(in_features=512, out_features=512, bias=True)\n",
       "          )\n",
       "          (self_attn_layer_norm): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n",
       "          (fc1): Linear(in_features=512, out_features=2048, bias=True)\n",
       "          (fc2): Linear(in_features=2048, out_features=512, bias=True)\n",
       "          (final_layer_norm): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n",
       "        )\n",
       "        (1): MarianEncoderLayer(\n",
       "          (self_attn): MarianAttention(\n",
       "            (k_proj): Linear(in_features=512, out_features=512, bias=True)\n",
       "            (v_proj): Linear(in_features=512, out_features=512, bias=True)\n",
       "            (q_proj): Linear(in_features=512, out_features=512, bias=True)\n",
       "            (out_proj): Linear(in_features=512, out_features=512, bias=True)\n",
       "          )\n",
       "          (self_attn_layer_norm): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n",
       "          (fc1): Linear(in_features=512, out_features=2048, bias=True)\n",
       "          (fc2): Linear(in_features=2048, out_features=512, bias=True)\n",
       "          (final_layer_norm): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n",
       "        )\n",
       "        (2): MarianEncoderLayer(\n",
       "          (self_attn): MarianAttention(\n",
       "            (k_proj): Linear(in_features=512, out_features=512, bias=True)\n",
       "            (v_proj): Linear(in_features=512, out_features=512, bias=True)\n",
       "            (q_proj): Linear(in_features=512, out_features=512, bias=True)\n",
       "            (out_proj): Linear(in_features=512, out_features=512, bias=True)\n",
       "          )\n",
       "          (self_attn_layer_norm): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n",
       "          (fc1): Linear(in_features=512, out_features=2048, bias=True)\n",
       "          (fc2): Linear(in_features=2048, out_features=512, bias=True)\n",
       "          (final_layer_norm): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n",
       "        )\n",
       "        (3): MarianEncoderLayer(\n",
       "          (self_attn): MarianAttention(\n",
       "            (k_proj): Linear(in_features=512, out_features=512, bias=True)\n",
       "            (v_proj): Linear(in_features=512, out_features=512, bias=True)\n",
       "            (q_proj): Linear(in_features=512, out_features=512, bias=True)\n",
       "            (out_proj): Linear(in_features=512, out_features=512, bias=True)\n",
       "          )\n",
       "          (self_attn_layer_norm): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n",
       "          (fc1): Linear(in_features=512, out_features=2048, bias=True)\n",
       "          (fc2): Linear(in_features=2048, out_features=512, bias=True)\n",
       "          (final_layer_norm): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n",
       "        )\n",
       "        (4): MarianEncoderLayer(\n",
       "          (self_attn): MarianAttention(\n",
       "            (k_proj): Linear(in_features=512, out_features=512, bias=True)\n",
       "            (v_proj): Linear(in_features=512, out_features=512, bias=True)\n",
       "            (q_proj): Linear(in_features=512, out_features=512, bias=True)\n",
       "            (out_proj): Linear(in_features=512, out_features=512, bias=True)\n",
       "          )\n",
       "          (self_attn_layer_norm): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n",
       "          (fc1): Linear(in_features=512, out_features=2048, bias=True)\n",
       "          (fc2): Linear(in_features=2048, out_features=512, bias=True)\n",
       "          (final_layer_norm): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n",
       "        )\n",
       "        (5): MarianEncoderLayer(\n",
       "          (self_attn): MarianAttention(\n",
       "            (k_proj): Linear(in_features=512, out_features=512, bias=True)\n",
       "            (v_proj): Linear(in_features=512, out_features=512, bias=True)\n",
       "            (q_proj): Linear(in_features=512, out_features=512, bias=True)\n",
       "            (out_proj): Linear(in_features=512, out_features=512, bias=True)\n",
       "          )\n",
       "          (self_attn_layer_norm): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n",
       "          (fc1): Linear(in_features=512, out_features=2048, bias=True)\n",
       "          (fc2): Linear(in_features=2048, out_features=512, bias=True)\n",
       "          (final_layer_norm): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n",
       "        )\n",
       "      )\n",
       "    )\n",
       "    (decoder): MarianDecoder(\n",
       "      (embed_tokens): Embedding(54796, 512, padding_idx=54795)\n",
       "      (embed_positions): MarianSinusoidalPositionalEmbedding(512, 512)\n",
       "      (layers): ModuleList(\n",
       "        (0): MarianDecoderLayer(\n",
       "          (self_attn): MarianAttention(\n",
       "            (k_proj): Linear(in_features=512, out_features=512, bias=True)\n",
       "            (v_proj): Linear(in_features=512, out_features=512, bias=True)\n",
       "            (q_proj): Linear(in_features=512, out_features=512, bias=True)\n",
       "            (out_proj): Linear(in_features=512, out_features=512, bias=True)\n",
       "          )\n",
       "          (self_attn_layer_norm): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n",
       "          (encoder_attn): MarianAttention(\n",
       "            (k_proj): Linear(in_features=512, out_features=512, bias=True)\n",
       "            (v_proj): Linear(in_features=512, out_features=512, bias=True)\n",
       "            (q_proj): Linear(in_features=512, out_features=512, bias=True)\n",
       "            (out_proj): Linear(in_features=512, out_features=512, bias=True)\n",
       "          )\n",
       "          (encoder_attn_layer_norm): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n",
       "          (fc1): Linear(in_features=512, out_features=2048, bias=True)\n",
       "          (fc2): Linear(in_features=2048, out_features=512, bias=True)\n",
       "          (final_layer_norm): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n",
       "        )\n",
       "        (1): MarianDecoderLayer(\n",
       "          (self_attn): MarianAttention(\n",
       "            (k_proj): Linear(in_features=512, out_features=512, bias=True)\n",
       "            (v_proj): Linear(in_features=512, out_features=512, bias=True)\n",
       "            (q_proj): Linear(in_features=512, out_features=512, bias=True)\n",
       "            (out_proj): Linear(in_features=512, out_features=512, bias=True)\n",
       "          )\n",
       "          (self_attn_layer_norm): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n",
       "          (encoder_attn): MarianAttention(\n",
       "            (k_proj): Linear(in_features=512, out_features=512, bias=True)\n",
       "            (v_proj): Linear(in_features=512, out_features=512, bias=True)\n",
       "            (q_proj): Linear(in_features=512, out_features=512, bias=True)\n",
       "            (out_proj): Linear(in_features=512, out_features=512, bias=True)\n",
       "          )\n",
       "          (encoder_attn_layer_norm): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n",
       "          (fc1): Linear(in_features=512, out_features=2048, bias=True)\n",
       "          (fc2): Linear(in_features=2048, out_features=512, bias=True)\n",
       "          (final_layer_norm): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n",
       "        )\n",
       "        (2): MarianDecoderLayer(\n",
       "          (self_attn): MarianAttention(\n",
       "            (k_proj): Linear(in_features=512, out_features=512, bias=True)\n",
       "            (v_proj): Linear(in_features=512, out_features=512, bias=True)\n",
       "            (q_proj): Linear(in_features=512, out_features=512, bias=True)\n",
       "            (out_proj): Linear(in_features=512, out_features=512, bias=True)\n",
       "          )\n",
       "          (self_attn_layer_norm): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n",
       "          (encoder_attn): MarianAttention(\n",
       "            (k_proj): Linear(in_features=512, out_features=512, bias=True)\n",
       "            (v_proj): Linear(in_features=512, out_features=512, bias=True)\n",
       "            (q_proj): Linear(in_features=512, out_features=512, bias=True)\n",
       "            (out_proj): Linear(in_features=512, out_features=512, bias=True)\n",
       "          )\n",
       "          (encoder_attn_layer_norm): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n",
       "          (fc1): Linear(in_features=512, out_features=2048, bias=True)\n",
       "          (fc2): Linear(in_features=2048, out_features=512, bias=True)\n",
       "          (final_layer_norm): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n",
       "        )\n",
       "        (3): MarianDecoderLayer(\n",
       "          (self_attn): MarianAttention(\n",
       "            (k_proj): Linear(in_features=512, out_features=512, bias=True)\n",
       "            (v_proj): Linear(in_features=512, out_features=512, bias=True)\n",
       "            (q_proj): Linear(in_features=512, out_features=512, bias=True)\n",
       "            (out_proj): Linear(in_features=512, out_features=512, bias=True)\n",
       "          )\n",
       "          (self_attn_layer_norm): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n",
       "          (encoder_attn): MarianAttention(\n",
       "            (k_proj): Linear(in_features=512, out_features=512, bias=True)\n",
       "            (v_proj): Linear(in_features=512, out_features=512, bias=True)\n",
       "            (q_proj): Linear(in_features=512, out_features=512, bias=True)\n",
       "            (out_proj): Linear(in_features=512, out_features=512, bias=True)\n",
       "          )\n",
       "          (encoder_attn_layer_norm): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n",
       "          (fc1): Linear(in_features=512, out_features=2048, bias=True)\n",
       "          (fc2): Linear(in_features=2048, out_features=512, bias=True)\n",
       "          (final_layer_norm): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n",
       "        )\n",
       "        (4): MarianDecoderLayer(\n",
       "          (self_attn): MarianAttention(\n",
       "            (k_proj): Linear(in_features=512, out_features=512, bias=True)\n",
       "            (v_proj): Linear(in_features=512, out_features=512, bias=True)\n",
       "            (q_proj): Linear(in_features=512, out_features=512, bias=True)\n",
       "            (out_proj): Linear(in_features=512, out_features=512, bias=True)\n",
       "          )\n",
       "          (self_attn_layer_norm): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n",
       "          (encoder_attn): MarianAttention(\n",
       "            (k_proj): Linear(in_features=512, out_features=512, bias=True)\n",
       "            (v_proj): Linear(in_features=512, out_features=512, bias=True)\n",
       "            (q_proj): Linear(in_features=512, out_features=512, bias=True)\n",
       "            (out_proj): Linear(in_features=512, out_features=512, bias=True)\n",
       "          )\n",
       "          (encoder_attn_layer_norm): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n",
       "          (fc1): Linear(in_features=512, out_features=2048, bias=True)\n",
       "          (fc2): Linear(in_features=2048, out_features=512, bias=True)\n",
       "          (final_layer_norm): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n",
       "        )\n",
       "        (5): MarianDecoderLayer(\n",
       "          (self_attn): MarianAttention(\n",
       "            (k_proj): Linear(in_features=512, out_features=512, bias=True)\n",
       "            (v_proj): Linear(in_features=512, out_features=512, bias=True)\n",
       "            (q_proj): Linear(in_features=512, out_features=512, bias=True)\n",
       "            (out_proj): Linear(in_features=512, out_features=512, bias=True)\n",
       "          )\n",
       "          (self_attn_layer_norm): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n",
       "          (encoder_attn): MarianAttention(\n",
       "            (k_proj): Linear(in_features=512, out_features=512, bias=True)\n",
       "            (v_proj): Linear(in_features=512, out_features=512, bias=True)\n",
       "            (q_proj): Linear(in_features=512, out_features=512, bias=True)\n",
       "            (out_proj): Linear(in_features=512, out_features=512, bias=True)\n",
       "          )\n",
       "          (encoder_attn_layer_norm): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n",
       "          (fc1): Linear(in_features=512, out_features=2048, bias=True)\n",
       "          (fc2): Linear(in_features=2048, out_features=512, bias=True)\n",
       "          (final_layer_norm): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n",
       "        )\n",
       "      )\n",
       "    )\n",
       "  )\n",
       "  (lm_head): Linear(in_features=512, out_features=54796, bias=False)\n",
       ")"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from transformers import AutoTokenizer, AutoModelForSeq2SeqLM, pipeline\n",
    "\n",
    "tokenizer = AutoTokenizer.from_pretrained(\"data/translation\")\n",
    "txt_model = AutoModelForSeq2SeqLM.from_pretrained(\"data/translation\")\n",
    "\n",
    "txt_model.cuda()\n",
    "txt_model.eval()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "hidden": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "translating texts\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "8e05176723194a809053109c1b6502ed",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=2141.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([16, 49])\n",
      "torch.Size([16, 45])\n",
      "torch.Size([16, 40])\n",
      "torch.Size([16, 35])\n",
      "torch.Size([16, 49])\n",
      "torch.Size([16, 59])\n",
      "torch.Size([16, 36])\n",
      "torch.Size([16, 41])\n",
      "torch.Size([16, 34])\n",
      "torch.Size([16, 51])\n",
      "torch.Size([16, 34])\n",
      "torch.Size([16, 30])\n",
      "torch.Size([16, 34])\n",
      "torch.Size([16, 48])\n",
      "torch.Size([16, 34])\n",
      "torch.Size([16, 33])\n",
      "torch.Size([16, 55])\n",
      "torch.Size([16, 78])\n",
      "torch.Size([16, 50])\n",
      "torch.Size([16, 43])\n",
      "torch.Size([16, 37])\n",
      "torch.Size([16, 49])\n",
      "torch.Size([16, 51])\n",
      "torch.Size([16, 55])\n",
      "torch.Size([16, 36])\n",
      "torch.Size([16, 90])\n",
      "torch.Size([16, 47])\n",
      "torch.Size([16, 46])\n",
      "torch.Size([16, 39])\n",
      "torch.Size([16, 40])\n",
      "torch.Size([16, 33])\n",
      "torch.Size([16, 38])\n",
      "torch.Size([16, 42])\n",
      "torch.Size([16, 34])\n",
      "torch.Size([16, 54])\n",
      "torch.Size([16, 48])\n",
      "torch.Size([16, 52])\n",
      "torch.Size([16, 38])\n",
      "torch.Size([16, 95])\n",
      "torch.Size([16, 79])\n",
      "torch.Size([16, 79])\n",
      "torch.Size([16, 54])\n",
      "torch.Size([16, 73])\n",
      "torch.Size([16, 35])\n",
      "torch.Size([16, 46])\n",
      "torch.Size([16, 42])\n",
      "torch.Size([16, 34])\n",
      "torch.Size([16, 45])\n",
      "torch.Size([16, 60])\n",
      "torch.Size([16, 51])\n",
      "torch.Size([16, 38])\n",
      "torch.Size([16, 73])\n",
      "torch.Size([16, 30])\n",
      "torch.Size([16, 67])\n",
      "torch.Size([16, 33])\n",
      "torch.Size([16, 54])\n",
      "torch.Size([16, 80])\n",
      "torch.Size([16, 41])\n",
      "torch.Size([16, 65])\n",
      "torch.Size([16, 27])\n",
      "torch.Size([16, 63])\n",
      "torch.Size([16, 30])\n",
      "torch.Size([16, 37])\n",
      "torch.Size([16, 49])\n",
      "torch.Size([16, 28])\n",
      "torch.Size([16, 50])\n",
      "torch.Size([16, 48])\n",
      "torch.Size([16, 51])\n",
      "torch.Size([16, 27])\n",
      "torch.Size([16, 44])\n",
      "torch.Size([16, 34])\n",
      "torch.Size([16, 30])\n",
      "torch.Size([16, 28])\n",
      "torch.Size([16, 35])\n",
      "torch.Size([16, 49])\n",
      "torch.Size([16, 44])\n",
      "torch.Size([16, 51])\n",
      "torch.Size([16, 53])\n",
      "torch.Size([16, 47])\n",
      "torch.Size([16, 36])\n",
      "torch.Size([16, 45])\n",
      "torch.Size([16, 28])\n",
      "torch.Size([16, 31])\n",
      "torch.Size([16, 31])\n",
      "torch.Size([16, 32])\n",
      "torch.Size([16, 51])\n",
      "torch.Size([16, 36])\n",
      "torch.Size([16, 34])\n",
      "torch.Size([16, 42])\n",
      "torch.Size([16, 54])\n",
      "torch.Size([16, 56])\n",
      "torch.Size([16, 49])\n",
      "torch.Size([16, 42])\n",
      "torch.Size([16, 48])\n",
      "torch.Size([16, 45])\n",
      "torch.Size([16, 41])\n",
      "torch.Size([16, 36])\n",
      "torch.Size([16, 34])\n",
      "torch.Size([16, 43])\n",
      "torch.Size([16, 49])\n",
      "torch.Size([16, 41])\n",
      "torch.Size([16, 35])\n",
      "torch.Size([16, 52])\n",
      "torch.Size([16, 38])\n",
      "torch.Size([16, 47])\n",
      "torch.Size([16, 72])\n",
      "torch.Size([16, 45])\n",
      "torch.Size([16, 38])\n",
      "torch.Size([16, 29])\n",
      "torch.Size([16, 65])\n",
      "torch.Size([16, 66])\n",
      "torch.Size([16, 56])\n",
      "torch.Size([16, 56])\n",
      "torch.Size([16, 46])\n",
      "torch.Size([16, 52])\n",
      "torch.Size([16, 44])\n",
      "torch.Size([16, 42])\n",
      "torch.Size([16, 25])\n",
      "torch.Size([16, 54])\n",
      "torch.Size([16, 38])\n",
      "torch.Size([16, 41])\n",
      "torch.Size([16, 47])\n",
      "torch.Size([16, 36])\n",
      "torch.Size([16, 53])\n",
      "torch.Size([16, 48])\n",
      "torch.Size([16, 45])\n",
      "torch.Size([16, 55])\n",
      "torch.Size([16, 33])\n",
      "torch.Size([16, 44])\n",
      "torch.Size([16, 37])\n",
      "torch.Size([16, 43])\n",
      "torch.Size([16, 42])\n",
      "torch.Size([16, 34])\n",
      "torch.Size([16, 36])\n",
      "torch.Size([16, 34])\n",
      "torch.Size([16, 41])\n",
      "torch.Size([16, 45])\n",
      "torch.Size([16, 35])\n",
      "torch.Size([16, 33])\n",
      "torch.Size([16, 45])\n",
      "torch.Size([16, 33])\n",
      "torch.Size([16, 48])\n",
      "torch.Size([16, 48])\n",
      "torch.Size([16, 41])\n",
      "torch.Size([16, 50])\n",
      "torch.Size([16, 44])\n",
      "torch.Size([16, 43])\n",
      "torch.Size([16, 47])\n",
      "torch.Size([16, 43])\n",
      "torch.Size([16, 45])\n",
      "torch.Size([16, 42])\n",
      "torch.Size([16, 41])\n",
      "torch.Size([16, 45])\n",
      "torch.Size([16, 40])\n",
      "torch.Size([16, 37])\n",
      "torch.Size([16, 114])\n",
      "torch.Size([16, 31])\n",
      "torch.Size([16, 37])\n",
      "torch.Size([16, 37])\n",
      "torch.Size([16, 32])\n",
      "torch.Size([16, 44])\n",
      "torch.Size([16, 47])\n",
      "torch.Size([16, 36])\n",
      "torch.Size([16, 34])\n",
      "torch.Size([16, 36])\n",
      "torch.Size([16, 55])\n",
      "torch.Size([16, 40])\n",
      "torch.Size([16, 44])\n",
      "torch.Size([16, 31])\n",
      "torch.Size([16, 35])\n",
      "torch.Size([16, 41])\n",
      "torch.Size([16, 46])\n",
      "torch.Size([16, 38])\n",
      "torch.Size([16, 38])\n",
      "torch.Size([16, 27])\n",
      "torch.Size([16, 43])\n",
      "torch.Size([16, 76])\n",
      "torch.Size([16, 38])\n",
      "torch.Size([16, 49])\n",
      "torch.Size([16, 53])\n",
      "torch.Size([16, 47])\n",
      "torch.Size([16, 32])\n",
      "torch.Size([16, 56])\n",
      "torch.Size([16, 35])\n",
      "torch.Size([16, 47])\n",
      "torch.Size([16, 78])\n",
      "torch.Size([16, 52])\n",
      "torch.Size([16, 49])\n",
      "torch.Size([16, 45])\n",
      "torch.Size([16, 44])\n",
      "torch.Size([16, 87])\n",
      "torch.Size([16, 52])\n",
      "torch.Size([16, 48])\n",
      "torch.Size([16, 65])\n",
      "torch.Size([16, 41])\n",
      "torch.Size([16, 30])\n",
      "torch.Size([16, 41])\n",
      "torch.Size([16, 55])\n",
      "torch.Size([16, 27])\n",
      "torch.Size([16, 53])\n",
      "torch.Size([16, 29])\n",
      "torch.Size([16, 47])\n",
      "torch.Size([16, 47])\n",
      "torch.Size([16, 57])\n",
      "torch.Size([16, 69])\n",
      "torch.Size([16, 38])\n",
      "torch.Size([16, 49])\n",
      "torch.Size([16, 74])\n",
      "torch.Size([16, 28])\n",
      "torch.Size([16, 49])\n",
      "torch.Size([16, 124])\n",
      "torch.Size([16, 43])\n",
      "torch.Size([16, 39])\n",
      "torch.Size([16, 50])\n",
      "torch.Size([16, 44])\n",
      "torch.Size([16, 30])\n",
      "torch.Size([16, 43])\n",
      "torch.Size([16, 40])\n",
      "torch.Size([16, 42])\n",
      "torch.Size([16, 47])\n",
      "torch.Size([16, 128])\n",
      "torch.Size([16, 74])\n",
      "torch.Size([16, 47])\n",
      "torch.Size([16, 35])\n",
      "torch.Size([16, 41])\n",
      "torch.Size([16, 66])\n",
      "torch.Size([16, 38])\n",
      "torch.Size([16, 37])\n",
      "torch.Size([16, 32])\n",
      "torch.Size([16, 75])\n",
      "torch.Size([16, 45])\n",
      "torch.Size([16, 59])\n",
      "torch.Size([16, 59])\n",
      "torch.Size([16, 47])\n",
      "torch.Size([16, 42])\n",
      "torch.Size([16, 40])\n",
      "torch.Size([16, 27])\n",
      "torch.Size([16, 40])\n",
      "torch.Size([16, 37])\n",
      "torch.Size([16, 91])\n",
      "torch.Size([16, 30])\n",
      "torch.Size([16, 40])\n",
      "torch.Size([16, 33])\n",
      "torch.Size([16, 28])\n",
      "torch.Size([16, 34])\n",
      "torch.Size([16, 42])\n",
      "torch.Size([16, 32])\n",
      "torch.Size([16, 31])\n",
      "torch.Size([16, 53])\n",
      "torch.Size([16, 44])\n",
      "torch.Size([16, 37])\n",
      "torch.Size([16, 45])\n",
      "torch.Size([16, 48])\n",
      "torch.Size([16, 47])\n",
      "torch.Size([16, 48])\n",
      "torch.Size([16, 62])\n",
      "torch.Size([16, 36])\n",
      "torch.Size([16, 43])\n",
      "torch.Size([16, 35])\n",
      "torch.Size([16, 80])\n",
      "torch.Size([16, 51])\n",
      "torch.Size([16, 46])\n",
      "torch.Size([16, 28])\n",
      "torch.Size([16, 47])\n",
      "torch.Size([16, 44])\n",
      "torch.Size([16, 43])\n",
      "torch.Size([16, 42])\n",
      "torch.Size([16, 60])\n",
      "torch.Size([16, 41])\n",
      "torch.Size([16, 41])\n",
      "torch.Size([16, 33])\n",
      "torch.Size([16, 42])\n",
      "torch.Size([16, 47])\n",
      "torch.Size([16, 50])\n",
      "torch.Size([16, 40])\n",
      "torch.Size([16, 87])\n",
      "torch.Size([16, 43])\n",
      "torch.Size([16, 43])\n",
      "torch.Size([16, 42])\n",
      "torch.Size([16, 49])\n",
      "torch.Size([16, 48])\n",
      "torch.Size([16, 37])\n",
      "torch.Size([16, 41])\n",
      "torch.Size([16, 29])\n",
      "torch.Size([16, 32])\n",
      "torch.Size([16, 61])\n",
      "torch.Size([16, 33])\n",
      "torch.Size([16, 29])\n",
      "torch.Size([16, 46])\n",
      "torch.Size([16, 41])\n",
      "torch.Size([16, 34])\n",
      "torch.Size([16, 54])\n",
      "torch.Size([16, 43])\n",
      "torch.Size([16, 54])\n",
      "torch.Size([16, 42])\n",
      "torch.Size([16, 30])\n",
      "torch.Size([16, 30])\n",
      "torch.Size([16, 29])\n",
      "torch.Size([16, 39])\n",
      "torch.Size([16, 39])\n",
      "torch.Size([16, 79])\n",
      "torch.Size([16, 38])\n",
      "torch.Size([16, 38])\n",
      "torch.Size([16, 47])\n",
      "torch.Size([16, 34])\n",
      "torch.Size([16, 29])\n",
      "torch.Size([16, 35])\n",
      "torch.Size([16, 46])\n",
      "torch.Size([16, 69])\n",
      "torch.Size([16, 34])\n",
      "torch.Size([16, 48])\n",
      "torch.Size([16, 32])\n",
      "torch.Size([16, 49])\n",
      "torch.Size([16, 42])\n",
      "torch.Size([16, 39])\n",
      "torch.Size([16, 32])\n",
      "torch.Size([16, 55])\n",
      "torch.Size([16, 45])\n",
      "torch.Size([16, 50])\n",
      "torch.Size([16, 39])\n",
      "torch.Size([16, 48])\n",
      "torch.Size([16, 51])\n",
      "torch.Size([16, 49])\n",
      "torch.Size([16, 45])\n",
      "torch.Size([16, 60])\n",
      "torch.Size([16, 35])\n",
      "torch.Size([16, 33])\n",
      "torch.Size([16, 44])\n",
      "torch.Size([16, 45])\n",
      "torch.Size([16, 73])\n",
      "torch.Size([16, 35])\n",
      "torch.Size([16, 44])\n",
      "torch.Size([16, 57])\n",
      "torch.Size([16, 50])\n",
      "torch.Size([16, 59])\n",
      "torch.Size([16, 32])\n",
      "torch.Size([16, 51])\n",
      "torch.Size([16, 55])\n",
      "torch.Size([16, 33])\n",
      "torch.Size([16, 45])\n",
      "torch.Size([16, 42])\n",
      "torch.Size([16, 54])\n",
      "torch.Size([16, 38])\n",
      "torch.Size([16, 41])\n",
      "torch.Size([16, 39])\n",
      "torch.Size([16, 48])\n",
      "torch.Size([16, 42])\n",
      "torch.Size([16, 51])\n",
      "torch.Size([16, 32])\n",
      "torch.Size([16, 42])\n",
      "torch.Size([16, 31])\n",
      "torch.Size([16, 29])\n",
      "torch.Size([16, 37])\n",
      "torch.Size([16, 59])\n",
      "torch.Size([16, 35])\n",
      "torch.Size([16, 45])\n",
      "torch.Size([16, 27])\n",
      "torch.Size([16, 51])\n",
      "torch.Size([16, 45])\n",
      "torch.Size([16, 30])\n",
      "torch.Size([16, 48])\n",
      "torch.Size([16, 51])\n",
      "torch.Size([16, 51])\n",
      "torch.Size([16, 52])\n",
      "torch.Size([16, 49])\n",
      "torch.Size([16, 49])\n",
      "torch.Size([16, 43])\n",
      "torch.Size([16, 40])\n",
      "torch.Size([16, 36])\n",
      "torch.Size([16, 49])\n",
      "torch.Size([16, 36])\n",
      "torch.Size([16, 44])\n",
      "torch.Size([16, 40])\n",
      "torch.Size([16, 54])\n",
      "torch.Size([16, 42])\n",
      "torch.Size([16, 42])\n",
      "torch.Size([16, 41])\n",
      "torch.Size([16, 80])\n",
      "torch.Size([16, 42])\n",
      "torch.Size([16, 34])\n",
      "torch.Size([16, 49])\n",
      "torch.Size([16, 49])\n",
      "torch.Size([16, 42])\n",
      "torch.Size([16, 38])\n",
      "torch.Size([16, 45])\n",
      "torch.Size([16, 47])\n",
      "torch.Size([16, 31])\n",
      "torch.Size([16, 45])\n",
      "torch.Size([16, 30])\n",
      "torch.Size([16, 29])\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([16, 41])\n",
      "torch.Size([16, 39])\n",
      "torch.Size([16, 39])\n",
      "torch.Size([16, 59])\n",
      "torch.Size([16, 31])\n",
      "torch.Size([16, 33])\n",
      "torch.Size([16, 52])\n",
      "torch.Size([16, 61])\n",
      "torch.Size([16, 46])\n",
      "torch.Size([16, 43])\n",
      "torch.Size([16, 44])\n",
      "torch.Size([16, 37])\n",
      "torch.Size([16, 51])\n",
      "torch.Size([16, 43])\n",
      "torch.Size([16, 40])\n",
      "torch.Size([16, 45])\n",
      "torch.Size([16, 33])\n",
      "torch.Size([16, 36])\n",
      "torch.Size([16, 52])\n",
      "torch.Size([16, 35])\n",
      "torch.Size([16, 50])\n",
      "torch.Size([16, 54])\n",
      "torch.Size([16, 56])\n",
      "torch.Size([16, 30])\n",
      "torch.Size([16, 52])\n",
      "torch.Size([16, 45])\n",
      "torch.Size([16, 45])\n",
      "torch.Size([16, 56])\n",
      "torch.Size([16, 28])\n",
      "torch.Size([16, 50])\n",
      "torch.Size([16, 55])\n",
      "torch.Size([16, 48])\n",
      "torch.Size([16, 58])\n",
      "torch.Size([16, 51])\n",
      "torch.Size([16, 45])\n",
      "torch.Size([16, 47])\n",
      "torch.Size([16, 53])\n",
      "torch.Size([16, 40])\n",
      "torch.Size([16, 49])\n",
      "torch.Size([16, 50])\n",
      "torch.Size([16, 42])\n",
      "torch.Size([16, 47])\n",
      "torch.Size([16, 43])\n",
      "torch.Size([16, 45])\n",
      "torch.Size([16, 47])\n",
      "torch.Size([16, 51])\n",
      "torch.Size([16, 37])\n",
      "torch.Size([16, 43])\n",
      "torch.Size([16, 32])\n",
      "torch.Size([16, 38])\n",
      "torch.Size([16, 52])\n",
      "torch.Size([16, 32])\n",
      "torch.Size([16, 32])\n",
      "torch.Size([16, 103])\n",
      "torch.Size([16, 26])\n",
      "torch.Size([16, 64])\n",
      "torch.Size([16, 44])\n",
      "torch.Size([16, 35])\n",
      "torch.Size([16, 47])\n",
      "torch.Size([16, 35])\n",
      "torch.Size([16, 37])\n",
      "torch.Size([16, 32])\n",
      "torch.Size([16, 47])\n",
      "torch.Size([16, 39])\n",
      "torch.Size([16, 47])\n",
      "torch.Size([16, 47])\n",
      "torch.Size([16, 44])\n",
      "torch.Size([16, 29])\n",
      "torch.Size([16, 50])\n",
      "torch.Size([16, 39])\n",
      "torch.Size([16, 33])\n",
      "torch.Size([16, 55])\n",
      "torch.Size([16, 38])\n",
      "torch.Size([16, 55])\n",
      "torch.Size([16, 66])\n",
      "torch.Size([16, 65])\n",
      "torch.Size([16, 36])\n",
      "torch.Size([16, 51])\n",
      "torch.Size([16, 62])\n",
      "torch.Size([16, 43])\n",
      "torch.Size([16, 33])\n",
      "torch.Size([16, 50])\n",
      "torch.Size([16, 47])\n",
      "torch.Size([16, 51])\n",
      "torch.Size([16, 29])\n",
      "torch.Size([16, 45])\n",
      "torch.Size([16, 30])\n",
      "torch.Size([16, 47])\n",
      "torch.Size([16, 117])\n",
      "torch.Size([16, 34])\n",
      "torch.Size([16, 47])\n",
      "torch.Size([16, 47])\n",
      "torch.Size([16, 32])\n",
      "torch.Size([16, 33])\n",
      "torch.Size([16, 48])\n",
      "torch.Size([16, 51])\n",
      "torch.Size([16, 31])\n",
      "torch.Size([16, 47])\n",
      "torch.Size([16, 50])\n",
      "torch.Size([16, 34])\n",
      "torch.Size([16, 50])\n",
      "torch.Size([16, 36])\n",
      "torch.Size([16, 33])\n",
      "torch.Size([16, 33])\n",
      "torch.Size([16, 39])\n",
      "torch.Size([16, 39])\n",
      "torch.Size([16, 35])\n",
      "torch.Size([16, 27])\n",
      "torch.Size([16, 58])\n",
      "torch.Size([16, 49])\n",
      "torch.Size([16, 90])\n",
      "torch.Size([16, 37])\n",
      "torch.Size([16, 45])\n",
      "torch.Size([16, 31])\n",
      "torch.Size([16, 40])\n",
      "torch.Size([16, 35])\n",
      "torch.Size([16, 38])\n",
      "torch.Size([16, 43])\n",
      "torch.Size([16, 42])\n",
      "torch.Size([16, 30])\n",
      "torch.Size([16, 59])\n",
      "torch.Size([16, 38])\n",
      "torch.Size([16, 41])\n",
      "torch.Size([16, 33])\n",
      "torch.Size([16, 33])\n",
      "torch.Size([16, 34])\n",
      "torch.Size([16, 55])\n",
      "torch.Size([16, 44])\n",
      "torch.Size([16, 41])\n",
      "torch.Size([16, 34])\n",
      "torch.Size([16, 48])\n",
      "torch.Size([16, 40])\n",
      "torch.Size([16, 32])\n",
      "torch.Size([16, 34])\n",
      "torch.Size([16, 42])\n",
      "torch.Size([16, 47])\n",
      "torch.Size([16, 55])\n",
      "torch.Size([16, 45])\n",
      "torch.Size([16, 39])\n",
      "torch.Size([16, 45])\n",
      "torch.Size([16, 31])\n",
      "torch.Size([16, 61])\n",
      "torch.Size([16, 62])\n",
      "torch.Size([16, 56])\n",
      "torch.Size([16, 39])\n",
      "torch.Size([16, 35])\n",
      "torch.Size([16, 41])\n",
      "torch.Size([16, 55])\n",
      "torch.Size([16, 83])\n",
      "torch.Size([16, 50])\n",
      "torch.Size([16, 41])\n",
      "torch.Size([16, 30])\n",
      "torch.Size([16, 36])\n",
      "torch.Size([16, 40])\n",
      "torch.Size([16, 37])\n",
      "torch.Size([16, 26])\n",
      "torch.Size([16, 79])\n",
      "torch.Size([16, 33])\n",
      "torch.Size([16, 64])\n",
      "torch.Size([16, 32])\n",
      "torch.Size([16, 41])\n",
      "torch.Size([16, 45])\n",
      "torch.Size([16, 39])\n",
      "torch.Size([16, 44])\n",
      "torch.Size([16, 37])\n",
      "torch.Size([16, 32])\n",
      "torch.Size([16, 44])\n",
      "torch.Size([16, 43])\n",
      "torch.Size([16, 54])\n",
      "torch.Size([16, 53])\n",
      "torch.Size([16, 40])\n",
      "torch.Size([16, 51])\n",
      "torch.Size([16, 31])\n",
      "torch.Size([16, 42])\n",
      "torch.Size([16, 49])\n",
      "torch.Size([16, 49])\n",
      "torch.Size([16, 47])\n",
      "torch.Size([16, 47])\n",
      "torch.Size([16, 42])\n",
      "torch.Size([16, 50])\n",
      "torch.Size([16, 47])\n",
      "torch.Size([16, 56])\n",
      "torch.Size([16, 41])\n",
      "torch.Size([16, 53])\n",
      "torch.Size([16, 39])\n",
      "torch.Size([16, 55])\n",
      "torch.Size([16, 31])\n",
      "torch.Size([16, 46])\n",
      "torch.Size([16, 50])\n",
      "torch.Size([16, 38])\n",
      "torch.Size([16, 42])\n",
      "torch.Size([16, 34])\n",
      "torch.Size([16, 36])\n",
      "torch.Size([16, 46])\n",
      "torch.Size([16, 42])\n",
      "torch.Size([16, 35])\n",
      "torch.Size([16, 53])\n",
      "torch.Size([16, 38])\n",
      "torch.Size([16, 86])\n",
      "torch.Size([16, 50])\n",
      "torch.Size([16, 50])\n",
      "torch.Size([16, 64])\n",
      "torch.Size([16, 42])\n",
      "torch.Size([16, 29])\n",
      "torch.Size([16, 50])\n",
      "torch.Size([16, 59])\n",
      "torch.Size([16, 30])\n",
      "torch.Size([16, 34])\n",
      "torch.Size([16, 30])\n",
      "torch.Size([16, 43])\n",
      "torch.Size([16, 47])\n",
      "torch.Size([16, 47])\n",
      "torch.Size([16, 23])\n",
      "torch.Size([16, 45])\n",
      "torch.Size([16, 41])\n",
      "torch.Size([16, 35])\n",
      "torch.Size([16, 57])\n",
      "torch.Size([16, 34])\n",
      "torch.Size([16, 31])\n",
      "torch.Size([16, 49])\n",
      "torch.Size([16, 48])\n",
      "torch.Size([16, 49])\n",
      "torch.Size([16, 43])\n",
      "torch.Size([16, 45])\n",
      "torch.Size([16, 42])\n",
      "torch.Size([16, 39])\n",
      "torch.Size([16, 40])\n",
      "torch.Size([16, 72])\n",
      "torch.Size([16, 45])\n",
      "torch.Size([16, 39])\n",
      "torch.Size([16, 45])\n",
      "torch.Size([16, 31])\n",
      "torch.Size([16, 46])\n",
      "torch.Size([16, 49])\n",
      "torch.Size([16, 32])\n",
      "torch.Size([16, 37])\n",
      "torch.Size([16, 59])\n",
      "torch.Size([16, 28])\n",
      "torch.Size([16, 44])\n",
      "torch.Size([16, 43])\n",
      "torch.Size([16, 37])\n",
      "torch.Size([16, 36])\n",
      "torch.Size([16, 38])\n",
      "torch.Size([16, 45])\n",
      "torch.Size([16, 52])\n",
      "torch.Size([16, 38])\n",
      "torch.Size([16, 46])\n",
      "torch.Size([16, 46])\n",
      "torch.Size([16, 108])\n",
      "torch.Size([16, 48])\n",
      "torch.Size([16, 28])\n",
      "torch.Size([16, 34])\n",
      "torch.Size([16, 48])\n",
      "torch.Size([16, 46])\n",
      "torch.Size([16, 72])\n",
      "torch.Size([16, 71])\n",
      "torch.Size([16, 39])\n",
      "torch.Size([16, 42])\n",
      "torch.Size([16, 46])\n",
      "torch.Size([16, 38])\n",
      "torch.Size([16, 44])\n",
      "torch.Size([16, 40])\n",
      "torch.Size([16, 36])\n",
      "torch.Size([16, 36])\n",
      "torch.Size([16, 50])\n",
      "torch.Size([16, 40])\n",
      "torch.Size([16, 46])\n",
      "torch.Size([16, 52])\n",
      "torch.Size([16, 33])\n",
      "torch.Size([16, 49])\n",
      "torch.Size([16, 33])\n",
      "torch.Size([16, 53])\n",
      "torch.Size([16, 45])\n",
      "torch.Size([16, 30])\n",
      "torch.Size([16, 46])\n",
      "torch.Size([16, 33])\n",
      "torch.Size([16, 45])\n",
      "torch.Size([16, 31])\n",
      "torch.Size([16, 54])\n",
      "torch.Size([16, 51])\n",
      "torch.Size([16, 33])\n",
      "torch.Size([16, 34])\n",
      "torch.Size([16, 37])\n",
      "torch.Size([16, 42])\n",
      "torch.Size([16, 33])\n",
      "torch.Size([16, 55])\n",
      "torch.Size([16, 26])\n",
      "torch.Size([16, 34])\n",
      "torch.Size([16, 28])\n",
      "torch.Size([16, 50])\n",
      "torch.Size([16, 38])\n",
      "torch.Size([16, 31])\n",
      "torch.Size([16, 46])\n",
      "torch.Size([16, 34])\n",
      "torch.Size([16, 39])\n",
      "torch.Size([16, 69])\n",
      "torch.Size([16, 40])\n",
      "torch.Size([16, 50])\n",
      "torch.Size([16, 47])\n",
      "torch.Size([16, 53])\n",
      "torch.Size([16, 43])\n",
      "torch.Size([16, 39])\n",
      "torch.Size([16, 32])\n",
      "torch.Size([16, 43])\n",
      "torch.Size([16, 40])\n",
      "torch.Size([16, 34])\n",
      "torch.Size([16, 37])\n",
      "torch.Size([16, 35])\n",
      "torch.Size([16, 34])\n",
      "torch.Size([16, 32])\n",
      "torch.Size([16, 35])\n",
      "torch.Size([16, 35])\n",
      "torch.Size([16, 35])\n",
      "torch.Size([16, 57])\n",
      "torch.Size([16, 48])\n",
      "torch.Size([16, 32])\n",
      "torch.Size([16, 39])\n",
      "torch.Size([16, 43])\n",
      "torch.Size([16, 49])\n",
      "torch.Size([16, 51])\n",
      "torch.Size([16, 48])\n",
      "torch.Size([16, 39])\n",
      "torch.Size([16, 46])\n",
      "torch.Size([16, 25])\n",
      "torch.Size([16, 38])\n",
      "torch.Size([16, 38])\n",
      "torch.Size([16, 38])\n",
      "torch.Size([16, 34])\n",
      "torch.Size([16, 37])\n",
      "torch.Size([16, 28])\n",
      "torch.Size([16, 48])\n",
      "torch.Size([16, 43])\n",
      "torch.Size([16, 58])\n",
      "torch.Size([16, 50])\n",
      "torch.Size([16, 38])\n",
      "torch.Size([16, 78])\n",
      "torch.Size([16, 49])\n",
      "torch.Size([16, 50])\n",
      "torch.Size([16, 45])\n",
      "torch.Size([16, 57])\n",
      "torch.Size([16, 34])\n",
      "torch.Size([16, 44])\n",
      "torch.Size([16, 36])\n",
      "torch.Size([16, 35])\n",
      "torch.Size([16, 45])\n",
      "torch.Size([16, 37])\n",
      "torch.Size([16, 47])\n",
      "torch.Size([16, 28])\n",
      "torch.Size([16, 31])\n",
      "torch.Size([16, 42])\n",
      "torch.Size([16, 44])\n",
      "torch.Size([16, 51])\n",
      "torch.Size([16, 35])\n",
      "torch.Size([16, 40])\n",
      "torch.Size([16, 31])\n",
      "torch.Size([16, 44])\n",
      "torch.Size([16, 47])\n",
      "torch.Size([16, 38])\n",
      "torch.Size([16, 50])\n",
      "torch.Size([16, 35])\n",
      "torch.Size([16, 38])\n",
      "torch.Size([16, 45])\n",
      "torch.Size([16, 37])\n",
      "torch.Size([16, 45])\n",
      "torch.Size([16, 45])\n",
      "torch.Size([16, 40])\n",
      "torch.Size([16, 48])\n",
      "torch.Size([16, 63])\n",
      "torch.Size([16, 45])\n",
      "torch.Size([16, 48])\n",
      "torch.Size([16, 48])\n",
      "torch.Size([16, 40])\n",
      "torch.Size([16, 42])\n",
      "torch.Size([16, 38])\n",
      "torch.Size([16, 47])\n",
      "torch.Size([16, 28])\n",
      "torch.Size([16, 43])\n",
      "torch.Size([16, 41])\n",
      "torch.Size([16, 38])\n",
      "torch.Size([16, 52])\n",
      "torch.Size([16, 34])\n",
      "torch.Size([16, 56])\n",
      "torch.Size([16, 51])\n",
      "torch.Size([16, 35])\n",
      "torch.Size([16, 38])\n",
      "torch.Size([16, 45])\n",
      "torch.Size([16, 49])\n",
      "torch.Size([16, 58])\n",
      "torch.Size([16, 41])\n",
      "torch.Size([16, 30])\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([16, 44])\n",
      "torch.Size([16, 34])\n",
      "torch.Size([16, 38])\n",
      "torch.Size([16, 45])\n",
      "torch.Size([16, 38])\n",
      "torch.Size([16, 50])\n",
      "torch.Size([16, 50])\n",
      "torch.Size([16, 37])\n",
      "torch.Size([16, 84])\n",
      "torch.Size([16, 62])\n",
      "torch.Size([16, 37])\n",
      "torch.Size([16, 45])\n",
      "torch.Size([16, 50])\n",
      "torch.Size([16, 53])\n",
      "torch.Size([16, 45])\n",
      "torch.Size([16, 39])\n",
      "torch.Size([16, 43])\n",
      "torch.Size([16, 40])\n",
      "torch.Size([16, 37])\n",
      "torch.Size([16, 41])\n",
      "torch.Size([16, 35])\n",
      "torch.Size([16, 40])\n",
      "torch.Size([16, 37])\n",
      "torch.Size([16, 30])\n",
      "torch.Size([16, 49])\n",
      "torch.Size([16, 30])\n",
      "torch.Size([16, 49])\n",
      "torch.Size([16, 96])\n",
      "torch.Size([16, 37])\n",
      "torch.Size([16, 37])\n",
      "torch.Size([16, 62])\n",
      "torch.Size([16, 28])\n",
      "torch.Size([16, 37])\n",
      "torch.Size([16, 34])\n",
      "torch.Size([16, 43])\n",
      "torch.Size([16, 40])\n",
      "torch.Size([16, 38])\n",
      "torch.Size([16, 73])\n",
      "torch.Size([16, 63])\n",
      "torch.Size([16, 50])\n",
      "torch.Size([16, 45])\n",
      "torch.Size([16, 129])\n",
      "torch.Size([16, 30])\n",
      "torch.Size([16, 48])\n",
      "torch.Size([16, 33])\n",
      "torch.Size([16, 28])\n",
      "torch.Size([16, 42])\n",
      "torch.Size([16, 44])\n",
      "torch.Size([16, 38])\n",
      "torch.Size([16, 35])\n",
      "torch.Size([16, 45])\n",
      "torch.Size([16, 47])\n",
      "torch.Size([16, 49])\n",
      "torch.Size([16, 33])\n",
      "torch.Size([16, 46])\n",
      "torch.Size([16, 46])\n",
      "torch.Size([16, 34])\n",
      "torch.Size([16, 49])\n",
      "torch.Size([16, 38])\n",
      "torch.Size([16, 48])\n",
      "torch.Size([16, 61])\n",
      "torch.Size([16, 33])\n",
      "torch.Size([16, 95])\n",
      "torch.Size([16, 41])\n",
      "torch.Size([16, 37])\n",
      "torch.Size([16, 33])\n",
      "torch.Size([16, 29])\n",
      "torch.Size([16, 34])\n",
      "torch.Size([16, 43])\n",
      "torch.Size([16, 62])\n",
      "torch.Size([16, 62])\n",
      "torch.Size([16, 33])\n",
      "torch.Size([16, 52])\n",
      "torch.Size([16, 38])\n",
      "torch.Size([16, 47])\n",
      "torch.Size([16, 44])\n",
      "torch.Size([16, 49])\n",
      "torch.Size([16, 56])\n",
      "torch.Size([16, 34])\n",
      "torch.Size([16, 35])\n",
      "torch.Size([16, 31])\n",
      "torch.Size([16, 54])\n",
      "torch.Size([16, 41])\n",
      "torch.Size([16, 34])\n",
      "torch.Size([16, 39])\n",
      "torch.Size([16, 51])\n",
      "torch.Size([16, 39])\n",
      "torch.Size([16, 49])\n",
      "torch.Size([16, 35])\n",
      "torch.Size([16, 45])\n",
      "torch.Size([16, 38])\n",
      "torch.Size([16, 49])\n",
      "torch.Size([16, 30])\n",
      "torch.Size([16, 29])\n",
      "torch.Size([16, 70])\n",
      "torch.Size([16, 42])\n",
      "torch.Size([16, 47])\n",
      "torch.Size([16, 30])\n",
      "torch.Size([16, 49])\n",
      "torch.Size([16, 28])\n",
      "torch.Size([16, 46])\n",
      "torch.Size([16, 32])\n",
      "torch.Size([16, 50])\n",
      "torch.Size([16, 31])\n",
      "torch.Size([16, 41])\n",
      "torch.Size([16, 40])\n",
      "torch.Size([16, 58])\n",
      "torch.Size([16, 26])\n",
      "torch.Size([16, 42])\n",
      "torch.Size([16, 28])\n",
      "torch.Size([16, 42])\n",
      "torch.Size([16, 62])\n",
      "torch.Size([16, 38])\n",
      "torch.Size([16, 68])\n",
      "torch.Size([16, 64])\n",
      "torch.Size([16, 48])\n",
      "torch.Size([16, 36])\n",
      "torch.Size([16, 40])\n",
      "torch.Size([16, 53])\n",
      "torch.Size([16, 40])\n",
      "torch.Size([16, 48])\n",
      "torch.Size([16, 50])\n",
      "torch.Size([16, 56])\n",
      "torch.Size([16, 40])\n",
      "torch.Size([16, 29])\n",
      "torch.Size([16, 42])\n",
      "torch.Size([16, 40])\n",
      "torch.Size([16, 52])\n",
      "torch.Size([16, 40])\n",
      "torch.Size([16, 46])\n",
      "torch.Size([16, 35])\n",
      "torch.Size([16, 49])\n",
      "torch.Size([16, 46])\n",
      "torch.Size([16, 38])\n",
      "torch.Size([16, 45])\n",
      "torch.Size([16, 48])\n",
      "torch.Size([16, 106])\n",
      "torch.Size([16, 42])\n",
      "torch.Size([16, 33])\n",
      "torch.Size([16, 75])\n",
      "torch.Size([16, 30])\n",
      "torch.Size([16, 54])\n",
      "torch.Size([16, 44])\n",
      "torch.Size([16, 42])\n",
      "torch.Size([16, 43])\n",
      "torch.Size([16, 66])\n",
      "torch.Size([16, 47])\n",
      "torch.Size([16, 40])\n",
      "torch.Size([16, 39])\n",
      "torch.Size([16, 39])\n",
      "torch.Size([16, 49])\n",
      "torch.Size([16, 45])\n",
      "torch.Size([16, 36])\n",
      "torch.Size([16, 40])\n",
      "torch.Size([16, 39])\n",
      "torch.Size([16, 45])\n",
      "torch.Size([16, 46])\n",
      "torch.Size([16, 33])\n",
      "torch.Size([16, 33])\n",
      "torch.Size([16, 52])\n",
      "torch.Size([16, 42])\n",
      "torch.Size([16, 69])\n",
      "torch.Size([16, 61])\n",
      "torch.Size([16, 41])\n",
      "torch.Size([16, 29])\n",
      "torch.Size([16, 68])\n",
      "torch.Size([16, 39])\n",
      "torch.Size([16, 32])\n",
      "torch.Size([16, 53])\n",
      "torch.Size([16, 41])\n",
      "torch.Size([16, 40])\n",
      "torch.Size([16, 42])\n",
      "torch.Size([16, 49])\n",
      "torch.Size([16, 82])\n",
      "torch.Size([16, 29])\n",
      "torch.Size([16, 45])\n",
      "torch.Size([16, 44])\n",
      "torch.Size([16, 41])\n",
      "torch.Size([16, 50])\n",
      "torch.Size([16, 47])\n",
      "torch.Size([16, 34])\n",
      "torch.Size([16, 46])\n",
      "torch.Size([16, 38])\n",
      "torch.Size([16, 37])\n",
      "torch.Size([16, 45])\n",
      "torch.Size([16, 31])\n",
      "torch.Size([16, 42])\n",
      "torch.Size([16, 50])\n",
      "torch.Size([16, 44])\n",
      "torch.Size([16, 46])\n",
      "torch.Size([16, 31])\n",
      "torch.Size([16, 47])\n",
      "torch.Size([16, 117])\n",
      "torch.Size([16, 38])\n",
      "torch.Size([16, 37])\n",
      "torch.Size([16, 55])\n",
      "torch.Size([16, 36])\n",
      "torch.Size([16, 41])\n",
      "torch.Size([16, 69])\n",
      "torch.Size([16, 42])\n",
      "torch.Size([16, 31])\n",
      "torch.Size([16, 39])\n",
      "torch.Size([16, 45])\n",
      "torch.Size([16, 159])\n",
      "torch.Size([16, 42])\n",
      "torch.Size([16, 35])\n",
      "torch.Size([16, 36])\n",
      "torch.Size([16, 29])\n",
      "torch.Size([16, 47])\n",
      "torch.Size([16, 40])\n",
      "torch.Size([16, 44])\n",
      "torch.Size([16, 39])\n",
      "torch.Size([16, 35])\n",
      "torch.Size([16, 35])\n",
      "torch.Size([16, 49])\n",
      "torch.Size([16, 35])\n",
      "torch.Size([16, 41])\n",
      "torch.Size([16, 59])\n",
      "torch.Size([16, 50])\n",
      "torch.Size([16, 38])\n",
      "torch.Size([16, 52])\n",
      "torch.Size([16, 53])\n",
      "torch.Size([16, 62])\n",
      "torch.Size([16, 50])\n",
      "torch.Size([16, 31])\n",
      "torch.Size([16, 55])\n",
      "torch.Size([16, 50])\n",
      "torch.Size([16, 49])\n",
      "torch.Size([16, 64])\n",
      "torch.Size([16, 28])\n",
      "torch.Size([16, 44])\n",
      "torch.Size([16, 41])\n",
      "torch.Size([16, 38])\n",
      "torch.Size([16, 63])\n",
      "torch.Size([16, 42])\n",
      "torch.Size([16, 35])\n",
      "torch.Size([16, 34])\n",
      "torch.Size([16, 36])\n",
      "torch.Size([16, 50])\n",
      "torch.Size([16, 41])\n",
      "torch.Size([16, 33])\n",
      "torch.Size([16, 66])\n",
      "torch.Size([16, 33])\n",
      "torch.Size([16, 56])\n",
      "torch.Size([16, 55])\n",
      "torch.Size([16, 42])\n",
      "torch.Size([16, 27])\n",
      "torch.Size([16, 44])\n",
      "torch.Size([16, 42])\n",
      "torch.Size([16, 29])\n",
      "torch.Size([16, 81])\n",
      "torch.Size([16, 44])\n",
      "torch.Size([16, 50])\n",
      "torch.Size([16, 29])\n",
      "torch.Size([16, 47])\n",
      "torch.Size([16, 48])\n",
      "torch.Size([16, 46])\n",
      "torch.Size([16, 67])\n",
      "torch.Size([16, 28])\n",
      "torch.Size([16, 46])\n",
      "torch.Size([16, 41])\n",
      "torch.Size([16, 52])\n",
      "torch.Size([16, 43])\n",
      "torch.Size([16, 35])\n",
      "torch.Size([16, 44])\n",
      "torch.Size([16, 36])\n",
      "torch.Size([16, 44])\n",
      "torch.Size([16, 84])\n",
      "torch.Size([16, 46])\n",
      "torch.Size([16, 51])\n",
      "torch.Size([16, 66])\n",
      "torch.Size([16, 44])\n",
      "torch.Size([16, 49])\n",
      "torch.Size([16, 40])\n",
      "torch.Size([16, 43])\n",
      "torch.Size([16, 48])\n",
      "torch.Size([16, 30])\n",
      "torch.Size([16, 54])\n",
      "torch.Size([16, 51])\n",
      "torch.Size([16, 34])\n",
      "torch.Size([16, 52])\n",
      "torch.Size([16, 44])\n",
      "torch.Size([16, 38])\n",
      "torch.Size([16, 45])\n",
      "torch.Size([16, 41])\n",
      "torch.Size([16, 24])\n",
      "torch.Size([16, 44])\n",
      "torch.Size([16, 46])\n",
      "torch.Size([16, 39])\n",
      "torch.Size([16, 38])\n",
      "torch.Size([16, 46])\n",
      "torch.Size([16, 63])\n",
      "torch.Size([16, 90])\n",
      "torch.Size([16, 71])\n",
      "torch.Size([16, 53])\n",
      "torch.Size([16, 50])\n",
      "torch.Size([16, 33])\n",
      "torch.Size([16, 70])\n",
      "torch.Size([16, 28])\n",
      "torch.Size([16, 31])\n",
      "torch.Size([16, 30])\n",
      "torch.Size([16, 40])\n",
      "torch.Size([16, 38])\n",
      "torch.Size([16, 39])\n",
      "torch.Size([16, 42])\n",
      "torch.Size([16, 53])\n",
      "torch.Size([16, 45])\n",
      "torch.Size([16, 43])\n",
      "torch.Size([16, 47])\n",
      "torch.Size([16, 30])\n",
      "torch.Size([16, 46])\n",
      "torch.Size([16, 31])\n",
      "torch.Size([16, 31])\n",
      "torch.Size([16, 56])\n",
      "torch.Size([16, 54])\n",
      "torch.Size([16, 45])\n",
      "torch.Size([16, 46])\n",
      "torch.Size([16, 32])\n",
      "torch.Size([16, 31])\n",
      "torch.Size([16, 47])\n",
      "torch.Size([16, 35])\n",
      "torch.Size([16, 27])\n",
      "torch.Size([16, 47])\n",
      "torch.Size([16, 41])\n",
      "torch.Size([16, 39])\n",
      "torch.Size([16, 40])\n",
      "torch.Size([16, 50])\n",
      "torch.Size([16, 27])\n",
      "torch.Size([16, 32])\n",
      "torch.Size([16, 110])\n",
      "torch.Size([16, 45])\n",
      "torch.Size([16, 41])\n",
      "torch.Size([16, 47])\n",
      "torch.Size([16, 45])\n",
      "torch.Size([16, 50])\n",
      "torch.Size([16, 32])\n",
      "torch.Size([16, 35])\n",
      "torch.Size([16, 51])\n",
      "torch.Size([16, 50])\n",
      "torch.Size([16, 38])\n",
      "torch.Size([16, 35])\n",
      "torch.Size([16, 46])\n",
      "torch.Size([16, 30])\n",
      "torch.Size([16, 46])\n",
      "torch.Size([16, 51])\n",
      "torch.Size([16, 48])\n",
      "torch.Size([16, 38])\n",
      "torch.Size([16, 38])\n",
      "torch.Size([16, 51])\n",
      "torch.Size([16, 50])\n",
      "torch.Size([16, 50])\n",
      "torch.Size([16, 56])\n",
      "torch.Size([16, 51])\n",
      "torch.Size([16, 40])\n",
      "torch.Size([16, 70])\n",
      "torch.Size([16, 45])\n",
      "torch.Size([16, 40])\n",
      "torch.Size([16, 47])\n",
      "torch.Size([16, 47])\n",
      "torch.Size([16, 45])\n",
      "torch.Size([16, 58])\n",
      "torch.Size([16, 42])\n",
      "torch.Size([16, 42])\n",
      "torch.Size([16, 36])\n",
      "torch.Size([16, 46])\n",
      "torch.Size([16, 34])\n",
      "torch.Size([16, 54])\n",
      "torch.Size([16, 35])\n",
      "torch.Size([16, 49])\n",
      "torch.Size([16, 47])\n",
      "torch.Size([16, 34])\n",
      "torch.Size([16, 38])\n",
      "torch.Size([16, 34])\n",
      "torch.Size([16, 41])\n",
      "torch.Size([16, 32])\n",
      "torch.Size([16, 37])\n",
      "torch.Size([16, 40])\n",
      "torch.Size([16, 58])\n",
      "torch.Size([16, 48])\n",
      "torch.Size([16, 45])\n",
      "torch.Size([16, 34])\n",
      "torch.Size([16, 42])\n",
      "torch.Size([16, 30])\n",
      "torch.Size([16, 32])\n",
      "torch.Size([16, 45])\n",
      "torch.Size([16, 42])\n",
      "torch.Size([16, 31])\n",
      "torch.Size([16, 38])\n",
      "torch.Size([16, 47])\n",
      "torch.Size([16, 69])\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([16, 74])\n",
      "torch.Size([16, 49])\n",
      "torch.Size([16, 40])\n",
      "torch.Size([16, 30])\n",
      "torch.Size([16, 67])\n",
      "torch.Size([16, 36])\n",
      "torch.Size([16, 43])\n",
      "torch.Size([16, 37])\n",
      "torch.Size([16, 32])\n",
      "torch.Size([16, 26])\n",
      "torch.Size([16, 47])\n",
      "torch.Size([16, 37])\n",
      "torch.Size([16, 29])\n",
      "torch.Size([16, 43])\n",
      "torch.Size([16, 40])\n",
      "torch.Size([16, 47])\n",
      "torch.Size([16, 42])\n",
      "torch.Size([16, 44])\n",
      "torch.Size([16, 43])\n",
      "torch.Size([16, 47])\n",
      "torch.Size([16, 52])\n",
      "torch.Size([16, 79])\n",
      "torch.Size([16, 38])\n",
      "torch.Size([16, 48])\n",
      "torch.Size([16, 36])\n",
      "torch.Size([16, 49])\n",
      "torch.Size([16, 45])\n",
      "torch.Size([16, 53])\n",
      "torch.Size([16, 44])\n",
      "torch.Size([16, 46])\n",
      "torch.Size([16, 41])\n",
      "torch.Size([16, 44])\n",
      "torch.Size([16, 34])\n",
      "torch.Size([16, 80])\n",
      "torch.Size([16, 46])\n",
      "torch.Size([16, 39])\n",
      "torch.Size([16, 41])\n",
      "torch.Size([16, 52])\n",
      "torch.Size([16, 41])\n",
      "torch.Size([16, 52])\n",
      "torch.Size([16, 44])\n",
      "torch.Size([16, 48])\n",
      "torch.Size([16, 33])\n",
      "torch.Size([16, 52])\n",
      "torch.Size([16, 38])\n",
      "torch.Size([16, 31])\n",
      "torch.Size([16, 40])\n",
      "torch.Size([16, 51])\n",
      "torch.Size([16, 41])\n",
      "torch.Size([16, 34])\n",
      "torch.Size([16, 68])\n",
      "torch.Size([16, 47])\n",
      "torch.Size([16, 52])\n",
      "torch.Size([16, 51])\n",
      "torch.Size([16, 29])\n",
      "torch.Size([16, 45])\n",
      "torch.Size([16, 38])\n",
      "torch.Size([16, 44])\n",
      "torch.Size([16, 35])\n",
      "torch.Size([16, 32])\n",
      "torch.Size([16, 35])\n",
      "torch.Size([16, 40])\n",
      "torch.Size([16, 37])\n",
      "torch.Size([16, 46])\n",
      "torch.Size([16, 43])\n",
      "torch.Size([16, 56])\n",
      "torch.Size([16, 40])\n",
      "torch.Size([16, 67])\n",
      "torch.Size([16, 47])\n",
      "torch.Size([16, 37])\n",
      "torch.Size([16, 38])\n",
      "torch.Size([16, 49])\n",
      "torch.Size([16, 45])\n",
      "torch.Size([16, 47])\n",
      "torch.Size([16, 50])\n",
      "torch.Size([16, 33])\n",
      "torch.Size([16, 40])\n",
      "torch.Size([16, 61])\n",
      "torch.Size([16, 53])\n",
      "torch.Size([16, 46])\n",
      "torch.Size([16, 38])\n",
      "torch.Size([16, 45])\n",
      "torch.Size([16, 36])\n",
      "torch.Size([16, 47])\n",
      "torch.Size([16, 47])\n",
      "torch.Size([16, 36])\n",
      "torch.Size([16, 52])\n",
      "torch.Size([16, 47])\n",
      "torch.Size([16, 67])\n",
      "torch.Size([16, 31])\n",
      "torch.Size([16, 51])\n",
      "torch.Size([16, 48])\n",
      "torch.Size([16, 81])\n",
      "torch.Size([16, 66])\n",
      "torch.Size([16, 36])\n",
      "torch.Size([16, 53])\n",
      "torch.Size([16, 34])\n",
      "torch.Size([16, 48])\n",
      "torch.Size([16, 42])\n",
      "torch.Size([16, 44])\n",
      "torch.Size([16, 50])\n",
      "torch.Size([16, 41])\n",
      "torch.Size([16, 48])\n",
      "torch.Size([16, 41])\n",
      "torch.Size([16, 24])\n",
      "torch.Size([16, 31])\n",
      "torch.Size([16, 49])\n",
      "torch.Size([16, 53])\n",
      "torch.Size([16, 44])\n",
      "torch.Size([16, 58])\n",
      "torch.Size([16, 39])\n",
      "torch.Size([16, 55])\n",
      "torch.Size([16, 42])\n",
      "torch.Size([16, 45])\n",
      "torch.Size([16, 46])\n",
      "torch.Size([16, 45])\n",
      "torch.Size([16, 61])\n",
      "torch.Size([16, 53])\n",
      "torch.Size([16, 35])\n",
      "torch.Size([16, 28])\n",
      "torch.Size([16, 40])\n",
      "torch.Size([16, 54])\n",
      "torch.Size([16, 46])\n",
      "torch.Size([16, 114])\n",
      "torch.Size([16, 113])\n",
      "torch.Size([16, 44])\n",
      "torch.Size([16, 39])\n",
      "torch.Size([16, 37])\n",
      "torch.Size([16, 27])\n",
      "torch.Size([16, 30])\n",
      "torch.Size([16, 35])\n",
      "torch.Size([16, 37])\n",
      "torch.Size([16, 31])\n",
      "torch.Size([16, 41])\n",
      "torch.Size([16, 36])\n",
      "torch.Size([16, 42])\n",
      "torch.Size([16, 36])\n",
      "torch.Size([16, 30])\n",
      "torch.Size([16, 40])\n",
      "torch.Size([16, 35])\n",
      "torch.Size([16, 31])\n",
      "torch.Size([16, 48])\n",
      "torch.Size([16, 51])\n",
      "torch.Size([16, 69])\n",
      "torch.Size([16, 46])\n",
      "torch.Size([16, 49])\n",
      "torch.Size([16, 38])\n",
      "torch.Size([16, 36])\n",
      "torch.Size([16, 50])\n",
      "torch.Size([16, 41])\n",
      "torch.Size([16, 55])\n",
      "torch.Size([16, 47])\n",
      "torch.Size([16, 37])\n",
      "torch.Size([16, 37])\n",
      "torch.Size([16, 40])\n",
      "torch.Size([16, 55])\n",
      "torch.Size([16, 50])\n",
      "torch.Size([16, 37])\n",
      "torch.Size([16, 49])\n",
      "torch.Size([16, 70])\n",
      "torch.Size([16, 47])\n",
      "torch.Size([16, 46])\n",
      "torch.Size([16, 45])\n",
      "torch.Size([16, 43])\n",
      "torch.Size([16, 52])\n",
      "torch.Size([16, 35])\n",
      "torch.Size([16, 41])\n",
      "torch.Size([16, 33])\n",
      "torch.Size([16, 30])\n",
      "torch.Size([16, 43])\n",
      "torch.Size([16, 28])\n",
      "torch.Size([16, 22])\n",
      "torch.Size([16, 43])\n",
      "torch.Size([16, 45])\n",
      "torch.Size([16, 31])\n",
      "torch.Size([16, 46])\n",
      "torch.Size([16, 53])\n",
      "torch.Size([16, 33])\n",
      "torch.Size([16, 30])\n",
      "torch.Size([16, 49])\n",
      "torch.Size([16, 43])\n",
      "torch.Size([16, 56])\n",
      "torch.Size([16, 49])\n",
      "torch.Size([16, 27])\n",
      "torch.Size([16, 44])\n",
      "torch.Size([16, 36])\n",
      "torch.Size([16, 36])\n",
      "torch.Size([16, 37])\n",
      "torch.Size([16, 46])\n",
      "torch.Size([16, 55])\n",
      "torch.Size([16, 45])\n",
      "torch.Size([16, 42])\n",
      "torch.Size([16, 53])\n",
      "torch.Size([16, 44])\n",
      "torch.Size([16, 36])\n",
      "torch.Size([16, 43])\n",
      "torch.Size([16, 54])\n",
      "torch.Size([16, 46])\n",
      "torch.Size([16, 46])\n",
      "torch.Size([16, 37])\n",
      "torch.Size([16, 41])\n",
      "torch.Size([16, 59])\n",
      "torch.Size([16, 45])\n",
      "torch.Size([16, 41])\n",
      "torch.Size([16, 29])\n",
      "torch.Size([16, 46])\n",
      "torch.Size([16, 36])\n",
      "torch.Size([16, 39])\n",
      "torch.Size([16, 52])\n",
      "torch.Size([16, 38])\n",
      "torch.Size([16, 36])\n",
      "torch.Size([16, 45])\n",
      "torch.Size([16, 51])\n",
      "torch.Size([16, 35])\n",
      "torch.Size([16, 97])\n",
      "torch.Size([16, 43])\n",
      "torch.Size([16, 46])\n",
      "torch.Size([16, 58])\n",
      "torch.Size([16, 43])\n",
      "torch.Size([16, 44])\n",
      "torch.Size([16, 43])\n",
      "torch.Size([16, 25])\n",
      "torch.Size([16, 77])\n",
      "torch.Size([16, 30])\n",
      "torch.Size([16, 41])\n",
      "torch.Size([16, 30])\n",
      "torch.Size([16, 36])\n",
      "torch.Size([16, 38])\n",
      "torch.Size([16, 41])\n",
      "torch.Size([16, 43])\n",
      "torch.Size([16, 34])\n",
      "torch.Size([16, 30])\n",
      "torch.Size([16, 44])\n",
      "torch.Size([16, 46])\n",
      "torch.Size([16, 35])\n",
      "torch.Size([16, 44])\n",
      "torch.Size([16, 40])\n",
      "torch.Size([16, 85])\n",
      "torch.Size([16, 38])\n",
      "torch.Size([16, 45])\n",
      "torch.Size([16, 30])\n",
      "torch.Size([16, 35])\n",
      "torch.Size([16, 39])\n",
      "torch.Size([16, 53])\n",
      "torch.Size([16, 37])\n",
      "torch.Size([16, 51])\n",
      "torch.Size([16, 42])\n",
      "torch.Size([16, 70])\n",
      "torch.Size([16, 34])\n",
      "torch.Size([16, 52])\n",
      "torch.Size([16, 51])\n",
      "torch.Size([16, 39])\n",
      "torch.Size([16, 53])\n",
      "torch.Size([16, 47])\n",
      "torch.Size([16, 41])\n",
      "torch.Size([16, 39])\n",
      "torch.Size([16, 63])\n",
      "torch.Size([16, 49])\n",
      "torch.Size([16, 39])\n",
      "torch.Size([16, 41])\n",
      "torch.Size([16, 43])\n",
      "torch.Size([16, 40])\n",
      "torch.Size([16, 44])\n",
      "torch.Size([16, 34])\n",
      "torch.Size([16, 42])\n",
      "torch.Size([16, 37])\n",
      "torch.Size([16, 44])\n",
      "torch.Size([16, 41])\n",
      "torch.Size([16, 43])\n",
      "torch.Size([16, 34])\n",
      "torch.Size([16, 45])\n",
      "torch.Size([16, 39])\n",
      "torch.Size([16, 40])\n",
      "torch.Size([16, 41])\n",
      "torch.Size([16, 30])\n",
      "torch.Size([16, 43])\n",
      "torch.Size([16, 41])\n",
      "torch.Size([16, 34])\n",
      "torch.Size([16, 62])\n",
      "torch.Size([16, 50])\n",
      "torch.Size([16, 35])\n",
      "torch.Size([16, 48])\n",
      "torch.Size([16, 39])\n",
      "torch.Size([16, 44])\n",
      "torch.Size([16, 32])\n",
      "torch.Size([16, 42])\n",
      "torch.Size([16, 36])\n",
      "torch.Size([16, 31])\n",
      "torch.Size([16, 29])\n",
      "torch.Size([16, 45])\n",
      "torch.Size([16, 77])\n",
      "torch.Size([16, 38])\n",
      "torch.Size([16, 58])\n",
      "torch.Size([16, 38])\n",
      "torch.Size([16, 74])\n",
      "torch.Size([16, 49])\n",
      "torch.Size([16, 36])\n",
      "torch.Size([16, 30])\n",
      "torch.Size([16, 46])\n",
      "torch.Size([16, 51])\n",
      "torch.Size([16, 37])\n",
      "torch.Size([16, 50])\n",
      "torch.Size([16, 51])\n",
      "torch.Size([16, 49])\n",
      "torch.Size([16, 41])\n",
      "torch.Size([16, 47])\n",
      "torch.Size([16, 34])\n",
      "torch.Size([16, 43])\n",
      "torch.Size([16, 35])\n",
      "torch.Size([16, 39])\n",
      "torch.Size([16, 46])\n",
      "torch.Size([16, 44])\n",
      "torch.Size([16, 22])\n",
      "torch.Size([16, 39])\n",
      "torch.Size([16, 39])\n",
      "torch.Size([16, 39])\n",
      "torch.Size([16, 38])\n",
      "torch.Size([16, 62])\n",
      "torch.Size([16, 36])\n",
      "torch.Size([16, 29])\n",
      "torch.Size([16, 37])\n",
      "torch.Size([16, 47])\n",
      "torch.Size([16, 52])\n",
      "torch.Size([16, 33])\n",
      "torch.Size([16, 40])\n",
      "torch.Size([16, 47])\n",
      "torch.Size([16, 52])\n",
      "torch.Size([16, 39])\n",
      "torch.Size([16, 33])\n",
      "torch.Size([16, 36])\n",
      "torch.Size([16, 43])\n",
      "torch.Size([16, 61])\n",
      "torch.Size([16, 38])\n",
      "torch.Size([16, 33])\n",
      "torch.Size([16, 52])\n",
      "torch.Size([16, 41])\n",
      "torch.Size([16, 36])\n",
      "torch.Size([16, 39])\n",
      "torch.Size([16, 39])\n",
      "torch.Size([16, 26])\n",
      "torch.Size([16, 33])\n",
      "torch.Size([16, 36])\n",
      "torch.Size([16, 50])\n",
      "torch.Size([16, 35])\n",
      "torch.Size([16, 38])\n",
      "torch.Size([16, 39])\n",
      "torch.Size([16, 30])\n",
      "torch.Size([16, 44])\n",
      "torch.Size([16, 33])\n",
      "torch.Size([16, 51])\n",
      "torch.Size([16, 42])\n",
      "torch.Size([16, 45])\n",
      "torch.Size([16, 29])\n",
      "torch.Size([16, 26])\n",
      "torch.Size([16, 46])\n",
      "torch.Size([16, 40])\n",
      "torch.Size([16, 40])\n",
      "torch.Size([16, 65])\n",
      "torch.Size([16, 111])\n",
      "torch.Size([16, 32])\n",
      "torch.Size([16, 36])\n",
      "torch.Size([16, 32])\n",
      "torch.Size([16, 55])\n",
      "torch.Size([16, 44])\n",
      "torch.Size([16, 69])\n",
      "torch.Size([16, 79])\n",
      "torch.Size([16, 49])\n",
      "torch.Size([16, 45])\n",
      "torch.Size([16, 31])\n",
      "torch.Size([16, 40])\n",
      "torch.Size([16, 49])\n",
      "torch.Size([16, 70])\n",
      "torch.Size([16, 115])\n",
      "torch.Size([16, 50])\n",
      "torch.Size([16, 36])\n",
      "torch.Size([16, 38])\n",
      "torch.Size([16, 60])\n",
      "torch.Size([16, 37])\n",
      "torch.Size([16, 39])\n",
      "torch.Size([16, 30])\n",
      "torch.Size([16, 47])\n",
      "torch.Size([16, 43])\n",
      "torch.Size([16, 57])\n",
      "torch.Size([16, 55])\n",
      "torch.Size([16, 47])\n",
      "torch.Size([16, 43])\n",
      "torch.Size([16, 24])\n",
      "torch.Size([16, 56])\n",
      "torch.Size([16, 31])\n",
      "torch.Size([16, 41])\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([16, 29])\n",
      "torch.Size([16, 88])\n",
      "torch.Size([16, 37])\n",
      "torch.Size([16, 38])\n",
      "torch.Size([16, 42])\n",
      "torch.Size([16, 34])\n",
      "torch.Size([16, 28])\n",
      "torch.Size([16, 48])\n",
      "torch.Size([16, 26])\n",
      "torch.Size([16, 45])\n",
      "torch.Size([16, 35])\n",
      "torch.Size([16, 48])\n",
      "torch.Size([16, 35])\n",
      "torch.Size([16, 40])\n",
      "torch.Size([16, 27])\n",
      "torch.Size([16, 48])\n",
      "torch.Size([16, 54])\n",
      "torch.Size([16, 27])\n",
      "torch.Size([16, 50])\n",
      "torch.Size([16, 45])\n",
      "torch.Size([16, 30])\n",
      "torch.Size([16, 55])\n",
      "torch.Size([16, 32])\n",
      "torch.Size([16, 44])\n",
      "torch.Size([16, 45])\n",
      "torch.Size([16, 45])\n",
      "torch.Size([16, 32])\n",
      "torch.Size([16, 29])\n",
      "torch.Size([16, 30])\n",
      "torch.Size([16, 45])\n",
      "torch.Size([16, 46])\n",
      "torch.Size([16, 29])\n",
      "torch.Size([16, 67])\n",
      "torch.Size([16, 45])\n",
      "torch.Size([16, 44])\n",
      "torch.Size([16, 49])\n",
      "torch.Size([16, 49])\n",
      "torch.Size([16, 48])\n",
      "torch.Size([16, 47])\n",
      "torch.Size([16, 39])\n",
      "torch.Size([16, 40])\n",
      "torch.Size([16, 45])\n",
      "torch.Size([16, 42])\n",
      "torch.Size([16, 23])\n",
      "torch.Size([16, 47])\n",
      "torch.Size([16, 72])\n",
      "torch.Size([16, 70])\n",
      "torch.Size([16, 33])\n",
      "torch.Size([16, 46])\n",
      "torch.Size([16, 39])\n",
      "torch.Size([16, 59])\n",
      "torch.Size([16, 79])\n",
      "torch.Size([16, 40])\n",
      "torch.Size([16, 46])\n",
      "torch.Size([16, 49])\n",
      "torch.Size([16, 47])\n",
      "torch.Size([16, 44])\n",
      "torch.Size([16, 29])\n",
      "torch.Size([16, 54])\n",
      "torch.Size([16, 36])\n",
      "torch.Size([16, 45])\n",
      "torch.Size([16, 43])\n",
      "torch.Size([16, 39])\n",
      "torch.Size([16, 31])\n",
      "torch.Size([16, 56])\n",
      "torch.Size([16, 54])\n",
      "torch.Size([16, 43])\n",
      "torch.Size([16, 48])\n",
      "torch.Size([16, 66])\n",
      "torch.Size([16, 47])\n",
      "torch.Size([16, 51])\n",
      "torch.Size([16, 40])\n",
      "torch.Size([16, 32])\n",
      "torch.Size([16, 46])\n",
      "torch.Size([16, 48])\n",
      "torch.Size([16, 76])\n",
      "torch.Size([16, 60])\n",
      "torch.Size([16, 48])\n",
      "torch.Size([16, 38])\n",
      "torch.Size([16, 40])\n",
      "torch.Size([16, 39])\n",
      "torch.Size([16, 43])\n",
      "torch.Size([16, 31])\n",
      "torch.Size([16, 48])\n",
      "torch.Size([16, 47])\n",
      "torch.Size([16, 26])\n",
      "torch.Size([16, 71])\n",
      "torch.Size([16, 43])\n",
      "torch.Size([16, 72])\n",
      "torch.Size([16, 52])\n",
      "torch.Size([16, 44])\n",
      "torch.Size([16, 43])\n",
      "torch.Size([16, 38])\n",
      "torch.Size([16, 38])\n",
      "torch.Size([16, 35])\n",
      "torch.Size([16, 42])\n",
      "torch.Size([16, 47])\n",
      "torch.Size([16, 40])\n",
      "torch.Size([16, 39])\n",
      "torch.Size([16, 47])\n",
      "torch.Size([16, 35])\n",
      "torch.Size([16, 48])\n",
      "torch.Size([16, 44])\n",
      "torch.Size([16, 58])\n",
      "torch.Size([16, 56])\n",
      "torch.Size([16, 35])\n",
      "torch.Size([16, 36])\n",
      "torch.Size([16, 200])\n",
      "torch.Size([16, 33])\n",
      "torch.Size([16, 47])\n",
      "torch.Size([16, 45])\n",
      "torch.Size([16, 71])\n",
      "torch.Size([16, 45])\n",
      "torch.Size([16, 64])\n",
      "torch.Size([16, 34])\n",
      "torch.Size([16, 35])\n",
      "torch.Size([16, 34])\n",
      "torch.Size([16, 40])\n",
      "torch.Size([16, 52])\n",
      "torch.Size([16, 28])\n",
      "torch.Size([16, 27])\n",
      "torch.Size([16, 47])\n",
      "torch.Size([16, 45])\n",
      "torch.Size([16, 54])\n",
      "torch.Size([16, 37])\n",
      "torch.Size([16, 51])\n",
      "torch.Size([16, 45])\n",
      "torch.Size([16, 40])\n",
      "torch.Size([16, 44])\n",
      "torch.Size([16, 41])\n",
      "torch.Size([16, 43])\n",
      "torch.Size([16, 67])\n",
      "torch.Size([16, 51])\n",
      "torch.Size([16, 77])\n",
      "torch.Size([16, 42])\n",
      "torch.Size([16, 33])\n",
      "torch.Size([16, 50])\n",
      "torch.Size([16, 44])\n",
      "torch.Size([16, 74])\n",
      "torch.Size([16, 38])\n",
      "torch.Size([16, 27])\n",
      "torch.Size([16, 43])\n",
      "torch.Size([16, 31])\n",
      "torch.Size([16, 43])\n",
      "torch.Size([16, 37])\n",
      "torch.Size([16, 47])\n",
      "torch.Size([16, 35])\n",
      "torch.Size([16, 33])\n",
      "torch.Size([16, 59])\n",
      "torch.Size([16, 45])\n",
      "torch.Size([16, 69])\n",
      "torch.Size([16, 36])\n",
      "torch.Size([16, 58])\n",
      "torch.Size([16, 31])\n",
      "torch.Size([16, 47])\n",
      "torch.Size([16, 52])\n",
      "torch.Size([16, 34])\n",
      "torch.Size([16, 35])\n",
      "torch.Size([16, 47])\n",
      "torch.Size([16, 44])\n",
      "torch.Size([16, 49])\n",
      "torch.Size([16, 47])\n",
      "torch.Size([16, 31])\n",
      "torch.Size([16, 58])\n",
      "torch.Size([16, 57])\n",
      "torch.Size([16, 47])\n",
      "torch.Size([16, 45])\n",
      "torch.Size([16, 46])\n",
      "torch.Size([16, 32])\n",
      "torch.Size([16, 33])\n",
      "torch.Size([16, 35])\n",
      "torch.Size([16, 43])\n",
      "torch.Size([16, 43])\n",
      "torch.Size([16, 48])\n",
      "torch.Size([16, 50])\n",
      "torch.Size([16, 35])\n",
      "torch.Size([16, 55])\n",
      "torch.Size([16, 48])\n",
      "torch.Size([16, 27])\n",
      "torch.Size([16, 29])\n",
      "torch.Size([16, 46])\n",
      "torch.Size([16, 35])\n",
      "torch.Size([16, 36])\n",
      "torch.Size([16, 44])\n",
      "torch.Size([16, 34])\n",
      "torch.Size([16, 33])\n",
      "torch.Size([16, 49])\n",
      "torch.Size([16, 39])\n",
      "torch.Size([16, 38])\n",
      "torch.Size([16, 35])\n",
      "torch.Size([16, 38])\n",
      "torch.Size([16, 31])\n",
      "torch.Size([16, 40])\n",
      "torch.Size([16, 30])\n",
      "torch.Size([16, 52])\n",
      "torch.Size([16, 26])\n",
      "torch.Size([16, 56])\n",
      "torch.Size([16, 45])\n",
      "torch.Size([16, 45])\n",
      "torch.Size([16, 36])\n",
      "torch.Size([16, 46])\n",
      "torch.Size([16, 42])\n",
      "torch.Size([16, 30])\n",
      "torch.Size([16, 51])\n",
      "torch.Size([16, 43])\n",
      "torch.Size([16, 41])\n",
      "torch.Size([16, 42])\n",
      "torch.Size([16, 51])\n",
      "torch.Size([16, 46])\n",
      "torch.Size([16, 57])\n",
      "torch.Size([16, 47])\n",
      "torch.Size([16, 33])\n",
      "torch.Size([16, 41])\n",
      "torch.Size([16, 47])\n",
      "torch.Size([16, 46])\n",
      "torch.Size([16, 51])\n",
      "torch.Size([16, 35])\n",
      "torch.Size([16, 52])\n",
      "torch.Size([16, 29])\n",
      "torch.Size([16, 45])\n",
      "torch.Size([16, 44])\n",
      "torch.Size([16, 36])\n",
      "torch.Size([16, 44])\n",
      "torch.Size([16, 36])\n",
      "torch.Size([16, 48])\n",
      "torch.Size([16, 44])\n",
      "torch.Size([16, 44])\n",
      "torch.Size([16, 45])\n",
      "torch.Size([16, 61])\n",
      "torch.Size([16, 52])\n",
      "torch.Size([16, 40])\n",
      "torch.Size([16, 43])\n",
      "torch.Size([16, 41])\n",
      "torch.Size([16, 44])\n",
      "torch.Size([16, 40])\n",
      "torch.Size([16, 31])\n",
      "torch.Size([16, 39])\n",
      "torch.Size([16, 34])\n",
      "torch.Size([16, 93])\n",
      "torch.Size([16, 34])\n",
      "torch.Size([16, 51])\n",
      "torch.Size([16, 54])\n",
      "torch.Size([16, 49])\n",
      "torch.Size([16, 37])\n",
      "torch.Size([16, 54])\n",
      "torch.Size([16, 42])\n",
      "torch.Size([16, 73])\n",
      "torch.Size([16, 55])\n",
      "torch.Size([16, 32])\n",
      "torch.Size([16, 35])\n",
      "torch.Size([16, 36])\n",
      "torch.Size([16, 52])\n",
      "torch.Size([16, 31])\n",
      "torch.Size([16, 33])\n",
      "torch.Size([16, 107])\n",
      "torch.Size([16, 40])\n",
      "torch.Size([16, 42])\n",
      "torch.Size([16, 50])\n",
      "torch.Size([16, 47])\n",
      "torch.Size([16, 56])\n",
      "torch.Size([16, 33])\n",
      "torch.Size([16, 119])\n",
      "torch.Size([16, 48])\n",
      "torch.Size([16, 63])\n",
      "torch.Size([16, 48])\n",
      "torch.Size([16, 96])\n",
      "torch.Size([16, 43])\n",
      "torch.Size([16, 50])\n",
      "torch.Size([16, 33])\n",
      "torch.Size([16, 47])\n",
      "torch.Size([16, 51])\n",
      "torch.Size([16, 39])\n",
      "torch.Size([16, 45])\n",
      "torch.Size([16, 46])\n",
      "torch.Size([16, 48])\n",
      "torch.Size([16, 33])\n",
      "torch.Size([16, 36])\n",
      "torch.Size([16, 45])\n",
      "torch.Size([16, 47])\n",
      "torch.Size([16, 116])\n",
      "torch.Size([16, 45])\n",
      "torch.Size([16, 77])\n",
      "torch.Size([16, 41])\n",
      "torch.Size([16, 39])\n",
      "torch.Size([16, 36])\n",
      "torch.Size([16, 24])\n",
      "torch.Size([16, 41])\n",
      "torch.Size([16, 28])\n",
      "torch.Size([16, 36])\n",
      "torch.Size([16, 43])\n",
      "torch.Size([16, 35])\n",
      "torch.Size([16, 44])\n",
      "torch.Size([16, 37])\n",
      "torch.Size([16, 45])\n",
      "torch.Size([16, 49])\n",
      "torch.Size([16, 44])\n",
      "torch.Size([16, 35])\n",
      "torch.Size([16, 35])\n",
      "torch.Size([16, 40])\n",
      "torch.Size([16, 32])\n",
      "torch.Size([16, 36])\n",
      "torch.Size([16, 31])\n",
      "torch.Size([16, 71])\n",
      "torch.Size([16, 40])\n",
      "torch.Size([16, 45])\n",
      "torch.Size([16, 42])\n",
      "torch.Size([16, 51])\n",
      "torch.Size([16, 40])\n",
      "torch.Size([16, 38])\n",
      "torch.Size([16, 28])\n",
      "torch.Size([16, 39])\n",
      "torch.Size([16, 45])\n",
      "torch.Size([16, 34])\n",
      "torch.Size([16, 37])\n",
      "torch.Size([16, 51])\n",
      "torch.Size([16, 40])\n",
      "torch.Size([16, 45])\n",
      "torch.Size([16, 32])\n",
      "torch.Size([16, 45])\n",
      "torch.Size([16, 37])\n",
      "torch.Size([16, 32])\n",
      "torch.Size([16, 46])\n",
      "torch.Size([16, 43])\n",
      "torch.Size([16, 39])\n",
      "torch.Size([16, 39])\n",
      "torch.Size([16, 32])\n",
      "torch.Size([16, 44])\n",
      "torch.Size([16, 36])\n",
      "torch.Size([16, 34])\n",
      "torch.Size([16, 27])\n",
      "torch.Size([16, 43])\n",
      "torch.Size([16, 48])\n",
      "torch.Size([16, 41])\n",
      "torch.Size([16, 41])\n",
      "torch.Size([16, 48])\n",
      "torch.Size([16, 34])\n",
      "torch.Size([16, 44])\n",
      "torch.Size([16, 48])\n",
      "torch.Size([16, 42])\n",
      "torch.Size([16, 47])\n",
      "torch.Size([16, 38])\n",
      "torch.Size([16, 31])\n",
      "torch.Size([16, 56])\n",
      "torch.Size([16, 44])\n",
      "torch.Size([16, 35])\n",
      "torch.Size([16, 29])\n",
      "torch.Size([16, 45])\n",
      "torch.Size([16, 38])\n",
      "torch.Size([16, 50])\n",
      "torch.Size([16, 30])\n",
      "torch.Size([16, 43])\n",
      "torch.Size([16, 30])\n",
      "torch.Size([16, 38])\n",
      "torch.Size([16, 40])\n",
      "torch.Size([16, 35])\n",
      "torch.Size([16, 40])\n",
      "torch.Size([16, 42])\n",
      "torch.Size([16, 46])\n",
      "torch.Size([16, 45])\n",
      "torch.Size([16, 45])\n",
      "torch.Size([16, 50])\n",
      "torch.Size([16, 45])\n",
      "torch.Size([16, 27])\n",
      "torch.Size([16, 48])\n",
      "torch.Size([16, 45])\n",
      "torch.Size([16, 43])\n",
      "torch.Size([16, 51])\n",
      "torch.Size([16, 45])\n",
      "torch.Size([16, 32])\n",
      "torch.Size([16, 41])\n",
      "torch.Size([16, 44])\n",
      "torch.Size([16, 52])\n",
      "torch.Size([16, 54])\n",
      "torch.Size([16, 50])\n",
      "torch.Size([16, 32])\n",
      "torch.Size([16, 55])\n",
      "torch.Size([16, 49])\n",
      "torch.Size([16, 35])\n",
      "torch.Size([16, 39])\n",
      "torch.Size([16, 51])\n",
      "torch.Size([16, 34])\n",
      "torch.Size([16, 45])\n",
      "torch.Size([16, 42])\n",
      "torch.Size([16, 54])\n",
      "torch.Size([16, 31])\n",
      "torch.Size([16, 43])\n",
      "torch.Size([16, 45])\n",
      "torch.Size([16, 47])\n",
      "torch.Size([16, 29])\n",
      "torch.Size([16, 40])\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([16, 51])\n",
      "torch.Size([16, 42])\n",
      "torch.Size([16, 45])\n",
      "torch.Size([16, 46])\n",
      "torch.Size([16, 36])\n",
      "torch.Size([16, 53])\n",
      "torch.Size([16, 67])\n",
      "torch.Size([16, 61])\n",
      "torch.Size([16, 41])\n",
      "torch.Size([16, 36])\n",
      "torch.Size([16, 48])\n",
      "torch.Size([16, 39])\n",
      "torch.Size([16, 48])\n",
      "torch.Size([16, 60])\n",
      "torch.Size([16, 30])\n",
      "torch.Size([16, 49])\n",
      "torch.Size([16, 44])\n",
      "torch.Size([16, 72])\n",
      "torch.Size([16, 50])\n",
      "torch.Size([16, 42])\n",
      "torch.Size([16, 112])\n",
      "torch.Size([16, 33])\n",
      "torch.Size([16, 36])\n",
      "torch.Size([16, 50])\n",
      "torch.Size([16, 31])\n",
      "torch.Size([16, 73])\n",
      "torch.Size([16, 47])\n",
      "torch.Size([16, 51])\n",
      "torch.Size([16, 26])\n",
      "torch.Size([16, 35])\n",
      "torch.Size([16, 42])\n",
      "torch.Size([16, 45])\n",
      "torch.Size([16, 35])\n",
      "torch.Size([16, 23])\n",
      "torch.Size([16, 68])\n",
      "torch.Size([16, 33])\n",
      "torch.Size([16, 45])\n",
      "torch.Size([16, 25])\n",
      "torch.Size([16, 54])\n",
      "torch.Size([16, 47])\n",
      "torch.Size([16, 39])\n",
      "torch.Size([16, 34])\n",
      "torch.Size([16, 47])\n",
      "torch.Size([16, 31])\n",
      "torch.Size([16, 29])\n",
      "torch.Size([16, 33])\n",
      "torch.Size([16, 73])\n",
      "torch.Size([16, 40])\n",
      "torch.Size([16, 59])\n",
      "torch.Size([16, 40])\n",
      "torch.Size([16, 45])\n",
      "torch.Size([16, 47])\n",
      "torch.Size([16, 56])\n",
      "torch.Size([16, 48])\n",
      "torch.Size([16, 41])\n",
      "torch.Size([16, 65])\n",
      "torch.Size([16, 51])\n",
      "torch.Size([16, 62])\n",
      "torch.Size([16, 33])\n",
      "torch.Size([16, 39])\n",
      "torch.Size([16, 41])\n",
      "torch.Size([16, 54])\n",
      "torch.Size([16, 41])\n",
      "torch.Size([16, 49])\n",
      "torch.Size([16, 44])\n",
      "torch.Size([16, 39])\n",
      "torch.Size([16, 35])\n",
      "torch.Size([16, 38])\n",
      "torch.Size([16, 48])\n",
      "torch.Size([16, 38])\n",
      "torch.Size([16, 32])\n",
      "torch.Size([16, 33])\n",
      "torch.Size([16, 46])\n",
      "torch.Size([16, 50])\n",
      "torch.Size([16, 37])\n",
      "torch.Size([16, 58])\n",
      "torch.Size([16, 57])\n",
      "torch.Size([16, 43])\n",
      "torch.Size([16, 33])\n",
      "torch.Size([16, 48])\n",
      "torch.Size([16, 36])\n",
      "torch.Size([16, 39])\n",
      "torch.Size([16, 28])\n",
      "torch.Size([16, 45])\n",
      "torch.Size([16, 28])\n",
      "torch.Size([16, 43])\n",
      "torch.Size([16, 82])\n",
      "torch.Size([16, 40])\n",
      "torch.Size([16, 40])\n",
      "torch.Size([16, 48])\n",
      "torch.Size([16, 30])\n",
      "torch.Size([16, 43])\n",
      "torch.Size([16, 38])\n",
      "torch.Size([16, 42])\n",
      "torch.Size([16, 40])\n",
      "torch.Size([16, 39])\n",
      "torch.Size([16, 36])\n",
      "torch.Size([16, 42])\n",
      "torch.Size([16, 43])\n",
      "torch.Size([16, 56])\n",
      "torch.Size([16, 28])\n",
      "torch.Size([16, 41])\n",
      "torch.Size([16, 31])\n",
      "torch.Size([16, 50])\n",
      "torch.Size([16, 49])\n",
      "torch.Size([16, 46])\n",
      "torch.Size([16, 37])\n",
      "torch.Size([16, 29])\n",
      "torch.Size([16, 28])\n",
      "torch.Size([16, 33])\n",
      "torch.Size([16, 51])\n",
      "torch.Size([16, 50])\n",
      "torch.Size([16, 42])\n",
      "torch.Size([16, 60])\n",
      "torch.Size([16, 49])\n",
      "torch.Size([16, 45])\n",
      "torch.Size([16, 43])\n",
      "torch.Size([16, 53])\n",
      "torch.Size([16, 33])\n",
      "torch.Size([16, 46])\n",
      "torch.Size([16, 38])\n",
      "torch.Size([16, 47])\n",
      "torch.Size([16, 42])\n",
      "torch.Size([16, 51])\n",
      "torch.Size([16, 39])\n",
      "torch.Size([16, 32])\n",
      "torch.Size([16, 37])\n",
      "torch.Size([16, 40])\n",
      "torch.Size([16, 54])\n",
      "torch.Size([16, 58])\n",
      "torch.Size([16, 53])\n",
      "torch.Size([16, 49])\n",
      "torch.Size([16, 36])\n",
      "torch.Size([16, 74])\n",
      "torch.Size([16, 75])\n",
      "torch.Size([16, 42])\n",
      "torch.Size([16, 65])\n",
      "torch.Size([16, 43])\n",
      "torch.Size([16, 53])\n",
      "torch.Size([16, 55])\n",
      "torch.Size([16, 117])\n",
      "torch.Size([16, 76])\n",
      "torch.Size([16, 32])\n",
      "torch.Size([16, 36])\n",
      "torch.Size([16, 33])\n",
      "torch.Size([16, 52])\n",
      "torch.Size([16, 30])\n",
      "torch.Size([16, 30])\n",
      "torch.Size([16, 39])\n",
      "torch.Size([16, 51])\n",
      "torch.Size([16, 38])\n",
      "torch.Size([16, 33])\n",
      "torch.Size([16, 44])\n",
      "torch.Size([16, 37])\n",
      "torch.Size([16, 41])\n",
      "torch.Size([16, 36])\n",
      "torch.Size([16, 51])\n",
      "torch.Size([16, 34])\n",
      "torch.Size([16, 33])\n",
      "torch.Size([16, 27])\n",
      "torch.Size([16, 39])\n",
      "torch.Size([16, 47])\n",
      "torch.Size([16, 53])\n",
      "torch.Size([16, 53])\n",
      "torch.Size([16, 48])\n",
      "torch.Size([16, 49])\n",
      "torch.Size([16, 39])\n",
      "torch.Size([16, 34])\n",
      "torch.Size([16, 44])\n",
      "torch.Size([16, 50])\n",
      "torch.Size([16, 42])\n",
      "torch.Size([16, 86])\n",
      "torch.Size([16, 52])\n",
      "torch.Size([16, 42])\n",
      "torch.Size([16, 76])\n",
      "torch.Size([16, 47])\n",
      "torch.Size([16, 48])\n",
      "torch.Size([16, 46])\n",
      "torch.Size([16, 45])\n",
      "torch.Size([16, 30])\n",
      "torch.Size([16, 39])\n",
      "torch.Size([16, 32])\n",
      "torch.Size([16, 52])\n",
      "torch.Size([16, 51])\n",
      "torch.Size([16, 38])\n",
      "torch.Size([16, 67])\n",
      "torch.Size([16, 37])\n",
      "torch.Size([16, 39])\n",
      "torch.Size([16, 34])\n",
      "torch.Size([16, 58])\n",
      "torch.Size([10, 32])\n",
      "\n"
     ]
    }
   ],
   "source": [
    "trans_texts = []\n",
    "CHUNK = 16\n",
    "print('translating texts')\n",
    "CTS = len(df)//CHUNK\n",
    "if len(df)%CHUNK!=0: CTS += 1\n",
    "for i,j in tqdm(enumerate(range(CTS)), total=CTS):\n",
    "    a = j*CHUNK\n",
    "    b = (j+1)*CHUNK\n",
    "    b = min(b,len(df))\n",
    "    with torch.no_grad() :\n",
    "        input_ids = tokenizer(list(df.iloc[a:b].title.values), return_tensors=\"pt\", truncation=True, padding=True, max_length=200).input_ids.cuda()\n",
    "        print(input_ids.shape)\n",
    "        outputs = txt_model.generate(input_ids=input_ids, num_return_sequences=1)    \n",
    "        val = tokenizer.batch_decode(outputs, skip_special_tokens=True)\n",
    "    trans_texts.extend(val)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "df['translated_titles'] = trans_texts"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "df.to_csv('data/train.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "from imports import *\n",
    "from clip_.train_text import *\n",
    "from clip_.data import TextDS\n",
    "from utils import load_data\n",
    "from arcface import ArcMarginProduct\n",
    "import matplotlib.pyplot as plt\n",
    "import clip\n",
    "np.random.seed(1337)\n",
    "device = torch.device('cuda')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "df, train_df, val_df, train_labels, val_labels = load_data(train_perc=0.3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "bs = 32\n",
    "bs_val = 128\n",
    "max_length=200"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "tr_ds = TextDS(train_df, max_length=max_length)\n",
    "val_ds = TextDS(val_df, max_length=max_length)\n",
    "full_ds = TextDS(df, max_length=max_length)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "torch.save(tr_ds, 'data/text_models/datasets/clip/tr_ds_0.3_{}.pth'.format(max_length))\n",
    "torch.save(val_ds, 'data/text_models/datasets/clip/val_ds_0.3_{}.pth'.format(max_length))\n",
    "torch.save(full_ds, 'data/text_models/datasets/clip/full_ds_0.3_{}.pth'.format(max_length))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "tr_ds = torch.load('data/text_models/datasets/clip/tr_ds_0.3_{}.pth'.format(max_length))\n",
    "val_ds = torch.load('data/text_models/datasets/clip/val_ds_0.3_{}.pth'.format(max_length))\n",
    "full_ds = torch.load('data/text_models/datasets/clip/full_ds_0.3_{}.pth'.format(max_length))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "tr_test_dl = DataLoader(tr_ds, batch_size = bs_val, shuffle = False, pin_memory = True)\n",
    "tr_dl = DataLoader(tr_ds, batch_size = bs, shuffle = True, pin_memory = True)\n",
    "val_dl = DataLoader(val_ds, batch_size = bs_val, shuffle = False, pin_memory = True)\n",
    "#full_dl = DataLoader(full_ds, batch_size = bs_val, shuffle = False, pin_memory = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "class CLIPTxt(torch.nn.Module) :\n",
    "    def __init__(self, model_name=\"RN50\", device='cuda') :\n",
    "        super().__init__()\n",
    "        self.model = clip.load(model_name, device=device, jit=False)[0]\n",
    "    def forward(self, imgs) :\n",
    "        return self.model.encode_text(imgs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# model\n",
    "model = CLIPTxt().to(device)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "centers = compute_centers(tr_test_dl, model, train_df)\n",
    "torch.save(centers, 'data/text_models/clip/centers_0.3_{}.pth'.format(max_length))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "centers = torch.load('data/text_models/clip/centers_0.3_{}.pth'.format(max_length))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Using center as wieghts\n"
     ]
    }
   ],
   "source": [
    "metric_fc = ArcMarginProduct(512, train_df['label_group'].nunique(), s=30, m=0.5,\n",
    "                             easy_margin=False, centers=centers, device=device).to(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "n_epochs, lf, params, optimizer, sched = get_hparams(tr_dl, model, metric_fc, lr=5e-4, n_epochs=10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "model.positional_embedding\n",
      "model.text_projection\n",
      "model.logit_scale\n",
      "model.visual.conv1.weight\n",
      "model.visual.bn1.weight\n",
      "model.visual.bn1.bias\n",
      "model.visual.conv2.weight\n",
      "model.visual.bn2.weight\n",
      "model.visual.bn2.bias\n",
      "model.visual.conv3.weight\n",
      "model.visual.bn3.weight\n",
      "model.visual.bn3.bias\n",
      "model.visual.layer1.0.conv1.weight\n",
      "model.visual.layer1.0.bn1.weight\n",
      "model.visual.layer1.0.bn1.bias\n",
      "model.visual.layer1.0.conv2.weight\n",
      "model.visual.layer1.0.bn2.weight\n",
      "model.visual.layer1.0.bn2.bias\n",
      "model.visual.layer1.0.conv3.weight\n",
      "model.visual.layer1.0.bn3.weight\n",
      "model.visual.layer1.0.bn3.bias\n",
      "model.visual.layer1.0.downsample.0.weight\n",
      "model.visual.layer1.0.downsample.1.weight\n",
      "model.visual.layer1.0.downsample.1.bias\n",
      "model.visual.layer1.1.conv1.weight\n",
      "model.visual.layer1.1.bn1.weight\n",
      "model.visual.layer1.1.bn1.bias\n",
      "model.visual.layer1.1.conv2.weight\n",
      "model.visual.layer1.1.bn2.weight\n",
      "model.visual.layer1.1.bn2.bias\n",
      "model.visual.layer1.1.conv3.weight\n",
      "model.visual.layer1.1.bn3.weight\n",
      "model.visual.layer1.1.bn3.bias\n",
      "model.visual.layer1.2.conv1.weight\n",
      "model.visual.layer1.2.bn1.weight\n",
      "model.visual.layer1.2.bn1.bias\n",
      "model.visual.layer1.2.conv2.weight\n",
      "model.visual.layer1.2.bn2.weight\n",
      "model.visual.layer1.2.bn2.bias\n",
      "model.visual.layer1.2.conv3.weight\n",
      "model.visual.layer1.2.bn3.weight\n",
      "model.visual.layer1.2.bn3.bias\n",
      "model.visual.layer2.0.conv1.weight\n",
      "model.visual.layer2.0.bn1.weight\n",
      "model.visual.layer2.0.bn1.bias\n",
      "model.visual.layer2.0.conv2.weight\n",
      "model.visual.layer2.0.bn2.weight\n",
      "model.visual.layer2.0.bn2.bias\n",
      "model.visual.layer2.0.conv3.weight\n",
      "model.visual.layer2.0.bn3.weight\n",
      "model.visual.layer2.0.bn3.bias\n",
      "model.visual.layer2.0.downsample.0.weight\n",
      "model.visual.layer2.0.downsample.1.weight\n",
      "model.visual.layer2.0.downsample.1.bias\n",
      "model.visual.layer2.1.conv1.weight\n",
      "model.visual.layer2.1.bn1.weight\n",
      "model.visual.layer2.1.bn1.bias\n",
      "model.visual.layer2.1.conv2.weight\n",
      "model.visual.layer2.1.bn2.weight\n",
      "model.visual.layer2.1.bn2.bias\n",
      "model.visual.layer2.1.conv3.weight\n",
      "model.visual.layer2.1.bn3.weight\n",
      "model.visual.layer2.1.bn3.bias\n",
      "model.visual.layer2.2.conv1.weight\n",
      "model.visual.layer2.2.bn1.weight\n",
      "model.visual.layer2.2.bn1.bias\n",
      "model.visual.layer2.2.conv2.weight\n",
      "model.visual.layer2.2.bn2.weight\n",
      "model.visual.layer2.2.bn2.bias\n",
      "model.visual.layer2.2.conv3.weight\n",
      "model.visual.layer2.2.bn3.weight\n",
      "model.visual.layer2.2.bn3.bias\n",
      "model.visual.layer2.3.conv1.weight\n",
      "model.visual.layer2.3.bn1.weight\n",
      "model.visual.layer2.3.bn1.bias\n",
      "model.visual.layer2.3.conv2.weight\n",
      "model.visual.layer2.3.bn2.weight\n",
      "model.visual.layer2.3.bn2.bias\n",
      "model.visual.layer2.3.conv3.weight\n",
      "model.visual.layer2.3.bn3.weight\n",
      "model.visual.layer2.3.bn3.bias\n",
      "model.visual.layer3.0.conv1.weight\n",
      "model.visual.layer3.0.bn1.weight\n",
      "model.visual.layer3.0.bn1.bias\n",
      "model.visual.layer3.0.conv2.weight\n",
      "model.visual.layer3.0.bn2.weight\n",
      "model.visual.layer3.0.bn2.bias\n",
      "model.visual.layer3.0.conv3.weight\n",
      "model.visual.layer3.0.bn3.weight\n",
      "model.visual.layer3.0.bn3.bias\n",
      "model.visual.layer3.0.downsample.0.weight\n",
      "model.visual.layer3.0.downsample.1.weight\n",
      "model.visual.layer3.0.downsample.1.bias\n",
      "model.visual.layer3.1.conv1.weight\n",
      "model.visual.layer3.1.bn1.weight\n",
      "model.visual.layer3.1.bn1.bias\n",
      "model.visual.layer3.1.conv2.weight\n",
      "model.visual.layer3.1.bn2.weight\n",
      "model.visual.layer3.1.bn2.bias\n",
      "model.visual.layer3.1.conv3.weight\n",
      "model.visual.layer3.1.bn3.weight\n",
      "model.visual.layer3.1.bn3.bias\n",
      "model.visual.layer3.2.conv1.weight\n",
      "model.visual.layer3.2.bn1.weight\n",
      "model.visual.layer3.2.bn1.bias\n",
      "model.visual.layer3.2.conv2.weight\n",
      "model.visual.layer3.2.bn2.weight\n",
      "model.visual.layer3.2.bn2.bias\n",
      "model.visual.layer3.2.conv3.weight\n",
      "model.visual.layer3.2.bn3.weight\n",
      "model.visual.layer3.2.bn3.bias\n",
      "model.visual.layer3.3.conv1.weight\n",
      "model.visual.layer3.3.bn1.weight\n",
      "model.visual.layer3.3.bn1.bias\n",
      "model.visual.layer3.3.conv2.weight\n",
      "model.visual.layer3.3.bn2.weight\n",
      "model.visual.layer3.3.bn2.bias\n",
      "model.visual.layer3.3.conv3.weight\n",
      "model.visual.layer3.3.bn3.weight\n",
      "model.visual.layer3.3.bn3.bias\n",
      "model.visual.layer3.4.conv1.weight\n",
      "model.visual.layer3.4.bn1.weight\n",
      "model.visual.layer3.4.bn1.bias\n",
      "model.visual.layer3.4.conv2.weight\n",
      "model.visual.layer3.4.bn2.weight\n",
      "model.visual.layer3.4.bn2.bias\n",
      "model.visual.layer3.4.conv3.weight\n",
      "model.visual.layer3.4.bn3.weight\n",
      "model.visual.layer3.4.bn3.bias\n",
      "model.visual.layer3.5.conv1.weight\n",
      "model.visual.layer3.5.bn1.weight\n",
      "model.visual.layer3.5.bn1.bias\n",
      "model.visual.layer3.5.conv2.weight\n",
      "model.visual.layer3.5.bn2.weight\n",
      "model.visual.layer3.5.bn2.bias\n",
      "model.visual.layer3.5.conv3.weight\n",
      "model.visual.layer3.5.bn3.weight\n",
      "model.visual.layer3.5.bn3.bias\n",
      "model.visual.layer4.0.conv1.weight\n",
      "model.visual.layer4.0.bn1.weight\n",
      "model.visual.layer4.0.bn1.bias\n",
      "model.visual.layer4.0.conv2.weight\n",
      "model.visual.layer4.0.bn2.weight\n",
      "model.visual.layer4.0.bn2.bias\n",
      "model.visual.layer4.0.conv3.weight\n",
      "model.visual.layer4.0.bn3.weight\n",
      "model.visual.layer4.0.bn3.bias\n",
      "model.visual.layer4.0.downsample.0.weight\n",
      "model.visual.layer4.0.downsample.1.weight\n",
      "model.visual.layer4.0.downsample.1.bias\n",
      "model.visual.layer4.1.conv1.weight\n",
      "model.visual.layer4.1.bn1.weight\n",
      "model.visual.layer4.1.bn1.bias\n",
      "model.visual.layer4.1.conv2.weight\n",
      "model.visual.layer4.1.bn2.weight\n",
      "model.visual.layer4.1.bn2.bias\n",
      "model.visual.layer4.1.conv3.weight\n",
      "model.visual.layer4.1.bn3.weight\n",
      "model.visual.layer4.1.bn3.bias\n",
      "model.visual.layer4.2.conv1.weight\n",
      "model.visual.layer4.2.bn1.weight\n",
      "model.visual.layer4.2.bn1.bias\n",
      "model.visual.layer4.2.conv2.weight\n",
      "model.visual.layer4.2.bn2.weight\n",
      "model.visual.layer4.2.bn2.bias\n",
      "model.visual.layer4.2.conv3.weight\n",
      "model.visual.layer4.2.bn3.weight\n",
      "model.visual.layer4.2.bn3.bias\n",
      "model.visual.attnpool.positional_embedding\n",
      "model.visual.attnpool.k_proj.weight\n",
      "model.visual.attnpool.k_proj.bias\n",
      "model.visual.attnpool.q_proj.weight\n",
      "model.visual.attnpool.q_proj.bias\n",
      "model.visual.attnpool.v_proj.weight\n",
      "model.visual.attnpool.v_proj.bias\n",
      "model.visual.attnpool.c_proj.weight\n",
      "model.visual.attnpool.c_proj.bias\n",
      "model.transformer.resblocks.0.attn.in_proj_weight\n",
      "model.transformer.resblocks.0.attn.in_proj_bias\n",
      "model.transformer.resblocks.0.attn.out_proj.weight\n",
      "model.transformer.resblocks.0.attn.out_proj.bias\n",
      "model.transformer.resblocks.0.ln_1.weight\n",
      "model.transformer.resblocks.0.ln_1.bias\n",
      "model.transformer.resblocks.0.mlp.c_fc.weight\n",
      "model.transformer.resblocks.0.mlp.c_fc.bias\n",
      "model.transformer.resblocks.0.mlp.c_proj.weight\n",
      "model.transformer.resblocks.0.mlp.c_proj.bias\n",
      "model.transformer.resblocks.0.ln_2.weight\n",
      "model.transformer.resblocks.0.ln_2.bias\n",
      "model.transformer.resblocks.1.attn.in_proj_weight\n",
      "model.transformer.resblocks.1.attn.in_proj_bias\n",
      "model.transformer.resblocks.1.attn.out_proj.weight\n",
      "model.transformer.resblocks.1.attn.out_proj.bias\n",
      "model.transformer.resblocks.1.ln_1.weight\n",
      "model.transformer.resblocks.1.ln_1.bias\n",
      "model.transformer.resblocks.1.mlp.c_fc.weight\n",
      "model.transformer.resblocks.1.mlp.c_fc.bias\n",
      "model.transformer.resblocks.1.mlp.c_proj.weight\n",
      "model.transformer.resblocks.1.mlp.c_proj.bias\n",
      "model.transformer.resblocks.1.ln_2.weight\n",
      "model.transformer.resblocks.1.ln_2.bias\n",
      "model.transformer.resblocks.2.attn.in_proj_weight\n",
      "model.transformer.resblocks.2.attn.in_proj_bias\n",
      "model.transformer.resblocks.2.attn.out_proj.weight\n",
      "model.transformer.resblocks.2.attn.out_proj.bias\n",
      "model.transformer.resblocks.2.ln_1.weight\n",
      "model.transformer.resblocks.2.ln_1.bias\n",
      "model.transformer.resblocks.2.mlp.c_fc.weight\n",
      "model.transformer.resblocks.2.mlp.c_fc.bias\n",
      "model.transformer.resblocks.2.mlp.c_proj.weight\n",
      "model.transformer.resblocks.2.mlp.c_proj.bias\n",
      "model.transformer.resblocks.2.ln_2.weight\n",
      "model.transformer.resblocks.2.ln_2.bias\n",
      "model.transformer.resblocks.3.attn.in_proj_weight\n",
      "model.transformer.resblocks.3.attn.in_proj_bias\n",
      "model.transformer.resblocks.3.attn.out_proj.weight\n",
      "model.transformer.resblocks.3.attn.out_proj.bias\n",
      "model.transformer.resblocks.3.ln_1.weight\n",
      "model.transformer.resblocks.3.ln_1.bias\n",
      "model.transformer.resblocks.3.mlp.c_fc.weight\n",
      "model.transformer.resblocks.3.mlp.c_fc.bias\n",
      "model.transformer.resblocks.3.mlp.c_proj.weight\n",
      "model.transformer.resblocks.3.mlp.c_proj.bias\n",
      "model.transformer.resblocks.3.ln_2.weight\n",
      "model.transformer.resblocks.3.ln_2.bias\n",
      "model.transformer.resblocks.4.attn.in_proj_weight\n",
      "model.transformer.resblocks.4.attn.in_proj_bias\n",
      "model.transformer.resblocks.4.attn.out_proj.weight\n",
      "model.transformer.resblocks.4.attn.out_proj.bias\n",
      "model.transformer.resblocks.4.ln_1.weight\n",
      "model.transformer.resblocks.4.ln_1.bias\n",
      "model.transformer.resblocks.4.mlp.c_fc.weight\n",
      "model.transformer.resblocks.4.mlp.c_fc.bias\n",
      "model.transformer.resblocks.4.mlp.c_proj.weight\n",
      "model.transformer.resblocks.4.mlp.c_proj.bias\n",
      "model.transformer.resblocks.4.ln_2.weight\n",
      "model.transformer.resblocks.4.ln_2.bias\n",
      "model.transformer.resblocks.5.attn.in_proj_weight\n",
      "model.transformer.resblocks.5.attn.in_proj_bias\n",
      "model.transformer.resblocks.5.attn.out_proj.weight\n",
      "model.transformer.resblocks.5.attn.out_proj.bias\n",
      "model.transformer.resblocks.5.ln_1.weight\n",
      "model.transformer.resblocks.5.ln_1.bias\n",
      "model.transformer.resblocks.5.mlp.c_fc.weight\n",
      "model.transformer.resblocks.5.mlp.c_fc.bias\n",
      "model.transformer.resblocks.5.mlp.c_proj.weight\n",
      "model.transformer.resblocks.5.mlp.c_proj.bias\n",
      "model.transformer.resblocks.5.ln_2.weight\n",
      "model.transformer.resblocks.5.ln_2.bias\n",
      "model.transformer.resblocks.6.attn.in_proj_weight\n",
      "model.transformer.resblocks.6.attn.in_proj_bias\n",
      "model.transformer.resblocks.6.attn.out_proj.weight\n",
      "model.transformer.resblocks.6.attn.out_proj.bias\n",
      "model.transformer.resblocks.6.ln_1.weight\n",
      "model.transformer.resblocks.6.ln_1.bias\n",
      "model.transformer.resblocks.6.mlp.c_fc.weight\n",
      "model.transformer.resblocks.6.mlp.c_fc.bias\n",
      "model.transformer.resblocks.6.mlp.c_proj.weight\n",
      "model.transformer.resblocks.6.mlp.c_proj.bias\n",
      "model.transformer.resblocks.6.ln_2.weight\n",
      "model.transformer.resblocks.6.ln_2.bias\n",
      "model.transformer.resblocks.7.attn.in_proj_weight\n",
      "model.transformer.resblocks.7.attn.in_proj_bias\n",
      "model.transformer.resblocks.7.attn.out_proj.weight\n",
      "model.transformer.resblocks.7.attn.out_proj.bias\n",
      "model.transformer.resblocks.7.ln_1.weight\n",
      "model.transformer.resblocks.7.ln_1.bias\n",
      "model.transformer.resblocks.7.mlp.c_fc.weight\n",
      "model.transformer.resblocks.7.mlp.c_fc.bias\n",
      "model.transformer.resblocks.7.mlp.c_proj.weight\n",
      "model.transformer.resblocks.7.mlp.c_proj.bias\n",
      "model.transformer.resblocks.7.ln_2.weight\n",
      "model.transformer.resblocks.7.ln_2.bias\n",
      "model.transformer.resblocks.8.attn.in_proj_weight\n",
      "model.transformer.resblocks.8.attn.in_proj_bias\n",
      "model.transformer.resblocks.8.attn.out_proj.weight\n",
      "model.transformer.resblocks.8.attn.out_proj.bias\n",
      "model.transformer.resblocks.8.ln_1.weight\n",
      "model.transformer.resblocks.8.ln_1.bias\n",
      "model.transformer.resblocks.8.mlp.c_fc.weight\n",
      "model.transformer.resblocks.8.mlp.c_fc.bias\n",
      "model.transformer.resblocks.8.mlp.c_proj.weight\n",
      "model.transformer.resblocks.8.mlp.c_proj.bias\n",
      "model.transformer.resblocks.8.ln_2.weight\n",
      "model.transformer.resblocks.8.ln_2.bias\n",
      "model.transformer.resblocks.9.attn.in_proj_weight\n",
      "model.transformer.resblocks.9.attn.in_proj_bias\n",
      "model.transformer.resblocks.9.attn.out_proj.weight\n",
      "model.transformer.resblocks.9.attn.out_proj.bias\n",
      "model.transformer.resblocks.9.ln_1.weight\n",
      "model.transformer.resblocks.9.ln_1.bias\n",
      "model.transformer.resblocks.9.mlp.c_fc.weight\n",
      "model.transformer.resblocks.9.mlp.c_fc.bias\n",
      "model.transformer.resblocks.9.mlp.c_proj.weight\n",
      "model.transformer.resblocks.9.mlp.c_proj.bias\n",
      "model.transformer.resblocks.9.ln_2.weight\n",
      "model.transformer.resblocks.9.ln_2.bias\n",
      "model.transformer.resblocks.10.attn.in_proj_weight\n",
      "model.transformer.resblocks.10.attn.in_proj_bias\n",
      "model.transformer.resblocks.10.attn.out_proj.weight\n",
      "model.transformer.resblocks.10.attn.out_proj.bias\n",
      "model.transformer.resblocks.10.ln_1.weight\n",
      "model.transformer.resblocks.10.ln_1.bias\n",
      "model.transformer.resblocks.10.mlp.c_fc.weight\n",
      "model.transformer.resblocks.10.mlp.c_fc.bias\n",
      "model.transformer.resblocks.10.mlp.c_proj.weight\n",
      "model.transformer.resblocks.10.mlp.c_proj.bias\n",
      "model.transformer.resblocks.10.ln_2.weight\n",
      "model.transformer.resblocks.10.ln_2.bias\n",
      "model.transformer.resblocks.11.attn.in_proj_weight\n",
      "model.transformer.resblocks.11.attn.in_proj_bias\n",
      "model.transformer.resblocks.11.attn.out_proj.weight\n",
      "model.transformer.resblocks.11.attn.out_proj.bias\n",
      "model.transformer.resblocks.11.ln_1.weight\n",
      "model.transformer.resblocks.11.ln_1.bias\n",
      "model.transformer.resblocks.11.mlp.c_fc.weight\n",
      "model.transformer.resblocks.11.mlp.c_fc.bias\n",
      "model.transformer.resblocks.11.mlp.c_proj.weight\n",
      "model.transformer.resblocks.11.mlp.c_proj.bias\n",
      "model.transformer.resblocks.11.ln_2.weight\n",
      "model.transformer.resblocks.11.ln_2.bias\n",
      "model.token_embedding.weight\n",
      "model.ln_final.weight\n",
      "model.ln_final.bias\n"
     ]
    }
   ],
   "source": [
    "for n, p in model.named_parameters() :\n",
    "    print(n)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "transformer.resblocks.0\n",
      "transformer.resblocks.1\n",
      "transformer.resblocks.2\n",
      "transformer.resblocks.3\n",
      "transformer.resblocks.4\n",
      "transformer.resblocks.5\n",
      "transformer.resblocks.6\n",
      "transformer.resblocks.7\n",
      "transformer.resblocks.8\n",
      "transformer.resblocks.9\n",
      "transformer.resblocks.10\n"
     ]
    }
   ],
   "source": [
    "mnp = list(model.named_parameters())\n",
    "\n",
    "param_groups = [{'params' : [p for n,p in mnp if n in ['model.token_embedding.weight', 'model.positional_embedding']]}]\n",
    "\n",
    "params_names = [n for n,p in mnp if n in ['model.token_embedding.weight', 'model.positional_embedding']]\n",
    "#print(params_names)\n",
    "n_blocks = 11\n",
    "for i in range(n_blocks):\n",
    "    print(f'transformer.resblocks.{i}')\n",
    "    ith_block = [p for n, p in mnp if f'transformer.resblocks.{i}.' in n]\n",
    "    ith_block_name = [n for n, p in mnp if f'transformer.resblocks.{i}.' in n]\n",
    "    params_names += ith_block_name\n",
    "    #for n in params_names :\n",
    "    #    print(n)\n",
    "    param_groups.append({'params' : ith_block})\n",
    "\n",
    "end = ['logit_scale', 'text_projection', 'final']\n",
    "param_groups.append({'params' : [p for n,p in mnp if any(end_n in n for end_n in end)]})\n",
    "params_names += [n for n,p in mnp if any(end_n in n for end_n in end)]\n",
    "\n",
    "param_groups.append({'params' : metric_fc.parameters()})\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "lf = nn.CrossEntropyLoss()\n",
    "optimizer = torch.optim.AdamW(param_groups, weight_decay = 1e-5)\n",
    "\n",
    "lr = list(np.linspace(5e-6,5e-4,len(param_groups)))\n",
    "\n",
    "n_epochs = 10\n",
    "sched = torch.optim.lr_scheduler.OneCycleLR(optimizer, max_lr=lr, pct_start=0.3,\n",
    "                                            total_steps=int(n_epochs * len(tr_dl)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "loss_hist = []\n",
    "best_thr_score={'val': {'thr': None, 'f1': None}, 'train': {'thr': None, 'f1': None}}\n",
    "thr_score_hist=[]\n",
    "ep_start = 0\n",
    "save_path = 'data/text_models/clip/test_21ap_epochs'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=10.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=327.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Thresholds', max=10.0, style=ProgressStyle(description_wi…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Checkpoint : saved model to data/text_models/clip/test_21ap_epochs_ep_0.pth\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=187.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Thresholds', max=10.0, style=ProgressStyle(description_wi…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Saved best model ep 0 with f score : 0.6049245574192079\n",
      "Ep 0: Train loss 6.6701 | Val f score 0.6049 with thresh 0.64, train f score 0.6513 with thresh 0.73\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=327.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Thresholds', max=5.0, style=ProgressStyle(description_wid…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=187.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Thresholds', max=5.0, style=ProgressStyle(description_wid…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Saved best model ep 1 with f score : 0.6132073974722503\n",
      "Ep 1: Train loss 6.6702 | Val f score 0.6132 with thresh 0.59, train f score 0.7104 with thresh 0.78\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=327.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Thresholds', max=5.0, style=ProgressStyle(description_wid…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Checkpoint : saved model to data/text_models/clip/test_21ap_epochs_ep_2.pth\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=187.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Thresholds', max=5.0, style=ProgressStyle(description_wid…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Saved best model ep 2 with f score : 0.6279998728346883\n",
      "Ep 2: Train loss 6.2191 | Val f score 0.6280 with thresh 0.69, train f score 0.7666 with thresh 0.88\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=327.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Thresholds', max=5.0, style=ProgressStyle(description_wid…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=187.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Thresholds', max=5.0, style=ProgressStyle(description_wid…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Saved best model ep 3 with f score : 0.6333510523520166\n",
      "Ep 3: Train loss 4.4237 | Val f score 0.6334 with thresh 0.74, train f score 0.8473 with thresh 0.93\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=327.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Thresholds', max=5.0, style=ProgressStyle(description_wid…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Checkpoint : saved model to data/text_models/clip/test_21ap_epochs_ep_4.pth\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=187.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Thresholds', max=5.0, style=ProgressStyle(description_wid…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Saved best model ep 4 with f score : 0.6372354247870237\n",
      "Ep 4: Train loss 2.3912 | Val f score 0.6372 with thresh 0.74, train f score 0.9145 with thresh 0.93\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=327.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Thresholds', max=5.0, style=ProgressStyle(description_wid…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=187.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Thresholds', max=5.0, style=ProgressStyle(description_wid…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Saved best model ep 5 with f score : 0.6447560419127103\n",
      "Ep 5: Train loss 1.0041 | Val f score 0.6448 with thresh 0.79, train f score 0.9624 with thresh 0.93\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=327.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Thresholds', max=5.0, style=ProgressStyle(description_wid…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Checkpoint : saved model to data/text_models/clip/test_21ap_epochs_ep_6.pth\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=187.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Thresholds', max=5.0, style=ProgressStyle(description_wid…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Saved best model ep 6 with f score : 0.6449589980744057\n",
      "Ep 6: Train loss 0.3737 | Val f score 0.6450 with thresh 0.79, train f score 0.9822 with thresh 0.88\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=327.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Thresholds', max=5.0, style=ProgressStyle(description_wid…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=187.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Thresholds', max=5.0, style=ProgressStyle(description_wid…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Saved best model ep 7 with f score : 0.6450074193692245\n",
      "Ep 7: Train loss 0.1815 | Val f score 0.6450 with thresh 0.79, train f score 0.9877 with thresh 0.83\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=327.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Thresholds', max=5.0, style=ProgressStyle(description_wid…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Checkpoint : saved model to data/text_models/clip/test_21ap_epochs_ep_8.pth\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=187.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Thresholds', max=5.0, style=ProgressStyle(description_wid…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Ep 8: Train loss 0.1411 | Val f score 0.6444 with thresh 0.79, train f score 0.9890 with thresh 0.78\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=327.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Thresholds', max=5.0, style=ProgressStyle(description_wid…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=187.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Thresholds', max=5.0, style=ProgressStyle(description_wid…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Ep 9: Train loss 0.1267 | Val f score 0.6445 with thresh 0.79, train f score 0.9896 with thresh 0.78\n",
      "\r"
     ]
    }
   ],
   "source": [
    "best_thr_score, thr_score_hist, losses = train(model, optimizer, lf, sched, metric_fc, tr_dl, val_dl,\n",
    "                                               n_epochs, train_df, val_df, \n",
    "                                               save_path=save_path, \n",
    "                                               prev_best_info=best_thr_score, info_history=thr_score_hist,\n",
    "                                               ep_start=ep_start)\n",
    "loss_hist.append(losses)\n",
    "ep_start += n_epochs"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Split no translation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from imports import *\n",
    "from clip_.train_text import *\n",
    "from clip_.data import TextDS\n",
    "from utils import load_data\n",
    "from arcface import ArcMarginProduct\n",
    "import matplotlib.pyplot as plt\n",
    "import clip\n",
    "np.random.seed(1337)\n",
    "device = torch.device('cuda')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "df, train_df, val_df, train_labels, val_labels = load_data(train_perc=0.3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "bs = 32\n",
    "bs_val = 128\n",
    "max_length=300"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "tr_ds = TextDS(train_df, column='title')\n",
    "val_ds = TextDS(val_df, column='title')\n",
    "full_ds = TextDS(df, column='title')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "torch.save(tr_ds, 'data/text_models/datasets/clip/tr_ds_0.3_{}_no_trans.pth'.format(max_length))\n",
    "torch.save(val_ds, 'data/text_models/datasets/clip/val_ds_0.3_{}_no_trans.pth'.format(max_length))\n",
    "torch.save(full_ds, 'data/text_models/datasets/clip/full_ds_0.3_{}_no_trans.pth'.format(max_length))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "tr_ds = torch.load('data/text_models/datasets/clip/tr_ds_0.3_{}_no_trans.pth'.format(max_length))\n",
    "val_ds = torch.load('data/text_models/datasets/clip/val_ds_0.3_{}_no_trans.pth'.format(max_length))\n",
    "full_ds = torch.load('data/text_models/datasets/clip/full_ds_0.3_{}_no_trans.pth'.format(max_length))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "tr_test_dl = DataLoader(tr_ds, batch_size = bs_val, shuffle = False, pin_memory = True)\n",
    "tr_dl = DataLoader(tr_ds, batch_size = bs, shuffle = True, pin_memory = True)\n",
    "val_dl = DataLoader(val_ds, batch_size = bs_val, shuffle = False, pin_memory = True)\n",
    "#full_dl = DataLoader(full_ds, batch_size = bs_val, shuffle = False, pin_memory = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "class CLIPTxt(torch.nn.Module) :\n",
    "    def __init__(self, model_name=\"RN50\", device='cuda') :\n",
    "        super().__init__()\n",
    "        self.model = clip.load(model_name, device=device, jit=False)[0]\n",
    "    def forward(self, imgs) :\n",
    "        return self.model.encode_text(imgs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "# model\n",
    "model = CLIPTxt().to(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f67a013721144f0aa2345b505778e0a7",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=82.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "centers = compute_centers(tr_test_dl, model, train_df)\n",
    "torch.save(centers, 'data/text_models/clip/centers_0.3_{}_no_trans.pth'.format(max_length))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "centers = torch.load('data/text_models/clip/centers_0.3_{}_no_trans.pth'.format(max_length))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Using center as wieghts\n"
     ]
    }
   ],
   "source": [
    "metric_fc = ArcMarginProduct(512, train_df['label_group'].nunique(), s=30, m=0.5,\n",
    "                             easy_margin=False, centers=centers, device=device).to(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "n_epochs, lf, params, optimizer, sched = get_hparams(tr_dl, model, metric_fc, lr=5e-4, n_epochs=10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "model.positional_embedding\n",
      "model.text_projection\n",
      "model.logit_scale\n",
      "model.visual.conv1.weight\n",
      "model.visual.bn1.weight\n",
      "model.visual.bn1.bias\n",
      "model.visual.conv2.weight\n",
      "model.visual.bn2.weight\n",
      "model.visual.bn2.bias\n",
      "model.visual.conv3.weight\n",
      "model.visual.bn3.weight\n",
      "model.visual.bn3.bias\n",
      "model.visual.layer1.0.conv1.weight\n",
      "model.visual.layer1.0.bn1.weight\n",
      "model.visual.layer1.0.bn1.bias\n",
      "model.visual.layer1.0.conv2.weight\n",
      "model.visual.layer1.0.bn2.weight\n",
      "model.visual.layer1.0.bn2.bias\n",
      "model.visual.layer1.0.conv3.weight\n",
      "model.visual.layer1.0.bn3.weight\n",
      "model.visual.layer1.0.bn3.bias\n",
      "model.visual.layer1.0.downsample.0.weight\n",
      "model.visual.layer1.0.downsample.1.weight\n",
      "model.visual.layer1.0.downsample.1.bias\n",
      "model.visual.layer1.1.conv1.weight\n",
      "model.visual.layer1.1.bn1.weight\n",
      "model.visual.layer1.1.bn1.bias\n",
      "model.visual.layer1.1.conv2.weight\n",
      "model.visual.layer1.1.bn2.weight\n",
      "model.visual.layer1.1.bn2.bias\n",
      "model.visual.layer1.1.conv3.weight\n",
      "model.visual.layer1.1.bn3.weight\n",
      "model.visual.layer1.1.bn3.bias\n",
      "model.visual.layer1.2.conv1.weight\n",
      "model.visual.layer1.2.bn1.weight\n",
      "model.visual.layer1.2.bn1.bias\n",
      "model.visual.layer1.2.conv2.weight\n",
      "model.visual.layer1.2.bn2.weight\n",
      "model.visual.layer1.2.bn2.bias\n",
      "model.visual.layer1.2.conv3.weight\n",
      "model.visual.layer1.2.bn3.weight\n",
      "model.visual.layer1.2.bn3.bias\n",
      "model.visual.layer2.0.conv1.weight\n",
      "model.visual.layer2.0.bn1.weight\n",
      "model.visual.layer2.0.bn1.bias\n",
      "model.visual.layer2.0.conv2.weight\n",
      "model.visual.layer2.0.bn2.weight\n",
      "model.visual.layer2.0.bn2.bias\n",
      "model.visual.layer2.0.conv3.weight\n",
      "model.visual.layer2.0.bn3.weight\n",
      "model.visual.layer2.0.bn3.bias\n",
      "model.visual.layer2.0.downsample.0.weight\n",
      "model.visual.layer2.0.downsample.1.weight\n",
      "model.visual.layer2.0.downsample.1.bias\n",
      "model.visual.layer2.1.conv1.weight\n",
      "model.visual.layer2.1.bn1.weight\n",
      "model.visual.layer2.1.bn1.bias\n",
      "model.visual.layer2.1.conv2.weight\n",
      "model.visual.layer2.1.bn2.weight\n",
      "model.visual.layer2.1.bn2.bias\n",
      "model.visual.layer2.1.conv3.weight\n",
      "model.visual.layer2.1.bn3.weight\n",
      "model.visual.layer2.1.bn3.bias\n",
      "model.visual.layer2.2.conv1.weight\n",
      "model.visual.layer2.2.bn1.weight\n",
      "model.visual.layer2.2.bn1.bias\n",
      "model.visual.layer2.2.conv2.weight\n",
      "model.visual.layer2.2.bn2.weight\n",
      "model.visual.layer2.2.bn2.bias\n",
      "model.visual.layer2.2.conv3.weight\n",
      "model.visual.layer2.2.bn3.weight\n",
      "model.visual.layer2.2.bn3.bias\n",
      "model.visual.layer2.3.conv1.weight\n",
      "model.visual.layer2.3.bn1.weight\n",
      "model.visual.layer2.3.bn1.bias\n",
      "model.visual.layer2.3.conv2.weight\n",
      "model.visual.layer2.3.bn2.weight\n",
      "model.visual.layer2.3.bn2.bias\n",
      "model.visual.layer2.3.conv3.weight\n",
      "model.visual.layer2.3.bn3.weight\n",
      "model.visual.layer2.3.bn3.bias\n",
      "model.visual.layer3.0.conv1.weight\n",
      "model.visual.layer3.0.bn1.weight\n",
      "model.visual.layer3.0.bn1.bias\n",
      "model.visual.layer3.0.conv2.weight\n",
      "model.visual.layer3.0.bn2.weight\n",
      "model.visual.layer3.0.bn2.bias\n",
      "model.visual.layer3.0.conv3.weight\n",
      "model.visual.layer3.0.bn3.weight\n",
      "model.visual.layer3.0.bn3.bias\n",
      "model.visual.layer3.0.downsample.0.weight\n",
      "model.visual.layer3.0.downsample.1.weight\n",
      "model.visual.layer3.0.downsample.1.bias\n",
      "model.visual.layer3.1.conv1.weight\n",
      "model.visual.layer3.1.bn1.weight\n",
      "model.visual.layer3.1.bn1.bias\n",
      "model.visual.layer3.1.conv2.weight\n",
      "model.visual.layer3.1.bn2.weight\n",
      "model.visual.layer3.1.bn2.bias\n",
      "model.visual.layer3.1.conv3.weight\n",
      "model.visual.layer3.1.bn3.weight\n",
      "model.visual.layer3.1.bn3.bias\n",
      "model.visual.layer3.2.conv1.weight\n",
      "model.visual.layer3.2.bn1.weight\n",
      "model.visual.layer3.2.bn1.bias\n",
      "model.visual.layer3.2.conv2.weight\n",
      "model.visual.layer3.2.bn2.weight\n",
      "model.visual.layer3.2.bn2.bias\n",
      "model.visual.layer3.2.conv3.weight\n",
      "model.visual.layer3.2.bn3.weight\n",
      "model.visual.layer3.2.bn3.bias\n",
      "model.visual.layer3.3.conv1.weight\n",
      "model.visual.layer3.3.bn1.weight\n",
      "model.visual.layer3.3.bn1.bias\n",
      "model.visual.layer3.3.conv2.weight\n",
      "model.visual.layer3.3.bn2.weight\n",
      "model.visual.layer3.3.bn2.bias\n",
      "model.visual.layer3.3.conv3.weight\n",
      "model.visual.layer3.3.bn3.weight\n",
      "model.visual.layer3.3.bn3.bias\n",
      "model.visual.layer3.4.conv1.weight\n",
      "model.visual.layer3.4.bn1.weight\n",
      "model.visual.layer3.4.bn1.bias\n",
      "model.visual.layer3.4.conv2.weight\n",
      "model.visual.layer3.4.bn2.weight\n",
      "model.visual.layer3.4.bn2.bias\n",
      "model.visual.layer3.4.conv3.weight\n",
      "model.visual.layer3.4.bn3.weight\n",
      "model.visual.layer3.4.bn3.bias\n",
      "model.visual.layer3.5.conv1.weight\n",
      "model.visual.layer3.5.bn1.weight\n",
      "model.visual.layer3.5.bn1.bias\n",
      "model.visual.layer3.5.conv2.weight\n",
      "model.visual.layer3.5.bn2.weight\n",
      "model.visual.layer3.5.bn2.bias\n",
      "model.visual.layer3.5.conv3.weight\n",
      "model.visual.layer3.5.bn3.weight\n",
      "model.visual.layer3.5.bn3.bias\n",
      "model.visual.layer4.0.conv1.weight\n",
      "model.visual.layer4.0.bn1.weight\n",
      "model.visual.layer4.0.bn1.bias\n",
      "model.visual.layer4.0.conv2.weight\n",
      "model.visual.layer4.0.bn2.weight\n",
      "model.visual.layer4.0.bn2.bias\n",
      "model.visual.layer4.0.conv3.weight\n",
      "model.visual.layer4.0.bn3.weight\n",
      "model.visual.layer4.0.bn3.bias\n",
      "model.visual.layer4.0.downsample.0.weight\n",
      "model.visual.layer4.0.downsample.1.weight\n",
      "model.visual.layer4.0.downsample.1.bias\n",
      "model.visual.layer4.1.conv1.weight\n",
      "model.visual.layer4.1.bn1.weight\n",
      "model.visual.layer4.1.bn1.bias\n",
      "model.visual.layer4.1.conv2.weight\n",
      "model.visual.layer4.1.bn2.weight\n",
      "model.visual.layer4.1.bn2.bias\n",
      "model.visual.layer4.1.conv3.weight\n",
      "model.visual.layer4.1.bn3.weight\n",
      "model.visual.layer4.1.bn3.bias\n",
      "model.visual.layer4.2.conv1.weight\n",
      "model.visual.layer4.2.bn1.weight\n",
      "model.visual.layer4.2.bn1.bias\n",
      "model.visual.layer4.2.conv2.weight\n",
      "model.visual.layer4.2.bn2.weight\n",
      "model.visual.layer4.2.bn2.bias\n",
      "model.visual.layer4.2.conv3.weight\n",
      "model.visual.layer4.2.bn3.weight\n",
      "model.visual.layer4.2.bn3.bias\n",
      "model.visual.attnpool.positional_embedding\n",
      "model.visual.attnpool.k_proj.weight\n",
      "model.visual.attnpool.k_proj.bias\n",
      "model.visual.attnpool.q_proj.weight\n",
      "model.visual.attnpool.q_proj.bias\n",
      "model.visual.attnpool.v_proj.weight\n",
      "model.visual.attnpool.v_proj.bias\n",
      "model.visual.attnpool.c_proj.weight\n",
      "model.visual.attnpool.c_proj.bias\n",
      "model.transformer.resblocks.0.attn.in_proj_weight\n",
      "model.transformer.resblocks.0.attn.in_proj_bias\n",
      "model.transformer.resblocks.0.attn.out_proj.weight\n",
      "model.transformer.resblocks.0.attn.out_proj.bias\n",
      "model.transformer.resblocks.0.ln_1.weight\n",
      "model.transformer.resblocks.0.ln_1.bias\n",
      "model.transformer.resblocks.0.mlp.c_fc.weight\n",
      "model.transformer.resblocks.0.mlp.c_fc.bias\n",
      "model.transformer.resblocks.0.mlp.c_proj.weight\n",
      "model.transformer.resblocks.0.mlp.c_proj.bias\n",
      "model.transformer.resblocks.0.ln_2.weight\n",
      "model.transformer.resblocks.0.ln_2.bias\n",
      "model.transformer.resblocks.1.attn.in_proj_weight\n",
      "model.transformer.resblocks.1.attn.in_proj_bias\n",
      "model.transformer.resblocks.1.attn.out_proj.weight\n",
      "model.transformer.resblocks.1.attn.out_proj.bias\n",
      "model.transformer.resblocks.1.ln_1.weight\n",
      "model.transformer.resblocks.1.ln_1.bias\n",
      "model.transformer.resblocks.1.mlp.c_fc.weight\n",
      "model.transformer.resblocks.1.mlp.c_fc.bias\n",
      "model.transformer.resblocks.1.mlp.c_proj.weight\n",
      "model.transformer.resblocks.1.mlp.c_proj.bias\n",
      "model.transformer.resblocks.1.ln_2.weight\n",
      "model.transformer.resblocks.1.ln_2.bias\n",
      "model.transformer.resblocks.2.attn.in_proj_weight\n",
      "model.transformer.resblocks.2.attn.in_proj_bias\n",
      "model.transformer.resblocks.2.attn.out_proj.weight\n",
      "model.transformer.resblocks.2.attn.out_proj.bias\n",
      "model.transformer.resblocks.2.ln_1.weight\n",
      "model.transformer.resblocks.2.ln_1.bias\n",
      "model.transformer.resblocks.2.mlp.c_fc.weight\n",
      "model.transformer.resblocks.2.mlp.c_fc.bias\n",
      "model.transformer.resblocks.2.mlp.c_proj.weight\n",
      "model.transformer.resblocks.2.mlp.c_proj.bias\n",
      "model.transformer.resblocks.2.ln_2.weight\n",
      "model.transformer.resblocks.2.ln_2.bias\n",
      "model.transformer.resblocks.3.attn.in_proj_weight\n",
      "model.transformer.resblocks.3.attn.in_proj_bias\n",
      "model.transformer.resblocks.3.attn.out_proj.weight\n",
      "model.transformer.resblocks.3.attn.out_proj.bias\n",
      "model.transformer.resblocks.3.ln_1.weight\n",
      "model.transformer.resblocks.3.ln_1.bias\n",
      "model.transformer.resblocks.3.mlp.c_fc.weight\n",
      "model.transformer.resblocks.3.mlp.c_fc.bias\n",
      "model.transformer.resblocks.3.mlp.c_proj.weight\n",
      "model.transformer.resblocks.3.mlp.c_proj.bias\n",
      "model.transformer.resblocks.3.ln_2.weight\n",
      "model.transformer.resblocks.3.ln_2.bias\n",
      "model.transformer.resblocks.4.attn.in_proj_weight\n",
      "model.transformer.resblocks.4.attn.in_proj_bias\n",
      "model.transformer.resblocks.4.attn.out_proj.weight\n",
      "model.transformer.resblocks.4.attn.out_proj.bias\n",
      "model.transformer.resblocks.4.ln_1.weight\n",
      "model.transformer.resblocks.4.ln_1.bias\n",
      "model.transformer.resblocks.4.mlp.c_fc.weight\n",
      "model.transformer.resblocks.4.mlp.c_fc.bias\n",
      "model.transformer.resblocks.4.mlp.c_proj.weight\n",
      "model.transformer.resblocks.4.mlp.c_proj.bias\n",
      "model.transformer.resblocks.4.ln_2.weight\n",
      "model.transformer.resblocks.4.ln_2.bias\n",
      "model.transformer.resblocks.5.attn.in_proj_weight\n",
      "model.transformer.resblocks.5.attn.in_proj_bias\n",
      "model.transformer.resblocks.5.attn.out_proj.weight\n",
      "model.transformer.resblocks.5.attn.out_proj.bias\n",
      "model.transformer.resblocks.5.ln_1.weight\n",
      "model.transformer.resblocks.5.ln_1.bias\n",
      "model.transformer.resblocks.5.mlp.c_fc.weight\n",
      "model.transformer.resblocks.5.mlp.c_fc.bias\n",
      "model.transformer.resblocks.5.mlp.c_proj.weight\n",
      "model.transformer.resblocks.5.mlp.c_proj.bias\n",
      "model.transformer.resblocks.5.ln_2.weight\n",
      "model.transformer.resblocks.5.ln_2.bias\n",
      "model.transformer.resblocks.6.attn.in_proj_weight\n",
      "model.transformer.resblocks.6.attn.in_proj_bias\n",
      "model.transformer.resblocks.6.attn.out_proj.weight\n",
      "model.transformer.resblocks.6.attn.out_proj.bias\n",
      "model.transformer.resblocks.6.ln_1.weight\n",
      "model.transformer.resblocks.6.ln_1.bias\n",
      "model.transformer.resblocks.6.mlp.c_fc.weight\n",
      "model.transformer.resblocks.6.mlp.c_fc.bias\n",
      "model.transformer.resblocks.6.mlp.c_proj.weight\n",
      "model.transformer.resblocks.6.mlp.c_proj.bias\n",
      "model.transformer.resblocks.6.ln_2.weight\n",
      "model.transformer.resblocks.6.ln_2.bias\n",
      "model.transformer.resblocks.7.attn.in_proj_weight\n",
      "model.transformer.resblocks.7.attn.in_proj_bias\n",
      "model.transformer.resblocks.7.attn.out_proj.weight\n",
      "model.transformer.resblocks.7.attn.out_proj.bias\n",
      "model.transformer.resblocks.7.ln_1.weight\n",
      "model.transformer.resblocks.7.ln_1.bias\n",
      "model.transformer.resblocks.7.mlp.c_fc.weight\n",
      "model.transformer.resblocks.7.mlp.c_fc.bias\n",
      "model.transformer.resblocks.7.mlp.c_proj.weight\n",
      "model.transformer.resblocks.7.mlp.c_proj.bias\n",
      "model.transformer.resblocks.7.ln_2.weight\n",
      "model.transformer.resblocks.7.ln_2.bias\n",
      "model.transformer.resblocks.8.attn.in_proj_weight\n",
      "model.transformer.resblocks.8.attn.in_proj_bias\n",
      "model.transformer.resblocks.8.attn.out_proj.weight\n",
      "model.transformer.resblocks.8.attn.out_proj.bias\n",
      "model.transformer.resblocks.8.ln_1.weight\n",
      "model.transformer.resblocks.8.ln_1.bias\n",
      "model.transformer.resblocks.8.mlp.c_fc.weight\n",
      "model.transformer.resblocks.8.mlp.c_fc.bias\n",
      "model.transformer.resblocks.8.mlp.c_proj.weight\n",
      "model.transformer.resblocks.8.mlp.c_proj.bias\n",
      "model.transformer.resblocks.8.ln_2.weight\n",
      "model.transformer.resblocks.8.ln_2.bias\n",
      "model.transformer.resblocks.9.attn.in_proj_weight\n",
      "model.transformer.resblocks.9.attn.in_proj_bias\n",
      "model.transformer.resblocks.9.attn.out_proj.weight\n",
      "model.transformer.resblocks.9.attn.out_proj.bias\n",
      "model.transformer.resblocks.9.ln_1.weight\n",
      "model.transformer.resblocks.9.ln_1.bias\n",
      "model.transformer.resblocks.9.mlp.c_fc.weight\n",
      "model.transformer.resblocks.9.mlp.c_fc.bias\n",
      "model.transformer.resblocks.9.mlp.c_proj.weight\n",
      "model.transformer.resblocks.9.mlp.c_proj.bias\n",
      "model.transformer.resblocks.9.ln_2.weight\n",
      "model.transformer.resblocks.9.ln_2.bias\n",
      "model.transformer.resblocks.10.attn.in_proj_weight\n",
      "model.transformer.resblocks.10.attn.in_proj_bias\n",
      "model.transformer.resblocks.10.attn.out_proj.weight\n",
      "model.transformer.resblocks.10.attn.out_proj.bias\n",
      "model.transformer.resblocks.10.ln_1.weight\n",
      "model.transformer.resblocks.10.ln_1.bias\n",
      "model.transformer.resblocks.10.mlp.c_fc.weight\n",
      "model.transformer.resblocks.10.mlp.c_fc.bias\n",
      "model.transformer.resblocks.10.mlp.c_proj.weight\n",
      "model.transformer.resblocks.10.mlp.c_proj.bias\n",
      "model.transformer.resblocks.10.ln_2.weight\n",
      "model.transformer.resblocks.10.ln_2.bias\n",
      "model.transformer.resblocks.11.attn.in_proj_weight\n",
      "model.transformer.resblocks.11.attn.in_proj_bias\n",
      "model.transformer.resblocks.11.attn.out_proj.weight\n",
      "model.transformer.resblocks.11.attn.out_proj.bias\n",
      "model.transformer.resblocks.11.ln_1.weight\n",
      "model.transformer.resblocks.11.ln_1.bias\n",
      "model.transformer.resblocks.11.mlp.c_fc.weight\n",
      "model.transformer.resblocks.11.mlp.c_fc.bias\n",
      "model.transformer.resblocks.11.mlp.c_proj.weight\n",
      "model.transformer.resblocks.11.mlp.c_proj.bias\n",
      "model.transformer.resblocks.11.ln_2.weight\n",
      "model.transformer.resblocks.11.ln_2.bias\n",
      "model.token_embedding.weight\n",
      "model.ln_final.weight\n",
      "model.ln_final.bias\n"
     ]
    }
   ],
   "source": [
    "for n, p in model.named_parameters() :\n",
    "    print(n)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "transformer.resblocks.0\n",
      "transformer.resblocks.1\n",
      "transformer.resblocks.2\n",
      "transformer.resblocks.3\n",
      "transformer.resblocks.4\n",
      "transformer.resblocks.5\n",
      "transformer.resblocks.6\n",
      "transformer.resblocks.7\n",
      "transformer.resblocks.8\n",
      "transformer.resblocks.9\n",
      "transformer.resblocks.10\n"
     ]
    }
   ],
   "source": [
    "mnp = list(model.named_parameters())\n",
    "\n",
    "param_groups = [{'params' : [p for n,p in mnp if n in ['model.token_embedding.weight', 'model.positional_embedding']]}]\n",
    "\n",
    "params_names = [n for n,p in mnp if n in ['model.token_embedding.weight', 'model.positional_embedding']]\n",
    "#print(params_names)\n",
    "n_blocks = 11\n",
    "for i in range(n_blocks):\n",
    "    print(f'transformer.resblocks.{i}')\n",
    "    ith_block = [p for n, p in mnp if f'transformer.resblocks.{i}.' in n]\n",
    "    ith_block_name = [n for n, p in mnp if f'transformer.resblocks.{i}.' in n]\n",
    "    params_names += ith_block_name\n",
    "    #for n in params_names :\n",
    "    #    print(n)\n",
    "    param_groups.append({'params' : ith_block})\n",
    "\n",
    "end = ['logit_scale', 'text_projection', 'final']\n",
    "param_groups.append({'params' : [p for n,p in mnp if any(end_n in n for end_n in end)]})\n",
    "params_names += [n for n,p in mnp if any(end_n in n for end_n in end)]\n",
    "\n",
    "param_groups.append({'params' : metric_fc.parameters()})\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "lf = nn.CrossEntropyLoss()\n",
    "optimizer = torch.optim.AdamW(param_groups, weight_decay = 1e-5)\n",
    "\n",
    "lr = list(np.linspace(5e-6,5e-4,len(param_groups)))\n",
    "\n",
    "n_epochs = 10\n",
    "sched = torch.optim.lr_scheduler.OneCycleLR(optimizer, max_lr=lr, pct_start=0.3,\n",
    "                                            total_steps=int(n_epochs * len(tr_dl)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "loss_hist = []\n",
    "best_thr_score={'val': {'thr': None, 'f1': None}, 'train': {'thr': None, 'f1': None}}\n",
    "thr_score_hist=[]\n",
    "ep_start = 0\n",
    "save_path = 'data/text_models/clip/test_21ap_epochs'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=10.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=327.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Thresholds', max=10.0, style=ProgressStyle(description_wi…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Checkpoint : saved model to data/text_models/clip/test_21ap_epochs_ep_0.pth\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=187.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Thresholds', max=10.0, style=ProgressStyle(description_wi…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Saved best model ep 0 with f score : 0.6468451045094107\n",
      "Ep 0: Train loss 6.2216 | Val f score 0.6468 with thresh 0.56, train f score 0.6907 with thresh 0.64\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=327.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Thresholds', max=5.0, style=ProgressStyle(description_wid…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=187.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Thresholds', max=5.0, style=ProgressStyle(description_wid…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Saved best model ep 1 with f score : 0.6520438288120098\n",
      "Ep 1: Train loss 6.2863 | Val f score 0.6520 with thresh 0.56, train f score 0.7341 with thresh 0.74\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=327.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Thresholds', max=5.0, style=ProgressStyle(description_wid…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Checkpoint : saved model to data/text_models/clip/test_21ap_epochs_ep_2.pth\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=187.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Thresholds', max=5.0, style=ProgressStyle(description_wid…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Saved best model ep 2 with f score : 0.6709357070379441\n",
      "Ep 2: Train loss 5.7792 | Val f score 0.6709 with thresh 0.66, train f score 0.7908 with thresh 0.84\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=327.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Thresholds', max=5.0, style=ProgressStyle(description_wid…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=187.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Thresholds', max=5.0, style=ProgressStyle(description_wid…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Saved best model ep 3 with f score : 0.6795619122663429\n",
      "Ep 3: Train loss 4.0533 | Val f score 0.6796 with thresh 0.71, train f score 0.8647 with thresh 0.89\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=327.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Thresholds', max=5.0, style=ProgressStyle(description_wid…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Checkpoint : saved model to data/text_models/clip/test_21ap_epochs_ep_4.pth\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=187.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Thresholds', max=5.0, style=ProgressStyle(description_wid…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Saved best model ep 4 with f score : 0.6832157171917493\n",
      "Ep 4: Train loss 2.1878 | Val f score 0.6832 with thresh 0.71, train f score 0.9210 with thresh 0.89\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=327.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Thresholds', max=5.0, style=ProgressStyle(description_wid…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=187.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Thresholds', max=5.0, style=ProgressStyle(description_wid…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Ep 5: Train loss 1.0056 | Val f score 0.6824 with thresh 0.76, train f score 0.9609 with thresh 0.89\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=327.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Thresholds', max=5.0, style=ProgressStyle(description_wid…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Checkpoint : saved model to data/text_models/clip/test_21ap_epochs_ep_6.pth\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=187.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Thresholds', max=5.0, style=ProgressStyle(description_wid…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Saved best model ep 6 with f score : 0.6846521698575314\n",
      "Ep 6: Train loss 0.4063 | Val f score 0.6847 with thresh 0.76, train f score 0.9816 with thresh 0.89\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=327.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Thresholds', max=5.0, style=ProgressStyle(description_wid…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=187.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Thresholds', max=5.0, style=ProgressStyle(description_wid…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Saved best model ep 7 with f score : 0.6874576656477703\n",
      "Ep 7: Train loss 0.1733 | Val f score 0.6875 with thresh 0.76, train f score 0.9894 with thresh 0.84\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=327.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Thresholds', max=5.0, style=ProgressStyle(description_wid…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Checkpoint : saved model to data/text_models/clip/test_21ap_epochs_ep_8.pth\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=187.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Thresholds', max=5.0, style=ProgressStyle(description_wid…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Ep 8: Train loss 0.1089 | Val f score 0.6869 with thresh 0.76, train f score 0.9919 with thresh 0.84\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=327.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Thresholds', max=5.0, style=ProgressStyle(description_wid…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=187.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Thresholds', max=5.0, style=ProgressStyle(description_wid…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Ep 9: Train loss 0.0887 | Val f score 0.6870 with thresh 0.76, train f score 0.9931 with thresh 0.79\n"
     ]
    }
   ],
   "source": [
    "best_thr_score, thr_score_hist, losses = train(model, optimizer, lf, sched, metric_fc, tr_dl, val_dl,\n",
    "                                               n_epochs, train_df, val_df, \n",
    "                                               save_path=save_path, \n",
    "                                               prev_best_info=best_thr_score, info_history=thr_score_hist,\n",
    "                                               ep_start=ep_start)\n",
    "loss_hist.append(losses)\n",
    "ep_start += n_epochs"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Multimodal training "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Testing zero shot "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Translated titles"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from imports import *\n",
    "from clip_.train_text import *\n",
    "from clip_.data import MultiDS\n",
    "from utils import load_data\n",
    "from arcface import ArcMarginProduct\n",
    "import matplotlib.pyplot as plt\n",
    "import clip\n",
    "np.random.seed(1337)\n",
    "device = torch.device('cuda')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "df, train_df, val_df, train_labels, val_labels = load_data(train_perc=0.3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "bs = 32\n",
    "bs_val = 128\n",
    "max_length=200"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "small_images_dir_train = 'data/small_train_images_250/'\n",
    "small_images_dir_val = 'data/small_train_images_224/'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "tr_ds = MultiDS(train_df, small_images_dir_train)\n",
    "val_ds = MultiDS(val_df, small_images_dir_val)\n",
    "full_ds = MultiDS(df, small_images_dir_val)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "torch.save(tr_ds, 'data/clip/multimodal/datasets/tr_ds_0.3.pth')\n",
    "torch.save(val_ds, 'data/clip/multimodal/datasets/val_ds_0.3.pth')\n",
    "torch.save(full_ds, 'data/clip/multimodal/datasets/full_ds_0.3.pth')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "tr_ds = torch.load('data/clip/multimodal/datasets/tr_ds_0.3.pth')\n",
    "val_ds = torch.load('data/clip/multimodal/datasets/val_ds_0.3.pth')\n",
    "full_ds = torch.load('data/clip/multimodal/datasets/full_ds_0.3.pth')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "tr_test_dl = DataLoader(tr_ds, batch_size = bs_val, shuffle = False, pin_memory = True)\n",
    "tr_dl = DataLoader(tr_ds, batch_size = bs, shuffle = True, pin_memory = True)\n",
    "val_dl = DataLoader(val_ds, batch_size = bs_val, shuffle = False, pin_memory = True)\n",
    "full_dl = DataLoader(full_ds, batch_size = bs_val, shuffle = False, pin_memory = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = clip.load('ViT-B/32', device=device, jit=False)[0]\n",
    "train_tfms, val_tfms = get_tfms(crop=224)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_path ='data/clip/multimodal/zero_shot/'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "model.eval()\n",
    "embs_imgs = []\n",
    "embs_txt = []\n",
    "for imgs, txts, _ in tqdm(full_dl, leave=False) :\n",
    "    with torch.no_grad():\n",
    "        imgs = val_tfms(imgs).to(device)\n",
    "        txts = txts.to(device)\n",
    "        embs_imgs.append(model.encode_image(imgs))\n",
    "        embs_txt.append(model.encode_text(txts))\n",
    "embs_imgs = F.normalize(torch.cat(embs_imgs, 0))\n",
    "embs_txt = F.normalize(torch.cat(embs_txt, 0))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "torch.save(embs_imgs, data_path + 'embs_imgs')\n",
    "torch.save(embs_txt, data_path + 'embs_txt')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "embs_imgs = torch.load(data_path + 'embs_imgs')\n",
    "embs_txt = torch.load(data_path + 'embs_txt')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#centers per label\n",
    "labels = torch.Tensor(full_ds.label_codes.values).to(device)\n",
    "print(labels.shape, labels.dtype)\n",
    "label_center_img = []\n",
    "label_center_txt = []\n",
    "for i in tqdm(range(int(labels.max().item())), leave=False) :\n",
    "    idxs = (labels == i).unsqueeze(-1)\n",
    "    embs_mean_img = (idxs*embs_imgs).sum(0) / idxs.sum().item()\n",
    "    embs_mean_txt = (idxs*embs_txt).sum(0) / idxs.sum().item()\n",
    "    label_center_img.append(embs_mean_img)\n",
    "    label_center_txt.append(embs_mean_txt)\n",
    "    #if i == 1 :\n",
    "    #    break\n",
    "label_center_img = torch.stack(label_center_img)\n",
    "label_center_txt = torch.stack(label_center_txt)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "torch.save(label_center_img, data_path + 'label_center_imgs')\n",
    "torch.save(label_center_txt, data_path + 'label_center_txt')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "label_center_img = torch.load(data_path + 'label_center_imgs')\n",
    "label_center_txt = torch.load(data_path + 'label_center_txt')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "labels = torch.Tensor(full_ds.label_codes.values).to(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_indexes = train_df.index.values\n",
    "val_indexes = val_df.index.values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Top 1 accuray : 0.18385401368141174\n",
      "Top 5 accuray : 0.3681167960166931\n",
      "Top 10 accuray : 0.4508613348007202\n",
      "Top 20 accuray : 0.5346569418907166\n",
      "Top 50 accuray : 0.6468613147735596\n"
     ]
    }
   ],
   "source": [
    "# image to text\n",
    "dists = embs_imgs @ label_center_txt.t()\n",
    "for top in [1, 5, 10, 20, 50] :\n",
    "    distances, indices = torch.topk(dists, top, dim=1, largest=True)\n",
    "    top_acc = (indices==labels.unsqueeze(-1)).sum() /len(labels)\n",
    "    print('Top {} accuray : {}'.format(top, top_acc))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Top 1 accuray : 0.18940146267414093\n",
      "Top 5 accuray : 0.38382482528686523\n",
      "Top 10 accuray : 0.4717080295085907\n",
      "Top 20 accuray : 0.5606131553649902\n",
      "Top 50 accuray : 0.6751241087913513\n"
     ]
    }
   ],
   "source": [
    "# text to image\n",
    "dists = embs_txt @ label_center_img.t()\n",
    "for top in [1, 5, 10, 20, 50] :\n",
    "    distances, indices = torch.topk(dists, top, dim=1, largest=True)\n",
    "    top_acc = (indices==labels.unsqueeze(-1)).sum() /len(labels)\n",
    "    print('Top {} accuray : {}'.format(top, top_acc))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Top 1 accuray : 0.8256350755691528\n",
      "Top 5 accuray : 0.9231241345405579\n",
      "Top 10 accuray : 0.9444087743759155\n",
      "Top 20 accuray : 0.9600583910942078\n",
      "Top 50 accuray : 0.9758248329162598\n"
     ]
    }
   ],
   "source": [
    "# image to image\n",
    "dists = embs_imgs @ label_center_img.t()\n",
    "for top in [1, 5, 10, 20, 50] :\n",
    "    distances, indices = torch.topk(dists, top, dim=1, largest=True)\n",
    "    top_acc = (indices==labels.unsqueeze(-1)).sum() /len(labels)\n",
    "    print('Top {} accuray : {}'.format(top, top_acc))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Top 1 accuray : 0.7429489493370056\n",
      "Top 5 accuray : 0.882861316204071\n",
      "Top 10 accuray : 0.9119124412536621\n",
      "Top 20 accuray : 0.9325547814369202\n",
      "Top 50 accuray : 0.952496349811554\n"
     ]
    }
   ],
   "source": [
    "# text to text\n",
    "dists = embs_txt @ label_center_txt.t()\n",
    "for top in [1, 5, 10, 20, 50] :\n",
    "    distances, indices = torch.topk(dists, top, dim=1, largest=True)\n",
    "    top_acc = (indices==labels.unsqueeze(-1)).sum() /len(labels)\n",
    "    print('Top {} accuray : {}'.format(top, top_acc))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### No translation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from imports import *\n",
    "from clip_.train_text import *\n",
    "from clip_.data import MultiDS\n",
    "from utils import load_data\n",
    "from arcface import ArcMarginProduct\n",
    "import matplotlib.pyplot as plt\n",
    "import clip\n",
    "np.random.seed(1337)\n",
    "device = torch.device('cuda')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "df, train_df, val_df, train_labels, val_labels = load_data(train_perc=0.3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "bs = 32\n",
    "bs_val = 128\n",
    "max_length=200"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "small_images_dir_train = 'data/small_train_images_250/'\n",
    "small_images_dir_val = 'data/small_train_images_224/'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "tr_ds = MultiDS(train_df, small_images_dir_train, column='title')\n",
    "val_ds = MultiDS(val_df, small_images_dir_val, column='title')\n",
    "full_ds = MultiDS(df, small_images_dir_val, column='title')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "torch.save(tr_ds, 'data/clip/multimodal/datasets/tr_ds_0.3_notrans.pth')\n",
    "torch.save(val_ds, 'data/clip/multimodal/datasets/val_ds_0.3_notrans.pth')\n",
    "torch.save(full_ds, 'data/clip/multimodal/datasets/full_ds_0.3_notrans.pth')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "tr_ds = torch.load('data/clip/multimodal/datasets/tr_ds_0.3_notrans.pth')\n",
    "val_ds = torch.load('data/clip/multimodal/datasets/val_ds_0.3_notrans.pth')\n",
    "full_ds = torch.load('data/clip/multimodal/datasets/full_ds_0.3_notrans.pth')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "tr_test_dl = DataLoader(tr_ds, batch_size = bs_val, shuffle = False, pin_memory = True)\n",
    "tr_dl = DataLoader(tr_ds, batch_size = bs, shuffle = True, pin_memory = True)\n",
    "val_dl = DataLoader(val_ds, batch_size = bs_val, shuffle = False, pin_memory = True)\n",
    "full_dl = DataLoader(full_ds, batch_size = bs_val, shuffle = False, pin_memory = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = clip.load('ViT-B/32', device=device, jit=False)[0]\n",
    "train_tfms, val_tfms = get_tfms(crop=224)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_path ='data/clip/multimodal/zero_shot/'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "model.eval()\n",
    "embs_imgs = []\n",
    "embs_txt = []\n",
    "for imgs, txts, _ in tqdm(full_dl, leave=False) :\n",
    "    with torch.no_grad():\n",
    "        imgs = val_tfms(imgs).to(device)\n",
    "        txts = txts.to(device)\n",
    "        embs_imgs.append(model.encode_image(imgs))\n",
    "        embs_txt.append(model.encode_text(txts))\n",
    "embs_imgs = F.normalize(torch.cat(embs_imgs, 0))\n",
    "embs_txt = F.normalize(torch.cat(embs_txt, 0))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "torch.save(embs_imgs, data_path + 'embs_imgs_notrans')\n",
    "torch.save(embs_txt, data_path + 'embs_txt_notrans')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "embs_imgs = torch.load(data_path + 'embs_imgs_notrans')\n",
    "embs_txt = torch.load(data_path + 'embs_txt_notrans')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "labels = df['label_group']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Computing label centers...\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=11014.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    }
   ],
   "source": [
    "unique_labels = labels.unique()\n",
    "labels = torch.Tensor(labels.values).to('cuda')\n",
    "print('Computing label centers...')\n",
    "label_center_img = []\n",
    "label_center_txt = []\n",
    "for label in tqdm(unique_labels, leave=False) :\n",
    "    idxs = (labels == label).unsqueeze(-1)\n",
    "    embs_mean_img = (idxs*embs_imgs).sum(0) / idxs.sum().item()\n",
    "    embs_mean_txt = (idxs*embs_txt).sum(0) / idxs.sum().item()\n",
    "    label_center_img.append(embs_mean_img)\n",
    "    label_center_txt.append(embs_mean_txt)\n",
    "        #if i == 1 :\n",
    "        #    break\n",
    "label_center_img = torch.stack(label_center_img)\n",
    "label_center_txt = torch.stack(label_center_txt)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ 249114794, 2937985045, 2395904891, ..., 1313560418,  763032672,\n",
       "         53836859])"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "unique_labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([34250]) torch.float32\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=11013.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    }
   ],
   "source": [
    "#centers per label\n",
    "labels = torch.Tensor(full_ds.label_codes.values).to(device)\n",
    "print(labels.shape, labels.dtype)\n",
    "label_center_img = []\n",
    "label_center_txt = []\n",
    "for i in tqdm(range(int(labels.max().item())), leave=False) :\n",
    "    idxs = (labels == i).unsqueeze(-1)\n",
    "    embs_mean_img = (idxs*embs_imgs).sum(0) / idxs.sum().item()\n",
    "    embs_mean_txt = (idxs*embs_txt).sum(0) / idxs.sum().item()\n",
    "    label_center_img.append(embs_mean_img)\n",
    "    label_center_txt.append(embs_mean_txt)\n",
    "    #if i == 1 :\n",
    "    #    break\n",
    "label_center_img = torch.stack(label_center_img)\n",
    "label_center_txt = torch.stack(label_center_txt)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "torch.save(label_center_img, data_path + 'label_center_imgs_notrans')\n",
    "torch.save(label_center_txt, data_path + 'label_center_txt_notrans')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "label_center_img = torch.load(data_path + 'label_center_imgs_notrans')\n",
    "label_center_txt = torch.load(data_path + 'label_center_txt_notrans')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Top 1 accuray : 0.1938978135585785\n",
      "Top 5 accuray : 0.38487592339515686\n",
      "Top 10 accuray : 0.4625401496887207\n",
      "Top 20 accuray : 0.5428029298782349\n",
      "Top 50 accuray : 0.6489050984382629\n"
     ]
    }
   ],
   "source": [
    "# image to text\n",
    "dists = embs_imgs @ label_center_txt.t()\n",
    "for top in [1, 5, 10, 20, 50] :\n",
    "    distances, indices = torch.topk(dists, top, dim=1, largest=True)\n",
    "    top_acc = (indices==labels.unsqueeze(-1)).sum() /len(labels)\n",
    "    print('Top {} accuray : {}'.format(top, top_acc))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Top 1 accuray : 0.17821897566318512\n",
      "Top 5 accuray : 0.3607591390609741\n",
      "Top 10 accuray : 0.44370803236961365\n",
      "Top 20 accuray : 0.5277372598648071\n",
      "Top 50 accuray : 0.6399416327476501\n"
     ]
    }
   ],
   "source": [
    "# text to image\n",
    "dists = embs_txt @ label_center_img.t()\n",
    "for top in [1, 5, 10, 20, 50] :\n",
    "    distances, indices = torch.topk(dists, top, dim=1, largest=True)\n",
    "    top_acc = (indices==labels.unsqueeze(-1)).sum() /len(labels)\n",
    "    print('Top {} accuray : {}'.format(top, top_acc))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Top 1 accuray : 0.8254890441894531\n",
      "Top 5 accuray : 0.9232116937637329\n",
      "Top 10 accuray : 0.9443503618240356\n",
      "Top 20 accuray : 0.9599708318710327\n",
      "Top 50 accuray : 0.9758832454681396\n"
     ]
    }
   ],
   "source": [
    "# image to image\n",
    "dists = embs_imgs @ label_center_img.t()\n",
    "for top in [1, 5, 10, 20, 50] :\n",
    "    distances, indices = torch.topk(dists, top, dim=1, largest=True)\n",
    "    top_acc = (indices==labels.unsqueeze(-1)).sum() /len(labels)\n",
    "    print('Top {} accuray : {}'.format(top, top_acc))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Top 1 accuray : 0.7583649754524231\n",
      "Top 5 accuray : 0.893868625164032\n",
      "Top 10 accuray : 0.9208175539970398\n",
      "Top 20 accuray : 0.941518247127533\n",
      "Top 50 accuray : 0.9617518186569214\n"
     ]
    }
   ],
   "source": [
    "# text to text\n",
    "dists = embs_txt @ label_center_txt.t()\n",
    "for top in [1, 5, 10, 20, 50] :\n",
    "    distances, indices = torch.topk(dists, top, dim=1, largest=True)\n",
    "    top_acc = (indices==labels.unsqueeze(-1)).sum() /len(labels)\n",
    "    print('Top {} accuray : {}'.format(top, top_acc))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Fine tuning clip"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Zero shot without fine tuning on split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from imports import *\n",
    "from clip_.train_text import *\n",
    "from clip_.data import MultiDS\n",
    "from utils import load_data\n",
    "from arcface import ArcMarginProduct\n",
    "import matplotlib.pyplot as plt\n",
    "import clip\n",
    "np.random.seed(1337)\n",
    "device = torch.device('cuda')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "df, train_df, val_df, train_labels, val_labels = load_data(train_perc=0.3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "bs = 32\n",
    "bs_val = 32\n",
    "max_length=200"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "small_images_dir_train = 'data/small_train_images_250/'\n",
    "small_images_dir_val = 'data/small_train_images_224/'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "tr_ds = MultiDS(train_df, small_images_dir_train, column='title')\n",
    "val_ds = MultiDS(val_df, small_images_dir_val, column='title')\n",
    "full_ds = MultiDS(df, small_images_dir_val, column='title')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "torch.save(tr_ds, 'data/clip/multimodal/datasets/tr_ds_0.3_notrans.pth')\n",
    "torch.save(val_ds, 'data/clip/multimodal/datasets/val_ds_0.3_notrans.pth')\n",
    "torch.save(full_ds, 'data/clip/multimodal/datasets/full_ds_0.3_notrans.pth')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "tr_ds = torch.load('data/clip/multimodal/datasets/tr_ds_0.3_notrans.pth')\n",
    "val_ds = torch.load('data/clip/multimodal/datasets/val_ds_0.3_notrans.pth')\n",
    "full_ds = torch.load('data/clip/multimodal/datasets/full_ds_0.3_notrans.pth')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "tr_test_dl = DataLoader(tr_ds, batch_size = bs_val, shuffle = False, pin_memory = True)\n",
    "tr_dl = DataLoader(tr_ds, batch_size = bs, shuffle = True, pin_memory = True)\n",
    "val_dl = DataLoader(val_ds, batch_size = bs_val, shuffle = False, pin_memory = True)\n",
    "full_dl = DataLoader(full_ds, batch_size = bs_val, shuffle = False, pin_memory = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = clip.load('ViT-B/32', device=device, jit=False)[0]\n",
    "train_tfms, val_tfms = get_tfms(crop=224)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "def compute_top_accs(embs_image, embs_texts, labels, tops=[1, 5, 10]) :\n",
    "    print('Computing label centers...')\n",
    "    label_center_img = []\n",
    "    label_center_txt = []\n",
    "    for label in tqdm(range(int(labels.max().item())), leave=False) :\n",
    "        idxs = (labels == label).unsqueeze(-1)\n",
    "        embs_mean_img = (idxs*embs_image).sum(0) / idxs.sum().item()\n",
    "        embs_mean_txt = (idxs*embs_texts).sum(0) / idxs.sum().item()\n",
    "        label_center_img.append(embs_mean_img)\n",
    "        label_center_txt.append(embs_mean_txt)\n",
    "        #if i == 1 :\n",
    "        #    break\n",
    "    label_center_img = torch.stack(label_center_img)\n",
    "    label_center_txt = torch.stack(label_center_txt)\n",
    "    \n",
    "    # image text\n",
    "    dists = embs_image @ label_center_txt.t()\n",
    "    for top in tops :\n",
    "        distances, indices = torch.topk(dists, top, dim=1, largest=True)\n",
    "        top_acc = (indices==labels.unsqueeze(-1)).sum() /len(labels)\n",
    "        print('Top {} accuray image-text : {}'.format(top, top_acc))\n",
    "    \n",
    "    # image text\n",
    "    dists = embs_texts @ label_center_img.t()\n",
    "    for top in [1, 5, 10] :\n",
    "        distances, indices = torch.topk(dists, top, dim=1, largest=True)\n",
    "        top_acc = (indices==labels.unsqueeze(-1)).sum() /len(labels)\n",
    "        print('Top {} accuray text-image : {}'.format(top, top_acc))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "model.eval()\n",
    "embs_imgs_val = []\n",
    "embs_txt_val = []\n",
    "ys_val = []\n",
    "for imgs, txts, labels in tqdm(val_dl, leave=False) :\n",
    "    with torch.no_grad():\n",
    "        ys_val.append(labels)\n",
    "        imgs = val_tfms(imgs).to(device)\n",
    "        txts = txts.to(device)\n",
    "        embs_imgs_val.append(model.encode_image(imgs))\n",
    "        embs_txt_val.append(model.encode_text(txts))\n",
    "embs_imgs_val = F.normalize(torch.cat(embs_imgs_val, 0))\n",
    "embs_txt_val = F.normalize(torch.cat(embs_txt_val, 0))\n",
    "ys_val = torch.cat(ys_val, 0).to(device)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "embs_imgs_val = F.normalize(torch.cat(embs_imgs_val, 0))\n",
    "embs_txt_val = F.normalize(torch.cat(embs_txt_val, 0))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "torch.save(embs_imgs_val, data_path + 'embs_imgs_val')\n",
    "torch.save(embs_txt_val, data_path + 'embs_txt_val')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.eval()\n",
    "embs_imgs_train = []\n",
    "embs_txt_train = []\n",
    "ys_val = []\n",
    "for imgs, txts, labels in tqdm(val_dl, leave=False) :\n",
    "    with torch.no_grad():\n",
    "        ys_val.append(labels)\n",
    "        imgs = val_tfms(imgs).to(device)\n",
    "        txts = txts.to(device)\n",
    "        embs_imgs_train.append(model.encode_image(imgs))\n",
    "        embs_txt_train.append(model.encode_text(txts))\n",
    "embs_imgs_train = F.normalize(torch.cat(embs_imgs, 0))\n",
    "embs_txt_train = F.normalize(torch.cat(embs_txt, 0))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_path ='data/clip/multimodal/zero_shot/'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "torch.save(embs_imgs_train, data_path + 'embs_imgs_train')\n",
    "torch.save(embs_txt_train, data_path + 'embs_txt_train')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Computing label centers...\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=7709.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Top 1 accuray image-text : 0.22469550371170044\n",
      "Top 5 accuray image-text : 0.4235195219516754\n",
      "Top 10 accuray image-text : 0.5037379264831543\n",
      "Top 1 accuray text-image : 0.20869381725788116\n",
      "Top 5 accuray text-image : 0.40470388531684875\n",
      "Top 10 accuray text-image : 0.4875682294368744\n"
     ]
    }
   ],
   "source": [
    "compute_top_accs(embs_imgs_val, embs_txt_val, ys_val)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Fine tuning split and monti"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from imports import *\n",
    "from clip_.train_text import *\n",
    "from clip_.data import MultiDS\n",
    "from utils import load_data\n",
    "from arcface import ArcMarginProduct\n",
    "import matplotlib.pyplot as plt\n",
    "import clip\n",
    "np.random.seed(1337)\n",
    "device = torch.device('cuda')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "df, train_df, val_df, train_labels, val_labels = load_data(train_perc=0.3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "bs = 32\n",
    "bs_val = 32\n",
    "max_length=200"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "small_images_dir_train = 'data/small_train_images_250/'\n",
    "small_images_dir_val = 'data/small_train_images_224/'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "tr_ds = MultiDS(train_df, small_images_dir_train, column='title')\n",
    "val_ds = MultiDS(val_df, small_images_dir_val, column='title')\n",
    "full_ds = MultiDS(df, small_images_dir_val, column='title')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "torch.save(tr_ds, 'data/clip/multimodal/datasets/tr_ds_0.3_notrans.pth')\n",
    "torch.save(val_ds, 'data/clip/multimodal/datasets/val_ds_0.3_notrans.pth')\n",
    "torch.save(full_ds, 'data/clip/multimodal/datasets/full_ds_0.3_notrans.pth')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "tr_ds = torch.load('data/clip/multimodal/datasets/tr_ds_0.3_notrans.pth')\n",
    "val_ds = torch.load('data/clip/multimodal/datasets/val_ds_0.3_notrans.pth')\n",
    "full_ds = torch.load('data/clip/multimodal/datasets/full_ds_0.3_notrans.pth')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "tr_test_dl = DataLoader(tr_ds, batch_size = bs_val, shuffle = False, pin_memory = True)\n",
    "tr_dl = DataLoader(tr_ds, batch_size = bs, shuffle = True, pin_memory = True)\n",
    "val_dl = DataLoader(val_ds, batch_size = bs_val, shuffle = False, pin_memory = True)\n",
    "full_dl = DataLoader(full_ds, batch_size = bs_val, shuffle = False, pin_memory = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = clip.load('ViT-B/32', device=device, jit=False)[0]\n",
    "train_tfms, val_tfms = get_tfms(crop=224)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_path ='data/clip/multimodal/zero_shot/'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "clip_loss = nn.CrossEntropyLoss()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "def compute_top_accs(embs_image, embs_texts, labels, tops=[1, 5, 10]) :\n",
    "    print('Computing label centers...')\n",
    "    label_center_img = []\n",
    "    label_center_txt = []\n",
    "    for label in tqdm(range(int(labels.max().item())), leave=False) :\n",
    "        idxs = (labels == label).unsqueeze(-1)\n",
    "        embs_mean_img = (idxs*embs_image).sum(0) / idxs.sum().item()\n",
    "        embs_mean_txt = (idxs*embs_texts).sum(0) / idxs.sum().item()\n",
    "        label_center_img.append(embs_mean_img)\n",
    "        label_center_txt.append(embs_mean_txt)\n",
    "        #if i == 1 :\n",
    "        #    break\n",
    "    label_center_img = torch.stack(label_center_img)\n",
    "    label_center_txt = torch.stack(label_center_txt)\n",
    "    \n",
    "    # image text\n",
    "    dists = embs_image @ label_center_txt.t()\n",
    "    for top in tops :\n",
    "        distances, indices = torch.topk(dists, top, dim=1, largest=True)\n",
    "        top_acc = (indices==labels.unsqueeze(-1)).sum() /len(labels)\n",
    "        print('Top {} accuray image-text : {}'.format(top, top_acc))\n",
    "    \n",
    "    # image text\n",
    "    dists = embs_texts @ label_center_img.t()\n",
    "    for top in [1, 5, 10] :\n",
    "        distances, indices = torch.topk(dists, top, dim=1, largest=True)\n",
    "        top_acc = (indices==labels.unsqueeze(-1)).sum() /len(labels)\n",
    "        print('Top {} accuray text-image : {}'.format(top, top_acc))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "def convert_models_to_fp32_(model):\n",
    "    for p in model.parameters():\n",
    "        p.data = p.data.float()\n",
    "\n",
    "def train_clip(model, optimizer, sched, train_dl, val_dl, n_epochs, train_df, val_df,\n",
    "          train_transforms, val_transforms, save_path, clip_loss = nn.CrossEntropyLoss()) :\n",
    "    tr_losses = []\n",
    "    val_losses = []\n",
    "    \n",
    "    convert_models_to_fp32_(model)\n",
    "    \n",
    "    for ep in tqdm(range(n_epochs), leave=False):\n",
    "        # TRAINING\n",
    "        model.train()\n",
    "        tr_loss = []\n",
    "        embs_imgs = []\n",
    "        embs_txt = []\n",
    "        ys = []\n",
    "        pbar = tqdm(train_dl, leave=False)\n",
    "        for imgs, txts, labels in pbar:\n",
    "            \n",
    "            ys.append(labels)\n",
    "            labels = labels.long().to(device)\n",
    "            clip_tgt = torch.arange(0, labels.size(0)).to(device)\n",
    "            \n",
    "            imgs = imgs.to('cuda')\n",
    "            txts = txts.to('cuda')\n",
    "\n",
    "            optimizer.zero_grad()\n",
    "            \n",
    "            image_features = model.encode_image(train_transforms(imgs))\n",
    "            text_features = model.encode_text(txts)\n",
    "            \n",
    "            logits_per_image, logits_per_text = model.forward_from_features(image_features, text_features)\n",
    "            loss_clip = clip_loss(logits_per_image, clip_tgt) + clip_loss(logits_per_text, clip_tgt)\n",
    "            \n",
    "            loss_clip.backward()\n",
    "            optimizer.step()\n",
    "            sched.step()\n",
    "    \n",
    "            tr_loss.append(loss_clip.item())\n",
    "            pbar.set_description(f\"Train loss: {round(np.mean(tr_loss),3)}\")\n",
    "            embs_imgs.append(image_features.detach().to('cpu'))\n",
    "            embs_txt.append(text_features.detach().to('cpu'))\n",
    "        ys = torch.cat(ys, 0).to(device)\n",
    "        embs_imgs = F.normalize(torch.cat(embs_imgs, 0).to(device))\n",
    "        embs_txt = F.normalize(torch.cat(embs_txt, 0).to(device))\n",
    "        compute_top_accs(embs_imgs, embs_txt, ys)\n",
    "\n",
    "        if ep % 2 == 0:\n",
    "            path =  save_path + '_ep_{}.pth'.format(ep)\n",
    "            print('Checkpoint : saved model to {}'.format(path))\n",
    "            torch.save(model.state_dict(\n",
    "            ),path)\n",
    "\n",
    "        # VALIDATION\n",
    "        model.eval()\n",
    "        ys = []\n",
    "        with torch.no_grad():\n",
    "            pbar = tqdm(val_dl, leave=False)\n",
    "            embs_imgs = []\n",
    "            embs_txt = []\n",
    "            val_loss = []\n",
    "            \n",
    "            for imgs, txts, labels in pbar:\n",
    "                ys.append(labels)\n",
    "                labels = labels.long().to(device)\n",
    "                clip_tgt = torch.arange(0, labels.size(0)).to(device)\n",
    "                \n",
    "                imgs = imgs.to('cuda')\n",
    "                txts = txts.to('cuda')\n",
    "\n",
    "                optimizer.zero_grad()\n",
    "\n",
    "                image_features = model.encode_image(val_transforms(imgs))\n",
    "                text_features = model.encode_text(txts)\n",
    "                \n",
    "                logits_per_image, logits_per_text = model.forward_from_features(image_features, text_features)\n",
    "                loss_clip = clip_loss(logits_per_image, clip_tgt) + clip_loss(logits_per_text, clip_tgt)\n",
    "                \n",
    "                val_loss.append(loss_clip.item())\n",
    "                pbar.set_description(f\"Val loss: {round(np.mean(val_loss),3)}\")\n",
    "                embs_imgs.append(image_features.detach().to('cpu'))\n",
    "                embs_txt.append(text_features.detach().to('cpu'))\n",
    "            ys = torch.cat(ys, 0).to(device)\n",
    "            embs_imgs = F.normalize(torch.cat(embs_imgs, 0).to(device))\n",
    "            embs_txt = F.normalize(torch.cat(embs_txt, 0).to(device))\n",
    "            compute_top_accs(embs_imgs, embs_txt, ys)\n",
    "\n",
    "        \n",
    "\n",
    "        tr_losses.append(tr_loss)\n",
    "        val_losses.append(val_loss)\n",
    "        summary = \"Ep {}: Train loss {:.4f} | Val loss {:.4f}\".format(\n",
    "            ep, np.asarray(tr_loss).mean(), np.asarray(val_loss).mean())\n",
    "        print(summary)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "for n, _ in model.named_parameters() :\n",
    "    print(n)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "mnp = list(model.named_parameters())\n",
    "\n",
    "start = ['visual.class_embedding', 'visual.positional_embedding', 'visual.conv1.weight',\n",
    "        'visual.ln_pre.weight', 'visual.ln_pre.bias', 'positional_embedding', 'token_embedding.weight']\n",
    "param_groups = [{'params' : [p for n,p in mnp if n in start]}]\n",
    "params_names = [n for n,p in mnp if n in start]\n",
    "\n",
    "n_blocks = 12\n",
    "for i in range(n_blocks):\n",
    "    ith_block = [p for n, p in mnp if f'resblocks.{i}.' in n]\n",
    "    ith_block_names = [n for n, p in mnp if f'resblocks.{i}.' in n] \n",
    "    param_groups.append({'params' : ith_block})\n",
    "    params_names += ith_block_names\n",
    "    \n",
    "end = ['visual.proj', 'visual.ln_post.weight', 'visual.ln_post.bias', 'ln_final.weight', 'ln_final.bias',\n",
    "      'text_projection', 'logit_scale']\n",
    "param_groups.append({'params' : [p for n,p in mnp if n in end]})\n",
    "params_names += [n for n,p in mnp if n in end]\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "for n, p in model.named_parameters() :\n",
    "    if n not in params_names :\n",
    "        print(n)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "lf = nn.CrossEntropyLoss()\n",
    "optimizer = torch.optim.AdamW(param_groups, weight_decay = 1e-5)\n",
    "\n",
    "lr = list(np.linspace(1e-6,1e-5,len(param_groups)))\n",
    "\n",
    "n_epochs = 2\n",
    "sched = torch.optim.lr_scheduler.OneCycleLR(optimizer, max_lr=lr, pct_start=0.3,\n",
    "                                            total_steps=int(n_epochs * len(tr_dl)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "save_path = 'data/clip/multimodal/tests_ft'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=2.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=327.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Computing label centers...\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=3303.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Top 1 accuray image-text : 0.24262452125549316\n",
      "Top 5 accuray image-text : 0.493486613035202\n",
      "Top 10 accuray image-text : 0.6063218712806702\n",
      "Top 1 accuray text-image : 0.2616858184337616\n",
      "Top 5 accuray text-image : 0.5310344696044922\n",
      "Top 10 accuray text-image : 0.6477011442184448\n",
      "Checkpoint : saved model to data/clip/multimodal/tests_ft_ep_0.pth\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=745.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Computing label centers...\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=7709.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Top 1 accuray image-text : 0.25674086809158325\n",
      "Top 5 accuray image-text : 0.4854682683944702\n",
      "Top 10 accuray image-text : 0.582822322845459\n",
      "Top 1 accuray text-image : 0.2503570020198822\n",
      "Top 5 accuray text-image : 0.4873162508010864\n",
      "Top 10 accuray text-image : 0.5856782793998718\n",
      "Ep 0: Train loss 1.5234 | Val loss 1.2039\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=327.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Computing label centers...\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=3303.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Top 1 accuray image-text : 0.3558429181575775\n",
      "Top 5 accuray image-text : 0.6887931227684021\n",
      "Top 10 accuray image-text : 0.8054597973823547\n",
      "Top 1 accuray text-image : 0.3854406177997589\n",
      "Top 5 accuray text-image : 0.7205938696861267\n",
      "Top 10 accuray text-image : 0.8299808502197266\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=745.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Computing label centers...\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=7709.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Top 1 accuray image-text : 0.25808483362197876\n",
      "Top 5 accuray image-text : 0.4959260821342468\n",
      "Top 10 accuray image-text : 0.5918521285057068\n",
      "Top 1 accuray text-image : 0.26161277294158936\n",
      "Top 5 accuray text-image : 0.502981960773468\n",
      "Top 10 accuray text-image : 0.6058798432350159\n",
      "Ep 1: Train loss 0.5667 | Val loss 1.1582\n",
      "\r"
     ]
    }
   ],
   "source": [
    "train_clip(model, optimizer, sched, tr_dl, val_dl, n_epochs, train_df, val_df, train_tfms, val_tfms, save_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": true,
   "toc_position": {
    "height": "calc(100% - 180px)",
    "left": "10px",
    "top": "150px",
    "width": "259.716px"
   },
   "toc_section_display": true,
   "toc_window_display": true
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}