cifar10/cifar10-super-convergence-more-aug.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## CIFAR 10"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "%reload_ext autoreload\n",
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import argparse\n",
    "import os\n",
    "import shutil\n",
    "import time\n",
    "\n",
    "from fastai.transforms import *\n",
    "from fastai.dataset import *\n",
    "from fastai.fp16 import *\n",
    "from fastai.conv_learner import *\n",
    "from pathlib import *\n",
    "\n",
    "import torch\n",
    "from torch.autograd import Variable\n",
    "import torch.nn as nn\n",
    "import torch.nn.parallel\n",
    "import torch.backends.cudnn as cudnn\n",
    "import torch.distributed as dist\n",
    "import torch.optim\n",
    "import torch.utils.data\n",
    "import torch.utils.data.distributed\n",
    "import torchvision.transforms as transforms\n",
    "import torchvision.datasets as datasets\n",
    "import models\n",
    "import models.cifar10 as cifar10models\n",
    "from distributed import DistributedDataParallel as DDP\n",
    "\n",
    "# print(models.cifar10.__dict__)\n",
    "model_names = sorted(name for name in models.__dict__\n",
    "                     if name.islower() and not name.startswith(\"__\")\n",
    "                     and callable(models.__dict__[name]))\n",
    "\n",
    "cifar10_names = sorted(name for name in cifar10models.__dict__\n",
    "                     if name.islower() and not name.startswith(\"__\")\n",
    "                     and callable(cifar10models.__dict__[name]))\n",
    "\n",
    "model_names = cifar10_names + model_names"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['resnet56', 'resnext29_16_64', 'resnext29_8_64', 'dpn107', 'dpn131', 'dpn68', 'dpn92', 'dpn98', 'inceptionresnetv2', 'inceptionresnetv2_conc', 'inceptionv4', 'load', 'load_block17', 'load_block35', 'load_block8', 'load_conv2d', 'load_conv2d_nobn', 'load_linear', 'load_mixed_4a_7a', 'load_mixed_5', 'load_mixed_5b', 'load_mixed_6', 'load_mixed_6a', 'load_mixed_7', 'load_mixed_7a', 'nasnetalarge', 'pre_resnet101', 'pre_resnet152', 'pre_resnet18', 'pre_resnet34', 'pre_resnet50', 'reduce', 'resnet101', 'resnet152', 'resnet18', 'resnet34', 'resnet50', 'resnext101', 'resnext152', 'resnext18', 'resnext34', 'resnext50', 'resnext_101_32x4d', 'resnext_101_64x4d', 'resnext_50_32x4d', 'se_resnet_101', 'se_resnet_152', 'se_resnet_18', 'se_resnet_34', 'se_resnet_50', 'se_resnet_50_conc', 'se_resnext_101', 'se_resnext_152', 'se_resnext_50', 'test', 'test_block17', 'test_block35', 'test_block8', 'test_conv2d', 'test_conv2d_nobn', 'test_mixed_4a_7a', 'test_mixed_5b', 'test_mixed_6a', 'test_mixed_7a', 'wrn_50_2f']\n"
     ]
    }
   ],
   "source": [
    "\n",
    "print(model_names)\n",
    "\n",
    "# Example usage: python run_fastai.py /home/paperspace/ILSVRC/Data/CLS-LOC/ -a resnext_50_32x4d --epochs 1 -j 4 -b 64 --fp16\n",
    "\n",
    "parser = argparse.ArgumentParser(description='PyTorch Cifar10 Training')\n",
    "parser.add_argument('data', metavar='DIR',\n",
    "                    help='path to dataset')\n",
    "parser.add_argument('--save-dir', type=str, default=Path.home()/'imagenet_training',\n",
    "                    help='Directory to save logs and models.')\n",
    "parser.add_argument('--arch', '-a', metavar='ARCH', default='resnet56',\n",
    "                    choices=model_names,\n",
    "                    help='model architecture: ' +\n",
    "                    ' | '.join(model_names) +\n",
    "                    ' (default: resnet56)')\n",
    "parser.add_argument('-j', '--workers', default=7, type=int, metavar='N',\n",
    "                    help='number of data loading workers (default: 4)')\n",
    "parser.add_argument('--epochs', default=1, type=int, metavar='N',\n",
    "                    help='number of total epochs to run')\n",
    "parser.add_argument('--cycle-len', default=95, type=float, metavar='N',\n",
    "                    help='Length of cycle to run')\n",
    "# parser.add_argument('--start-epoch', default=0, type=int, metavar='N',\n",
    "#                     help='manual epoch number (useful on restarts)')\n",
    "parser.add_argument('-b', '--batch-size', default=512, type=int,\n",
    "                    metavar='N', help='mini-batch size (default: 256)')\n",
    "parser.add_argument('--lr', '--learning-rate', default=0.8, type=float,\n",
    "                    metavar='LR', help='initial learning rate')\n",
    "parser.add_argument('--momentum', default=0.9, type=float, metavar='M', help='momentum')\n",
    "parser.add_argument('--weight-decay', '--wd', default=1e-4, type=float,\n",
    "                    metavar='W', help='weight decay (default: 1e-4)')\n",
    "# parser.add_argument('--print-freq', '-p', default=10, type=int,\n",
    "#                     metavar='N', help='print frequency (default: 10)')\n",
    "# parser.add_argument('--resume', default='', type=str, metavar='PATH',\n",
    "#                     help='path to latest checkpoint (default: none)')\n",
    "# parser.add_argument('-e', '--evaluate', dest='evaluate', action='store_true',\n",
    "#                     help='evaluate model on validation set')\n",
    "parser.add_argument('--pretrained', dest='pretrained', action='store_true', help='use pre-trained model')\n",
    "parser.add_argument('--fp16', action='store_true', help='Run model fp16 mode.')\n",
    "parser.add_argument('--use-tta', default=True, type=bool, help='Validate model with TTA at the end of traiing.')\n",
    "parser.add_argument('--train-half', action='store_true', help='Train model on half images. TODO: allow custom epochs and LR')\n",
    "parser.add_argument('--sz',       default=32, type=int, help='Size of transformed image.')\n",
    "# parser.add_argument('--decay-int', default=30, type=int, help='Decay LR by 10 every decay-int epochs')\n",
    "parser.add_argument('--use-clr', default='10,13.68,0.95,0.85', type=str, \n",
    "                    help='div,pct,max_mom,min_mom. Pass in a string delimited by commas. Ex: \"20,2,0.95,0.85\"')\n",
    "parser.add_argument('--loss-scale', type=float, default=128,\n",
    "                    help='Loss scaling, positive power of 2 values can improve fp16 convergence.')\n",
    "parser.add_argument('--prof', dest='prof', action='store_true', help='Only run a few iters for profiling.')\n",
    "\n",
    "parser.add_argument('--dist-url', default='file://sync.file', type=str,\n",
    "                    help='url used to set up distributed training')\n",
    "parser.add_argument('--dist-backend', default='nccl', type=str, help='distributed backend')\n",
    "\n",
    "parser.add_argument('--world-size', default=1, type=int,\n",
    "                    help='Number of GPUs to use. Can either be manually set ' +\n",
    "                    'or automatically set by using \\'python -m multiproc\\'.')\n",
    "parser.add_argument('--rank', default=0, type=int,\n",
    "                    help='Used for multi-process training. Can either be manually set ' +\n",
    "                    'or automatically set by using \\'python -m multiproc\\'.')\n",
    "\n",
    "class TorchModelData(ModelData):\n",
    "    def __init__(self, path, trn_dl, val_dl, aug_dl=None):\n",
    "        super().__init__(path, trn_dl, val_dl)\n",
    "        self.aug_dl = aug_dl\n",
    "\n",
    "def torch_loader(data_path, size):\n",
    "    # Data loading code\n",
    "    traindir = os.path.join(data_path, 'train')\n",
    "    valdir = os.path.join(data_path, 'test')\n",
    "    normalize = transforms.Normalize(mean=[0.4914 , 0.48216, 0.44653], std=[0.24703, 0.24349, 0.26159])\n",
    "    \n",
    "    scale_size = 40\n",
    "    padding = int((scale_size - size) / 2)\n",
    "    train_tfms = transforms.Compose([\n",
    "        transforms.RandomCrop(size, padding=padding),\n",
    "        transforms.ColorJitter(.2,.2,.2),\n",
    "#         transforms.RandomRotation(2),\n",
    "        transforms.RandomHorizontalFlip(),\n",
    "        transforms.ToTensor(),\n",
    "        normalize,\n",
    "    ])\n",
    "    train_dataset = datasets.ImageFolder(traindir, train_tfms)\n",
    "    train_sampler = (torch.utils.data.distributed.DistributedSampler(train_dataset)\n",
    "                     if args.distributed else None)\n",
    "    train_loader = torch.utils.data.DataLoader(\n",
    "        train_dataset, batch_size=args.batch_size, shuffle=(train_sampler is None),\n",
    "        num_workers=args.workers, pin_memory=True, sampler=train_sampler)\n",
    "\n",
    "    val_tfms = transforms.Compose([\n",
    "    #   transforms.Resize(int(size*1.14)),\n",
    "    #   transforms.CenterCrop(size),\n",
    "        transforms.ToTensor(),\n",
    "        normalize,\n",
    "    ])\n",
    "    val_loader = torch.utils.data.DataLoader(\n",
    "        datasets.ImageFolder(valdir, val_tfms),\n",
    "        batch_size=args.batch_size, shuffle=False,\n",
    "        num_workers=args.workers, pin_memory=True)\n",
    "\n",
    "\n",
    "    aug_loader = torch.utils.data.DataLoader(\n",
    "        datasets.ImageFolder(valdir, train_tfms),\n",
    "        batch_size=args.batch_size, shuffle=False,\n",
    "        num_workers=args.workers, pin_memory=True)\n",
    "\n",
    "    train_loader = DataPrefetcher(train_loader)\n",
    "    val_loader = DataPrefetcher(val_loader)\n",
    "    aug_loader = DataPrefetcher(aug_loader)\n",
    "    if args.prof:\n",
    "        train_loader.stop_after = 200\n",
    "        val_loader.stop_after = 0\n",
    "\n",
    "    data = TorchModelData(data_path, train_loader, val_loader, aug_loader)\n",
    "    return data, train_sampler\n",
    "\n",
    "\n",
    "# Seems to speed up training by ~2%\n",
    "class DataPrefetcher():\n",
    "    def __init__(self, loader, stop_after=None):\n",
    "        self.loader = loader\n",
    "        self.dataset = loader.dataset\n",
    "        self.stream = torch.cuda.Stream()\n",
    "        self.stop_after = stop_after\n",
    "        self.next_input = None\n",
    "        self.next_target = None\n",
    "\n",
    "    def __len__(self):\n",
    "        return len(self.loader)\n",
    "    \n",
    "    def preload(self):\n",
    "        try:\n",
    "            self.next_input, self.next_target = next(self.loaditer)\n",
    "        except StopIteration:\n",
    "            self.next_input = None\n",
    "            self.next_target = None\n",
    "            return\n",
    "        with torch.cuda.stream(self.stream):\n",
    "            self.next_input = self.next_input.cuda(async=True)\n",
    "            self.next_target = self.next_target.cuda(async=True)\n",
    "\n",
    "    def __iter__(self):\n",
    "        count = 0\n",
    "        self.loaditer = iter(self.loader)\n",
    "        self.preload()\n",
    "        while self.next_input is not None:\n",
    "            torch.cuda.current_stream().wait_stream(self.stream)\n",
    "            input = self.next_input\n",
    "            target = self.next_target\n",
    "            self.preload()\n",
    "            count += 1\n",
    "            yield input, target\n",
    "            if type(self.stop_after) is int and (count > self.stop_after):\n",
    "                break\n",
    "                \n",
    "def top5(output, target):\n",
    "    \"\"\"Computes the precision@k for the specified values of k\"\"\"\n",
    "    top5 = 5\n",
    "    batch_size = target.size(0)\n",
    "    _, pred = output.topk(top5, 1, True, True)\n",
    "    pred = pred.t()\n",
    "    correct = pred.eq(target.view(1, -1).expand_as(pred))\n",
    "    correct_k = correct[:top5].view(-1).float().sum(0, keepdim=True)\n",
    "    return correct_k.mul_(1.0 / batch_size)\n",
    "\n",
    "\n",
    "class ImagenetLoggingCallback(Callback):\n",
    "    def __init__(self, save_path, print_every=50):\n",
    "        super().__init__()\n",
    "        self.save_path=save_path\n",
    "        self.print_every=print_every\n",
    "    def on_train_begin(self):\n",
    "        self.batch = 0\n",
    "        self.epoch = 0\n",
    "        self.f = open(self.save_path, \"a\", 1)\n",
    "        self.log(\"\\ton_train_begin\")\n",
    "    def on_epoch_end(self, metrics):\n",
    "        log_str = f'\\tEpoch:{self.epoch}\\ttrn_loss:{self.last_loss}'\n",
    "        for (k,v) in zip(['val_loss', 'acc', 'top5', ''], metrics): log_str += f'\\t{k}:{v}'\n",
    "        self.log(log_str)\n",
    "        self.epoch += 1\n",
    "    def on_batch_end(self, metrics):\n",
    "        self.last_loss = metrics\n",
    "        self.batch += 1\n",
    "        if self.batch % self.print_every == 0:\n",
    "            self.log(f'Epoch: {self.epoch} Batch: {self.batch} Metrics: {metrics}')\n",
    "    def on_train_end(self):\n",
    "        self.log(\"\\ton_train_end\")\n",
    "        self.f.close()\n",
    "    def log(self, string):\n",
    "        self.f.write(time.strftime(\"%Y-%m-%dT%H:%M:%S\")+\"\\t\"+string+\"\\n\")\n",
    "\n",
    "# Logging + saving models\n",
    "def save_args(name, save_dir):\n",
    "    if (args.rank != 0) or not args.save_dir: return {}\n",
    "\n",
    "    log_dir = f'{save_dir}/training_logs'\n",
    "    os.makedirs(log_dir, exist_ok=True)\n",
    "    return {\n",
    "        'best_save_name': f'{name}_best_model',\n",
    "        'cycle_save_name': f'{name}',\n",
    "        'callbacks': [\n",
    "            ImagenetLoggingCallback(f'{log_dir}/{name}_log.txt')\n",
    "        ]\n",
    "    }\n",
    "\n",
    "def save_sched(sched, save_dir):\n",
    "    if (args.rank != 0) or not args.save_dir: return {}\n",
    "    log_dir = f'{save_dir}/training_logs'\n",
    "    sched.save_path = log_dir\n",
    "    sched.plot_loss()\n",
    "    sched.plot_lr()\n",
    "\n",
    "def update_model_dir(learner, base_dir):\n",
    "    learner.tmp_path = f'{base_dir}/tmp'\n",
    "    os.makedirs(learner.tmp_path, exist_ok=True)\n",
    "    learner.models_path = f'{base_dir}/models'\n",
    "    os.makedirs(learner.models_path, exist_ok=True)\n",
    "    "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Resnet block"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "args_input = [\n",
    "    '/home/paperspace/imagenet-fast/fp16/data/cifar10', \n",
    "    '--save-dir', '/home/paperspace/data/cifar_training/test1', \n",
    "#     '-a', 'resnext29_8_64', \n",
    "#     '-j', '6', \n",
    "#     '--prof', \n",
    "    '-b', '512', \n",
    "#     '--sz', '32',\n",
    "#     '--loss-scale', '128',\n",
    "    '--fp16',\n",
    "    '--cycle-len', '110',\n",
    "#     '--epochs', '1',\n",
    "#     '--use-clr', '10,13.68,0.95,0.85',\n",
    "    '--wd', '2e-4',\n",
    "    '--lr', '1',\n",
    "#     '--train-half' # With fp16, iterations are so fast this doesn't matter\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "# This is important for speed\n",
    "cudnn.benchmark = True\n",
    "global arg\n",
    "args = parser.parse_args(args_input); args\n",
    "if args.cycle_len > 1: args.cycle_len = int(args.cycle_len)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "args.distributed = args.world_size > 1\n",
    "args.gpu = 0\n",
    "if args.distributed:\n",
    "    args.gpu = args.rank % torch.cuda.device_count()\n",
    "\n",
    "if args.distributed:\n",
    "    torch.cuda.set_device(args.gpu)\n",
    "    dist.init_process_group(backend=args.dist_backend, init_method=args.dist_url,\n",
    "                            world_size=args.world_size)\n",
    "\n",
    "if args.fp16:\n",
    "    assert torch.backends.cudnn.enabled, \"fp16 mode requires cudnn backend to be enabled.\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "=> creating model 'resnet56'\n"
     ]
    }
   ],
   "source": [
    "# create model\n",
    "model = cifar10models.__dict__[args.arch] if args.arch in cifar10_names else models.__dict__[args.arch] \n",
    "if args.pretrained:\n",
    "    print(\"=> using pre-trained model '{}'\".format(args.arch))\n",
    "    model = model(pretrained=True)\n",
    "else:\n",
    "    print(\"=> creating model '{}'\".format(args.arch))\n",
    "    model = model()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = model.cuda()\n",
    "if args.distributed:\n",
    "    model = DDP(model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "if args.train_half:\n",
    "    data, train_sampler = torch_loader(args.data, 16)\n",
    "else:\n",
    "    data, train_sampler = torch_loader(args.data, args.sz)\n",
    "\n",
    "learner = Learner.from_model_data(model, data)\n",
    "# learner.crit = F.nll_loss\n",
    "learner.crit = F.cross_entropy\n",
    "learner.metrics = [accuracy]\n",
    "if args.fp16: learner.half()\n",
    "\n",
    "if args.prof:\n",
    "    args.epochs = 1\n",
    "    args.cycle_len=.01\n",
    "if args.use_clr:\n",
    "    args.use_clr = tuple(map(float, args.use_clr.split(',')))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "# x,y = next(iter(data.trn_dl))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "# plt.imshow(np.transpose(x[50], (1, 2, 0)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "# %pdb off"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 128x128\n",
    "if args.train_half: \n",
    "    save_dir = args.save_dir+'/128'\n",
    "    update_model_dir(learner, save_dir)\n",
    "    sargs = save_args('first_run_128', save_dir)\n",
    "    learner.fit(args.lr,args.epochs, cycle_len=45,\n",
    "                train_sampler=train_sampler,\n",
    "                wds=args.weight_decay,\n",
    "                use_clr_beta=args.use_clr,\n",
    "                loss_scale=args.loss_scale,\n",
    "                **sargs\n",
    "            )\n",
    "    save_sched(learner.sched, save_dir)\n",
    "    data, train_sampler = torch(args.data, args.sz)\n",
    "    learner.set_data(data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "99d65b6899914c039b46f03dab37ae99",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epoch', max=110), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch      trn_loss   val_loss   accuracy                 \n",
      "    0      1.812441   1.859512   0.3174    \n",
      "    1      1.623513   1.485934   0.4409                   \n",
      "    2      1.43354    1.352941   0.5064                   \n",
      "    3      1.20538    1.823247   0.44                     \n",
      "    4      1.047512   1.031067   0.6364                   \n",
      "    5      0.903786   0.990384   0.6529                    \n",
      "    6      0.794144   0.937554   0.6923                    \n",
      "    7      0.719986   0.794384   0.7304                    \n",
      "    8      0.663548   0.824012   0.7209                    \n",
      "    9      0.616721   0.684154   0.7641                    \n",
      "    10     0.599153   0.760171   0.7422                    \n",
      "    11     0.569865   0.973621   0.6983                    \n",
      "    12     0.544941   0.826669   0.7262                    \n",
      "    13     0.524519   0.852246   0.7406                    \n",
      "    14     0.513213   0.753155   0.7662                    \n",
      "    15     0.497413   0.846931   0.7504                    \n",
      "    16     0.477519   0.668112   0.7875                    \n",
      "    17     0.473021   0.655387   0.783                     \n",
      "    18     0.466157   0.857789   0.7328                    \n",
      "    19     0.450976   0.666845   0.7821                    \n",
      "    20     0.443292   0.860191   0.7377                    \n",
      "    21     0.439205   1.131203   0.6821                    \n",
      "    22     0.446406   1.028122   0.7189                    \n",
      "    23     0.426874   1.017566   0.7207                    \n",
      "    24     0.427285   0.96748    0.7216                    \n",
      "    25     0.425151   0.703964   0.7811                    \n",
      "    26     0.418674   0.983397   0.7092                    \n",
      "    27     0.418991   0.744101   0.7635                    \n",
      "    28     0.415415   0.681863   0.7864                    \n",
      "    29     0.413028   0.61349    0.8049                    \n",
      "    30     0.41116    0.649954   0.789                     \n",
      "    31     0.395728   1.480305   0.6203                    \n",
      "    32     0.402052   0.689628   0.7703                    \n",
      "    33     0.39418    1.019106   0.7183                    \n",
      "    34     0.402665   0.668238   0.7811                    \n",
      "    35     0.408001   0.809439   0.7619                    \n",
      "    36     0.397961   0.859566   0.7617                    \n",
      "    37     0.394969   0.698795   0.7666                    \n",
      "    38     0.390791   0.685463   0.7879                    \n",
      "    39     0.393455   0.872631   0.7195                    \n",
      "    40     0.391663   0.686177   0.7817                    \n",
      "    41     0.395742   0.608006   0.8057                    \n",
      "    42     0.382299   0.761684   0.7698                    \n",
      "    43     0.390123   0.580282   0.8171                    \n",
      "    44     0.383594   0.948203   0.7183                    \n",
      "    45     0.386429   1.288566   0.675                     \n",
      "    46     0.386106   0.783413   0.7616                    \n",
      "    47     0.387071   0.907251   0.7336                    \n",
      "    48     0.376726   0.665081   0.8034                    \n",
      "    49     0.389123   0.749052   0.7737                    \n",
      "    50     0.383904   0.616589   0.801                     \n",
      "    51     0.374272   0.730153   0.7736                    \n",
      "    52     0.371789   0.712743   0.7745                    \n",
      "    53     0.377667   0.703202   0.7764                    \n",
      "    54     0.36974    0.61975    0.802                     \n",
      "    55     0.358827   0.551073   0.8208                    \n",
      "    56     0.356373   0.862586   0.7476                    \n",
      "    57     0.366815   0.699805   0.7845                    \n",
      "    58     0.358783   0.637675   0.8054                    \n",
      "    59     0.357312   0.856862   0.7544                    \n",
      "    60     0.353695   0.657164   0.7961                    \n",
      "    61     0.349274   0.767159   0.7683                    \n",
      "    62     0.346976   0.767261   0.7736                    \n",
      "    63     0.340436   0.628997   0.7969                    \n",
      "    64     0.341639   0.806389   0.7749                    \n",
      "    65     0.334653   0.665875   0.7943                    \n",
      "    66     0.34402    0.803923   0.7592                    \n",
      "    67     0.339453   0.633359   0.8014                    \n",
      "    68     0.332211   0.563867   0.8298                    \n",
      "    69     0.33266    0.641863   0.7961                    \n",
      "    70     0.329087   0.530855   0.8222                    \n",
      "    71     0.323273   0.578356   0.8087                    \n",
      "    72     0.319474   0.788859   0.7752                    \n",
      "    73     0.315593   0.51844    0.8351                    \n",
      "    74     0.314834   0.528026   0.8364                    \n",
      "    75     0.310698   0.609739   0.8137                    \n",
      "    76     0.310191   0.567951   0.82                      \n",
      "    77     0.307431   0.445347   0.8574                    \n",
      "    78     0.301006   0.643344   0.8068                    \n",
      "    79     0.301107   0.801222   0.7837                    \n",
      "    80     0.285171   0.49959    0.835                     \n",
      "    81     0.282677   0.486095   0.8435                    \n",
      "    82     0.271954   0.684845   0.8037                    \n",
      "    83     0.270613   0.485216   0.8463                    \n",
      "    84     0.262175   0.629983   0.8131                    \n",
      "    85     0.267298   0.418199   0.8652                    \n",
      "    86     0.253407   0.545097   0.8377                    \n",
      "    87     0.250527   0.376923   0.8776                    \n",
      "    88     0.252519   0.616157   0.8102                    \n",
      "    89     0.230011   0.489934   0.8478                    \n",
      "    90     0.218822   0.430929   0.8712                    \n",
      "    91     0.203619   0.395288   0.8746                    \n",
      "    92     0.192234   0.372025   0.8803                    \n",
      "    93     0.180982   0.338984   0.8912                    \n",
      "    94     0.167879   0.350927   0.8904                    \n",
      "    95     0.155493   0.402861   0.8785                    \n",
      "    96     0.148298   0.349918   0.8914                    \n",
      "    97     0.136534   0.316343   0.9028                    \n",
      "    98     0.124189   0.3294     0.906                     \n",
      "    99     0.113916   0.31101    0.9081                    \n",
      "   100     0.107916   0.299103   0.9114                    \n",
      "   101     0.097518   0.290329   0.9201                     \n",
      "   102     0.088652   0.305113   0.9159                     \n",
      "   103     0.076407   0.297681   0.9155                     \n",
      "   104     0.065395   0.297327   0.9162                     \n",
      "   105     0.055489   0.270902   0.9239                     \n",
      "   106     0.04417    0.259012   0.9284                     \n",
      "   107     0.034333   0.247566   0.9341                     \n",
      "   108     0.028114   0.247997   0.9339                     \n",
      "   109     0.022313   0.247694   0.9345                     \n",
      "\n",
      "Finished!\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAD8CAYAAACMwORRAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAHp9JREFUeJzt3Xl8VPW9//HXJzvZyB4CCYRdUFEhRXArijv22l3stS7Va2u11Wt/ttpeu93+fvX21vtrtQtatVrbqq1ad+uKe12CUnYkrAlbEhICBBKSzPf+MYc4hGzChJM5834+HvNg5jtnzny+OfDm5HuWrznnEBGRYEnwuwAREYk+hbuISAAp3EVEAkjhLiISQAp3EZEAUriLiASQwl1EJIAU7iIiAaRwFxEJoCS/vrigoMCVl5f79fUiIjFpwYIF9c65wr6W8y3cy8vLqays9OvrRURikpmt789yGpYREQkghbuISAAp3EVEAkjhLiISQAp3EZEAUriLiASQwl1EJIBiLtxXbtnJrc+vZNuuVr9LEREZtGIu3NfU7eL2l6uoU7iLiPQo5sI9NTlccktbyOdKREQGr9gL96REAFrbOnyuRERk8IrBcA+X3NquPXcRkZ7EXLinJYf33Fu05y4i0qOYC3ftuYuI9C0Gw90bc1e4i4j0KObCPa3zbBkNy4iI9CTmwl177iIifYu9cE/eN+auPXcRkZ7EXrgn6SImEZG+xFy4mxmpSQm6iElEpBcxF+4AmalJ7Gpt97sMEZFBKzbDPU3hLiLSm5gM94yUJJoV7iIiPYrJcM9MS2Jni8JdRKQnMRnuWRpzFxHpVZ/hbmZlZjbfzJab2VIzu7abZczMbjOzKjNbZGZTB6bcsMw0DcuIiPQmqR/LtAPfcs69b2ZZwAIze8E5tyximXOA8d7jeOC33p8DIkN77iIivepzz905t9k59773fCewHBjRZbHzgT+4sLeBHDMriXq1nqxUjbmLiPTmY425m1k5cBzwTpe3RgDVEa9rOPA/gKjJTE2itT3EXt1fRkSkW/0OdzPLBB4BrnPO7ej6djcfcd2s40ozqzSzyrq6uo9XaYTcjBQAGnfvPeh1iIgEWb/C3cySCQf7n5xzj3azSA1QFvG6FNjUdSHn3J3OuQrnXEVhYeHB1AtAvhfu23Yp3EVEutOfs2UMuBtY7pz7nx4WewK42DtrZgbQ5JzbHMU695OnPXcRkV7152yZE4EvA4vNbKHX9l1gJIBzbh7wDHAuUAXsBi6Lfqkfyc/09tybFe4iIt3pM9ydc2/Q/Zh65DIOuDpaRfUlNz0c7g27Wg/XV4qIxJSYvEI1Jz0FM2jQnruISLdiMtwTE4zc9BQNy4iI9CAmwx3CB1W15y4i0r2YDfeCzBSdCiki0oMYDvdU6nRAVUSkWzEb7oVZqdTvVLiLiHQnZsO9IDOVna3ttGiibBGRA8RsuJcMTQNg4/Y9PlciIjL4xGy4j8rPAGD9tmafKxERGXxiNtzL89MBWFe/2+dKREQGn5gN97yMFLLSklinPXcRkQPEbLibGeX5GaytV7iLiHQVs+EOUF6QwfptGpYREekqtsM9P52axt2abk9EpIsYD/cMQg5qGrX3LiISKbbDvSB8OqQOqoqI7C+2w12nQ4qIdCumw12nQ4qIdC+mw93MGF2Qweq6XX6XIiIyqMR0uAMcMSyL5Zt3Ep7GVUREIADhftSIoTQ072VTU4vfpYiIDBoxH+5TSnMAWFS93edKREQGj5gP9yOGZZFgsHzzDr9LEREZNGI+3NOSExmZl87qOp0xIyKyT8yHO8DYwkydMSMiEiEY4V6UyZr6ZjpCOmNGRASCEu6FGextD7GxUVPuiYhAYMI9E0BDMyIiHoW7iEgABSLcczNSyM9IoapW4S4iAgEJd9AZMyIikYIT7kUZOtddRMQTnHAvzKSheS8NzXv9LkVExHfBCfei8EHVNRqaEREJTriP0xkzIiKdAhPuw3OGkJqUoDNmREToR7ib2T1mVmtmS3p4f5aZNZnZQu/x/eiX2bfEhPCsTKsU7iIi/dpzvxc4u49lXnfOHes9fnzoZR2co0cMZVFNk2ZlEpG412e4O+deAxoOQy2HbOqoXBqa97J+226/SxER8VW0xtxnmtk/zexZMzuyp4XM7EozqzSzyrq6uih99Ucml2QD8OHWnVFft4hILIlGuL8PjHLOHQPcDjzW04LOuTudcxXOuYrCwsIofPX+RuWnA2jPXUTi3iGHu3Nuh3Nul/f8GSDZzAoOubKDkJOue8yIiEAUwt3MhpmZec+ne+vcdqjrPVjjijJZsUXzqYpIfEvqawEzewCYBRSYWQ3wAyAZwDk3D/g8cJWZtQN7gLnOx9NVxhdn8vjCTTjn8P7PERGJO32Gu3Puwj7e/xXwq6hVdIjK8zPY2dLO9t1t5Gak+F2OiIgvAnOF6j6j8jMA2NCgg6oiEr8CF+5leUMAhbuIxLfghXtu+HTI6kaFu4jEr8CFe0ZqEgWZKVRrz11E4ljgwh2gLC9dwzIiEteCGe65CncRiW+BDPeReels2t5Ce0fI71JERHwR2HDvCDk2N7X4XYqIiC8CGe5leeEzZjQ0IyLxKqDhHj7XXXeHFJF4FchwLxk6hORE0567iMStQIZ7YoJRmpvOhoZmv0sREfFFIMMdwhN3rKvXnruIxKfAhnt5fgYbGnZrsmwRiUuBDfdR+ensam1nW/Nev0sRETnsAhvu5d6tf9dv07i7iMSfwIa7JssWkXgW2HAvzU0nwWCdwl1E4lBgwz0lKYHhOUM0LCMicSmw4Q7hcXcNy4hIPAp0uI/KT9eeu4jEpcCHe+PuNpp2t/ldiojIYRXwcPdOh9RtCEQkzgQ63D86113j7iISXwId7iPz9p3rrj13EYkvgQ73ISmJFGenslY3EBOROBPocAeYUJzF8s07/C5DROSwCny4TykdysqtO2lp6/C7FBGRwyYOwj2HjpBjmfbeRSSOBD7cJxZnAXDvm+v8LURE5DAKfLjvuzukxt1FJJ4EPtzNjAsqyqjb1apZmUQkbgQ+3AGOGpHN9t1tVDfs8bsUEZHDIi7CvTg7DYBbX1jpcyUiIodHXIT7GZOL/S5BROSwiotwNzNmTSxk1dZdfpciInJY9BnuZnaPmdWa2ZIe3jczu83MqsxskZlNjX6Zh25cYSar63bREdJBVREJvv7sud8LnN3L++cA473HlcBvD72s6BtXlElre4iNjTqoKiLB12e4O+deAxp6WeR84A8u7G0gx8xKolVgtIwrygTg2SWbfa5ERGTgRWPMfQRQHfG6xms7gJldaWaVZlZZV1cXha/uv0kl2QD8XleqikgciEa4Wzdt3Q5sO+fudM5VOOcqCgsLo/DV/ZeRmgTAlh0th/V7RUT8EI1wrwHKIl6XApuisN6ou+zEcgB2tGhOVREJtmiE+xPAxd5ZMzOAJufcoBzY/uSE8G8LS2qafK5ERGRgJfW1gJk9AMwCCsysBvgBkAzgnJsHPAOcC1QBu4HLBqrYQzV1VC5JCcYbVfWcMK7A73JERAZMn+HunLuwj/cdcHXUKhpA2WnJHDViKO+t6+3kHxGR2BcXV6hGmjYql0U1TbR1hPwuRURkwMRduB9TlkNre4h31mjvXUSCK+7C/biyHABeXL7V50pERAZO3IV7WV46WalJbNqu2xCISHDFXbgDjC7M4Pll2nMXkeCKy3CfXp4HQK2uVhWRgIrLcP/0ceFb38xfWetzJSIiAyMuw/3I4dmMzEvnqUWD8kJaEZFDFpfhbmbMmVLCW6u30di81+9yRESiLi7DHWDO0SV0hBx/+Md6v0sREYm6uA33I4eH7+9euV4XM4lI8MRtuJsZF04fycLq7ZpXVUQCJ27DHWDGmDx2trSzbNMOv0sREYmquA73mWPzAXhrdb3PlYiIRFdch3tRVhrl+em8s1bj7iISLHEd7gCzJhbxxqp6drW2+12KiEjUxH24n3lkMXs7Qlz1xwV+lyIiEjVxH+4Vo8L3mXl9VT2PfbDR52pERKIj7sM9JSmBN75zKgDXPbTQ52pERKIj7sMdoDQ3na/PGgug+VVFJBAU7p4rTxkDwJ/f2eBzJSIih07h7slJT+Hco4fxtw82sra+2e9yREQOicI9wvVnTATg+48v8bkSEZFDo3CPMK4oEwifOfP7N9f6XI2IyMFTuHfxf86cAMCPnlyGc7qhmIjEJoV7F9ecNp6LZowE4OUVmoZPRGKTwr0b3zn7CABeXL7V50pERA6Owr0bWWnJjC/K5IF3qym/8WlCut+7iMQYhXsPbv3iMZ3Pr7y/0sdKREQ+PoV7D6aU5vDhT84B4MXltZTf+DTlNz7N5qY9PlcmItI3hXsvUpISeO66U/Zr+8lTy32qRkSk/xTufZg4LIuzjiwG4OTxBTy9eDPlNz7NP1Zv87kyEZGeKdz74Y4vV7DuljlcekJ5Z9uFv3ub1z6s868oEZFeKNw/htmTipl30dTO1xff8y6vr1LAi8jgo3D/mM4+qoR1t8zh8pNGA/Dlu9/lU7e/4XNVIiL761e4m9nZZrbSzKrM7MZu3r/UzOrMbKH3uCL6pQ4uN51zBLnpyQAs3tjEW1X11O9q9bkqEZEw6+v+KWaWCHwInAHUAO8BFzrnlkUscylQ4Zy7pr9fXFFR4SorY//88S1NLcz46UsHtL92w6mMzE/3oSIRCTIzW+Ccq+hruf7suU8Hqpxza5xze4EHgfMPtcCgGDY0jdlHFB3Qfsp/z+d6TdsnIj7pT7iPAKojXtd4bV19zswWmdnDZlYWlepixN2XfoJ1t8zhvq9MB+DC6eHuP/rBRo76wXOceMvLdOgWBiJyGPUn3K2btq5J9SRQ7pybArwI3NftisyuNLNKM6usqwveWSafnFDIulvm8NPPTuHeyz4BwK7WdjZu38PY7z7DXa+voaWtw+cqRSQe9Cfca4DIPfFSYFPkAs65bc65fUcTfwdM625Fzrk7nXMVzrmKwsLCg6k3ZsyaWMQNZ03cr+0nTy/niJv/zgvLtvKTp5Yxf2X4lsK6b7yIRFt/DqgmET6gOhvYSPiA6pecc0sjlilxzm32nn8G+I5zbkZv6w3KAdX+aNrTxtOLNvPdvy3ucZmvfnIM184eT3pK0mGsTERiTX8PqPYZ7t7KzgV+ASQC9zjn/q+Z/RiodM49YWY/Bf4FaAcagKuccyt6W2c8hXukv7xXzbcfWdTj+1mpSbzzvdmdIb9s0w4mD88GIBRytIccKUm6PEEkXkU13AdCvIZ7pFDI8cLyrUwozuLUn7/S67JXnzqWX89f3fn6h5+azKUnjh7gCkVksFG4x6DG5r0c958v9Hv5rLQkvn3WRG5+fClXzRrL6ZOKeGPVNr5x2jgSEro7Di4isU7hHsOWbdpBSpKxpq6Z8cVZvLKylh89uYy7L6lg9qRifvNKFT/7+8p+rev6Mybw/oZGlm7awX9/fgqX/v49hg9N46GvzuTkn83nvq9M55EFNVx7+njGFmYOcM9E5FAp3AOudkcL0/9f+MrY//z0Udz82JKorPfv151MeX4GacmJAKyu20VxdhqZqQce6HXO8c0HFzK9PJeHKqv5/nlHMn10XlTqEJHuKdzjQFXtLpITjVH5Gfu1N7e2c8erq7l9fhX7Nu+w7DS27GjhlAmFnbcqPnrEUBZvbOp23acdUcTLK2o7X79142ks37yDy+/rfZs9/++n8Ov5Vcwck88XKsr4S2U1J44tYGR+On+trObR9zdy/+XTaQ85WttCtLZ38LcPNnLBJ8rISU8Bwv9p/O2DjcyeVMzQIckH9bMJhRw7W9pJSAjPiSsSFAp3AaAj5EjsZvy9pa2D5MQExn73GT43tZSff2EKP3pyGfe+te6gvicpwWjv5SrcK04azV1vrO1zPXd8eRrXP7SQ5r3hi72evOYkPjfvLfa2hzqXuWrWWH77ymqmjswhMcF4b10j500p4Zdzj2PHnjZyM1Iov/Hp/db7zdnjuf6MCQfVN5HBROEuB6WtI8Tv31zLo+9v5IazJlLdsJshKYl855HwOfqXnlDOzedNJjHB2LarleSkBLK9PeOr//w+Ty/a7Gf5AJw+qZgXl289oP1Lx4/k2NIczODet9Zx2hFFXHHSGO5/ex23vvAhp4wv5IsVZcyZUuJD1SL9o3AXXyyq2c7RI4ayfXcbIefIz0zllZW1XH5fJfkZKbx6w6kMSQmP5y/d1MTIvHSy0pLZ0dLGlB8+37me9753Oj9/biUPVVZTmJVK3c5W5n6ijFH5GfzX31fw6WOH89jC8IXS8y6axtf+uOCAWv5x02kUZqayeGMTn/nNWx+7L7/916mMzE/nyOFDO9ta2jr4/uNLmDWxiHOOGkZHyLFpewvPLtnMuUeXUJanO4HKwFK4S+B1hBxr63cxriirs+3tNduYe+fbTCkdyhPXnNTZvq6+mVNvfYWe/rp/47Rx3P5y1SHXdPL4AjY27uGKk8cwuiCDV1bWMnNsPrnpKRRnpzFsaNohf4fEN4W7xK0PNjQyImcIRdl9B+kbq+o5cng2uRkpnW0NzXuZ+jGuNzgYKUkJvPvd2XxQvZ2Xlm9lz94Qc6YM47Qjilm6qYk5t4Vn97qgooz01ET+Y87k/Y6dLFjfyDGlQ0lK1NXK8UbhLhIF765toHJ9A2W56Zx91DCSvTCt39VKWnLifqeI/urlVTy3dOt+ZyDNmljIKysH5g6ox4/O4/7Lj+fhBTWcMbmYwqxU2jtCJCYYZrqILagU7iI+2tLUQmFWKokJhnOO3Xs7yPD+I2jvCDHue8/2+vljy3IYU5jBO2sa2Lh9T5/fN2NMHrd+8VhOvOVlAG44ayKLarbzs88dw2d+8ybXnDaOz04tJRRyvLqqDhwcNWIohVmph95ZOawU7iIxpiPkeGHZVk49opDUpMTO9qraXXz6129y2YnltLR18LvX13LPpRV85d6P9+9n34Hp7mSkJDLvy9OYUJzFBxsaOfsonTE0WCncRQLOOYeZ0dYR4qz//xpr6puZd9FUirPT+OVLqzqHg04aV8AbVfUH/T2pSQm0tof46WeP5sLpI6NVvhwkhbuIdKqq3cldr6/lm7PHMzxnCADvb2jkzlfX8K0zJ/DI+xuZ9+rqPtYS9pUTR3PDWRNp2tPGXyurOXdKCWMLM2lubSctObHbi+YkehTuInJQane28NzSrRw/Oo+hQ5L5YEMjbR2ObzzwQY+f+eXcY7n2wfCE8GdMLua2ucd1Xs8g0aVwF5Goam5t52t/XEBzazvvb9je5/JJCcY5R5cwc0w+P3hiCW0djldvmEVZbrpuSX0IFO4iMmDaOkIkmJGYYDy/dAtX3r+A848dzmeOG8FfKqt5ZvGWXj8/qSSbx64+gcU1TQzPGUJeRgpH/uA5HrpyBhXlurNobxTuInLYNO1uIystqXOP/LUP65hQnMXDC6r5+fMf8tQ3TqJyXQM/fHJZt5+fc3QJTy8O35do6Y/O6jxtVA6kcBeRQakj5Lj6T+/z96W9791/6pjh3H7hcYepqtihcBeRmPBWVT1fuusdXrz+k9z6/EqeXbJ/6F9+0mhuPm+yT9UNPgp3EYk5oZBj0cYm9raH+OId/+hsv2rWWC6ZWU56amLnLabjlcJdRGJaKOR4bVUd3354EbURV9Y+/c2TmDQsm+rG3YzMS4+7++go3EUkEFraOrjgjn/wz5rup4QsyxvCazecGjch399w1yFpERnU0pITedy7N/+2Xa2cd/sbbG5q6Xy/umEP97+9notnlvtU4eCkPXcRiVkdIccV973H/IjbKv/k00dx0YxRPlY1sDQsIyJxYW97iP94bDF/qaw54L1/O3k03zpzIokJ1nkv/lincBeRuFPTuJv/eGzJAROk5KQnc82p4zhz8jDK8obE9Pi8wl1E4trOlja+ev8CNjTspqbxowlPpo/O446LpjF0SHJM3uNG4S4i4nlx2VbueG01NY179jsYe8VJozluZC6nTy7ab4KUwUzhLiLSjRVbdnDtAwtZXbeL9tBH+TcyL51R+elccfIYPjmh0McKe6dwFxHpw5amFl5bVcctz66goXlvZ/u0UbkUZqZy/rHDmT2pmJSkwXMwVuEuItJPO1ra+Gf1dlISE7jt5VW8WbWNnPRktu9uA2BK6VAuPaGcUyYUUpDp76TiCncRkUPQ2t7BKyvDe/Vr65s72y+oKOPmT00m06fbEivcRUSipHZHC9c9tJDstGSeW7YFAwqzUhldkMG5R5dw9pHDKMpOOyy1KNxFRAbAgvUNzF9Rx6ranby0vJb2kCMxwfjCtFLmTCnhhLEFAzpJuO4tIyIyAKaNymPaqPBUgK3tHSxY18gzSzbzx7c38OB71RRlpfL1WWM5d0oJRVmHZ2++O/3aczezs4FfAonAXc65W7q8nwr8AZgGbAMucM6t622d2nMXkSCpadzNXa+vZWH1dhZWhycQLxmaxsnjCzi6NIdh2WkcUzb0kAM/asMyZpYIfAicAdQA7wEXOueWRSzzdWCKc+5rZjYX+Ixz7oLe1qtwF5Egcs6xYstOnl+6lQ+37uSVlbU07+3ofL80dwiXzCzn304Zc1Drj+awzHSgyjm3xlvxg8D5QORMt+cDP/SePwz8yszM+TWgLyLiEzNjUkk2k0qygfDQTd3OVlZs3smijU18uGUnw3OGDHgd/Qn3EUB1xOsa4PielnHOtZtZE5AP1EejSBGRWJWalEhpbjqluemcPrn4sH1vfy676u6wb9c98v4sg5ldaWaVZlZZV1fXzUdERCQa+hPuNUBZxOtSYFNPy5hZEjAUaOi6Iufcnc65CudcRWHh4L13g4hIrOtPuL8HjDez0WaWAswFnuiyzBPAJd7zzwMva7xdRMQ/fY65e2Po1wDPET4V8h7n3FIz+zFQ6Zx7ArgbuN/Mqgjvsc8dyKJFRKR3/bqIyTn3DPBMl7bvRzxvAb4Q3dJERORgDZ77WIqISNQo3EVEAkjhLiISQL7dFdLM6oD1B/nxAoJ/gVTQ+xj0/kHw+xj0/sHg7OMo51yf55L7Fu6Hwswq+3NvhVgW9D4GvX8Q/D4GvX8Q233UsIyISAAp3EVEAihWw/1Ovws4DILex6D3D4Lfx6D3D2K4jzE55i4iIr2L1T13ERHpRcyFu5mdbWYrzazKzG70u57+MrMyM5tvZsvNbKmZXeu155nZC2a2yvsz12s3M7vN6+ciM5sasa5LvOVXmdklPX2nH8ws0cw+MLOnvNejzewdr9aHvJvPYWap3usq7/3yiHXc5LWvNLOz/OlJ98wsx8weNrMV3racGaRtaGb/7v39XGJmD5hZWqxvQzO7x8xqzWxJRFvUtpmZTTOzxd5nbjOzgZsd++NwzsXMg/CNy1YDY4AU4J/AZL/r6mftJcBU73kW4akLJwM/A2702m8E/st7fi7wLOF75c8A3vHa84A13p+53vNcv/sX0c/rgT8DT3mv/wLM9Z7PA67ynn8dmOc9nws85D2f7G3XVGC0t70T/e5XRP/uA67wnqcAOUHZhoQn3VkLDInYdpfG+jYETgGmAksi2qK2zYB3gZneZ54FzvF7WzrnYi7cZwLPRby+CbjJ77oOsi+PE56XdiVQ4rWVACu953cQnqt23/IrvfcvBO6IaN9vOZ/7VAq8BJwGPOX9Za8HkrpuP8J3GZ3pPU/ylrOu2zRyOb8fQLYXftalPRDbkI9mVMvztslTwFlB2IZAeZdwj8o2895bEdG+33J+PmJtWKa7Kf9G+FTLQfN+fT0OeAcods5tBvD+LPIW66mvg/ln8Avg20DIe50PbHfOtXuvI2vdb2pGYN/UjIO5f2OAOuD33tDTXWaWQUC2oXNuI/BzYAOwmfA2WUCwtuE+0dpmI7znXdt9F2vh3q/p/AYzM8sEHgGuc87t6G3RbtpcL+2+MrPzgFrn3ILI5m4WdX28Nyj750ki/Ov9b51zxwHNhH+l70lM9dEbdz6f8FDKcCADOKebRWN5G/bl4/Zp0PY11sK9P1P+DVpmlkw42P/knHvUa95qZiXe+yVArdfeU18H68/gROBfzGwd8CDhoZlfADkWnnoR9q+1p6kZB2v/IFxbjXPuHe/1w4TDPijb8HRgrXOuzjnXBjwKnECwtuE+0dpmNd7zru2+i7Vw78+Uf4OSdwT9bmC5c+5/It6KnKLwEsJj8fvaL/aO3s8AmrxfH58DzjSzXG9P60yvzVfOuZucc6XOuXLC2+Vl59y/AvMJT70IB/avu6kZnwDmemdijAbGEz5g5Tvn3Bag2swmek2zgWUEZBsSHo6ZYWbp3t/Xff0LzDaMEJVt5r2308xmeD+ziyPW5S+/B/0P4sDIuYTPNFkNfM/vej5G3ScR/nVtEbDQe5xLeIzyJWCV92eet7wBv/b6uRioiFjXV4Aq73GZ333rpq+z+OhsmTGE/2FXAX8FUr32NO91lff+mIjPf8/r90oGyZkHEbUdC1R62/ExwmdOBGYbAj8CVgBLgPsJn/ES09sQeIDwMYQ2wnval0dzmwEV3s9rNfAruhxw9+uhK1RFRAIo1oZlRESkHxTuIiIBpHAXEQkghbuISAAp3EVEAkjhLiISQAp3EZEAUriLiATQ/wIQ+RKnnVfBFQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAtQAAAEKCAYAAAAy8cIyAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJzs3XlclXX6//HXxSaCgAuICyouIAKimWmZZWa5pWIuMzX1nZlqauY7Nc1MadnibmVp08w0Td+ppqZZm7RyS1Mz2zdtAQEBETdccQNB2a/fHxznx5ALIof7HLiej8f9kHOf+3DeB4Fzcd/X5/MRVcUYY4wxxhhTPz5OBzDGGGOMMcabWUFtjDHGGGPMRbCC2hhjjDHGmItgBbUxxhhjjDEXwQpqY4wxxhhjLoIV1MYYY4wxxlwEK6iNMcYYY4y5CFZQG2OMMcYYcxGsoDbGGGOMMeYi+Dkd4EKFh4drdHS00zGMMeaCffXVV4dVNcLpHI3JfmcbY7xZXX9ve11BHR0dzebNm52OYYwxF0xEdjmdobHZ72xjjDer6+9ta/kwxhhjjDHmIlhBbYwxxhhjzEWwgtoYY4wxxpiLYAW1McYYY4wxF8EKamOMMcYYYy6C2wpqEXlZRA6JSNpZ7hcR+b2I5IhIqogMcFcWY4wxxhhj3MWdZ6j/Aow+x/1jgBjXdhfwvBuzGGOMMcYY4xZum4daVT8UkehzHJIM/FVVFfhcRFqLSEdV3e+uTKbpyT9Rysc5+Uzs3xkRcTqOMaYBfLvnOFvyjvM/V0Q7HcUYcwG27i9kzRbPLOOSolpzXXyk2z6/kwu7dAb21Lid59r3nf8JEbmL6rPYdO3atVHCGc9XVaX88OUv2bq/kFNlVfxgsH1vGNMU/HvTbpZszmNwj3bERoY4HccYUweFJeX88OUvyT9Riiee37plcNcmW1Cf6cutZzpQVV8AXgAYOHDgGY8xzc/fPt/F1v2FADz2dgZXxYTTpW2Qw6mMMRdr+qg41qQd4NFlafz7rsvt6pMxXuCZ9dkcLiplxT1XkhTV2uk4jc7JWT7ygC41bkcB+xzKYrzMzsPFLFyTybDYCD56YDgAD76RSlWV/b1ljLdrGxzAg6Pj+HLHUZZ/a28Lxni6jH2FvPrpTn4wqGuzLKbB2YJ6BfBD12wflwMF1j9t6qKqSnlgaSp+vsLCyX3p0jaIR8fF8+n2I/zji11OxzPGNIDvD+xCvy6tWfD2VgpLyp2OY4w5i6oqZdbyNFoHBTB9VG+n4zjGndPm/Qv4DOgtInkicoeI/ExEfuY6ZDWQC+QALwI/d1cW07T85dOdfLnzKLPGxdMxrCUAN13Whatiwnl8dSa7j5x0OKEx5mL5+AgLkhM5UlzKb9ZlOx3HGHMWb3ydx+Zdx5gxOo7WQQFOx3GM2wpqVb1ZVTuqqr+qRqnqn1X1/1T1/1z3q6rerao9VbWvqm52VxbTdOTmF/HU2kyujWvPlEuj/rNfRHhychJ+PsL0pSnW+mFME9A3KoxbB3fjr5/tJGNfodNxjDG1FJwsZ+GaTAZ0bf1f78nNka2UaLxGZZUyfWkqAb4+PDGp73cGKnVq3ZKZ4+L5YsdR/vrZTkcyGmMa1rSRvWkTFMDM5Wn2h7IxHmbRukyOnSxj/sREfHya9+BhK6iN13jlkx18tesYcyYkEBkaeMZjpg6M4preESx8J5Odh4sbOaExpqGFBfkzY0wcX+06xtKv85yOY4xx2ZJXwD++2M0Pr4gmoVOY03EcZwW18Qo5h4pYtDaL6/pEcuMlnc96nIiwcFIS/r4+TFuSQqWd0TLG600eEMXAbm1YuCaTgpM2QNEYp1VVKY8uT6NdcAvuGxnrdByPYAW18XjVrR4ptAzw5fFJieedk7ZDWCBzxiewedcxXvlkRyOlNMa4i4+PMH9iIgWnylm0LtPpOMY0e69t2kPKnuM8ckMcoYH+TsfxCFZQG4/30ke5fLP7OHMnJNA+5MytHrVNGtCZ6/q0Z9HaLHLzi9yc0Bjjbn06hvLDK7rxjy92k5p33Ok4xjRbR4vLeGptJoO7t2Vi/7NfMW5urKA2Hm3bwRM8vT6bUQmRTOjXqc6PExEev7Evgf6+1vphTBPx6+tjCW/VgpnLbICiMU55ck0mRSUVzJ94/ivGzYkV1MZjVVRWMW1JCsEBviyY+N1ZPc6nfWggcyck8PXu4/z541w3pTTGNJbQQH8eGduHlLwCXtu0x+k4xjQ7X+8+xr837+H2od2JjQxxOo5HsYLaeKwXPsolJa+AecmJRIS0qNfnSO7fiZHxkSxel03OoRMNnNAY09iS+3fi8h5teWptJkeLy5yOY0yzUVmlzFyWRofQQO4dEeN0HI9jBbXxSFkHTvDb9dsY27cD45I61vvziAiP3diX4ABf7l+SSkVlVQOmNMY0NhFhXnIiRSUVPPWODVA0prH8/fNdpO8r5NFxfWjVws/pOB7HCmrjccpdrR4hgX7MT774Hq2IkBbMS04kZc9xXvjIWj+M8XaxkSHcPrQ7r23aw9e7jzkdx5gmL/9EKYvXZTG0Vzg39K3/Sa6mzApq43H+9MF2tuwtYP7ERNq1ql+rR23jkjoytm8Hfrt+G9kHrfXDNG8iMlpEskQkR0RmnOH+biKyQURSReR9EYmqdX+oiOwVkT80Xur/9ssRMXQIDWTmsjQbdGyMmz2xeisl5ZXMTU6wgYhnYQW18Shb9xfyuw3bXAVww/0VfPoycatAP+5/PYVya/0wzZSI+ALPAWOAeOBmEYmvddhi4K+qmgTMA56odf984AN3Zz2X4BZ+zBwXT/q+Qv7xxS4noxjTpH2Re4Q3v9nLXVf3oGdEK6fjeCwrqI3HON3qEdbSn3nJiQ3++cNbtWB+ciJb9hbwpw+2N/jnN8ZLDAJyVDVXVcuA14DkWsfEAxtcH2+seb+IXApEAusaIes5je3bgaG9wlm0Nov8E6VOxzGmySmvrGLW8nQ6t27JPcNtIOK5WEFtPMYfN24nfV8hCyb2pW1wgFue44akjoxL6sjvNmxj6/5CtzyHMR6uM1Bzzrk8176aUoDJro9vBEJEpJ2I+ABPA9PdnrIORIS5yQmUlFeycI0NUDSmob366U6yDp5g1vh4Wgb4Oh3Ho1lBbTxC+r4Cnn1vG8n9OzE6sYNbn2teciJhLf2t9cM0V2dqgKzdhDwNGCYi3wDDgL1ABfBzYLWqnnMSaBG5S0Q2i8jm/Pz8hsh8Vj0jWnHX1T144+s8Nu086tbnMqY5OVBQwjPrsxneO4KR8ZFOx/F4VlAbx5VVVDFtSSptggOYMz7B7c/XNjiABRP7krG/kOc25rj9+YzxMHlAlxq3o4B9NQ9Q1X2qOklVLwEece0rAK4A7hGRnVT3Wf9QRBbWfgJVfUFVB6rqwIiICDe9jP/v7uG96Ny6JTOXpdnUmMY0kAVvZ1BepcyZYAMR68IKauO4P2zMYev+Qh6/sS9t3NTqUdvoxA4k9+/EH97LIX1fQaM8pzEeYhMQIyLdRSQAuAlYUfMAEQl3tXcAPAS8DKCqt6hqV1WNpvos9l9V9TuzhDS2oAA/Zo2PJ/PACV79zAYoGnOxPsk5zKrU/fz8mp50axfsdByvYAW1cVTa3gKe25jDpEs6c30jX1KaMz6BNsEB3P96CmUVdlbLNA+qWgHcA6wFtgKvq2q6iMwTkQmuw64BskQkm+oBiI85EvYCjIyPZHjvCJ5Zn83BwhKn4xjjtcoqqpi5PI2ubYP42bCeTsfxGlZQG8eUVlQybUkK7YIDmN0IrR61tQkO4PEb+5J54AR/eG9boz+/MU5R1dWqGquqPVX1Mde+Waq6wvXxUlWNcR3zE1X9zhQaqvoXVb2nsbOfjYgwZ0ICZZVVPPb2VqfjGOO1Xvo4l9z8YuZOSCDQ3wYi1pUV1MYxz27IIfPACRZO7ktYkL8jGa6Pj2TSgM489/52tuRZ64cx3qxbu2D+d1hPVqTs49Pth52OY4zX2Xv8FM9uyKm+4hPX3uk4XsUKauOI1LzjPP/BdqZcGsW1cc6OHp49LoHwVgHcv+RbSisqHc1ijLk4/3tNT7q2DWLW8nRr5TLmAs1bmY6izBpfe60ncz5WUJtGV1pRyf2vpxDRqgUzxzn/QxsW5M/CSUlkHyzi9xus9cMYbxbo78ucCfHkHCri5U92OB3HGK+xMesQa9MP8otrY4hqE+R0HK9jBbVpdL99dxvbDhXxxOS+hLV0ptWjtuFx7Zl6aRTPv7+dlD3HnY5jjLkI18ZFcn18JL/fsI19x085HccYj1dSXsmcFen0iAjmzqt6OB3HK1lBbRrVN7uP8acPtvP9gV0Y3tuz+rMeHRdPZGgg9y9JoaTcWj+M8WazxsVTpcqCtzOcjmKMx/vTB7nsOnKSeRMSCfCz0rA+7KtmGk1JefWsHpGhgTwyro/Tcb4jrKU/CycnkXOoiGfezXY6jjHmInRpG8Q9w3uxessBPsx272qNxniz3UdO8sf3c7ghqSNDY8KdjuO1rKA2jeaZ9dlszy/myclJhAZ6RqtHbcNiI7h5UBde/DCXr3YdczqOMeYi3Hl1D7qHBzN7RboNODbmDFSVOSvT8fMRZt7g/Jgmb2YFtWkUX+06xgsf5XLzoK5cHev+pYgvxsNj+9AxrCXTrfXDGK/Wws+XuRMS2HG4mBc/zHU6jjEeZ33GQd7LPMSvroulQ1ig03G8mhXUxu1KyiuZviSFTmEteXhsnNNxzisk0J8nJyeRe7iYp9dlOR3HGHMRro6NYGzfDvxhYw57jp50Oo4xHuNUWSVzV2YQG9mKH18Z7XQcr2cFtXG7xWuzyD1c3eoR4qGtHrUNjQnnlsFdeenjHWzeedTpOMaYizBzXDw+IsxbZQMUjTntuY057D1+ivnJifj7Wjl4sewraNxq086j/PmTHdx6eVevG+zw0Ng+dG7dkmlLUjhVZq0fxnirjmEtuXdEjOvy9kGn4xjjuNz8Il74MJcbL+nM4B7tnI7TJLi1oBaR0SKSJSI5IjLjDPd3FZGNIvKNiKSKyFh35jGN61RZdatH59YteWiM583qcT6tWvjx1JQkdh45yVNrM52OY4y5CLdf2Z1e7Vsxe0W6jY0wzZqqMntFOi38fHjIC9owvYXbCmoR8QWeA8YA8cDNIlJ7COmjwOuqeglwE/BHd+Uxje+ptZnVxeiUJIJb+Dkdp16G9AznR1d045VPdvJF7hGn4xhj6inAz4d5yQnsOXqK59/f7nQcYxyzessBPtp2mPtHxtI+xAYiNhR3nqEeBOSoaq6qlgGvAcm1jlEg1PVxGLDPjXlMI/oi9wivfLKTH13RjSE9vavVo7YHx8TRtW0Q05emcrKswuk4xph6GtIznAn9OvH8B9vZdaTY6TjGNLqi0grmr8ogvmMot17ezek4TYo7C+rOwJ4at/Nc+2qaA9wqInnAauAXbsxjGsnJsgqmL02la9sgHhzj/ZeTggL8WDQlid1HT/LkGmv9MMabPXpDHwJ8fZi9Ih1VdTqOMY3q9xu2caCwhPkTE/GzgYgNyp1fTTnDvtq/vW4G/qKqUcBY4G8i8p1MInKXiGwWkc35+bbilad7ck0mu4+eZNGUJIICvLPVo7bBPdpx25XRvPrZLj7dftjpOMaYemofGsivr4/l/ax81mXYAEXTfGQfPMHLH+/g+wO7cGm3Nk7HaXLcWVDnAV1q3I7iuy0ddwCvA6jqZ0Ag8J3+AFV9QVUHqurAiAjPXhSkuft0+2Fe/WwXt10Z3eRGDj8wKo7odkE8sDSVolJr/TDGW/3oim7EdQhh3soMa+MyzYKqMnNZGsEt/HhgdG+n4zRJ7iyoNwExItJdRAKoHnS4otYxu4ERACLSh+qC2k5Be6ni0goeWJpaXXSO8v5Wj9paBviyeGo/9h4/xROrtzodxxhTT36+PsyfmMje46f4w3s5Tscxxu2Wf7uPL3Yc5YHRvWnXqoXTcZoktxXUqloB3AOsBbZSPZtHuojME5EJrsPuB+4UkRTgX8CP1ZravNYTa7ay9/gpFk3tR8sAX6fjuMXA6Lb8ZGh3/vHFbj7eZq0fxniry6LbMnlAFC9+lMv2/CKn4xjjNoUl5Sx4eyv9osK46bKuTsdpstzaka6qq1U1VlV7qupjrn2zVHWF6+MMVb1SVfupan9VXefOPMZ9Psk5zN8/380dV3bnsui2Tsdxq/tH9qZHRDAPvpHKiZJyp+MYY+ppxpg4Av19mb3cBiiapus367I5UlzK/ImJ+PqcaXibaQg2xNNctBMl5TywNJUe4cFMG9X0e7MC/atbP/YXnOJxa/0wxmtFhLRg+qjefJxzmLe37Hc6jjENLmNfIX/9bCe3DO5KUlRrp+M0aVZQm4v2+OpM9hdUt3oE+jfNVo/aBnRtw51X9+BfX+7hg2xr+zfGW90yuBsJnUKZvyrDBhubJqWqSpm5PI3WQQFMG9n0T3Y5zQpqc1E+zM7nX1/u5s6rejS7aXh+fV0svdq34sGlqRScstYPY7yRr48wf2IiBwtL+f2GbU7HMabBLP06j692HWPGmDhaBwU4HafJs4La1FthSTkz3kilZ0Qwv74+1uk4jS7Q35enp/Yjv6iUBasynI5jjKmnAV3bcNNlXXj54x1kHzzhdBxjLtrxk2UsXJPJpd3aMGVAlNNxmgUrqE29PbZqKwcKS1jcjFo9auvXpTU/G9aDJV/lsTHzkNNxjDH19MDoOFoF+jFzWZoNUDReb9HaLI6fLGN+ciI+NhCxUVhBberl/axD/HvzHn46rCeXdG1erR613Tsiht6RIcx4M5WCk9b6YYw3ahscwAOj4vhix1GWf1t7DTJjvEdq3nH++eVufjQkmvhOoU7HaTasoDYXrOBUOTPe2EJM+1b86roYp+M4roVf9awfh4vKmLsq3ek4xph6uumyLvTr0poFb2+l0KbENF6osqp6RcTwVi2aZSumk6ygNhds/qoM8otKWTy1Hy38mmerR219o8K4+5qevPn1XtZnHHQ6jjGmHnx8hPnJCRwpLuWZ9dlOxzHmgr22aTcpeQU8MrYPoYH+TsdpVqygNhfkvcyDLP0qj/8d1pN+XWxOy5ruuTaGuA4hPPzWFo4VlzkdxxhTD0lRrbllcFde/XQnGfsKnY5jTJ0dKSrlqXeyuLxHW5L7d3I6TrNjBbWps4KT1a0evSND+MWIXk7H8TgBfj48/b1+HCsuY85Ka/0wxltNH1k9zdis5WlUVdkAReMdnnwnk+LSCuYnJyJiAxEbmxXUps7mrkznSHEZT3/PWj3OJqFTGPdc24vl3+7jnbQDTscxxtRDWJA/M8bEsXnXMd74Os/pOMac11e7jvL65jzuGNqdmMgQp+M0S1ZQmzpZn3GQN7/Zy93De5HYOczpOB7t7uG9iO8YyqPLtnDUWj+MBxKR0SKSJSI5IjLjDPd3E5ENIpIqIu+LSJRrf38R+UxE0l33fb/x0zeOKQOiuLRbGxauybTZe4xHq6isYuaydDqEBnLvCJsowClWUJvzOlZcxsNvbaFPx1DuGW6tHufj71vd+lFwqpxZy9OcjmPMfxERX+A5YAwQD9wsIvG1DlsM/FVVk4B5wBOu/SeBH6pqAjAa+K2INMnBFNUDFBM5drKMxeuynI5jzFn9/fNdZOwvZOa4eIJb+Dkdp9mygtqc15yV6RwrLmPx1CQC/Oxbpi76dAzllyNiWJW6n7dT9zsdx5iaBgE5qpqrqmXAa0ByrWPigQ2ujzeevl9Vs1V1m+vjfcAhIKJRUjsgvlMoP7wimr9/sYsteQVOxzHmOw6dKOHpddlcFRPO2L4dnI7TrFl1ZM7pnbQDLP92H/dc24uETtbqcSF+NqwnfTuHMXN5GoeLSp2OY8xpnYE9NW7nufbVlAJMdn18IxAiIu1qHiAig4AAYLubcnqE+0bGEt6qBY/aAEXjgZ5YnUlpRRVzJyTYQESHWUFtzupocRmPLttCfMdQ7rZWjwvm52r9KCqpsOWMjSc507tu7W/OacAwEfkGGAbsBSr+8wlEOgJ/A25T1arvPIHIXSKyWUQ25+fnN1xyB4QG+vPI2D6k7DnOvzfvOf8DjGkkn+ce4a1v9nLX1T3oEdHK6TjNnhXU5qxmLU+j4FQ5T3+vH/6+9q1SH7GRIfzq+hjWpB1glbV+GM+QB3SpcTsK+K+1tlV1n6pOUtVLgEdc+woARCQUeBt4VFU/P9MTqOoLqjpQVQdGRHh/R0hy/04M7t6WJ9/JtIHGxiOUV1Yxa3kanVu3tBNeHsKqJHNGq7fsZ1Xqfu69NoY+HUOdjuPV7rqqB/26tGbm8jQOnShxOo4xm4AYEekuIgHATcCKmgeISLiInH5/eAh42bU/AHiL6gGLSxoxs6NEhPkTEykqqeCpdzKdjmMMf/lkJ9kHi5gzIYGWATaNrSewgtp8x+GiUh5dlkbfzmH87JqeTsfxen6+Pjw9NYmTZZU88pa1fhhnqWoFcA+wFtgKvK6q6SIyT0QmuA67BsgSkWwgEnjMtf97wNXAj0XkW9fWv3FfgTNiI0O4fWh3Xtu0h693H3M6jmnGDhSU8Nt3s7k2rj3X9WnvdBzjYgW1+Y5Zy9MoKqlg8VRr9WgovdqHMG1kLOszDrL8233nf4AxbqSqq1U1VlV7qupjrn2zVHWF6+OlqhrjOuYnqlrq2v93VfVX1f41tm+dfC2N6d4RMUSGtmDW8jQqbYCiccj8tzOoqFLmjLeBiJ7EqiXzX1al7mP1lgP88roYenew1ZYa0h1DezCga2tmr0jnUKG1fhjjbVq18GPmuHjS9hbyjy92OR3HNEMfbzvM26n7+fk1vejaLsjpOKYGK6jNf+SfKGXmsjT6RYXx06t7OB2nyfH1ERZP7UdJeSUPv7XFWj+M8UI39O3I0F7hLFqbZdNhmkZVWlHJrOVpdGsXxE+H2Xu0p7GC2gCgqjy6bAvFZZUsntoPP2v1cIseEa2YPqo37249xJtf73U6jjHmAokIc5MTKCmv5InVNkDRNJ6XPtpB7uFi5k5IINDfBiJ6mjpXTSIS7M4gxlkrUvaxNv0g910fS0yktXq4021Xduey6DbMWZnOgQJr/TDG2/SMaMWdV/Xgja/z2LTzqNNxTDOQd+wkz763jVEJkVzT2wYieqLzFtQiMkREMqgeDY6I9BORP7o9mWk0hwpLmLU8nUu6tubOq+wykrv5+giLpvSjvLKKGW+mWuuHMV7onmt70bl1S2YuS6Oi8jtr2xjToOatzEAQZo1PcDqKOYu6nKF+BhgFHAFQ1RSqp00yTYCq8vBbaZSUV7d6+PrYiOHGEB0ezIzRcbyflc+SzXlOxzHGXKCggOoBipkHTvDqZzZA0bjPxsxDrMs4yC9GVP8RZzxTnVo+VLX2equVbshiHLDs2728u/Ug00b2pqctXdqofnhFNIO7t2X+qgz2HT/ldBxjzAWqvvwewTPrszloM/cYNygpr2T2inR6RgTzk6F2BdmT1aWg3iMiQwAVkQARmYar/cN4t4OFJcxens6l3dpw+9DuTsdpdnxcrR+Vqjz4hrV+GONtRIQ54xMoq6zi8dX2tmga3v99sJ3dR08yPzmRAD+bLMCT1eV/52fA3UBnIA/oD/zcnaGM+6kqD725hbLKKhZNSbJWD4d0bRfEQ2Pi+GjbYV7bVPtCkDHnJiIDReQtEflaRFJFZIuIpDqdqzmJDg/mZ8N6svzbfXy6/bDTcUwTsutIMX98fzvj+3ViSK9wp+OY86hLQd1bVW9R1UhVba+qtwJ93B3MuNcbX+/lvcxDTB8VRw9r9XDULYO7MaRnOxasyiDv2Emn4xjv8g/gFWAyMB4Y5/rXNKKfX9OTLm1bMmt5OmUVNkDRXDxVZc6KdPx9hEfGWsnlDepSUD9bx33GSxwoKGHuynQGRbfltiHRTsdp9nx8hCcnJwHwwNJUqmxJY1N3+aq6QlV3qOqu05vToZqbQH9f5oxPIOdQEa98ssPpOKYJWJdxkI1Z+fz6+lg6hAU6HcfUwVkLahG5QkTuByJE5L4a2xygTjOKi8hoEckSkRwRmXGWY74nIhkiki4i/6zXqzB1pqrMeDOVikrlqSlJ+Firh0fo0jaIR26I59PtR/jHl7udjmO8x2wReUlEbhaRSac3p0M1RyP6RHJdn0h+t2Eb+wtskLGpv5NlFcxbmUHvyBB+ZCe9vMa5zlAHAK0APyCkxlYITDnfJxYRX+A5YAwQD9wsIvG1jokBHgKuVNUE4Ff1eA3mAizZnMf7Wfk8OLo30eG2Vo8nuXlQF66KCeeJ1VvZc9RaP0yd3Eb1uJbRVLd6nG77MA6YPT6eyipl/qoMp6MYL/bcxhz2Hj/F/ImJ+NuqxV7D72x3qOoHwAci8pd6XkIcBOSoai6AiLwGJAM1f9PcCTynqsdcz3moHs9j6mjf8VPMX5XB4O5t+eEV0U7HMbWICAsnJzHqmQ+ZvjSFf/7kcruCYM6nn6r2dTqEqdalbRD3DO/F0+uz+TA7n6tjI5yOZLzM9vwiXvgwl0kDOjOoe1un45gLUJc/fU6KyCIRWS0i753e6vC4zkDNaQvyXPtqigViReQTEflcREaf6ROJyF0isllENufn59fhqU1t6pqarVKVRVP6WaHmoTq3bsnMcX34PPcof/vcWmHNeX1e+8qfcdZdw3rQPTyY2SvSKa2wJRtM3akqs5enE+jvy0NjbCCit6lLQf0PIBPoDswFdgKb6vC4M1VstUdb+QExwDXAzcBLItL6Ow9SfUFVB6rqwIgI+4u/Pl7btIePth3moTFxdG0X5HQccw7fG9iFYbERLFyTyc7DxU7HMZ5tKPCta6yKTZvnAVr4+TJnQgI7Dhfz4oe5TscxXuTtLfv5OOcw00b2JiKkhdNxzAWqS0HdTlX/DJSr6geqejtweR0elwd0qXE7Cth3hmOWq2q5qu4AsqgusE0Dyjt2kgWrMhjSsx23DO7mdBxzHtWtH33x8xWmL02xWT/MuYym+nfmSGzaPI8xLDaCMYkd+MPGHBsPYeqkqLSC+asySOgUyq2X2/u0N6pLQV3u+ne/iNwgIpdQXRyfzyYgRkS6i0gAcBOwotYxy4DhACISTnULiP1J34BIr2JRAAAgAElEQVROt3oAPDnZZvXwFh3DWjJ7fAKbdh7jlU93Oh3HeC49y2YcNnNcPD4izLMBiqYOfvduNgcLS5k/MdEWWvNSdSmoF4hIGHA/MA14Cfj1+R6kqhXAPcBaqpcqf11V00VknohMcB22FjgiIhnARmC6qh6px+swZ/GPL3bzSc4RHr6hD13aWquHN5k8oDPXxrVn0dpMcvOLnI5jPNPbwCrXvxuoPiGxxtFEBoBOrVty74gY1mcc5L3Mg07HMR4s68AJXv5kJzdd1oUBXds4HcfU0zkLatfUdzGqWqCqaao6XFUvVdXaZ5rPSFVXq2qsqvZU1cdc+2adfrxWu09V41W1r6q+dtGvyPzHnqMneXz1Vob2CucHg7o6HcdcIBHhiUl9CfD1YfrSVCqt9cPU4vq9meT6N4bq2ZU+djqXqXb7ld3p1b4Vs1ekU1JuAxTNd6kqM5enERLoxwOj45yOYy7COQtqVa0EJpzrGOOZqqqUB5am4iPCk1OSELFLSN4oMjSQuckJfLXrGC9/bCuwmXNT1a+By5zOYaoF+PkwLzmBPUdP8fz7252OYzzQsm/38uWOozwwKo62wQFOxzEX4azzUNfwqYj8Afg38J8pB1y/uI2H+vsXu/gs9wgLJ/Wlc+uWTscxF2Fi/86s3nKAReuyGB7Xnl7tWzkdyXgIEbmvxk0fYABgc4t6kCE9w5nQrxPPf7CdSQM6062dLahlqhWcKuextzPp16U1N13W5fwPMB6tLj3UQ4AEYB7wtGtb7M5Q5uLsOlLME6szuTo2gu/bD6nXExEeuzGRoABfpi1JsdYPU1PNVWxbUN1LnexoIvMdj9zQhwBfH2avSEfVfn5NtWfWZ3OkuJQFyYk2YUATcN4z1Ko6vDGCmIZRVaVMX5qKn4/w5OS+1urRRLQPCWTuhAR++dq3vPhRLj8b1tPpSMYzZKjqkpo7RGQqsOQsxxsHRIYG8qvrYljw9lbWZRxkVEIHpyMZh6XvK+Cvn+3k1sHd6BsV5nQc0wBskfgm5tXPdvLljqPMHB9PxzBr9WhKJvTrxOiEDvxmXTbbDp5wOo7xDA/VcZ9x2I+HRBPXIYR5KzM4WVbhdBzjoKoqZeayNNoEBTBtZG+n45gGYgV1E7LjcDFPvpPJ8N4RTL20LlOFG28iIiy4MZFWgX7cvySFisoqpyMZh4jIGBF5FugsIr+vsf0FsGrNA/n5+jAvOZG9x0/x3MYcp+MYBy39Ko+vdx9nxpg4woL8nY5jGogV1E1EZZUyfUkK/r4+PDHJZvVoqsJbtWB+ciKpeQX8yZY1bs72AZuBEuCrGtsKYJSDucw5DOrelkkDOvPCh7lst7nlm6XjJ8tY+E4mA7u1YfIAO/HVlJy3h1pEJp1hdwGwRVUPNXwkUx+vfLKDzbuO8fTUfnQIC3Q6jnGjG5I6sjqtI799N5sRfdoT1yHU6UimkalqCpAiIv9U1fLzPsB4jIfG9GF9xkHmrEjnr7cPspMfzcxTa7MoOFXO/Ik2ELGpqcsZ6juoXh3xFtf2InAf8ImI/I8bs5k62p5fxKK1WVzXpz2TBnR2Oo5pBPOTEwlr6c/9r6dQbq0fzdkgEVkvItkikisiO0TELl14sIiQFkwb2ZuPth1m9ZYDTscxjShlz3H+9eVufnRFNH062omQpqYuBXUV0EdVJ6vqZCAeKAUGAw+6M5w5v9OtHoH+vjx+o83q0Vy0DQ5gwcRE0vcV2oIRzdufgd8AQ6le0GUgtrCLx7v18m4kdApl/qoMikqt5b05qKyqXhExvFULfn19jNNxjBvUpaCOVtWDNW4fAmJV9Shglxod9uePc/l693HmTkigfai1ejQnoxM7MqFfJ36/YRvp+wqcjmOcUaCqa1T1kKoeOb2d70EiMlpEskQkR0RmnOH+biKyQURSReR9EYmqcd+PRGSba/tRQ7+g5sDXR5iXnMiBwhKe3bDN6TimEfzry92k5hXw6A19CAm0gYhNUV0K6o9EZJXrl+iPgOXAhyISDBx3bzxzLjmHTrB4XTYj4yNJ7t/J6TjGAXMnJNA6KIBpS1Ipq7DWj2Zoo4gsEpErRGTA6e1cDxARX+A5YAzVVxxvFpH4WoctBv6qqklUL+r1hOuxbYHZVF+hHATMFpE2DfuSmodLu7Xh+wO78OePd5Bt02A2aUeKSlm0NosrerRjQj97r26q6lJQ3w38BegPXAL8FbhbVYtt0RfnVFRWcf+SVIICfFlwY6K1ejRTbYIDePzGRLbuL+QPNhVXczSY6jaPx6n7SraDgBxVzVXVMuA1vru6YjywwfXxxhr3jwLWq+pRVT0GrAdGX/SraKYeHBNHcAs/Zi5LsxUUm7CFazIpLq1gXnKCvVc3YXVZKVGBpa7NeIgXP9pByp7j/P7mS2gfYq0ezdnIhA5MuqQzz23MYWR8JImdbdWt5qKeJzU6A3tq3M6jujCvKQWYDPwOuBEIEZF2Z3msjYSup7bBATwwujePvJXGipR9JPe3L2VTs3nnUZZ8lcdPh/UgJjLE6TjGjc57hlpEJrl65QpEpFBETohIYWOEM2eWffAEz6zPZkxiB8YndXQ6jvEAs8cn0C44gPtfT6G0otLpOKaRiEikiPxZRNa4bseLyB3ne9gZ9tU+PToNGCYi3wDDgL1ULxhTl8ciIneJyGYR2Zyfn3/e19Gc3XRZV/pFhbHg7a0UltiwpKakorKKmcvT6RgWyL3X2kDEpq4uLR9PARNUNUxVQ1U1RFVtvheHVFRWMW1JCq0C/Zg/0Vo9TLWwIH+emNSXrIMneHaDtX40I38B1gKnGzOzgV+d5zF5QJcat6OoXijmP1R1n6pOUtVLgEdc+wrq8ljXsS+o6kBVHRgREVH3V9MM+foI8ycmcriolGfWZzsdxzSgv32+i637C5k1Lp7gFudtCDBeri4F9UFV3er2JKZO/vRhLql5BcxPTiS8VQun4xgPMqJPJFMujeL5D7aTssfGCzcT4ar6OtXTm6KqFcD5LlFsAmJEpLuIBAA3Ub3C4n+ISLiInH5/eAh42fXxWmCkiLRxDUYc6dpnLkJSVGt+MKgrr366k4x9dgG4KThUWMJv1mVzVUw4oxM7OB3HNIK6FNSbReTfInKzq/1j0llWTzRulnmgkN++m80NSR25wVo9zBnMHBdPRKsWTFuSQkm5tX40A8Wu3mYFEJHLqV7J9qxcRfc9VBfCW4HXVTVdROaJyATXYdcAWSKSDUQCj7keexSYT3VRvgmY59pnLtL0Ub1pHRTArOVpVFXZAEVv9/jqrZRWVDEv2a4kNxd1KahDgZNUn4kY79rGuTOU+a5yV6tHaKA/8yYkOB3HeKiwlv4snNyXbYeK+O27Nr9tM3Af1WeXe4rIJ1TPwvSL8z1IVVeraqyq9lTV08XyLFVd4fp4qarGuI75iaqW1njsy6ray7W94p6X1fy0Dgpgxpg4Nu86xhtf5zkdx1yEz7YfYdm3+/jpsB50Dw92Oo5pJHWZ5eO2xghizu3597eTtreQ528ZQDtr9TDncE3v9tx0WRde+HA7IxMiGdDVpgluqlT1axEZBvSmesBglqrayDYvNWVAFK99uZuFazIZGd+BsCBbAMTblFdWMWt5GlFtWvLza3o5Hcc0orOeoRaRB1z/Pisiv6+9NV5Ek7GvkGff28aEfp0Y09daPcz5PXJDHzqEBjLdWj+aNNciLWOBEVRfRfyFiNznbCpTXz6uAYrHTpaxeF2W03FMPbzyyQ62HSpizvgEWgb4Oh3HNKJztXycHoi4GfjqDJtpBGUV1a0eYS0DmGutHqaOQgL9eXJKEtvzi/mNzRzQlK0Efgy0A0JqbMZLJXQK44dXRPP3L3axJe+c7fDGw+wvOMVv393GiLj2XBcf6XQc08jO2vKhqitd/77aeHFMbc9tzCFjfyEv/M+ltAkOcDqO8SJXxUTwg8FdefGjXEYlRHJpt7ZORzINL8q1PLhpQu4bGcuq1P08ujyNt/53CD4+NqjNGyxYtZXKKmWOnfxqluqysEusiLwgIutE5L3TW2OEa+7S9hbw3MYcbrykMyMTbNodc+EeHtuHTmEtmbYklVNl1vrRBK0RkZFOhzANKzTQn0duiCNlz3H+vXnP+R9gHPdhdj5vb9nP3cN70aVtkNNxjAPqMsvHEuAb4FFgeo3NuNHpVo82wQHMHh/vdBzjpVq18GPRlCR2HC5m0VrryWyCPgfeEpFTtpJt0zKxf2cGdW/Lk+9kcrS4zOk45hxKKyqZvSKd6HZB3HV1D6fjGIfUpaCuUNXnVfVLVf3q9Ob2ZM3cs+9tI/PACZ64sS+tg6zVw9TfkF7h/M/l3Xjl0x18ucOmDG5ingauAIJsJdumRUSYn5zIiZIKFq3NdDqOOYeXPtrBjsPFzE1OJNDfBiI2V3UpqFeKyM9FpKOItD29uT1ZM7Ylr4A/vr+dyQOibGCDaRAzxsQR1aYl05emcLKswuk4puFsA9JU1VYCaYJ6dwjh9iujeW3THr7ZfczpOOYM9hw9ybPvbWNMYgeGxUY4Hcc4qC4F9Y+obvH4lP8/w8dmd4ZqzkorKrl/ybeEtwpglrV6mAYS3MKPRVP6sevISZ56x1o/mpD9wPsi8pCI3Hd6czqUaTi/vC6W9iEtmLk8jUpbQdHjzFuVgSDMHGfv183dOQtqEfEBblXV7rU2axJyk9+9u43sg0UsnJREWEub1N80nMt7tOPHQ6L5y6c7+Wz7EafjmIaxA9gABGDT5jVJrVr48egN8aTtLeSfX+xyOo6p4b3Mg6zPOMi9I2Lo1Lql03GMw865UqKqVonIYqp79Iybpew5zv99sJ3vDYxieFx7p+OYJuiB0b15P+sQ05emsPZXVxPc4ryLpRoPpqpzAUQkpPqmFjkcybjBuKSOvLZpN4vWZjGmb0fCbbVcx5WUVw9E7NW+FXcM7e50HOMB6tLysU5EJovIBU+EKSKjRSRLRHJEZMY5jpsiIioiAy/0OZqKkvJK7l+SQmRoII/apSPjJkEBfiya2o+9x0/xxJqt53+A8Wgikigi3wBpQLqIfCUiNgluEyMizJ2QyKnySp5YbQMUPcHz729nz9FTzEtOIMCvLqWUaerq8l1wH9VT55VeyLRMriVxnwPGAPHAzSLynUrRdWblXuCLC0rexDzzbjY5h4pYODmJ0EBr9TDuc1l0W26/sjt//3w3n+QcdjqOuTgvAPepajdV7QbcD7zocCbjBr3at+InV/Xgja/z2LTTZutx0q4jxTz/wXYm9OvEkJ7hTscxHuK8BbVrGiYfVQ24wGmZBgE5qpqrqmXAa0DyGY6bDzwFlFxQ8ibk693HePHDXG66rIuNEjaNYtrI3vQID+aBpamcKCl3Oo6pv2BV3Xj6hqq+DwQ7F8e40y+u7UWnsEBmLkujorLK6TjNkqoye0U6Ab4+PHJDH6fjGA9Sp+sUItJGRAaJyNWntzo8rDNQc4mnPNe+mp/3EqCLqq6qc+ImpqS8kmlLUugQGmg/nKbRtAzwZdHUfuwvOMXjdgnZm+WKyEwRiXZtj1I9UNE0QUEBfswaH0/mgRO8+pkNUHTC2vSDvJ+Vz6+uiyEyNNDpOMaD1GXp8Z8AHwJrgbmuf+fU4XOfqef6P3P+uGYQeYbqS5Tny3CXiGwWkc35+fl1eGrv8fS6LHLzi3lqSj9CrNXDNKJLu7Xhzqt68K8vd/NhdtP6uWpGbgcigDeAN4Fw4MdOBjLuNSqher7jZ9Znc6iw2V7YdcTJsgrmrUwnrkMIPx4S7XQc42Hqcob6l8BlwC5VHQ5cAtTl3TcP6FLjdhSwr8btECCR6jlUdwKXAyvONDBRVV9Q1YGqOjAioum0RHy16ygvfbyDHwzuytAY68Myje/X18fSMyKYB99IpdBaP7xRT6p/z/oA/sAIqk+AmCaqeoBiAmUVVTy22gYWN6Zn38thX0EJ8ycm4udrAxHNf6vLd0SJqpYAiEgLVc0EetfhcZuAGBHpLiIBwE3AitN3qmqBqoararSqRgOfAxNUtVksGnOqrJJpS1LpFNaSh8daq4dxRqC/L09/rz8HC0tYsCrD6Tjmwv0DeBmYBIxzbeMdTWTcLjo8mJ8N68Hyb/fx6XYbWNwYcg4V8dJHuUweEMVl0bZYtPmuuhTUeSLSGlgGrBeR5fz3meYzUtUK4B6qW0S2Aq+rarqIzBORCRcTuilYtDaLHYeLWTQliVY2F7BxUP8urfnpsJ68vjmPjVmHnI5jLky+qq5U1R2quuv05nQo434/H96LLm1bMmt5OuU2QNGtqgciphHo78tDY+OcjmM8VF1m+bhRVY+r6hxgJvBnYGJdPrmqrlbVWFXtqaqPufbNUtUVZzj2muZydvrLHUd55dMd/M/l3RjSy1o9jPN+dV0MsZGtmPFGKgUnrfXDi8wWkZdE5GYRmXR6czqUcb9Af1/mjE8g51ARL39s41DdaVXqfj7JOcL0Ub1tUR1zVnWd5WOoiNymqh8An1Frtg5TdyfLKpi+NIWoNi2ZMcb+0jWeoYWfL4un9uNwURnzrPXDm9wG9AdGU93qMZ7qtg/TDIzoE8l1fdrzuw3b2F9wyuk4TVJRaQUL3s4gsXMotwzu5nQc48HqMsvHbOBB4CHXLn/g7+4M1ZQ99U4Wu46cZNGUfrbss/EoSVGt+fk1PXnj6zzezTjodBxTN/1cA7Z/pKq3ubbbnQ5lGs/s8QlUVikLVtkARXf47fpsDp0oZX5yIr4+F7xgtGlG6nKG+kZgAlAMoKr7qJ6hw1ygz3OP8JdPd/LjIdFc3qOd03GM+Y5fXBtDXIcQHnprC8dPljkdx5zf52dagdY0H13aBnHP8F68vWW/TX/ZwLIOnOCVT3dy02VduKRrG6fjGA9Xl4K6TFUV1xzSImKrcNVDcWl1q0e3dkE8MLouk6QY0/gC/HxYPLUfx4rLmLvSWj+8wFDgWxHJEpFUEdkiIqlOhzKN686rexDdLojZK9Iprah0Ok6ToKrMXJ5GSKAfD4yy9kxzfnUpqF8XkT8BrUXkTuBd4EX3xmp6Fq7JJO/YKRZN6UdQgLV6GM+V2DmMu4f34q1v9rI2/YDTccy5jQZigJH8//5pmzavmQn092VuciI7Dhfz0kc2QLEhvPXNXr7ccZQHR8fRJjjA6TjGC9Rllo/FwFKqV+LqDcxS1WfdHawp+TTnMH/7fBe3DenOoO42f6XxfHcP70V8x1AeeWsLR4ut9cNT1Zwqz6bNa96GxUYwJrEDz763jT1HTzodx6sVnCrn8dVb6d+lNd8f2OX8DzCGOs7yoarrVXW6qk5T1fXuDtWUFJVWMH1pKt3Dg5k+ylo9jHc43fpRcKqc2SvSnY5jjKmDmePiEcRm6rlIv1mXxdHiMhZMTMTHBiKaOjprQS0iJ0Sk8AzbCREpbMyQ3uzx1VvZV3CKxVOTaBng63QcY+osvlMo914bw8qUfazest/pOMaY8+jUuiX3johhfcZB3su0mXrqI21vAX/7fBe3Xt6NxM5hTscxXuSsBbWqhqhq6Bm2EFUNbcyQ3uqjbfn884vd/GRody7tZq0exvv87JqeJHYOZeayNI4UlTodxxhzHncM7U7PiGDmrMigpNwGKF6IqqrqgYhtgwO4f6RdUTYXpk4tH+bCnSgp58GlqfSICLYfTOO1/H19eHpqfwpLypm13Fo/jPF0AX4+zE9OZPfRk/zfB9udjuNVlny1h292H+ehMX0Ia+nvdBzjZaygdpPHV2/lQGEJi6f2I9DfWj2M9+rdIYRfXRfL21v2syp1n9NxjDHnMaRXOOP7deKP729n15Fip+N4hWPFZSxck8ll0W2YNMAWgzYXzgpqN/ggO59/fbmHO6/uwQCbDN40AT+9ugf9osKYuSyN/BPW+uHtRGS0a+7qHBGZcYb7u4rIRhH5xjW/9VjXfn8RedU13/VWEXnou5/deIJHxvbB30eYsyKd6qUkzLk8tTaLwpIK5k9MRMQGIpoLZwV1AyssKWfGG6n0at+KX18X63QcYxqEn2/1rB/FZZU8umyLvUF7MRHxBZ4DxgDxwM1nWG3xUeB1Vb0EuAn4o2v/VKCFqvYFLgV+KiLRjZHbXJgOYYH8+vpYNmblsz7DBiiey7d7jvPapt38eEg0cR1siJipHyuoG9iCVRkcOlHK09bqYZqYmMgQ7rs+lrXpB1mRYq0fXmwQkKOquapaBrwGJNc6RoHTlUUYsK/G/mAR8QNaAmWAzfrkoX40JJrekSHMXZnBqTIboHgmlVXKzGVpRLRqwa+ui3E6jvFiVlA3oI2Zh3h9c1715fEurZ2OY0yDu/OqHlzStTWzlqdzqLDE6TimfjoDe2rcznPtq2kOcKuI5AGrgV+49i8FioH9wG5gsaoedWtaU2/+vj7MS05g7/FT/GHjNqfjeKR/frmbLXsLeHRcPCGBNhDR1J8V1A2k4GQ5M95MJTayFb+0v3JNE+XrIyye2o+S8koefstaP7zUmRpEa/9H3gz8RVWjgLHA30TEh+qz25VAJ6A7cL+I9PjOE4jcJSKbRWRzfn5+w6Y3F2Rwj3ZMuqQzL3yYS25+kdNxPMrholIWvZPJFT3aMT6po9NxjJezgrqBzFuVweGiMp6e2p8WftbqYZqunhGtmD6qN+9uPcRb3+x1Oo65cHlAzfWUo/j/LR2n3QG8DqCqnwGBQDjwA+AdVS1X1UPAJ8DA2k+gqi+o6kBVHRgREeGGl2AuxENj+xDo58tsG6D4XxauyeRUeSXzJybYQERz0aygbgDvZhzkja/z+Pk1PekbZSsrmabvtiu7M7BbG+asSOdAgbV+eJlNQIyIdBeRAKoHHa6odcxuYASAiPShuqDOd+2/VqoFA5cDmY2W3NRLREgL7h8Zy0fbDrN6ywGn43iEzTuPsvSrPO4Y2oNe7UOcjmOaACuoL9Lxk2U89NYW4jqE8ItrrdXDNA++PsKiqf0oq6zioTdT7ayXF1HVCuAeYC2wlerZPNJFZJ6ITHAddj9wp4ikAP8CfqzV/8nPAa2ANKoL81dUNbXRX4S5YLde3o34jqHMX5VBcWmF03EcVVFZxaPL0ugUFsi9I3o5Hcc0EVZQX6S5KzM4VlzG4qn9CPCzL6dpPrqHB/PAqDg2ZuWz5Ks8p+OYC6Cqq1U1VlV7qupjrn2zVHWF6+MMVb1SVfupan9VXefaX6SqU1U1QVXjVXWRk6/D1J2frw/zJyZyoLCE329o3gMU//rZLjIPnGDW+HiCAvycjmOaCKsAL8La9AO89c1e7h7ei8TO1uphmp8fD4lmUHRb5q/MYH/BKafjGGPO4dJubfjewCj+/PEOsg+ecDqOIw4VlvCb9dkMi41gVEIHp+OYJsQK6no6VlzGI2+lEd8xlLuH2yUj0zz5+AiLpiZRUaU8+IbN+mGMp3twdBzBLfyYtTytWf68PrZ6K2UVVcyZYAMRTcOygrqeZq9I5/hJa/Uwplu7YB4aG8eH2fn8e9Oe8z/AGOOYdq1a8MDo3nyee7TZLdD06fbDLP92Hz8b1oPu4cFOxzFNjFWC9bBmy35WpOzj3hExxHeyZUqNuXVwN67o0Y4Fb28l79hJp+MYY87hpsu6khQVxoK3t1JYUu50nEZRVlHFrOXpdGnbkp/bVWXjBlZQX6AjRaU8uiyNxM6h/O81PZ2OY4xH8PERnpqSRJUqD75hs34Y48l8fYQFExM5XFTKb9c3jwGKr3yyg5xDRcwZn0Cgv60VYRqeFdQXaNaKdApLylk8tR/+vvblM+a0Lm2DeHhsHz7JOcI/vtjtdBxjzDkkRbXmB4O68upnO9m6v9DpOG61v+AUv9uwjev6RDKiT6TTcUwTZRXhBViVuo+3U/fzq+tiietgrR7G1HbL4K4M7RXO46u3sueotX4Y48mmj+pNWEt/Zi5Lo6qq6V5Vmr8qg8oqZfb4eKejmCbMCuo6OlxUyqzl6SRFhfHTq3s4HccYjyQiLJzcFx8RHlia2qTfpI3xdq2DApgxOo7Nu47x5jd7nY7jFh9m57N6ywHuGd6LLm2DnI5jmjArqOtAVZm5LI2ikgqentoPP2v1MOasotoE8egNffgs9wh//2KX03GMMecw5dIoBnRtzROrt1JwsmkNUCytqGT2inS6hwdz1zA7EWbcyyrDOliZup81aQf49fWxxESGOB3HGI/3/cu6cHVsBE+szmTXkWKn4xhjzsLHR5g/MZFjJ8tYvC7L6TgN6sUPc9lxuJi5ExJo4WcDEY17WUF9HodOlDBreRr9u7Tmzqu6Ox3HGK8gIjw5uS9+PsJ0a/0wxqMldArjh1dE8/cvdrElr8DpOA1iz9GT/GFjDmP7duDq2Ain45hmwK0FtYiMFpEsEckRkRlnuP8+EckQkVQR2SAi3dyZ50KpKo+8lcbJskoWW6uHMRekY1hLZo6P58sdR3n1s51OxzHGnMOvr4+lXXALZi5vGgMU567MwEeEmeNsIKJpHG6rEEXEF3gOGAPEAzeLSO3v7G+AgaqaBCwFnnJXnvpY/u0+1mccZNrIWHq1b+V0HGO8ztRLoxjeO4In38lkx2Fr/TDGU4W19OfhsXF8u+c4/97s3Suebth6kHe3HuTeETF0DGvpdBzTTLjzlOsgIEdVc1W1DHgNSK55gKpuVNXTc2t9DkS5Mc8FOVRYwuwV6Qzo2po7htpgBmPqQ0R4YlISAb4+TF+SQmUTOPNlTFN14yWdGRTdliffyeRYcZnTceqlpLySOSvT6dW+FbdfaW2apvG4s6DuDNT8MzfPte9s7gDWuDFPnakqD7+1hZLy6lYPXx9xOpIxXqtDWCBzJiSwedcxXvlkh9NxjDFnISLMm5jAiZIKnlqb6XScevnjxhz2HD3F/OREAvysTdM0Hnd+t52pCj3j6SkRuRUYCCw6y/13ichmEdmcn5/fgBHP7M2v9/Lu1kNMH9WbHhHW6mHMxbrxks5c1+afxesAABdGSURBVKc9i9ZmkXOoyOk4xpiziOsQym1Donlt0x6+2X3M6TgXZOfhYv7vg1yS+3fiip7tnI5jmhl3FtR5QJcat6OAfbUPEpHrgEeACapaeqZPpKovqOpAVR0YEeHe0boHCkqYszKdy6LbcJtdLvp/7d15fFTV+cfxz0NWCBDWIBIQkM0EECUqKLKIiiCr0ora1lpbf9raVloXqBuLVtFWra2tWm3tolZFK6AiokawisiiQAhLArKENci+CSHn98dc6jQmCiYzd+bO9/16zSv3nnvmznNyJifP3HvuHZEaYWb8ekQX0lOSuHmypn6IxLIbL+hAVr3QBYrx8rfqnOOuqUtJTa7FbYNO8TscSUCRTKjnAe3NrI2ZpQKjgKnhFczsNOBxQsn01gjGckycc4x9eTGHj5TzwEhN9RCpSVn105kwLJeP1+3kyfdW+x2OiFShbloyt12cQ8GG3TwbJ1/ONGPpZmatLGX0BR3Iqp/udziSgCKWUDvnyoAbgBnAMuAF59xSM5tgZkO9ag8AdYEXzewTM5taxe6i4sUFJeSvKOXWizrRukmGn6GIBNLQU09kQG4zfjtzJUVb9vgdjohUYUjX5px9cmMemLGCbXsrPXkcM/YfKmPCtEI6nVCPq3rG1N13JYFEdMa+c+5151wH59zJzrl7vLI7nXNTveXznXPNnHPdvMfQr95j5GzadYCJ0wo5s00jrurZ2q8wRALNzLh7eBcyUpO46cVFlB0p9zskEamEmTFhWGcOHD7CfdNj+wLFR94uZuOug9w9vLO+L0J8o3ceoaket760hLJyx29GnkotTfUQiZim9dKYOLwzi0p28fhsTf0QiVXtsupyTa+2TF5Qwvw12/0Op1LFW/fw5HurGdk9m7zWjfwORxKYEmrg+Xnrmb2ylLGDOtGqcR2/wxEJvMFdT2RQlxN4+K2VrNisqR8isepn/dtxYmY6t79SEHNnlJxz3DllKXVSkxgzsJPf4UiCS/iEesPOA9z92jJ6tm3Md87S3CuRaJk4rDP101O46cVFHI6xf9QiElInNZk7h+SwfPMe/j4nti5QnLZ4Ex+s+oybB3SkSd00v8ORBJfQCbVzjlsnL6bcOe4f2VVTPUSiqHHdNO4e3pklG3bx2Lur/A5HRKowIPcEendoyoMzV7J190G/wwFgz8HD3P1qIV1aZHKFDoZJDEjohPrZj9bxn+Jt/GrQKbRspKkeItE2sEtzhpx6Io+8U0Thxt1+hyMilTAzxg/N5VBZOfe8vszvcAB4+K0iSvd+zsThnXWLW4kJCZtQr9++n1+/toxe7Zpw5Vmt/A5HJGFNGJpLZu1UbnpxEYfKNPVDJBa1aZLBdX3aMuWTjcxZ9ZmvsSzfvJunP1jDqDNa0a1lA19jETkqIRPq8nLHrS8txsy479IumOnTrYhfGmakcs+IzhRu2s2j+cV+hyMiVbi+bzuyG9bmzikFvl334JzjjlcKqJ+ezC0DOvoSg0hlEjKhfmbuWj5Y9Rm3XXwK2Q011UPEbwNyT2B4txN5NL+Ygg27/A5HRCpROzWJcUNyKdq6l7++/6kvMby8cAPz1uxgzMBONMxI9SUGkcokXEK97rP93Dt9Oee2b8KoM1r6HY6IeMYNzaVhhqZ+iMSy83Oa0b9TFg+/VcSmXQei+tq7Dhzm3unLOK1VA77VXf+/JbYkVEJdXu64efIiksyYdGlXTfUQiSEN6qRy74guLN+8h9+/U+R3OCJShXFDczlS7rj71eheoPjbN1ewfd8hJg7rrLtyScxJqIT673PWMPfT7dwxOIcTG9T2OxwRqeD8nGZceno2f3x3FYtLdvodTmCZ2UVmtsLMis1sTCXbW5lZvpl9bGaLzWxQ2LauZjbHzJaa2RIzS49u9OK3lo3q8JN+7XhtySbeKyqNymsWbNjFPz9cy3d7nETnFplReU2R45EwCfWabfuY9MYK+nZsyrfysv0OR0SqcOeQHJrUTeWXLyzi87IjfocTOGaWBDwKDARygMvNLKdCtduBF5xzpwGjgD96z00G/glc55zLBfoCh6MUusSQa3u35aTGdbhrytKI/52Wlztuf6WARhlp/OJCXYgosSkhEuqjUz2Sk4x7L9FdPURiWWbtFO67tCtFW/fy8Fua+hEBZwLFzrnVzrlDwL+AYRXqOKC+t5wJbPSWLwQWO+cWATjnPnPO6VNPAkpPSWL80FxWb9vHk+9F9gLFF+av55P1O/nVoE5k1k6J6GuJfFMJkVD/9YM1zFuzg7uG5NI8U1M9RGJdv45ZfDsvm8dnrWLB2u1+hxM0LYD1YeslXlm4ccB3zKwEeB34qVfeAXBmNsPMFprZLZEOVmJX345ZXJR7Ar9/p4iSHfsj8ho79h1i0hvLObN1I0acVvFtKhI7Ap9QHz5Szt/nrOG8Tllcerr+GEXixe2Dc2ieWZvLHv+QsS8vifodBQKsslN0rsL65cDTzrlsYBDwDzOrBSQDvYArvZ8jzKz/l17A7Fozm29m80tLozPHVvxxx5AcDGPCtMKI7P/+GcvZfbCMCcNzdXZZYlrgE+qUpFpM+ck5uquHSJypn57Cv39yNlee1YrJC9bT54F3GT9tKaV7Pvc7tHhXAoTfcyybL6Z0HHUN8AKAc24OkA408Z47yzm3zTm3n9DR69MrvoBz7gnnXJ5zLq9p06YRaILEihYNavPT/u14s3AL+cu31ui+P163g3/NW8/VZ7em0wn1v/4JIj4KfEINodtxNa2X5ncYInKcsuqlM35YZ/Jv6suIbi34+5y19L4/n0lvLGfn/kN+hxev5gHtzayNmaUSuuhwaoU664D+AGZ2CqGEuhSYAXQ1szreBYp9gMgcmpS48cNebWnbNIO7pi7l4OGamVJ/pNxxx5QCsuqlceMFHWpknyKRlBAJtYjEt+yGdZg0sitv/aIPF+Y247FZqzh3Uj6/e6uIPQd1k4nj4ZwrA24glBwvI3Q3j6VmNsHMhnrVfgn8yMwWAc8B33chO4AHCSXlnwALnXOvRb8VEktSk2sxcVhn1m3fz2OzVtXIPp+du5aCDbu5/eIc6qYl18g+RSLJnKs4dS625eXlufnz5/sdhoj4aMXmPTw0cyVvLN1MgzopXNfnZL7X8yTqpMb2P14zW+Ccy/M7jmjSmJ04bnh2IW8WbmHm6N6c1DjjG+9n297POe8379IlO5N/XnOWpmuKr4513NYRahGJOx1PqMdj3+3OtBt60a1lA+6bvpze97/L0+9/qntXi/jk9otzSKlljJu6lOocrLv39eUcOHyE8UM7K5mWuKGEWkTiVpfsTJ6++kwmX9eTdlkZjJtWSL8H3uW5j9Zx+Ei53+GJJJQTMtO58fwO5K8oZWbhlm+0j3lrtvPSwhJ+eG5b2mXVreEIRSJHCbWIxL281o147kc9eOaHZ5FVP52xLy/h/Adn8e+PSzhSHl/T2kTi2ffPaU2HZnUZP62QA4eO72xR2ZFy7nilIHTnkPPaRShCkchQQi0igWBmnNOuCf/+8dk8dVUedVKTGf38Ii56eDbTl2yiXIm1SMSlJIUuUNyw8wCP5hcf13P/Nmctyzfv4Y7BOTF/PYRIRUqoRSRQzIz+pzTjtZ/24tErTqfcOa5/ZiFD/vAf8pdvrdbcThH5eme1bcyI01rwxOzVrC7de0zP2bL7IA/NXEnfjk0ZkNsswhGK1Dwl1CISSLVqGRd3bc6bo/vw4LdPZc/BMq5+eh4jH5vDB6u2+R2eSKCNHdSJtORa3HWMFyje89oyDh0pZ9wQfSOixCcl1CISaEm1jEtOz+btX/bh1yO6sHHnAa7481yu+POHLFi7w+/wRAIpq146v7ywA+8VbWN6weavrPtB8TamLtrIdX1OpnWTb367PRE/KaEWkYSQklSLK85qRf5NfblzcA4rt+zh0j99wNV//YiCDbv8Dk8kcL7T4yRymtdnwrRC9n1eVmmdQ2Xl3DGlgJaNavPjvidHOUKRmqOEWkQSSnpKEj/o1YbZt/Tj1os6sXDdTgb//j9c/88FFG3Z43d4IoGRnFSLicNz2bz7II+8XVRpnb+8/ymrSvcxfmgu6SlJUY5QpOYooRaRhFQnNZnr+57Me7f24+f92/Ne0TYufHg2o5//hDXb9vkdnkggdD+pEd/qns1T//n0Sx9YN+48wO/eKuKCnGac10kXIkp8U0ItIgmtfnoKoy/owHu39OPa3m2ZXrCJ/g/OYuzLi9mw84Df4YnEvTEDO5GRlswdUwr+5wLFia8W4nDcOTjHx+hEakZEE2ozu8jMVphZsZmNqWR7mpk9722fa2atIxmPiEhVGmakMnbgKcy+pR/f7XESLy3YQL8H3mXc1KVs3XPQ7/BE4lbjumncPKAjH67eztRFGwGYtbKU6QWbuaFfO1o2quNzhCLVF7GE2sySgEeBgUAOcLmZVfwYeg2wwznXDngImBSpeEREjkVWvXTGDc0l/+a+XNq9Bf/4cC2978/n3unL2LHvkN/hicSly89sRdfsTO5+bRmf7f2cu6YU0LZJBj/q3dbv0ERqRCSPUJ8JFDvnVjvnDgH/AoZVqDMM+Ju3PBnob7oBpYjEgBYNanPvJV15+xd9GNi5OU/MXs259+fz0MyV7D542O/wROJKUi1j4rDObNv7Od3vfos1n+1n/LBc0pJ1IaIEQyQT6hbA+rD1Eq+s0jrOuTJgF9A4gjGJiByX1k0yeOiybsy4sTfntm/C794u4txJ+cws3OJ3aCJx5dSWDbgsryUA55/SjHPbN/U5IpGakxzBfVd2pLni1yUdSx3M7FrgWoBWrVpVPzIRkePUoVk9/vSd7hRs2MVDM1fSponmfYocr7EDT6FhRipXn9Pa71BEalQkj1CXAC3D1rOBjVXVMbNkIBPYXnFHzrknnHN5zrm8pk31iVZE/NO5RSZPff8M2mXV8zsUkbiTWSeFWy/qRFa9dL9DEalRkUyo5wHtzayNmaUCo4CpFepMBa7ylkcC77jwe+qIiIiIiMS4iE35cM6VmdkNwAwgCfiLc26pmU0A5jvnpgJPAf8ws2JCR6ZHRSoeEREREZFIiOQcapxzrwOvVyi7M2z5IPCtSMYgIiIiIhJJ+qZEEREREZFqUEItIiIiIlINSqhFRERERKpBCbWIiIiISDUooRYRERERqQaLt9s+m1kpsPYbPLUJsK2Gw/FLkNoCwWqP2hK7YqE9JznnEurbqTRmA8FqCwSrPWpL7IqV9hzTuB13CfU3ZWbznXN5fsdRE4LUFghWe9SW2BW09gRdkPorSG2BYLVHbYld8dYeTfkQEREREakGJdQiIiIiItWQSAn1E34HUIOC1BYIVnvUltgVtPYEXZD6K0htgWC1R22JXXHVnoSZQy0iIiIiEgmJdIRaRERERKTGBT6hNrOLzGyFmRWb2Ri/46mMmbU0s3wzW2ZmS83s5155IzObaWZF3s+GXrmZ2SNemxab2elh+7rKq19kZlf51SYvliQz+9jMXvXW25jZXC+2580s1StP89aLve2tw/Yx1itfYWYDfGpHAzObbGbLvT7qGa99Y2ajvfdYgZk9Z2bp8dQvZvYXM9tqZgVhZTXWF2bW3cyWeM95xMwsWm2TL2jc9q1NGrNjs1/idtxOqDHbORfYB5AErALaAqnAIiDH77gqibM5cLq3XA9YCeQA9wNjvPIxwCRveRAwHTCgBzDXK28ErPZ+NvSWG/rYrl8AzwKveusvAKO85ceA673lHwOPecujgOe95Ryvz9KANl5fJvnQjr8BP/SWU4EG8dg3QAvgU6B2WH98P576BegNnA4UhJXVWF8AHwE9vedMBwZG+/2W6A80bvs2bqMxO+b6hTgft0mgMduXF43iG7EnMCNsfSww1u+4jiHuKcAFwAqguVfWHFjhLT8OXB5Wf4W3/XLg8bDy/6kX5TZkA28D5wGvem/2bUByxb4BZgA9veVkr55V7K/welFsR31vMLMK5XHXN97AvN4blJK9fhkQb/0CtK4wONdIX3jbloeV/089PaLWvxq3fRi3NWbHbL/E/bidKGN20Kd8HH0jHlXilcUs7/TMacBcoJlzbhOA9zPLq1ZVu2KpvQ8DtwDl3npjYKdzrsxbD4/tv3F723d59WOhPW2BUuCv3qnQJ80sgzjsG+fcBuA3wDpgE6Hf8wLis1/C1VRftPCWK5ZLdMXa++trBWTc1pgdElP9EtBxO5BjdtAT6srm0rioR3GMzKwu8BJwo3Nu91dVraTMfUV5VJnZYGCrc25BeHElVd3XbIuF9iQTOl31J+fcacA+QqeoqhKzbfHmqQ0jdLrvRCADGPgVccVsW47R8cYfL+0KurjqhyCM2xqzvyRW2pJo43Zcj9lBT6hLgJZh69nARp9i+UpmlkJoUH7GOfeyV7zFzJp725sDW73yqtoVK+09BxhqZmuAfxE6hfgw0MDMkiuJ7b9xe9szge3ERntKgBLn3FxvfTKhwToe++Z84FPnXKlz7jDwMnA28dkv4WqqL0q85YrlEl2x9v6qUoDGbY3ZX4ilfoFgjtuBHLODnlDPA9p7V8OmEpqgP9XnmL7Euyr1KWCZc+7BsE1Tgau85asIzdE7Wv4974rYHsAu77TJDOBCM2vofaq90CuLKufcWOdctnOuNaHf+TvOuSuBfGCkV61ie462c6RX33nlo7yrltsA7QldgBA1zrnNwHoz6+gV9QcKic++WQf0MLM63nvuaFvirl8qqJG+8LbtMbMe3u/ne2H7kujRuB3lsUFjdmz2iyeI43Ywx2y/Jm9H60HoqtGVhK5ovc3veKqIsReh0xSLgU+8xyBC857eBoq8n428+gY86rVpCZAXtq8fAMXe4+oYaFtfvrhivC2hP+Bi4EUgzStP99aLve1tw55/m9fOFfh09S7QDZjv9c8rhK4yjsu+AcYDy4EC4B+ErviOm34BniM0j/AwoaMT19RkXwB53u9mFfAHKlzYpEfU+lnjtn/t0pgdY/0Sz+N2Io3Z+qZEEREREZFqCPqUDxERERGRiFJCLSIiIiJSDUqoRURERESqQQm1iIiIiEg1KKEWEREREakGJdQSl8zsA+9nazO7oob3/avKXktERL4ZjdkSdLptnsQ1M+sL3OScG3wcz0lyzh35iu17nXN1ayI+ERH5gsZsCSodoZa4ZGZ7vcX7gHPN7BMzG21mSWb2gJnNM7PFZvZ/Xv2+ZpZvZs8SumE8ZvaKmS0ws6Vmdq1Xdh9Q29vfM+Gv5X170wNmVmBmS8zssrB9v2tmk81suZk9431rE2Z2n5kVerH8Jpq/IxGRWKExW4Iu+euriMS0MYQd7fAG2V3OuTPMLA1438ze9OqeCXR2zn3qrf/AObfdzGoD88zsJefcGDO7wTnXrZLXuoTQN3CdCjTxnjPb23YakAtsBN4HzjGzQmAE0Mk558ysQY23XkQkvmjMlkDSEWoJmguB75nZJ8BcQl9x2t7b9lHYwAzwMzNbBHwItAyrV5VewHPOuSPOuS3ALOCMsH2XOOfKCX0FcWtgN3AQeNLMLgH2V7t1IiLBojFbAkEJtQSNAT91znXzHm2cc0ePduz7b6XQPL7zgZ7OuVOBj4H0Y9h3VT4PWz4CJDvnyggdYXkJGA68cVwtEREJPo3ZEghKqCXe7QHqha3PAK43sxQAM+tgZhmVPC8T2OGc229mnYAeYdsOH31+BbOBy7w5f02B3sBHVQVmZnWBTOfc68CNhE49iogkMo3ZEkiaQy3xbjFQ5p0GfBr4HaFTdwu9i0xKCR1pqOgN4DozWwysIHQK8agngMVmttA5d2VY+b+BnsAiwAG3OOc2e4N7ZeoBU8wsndCRktHfrIkiIoGhMVsCSbfNExERERGpBk35EBERERGpBiXUIiIiIiLVoIRaRERERKQalFCLiIiIiFSDEmoRERERkWpQQi0iIiIiUg1KqEVEREREqkEJtYiIiIhINfw/MdD/XI5reAEAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 864x288 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Full size\n",
    "update_model_dir(learner, args.save_dir)\n",
    "sargs = save_args('first_run', args.save_dir)\n",
    "learner.fit(args.lr,args.epochs, cycle_len=args.cycle_len,\n",
    "            sampler=train_sampler,\n",
    "            wds=args.weight_decay,\n",
    "            use_clr_beta=args.use_clr,\n",
    "            loss_scale=args.loss_scale,\n",
    "            **sargs\n",
    "           )\n",
    "save_sched(learner.sched, args.save_dir)\n",
    "\n",
    "print('Finished!')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "learner.save('cifar10-resnext-lr1')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "learner.sched.plot()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "learner.lr_find()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "learner.sched.plot()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "learner.fit(1e-5,1, cycle_len=15,\n",
    "            sampler=train_sampler,\n",
    "            wds=args.weight_decay,\n",
    "            loss_scale=args.loss_scale,\n",
    "            **sargs\n",
    "           )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "TTA acc: 0.9389                              \n"
     ]
    }
   ],
   "source": [
    "\n",
    "if args.use_tta:\n",
    "    log_preds,y = learner.TTA()\n",
    "    preds = np.mean(np.exp(log_preds),0)\n",
    "    acc = accuracy(torch.FloatTensor(preds),torch.LongTensor(y))\n",
    "    print('TTA acc:', acc)\n",
    "    \n",
    "    with open(args.save_dir+'/tta_accuracy.txt', \"a\", 1) as f:\n",
    "        f.write(time.strftime(\"%Y-%m-%dT%H:%M:%S\")+f\"\\tTTA accuracty: {acc}\\n\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "TTA acc: 0.9402                              \n"
     ]
    }
   ],
   "source": [
    "\n",
    "if args.use_tta:\n",
    "    log_preds,y = learner.TTA()\n",
    "    preds = np.mean(np.exp(log_preds),0)\n",
    "    acc = accuracy(torch.FloatTensor(preds),torch.LongTensor(y))\n",
    "    print('TTA acc:', acc)\n",
    "    \n",
    "    with open(args.save_dir+'/tta_accuracy.txt', \"a\", 1) as f:\n",
    "        f.write(time.strftime(\"%Y-%m-%dT%H:%M:%S\")+f\"\\tTTA accuracty: {acc}\\n\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  },
  "toc": {
   "nav_menu": {
    "height": "266px",
    "width": "252px"
   },
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": "block",
   "toc_window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}