transformer.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "view-in-github"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/Arnouux/NLP_tweets/blob/main/transformer.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "# Initialization"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "## Libraries"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "### Install the ones that may not be present on the system"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 1,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "MCLczXcXt2fe",
        "outputId": "a8e79295-517b-46c1-e52d-e60539c0d02c"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Collecting datasets\n",
            "  Downloading datasets-1.16.1-py3-none-any.whl (298 kB)\n",
            "\u001b[K     |████████████████████████████████| 298 kB 5.2 MB/s \n",
            "\u001b[?25hRequirement already satisfied: importlib-metadata in /usr/local/lib/python3.7/dist-packages (from datasets) (4.8.2)\n",
            "Collecting huggingface-hub<1.0.0,>=0.1.0\n",
            "  Downloading huggingface_hub-0.2.1-py3-none-any.whl (61 kB)\n",
            "\u001b[K     |████████████████████████████████| 61 kB 445 kB/s \n",
            "\u001b[?25hRequirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.7/dist-packages (from datasets) (1.19.5)\n",
            "Requirement already satisfied: requests>=2.19.0 in /usr/local/lib/python3.7/dist-packages (from datasets) (2.23.0)\n",
            "Requirement already satisfied: tqdm>=4.62.1 in /usr/local/lib/python3.7/dist-packages (from datasets) (4.62.3)\n",
            "Requirement already satisfied: multiprocess in /usr/local/lib/python3.7/dist-packages (from datasets) (0.70.12.2)\n",
            "Collecting fsspec[http]>=2021.05.0\n",
            "  Downloading fsspec-2021.11.1-py3-none-any.whl (132 kB)\n",
            "\u001b[K     |████████████████████████████████| 132 kB 46.5 MB/s \n",
            "\u001b[?25hRequirement already satisfied: pandas in /usr/local/lib/python3.7/dist-packages (from datasets) (1.1.5)\n",
            "Requirement already satisfied: packaging in /usr/local/lib/python3.7/dist-packages (from datasets) (21.3)\n",
            "Collecting xxhash\n",
            "  Downloading xxhash-2.0.2-cp37-cp37m-manylinux2010_x86_64.whl (243 kB)\n",
            "\u001b[K     |████████████████████████████████| 243 kB 37.2 MB/s \n",
            "\u001b[?25hCollecting aiohttp\n",
            "  Downloading aiohttp-3.8.1-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl (1.1 MB)\n",
            "\u001b[K     |████████████████████████████████| 1.1 MB 32.7 MB/s \n",
            "\u001b[?25hRequirement already satisfied: dill in /usr/local/lib/python3.7/dist-packages (from datasets) (0.3.4)\n",
            "Requirement already satisfied: pyarrow!=4.0.0,>=3.0.0 in /usr/local/lib/python3.7/dist-packages (from datasets) (3.0.0)\n",
            "Requirement already satisfied: typing-extensions>=3.7.4.3 in /usr/local/lib/python3.7/dist-packages (from huggingface-hub<1.0.0,>=0.1.0->datasets) (3.10.0.2)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.7/dist-packages (from huggingface-hub<1.0.0,>=0.1.0->datasets) (3.4.0)\n",
            "Requirement already satisfied: pyyaml in /usr/local/lib/python3.7/dist-packages (from huggingface-hub<1.0.0,>=0.1.0->datasets) (3.13)\n",
            "Requirement already satisfied: pyparsing!=3.0.5,>=2.0.2 in /usr/local/lib/python3.7/dist-packages (from packaging->datasets) (3.0.6)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests>=2.19.0->datasets) (2021.10.8)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests>=2.19.0->datasets) (1.24.3)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests>=2.19.0->datasets) (2.10)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests>=2.19.0->datasets) (3.0.4)\n",
            "Requirement already satisfied: charset-normalizer<3.0,>=2.0 in /usr/local/lib/python3.7/dist-packages (from aiohttp->datasets) (2.0.8)\n",
            "Collecting frozenlist>=1.1.1\n",
            "  Downloading frozenlist-1.2.0-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl (192 kB)\n",
            "\u001b[K     |████████████████████████████████| 192 kB 51.0 MB/s \n",
            "\u001b[?25hCollecting yarl<2.0,>=1.0\n",
            "  Downloading yarl-1.7.2-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl (271 kB)\n",
            "\u001b[K     |████████████████████████████████| 271 kB 47.9 MB/s \n",
            "\u001b[?25hCollecting asynctest==0.13.0\n",
            "  Downloading asynctest-0.13.0-py3-none-any.whl (26 kB)\n",
            "Requirement already satisfied: attrs>=17.3.0 in /usr/local/lib/python3.7/dist-packages (from aiohttp->datasets) (21.2.0)\n",
            "Collecting async-timeout<5.0,>=4.0.0a3\n",
            "  Downloading async_timeout-4.0.2-py3-none-any.whl (5.8 kB)\n",
            "Collecting aiosignal>=1.1.2\n",
            "  Downloading aiosignal-1.2.0-py3-none-any.whl (8.2 kB)\n",
            "Collecting multidict<7.0,>=4.5\n",
            "  Downloading multidict-5.2.0-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl (160 kB)\n",
            "\u001b[K     |████████████████████████████████| 160 kB 39.1 MB/s \n",
            "\u001b[?25hRequirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.7/dist-packages (from importlib-metadata->datasets) (3.6.0)\n",
            "Requirement already satisfied: python-dateutil>=2.7.3 in /usr/local/lib/python3.7/dist-packages (from pandas->datasets) (2.8.2)\n",
            "Requirement already satisfied: pytz>=2017.2 in /usr/local/lib/python3.7/dist-packages (from pandas->datasets) (2018.9)\n",
            "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.7/dist-packages (from python-dateutil>=2.7.3->pandas->datasets) (1.15.0)\n",
            "Installing collected packages: multidict, frozenlist, yarl, asynctest, async-timeout, aiosignal, fsspec, aiohttp, xxhash, huggingface-hub, datasets\n",
            "Successfully installed aiohttp-3.8.1 aiosignal-1.2.0 async-timeout-4.0.2 asynctest-0.13.0 datasets-1.16.1 frozenlist-1.2.0 fsspec-2021.11.1 huggingface-hub-0.2.1 multidict-5.2.0 xxhash-2.0.2 yarl-1.7.2\n",
            "Collecting transformers\n",
            "  Downloading transformers-4.14.1-py3-none-any.whl (3.4 MB)\n",
            "\u001b[K     |████████████████████████████████| 3.4 MB 5.3 MB/s \n",
            "\u001b[?25hRequirement already satisfied: importlib-metadata in /usr/local/lib/python3.7/dist-packages (from transformers) (4.8.2)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.7/dist-packages (from transformers) (2.23.0)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.7/dist-packages (from transformers) (3.4.0)\n",
            "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.7/dist-packages (from transformers) (2019.12.20)\n",
            "Requirement already satisfied: huggingface-hub<1.0,>=0.1.0 in /usr/local/lib/python3.7/dist-packages (from transformers) (0.2.1)\n",
            "Collecting sacremoses\n",
            "  Downloading sacremoses-0.0.46-py3-none-any.whl (895 kB)\n",
            "\u001b[K     |████████████████████████████████| 895 kB 44.6 MB/s \n",
            "\u001b[?25hCollecting tokenizers<0.11,>=0.10.1\n",
            "  Downloading tokenizers-0.10.3-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl (3.3 MB)\n",
            "\u001b[K     |████████████████████████████████| 3.3 MB 50.1 MB/s \n",
            "\u001b[?25hRequirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.7/dist-packages (from transformers) (4.62.3)\n",
            "Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.7/dist-packages (from transformers) (21.3)\n",
            "Requirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.7/dist-packages (from transformers) (1.19.5)\n",
            "Collecting pyyaml>=5.1\n",
            "  Downloading PyYAML-6.0-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl (596 kB)\n",
            "\u001b[K     |████████████████████████████████| 596 kB 43.1 MB/s \n",
            "\u001b[?25hRequirement already satisfied: typing-extensions>=3.7.4.3 in /usr/local/lib/python3.7/dist-packages (from huggingface-hub<1.0,>=0.1.0->transformers) (3.10.0.2)\n",
            "Requirement already satisfied: pyparsing!=3.0.5,>=2.0.2 in /usr/local/lib/python3.7/dist-packages (from packaging>=20.0->transformers) (3.0.6)\n",
            "Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.7/dist-packages (from importlib-metadata->transformers) (3.6.0)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (1.24.3)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (2021.10.8)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (2.10)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (3.0.4)\n",
            "Requirement already satisfied: six in /usr/local/lib/python3.7/dist-packages (from sacremoses->transformers) (1.15.0)\n",
            "Requirement already satisfied: joblib in /usr/local/lib/python3.7/dist-packages (from sacremoses->transformers) (1.1.0)\n",
            "Requirement already satisfied: click in /usr/local/lib/python3.7/dist-packages (from sacremoses->transformers) (7.1.2)\n",
            "Installing collected packages: pyyaml, tokenizers, sacremoses, transformers\n",
            "  Attempting uninstall: pyyaml\n",
            "    Found existing installation: PyYAML 3.13\n",
            "    Uninstalling PyYAML-3.13:\n",
            "      Successfully uninstalled PyYAML-3.13\n",
            "Successfully installed pyyaml-6.0 sacremoses-0.0.46 tokenizers-0.10.3 transformers-4.14.1\n"
          ]
        }
      ],
      "source": [
        "!pip install datasets\n",
        "!pip install transformers"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "### Import the others"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 2,
      "metadata": {
        "id": "PD2pdQ4tt1Qa"
      },
      "outputs": [],
      "source": [
        "import functools\n",
        "import sys\n",
        "\n",
        "import datasets\n",
        "import matplotlib.pyplot as plt\n",
        "import numpy as np\n",
        "import re\n",
        "import torch\n",
        "import torch.nn as nn\n",
        "import torch.optim as optim\n",
        "import torchtext\n",
        "import tqdm\n",
        "import transformers"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "## Seed randomness"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 3,
      "metadata": {
        "id": "ZuQ4HRwSt1Qd"
      },
      "outputs": [],
      "source": [
        "seed = 0\n",
        "\n",
        "torch.manual_seed(seed)\n",
        "np.random.seed(seed)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "## Form the correct datasets"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "### Make the files accessible to the script"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "We use the drive to store the file, then grant access to Colab to the particular folder containing the relevant files"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 5,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "NcDVB4CAuPRU",
        "outputId": "d147c9d2-8683-4e7a-ae1e-c64baf6975da"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Mounted at /content/drive\n"
          ]
        }
      ],
      "source": [
        "# Specific to Colab\n",
        "from google.colab import drive\n",
        "drive.mount('/content/drive')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 6,
      "metadata": {
        "id": "UkvmTLa-uQiI"
      },
      "outputs": [],
      "source": [
        "# Specific to Colab\n",
        "!rm -rf \"twitter-datasets\"\n",
        "!cp -r \"drive/MyDrive/data\" \"twitter-datasets\"\n",
        "# WARNING: You may need to change this path (\"drive/MyDrive/data\"). \n",
        "# This path is valid if you have a folder \"data\" on your drive that contains all the necessary files.\n",
        "# The root folder of your Google Drive is \"drive/MyDrive\", then the path works as usual."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 7,
      "metadata": {
        "id": "idiUovAyt1Qf"
      },
      "outputs": [],
      "source": [
        "FILE_NEG = \"twitter-datasets/train_neg.txt\"\n",
        "FILE_POS = \"twitter-datasets/train_pos.txt\"\n",
        "# FILE_NEG_FULL = \"twitter-datasets/train_neg_full.txt\"\n",
        "# FILE_POS_FULL = \"twitter-datasets/train_pos_full.txt\"\n",
        "FILE_TEST = \"twitter-datasets/test_data.txt\""
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "### Generating the train entries dataset"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 8,
      "metadata": {
        "id": "wyb8CqDWt1Qg"
      },
      "outputs": [],
      "source": [
        "entries_dict = {\"label\":[], \"text\":[]}"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "#### Reading the files"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 9,
      "metadata": {
        "id": "jqrICqyJt1Qi"
      },
      "outputs": [],
      "source": [
        "with open(FILE_NEG, \"r\") as f:\n",
        "    content = f.read()\n",
        "content = content.split(\"\\n\")\n",
        "for entry in content:\n",
        "    if entry == \"\" or entry == \" \":\n",
        "        continue\n",
        "    entries_dict[\"text\"].append(entry)\n",
        "    entries_dict[\"label\"].append(0)\n",
        "with open(FILE_POS, \"r\") as f:\n",
        "    content = f.read()\n",
        "content = content.split(\"\\n\")\n",
        "for entry in content:\n",
        "    if entry == \"\" or entry == \" \":\n",
        "        continue\n",
        "    entries_dict[\"text\"].append(entry)\n",
        "    entries_dict[\"label\"].append(1)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "#### Splitting the dataset into two"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 10,
      "metadata": {
        "id": "am5E34dzt1Qj"
      },
      "outputs": [],
      "source": [
        "test_size = 0.25\n",
        "dataset = datasets.Dataset.from_dict(\n",
        "    entries_dict\n",
        ")\n",
        "train_valid_data = dataset.train_test_split(test_size=test_size)\n",
        "train_data = train_valid_data[\"train\"]\n",
        "valid_data = train_valid_data[\"test\"]"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "#### Transform the inputs from sentence of words to numerical values"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 145,
          "referenced_widgets": [
            "50c39ab0aa964670b19fc891d81f216f",
            "7a6e6022927c4bcc8f416ffb3cabfe38",
            "463bbaa6a226454781e6a9c26d39e39c",
            "cd1b47860cf94aca922f2362dc80aa32",
            "f7f1b5a617e547fc926e79a1b4832cee",
            "e1f34a29979d495188a6978a0bedcc93",
            "b179ad0042d1435789b4869d2d0b5d2a",
            "986b60c69548453396e76b4af9aef8d8",
            "1d450c26c0ba4111bc7c60376c7cee50",
            "df3678265f154c90bbaa0e93eaa0ce0b",
            "5d30bb5238244bc79fd823c44eac3679",
            "78f892242f924ece821ec4fd051fdafe",
            "98ff1d8cf2cc4e96bd3bf894b82640c1",
            "0d8ab1ca91374a698e3a03547b8da661",
            "c5d1a0bb1ba34862957403f49dd0188b",
            "1f9ea842afda46deb43a4cdc9df3e5d0",
            "a7dbafc0087748179dc6366dd3db8699",
            "6a98ec97cf934d56828db462f1237f36",
            "ee35e0788cb041fa8f56162f973335f2",
            "79b8ebb4962d4f498c966f382ee4f667",
            "e0632f7b600d48a19b0bca94d10b91fe",
            "af0e47a41c404036a01baae5b7dc473d",
            "41f8784804d34390b6c8220521fd8996",
            "267ada4ebb904acd83c8591a61e63221",
            "996c54b6fc0f421fa1d1cc31e27134e2",
            "c12c22bcfebf4362a6963ced56d11524",
            "552853a16c0e45a88185aee8302e96c9",
            "4dfebf84c7e342cfa0af9c73476b5676",
            "3470b025ac094eae8f3623897a51a713",
            "deef4bd3e8344c05ab6dc1f3a5bb4b10",
            "041969198b05460da479d33496de3ea3",
            "51c21294fb964c69ab6baaf1c5068a37",
            "ee5e1617624a4345bee82d6619bf3cf3",
            "6affffe403fc4fad8fa60276030a10f4",
            "b2ee84b674d746aaa1d2b66538176125",
            "68440f8633ca458cb1a07896ae6eb54a",
            "94a205668ba74caeb8942547a2c8ac25",
            "b73121d550ad4697b8d89ad80139c7a4",
            "6945c0767f1b44bcbb002e6a73427ad9",
            "1ae6dd8a46124404961f9c3439faf584",
            "53808da8556d4f3da2cf7430fc950810",
            "74ba5812390a4f4d8fbaf462e80c4bac",
            "07dde21456444417a82bad4df05736f2",
            "12d400a620844b47b218a09780a87d84"
          ]
        },
        "id": "hdkCW0uft1Qk",
        "outputId": "35f164dd-3aff-4513-a7db-ea36d3465f2d"
      },
      "outputs": [],
      "source": [
        "transformer_name = 'bert-base-uncased'\n",
        "\n",
        "tokenizer = transformers.AutoTokenizer.from_pretrained(transformer_name)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 12,
      "metadata": {
        "id": "WXho_tzit1Ql"
      },
      "outputs": [],
      "source": [
        "def tokenize_and_numericalize_data(example, tokenizer):\n",
        "    ids = tokenizer(example['text'], truncation=True)['input_ids']\n",
        "    return {'ids': ids}"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 81,
          "referenced_widgets": [
            "5fb0bf3b44e7447d8df112f87d97c7c1",
            "b115396ac40f4579a456902ea0c2c27c",
            "104c7170bf1b4ecdbf7927e4dc74aeff",
            "7f34e6470b374c258cc899d3d03666e3",
            "060a09256c5c4aca87f3ee2eaa097e1f",
            "ec712bff1ad549e49411961602aae93f",
            "8f9b723c723b4ce0a59f8d4514acd305",
            "c6b3d5719f7344a08e123505e2e65c1a",
            "104e485bc4ba4793accff6b757d8487b",
            "1180e4a7dbbb4200b88e1dac732faaa7",
            "fe3eba48e9de4195b933f377dcc64872",
            "10559067a3d04ade93e711d4114e10c3",
            "66d7bd3924864c528abfd1db275aff37",
            "60457332cc084713b99cfd738569afde",
            "380a869110f445a3b747378bf45e606d",
            "9bb6f10207534f0f8dc8dec1300b3e00",
            "0082375dcc8a446b80e5bd172a9e649d",
            "52b9d6a47ab140eba79fde8d0f214295",
            "dc67f5506eb1440f9dcc7934d46ed021",
            "69f4be1ffad34246a535b7e484baec57",
            "e6a0ff19e74d4ff89328b0d9968b83b4",
            "d42e08263a774255a7dcff046b8592ab"
          ]
        },
        "id": "MzeMxJAkt1Qm",
        "outputId": "421560f4-f48a-4601-fb57-4cf0267e75ca"
      },
      "outputs": [],
      "source": [
        "train_data = train_data.map(\n",
        "    tokenize_and_numericalize_data, \n",
        "    fn_kwargs={'tokenizer': tokenizer}\n",
        ")\n",
        "valid_data = valid_data.map(\n",
        "    tokenize_and_numericalize_data,\n",
        "    fn_kwargs={'tokenizer': tokenizer}\n",
        ")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 14,
      "metadata": {
        "id": "AT-O0Jj-t1Qn"
      },
      "outputs": [],
      "source": [
        "pad_index = tokenizer.pad_token_id"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 15,
      "metadata": {
        "id": "PyKw560Et1Qo"
      },
      "outputs": [],
      "source": [
        "train_data = train_data.with_format(\n",
        "    type=\"torch\",\n",
        "    columns=[\"ids\", \"label\"]\n",
        ")\n",
        "valid_data = valid_data.with_format(\n",
        "    type=\"torch\",\n",
        "    columns=[\"ids\", \"label\"]\n",
        ")"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "#### Batchs formation"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 23,
      "metadata": {
        "id": "422szw6bt1Qt"
      },
      "outputs": [],
      "source": [
        "def collate(batch, pad_index):\n",
        "    batch_ids = [i['ids'] for i in batch]\n",
        "    batch_ids = nn.utils.rnn.pad_sequence(\n",
        "        batch_ids,\n",
        "        padding_value=pad_index,\n",
        "        batch_first=True\n",
        "    )\n",
        "    batch_label = [i['label'] for i in batch]\n",
        "    batch_label = torch.stack(batch_label)\n",
        "    batch = {'ids': batch_ids,\n",
        "             'label': batch_label}\n",
        "    return batch"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 24,
      "metadata": {
        "id": "iVDs6xKct1Qt"
      },
      "outputs": [],
      "source": [
        "batch_size = 8\n",
        "\n",
        "collate = functools.partial(collate, pad_index=pad_index)\n",
        "\n",
        "train_dataloader = torch.utils.data.DataLoader(\n",
        "    train_data, \n",
        "    batch_size=batch_size, \n",
        "    collate_fn=collate, \n",
        "    shuffle=True)\n",
        "valid_dataloader = torch.utils.data.DataLoader(\n",
        "    valid_data,\n",
        "    batch_size=batch_size,\n",
        "    collate_fn=collate\n",
        ")"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "# Neural Network"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 125,
          "referenced_widgets": [
            "910f8c4e6ef041f4b23b2ef4be17590a",
            "ed956969e15c45a3818b4f175473a962",
            "31d606615b7e4ee28c3fb10b5421ac89",
            "5857deca493c4a289009903b8cca7eaa",
            "132a9aafe9d247038231fe510e7e7283",
            "85bc4b6579524db2bd47bbfc3c3ffe2f",
            "ece16fbcf31f4ed9afbf824aff0115dd",
            "7953f605bed04f27b1c1ec4767d49614",
            "7f8e663a518849f19f530e9a7e04f84e",
            "b2dffd1b7b984770a65817cc95c805d6",
            "736e5a4017314707a723646f8f492258"
          ]
        },
        "id": "AgBMskKbt1Qo",
        "outputId": "75b32ce6-af3a-4fa2-9239-2c7cbe45ca69"
      },
      "outputs": [],
      "source": [
        "transformer = transformers.AutoModel.from_pretrained(transformer_name)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "## Transformer Class"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "The class containing the neural network"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 17,
      "metadata": {
        "id": "7I-9sTxet1Qo"
      },
      "outputs": [],
      "source": [
        "class Transformer(nn.Module):\n",
        "    def __init__(self, transformer, output_dim, freeze):\n",
        "        super().__init__()\n",
        "        self.transformer = transformer\n",
        "        hidden_dim = transformer.config.hidden_size\n",
        "        self.fc = nn.Linear(hidden_dim, output_dim)\n",
        "        \n",
        "        if freeze:\n",
        "            for param in self.transformer.parameters():\n",
        "                param.requires_grad = False\n",
        "        \n",
        "    def forward(self, ids):\n",
        "        output = self.transformer(ids, output_attentions=True)\n",
        "        hidden = output.last_hidden_state\n",
        "        attention = output.attentions[-1]\n",
        "        cls_hidden = hidden[:,0,:]\n",
        "        prediction = self.fc(torch.tanh(cls_hidden))\n",
        "        return prediction"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "## Initialization"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 18,
      "metadata": {
        "id": "unnlFrcKt1Qq"
      },
      "outputs": [],
      "source": [
        "output_dim = len(train_data['label'].unique())\n",
        "freeze = False\n",
        "\n",
        "model = Transformer(transformer, output_dim, freeze)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "MMqvE7snt1Qq",
        "outputId": "24c836de-bdb0-41b2-82a5-ff789b6970ed"
      },
      "outputs": [],
      "source": [
        "def count_parameters(model):\n",
        "    return sum(p.numel() for p in model.parameters() if p.requires_grad)\n",
        "\n",
        "print(f'The model has {count_parameters(model):,} trainable parameters')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 20,
      "metadata": {
        "id": "f49sG3h0t1Qq"
      },
      "outputs": [],
      "source": [
        "lr = 1e-5\n",
        "\n",
        "optimizer = optim.Adam(model.parameters(), lr=lr)\n",
        "criterion = nn.CrossEntropyLoss()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 21,
      "metadata": {
        "id": "6bgWheZ8t1Qs"
      },
      "outputs": [],
      "source": [
        "device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 22,
      "metadata": {
        "id": "hk1R9ko_t1Qs"
      },
      "outputs": [],
      "source": [
        "model = model.to(device)\n",
        "criterion = criterion.to(device)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "## Train/Evaluate functions"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 25,
      "metadata": {
        "id": "hchejXdjt1Qt"
      },
      "outputs": [],
      "source": [
        "def get_accuracy(prediction, label):\n",
        "    batch_size, _ = prediction.shape\n",
        "    predicted_classes = prediction.argmax(dim=-1)\n",
        "    correct_predictions = predicted_classes.eq(label).sum()\n",
        "    accuracy = correct_predictions / batch_size\n",
        "    return accuracy"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 26,
      "metadata": {
        "id": "E8Co9kadt1Qt"
      },
      "outputs": [],
      "source": [
        "def train(dataloader, model, criterion, optimizer, device):\n",
        "\n",
        "    model.train()\n",
        "    epoch_losses = []\n",
        "    epoch_accs = []\n",
        "\n",
        "    for batch in tqdm.tqdm(dataloader, desc='training...', file=sys.stdout):\n",
        "        ids = batch['ids'].to(device)\n",
        "        label = batch['label'].to(device)\n",
        "        prediction = model(ids)\n",
        "        loss = criterion(prediction, label)\n",
        "        accuracy = get_accuracy(prediction, label)\n",
        "        optimizer.zero_grad()\n",
        "        loss.backward()\n",
        "        optimizer.step()\n",
        "        epoch_losses.append(loss.item())\n",
        "        epoch_accs.append(accuracy.item())\n",
        "\n",
        "    return epoch_losses, epoch_accs"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 27,
      "metadata": {
        "id": "9FzcLVrPt1Qt"
      },
      "outputs": [],
      "source": [
        "def evaluate(dataloader, model, criterion, device):\n",
        "    \n",
        "    model.eval()\n",
        "    epoch_losses = []\n",
        "    epoch_accs = []\n",
        "\n",
        "    with torch.no_grad():\n",
        "        for batch in tqdm.tqdm(dataloader, desc='evaluating...', file=sys.stdout):\n",
        "            ids = batch['ids'].to(device)\n",
        "            label = batch['label'].to(device)\n",
        "            prediction = model(ids)\n",
        "            loss = criterion(prediction, label)\n",
        "            accuracy = get_accuracy(prediction, label)\n",
        "            epoch_losses.append(loss.item())\n",
        "            epoch_accs.append(accuracy.item())\n",
        "\n",
        "    return epoch_losses, epoch_accs"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "## Training"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 28,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "6cYjpRQHt1Qt",
        "outputId": "5eb2b8ae-ad8b-437a-ecea-889ca9747f34"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "training...: 100%|██████████| 18750/18750 [1:06:29<00:00,  4.70it/s]\n",
            "evaluating...: 100%|██████████| 6250/6250 [06:08<00:00, 16.95it/s]\n",
            "epoch: 1\n",
            "train_loss: 0.316, train_acc: 0.860\n",
            "valid_loss: 0.273, valid_acc: 0.883\n",
            "training...: 100%|██████████| 18750/18750 [1:06:31<00:00,  4.70it/s]\n",
            "evaluating...: 100%|██████████| 6250/6250 [06:09<00:00, 16.92it/s]\n",
            "epoch: 2\n",
            "train_loss: 0.225, train_acc: 0.907\n",
            "valid_loss: 0.278, valid_acc: 0.885\n",
            "training...: 100%|██████████| 18750/18750 [1:06:36<00:00,  4.69it/s]\n",
            "evaluating...: 100%|██████████| 6250/6250 [06:08<00:00, 16.94it/s]\n",
            "epoch: 3\n",
            "train_loss: 0.146, train_acc: 0.942\n",
            "valid_loss: 0.326, valid_acc: 0.883\n"
          ]
        }
      ],
      "source": [
        "n_epochs = 3\n",
        "best_valid_loss = float('inf')\n",
        "\n",
        "train_losses = []\n",
        "train_accs = []\n",
        "valid_losses = []\n",
        "valid_accs = []\n",
        "\n",
        "for epoch in range(n_epochs):\n",
        "\n",
        "    train_loss, train_acc = train(\n",
        "        train_dataloader,\n",
        "        model,\n",
        "        criterion,\n",
        "        optimizer,\n",
        "        device\n",
        "    )\n",
        "    valid_loss, valid_acc = evaluate(\n",
        "        valid_dataloader,\n",
        "        model,\n",
        "        criterion,\n",
        "        device\n",
        "    )\n",
        "\n",
        "    train_losses.extend(train_loss)\n",
        "    train_accs.extend(train_acc)\n",
        "    valid_losses.extend(valid_loss)\n",
        "    valid_accs.extend(valid_acc)\n",
        "    \n",
        "    epoch_train_loss = np.mean(train_loss)\n",
        "    epoch_train_acc = np.mean(train_acc)\n",
        "    epoch_valid_loss = np.mean(valid_loss)\n",
        "    epoch_valid_acc = np.mean(valid_acc)\n",
        "    \n",
        "    if epoch_valid_loss < best_valid_loss:\n",
        "        best_valid_loss = epoch_valid_loss\n",
        "        torch.save(model.state_dict(), 'transformer.pt')\n",
        "    \n",
        "    print(f'epoch: {epoch+1}')\n",
        "    print(f'train_loss: {epoch_train_loss:.3f}, train_acc: {epoch_train_acc:.3f}')\n",
        "    print(f'valid_loss: {epoch_valid_loss:.3f}, valid_acc: {epoch_valid_acc:.3f}')"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "## Training stats display"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 29,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 388
        },
        "id": "VrgKu_9Rt1Qv",
        "outputId": "4211f522-815d-41f3-acf0-b1938a260595"
      },
      "outputs": [
        {
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAmEAAAFzCAYAAAB2A95GAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3dd3wUdf4/8NebJBCRXlQUvMB5SgcVPe6wnw311DsbCnfqnV/Pfv68O8VyinoqVlREsRdUULGBIEjvLdSEnkCAJATSC+nJ5/fHbsJms2V2d2Y/M9nX8/HII7uzszPvnWxm3vOpopQCEREREUVXK90BEBEREcUiJmFEREREGjAJIyIiItKASRgRERGRBkzCiIiIiDRgEkZERESkQbzuAELVrVs3lZSUpDsMIiIioqDWr1+fp5Tq7us1xyVhSUlJSE5O1h0GERERUVAiss/fa6yOJCIiItKASRgRERGRBkzCiIiIiDRwXJswIiIiMldNTQ0yMzNRWVmpOxTHSkxMRM+ePZGQkGD4PUzCiIiIYlxmZibat2+PpKQkiIjucBxHKYX8/HxkZmaid+/eht/H6kgiIqIYV1lZia5duzIBC5OIoGvXriGXJDIJIyIiIiZgEQrn+DEJIyIiIq2Kiorw9ttvh/XeK664AkVFRYbXHzduHF555ZWw9mU2JmFERESkVaAkrLa2NuB7Z8+ejU6dOlkRluWYhBEREZFWY8eORXp6OoYOHYr//Oc/WLx4Mc4991xcffXV6N+/PwDg2muvxZlnnokBAwbgvffea3xvUlIS8vLykJGRgX79+uH//u//MGDAAFx66aWoqKgIuN9NmzZh+PDhGDx4MP70pz+hsLAQAPDmm2+if//+GDx4MEaNGgUAWLJkCYYOHYqhQ4fi9NNPR2lpacSfm70jiYiIqNHTM7diW3aJqdvsf2IHPPXHAX5fHz9+PFJTU7Fp0yYAwOLFi7FhwwakpqY29jb86KOP0KVLF1RUVOCss87Cddddh65duzbZzu7duzF16lS8//77uPHGG/Htt99izJgxfvf717/+FRMnTsT555+PJ598Ek8//TRef/11jB8/Hnv37kWbNm0aqzpfeeUVTJo0CSNGjEBZWRkSExMjPSwsCSMyXXkBUJarOwoiIkc7++yzmwz38Oabb2LIkCEYPnw4Dhw4gN27dzd7T+/evTF06FAAwJlnnomMjAy/2y8uLkZRURHOP/98AMCtt96KpUuXAgAGDx6M0aNH4/PPP0d8vKu8asSIEXjooYfw5ptvoqioqHF5JFgSRmS2l9wnjXHFeuMgIgpDoBKraDr22GMbHy9evBjz58/HqlWr0LZtW1xwwQU+h4No06ZN4+O4uLig1ZH+zJo1C0uXLsXMmTPx3HPPISUlBWPHjsWVV16J2bNnY8SIEZg7dy769u0b1vYbsCSMWqbtM4Hqct1REBGRAe3btw/Yxqq4uBidO3dG27ZtsWPHDqxevTrifXbs2BGdO3fGsmXLAABTpkzB+eefj/r6ehw4cAAXXnghXnzxRRQXF6OsrAzp6ekYNGgQHnnkEZx11lnYsWNHxDGwJIxanuxNwFdjgKGjgWvD6/JMRETR07VrV4wYMQIDBw7EyJEjceWVVzZ5/fLLL8fkyZPRr18/nHbaaRg+fLgp+/30009x1113oby8HH369MHHH3+Muro6jBkzBsXFxVBK4YEHHkCnTp3w3//+F4sWLUKrVq0wYMAAjBw5MuL9i1LKhI8RPcOGDVPJycm6wyA727sU+PSPQNK5wG0/RX//4zq6f7M6koicYfv27ejXr5/uMBzP13EUkfVKqWG+1md1JBEREZEGTMKIiIiINGASRkRERKQBkzAiIiIiDZiEEREREWnAJIyIiIhIAyZhRERE5Djt2rUDAGRnZ+P666/3uc4FF1wAX8Na+VsebUzCiIiIyLFOPPFETJ8+XXcYYWESRkRERFqNHTsWkyZNanw+btw4vPLKKygrK8Mf/vAHnHHGGRg0aBB+/PHHZu/NyMjAwIEDAQAVFRUYNWoU+vXrhz/96U+G5o6cOnUqBg0ahIEDB+KRRx4BANTV1eG2227DwIEDMWjQIEyYMAGAaxLx/v37Y/DgwRg1alTEn5vTFhEREdFRP48FclLM3eYJg4CR4/2+fNNNN+HBBx/EvffeCwD4+uuvMXfuXCQmJuL7779Hhw4dkJeXh+HDh+Pqq6+GiPjczjvvvIO2bdti+/bt2LJlC84444yAYWVnZ+ORRx7B+vXr0blzZ1x66aX44Ycf0KtXL2RlZSE1NRUAUFRUBAAYP3489u7dizZt2jQuiwRLwoiIiEir008/HYcPH0Z2djY2b96Mzp07o1evXlBK4bHHHsPgwYNx8cUXIysrC4cOHfK7naVLl2LMmDEAgMGDB2Pw4MEB97tu3TpccMEF6N69O+Lj4zF69GgsXboUffr0wZ49e3D//fdjzpw56NChQ+M2R48ejc8//xzx8ZGXY7EkjCiQuhpg8zTXZOCteM9CRDEgQImVlW644QZMnz4dOTk5uOmmmwAAX3zxBXJzc7F+/XokJCQgKSkJlZWVlsfSuXNnbN68GXPnzsXkyZPx9ddf46OPPsKsWbOwdOlSzJw5E8899xxSUlIiSsZ4VSEKZOVEYMZ9wKYvdEdCRNSi3XTTTZg2bRqmT5+OG264AQBQXFyM4447DgkJCVi0aBH27dsXcBvnnXcevvzySwBAamoqtmzZEnD9s88+G0uWLEFeXh7q6uowdepUnH/++cjLy0N9fT2uu+46/O9//8OGDRtQX1+PAwcO4MILL8SLL76I4uJilJWVRfSZWRJGFEh5vut3ZeR1/0RE5N+AAQNQWlqKk046CT169AAAjB49Gn/84x8xaNAgDBs2DH379g24jbvvvhu33347+vXrh379+uHMM88MuH6PHj0wfvx4XHjhhVBK4corr8Q111yDzZs34/bbb0d9fT0A4IUXXkBdXR3GjBmD4uJiKKXwwAMPoFOnThF9ZiZhREREZAspKU07BHTr1g2rVq3yuW5DKVRSUlJjA/pjjjkG06ZNC7qfxYsXNz6++eabcfPNNzd5fciQIdiwYUOz9y1fvjzotkPB6kgiIiIiDZiEEREREWnAJIyIiIhIAyZhREREBKWU7hAcLZzjxySMWh6eSIiIQpKYmIj8/HwmYmFSSiE/Px+JiYkhvY+9I4mIiGJcz549kZmZidzcXN2hOFZiYiJ69uwZ0nuYhBEREcW4hIQE9O7dW3cYMYfVkUREREQaMAkjIiIi0oBJGBEREZEGTMKIiIiINGASRkRERKQBkzAiIzh2DhERmYxJGBEREZEGTMKIiIiINGASRkRERKQBkzAiIiIiDSxLwkSkl4gsEpFtIrJVRP7pYx0RkTdFJE1EtojIGVbFQ0RERGQnVs4dWQvgX0qpDSLSHsB6EZmnlNrmsc5IAL9x//wWwDvu30REREQtmmUlYUqpg0qpDe7HpQC2AzjJa7VrAHymXFYD6CQiPayKiYiIiMguotImTESSAJwOYI3XSycBOODxPBPNEzWIyJ0ikiwiybm5uVaFSURERBQ1lidhItIOwLcAHlRKlYSzDaXUe0qpYUqpYd27dzc3QCIiIiINLE3CRCQBrgTsC6XUdz5WyQLQy+N5T/cyIiIiohbNyt6RAuBDANuVUq/5WW0GgL+6e0kOB1CslDpoVUxEREREdmFl78gRAP4CIEVENrmXPQbgZABQSk0GMBvAFQDSAJQDuN3CeIiIiIhsw7IkTCm1HIAEWUcBuNeqGIiIiIjsiiPmExEREWnAJIyIiIhIAyZhRIYo3QEQEVELwySMKBAJ2KyRiIgobEzCiIiIiDRgEkZERESkAZMwIiIiIg2YhBERERFpwCSMKJDaKuu2vfItYMMU67ZPRES2ZuW0RUTOt/Y91++qMvO3/cvjrt9n/MX8bRMRke2xJIzIiNpK3REQEVELwySMiIiISAMmYURGcNBWIiIyGZMwIiIiIg2YhBERERFpwCSMyJ+U6bojICKiFoxJGJE/3/5ddwRERNSCMQkjMkIp3REQEVELwySMiIiISAMmYUREREQaMAkjIiIi0oBJGLVAVrTfYpswIiIyF5Mwark4yj0REdkYkzAiIiIiDZiEEXk7vB2oLNYdBRERtXBMwoi8vT0c+PSP5mzrlyeAzGRztkVERC0KkzAiXw5uNmc7KycCH/yh6bJ3zwdWTTJn+0RE5FjxugMg8qtwH9Dp5JbXwP7gJtcPERHFNJaEkT3lpAJvDAZWv23N9nfPA2oqQ3iD5kRwxyxg2mi9MRARkamYhJE9Fe51/d630vxtZ28EvrgemPuY+du2yrRbgB0/6Y6CiIhMxCSMYk9Foet3QbreOMK15GVgXEegvk53JEREFAEmYUROs+wV1++6ar1xEBFRRJiExbKPrwD+d7zuKIiIiGISe0fGsn0rdEfgIJw7koiIzMWSMCIjFJMwIiIyF5MwIiIiIg2YhBERERFpwCSM7I3VgP7x2BARORqTMIrMD/cAEwZZsOEWNlWRqXhsiIhaAvaOpMhs+kJ3BPbl8JKq4ooadDwmQXcYREQtFkvCiOxi589AxgpbJG/ztx3CkKd/wdq9BbpDISJqsZiEUctlZjJTXWbetvyZOgr45Argpwet31cQa/bmAwA2HSjUHAkRUcvFJIxaIINtpkJJ0tZ/ElYkYYnmvoiISBsmYdQCBUuuwmzYvmtueO8jIiLygUkYtVxici/CL28M/HpdDVBfZ+4+fdLfZoyIiCLH3pFEZnm2G9D1FN1REBGRQ7AkjGzOYaU++WlR2AnHCSMiagmYhJE9mV2VSES2sTW7GM/P3g5lg+FYiHRiEuZUO2YB00brjkK/vcuANe/6f10p1/hb9fXRi4mIArr+nVV4b+kelFdHow0lkX0xCXOqabcAO37SHYV+n14F/Pyw/9e3fO0af2vdB9GLyZ/Vk11JIxEBYIE3ERvmU8tWmu36XZKpNw4AmPOI7giIiMhGWBJG0ZWfDozrCBzeoTEIB7dDGdcRqK3QHQUREZmASRhF19bvXL9TvtYbB2B9XYjRRsdzHwdKD4WzgzDeQ0REdsHqSCLdVr0VpaEtQsfOa0RE1mFJWCxa94GrWstspYeAvN3mbjNWsoD6Wt0RNCFsMU1EZDnLkjAR+UhEDotIqp/XLxCRYhHZ5P550qpYyMvyN6zZ7qunAm8NM2ljGpIADmPRTIykwEREWlhZEvYJgMuDrLNMKTXU/fOMhbEQNedZypa9EXimM5C+UF88NsJyMCIi61mWhCmllgIosGr75FCVJa7fG6ZoDMJHirFvpev37nnRDYWa+GFjFl6eq7PnLEVTrLQ2IPJHd5uw34nIZhH5WUQGaI7FWvX1wLPHAes+1B2JXsXu8bqOHNYbhz+Ouiq0vPKqB7/ahEmL0nWHQSYpOFKNAwXlzZazySGRi84kbAOAXymlhgCYCOAHfyuKyJ0ikiwiybm5uVEL0FT1NUBdFTBnrO5IYlPQ5MqJVwUnJYwUi855cSHOfWmR7jCIbEtbEqaUKlFKlbkfzwaQICLd/Kz7nlJqmFJqWPfu3aMaZyx57PsUJI2dpTsMk4WaXIWY2JQXAHMeBWqrQ9wPUcvHuSGJAtOWhInICeLuBy8iZ7tjydcVj2VWvwPk2W0MKN+Jxpdr9kc5DhtpqB8JtTpy3pPA6rePDkJLRERkkGWDtYrIVAAXAOgmIpkAngKQAABKqckArgdwt4jUAqgAMEopmzXIqT4CxB8DtAozV62tdlU/HvMS8C82Nra3MKsjG8b3UhzegoiIQmNZEqaUujnI628BeMuq/Ufkg0uAEwYByR8Cw+8BLn8hzA25c8rqMmOr71sFfHw58O/dQLvjwtxnSxPtvNxe9wFERNRy6e4daU+Za10JGABs+jJ6+109yfV7/ypr92OzAkefIuk+Fc7nC3V/1UeAGk6kTURE4WMSZke/POGMRMn2wkjkjB73508EJgwMtrHQ909ERD5V19ajpLJGdximYhJmR0X7gdIc3VEEVx9GzyfbDhDUEFcIiVN5HpCxwvW47JDpEQUVhUSd9wJkJX69KBR3fJaMweN+0R2GqZiERUuoVzPbJitu6YuAZ7rojsIAg8fd6PHO3tS0BKzY3aN0/riQorI9m3/9yNn49aJwLN3l0HFCA2ASFnV2OP2YcP+ZNj/ybUST0SSrojDw60teBIoPRB4PEZGNDHhyDsbN2Ko7jJjDJIxiS/oiYOPn7ic+ktHUb63Z7/IJ1myXiMgER6rr8MnKDN1hxBzLhqggZ2nxd0AN1cFTro1sOztnh/c+h1ZXKrbaISKyDEvCCACM3wFt/R4Y1xEo2GtpPOYxWA1p9zZ4USa2qDYnImrZmIRRaBqq63JSorM/M7vneZdGHd7u8cSmSUd5AfD8Sb5fY+JIRORoTMLsqoWNDfDL1hwkjZ2Fqlqj0/uYkGDsXdJ0QFXvdllHHNDTJnOd8RkXiIhi0OYDRaiqdeZk8UzCKCo+WO6qviytrI3ujpeMN7aeE0uVWliiTkQtn9nJ0oGCclwzaYVj2zUzCYsaG10wdV68nZjsRIP332T5BFfbu5wteuKhmJBZWI5qw6XT5ATr9xUiPdeepefbsktw2hNzMHereYORF5W7RtBPySo2bZvRxCQs2myRhFibhNXVK0xalIYjVSaXeh3JA4qzzN1mIzv8XTw0tF9LW6g1DLKH6eszscfkC2tJZQ3OeXERnvghSu07KSque2cl/vDqEt1h+LQ5swgAsGjHYc2R2AeTsGBskTQ5y+yUg3h57k6M/3mHuRt++dfAhP7mbrOBXf/Odo2Lourf32zGyDeWmbrNhpukpbvyTN0uERnHJCwaYqztTkPj+yPVFrX/yt5kzXaJbMx4pxYicgomYZbyUYoRYwmZEfX1CjnFlcbfYMlk2RaUOLWAv3UL+AhELd5zs7ZhZRpLNJ2ISVjUGLnIx2bV0xsLdmP4CwuQWViuJ4B3zwdmPqBn3zbFWlAi53h/2V7c8sEa3WFo5dQbRiZhJsstrUJNnRnVBhZ8oxY84+pxZzNLdrnG6zpcWgUAqKmrR3XjMWx+HB6YthEPfR1qlWSA43nQourN9AXWbLeRQ886REQmcfoNI5MwE1XW1OGs5+bjse9s2tto2auu35HcMjS818Iv/p/eXoF/fL7B7+ullbX4boNVvSRNtG1GxJsoKq82IRCyUn5ZFVId2j2eYk9ZVS1W78nXHQa5MQkzUUPpzZxUX2OgtLRSi/CyMGXgfalZJWFtu5Fdbo02fR7xJnYeKjUhELLSVROX46qJy3WHQWTI/V9uwKj3ViO/rEp3KAQmYdFnlwTBLl4bgAkF9+mOwraMJK2k18FQOpUQabYjx3Vjx9629hCvOwD703URtHK/NiqVK8lEb7O3uf0ns7eozfBW24OvRORQyqmtqYlMwpIwAgDEoQ7dUag7DHOse9/HwpZYomTdZ2qJR4vsQ1gjQCZzaj7PJIwAAI/Hf4F1ifeiE9gGicK3bHcuJi1K0x0GGeDUixZRS8LqSAIA/KGVqzdiBwlzrK7ygohj0HJvnLFCx15NYr+r6F8+XAsAuPfCUzRHQkaxUIpIH5aEBaPtDGXhBdaKW+CVb/rYj/m7Md0nV+iOgIiIYhSTMLtL/dY1wGpZru5IDGtVX4PjEXnJGADWmRARtTBWntbX7i3Ade+sRLVDen8yCYuWcL916z50/c7baV4sFhdRnZ7yLNYk3ofW9T6qNg2WLJofIZM5sofaunpUVNfpDoPIJ6f3WB377Ras31eIA7qmwQsRk7Cos756c9OBIizYbsUk18b0yFkEAJi3OSOM6YUssnKi7ggcyeknZDu6c8p69Htyju4wiHyKxmwkVrTyceqZikmYVUpzTC69MiB9EVBRhGsnrcDfP022dl8G/4scMb0QNcPG2tZZuOOw7hCI/Co44qyp0px+rmISZpVXTwMmnxO9/ZUXAFOuBb4aE719moxjB5mjvl5hZVqe7jDIQVIyi027+O7MKXX0lDgFR6rxw0brbx4/W5WBVemcwzHWMQkzkZTn4R9xM6GaFYx6PZ/5IDDjgWbvzy+vCX/nde4TaN6u4KvWO6PBIgXhp6rw/WV7cMsHa7CIJS5k0B/fWo6r3zJn/svLXl+KSyYsNbSuHauQ7pqyHg9+tQnZRRWW7ufJH7fi5vdXW7oPsj8mYSY6Ztb9eDRhKoZgd+AV138MbPi0yaL9+eVRm9lebzsf/6VdVTVsrGyG95ftBQDklFg/p+Hj36cgaewsy/fjraK6DsUVEdy0UDOZheYlHcFK1exc5n2wxHUcauvsmCJSS2MoCRORf4pIB3H5UEQ2iMilVgfnOFWu0eafwTshv7W0yuuCYlaiVFEI1AavZhAT70kbthWH0ErcftgUy+3HQj/+T/+0zefyvChWBX2xZn/U9uXp4teWYMjTv2jZdyiumrgMpz8T/TiziypQVcubGiK7M1oS9jelVAmASwF0BvAXAOMti8rhfi3eyUQE932RJmMvJgFf3mh8d4bX9P5MzT/j2sR7kYBaw1v0fZxi4250b96RkN/zTfKBkN+z61ApHvs+BfX1zj6uWRZXFfmyLqMAU9eGlnSmZpWgMJJmBmGoqavH78cvxENfbQ64nrO/AURNObUnt9EkrOHqeAWAKUqprbB3iXILFMHh3rPIvDBC1JCE3Vf0Ei5vtRbGP4f/9eJRh/bwGgOmYG94AdrEyUVro7KfOz5Nxpdr9ttyDB27N1K+YfIqPPpdiu4wAAB19Qo7c3zP81rnTrDnGxymhidy8tS8TbO9icO/wUaTsPUi8gtcSdhcEWkPhFjX5FTl+UDWBqAulBId8nZexUJMbv26Kdt6PuEDpCTeAXh2MHhzqCnb1iVO7Hnii9bNZVF5NRsph+DNBbtx2etLsf1gSdjbcGrJAVFLYjQJ+zuAsQDOUkqVA0gAcLtlUdnN+xcCH1ykOwrz2PTka3SEiu7ivvBUFlkXTIyL9t2lXaYYmb/tEJ6euVV3GEFtOuD67pvR+cJpQ8NkFpb7LQUkchqjSdjvAOxUShWJyBgATwAoti4sGzoYuH2FbZTpGynfiIRKE8evmjbavG0RAbjjs2R8vCJDdxgUwDkvLsJlrxsbAoPI7owmYe8AKBeRIQD+BSAdwGeWRWV3+enAEVcy0efRWbj3yw3m78PSu9MISsIiLEXrlO3/5BnyJ86xR/scXZxVfkE6HSgox46c8KsuY4lNKwqohTKahNUqVwOCawC8pZSaBKC9dWHZ3MQzgNf6AQDqFTBry8HA69fXuMYGs4n4KhOq8QIkieEOBhtp3nnTu6s4MbLJeD2yzoR5wQdWNsu5Ly3C5a8vi9r+jLJzwuOwWlpHsfPfPdqMJmGlIvIoXENTzBKRVnC1C4tddV5jb+WkolVxgO7rcx/z+1JeWRW2ZnndpTZ+S12/y6prMTslSLKn0/LXGh+2rdVTU71mbwHWZRRo2bfTBDsJ8gJkvYkLgwzqTNTCWHFacfq5ymgSdhOAKrjGC8sB0BPAy5ZF5USTR6BVaXZYb73ijWV4+NstTZbtzSvDYY9Gt28tTMc9X2wIaTypJtPWzH864LpBv8dKAXX2HqH8slbrdIegxf1TN+L52dsNrev0ExY1VVenHD3mG7+P9mNlKZVzv6nWMZSEuROvLwB0FJGrAFQqpWK3TZjJDpc2H+H85vfX4HfjFzY+z3WPgl4ZwtQ+t3/ikZR4lFQFovylYwueAZ7tBtS6B8ks2GM4DqN89sgLdEaobtpD6t3WE0yOyBlmbs7Ge0vN/3uQfTUkL3d8lozbPrH25iOzsByTl6Rbug+KLUy+jzI6bdGNANYCuAHAjQDWiMj1VgbWYtX6Hun7hrjFGOlVklNnhzvcV04DZtx/dK7L9IWB14+A5yCBfpNBIh/Kq2uRNHYWpoU4on1LsHRXrqXbv/3jdRj/8w7TJrTOKqpAaRXHXbSrF37eoTuEmGK0OvJxuMYIu1Up9VcAZwP4r3Vh2ZOVI3q/nPAeWnkM2DmgVYZl+2rms2saH/4tbk7T18pygA16Cz1fnrsDP8b0vJIUTK67NPntxa4Sm09W7EXa4TKdIVnKjCojo9s44k6YzLolTGa7TYqC1XvyUVpp7yY0gPEkrJVSyqOBEfJDeG+LEc0RvT9s/SoAoNZoaVhuBHcvexY3Pvxb/Bz/65nsfz82HXutoTry1YR38Me4VY3LJy1Kxz+nbYpaXE5h5qTrOpRX1+KLNftcI7ebWPCplMK4mdtw7aQV5m3UBKlZxXhl7k7dYTjSzpxSJI2dhbTDHKSVfPO+qXj8+1Tc++VGPcGEIN7genNEZC6Aqe7nNwGYbU1I5GnD/kKc7ZHu+r173fJN48OMxFswsfZavFrbdOLuegha2ejC/cTG83wuvy7Ooyt9+gK4Jmwwptv+nyOMiqLl2Z+2YeraA+jVuS369jB/xJsym1V5XTtpRZObKqeNVB+qm98z76Z1xmZXSfic1BzTtkktQ6B/o50OGBvPaMP8/wB4D8Bg9897SqlHrAzMrv7fV84okbkzblb4b64oAuqdOd5W/+X36w7BEeyQiueXuYZ5Kdc0tptSKqoDmNY7dHCk7OLwpkZatcfeE7L749A/EzmU4SpFpdS3SqmH3D/fWxmUnX2/8WjbpHEzrJ1jrjVqglc5WZEsvfgrYOrNTZep8AZg5fnMXlp22UvTzh3BTFm9D5e/vszStp6hKCqvRmqWvWaDK66wtk2NnROeFl5QSTYRMAkTkVIRKfHxUyoi9i/ns8CJODr34ScrMyzd18sJ7wZfKcAgsM2EcsLbPbfpGbKiMIQ3h6dtXYmrJ6YPx8KcnlkUGjtfJD2FM+F4wwDJ+/KNj71npSvfXI6rJi43tG60EgRlwRfAidWwSikkjZ2Fl+ey56AV9uSWISdIiWs0Z5iIpoBJmFKqvVKqg4+f9kqpDtEK0i4Gyh6sTHwgavs7r9WWoOuoLV9h4oLdqPWaKigOzqtOvDDnQ789Me+L/yHK0cQ2510m7aW+XuHUx423Tywur0GWSUNA6LRhv/U3azoccVeZT1oUe+OlFZfXYInFw6Bc9LablOMAACAASURBVOoSDH9hgc/Xpq7dj715R/DGgpY5w0TM9XAMqsj/OEN9xIJGoSXZ6IDwu9JX1dbj1Xm7sDOnaa+heAmt+lDEBkUeAe664xBedWgsc0opViBKKXy0fK8jupp7qq1XqK4L/J31THTzjzQfsFmHQGMTBkvMUzKL8ee3V/p87evkAxFEpd9/f0jVHYI2//g8Gbd+tBaFR6qDr2yBR79LwdUGS4i9hVNCHm1MwrzlRq8LuVIKeK0flrf5Z9B1G4rwm7V5cT+tawFX3EMl9rgQOYf1f/NQ2lhZYUVaPp75aRue+jFw+0sjca5Mz8Nj36c0Pq9wzz4xZ6ueHne19QpVtfYqsX5ulrHpr3zJLfNfnfTJioywt2sHwarKWrL0XFd1fU2QmworGRncV/e5KlxMwjRqmAeygxivhvjPN8GrKJ2qsFzPnRa5bgi+Xnfg6LRYUW234//k2RCPGQ3Eb3l/Db5cc7Sku6H6b/FOa6taAvk4zOQk0D3XT1sOhhcMgHnbmyaknvspq6pFcbmzSiSp5XNCaVcgTMIcZtvB6PWHeNdG88U5fWBSHaoDlLJ4N7iet+0QHv52i2mDia5Mzwu6jll53uHSSkxalOb39QMF5Vhn01HarRie49/fuAZBNvs/5tIJSzHkmV8i3o6zL5lkBp0VN1lFFbaaTcOyJExEPhKRwyLiszJdXN4UkTQR2SIiZ1gVi5M1nLAy8nz34Gr4Lp+WH9mcjvE+GvK/MMc+PYFOkuAXdWpq3MxtjaWtDfz1TCutdBX3F5jU7uOW99c0Pl62OxdJY2fhcIk1VToPTtuEr5Mz/b5+7kuLcMPkozMwlFbWYP2+8JOyPblljioR8pwmqLKmztRhJ8qr7TUgrk6TFqXhb0EmUy+rqkWFpnHx7CDUBHy7BYUOI8YvxMWvLTF9u+GysiTsEwCXB3h9JIDfuH/uBPCOhbGEwJ73acEmtG5TXx50G60CNL5vC71tHiYkTMJt8f7vslkSFp7dh/RP8/Lpyn0AgE0HigC4SuHmpB4MaYL6QGt6lyY1zCPpz52frcd176wKuxTqoleX4MqJy4KvGIJQv91GShEbVnnWo53X1W8tx5CnIy/NAlzJdf8n52K1Qwdl9cfXsBxGjvfLc3di4Y7DAdcZ+NRc/H68716AdmfkOzppUVrAmRJC/Z6PfMPc/zM7MjptUciUUktFJCnAKtcA+Ey5vvGrRaSTiPRQSoXfoMFi0U4DOksZholrbBR/SUgoFzI7+1Ocveb5I+vMSc3B3V9sMLRusIvfB8v2NBmJXingwSCzWjQMiFpXH35D48zCwO047ToU1q5DzathPGM12r5GKdU4yG1yRgGG9+lqSnyhqK9XaNXKugPtWWpsZvVZoYNKUUP1ssHmDNH6/xBxTZ91Yqdj8OqNQ6Kz0xDpbBN2EgDPfsuZ7mXNiMidIpIsIsm5ufoa0fZr5X/4inCYN2hhy0jE/MlIvEV3CC1OcUVNkxKMUL9BO3NKMfz58O7o80zs6v6/WduxJbPpKPO6pkHy5Os/2wmDlO4vKG9SEuTve/HUjK14e7EJbUYjOHVNXOi/HSCZz+7f3savrVegq/bk49sN/psr6OaIhvlKqfeUUsOUUsO6d+9u7c4CfNPuiZ9h7b7D9FL8ez6Xx8N4e41QemiSPXh/VevqFR6cttHQe2/7eB1Gvbe6eYN2g2fad5emI8e0Nl56Tu+26FXlvnLYqZ1Qw5AE/mzYX4jPVu0Le/tm5aILdhwyZ0PkaA64twlIZxKWBaCXx/Oe7mUxY0tmkSnbuTHedyPD71s/acr27WBkXOAGrwRkFpbjh03ZhtZtaJxttPrAWsGLQ6yYPscO9ua72nLq+Hj+9jllVQbyyvy3qyuvsk/CSOGLxldu0c7D2JNrn56IdqQzCZsB4K/uXpLDARTbuT2YFf45LXDbFV+6owh4ManxuQpwGzCoVUYYUR3FxvD21l4qgs576PS/oJV3uXYY3HHm5mzklVVFPZadOaVYvddVHe19jD9dtQ/3GGyzFyv8fQ+/XZ+JhQ4rkQv3Xyqcb+jtH6/DRa/apyeiHVk5RMVUAKsAnCYimSLydxG5S0Tucq8yG8AeAGkA3gdwj1WxhMb4V7S/ZGB9m3+gK4qDr2ySi+I2NplMW2xeQhCsVyeF7+JW63H356FdLP39NVpqSZNPUfhKhlJVW1pZG/WSsMteX4qHp/sf+FnXFDV25e/v869vNuNvnyRHN5gQ1Jowyn00z+Avz93RYjqbGWVZEqaUulkp1UMplaCU6qmU+lApNVkpNdn9ulJK3auU+rVSapBSyr7fZD/ujP8JXaUU57RKCb5yhPwlM+3qozd4a9jK7TlQZktQb9LV+2ibVnNPubNTDmL+9uAlBe8u2RPwdc9PuTU7ejc94dqfH3zIGKMiTZDtdhtki7Z4Xqau3Y9zXoxsrMUGy3fnobq2efIT7eRiZ04pTnn8Z8xJDTwt146cEjz5o/+5MY1GbcY4gJMWpWOpiZOF2++b1pwjGuaTc10Vtxp4qbfuMMgg72qXSHO8lwwM+FtXD3y4fK/veHycRrdmm3/jYfoFMsSzv10bF1tRQhpO1avVJYWPfpfSOOyIn052ho35cA1e+Ln5HJwfeXzHDxZb3xFqs7vNcbCboFs/Wmuoo0Ww43Hz+/7HBwtFuP+LTi0/YxLmJZRqhIQQeh/GqlcS3tUdAoVhlXvaoYYTm1IKpZXOGN+oYVDYYDwTvLJK///LtXX1R+fUNMgOpT3RuihFc+gNq/dVcKS6MeGLZFe+5gTN9ki83l4U+fAer8zdiadnbrVNU4JgY+dZYfnuPJt0LgqfZYO1OlWZgdnaG1wZt9bCSHxrBX0z2ZPzhHt+XpdR2OT5pyszMG7mNix7+EL06tLW0DYKj1Qj30Dbolkpxnp0Rtvbi9Pw0pydOCupc7PjQda4/eO1KK2sxfS7f69l/zdMXqllv6E6UFCOt9zDy9xy9slR2ac9Ur2mxny4JvhKNseSMC/hfNHaSfSm/BkX/2nU9tWaJX22Fs3eq79sc1Vp7C8ox6r0fOSXBU+uTn92XuOclADwyi87cc2k5jMjzE7x32aloerKaDJpZD2jBRzvuEsrQknAckurDE1e7s3o5wtlPDGryoyyi3yXeIRT+uddNbloZy6S9+lLeP2NkbYqilMzfb8xE48E6DQBADUeDe7rLC4Ji3aZ7k73VGtTVmVEec96MAnzEmzeOV+Ol+g1PG8j0UuMXkt4O2r7ovDsyDk6N+QeP5O8GxLCefzm91djiVfj2dSs4qDVIrsOlWGzwarCBpH2PKuqtXZMK++JvP/8zoomk5d7CqWty60frcUrPqpZ+j05x/Sei6EmTw9/2zRBCLU6zA5VtQ0Kj1Tj2/X2Gk39/321GV8lHwi+okGR5mjRLgF7ee5OHCqpxH9/3BrxtpwwSwWTMC9Khf5Hs/+fOTw6qlspfJN8TOOy7WAJJi7YDcB1sawO0mU93O/yVROXRzSKulUmzNtt2bZr6uox5Jmmk2EfKAivXYzntaLgSDWW7MrFW4vS8O9vNjdL3vLKqpr9na5yTyhudvsgu1VBmT2zwH1TN+Bf32w2dZtmm5N6EFNWZaCmrh53TVmPnR43XoFEcl2atCgN9V7fu0i2l1Nc2aT3ZLCvaTjtT516HWYSRtSCvblgN16d55oE/os1+/02njVjsNAdOdYNl7Iv/wiSxs7C2r2BS52zvKrKDpf6byoQ6U2yr2EIzNj2st1HSxmnr89sUtrpT2qW69hPWW2/RDiQUEvF3vKeZisE172zEj9uajopy6GS0Gs+whVufnzX5xvw3x+3Ylt2CeZszcF/pm82NTn29Td4ee5OLNlt3lARw19YgLOfX2D4rx3OsQr1LSWVNUgaOwvfb9RbEsokzEs4J89w2+Zsa3O74XU56ClFataW4BNS2LX0PsM97tb3G7NM+09ocqLX+LnNLL36ycDf2IlC6TDlz/p9hWHNUuJ0S3YdDut9tXX6ykHD2XPa4TJDQ+I0aBjL7/2lvofHiRb2jtSorUTvLoxaPrMSqIZ5JQHXtDqZBeVaT8hmK3F3FojmVEGBEq2LXl2CYb/qHNr2Ig3Iw/4C8waWtYodq7qjrTzMqti8smrkFFfihI6JPl83OiyTGd85o9sIdxDqtxeno0/3Y8N6ry5MwrzZtCjgd622YUdddLoikzOYWTrqec77x5SjjeH35h3B3rwjSIiz5/+FTlsyjY/cXxMkiQ2lR6BNT1GG3fulfeal9JUc22FOUV/un7ox7PdW1dYh7XAZ2ieGfsk3XIUYyjaDbNQmQ59FBasjHeI/CV9jAnsrkodQq8GNXFwEgt2HysINyVF2eX3O7QebtmkLdrTMGiHcLG/M3x20zVyoDA35YSAr9LeZqyctx799NIwf+23gIRqiwcpenOEk0nllkdWcXPzaEvz2+QUhv6+l5kMpWfaY/oxJmAk6IToXrb6yPyr7odhTG2OT5voy8o1lTZ6b0Q7JKr6u4RPm7zJv++4dWF0qtC+/HNN9DBExbZ15QzS0RGYlxwHfH+S5lSIpCdvjZ6w3b/O3BZ/TNhqYhDUT+ldtTHzodxdEdvLED/4n8I0F3terwePm6gkEaNZ4vPm1NLRzlFOqL1ebPCDqop2HUWKjqbasml7IX4nd7kPGhrJwqpmbI5tpY8EOV4cF3f8fTMK8OOR8RRQyoyU74Z6Upq41r/QiNUBVQUh3+Aavexe+vLjJ85IAc0mGI5ISJe9rt+6LhhV+2JiFUe+ZV717qKQSt3+8Dvd92bwdVX29wrtL0nEkwpLO136x95yF5n+H7SWSNnJ2wiTMm41PcNGcpoZanoaxpLx53qH7u8CbeRNvZCDG+6LceNvIHJdOVRNkgF5frCi0ufOzZCSNneXztQe/MnfoiKoa12fem9e8qcjPqTl44ecdeGnOjojOqG/6GBzZ03cb/I8/NXVt+E1LzPrbeA/G6o+OS6JdO0dYgUkYUYzr/ejsxsfRKGV5aY6xEoTKGl9d8u17ct6Wbd1gtZGww/AigqPzj+pW4f5elVrc5u+hrzf7nQYvWG/Z4CL/m5770qKIt7Ev/wjSDhtvE/3odykR7zMUTig1ZhLmxU7zmhHp0fx/QEfD/Ye+dtbAmn9+p/nk5A2c2uW+qDx4qaVVbZ2MyC+rwt4w50yNRtIcynyhVjrfq7rdiKOdMwJv9+LXloQVUyBmfaWc8H/HJIzIoay6XQh09/jUjMgn1TVqyU7zpk2JhvoAtX6RNCJeuKPpiOfRvE2cEsIgqeEOsBmJ349fiAtfWez39UAhGZkOyqhgHTl+2ZoTsMfnaU/8jIenNx2qo7q2HuXV4ZfWRVLl6UtLLZ7QXVrGJMxL13atdYdAZFuhVD3Esge9ejj+Z3r44169Ni+yoSd2RtBLrqgieElYQ0eJv3+aHGRN81X5mb/T8IXVR5LWkLiFcnEO1gj+zinr/cYKuD7H18lN25BdP3kl+j8ZvJduoPlRfQk3VQ76PgeUOtkRkzAviQn2nUTg5FbOKhkga1nV4d1fO5Zoysgvd/Q53ehUMNEQSQFVpMMAOFHD4dJd8mN0Roa/fLg2pO0eLDb23dx0oAh9//tz4/ngxndX4YNlewAAK9Ly8PJc4/M0egv2nbSiUPX7jZlaq879YRJG1EKsyzA+9Y0T+Jorb+raA3hjgXmDkpopWj26jA7REajkpSUwcj3NLKxA0thZ+Dr5aFWg0Qvx2c8v8DlKvVLK8I3K7Z+sw9Zs4yOzv7sk3dB60cglJi5MQ2XN0e/Qvvxy/G/WdgDA6A/WYNIiY7H68uPmrMbH0UqM/t9Xm7Fsd15U9hUKJmFE5CgHCip0h+BT5D3ejCkqNz6cRjQahr88dyfWmDzQaqhSs4qRVVSBR79LQa2PITkeDrM6eMfB5lW5c7fm+JxqyZftB0vw57dXGt7fCz+HX7rkJJ7J3ccrMizbj/f9SqnJY6eZwb51b7roLoMmIgogu8h4VWe0Shm+So7+NEOeCeZVE5ejy7GtUXCkGlcO6oFfdW1r2X7zyqpR4aOU1h+z/wROGENrTmoOBpzYwdC6X/v47kTzM+oeEYFJmBfmYOQUKoRv6wuztxtaz+weVRQbvtuQFXylMPkb4HWb14TrBe4Bd0WMN6oP91K/Nz+8YTHM4G/QZTu56/P1aN+G6YURrI4kcqhQZlB4d+keCyOhaIt0cubg27d087bmOZL8Sz4anytAe+cV+5eFRTYYrg3bz1uGqSoRtViHbdDT02wirp5rRsTQtaxRVlFFSI3hvXlOYWW0h2IgwarW7tAwtEesyCy0Z/tRT0zCmonhW0CiFmZ5mv16Q0Uq7XBZY9WblZbucuaQOOE2wjeqIoIBVH2Zv90e0zkZVVVrvD1cuGLp5oHVkV5iuRieiOwv0sFbjfrrR6GNP+UUoV7gz3x2XpPn7y/ba14wDnTaE3OaLVuZloeaQFNG2Jjuaz5LwoiIqIldh2JjZgQjvUfzvUoddbcHs6NbPlgT9nutbt/oyY49S1kSRuRQofSOpNjk5AFbK2vqcO0k/5OihyuUcdbMYMX4cS294fp3GzIDvu6vx2ww1T7+H3QfSyZhXlJMaIhJRGQHA58KPvegXb00Z6fhDgiheH52bAyI6mSfhTBxfCimrY3+eHbBMAnzYmTCWiIistb2g/YfD0uX1+bt9LncjnMj2kmdDY8PkzAvPTtbN9IykZlCGSeMSCe7flPtGlcwc7f67lF5ulcngmDKTe7p6US6G+YzCfOSEMdDQkSkmx0bUdtdUXloNTnph/WN/E8uzDiIiCgm2XFC51iju3tRqImr2ZiEedP9jSAioqiIxqC3ZG/7C8q17p9JWDM8JEREuu2OkbHKdHrs+xTdIUTV+n2FukNohoO1etHdSI+IiJoPkkrmS8nSNyRT0thZeOCiU7Tt3y5Y7ENERERR9+bCNN0haMckzAtHISciMlchS7WIfGIS5oUpGBGRuQ6XVuoOgWwq1psAMQkjcqjuwim2yBnqOeQXkU9MwrzwXEFOcU/8DN0hEBFFJLuoQncIWjEJIyIiIi0KNQ+WqhuTMCIiIiINmIQRERERacAkzAvbhBEREVE0MAkjIiKimKR7iAwmYUREREQaMAnz0jaB02kSERGR9ZiEeWnVKsaH7yUiIqKoYBLmjTkYERFRTNB9yWcSRkRERKQBkzAvoj0vJiIioljAJIyIiIhIA0uTMBG5XER2ikiaiIz18fptIpIrIpvcP3dYGY8RonvQECIiIooJlo3HICJxACYBuARAJoB1IjJDKbXNa9WvlFL3WRUHERERkR1ZWRJ2NoA0pdQepVQ1gGkArrFwf6ZgQRgRERFFg5VJ2EkADng8z3Qv83adiGwRkeki0svCeIiIiIhsQ3fD/JkAkpRSgwHMA/Cpr5VE5E4RSRaR5Nzc3KgGSERERGQFK5OwLACeJVs93csaKaXylVJV7qcfADjT14aUUu8ppYYppYZ1797dkmCPYn0kERERWc/KJGwdgN+ISG8RaQ1gFIAZniuISA+Pp1cD2G5hPIb8+rhjdYdAREREUaB7RATLekcqpWpF5D4AcwHEAfhIKbVVRJ4BkKyUmgHgARG5GkAtgAIAt1kVj1Gt4+J0h0BEREQxwLIkDACUUrMBzPZa9qTH40cBPGplDKFjdSQREVEs0H3F190wn4iIiCgmMQkjIiIi0oBJGBEREcUk3QO0MwnzpvsvQkRERDGBSRgRERGRBkzCiIiIiDRgEtYMqyOJiIjIekzCmlG6AyAiIqIoEM0FL0zCiIiIKDaxdyQRERFR7GES1gzbhBEREZH1mIQRERERacAkjIiIiEgDJmHeWBtJREQUGzQPiMAkjIiIiEgDJmFEREREGjAJa4b1kURERDGB44QRERERRZ/uYhcmYUREREQaMAkjIiIi0oBJmDfRXThJREREsYBJmDeledAQIiIiiglMwoiIiCgm6a78YhLmTfdfhIiIiKJCd+UXkzAiIiKKSVW19Vr3zySMiIiISAMmYUREREQaMAlrhm3CiIiIyHpMwoiIiIg0YBJGREREpAGTMG8cooKIiIiigEkYERERkQZMwrxVFuuOgIiIiGIAkzBvRft1R0BEREQxgElYM2wTRkRERNZjEtaM5omkiIiIKCYwCSMiIiLSgElYM6yOJCIiIusxCSMiIiLSgEmYNw7WSkRERFHAJKwZJmFERERkPSZhRERERBowCSMiIiLSgEmYN7YJIyIioihgEtYMkzAiIiKyHpMwIiIiIg2YhBERERFpwCTMG9uEERERURQwCSMiIiLSgEmYt6RzdEdAREREMYBJmLcOJ+qOgIiIiGIAkzAiIiIiDZiEEREREWnAJIyIiIhIAyZhRERERBowCSMiIiLSwNIkTEQuF5GdIpImImN9vN5GRL5yv75GRJKsjIeoJVlYN1R3CEREFAHLkjARiQMwCcBIAP0B3Cwi/b1W+zuAQqXUKQAmAHjRqnhM0fHkps/bdNATRxgqu3ofenK6zfW/1h2CTwlx4c868efTT/K5/L4LT8E9F1jzeZc9fCHeGX0GRv/25OArm+y6M3ri/otOwfEd2hh+T6swD++z1w7E2JF9my3/k59jTkTWi7dw22cDSFNK7QEAEZkG4BoA2zzWuQbAOPfj6QDeEhFRSikL4wrujoXAiteBQdcDfa8C0hcCC591La+tBOY/BfzuPqBLb6CiEHgxCWh3PJDQFuh5FnDef1zbmXSW6/fffgEK97pe27MYOKYz0P004J3fA+ePBY7rC0gcMP12YOgtwOl/AT68BPj1H4AbPwWUAn64Gxj5IjBhgGubfa8Cev0WmPdf4OKngfY9gN9cAqx4w7WPGz8FvrkNyN4IPJGLxPjWwLyngPw0YMdP/j970rlAxjJg4HXA9R8Ba94Ffn646TptuwID/uzaT/5u48f12O7AkVzkJ12FrhkBYghgY9ercHr+T0C7E4CynKMv3JcMZK4DijOBRc+FtW2neb/uSpx3andccGp3PPPTNow4pStWpOUDAGbedw7++NbyZu9Jf/4KxLUSfLVuPx75NgVpz43E1W+twLaDJTg7qQsuHXA8/jdrOwad1BHt2sRj1R7X9n7Xpytyy6qQdrgMAHD3Bb/GrC0H8dnfzkZpZS325JWhpLIWWYUVGDuyL2rq6rE1uwRVNXUQESzYcQg5xZU4qdMx2JxZhBGndMO+vHI8fPlpeHXeLoz+7clYsisX91xwCr7bmAUAePyKfrh84AmYk5qD/zuvD5btzsXbi9Pxxqih+Oe0TVjwr/Px6+7toJRCTkkljlTV4uLXljb5vD/eOwLXTFoBANj7whUQEXy4fC+e/WkbTup0DN7/6zD06tIWvbq0xchBPfDnM05CuzYJ6NP9WAz733wUV9Rgwk1D8NysHcgrq8LHt5+Fqpp6DDixA859aVGz4zvqrF6Ytu5A4/P5D52PPbllOLZNPCYu3I0v7hiOWSkH0b5NPNq2jsNv+3QFAPxtRG+c/uw8XNzveMzffgjv/uVMfLchE3O3HsK3d/8O6blH8PD0LQCA3c9dgR83ZeHpmdtwxaATUFxRg9kpORj/50G4pP/x+GD5XryzOL1JXKcd3x5/Gf4rAMBd5/8a1bX1GDdzK/bklmHCTUMxZvivACh0a9cG57+8uNnnSn7iYhzbOh4b9heia7vW6HtCB+SWVuGs5+bjnFO6YXlaHgDgpmG9cO6p3TDgxI5YvjsX//1xa4BvMJF+Om6+PIlV+Y6IXA/gcqXUHe7nfwHwW6XUfR7rpLrXyXQ/T3evk+e1rTsB3AkAJ5988pn79u2zJOawlWQDbdq7fjzVVLh+Jxzj+32HtwPdTgVaxTVdrhSw9GXgjFuB9sc3fS1jORCfCPQcFnncdTWAtHLtv6IISOzoe+7MI3mAqgfaHdc8zppyoPWxrmNQmAH86vdHXz+4BYhr7UoyvaXNByDAKX/w2lc+kDYPGDIKqK8DdswCThvp2n99rWtfBXuALn2ADVOAvlcCbbs03UZeGnAoFcjbBZQXAEX7XJ+vz/nAWXcAreKB3J3ASWe6Pu+K14EFzwBn3g6s/xh7blqIPqcMAPavBJZPcH3O7qcBpTmuz3NoK9D/amDXHFcyeNMUV5Lc67fAnLHANW8DPYYAG6cAw+8BFj3viuv0Ma6/3ff/AC57Duj6G6D0IJC305VEb/kKuOIVYNVbwG8uQ1GbHuiYmwzp2As4kuv6fm39DkjsBPQchg3tzsOpx7dHuzbxOFhcgR4dj0FxeQ2qautwXIdE16Eoq0K9UjiufaLfr8Hh0kp8tnIfHrrkVIgAC3ccxoWnHYdWrQTrMgpwcpe2OL7D0ffvzCnFaSe097u9SB2pqkVJZQ16dGz+f1NeXYu2ra28dwxNbV09CsqrmxzfwiPVmDB/F64eciKGJXUJ8O7AqmrrUFRe03jsj1TV4tg2vj974ZFqdD62dePzVen5GNqrE45pHedzfaMqa+qQmBB4G0opzN16CBf3Ow7xcU0rV8qqajFvWw4yCypwYV/X+eOqictx/0Wn4I5z+mBWykGcd2o3lFTU4qROx6BOKfycehDHtU/EDxuzcNuIJNwweRUA4JxTuuHO8/rglOPa4bsNmbjzvF/jrUVpOKlTIrKLKtGtfRvMST3YeCPywEWn4NIBJ6DgSDVWpOfh3SV70PeE9jj1+PZ4ZGRfdGnbGiPfWIqM/HIc36ENDpVU4cSOicgurkTb1nG4ZuiJuHTACZgwbxe2ZBbjvFO74zfHtcNtv0/CxysyMDvlIHp0SsTG/UV4e/QZqK1XuHrIia7lHRMx8KSOSIhrhbTDZTihYyK+35iF7KIKPHTJqdiaXYKK6jr8tncXfLP+AC7tfwLi4gQ/bspGXV09Lh/YA9dOWoGckkoAwJjhJ2N4n67okJiAlen5GN6nC9rEx6HjvkwkpAAACYFJREFUMQk47YT2yCwsR229wicrMvDQJaciLbcMdfUKOw6WYEivTujd7Vh0SEwAAPyUchAZeUfQJr4V9uQeQVyc4I5zeiO3tAopWcW4+eyT0bZ1HMqqanHD5FXYkVOKN28+He0T45GRdwRnnNwZ7yxOxxNX9cM3yZm47oyeEAG2ZBaje/s26NauNbZml2B4n65YuisX//pmMwDgy//7LXKKK7F8dx6+25iFm4b1wmkntMcl/Y/HhHm7MHr4yUjJLEZSt2Mxbe0BzNmag6SubXHr75Pw7pI9yCmpxAkdElFbr3DBad0xfX0mnr12IH7Xpyvyyqrw1I9bcdNZvTB17X48cnlfnOD+G0xZvQ9nJ3XBr7q2Rf6RarRPjEfhkWrcP3UjLhtwAu48rw8S4qxtHi8i65VSPi/ajkjCPA0bNkwlJydbEjMRERGRmQIlYVamf1kAenk87+le5nMdEYkH0BFAvoUxEREREdmClUnYOgC/EZHeItIawCgAM7zWmQHgVvfj6wEs1N4ejIiIiCgKLGtcoZSqFZH7AMwFEAfgI6XUVhF5BkCyUmoGgA8BTBGRNAAFcCVqRERERC2epS1clVKzAcz2Wvakx+NKADdYGQMRERGRHXHEfCIiIiINmIQRERERacAkjIiIiEgDJmFEREREGjAJIyIiItKASRgRERGRBkzCiIiIiDRgEkZERESkAZMwIiIiIg3EaVM1ikgugH1R2FU3AHlR2E+s43GODh7n6OBxjg4e5+jgcTbHr5RS3X294LgkLFpEJFkpNUx3HC0dj3N08DhHB49zdPA4RwePs/VYHUlERESkAZMwIiIiIg2YhPn3nu4AYgSPc3TwOEcHj3N08DhHB4+zxdgmjIiIiEgDloQRERERacAkzIuIXC4iO0UkTUTG6o7HCUTkIxE5LCKpHsu6iMg8Ednt/t3ZvVxE5E338d0iImd4vOdW9/q7ReRWj+VnikiK+z1viohE9xPag4j0EpFFIrJNRLaKyD/dy3msTSQiiSKyVkQ2u4/z0+7lvUVkjfvYfCUird3L27ifp7lfT/LY1qPu5TtF5DKP5TzPuIlInIhsFJGf3M95nC0gIhnu/+1NIpLsXsZzh25KKf64fwDEAUgH0AdAawCbAfTXHZfdfwCcB+AMAKkey14CMNb9eCyAF92PrwDwMwABMBzAGvfyLgD2uH93dj/u7H5trXtdcb93pO7PrOk49wBwhvtxewC7APTnsTb9OAuAdu7HCQDWuI/J1wBGuZdPBnC3+/E9ACa7H48C8JX7cX/3OaQNgN7uc0sczzPNjvdDAL4E8JP7OY+zNcc5A0A3r2U8d2j+YUlYU2cDSFNK7VFKVQOYBuAazTHZnlJqKYACr8XXAPjU/fhTANd6LP9MuawG0ElEegC4DMA8pVSBUqoQwDwAl7tf66CUWq1c/+mfeWwrpiilDiqlNrgflwLYDuAk8Fibyn28ytxPE9w/CsBFAKa7l3sf54bjPx3AH9ylANcAmKaUqlJK7QWQBtc5hucZNxHpCeBKAB+4nwt4nKOJ5w7NmIQ1dRKAAx7PM93LKHTHK6UOuh/nADje/djfMQ60PNPH8pjmroo5Ha5SGh5rk7mryDYBOAzXhSYdQJFSqta9iuexaTye7teLAXRF6Mc/Fr0O4GEA9e7nXcHjbBUF4BcRWS8id7qX8dyhWbzuAKjlU0opEWE3XJOISDsA3wJ4UClV4tn0gsfaHEqpOgBDRaQTgO8B9NUcUosjIlcBOKyUWi8iF+iOJwaco5TKEpHjAMwTkR2eL/LcoQdLwprKAtDL43lP9zIK3SF3ETXcvw+7l/s7xoGW9/SxPCaJSAJcCdgXSqnv3It5rC2ilCoCsAjA7+Cqkmm4cfU8No3H0/16RwD5CP34x5oRAK4WkQy4qgovAvAGeJwtoZTKcv8+DNeNxdnguUM7JmFNrQPwG3fvnNZwNf6coTkmp5oBoKHnzK0AfvRY/ld375vhAIrdxeFzAVwqIp3dPXQuBTDX/VqJiAx3t//4q8e2Yor7838IYLtS6jWPl3isTSQi3d0lYBCRYwBcAlf7u0UArnev5n2cG47/9QAWutvFzAAwyt2rrzeA38DVeJnnGQBKqUeVUj2VUklwHYOFSqnR4HE2nYgcKyLtGx7D9T+fCp479NPdM8BuP3D1CtkFVxuQx3XH44QfAFMBHARQA1dbgL/D1VZjAYDdAOYD6OJeVwBMch/fFADDPLbzN7ga1aYBuN1j+TC4ThjpAN6Ce5DhWPsBcA5c7Tq2ANjk/rmCx9r04zwYwEb3cU4F8KR7eR+4Lu5pAL4B0Ma9PNH9PM39eh+PbT3uPpY74dFbjOeZZsf8AhztHcnjbP7x7QNX79DNALY2HAueO/T/cMR8IiIiIg1YHUlERESkAZMwIiIiIg2YhBERERFpwCSMiIiISAMmYUREREQaMAkjopglIhki0i3IOo9FKx4iii1MwoiIAmMSRkSWYBJGRI4lIkkikurx/N8iMk5EFovIGyKySURSReRs9+tdReQXEdkqIh/ANShlw3t/cE9uvLVhgmMRGQ/gGPd2vnAvGyMia93L3nVP9h0nIp+495UiIv8vukeCiJyISRgRtVRtlVJDAdwD4CP3sqcALFdKDYBr/ryTPdb/m1LqTLhG/n5ARLoqpcYCqFBKDVVKjRaRfgBuAjDCve06AKMBDAVwklJqoFJqEICPo/IJicjR4oOvQkTkSFMBQCm1VEQ6uOeDPA/An93LZ4lIocf6D4jIn9yPe8E1B2G+1zb/AOBMAOtcU+ThGLgmPZ4JoI+ITAQwC8Av1nwkImpJmIQRkZPVommJfqLHY+852fzO0SYiFwC4GMDvlFLlIrLYa1uNqwL4VCn1qI9tDAFwGYC7ANwI1xx7RER+sTqSiJzsEIDj3G292gC4yuO1mwBARM4BUKyUKgawFMAt7uUjAXR2r9sRQKE7AesLYLjHdmpEJMH9eAGA60XkOPc2uojIr9w9LFsppb4F8ASAM6z4sETUsrAkjIgcSylVIyLPAFgLIAvADo+XK0VkI4AEHC2VehrAVBHZCmAlgP3u5XMA3CUi2wHsBLDaYzvvAdgiIhvc7cKeAPCLiLQCUAPgXgAVAD52LwOAZiVlRETeRCm/JfRERI7krk78t1IqWXcsRET+sDqSiIiISAOWhBERERFpwJIwIiIiIg2YhBERERFpwCSMiIiISAMmYUREREQaMAkjIiIi0oBJGBEREZEG/x/jJgqBiC4GrwAAAABJRU5ErkJggg==",
            "text/plain": [
              "<Figure size 720x432 with 1 Axes>"
            ]
          },
          "metadata": {
            "needs_background": "light"
          },
          "output_type": "display_data"
        }
      ],
      "source": [
        "fig = plt.figure(figsize=(10,6))\n",
        "ax = fig.add_subplot(1,1,1)\n",
        "ax.plot(train_losses, label='train loss')\n",
        "ax.plot(valid_losses, label='valid loss')\n",
        "plt.legend()\n",
        "ax.set_xlabel('updates')\n",
        "ax.set_ylabel('loss');"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 30,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 388
        },
        "id": "NfqDTZGHt1Qv",
        "outputId": "978ea7a8-684a-49fe-d643-a2f76bad6de1"
      },
      "outputs": [
        {
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAmEAAAFzCAYAAAB2A95GAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3deXxcdb3/8dcnyWRP2qTpnrZpsfsS2qal0gKFUikgZdFSNtkUriCgomgFxV65ehHRq7j9RGVVWQRlRxAp4AZ0YSst2AIFWrqv6ZJm+/7+mMl0kkySSTIzZ07m/Xw8+ujMmTPnfM/3nDnnne/5nnPMOYeIiIiIJFeG1wUQERERSUcKYSIiIiIeUAgTERER8YBCmIiIiIgHFMJEREREPKAQJiIiIuKBLK8L0FllZWWuoqLC62KIiIiIdGj58uXbnHN9o33muxBWUVHBsmXLvC6GiIiISIfM7P22PtPpSBEREREPKISJiIiIeEAhTERERMQDCmEiIiIiHlAIExEREfGAQpiIiIiIBxTCRERERDygECYiIiLiAYUwEREREQ8kLISZ2W1mtsXMVrbxuZnZLWa21sxeN7MpiSqLiIiISKpJZEvYHcC8dj4/ERgZ+ncp8MsElkVEREQkpSTs2ZHOuRfMrKKdUU4F7nLOOeBFM+ttZgOdcxsTVaaYLbsNHvsyAGsaB5NLLUMytgKwyZVQzH7y7SAAG1wf6rJLqKhbyweuH0Xsx3DscEUU237KbA8AW10xfUOv32UwI9jQbJbr6U85m1kTGM2QuvfYQwH92Mlel0uh1dDgjExzHRZ9kyvBYTRi7HaFjMt4n4Nkk0MtAG82DqMwq5G6hkYyaaDa5THO3icDR0Zo+tUujyI7wEZXykDbwT6Xwz7y6Ge7AKh1mRy0XOoyciht3AHA+439GJaxpVlZ9rkcsqknYA2tyrm8cSTVLp9xmR/Qj52tPt/iSmiwTPqxg0waAThINusa+zE6Y32zcbe7IvpYNdtcMWW2h4MuQI7VhT+PrPv/NA5mVEbzut/lCvjIlTEuI/h4r52ukBLbCxCug2CZetPPdlHrMqkni42ulMMyNrK6cQiNZDA+49DjwepdBlnWGFGGXpSyJ7wOGzAyab4+68ngQ9ef4db6J7Df5dBABkV2gP80DubfObP4TeAsPtxxIDxObiCDmrpGygpz6FuUw9ub9lCQk0V1TT05WRmUFeawYdeh8QtzssjLzmRr9cFW8wMYM6CI+kaHc47ahkY27zlIbX1wmQb3zgtPK8Og0UG/ohwyM4yNu2vCw4tyA+w+EFwXQ0rzyA9ksbm6hpq6BrIyMsjJyqDROXburwuXf8yAIrZWH2T7vtpwWcYNLGbVxj3NlrV3XjYAm/bUhIcHMo26hub12isvWIb87Exq6hooyMniYH0juVkZ9CvOZe2W4LrOycqgV16Akf0LWbN5L9v31dLQ6CjKDdZhINNoaHRkZhyaR7+iHAb2zqN3XoD/bK4OL3tRbhZZGYYDdu0/tC1G1kFpQTbvbt3HyH6FrAmVoTg3izEDiln+wU4aGg8thxm40NuP9Stk0+4a9h6sb7au3tpU3Wy5MwwmDO7Fzv21fLjjAH0KspvVaVaGcVjfQtZt38fAXrnsPVjPoN55mBnVNXV8uGM/vfOzyTDIysjgo90HKMzJom9RDrlZmeyvrWfd9v3N6r4oN0B1TR2DeufxfuizssJsApkZbN5TQ9MilRVmU5CTxZY9BzlQd2j/kBfI5LB+BRysawzXCUDlkN6s3Rxcvn21Da3qJJIZBDIyqG1opE9BNuWl+ezcV8sHO/a3HjlivoW5WfQvzuH9bfupDtVtWWE2ew/W07coh/oGF16/OVkZjBlQxGvrd0et/0CmkZWRQWlBNht2HaBqWAn7axtYtXEPI/oW8O7WfQQyjezMDPJzsnDO0dAY/B1AcJutb2ikvtGRlWHsq21geFkB723bB0BRThZmkJlh4e9kZ2XQvzin2T6hLYU5WQwpzWf1xj1kZhgl+QGKcgMMKc1n3bZ97K+tZ/eBOnrlZZMbyGD9zuA0K/rks277frIyjEBmRrN1l5lh4W12QHEuJQXZrA79ZnvnBxjVr4iNew6Eyxe5D4FD+xGAUf0LeXfrPnrlBeidH+Cdrfuo6JPP4JI8/rl2O6UF2QQyjc17gvuusQOL+WjXAQ7UNpCdlRH+bWRnBtfBpj01mEFpfvA3kBvIIJCRgQMeu3IWFWUFHdZZopiLthXHa+LBEPaYc25ClM8eA250zv0j9P5vwNedc62ezm1mlxJsLWPo0KFT33+/zWdhdt+6f8AdJydu+iJxcmntl3m6cZrXxRAR8bV1Nyb2mG9my51zVdE+80XHfOfcrc65KudcVd++fRM7s+1rEzt9kTiZmPGe10UQEZFu8DKEbQCGRLwvDw0TERER6fG8DGGPAOeHrpKcAexOif5gIj5hJK4rgYiIJF7COuab2T3AbKDMzNYD3wYCAM65/wc8AZwErAX2AxclqiwiIiIiqSaRV0ee3cHnDvhCoubfdeZ1AURiopYwERF/80XHfBFpTX8uiIj4m0KYiIiIiAcUwkREREQ8oBAmIiIi4gGFMBGfUsd8ERF/UwgTERER8YBCmIhP6epIERF/UwgTERER8YBCWEum9gURERFJPIUwEREREQ8ohIn4lq6OFBHxM4UwEZ/SLSpERPxNIawV9QkTERGRxFMIa0WtCyIiIpJ4CmEiPqU2WxERf1MIExEREfGAQlgral8Qf9CJcxERf1MIE/Ep/bkgIuJvCmEiIiIiHlAIExEREfGAQlhLenak+IRu1ioi4m8KYSI+pRAmIuJvCmEtOR3YxB/UZisi4m8KYSI+pZYwERF/UwgT8SmFMBERf1MIa0kd80VERCQJFMJEfEotYSIi/qYQJuJTCmEiIv6mECbiUxkKYSIivqYQ1or6hIk/qCVMRMTfFMJEfEp/LoiI+JtCmIhPqSVMRMTfFMJEfEshTETEzxTCRHxKpyNFRPxNIUzEp3Q6UkTE3xTCRHxKLWEiIv6mENbC9lcf9boIIjE5SMDrIoiISDcohLWw8oNtXhdBJCZ3NnzC6yKIiPjalKG9PZ2/QpiIT73jBntdBBERX+uV5+0ZBYUwERERSUtm3vauVQgTERER8YBCWAu64kxERCQ9eH3MVwgTERGRtOTx2UiFMBEREUlX6hMmIiIiknRqCRMRERFJQwphIiIiIh5QCGvBeX6thIiIiKQDhbCWlMFERETSgteHfIWwlpzXBRAREZFkUMf8FGNKYSIiImnBdIsKERERkeRTS5iIiIhIGlIIExEREfGAQpiIiIikJZ2OFBEREfGAOuanGq9vGiIiIiJpQSFMRERExAMKYS3osUUiIiKSDAkNYWY2z8zeNrO1ZrYoyudDzWyJmb1iZq+b2UmJLE8sFMFEREQkGRIWwswsE/g5cCIwDjjbzMa1GO2bwP3OucnAWcAvElWeWOmO+SIiIpIMiWwJmw6sdc6965yrBe4FTm0xjgOKQ697AR8lsDwxUQQTERFJEz34FhWDgQ8j3q8PDYu0GDjPzNYDTwBXRpuQmV1qZsvMbNnWrVsTUVYRERFJM153QfK6Y/7ZwB3OuXLgJOBuM2tVJufcrc65KudcVd++fZNeSBEREel5zOO7tSYyhG0AhkS8Lw8Ni/RZ4H4A59y/gVygLIFlEhEREUkJiQxhS4GRZjbczLIJdrx/pMU4HwBzAMxsLMEQ5un5Rq+bJkVERCQ9JCyEOefqgSuAp4DVBK+CfNPMvmNm80OjfQW4xMxeA+4BLnTOqW+8iIiIJJzXDS9ZiZy4c+4Jgh3uI4ddH/F6FTAzkWUQERERSUVed8xPOWqGExERSQ8e98tXCBMREZH05PXpSIWwFrxeISIiIpIcPfkWFSIiIiIpy+trARXCREREJC2pJSzFqGO+iIiIJINCmIiIiIgHFMJaUMd8ERERSQaFMBEREREPKISJiIiIeEAhTERERMQDCmEt1NQ1el0EERERSYKP9Sv0dP4KYSIi4qlTKgc1e//3rx1L1bASAAKZwculfv+5I5qNc+GRFay78WTW3Xhy1Gl+Ze6oNudXVpjT5mfXnjQmPN1jR/eNqfwAPztnMutuPJnnr5kd83eaFOVktbss0Vx4ZEWz9w9/YWbU73dmmk1evX5ut6fz6anlAJTkB9od77/nj6d3B+NEK0tkeVZ954ROfT9SQXZml78bDwphIiLiqVjuWu7FletduZGnxzdgF59RCBMRkZSTyCzj8U3S46InLEOT7gZXPwdfhTAREfFUey1O5uHdGzsz524FgS4sYst66UmhLJn02CIREUlr7Z2OdAloE/Nzy4n0LAphIiKS+jrZYNHVnBUZ0FK5dSmVy9ZZsfQJ7KkUwkRExFNdOQR3J4TE/l3/JJ14nrb18hRwulEIExGRlJWIRpJENrz0pBaqeEhGG5ef29EUwlrx8+oUEemZOts6E48s1JlApSNHc526qCFhpeiY16FZIUxERLzVgxJMV1rZupIDWn7H6zAhXaMQ1oq2ZBGRZIp2BWQ6d9aOhUJXz6AQJiIinkpIv684fC+Vc47X97eS+FAIExGRlNMUMlz4vXdlkQTr9h3z/dtqqhAmIiKeSn7Aiu2gncrBL4WL5ite16NCmIiIpCwvWzmSdb+sLp1aTGTRvE4maUQhTEREPBVLzlIuaK4n3VDVvycTu08hTEREPJVKN2Tt7mOLkhUoWpYt1U6dpnOw6gyFMBERSTmJPA3p437cvtNRNoxHePTz6lQIa8XPq1NEpGdq2W8qGafjOnXH/B6U7JLdquZp3XnchKgQJiIiEkUq97tqdcf8OJa1B+XJlKcQ1krq/uhERHqiaHfMT6RU6z/VFT1hGUQhTEREPNZey0vX73zf9jfbn19kz/wuzlw6pbsR3M8tdwphIiKS8rxo+UnlDNby9KNaxrrG62pTCBMREYkTrw/q4i8KYSIiknJS4QxTV+5k79V9wuIqjguRCusxlSmEiYiIp2I5UHc2c6TylY0tpdpTi+KhM+Xzc5+u7lIIExERTyXkjvldbINxPu2Xn9Z9wnwc4hTCWvHx2hQR6WESE9Bi07mbtXapKF2X1qkrfryuRoUwERFJefE8WPaEu9snNDsk+475adz4oRAmIiJpJdYO935qa/JTHzg5RCFMREQ81rolRJFC0oFCWCv66YuIJFM8zw42NXK1e1f8GGfYpVtUdGFhunLU8bovUzx1d/37+XSmQpiIiKScrraNZcQxnXTqNgvJfv6l7pgfF16fxlUIExGRtNKVFq5U0wMWQVAIExGRNBPzKUMFHUkwhTAREUl5LVt+/NwPKB56Uj5M5zWpENZKOm8OIiLJF8+9blM46eo0I1vJOtNfyOtbj8U1lPnsMOh13XeHQpiIiHgqnjdPjWvH/CQ1N3Wlj1pGRk9qC/OO133rFMJERETipCd0+k86H7dkdZdCmIiIpJyWjWNeRJuuzNOrRyIp+/mTQpiIiEgUqRxsUrlsEjuFsBZ2U+h1EUREepSS/EC7n08f3qfZ+4KcrPDraRUlAJQV5jQbZ/ygXlGn9cnKgQCM7l/U5vzmjO3f5mcjI743cXD0eUQzrE9+zOO2NOtjZZ3+ztgBxc3eF+e1X8f52ZkxTzsz81DCG9W/a8fEw4f2BjpethFlhRwzum+X5tEkJ9D1KFPRp6Bb8+6urI5HSTOF/eBA577yUMORnJb5r/D7epfBTgrpa3viUqSFB7/FBsq4KvNPlFg1czNXhD9b3TiEsRkfdmp6Pym+hi/u+UHM499Ut5CvBe5rNuzvDRN43/XnvKy/AXBH/Se4MOvpTpUD4Lq6i5me8RZvN5YzOeMd5mYu7/A71S6PIuvkSmrD/fXH8GzjZAbYDhYH7orLNKO5pf40alw25baVc7KWhIc/2HAUqxqHMdo+5Mys5wFY78rY4Mo4IuOtTs3j6/PGUNEnnzVb9jKkNI8+BTmYwc+eXctL7+0A4JwjhvKHlz5o9r1PTSnnwRXrAfjBpydRlBvg878LrodHr5jFKT/7BwA/Xng4YwYWMe/Hfwfgu6dPoK6+kcWPrgouy2VH8vCrG8gLZPKrF96NWsZzjhjKhEG96FuUwyV3LQNgwdRyXl63g/e37w+P9+OFh/Ol+14F4MQJAzh2dD/e/Gg304aXkp+diXPw2TuD3795QSV3/msdb2zYDcBPz57Mlfe8AsBL187hiO8Ft9HR/Yt4e3N1p+oUYPEp4xjUO49L7z60bb7yrbncs/QDinKyGNqngB37DvKnFRsoL8njnpeb/x6/OGckP/nbmvD7ssIctu092Gycc48YSnFegPU7D/Doax+Fh1/w8WEMLsnj7U17w+vo5+dMwQze3lTN1GEl7Nxfy4r3d7KvtoEHlq8Pf/d/z5jIA8vXs/z9na2W6VufHMfEwb0481f/BmDGiFJG9y9iyrASauoa+PiIMvJzMqn6n2cAuOaE0WzaXcOoAUVUlvdi/s/+CcDUYSWcNnkw33poZat5XHHsxzh50kBe/XAXC6aWs21vLas27mb8oF68u3UfAJkZRl1DI0ce1ocxA4rYX9tAeUkepQXZ4el8bd4YSguyGVKaz0vXzuFvq7fQKy/ASRMHhMd5+bo5NDQ6Vn20hzlj+/P5Yw5jVP8ipg8/nt0HatlTU8/g3nnUNzoO1jUwrE8Bnz/mMI7/UfA399q3P8GbH+2md1424wYdCjfnzRjGjBF9mPt/LwDwzNXHkJVhbNt7kIG989hafZA31u9ixog+zcJbkwcv+zh9CnLIMGN/XX34t/PiN+bwygc72bm/jqnDSqgoOxTgVnxrLg2NDjNYtm4HU4aVsGzdTuaNH8BfV2/m2NH9eH/7Pkb2L+K5r86mb1EOm/fU0K8ot9m8l33z+PAp3ZeunUNuViZvbtzNOb9+CYB/LjqOpe/tIC87k6KcLBodHKxvoLQgm0BECPvT5TPZfaAuWM/XzmHXgTr6FeXwzta9VNfUkxfIxAFn3foiALddWEV5ST4j+xUy87AyBvTK5dqTxlKQk8UrH+xkUnlvVm/cQ3lJsP6qKkqpqihh7Za9fPKnwX3N018+mt75Ad7Zso+zf/1ieN7TQ7/llvKzs1jy1dkce/NzANz92ekMKM4Nr7efnj2ZsQOLaHQwtDSfO/61jvGDiskLZFJVURp1mskSUwgzsz8BvwWedM41JrZI3soPZLYZwl5qHNPswHjAZZNntTza8PFmIexjB38HwKyMN/hd9v8C8GLjWGZkrKbGBci1uk6V6SU3FoCv1f8XAOsyzwl/dkP9Z/hD9vc6Nb0vXv1N3r7+HkZnrO94ZOAXDae2CmH/VXc1+8kNh7DF9Rd2KYT9vuF4ft9wfPBNA3zZPcAXs/7UbJyWIXDiwd/y+8B3mZn5ZnjYfpfDS41jODbztU7Nf+W073FF1RA++dN/dCuEfdDYl6EZW9v8/Ef1ZxLINOrqXTiEVbs8vlJ3GQD51IRD2MVZN3Jw/x6ez7k6/P1XGw/jtaKjuWDf7W3O47LZhwFwYovhU4aWMP7bT5EbyOB7p09sFcJ+eGZl+AC/oGpIs88mlh9qCTht8uBmn517xDCAcAibOqyEqcOCrRZvbNjNv97Z3qqM3zt9Yvh1aUE2O/bVsujEMaz4YFc4lDXNqymE/fK8qaGhzcs2qn8h/9m8l4mDe3HxrAq+fN9rnHr4IIaUBg9oleW96F+cy+DeeWzYdYBrTx7LBbe93KpMHblw5vBWw0oKsrl89seaDTt9cjkAS9ftZO2WveHhX547qlkIu+nTE7n4jmXNvvvdUL08+cbGZiHs87MPY2CvPIDwOjp5UrCl56SJA8PjnXr44PBnF92+lKNH9eXs6UM5e/pQKhY93qr8n53VfJnuvfTjUZe9yeWzD4va4fzTU8s5pXJQqxCWnZXBV08YDcDYgcFQM6BXLgN6BYNC/+LmgQHg2DH9os47M8M4rG9h+HvnHDG01ThNAaSprkaFAlHfohz6FuW0Gh/gY/0OtfD0ygtw5GGtW2zMrFm4avpORVmw9WRw7zwOH9I76vSHlxUwdVj0A/yAXrmcGLH+IkUG0HkTmq/rE8YHg2dTmZrKMaJv69aq0vzs8BWUTfUduYyDe+cxuMVvusnB+gYAAplGYU4WhaGWyX7FufQLTavlsg0ozmXTnhrGDChmUO+8ZuVrej97dHAdzwy1jg0LtULlBjKbrY+m9RcZLPtF2WYiDS871KJ11MjmLWunVA5q9v7zxxzW7rSSKdY2vF8A5wBrzOxGMxudwDL5jvPhbfP8WOZIloaX03R3jSWzv7BX/VX8vVW31tl1luzlb2t+PW09pKuuPFdRfdU6J6YQ5px7xjl3LjAFWAc8Y2b/MrOLzKz9E9Ei0mmtQ2bXE5R2iunHqyv0JHXod+8PMfdmM7M+wIXA54BXgJ8QDGV/TUjJPJIu221PbEnq6rpLtZ1VW+smxYoZNz1tS1QASm9a+12TavvhZIkphJnZn4G/A/nAKc65+c65+5xzV0LblxOa2Twze9vM1prZojbGOdPMVpnZm2b2h64shFc6c0qvJ4aeVNKzajfadmX0tKVMlLZCkB/CUeqXMDbpekCNpCqQWMR6deQtzrkl0T5wzlVFG25mmcDPgbnAemCpmT3inFsVMc5I4BvATOfcTjOL3jMzxbUVxqLtUFOlL1aqlKOniKU2LdYgFfUI1v3Dc6oeGI2uBaTIr0T9eooucGf62XR2EZJ9t3Yzf4RbkVQV6+nIcWYWvgTEzErM7PIOvjMdWOuce9c5VwvcC5zaYpxLgJ8753YCOOe2xFielKBdT/zFWqepeXgVL0TmDj0yRkT8JNYQdolzblfTm1BouqSD7wwGIm+Ysz40LNIoYJSZ/dPMXjSzedEmZGaXmtkyM1u2dWvbtwFINTocpA+z+Eby6CckuzePdG+wSIeA1vOXUFKV6+b+qStXYvYEsYawTIvYg4VONWa3M36ssoCRwGzgbODXkS1uTZxztzrnqpxzVX37du/Ouh3qwnbgx2ObH8ucKF79+Du9DroYIrxYvnTdoXZXvIJysgK31nPP1J3tR9tE58Qawv4C3Gdmc8xsDnBPaFh7NtD87orloWGR1gOPOOfqnHPvAf8hGMo8k6idlzrmx6Yn/3zb+0sxcvswXKvtxejpdZOIiXrzm/Pyl54GjX2STNqeEi7WEPZ1YAlwWejf34CvdfCdpcBIMxtuZtnAWcAjLcZ5iGArGGZWRvD0ZPTnnaS0jrfUpoOqOsSnFq0N70R26O5uXor8flunHdOpA3l3Tw211N6p3HQ4zetHWi/+ENPVkaFHFf0y9C8mzrl6M7sCeArIBG5zzr1pZt8BljnnHgl99gkzWwU0ANc451o/66QH6SktYl4vRTz7YCVrXxXr1ZGJKk6q7pPjebCI2pcuycvd4ewSWJ5UORUUj3J4vY+R5ErV/VOixfrsyJHA/wLjgPADnJxzI9r7nnPuCeCJFsOuj3jtgKtD/1JCZ7YDf7dq+bns8ZNOO/o0agjyrXi1YCVtXSfhFhXaU3lI+4yEi/V05O0EW8HqgWOBu4DfJapQXurMNteZVi3tSGKj33xQPFtM0+0vzLZCQTqcnkmDRZQk6Mp2pD/yuibWEJbnnPsbYM65951zi4GTE1csf/F3i5jE48DVU04zx0tKhAGPCpEKW4LXB8SUWP/iia6u+3TdZGK9Y/5BM8sA1oT6eW2gnccV+Vk8T0fqwJw8CsJ+1817oMWpFH6nX4GIv8TaEvZFgs+NvAqYCpwHXJCoQkni+T20RAu4fg69zW5REWXVdGfZvG4VaUtCitXOZp0qV0cm45cX76sjJb2kyE8lLXTYEha6MetC59xXgb3ARQkvlU90Jcj4PfxIN8S86qOP2N0tJ1VPESW6WKm23Antm5Ziyyo+p+0p4TpsCXPONQCzklCWHq2pJcPPrTWpTgG3fen+1206dMxv4vW6jktNe70QXeTTYnsunX6fkWLtE/aKmT0C/BHY1zTQOfenhJTKJ/z8W/Nz2eMtafdWaqfSrdnraHfMd10+svlh39aVA1eHX/HqaOjhjyvZ9wmL7c533ZyHHzbgaHxabEmuWENYLrAdOC5imAN6XAhTOEkv2k+mhu787ozm/b18e9Amjs+OjM9kRGKmba5rYr1jvvqBtUMbn+g0c2pIlTvGeyVV8qefg7B0j9Z858R6x/zbiZI1nHMXx71EPVRPO0jrh5Z8Xa3zdOuj0tZVkKlydaR0jtZbekjXY0qspyMfi3idC5wOfBT/4oj0YDHuZdSIECehiky11rHOlCbVyt5SYq/0TO1lF4mHWE9HPhj53szuAf6RkBL5SvruJFKxZc/LMsV7S4jn9HQs84+4bcE94epIkTQQ681aWxoJ9ItnQUS8kmohpa3ipFgx46q7Z5xS708Cb/TkbUSST9tT4sXaJ6ya5vu5TcDXE1KiHi4VW5D8qGU9utCNHTrL4Y8djR/KmApSLVB7JZl3zE9Yly31BfNcMtdAuv52Yz0dWZTogqSKRG0HqbZ9peqNTZ1LzXJ1ROFaEqGzYSrZVyUmY2660tIflJm7JqbTkWZ2upn1injf28xOS1yx/CVVA00iKXQkTry3Ji92jrEcOJNdLj9ssfG6EtDzA2L67RKlidZ9p8TaJ+zbzrndTW+cc7uAbyemSN5K9L4rHQNbOohlvcb+6EhLq5Cb6IaOVGtI0QWFItIk1hAWbbxYb2/RY/n5MNndsifstK0lv1aTdeBqb8kiQ5fRuqWxO6HMDwfmbvdhivZ117IOkyMV9gvJKoMfti3xh3Q97RxrCFtmZj8ys8NC/34ELE9kwXqadGrZSIZo9dmVOo7Xz17rt3vidfosPXfjh6TK8qdKOURSXawh7EqgFrgPuBeoAb6QqEL5RUenoHrygbknL1tXJLo+dBq7C3rAX9ZdvVmr7jIv4g+xXh25D1iU4LKkBYWXxOnJQcVwPSFTxJWCRmuebCMJWg3+Xbv+LXlL2uUkXqxXR/7VzHpHvC8xs6cSVyxJtO4HlsTsaPx7i4rUlapZJV4hKtbgkaLVkBA9qU9YKv+22uPXcnddOv3C4ifW05FloSsiAYA49J8AACAASURBVHDO7aSH3jG/Kz+cntwCI/ET+1bi/y2qs89H7O7uu72O/cmuy47CZWKfB5kaW066drKW1H/eaaqJNYQ1mtnQpjdmVoFibyf1rOry+mfWcv7dq12vlyax0ul46PdljVerZaq2fopIc7HeZuI64B9m9jzBI9ZRwKUJK5WHOrPv8n97hfiFtjSJhd9DqEi6ibVj/l/MrIpg8HoFeAg4kMiCSWJ1N0DqAgOR1JW0PmFt7EcUBkViE+sDvD8HfBEoB14FZgD/Bo5LXNFSn2JIalEwlHSn7CPiL7H2CfsiMA143zl3LDAZ2NX+V/ypzgJtfrbf5TR7v8/lAdDQRjXWkxl+XUPwu9XkdbeIcXGAnI5HakeiTsXW0rr+D0b5W6HluthPbrP6jlV2VgaZGd1fltoY/p4pzGk+zn5yw68j6zM/O6vVNrWfHCwru1tlLMjp3EMu8rM7X59N8gIdfzdcHwaBLqyDpuUxMwKZwfrKycokKyMjVIbg//nZwfECGbHu7rqnaX5tycpse1lbftbZFqWmbblp2bsrJ6v96WRmWNTkV9BBHcSiaRvK8FmzWlN5O9oOEiU7M37beWf2GYd+j3GbfVqItYZrnHM1ZoaZ5Tjn3jKz0QktmUeeKF7I+u17aCCDq7Ie4qraK/ivrEf5Vt1FfOD6c07j37g68AAAp9bewPlZT/Ni41h+W3w5fXe+Qi614Wn9u3EcP6k/nX6Z+/hB7Rmckfl3Puh3LOds+ymzM19j/sEbeCTnW83mfyCziLyGagCeb5hE5rwb4NF9zcZ5vGE6J2e+HH5/6sHv8HDO9awsmME9u8YxqU8jz28r5oiM1VyQ9VcALqz9Gndk3xT+zvpjb4Hn5wKwtOAY1u7O4OysJQCsGXclN2+Zxpe3XMct9acDcFPR1/la9fcBeL3iYubkjaakIBuWHSrXxbVf5bbsm3mzcRgHCfCnhqNY48r5v8DP+UbdJfww8Et2l07ksJ3/5P8Kvsxb+wrC3/3myWP5n8dXc1vDPIbbRk7MfJlasni638Xc/eEUvh64nwwa+fCom/iv2hF85fnLuL/yLYavvAWAJTNuZ8yWJ+C9V8LTfCVzEpMbXufpxioerj+SS7Ie5699L2BQXj01LsDSDQf54fGjyM/O5PAhvVm862qumVTDewziqRdfZcawQmZ+dEezun+ncSCHZWykIbeEp2snsTezF6P6FUDJMMpX3hh1m/r3qK/z4MwjKckPcNwPn2eB3cyvcn7CB3Nvhz9uAaCafH7XeALnTRvID2adwKOvfkR1/Tf4zLN5PJRzPV8OXM+fzz+ag7c+wqP7J/KpC7/MQ5njWbO5mroGx6TyXlHnDZAbyOSbJ4/luDHBC5rvung6+2sbKC/J440NwUfC3nZhVbPvfPuUcRw1sm/U6T3w+Y+zfueh3gg/O2cypQXNA+L3zpjIqAFFnD19CA8uX8/NT/+n1XTuuXQGT725iV55AT4xfgBXHvcxKvoUUFoYnNZDX5jJms3VbS7X/ztvKn9+ZQMjygoYVprPu1v38V/HjKAwJ4uvzB3FwmlDAPj1BVU8/OoGZn6sD1+bN5qb/vI2AM9fM5tjb36Ov3zpaM759Yv88fNH8tGuA+w5UEdxXoBzf/NSs/n99/zxbN97kFlt1EuTX19QxUOvbGBaRSkbdwfr6aZPT2LfwXp65QWYXlHKF+eMZNf+WqZWlBKZP0+cMJArj6tm3oQB/H3NNvoXHwrqNy+o5LC+BS1n18zEwb24eu4ozgotO8CPzqykvsGxv7aefsW59M5r+w/Nlh6/ahZ/X7Ot9TKeX8V3H1/FiRMGkJWZQWV5L4pyA0we2puX3t3BDxZMinkebfnp2ZO5d+mHTBhc3O1pteXmBZWM6KBOAb53+kQqh7T9G4s0vKyAa04YzemTB7f67H/PmMjEwbFNp6sev2oW/3pne5ufP3blrPDvvi25gUy+9clxHDu6/W090t0XH8GTKzdSVtj1P/CPH9ufyUN7Nxv26BWzWLUxWN4/XHIEO/fVhT9ruSwt3//2gqqUD/EWy716zOzPwEXAlwiegtwJBJxzJyW2eK1VVVW5ZcuWdTxiF110+8sseXtrs2FnTBnMn1ZsCL9fl3sOABU1fwi+v/FkAJ59azMX37GMY0f35faLpked/ovvbuesW19sNa2wxbupa2hk5HVPNpt2xaLHg//3yee5a46FO0+B917gW72+x92bK4LT/sYcBvTKDY8/3tbxeM61vNk4jJNr//fQvBY3/wG+tWkP/3fLj/hV9v/xVEMVJ9zwNwBm/2AJ67bvD5ejqQxNZQpOq1d4mk2fR3riqqMYN6iY4d94HOdgzXdPDLdafOX+13hwxXoA7rlkBh8/rA+vfLCT03/xLwAqy3vx8BWzotZjtPmz5Hvw/Pdh9jdg9qF7C9/6wjt874m3uOSo4Vx38rj2p9fW9EMqav7AK9+aGwygkZbdDo99CaZcwF9GXMvnfxd8qtfS646nb1HbO6U/rVjP1fe/BsCXjx/FF48f2Xx+0eo8yeJRhlRYjiaxliWVypwo6bCMIl4zs+XOuapon8XaMf/00MvFZrYE6AX8JU7lS30xdjVKxvPv/NrrqSt149dlFRERiUWnT1o7555PREF6Eq9uVNjeDStTRWo3DIuIiCRPcnqq+kiq3+m5VYuSj+/K6IfQ2F0pvjmJiIiHFMISIJEPFvZbcIm1vE3jdWvp2qj35OZUf60fERHxjkJYHKnVI1W0dQPJBK4grXwREekkhbBYxPH42t1J+fjsY7v00FcREUk3CmE+0zKEtZ/JYjwV2EODnYiISCpTCIuFz0OKHjSeuhSARUTSl0JYCkr1KzQlMfx20YWIiHSPQpjPtLzyMpGH7USEwfQImApTIiLSMYWwOOppp5YSeauNWKRDXBMRkfSlEJYAiWztaRmL4jGntGiciiPVl4iIxINCWAx6WANXUrXbmBblM9+3Jvp+AUREJFkUwlrw+hRcRzp3iwp/89eyqXlMREQ6RyEsybp7qrK9K+hSPD/2cJGVr0AmIiIdUwiTlNBj+ll1ckGUm0VE0pdCWAySfYqyvcN466J0XDZL80N909LHK+epxVFEROJBIcxn/H78tzbfJI6LdwrreI7JmpGIiPiYQlgM/H5ITZ/HFvl9TYmISDpRCGsh1Q/jra6OjEOBU+H0WlzL0GM6mImISE+mEOY7yUtM6fGIIREREW8ohLWg2JFEUfNkCjTLdUc3mvRSoUVSRESSRyGshVRv/dGBOlVF325Se2sSEREvKYTFIJ4H0u5mvJYZTKHM31L9CQ0iIpI4CmExSPZhMsUb4xKiJy2zcpWIiMRCIcxn/N5y0pmw1elc5vO6ERGR9KIQ5jOKGbHoQc1qIiLSYymEteDnlib/ljy6nrY8IiIikRIawsxsnpm9bWZrzWxRO+N9ysycmVUlsjyJloz85uOM6BmX5DjXk/q3iYhI4iQshJlZJvBz4ERgHHC2mY2LMl4R8EXgpUSVpTOiHa47G3wSeQzuTEtduj+4uyXTaUoREUkhiWwJmw6sdc6965yrBe4FTo0y3g3A94GaBJalx4olk3U1iiU6sigiiohIOktkCBsMfBjxfn1oWJiZTQGGOOceT2A5OiXV20oSEVwSeYqzs9P2/+lW3y+AiIgkiWcd880sA/gR8JUYxr3UzJaZ2bKtW7cmvnAtxPXZ0t2dgI7x7fCwciI6gnUmSGp1ioikr0SGsA3AkIj35aFhTYqACcBzZrYOmAE8Eq1zvnPuVudclXOuqm/fvgkscmpI9UcndUdSls2n1adAJiKSXhIZwpYCI81suJllA2cBjzR96Jzb7Zwrc85VOOcqgBeB+c65ZQksU5d09rYViTyY6kAtIiLSMyQshDnn6oErgKeA1cD9zrk3zew7ZjY/UfPtru6EnGQ08vj5PmYiIiJySFYiJ+6cewJ4osWw69sYd3Yiy9JTKZKJiIj4k+6YH0dJuVlr4mchIiIiSaAQlgCJvVlrAicuIiIiSaMQFoOefLWiiIiIeEMhrIXszNZVsmBqefj18LICAG6qO5MJg4uZPry0U9P/WL/CLpUrJytYrq+eMDo44IjPA3D0UUeHx+lTkN1sPqPGTALg7oz5XHhkBUz+DBQOiDr9VxsPA+DZwpPDwy4/9mPNxinMyeKimRVtlnFQr1wAinMPdTUc2icfgK/PG9Nq/AVV5a2GRdbP544a0ea8wkbMhvJpwdfjQg9kGH1Ss1HmjOkPwMkTB3Y8vZZmXA5AIxnUu+A6KMiJ0pWy4qjg/5XnMHloSXhwUW6g3cl/fESf8OsTxvdv9fm88QMY2cVtJl5G9S+MWrbOOH5sf8YMKIpTibpnWJ985lcO6nC8w4f05qiRZUkokXeuadqfiIgnzG9X21VVVbllyxJ3F4vLfrecJ1duajZsxbfmUhoRcNryzKrNfO6uZcwZ04/fXjgtthku7tXi/e5Yixo3qz7aw0m3/J0xA4r4y5eO7vgLcXTmr/7Ny+/t4N5LZzAjIpCIiIj0BGa23DnX6h6ooJYwEREREU8ohLXgs4ZB/1N9i4hImlIIE1wKJCFd+iAiIulGIayFVAgkXtFVoCIiIsmjEBaDWKOJ3+Ob3y7SEBER8TOFsATwW4OS6WSgiIhI0imEiYiIiHhAIUxERETEAwphCaCuVZ2nKhMRkXSjENZCdwKUelZ1gSpNRETSlEKYiIiIiAcUwsRbOg8pIiJpSiEsBn675YQfqYpFRCTdKITFkRp1REREJFYKYQmgljMRERHpiEKYpPXzMkVERLyiECZheoC3iIhI8iiEtaA2IREREUkGhTARERERDyiEtRDtjvmmGyiIiIhInCmEiYiIiHhAIUw8pSszRUQkXSmExZHrztO/05yuzBQRkXSjEJYQChQiIiLSPoWwhFCLWGepFVFERNKNQlgrXQ8DOqXWebryVERE0pVCWCyUE0RERCTOFMIk6r3RREREJLEUwiRMDX4iIiLJoxAmIiIi4gGFMPGUbtYqIiLpSiEsjnSbha7TlaUiIpJuFMISQoFCRERE2qcQJiIiIuIBhbAWop1R1JkyERERiTeFsBbUq0tERESSQSFMRERExAMKYSIiIiIeUAgTERER8YBCmIiIiIgHFMJaGNQ7t9WwQEZs1dSvOPjd0QMKOzfTAZOC/xcN7Nz34qRXXgCACYOLkz7vcQOD8yzJDyR93iIiIl4yv93lvaqqyi1btixh03/6zU1cevfy8PvrThrLJUePiPn7y9/fQWV5b7IyY8y3m96AXkOgeiMU9of80s4WOS5e+WAnYwcWkxvITOp8a+sbWfnRbqYMLUnqfEVERJLBzJY756qifZaV7MKkupaRdPygzrUOTR3WyRA1YGLw/7zenftenE32KARlZ2UogImISFrS6UgRERERDyiEiYiIiHhAIUxERETEAwphIiIiIh5QCBMRERHxgEKYiIiIiAcUwkREREQ8oBDWgs/uXSsiIiI+pRDWilKYiIiIJJ5CWAcUyURERCQRFMJEREREPJDQEGZm88zsbTNba2aLonx+tZmtMrPXzexvZjYskeURERERSRUJC2Fmlgn8HDgRGAecbWbjWoz2ClDlnJsEPADclKjyiIiIiKSSRLaETQfWOufedc7VAvcCp0aO4Jxb4pzbH3r7IlCewPKIiIiIpIxEhrDBwIcR79eHhrXls8CTCSyPiIiISMrI8roAAGZ2HlAFHNPG55cClwIMHTo0oWXRfcJEREQkGRLZErYBGBLxvjw0rBkzOx64DpjvnDsYbULOuVudc1XOuaq+ffsmpLAiIiIiyZTIlrClwEgzG04wfJ0FnBM5gplNBn4FzHPObUlgWURERHynrq6O9evXU1NT43VRpAO5ubmUl5cTCARi/k7CQphzrt7MrgCeAjKB25xzb5rZd4BlzrlHgB8AhcAfzQzgA+fc/ESVSURExE/Wr19PUVERFRUVhI6TkoKcc2zfvp3169czfPjwmL+X0D5hzrkngCdaDLs+4vXxiZx/PKiPmIiIeKWmpkYBzAfMjD59+rB169ZOfU93zBcREUlhCmD+0JX1pBAmIiIiUe3atYtf/OIXXfruSSedxK5du+Jcop5FIUxERESiai+E1dfXt/vdJ554gt69eyeiWN3inKOxsdHrYgAKYa2oC5iIiEjQokWLeOeddzj88MO55ppreO655zjqqKOYP38+48YFn0R42mmnMXXqVMaPH8+tt94a/m5FRQXbtm1j3bp1jB07lksuuYTx48fziU98ggMHDrSa16OPPsoRRxzB5MmTOf7449m8eTMAe/fu5aKLLmLixIlMmjSJBx98EIC//OUvTJkyhcrKSubMmQPA4sWLufnmm8PTnDBhAuvWrWPdunWMHj2a888/nwkTJvDhhx9y2WWXUVVVxfjx4/n2t78d/s7SpUs58sgjqaysZPr06VRXV3P00Ufz6quvhseZNWsWr732WrfrNyVu1ppK1BFfRERS0X8/+iarPtoT12mOG1TMt08Z3+bnN954IytXrgwHkOeee44VK1awcuXK8FWAt912G6WlpRw4cIBp06bxqU99ij59+jSbzpo1a7jnnnv49a9/zZlnnsmDDz7Ieeed12ycWbNm8eKLL2Jm/OY3v+Gmm27ihz/8ITfccAO9evXijTfeAGDnzp1s3bqVSy65hBdeeIHhw4ezY8eODpd1zZo13HnnncyYMQOA7373u5SWltLQ0MCcOXN4/fXXGTNmDAsXLuS+++5j2rRp7Nmzh7y8PD772c9yxx138OMf/5j//Oc/1NTUUFlZGXtFt0EhTERERGI2ffr0ZrdhuOWWW/jzn/8MwIcffsiaNWtahbDhw4dz+OGHAzB16lTWrVvXarrr169n4cKFbNy4kdra2vA8nnnmGe69997weCUlJTz66KMcffTR4XFKS0s7LPewYcPCAQzg/vvv59Zbb6W+vp6NGzeyatUqzIyBAwcybdo0AIqLiwFYsGABN9xwAz/4wQ+47bbbuPDCCzucXywUwkRERHygvRarZCooKAi/fu6553jmmWf497//TX5+PrNnz456Y9mcnJzw68zMzKinI6+88kquvvpq5s+fz3PPPcfixYs7XbasrKxm/b0iyxJZ7vfee4+bb76ZpUuXUlJSwoUXXtjuDXHz8/OZO3cuDz/8MPfffz/Lly/vdNmiUZ8wERERiaqoqIjq6uo2P9+9ezclJSXk5+fz1ltv8eKLL3Z5Xrt372bw4MEA3HnnneHhc+fO5ec//3n4/c6dO5kxYwYvvPAC7733HkD4dGRFRQUrVqwAYMWKFeHPW9qzZw8FBQX06tWLzZs38+STTwIwevRoNm7cyNKlSwGorq4OX4Dwuc99jquuuopp06ZRUlLS5eWMpBAmIiIiUfXp04eZM2cyYcIErrnmmlafz5s3j/r6esaOHcuiRYuane7rrMWLF7NgwQKmTp1KWVlZePg3v/lNdu7cyYQJE6isrGTJkiX07duXW2+9lTPOOIPKykoWLlwIwKc+9Sl27NjB+PHj+dnPfsaoUaOizquyspLJkyczZswYzjnnHGbOnAlAdnY29913H1deeSWVlZXMnTs33EI2depUiouLueiii7q8jC2Z81lP9KqqKrds2bKETf/x1zfyhT+sCL+/+7PTOWqkHhouIiLJt3r1asaOHet1MQT46KOPmD17Nm+99RYZGdHbsKKtLzNb7pyrija+WsJERERE2nHXXXdxxBFH8N3vfrfNANYV6pjfgtOdwkRERCTC+eefz/nnnx/36aolTERERMQDCmEiIiIiHlAIExEREfGAQpiIiIiIBxTCREREJG4KCwuB4C0dPv3pT0cdZ/bs2STydlN+oRAmIiIicTdo0CAeeOABr4sRVdNd8L2mECYiIiJRLVq0qNkjgxYvXszNN9/M3r17mTNnDlOmTGHixIk8/PDDrb67bt06JkyYAMCBAwc466yzGDt2LKeffnrUZ0cCfOc732HatGlMmDCBSy+9lKYbyq9du5bjjz+eyspKpkyZwjvvvAPA97//fSZOnEhlZSWLFi0Cmreybdu2jYqKCgDuuOMO5s+fz3HHHcecOXPaXYa77rqLSZMmUVlZyWc+8xmqq6sZPnw4dXV1QPCxR5Hvu0r3CWvBMK+LICIi0tqTi2DTG/Gd5oCJcOKNbX68cOFCvvSlL/GFL3wBgPvvv5+nnnqK3Nxc/vznP1NcXMy2bduYMWMG8+fPxyz6MfSXv/wl+fn5rF69mtdff50pU6ZEHe+KK67g+uuvB+Azn/kMjz32GKeccgrnnnsuixYt4vTTT6empobGxkaefPJJHn74YV566SXy8/PDz49sz4oVK3j99dcpLS2lvr4+6jKsWrWK//mf/+Ff//oXZWVl7Nixg6KiImbPns3jjz/Oaaedxr333ssZZ5xBIBDocJ7tUUtYC3PH9Wdw7zwAplWU8PERfTwukYiIiDcmT57Mli1b+Oijj3jttdcoKSlhyJAhOOe49tprmTRpEscffzwbNmxg8+bNbU7nhRde4LzzzgNg0qRJTJo0Kep4S5Ys4YgjjmDixIk8++yzvPnmm1RXV7NhwwZOP/10AHJzc8nPz+eZZ57hoosuIj8/H4DS0tIOl2fu3Lnh8dpahmeffZYFCxaEn1/ZNP7nPvc5br/9dgBuv/32uDxDUi1hLWRnZfDPRcd5XQwREZHm2mmxSqQFCxbwwAMPsGnTpvCDsn//+9+zdetWli9fTiAQoKKiIvyg666qqanh8ssvZ9myZQwZMoTFixd3aZpZWVk0NjaGpxmpoKAg/LqzyzBz5kzWrVvHc889R0NDQ/hUa3eoJUxERETatHDhQu69914eeOABFixYAMDu3bvp168fgUCAJUuW8P7777c7jaOPPpo//OEPAKxcuZLXX3+91ThNAaisrIy9e/eGO/UXFRVRXl7OQw89BMDBgwfZv38/c+fO5fbbb2f//v0A4dORFRUVLF++HKDdCwPaWobjjjuOP/7xj2zfvr3ZdCH4+KJzzjknLq1goBAmIiIi7Rg/fjzV1dUMHjyYgQMHAnDuueeybNkyJk6cyF133cWYMWPancZll13G3r17GTt2LNdffz1Tp05tNU7v3r255JJLmDBhAieccALTpk0Lf3b33Xdzyy23MGnSJI488kg2bdrEvHnzmD9/PlVVVRx++OHcfPPNAHz1q1/ll7/8JZMnT2bbtm1tlqmtZRg/fjzXXXcdxxxzDJWVlVx99dXNvrNz507OPvvs2CuwHdZ05YFfVFVVOd1bRERE0sHq1asZO3as18WQkAceeICHH36Yu+++O+rn0daXmS13zlVFG199wkREREQ6cOWVV/Lkk0/yxBNPxG2aCmEiIiIiHfjpT38a92mqT5iIiIiIBxTCREREUpjf+m6nq66sJ4UwERGRFJWbm8v27dsVxFKcc47t27eTm5vbqe+pT5iIiEiKKi8vZ/369WzdutXrokgHcnNzKS8v79R3FMJERERSVCAQYPjw4V4XQxJEpyNFREREPKAQJiIiIuIBhTARERERD/jusUVmthVo/0mh3VcGtP3AKYkn1XVyqJ6TQ/WcHKrn5FA9x8cw51zfaB/4LoQlg5kta+s5TxJfquvkUD0nh+o5OVTPyaF6TjydjhQRERHxgEKYiIiIiAcUwqK71esCpBHVdXKonpND9ZwcqufkUD0nmPqEiYiIiHhALWEiIiIiHlAIa8HM5pnZ22a21swWeV0ePzCz28xsi5mtjBhWamZ/NbM1of9LQsPNzG4J1e/rZjYl4jsXhMZfY2YXRAyfamZvhL5zi5lZcpcwNZjZEDNbYmarzOxNM/tiaLjqOo7MLNfMXjaz10L1/N+h4cPN7KVQ3dxnZtmh4Tmh92tDn1dETOsboeFvm9kJEcO1nwkxs0wze8XMHgu9Vz3HmZmtC/2uXzWzZaFh2m+kAuec/oX+AZnAO8AIIBt4DRjndblS/R9wNDAFWBkx7CZgUej1IuD7odcnAU8CBswAXgoNLwXeDf1fEnpdEvrs5dC4FvruiV4vs0f1PBCYEnpdBPwHGKe6jns9G1AYeh0AXgrVyf3AWaHh/w+4LPT6cuD/hV6fBdwXej0utA/JAYaH9i2Z2s+0qu+rgT8Aj4Xeq57jX8frgLIWw7TfSIF/aglrbjqw1jn3rnOuFrgXONXjMqU859wLwI4Wg08F7gy9vhM4LWL4XS7oRaC3mQ0ETgD+6pzb4ZzbCfwVmBf6rNg596IL/trviphWWnHObXTOrQi9rgZWA4NRXcdVqL72ht4GQv8ccBzwQGh4y3puqv8HgDmhloBTgXudcwedc+8BawnuY7SfCTGzcuBk4Deh94bqOVm030gBCmHNDQY+jHi/PjRMOq+/c25j6PUmoH/odVt13N7w9VGGp7XQqZjJBFtpVNdxFjpF9iqwheDB5h1gl3OuPjRKZN2E6zP0+W6gD52v/3T0Y+BrQGPofR9Uz4nggKfNbLmZXRoapv1GCsjyugDS8znnnJnpMtw4MbNC4EHgS865PZHdL1TX8eGcawAON7PewJ+BMR4Xqccxs08CW5xzy81sttfl6eFmOec2mFk/4K9m9lbkh9pveEctYc1tAIZEvC8PDZPO2xxqpib0/5bQ8LbquL3h5VGGpyUzCxAMYL93zv0pNFh1nSDOuV3AEuDjBE/LNP3hGlk34foMfd4L2E7n6z/dzATmm9k6gqcKjwN+guo57pxzG0L/byH4R8V0tN9ICQphzS0FRoauzskm2PnzEY/L5FePAE1Xz1wAPBwx/PzQFTgzgN2hJvGngE+YWUnoKp1PAE+FPttjZjNC/T/Oj5hWWgkt/2+B1c65H0V8pLqOIzPrG2oBw8zygLkE+98tAT4dGq1lPTfV/6eBZ0N9Yx4Bzgpd1TccGEmwA7P2M4Bz7hvOuXLnXAXBOnjWOXcunwBd2gAAAzNJREFUque4MrMCMytqek3w974S7TdSg9dXBqTaP4JXhvyHYB+Q67wujx/+AfcAG4E6gv0BPkuwr8bfgDXAM0BpaFwDfh6q3zeAqojpXEywU+1a4KKI4VUEdxrvAD8jdJPhdPsHzCLYt+N14NXQv5NU13Gv50nAK6F6XglcHxo+guDBfS3wRyAnNDw39H5t6PMREdO6LlSXbxNxxZj2M63qfDaHro5UPce3bkcQvDL0NeDNpnrQfiM1/umO+SIiIiIe0OlIEREREQ8ohImIiIh4QCFMRERExAMKYSIiIiIeUAgTERER8YBCmIikLTNbZ2ZlHYxzbbLKIyLpRSFMRKR9CmEikhAKYSLiW2ZWYWYrI95/1cwWm9lzZvYTM3vVzFaa2fTQ533M7Gkze9PMfkPwxpRN330o9IDjN5secmxmNwJ5oen8PjTsPDN7OTTsV6GHfWea2R2heb1hZl9Obk2IiB8phIlIT5XvnDscuBy4LTTs28A/nHPjCT5Db2jE+Bc756YSvPv3VWbWxzm3CDjgnDvcOXeumY0FFgIzQ9NuAM4FDgcGO+cmOOcmArcnZQlFxNeyOh5FRMSX7gFwzr1gZsWh50EeDZwRGv64me2MGP8qMzs99HoIwWcQbm8xzTnAVGBp8DF55BF88PGjwAgz+ynwOPB0YhZJRHoShTAR8bN6mrfo50a8bvlMtjaf0WZms4HjgY875/ab2XMtphUeFbjTOfeNKNOoBE4APg+cSfA5eyIibdLpSBHxs81Av1BfrxzgkxGfLQQws1nAbufcbuAF4JzQ8BOBktC4vYCdoQA2BpgRMZ06MwuEXv8N+LSZ9QtNo9TMhoWusMxwzj0IfBOYkoiFFZGeRS1hIuJbzrk6M/sO8DKwAXgr4uMaM3sFCHCoVeq/gXvM7E3gX8AHoeF/AT5vZquBt4EXI6ZzK/C6ma0I9Qv7JvC0mWUAdcAXgAPA7aFhAK1aykREWjLn2myhFxHxpdDpxK8655Z5XRYRkbbodKSIiIiIB9QSJiIiIuIBtYSJiIiIeEAhTERERMQDCmEiIiIiHlAIExEREfGAQpiIiIiIBxTCRERERDzw/wEn8bfXlQt5ZwAAAABJRU5ErkJggg==",
            "text/plain": [
              "<Figure size 720x432 with 1 Axes>"
            ]
          },
          "metadata": {
            "needs_background": "light"
          },
          "output_type": "display_data"
        }
      ],
      "source": [
        "fig = plt.figure(figsize=(10,6))\n",
        "ax = fig.add_subplot(1,1,1)\n",
        "ax.plot(train_accs, label='train accuracy')\n",
        "ax.plot(valid_accs, label='valid accuracy')\n",
        "plt.legend()\n",
        "ax.set_xlabel('updates')\n",
        "ax.set_ylabel('accuracy');"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "# Predictions"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "## Load the best model"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 31,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ZZtri4nlt1Qv",
        "outputId": "eee6a114-55f3-4455-b3e8-2c1d321cd331"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "<All keys matched successfully>"
            ]
          },
          "execution_count": 31,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "model.load_state_dict(torch.load('transformer.pt'))"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "## Prediction function"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 32,
      "metadata": {
        "id": "-cFpdpRlt1Qv"
      },
      "outputs": [],
      "source": [
        "def predict_sentiment(text, model, tokenizer, device):\n",
        "    ids = tokenizer(text)['input_ids']\n",
        "    tensor = torch.LongTensor(ids).unsqueeze(dim=0).to(device)\n",
        "    prediction = model(tensor).squeeze(dim=0)\n",
        "    probability = torch.softmax(prediction, dim=-1)\n",
        "    predicted_class = prediction.argmax(dim=-1).item()\n",
        "    predicted_probability = probability[predicted_class].item()\n",
        "    return predicted_class, predicted_probability"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 33,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "oMa99Habt1Qv",
        "outputId": "eed01cce-72bc-4e79-9b45-389c76769d11"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "(0, 0.991040050983429)"
            ]
          },
          "execution_count": 33,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "text = \"This film is terrible!\"\n",
        "\n",
        "predict_sentiment(text, model, tokenizer, device)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "## Submission generation"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 34,
      "metadata": {
        "id": "ussQlCWqt1Qv"
      },
      "outputs": [],
      "source": [
        "with open(FILE_TEST, \"r\", encoding=\"utf-8\") as f:\n",
        "    content = f.read()\n",
        "content = content.split(\"\\n\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 35,
      "metadata": {
        "id": "TTZSfMcht1Qx"
      },
      "outputs": [],
      "source": [
        "preds = {\"ids\":[], \"text\":[], \"label\":[]}\n",
        "for entry in content:\n",
        "    if entry == \"\" or entry == \" \":\n",
        "        continue\n",
        "    entry_split = re.split(\",\", entry, 1)\n",
        "    preds[\"ids\"].append(entry_split[0])\n",
        "    preds[\"text\"].append(entry_split[1])\n",
        "    preds[\"label\"].append(predict_sentiment(\n",
        "        entry_split[1],\n",
        "        model,\n",
        "        tokenizer,\n",
        "        device\n",
        "    )[0])"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 36,
      "metadata": {
        "id": "DDiJ_9Qut1Qx"
      },
      "outputs": [],
      "source": [
        "with open(\"submission.csv\", \"w\", encoding=\"utf-8\") as f:\n",
        "    f.write(\"Id,Prediction\\n\")\n",
        "    for i in range(len(preds[\"ids\"])):\n",
        "        f.write(\"{},{}\\n\".format(preds[\"ids\"][i], -1 if preds[\"label\"][i]==0 else 1))"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "## Submission downloading"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "We also save the model for eventual posterior re-use"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 39,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 17
        },
        "id": "CiPZnraDujG1",
        "outputId": "db826f61-b85b-4d21-92a7-1f08c053ae14"
      },
      "outputs": [
        {
          "data": {
            "application/javascript": "\n    async function download(id, filename, size) {\n      if (!google.colab.kernel.accessAllowed) {\n        return;\n      }\n      const div = document.createElement('div');\n      const label = document.createElement('label');\n      label.textContent = `Downloading \"${filename}\": `;\n      div.appendChild(label);\n      const progress = document.createElement('progress');\n      progress.max = size;\n      div.appendChild(progress);\n      document.body.appendChild(div);\n\n      const buffers = [];\n      let downloaded = 0;\n\n      const channel = await google.colab.kernel.comms.open(id);\n      // Send a message to notify the kernel that we're ready.\n      channel.send({})\n\n      for await (const message of channel.messages) {\n        // Send a message to notify the kernel that we're ready.\n        channel.send({})\n        if (message.buffers) {\n          for (const buffer of message.buffers) {\n            buffers.push(buffer);\n            downloaded += buffer.byteLength;\n            progress.value = downloaded;\n          }\n        }\n      }\n      const blob = new Blob(buffers, {type: 'application/binary'});\n      const a = document.createElement('a');\n      a.href = window.URL.createObjectURL(blob);\n      a.download = filename;\n      div.appendChild(a);\n      a.click();\n      div.remove();\n    }\n  ",
            "text/plain": [
              "<IPython.core.display.Javascript object>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/javascript": "download(\"download_7d584216-46f3-46f4-bb84-0a4a2d176121\", \"transformer.pt\", 438022061)",
            "text/plain": [
              "<IPython.core.display.Javascript object>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        }
      ],
      "source": [
        "# Specific to Colab\n",
        "from google.colab import files\n",
        "files.download('transformer.pt')\n",
        "files.download('submission.csv')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "UxveKpYS8VN1"
      },
      "outputs": [],
      "source": []
    }
  ],
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "include_colab_link": true,
      "name": "4_transformer.ipynb",
      "provenance": []
    },
    "interpreter": {
      "hash": "1c19f1ca3afd4998816691da9a36584d647e992dcfc819c425cd38dca0566a7c"
    },
    "kernelspec": {
      "display_name": "Python 3.9.6 64-bit",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.9.6"
    },
    "orig_nbformat": 4,
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "0082375dcc8a446b80e5bd172a9e649d": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "041969198b05460da479d33496de3ea3": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "060a09256c5c4aca87f3ee2eaa097e1f": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_fe3eba48e9de4195b933f377dcc64872",
            "placeholder": "​",
            "style": "IPY_MODEL_1180e4a7dbbb4200b88e1dac732faaa7",
            "value": " 150000/150000 [00:56&lt;00:00, 2580.57ex/s]"
          }
        },
        "07dde21456444417a82bad4df05736f2": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "0d8ab1ca91374a698e3a03547b8da661": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_6a98ec97cf934d56828db462f1237f36",
            "placeholder": "​",
            "style": "IPY_MODEL_a7dbafc0087748179dc6366dd3db8699",
            "value": "Downloading: 100%"
          }
        },
        "104c7170bf1b4ecdbf7927e4dc74aeff": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_8f9b723c723b4ce0a59f8d4514acd305",
            "placeholder": "​",
            "style": "IPY_MODEL_ec712bff1ad549e49411961602aae93f",
            "value": "100%"
          }
        },
        "104e485bc4ba4793accff6b757d8487b": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "10559067a3d04ade93e711d4114e10c3": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HBoxModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_60457332cc084713b99cfd738569afde",
              "IPY_MODEL_380a869110f445a3b747378bf45e606d",
              "IPY_MODEL_9bb6f10207534f0f8dc8dec1300b3e00"
            ],
            "layout": "IPY_MODEL_66d7bd3924864c528abfd1db275aff37"
          }
        },
        "1180e4a7dbbb4200b88e1dac732faaa7": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "12d400a620844b47b218a09780a87d84": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "132a9aafe9d247038231fe510e7e7283": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_736e5a4017314707a723646f8f492258",
            "placeholder": "​",
            "style": "IPY_MODEL_b2dffd1b7b984770a65817cc95c805d6",
            "value": " 420M/420M [00:14&lt;00:00, 32.5MB/s]"
          }
        },
        "1ae6dd8a46124404961f9c3439faf584": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "1d450c26c0ba4111bc7c60376c7cee50": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "1f9ea842afda46deb43a4cdc9df3e5d0": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_af0e47a41c404036a01baae5b7dc473d",
            "placeholder": "​",
            "style": "IPY_MODEL_e0632f7b600d48a19b0bca94d10b91fe",
            "value": " 570/570 [00:00&lt;00:00, 13.4kB/s]"
          }
        },
        "267ada4ebb904acd83c8591a61e63221": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "31d606615b7e4ee28c3fb10b5421ac89": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_ece16fbcf31f4ed9afbf824aff0115dd",
            "placeholder": "​",
            "style": "IPY_MODEL_85bc4b6579524db2bd47bbfc3c3ffe2f",
            "value": "Downloading: 100%"
          }
        },
        "3470b025ac094eae8f3623897a51a713": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "380a869110f445a3b747378bf45e606d": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "FloatProgressModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_69f4be1ffad34246a535b7e484baec57",
            "max": 50000,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_dc67f5506eb1440f9dcc7934d46ed021",
            "value": 50000
          }
        },
        "41f8784804d34390b6c8220521fd8996": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HBoxModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_996c54b6fc0f421fa1d1cc31e27134e2",
              "IPY_MODEL_c12c22bcfebf4362a6963ced56d11524",
              "IPY_MODEL_552853a16c0e45a88185aee8302e96c9"
            ],
            "layout": "IPY_MODEL_267ada4ebb904acd83c8591a61e63221"
          }
        },
        "463bbaa6a226454781e6a9c26d39e39c": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_b179ad0042d1435789b4869d2d0b5d2a",
            "placeholder": "​",
            "style": "IPY_MODEL_e1f34a29979d495188a6978a0bedcc93",
            "value": "Downloading: 100%"
          }
        },
        "4dfebf84c7e342cfa0af9c73476b5676": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "50c39ab0aa964670b19fc891d81f216f": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HBoxModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_463bbaa6a226454781e6a9c26d39e39c",
              "IPY_MODEL_cd1b47860cf94aca922f2362dc80aa32",
              "IPY_MODEL_f7f1b5a617e547fc926e79a1b4832cee"
            ],
            "layout": "IPY_MODEL_7a6e6022927c4bcc8f416ffb3cabfe38"
          }
        },
        "51c21294fb964c69ab6baaf1c5068a37": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "52b9d6a47ab140eba79fde8d0f214295": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "53808da8556d4f3da2cf7430fc950810": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "ProgressStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "552853a16c0e45a88185aee8302e96c9": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_ee5e1617624a4345bee82d6619bf3cf3",
            "placeholder": "​",
            "style": "IPY_MODEL_51c21294fb964c69ab6baaf1c5068a37",
            "value": " 226k/226k [00:00&lt;00:00, 1.01MB/s]"
          }
        },
        "5857deca493c4a289009903b8cca7eaa": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "FloatProgressModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_7f8e663a518849f19f530e9a7e04f84e",
            "max": 440473133,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_7953f605bed04f27b1c1ec4767d49614",
            "value": 440473133
          }
        },
        "5d30bb5238244bc79fd823c44eac3679": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "5fb0bf3b44e7447d8df112f87d97c7c1": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HBoxModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_104c7170bf1b4ecdbf7927e4dc74aeff",
              "IPY_MODEL_7f34e6470b374c258cc899d3d03666e3",
              "IPY_MODEL_060a09256c5c4aca87f3ee2eaa097e1f"
            ],
            "layout": "IPY_MODEL_b115396ac40f4579a456902ea0c2c27c"
          }
        },
        "60457332cc084713b99cfd738569afde": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_52b9d6a47ab140eba79fde8d0f214295",
            "placeholder": "​",
            "style": "IPY_MODEL_0082375dcc8a446b80e5bd172a9e649d",
            "value": "100%"
          }
        },
        "66d7bd3924864c528abfd1db275aff37": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "68440f8633ca458cb1a07896ae6eb54a": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_1ae6dd8a46124404961f9c3439faf584",
            "placeholder": "​",
            "style": "IPY_MODEL_6945c0767f1b44bcbb002e6a73427ad9",
            "value": "Downloading: 100%"
          }
        },
        "6945c0767f1b44bcbb002e6a73427ad9": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "69f4be1ffad34246a535b7e484baec57": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "6a98ec97cf934d56828db462f1237f36": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "6affffe403fc4fad8fa60276030a10f4": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HBoxModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_68440f8633ca458cb1a07896ae6eb54a",
              "IPY_MODEL_94a205668ba74caeb8942547a2c8ac25",
              "IPY_MODEL_b73121d550ad4697b8d89ad80139c7a4"
            ],
            "layout": "IPY_MODEL_b2ee84b674d746aaa1d2b66538176125"
          }
        },
        "736e5a4017314707a723646f8f492258": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "74ba5812390a4f4d8fbaf462e80c4bac": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "78f892242f924ece821ec4fd051fdafe": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HBoxModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_0d8ab1ca91374a698e3a03547b8da661",
              "IPY_MODEL_c5d1a0bb1ba34862957403f49dd0188b",
              "IPY_MODEL_1f9ea842afda46deb43a4cdc9df3e5d0"
            ],
            "layout": "IPY_MODEL_98ff1d8cf2cc4e96bd3bf894b82640c1"
          }
        },
        "7953f605bed04f27b1c1ec4767d49614": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "ProgressStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "79b8ebb4962d4f498c966f382ee4f667": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "7a6e6022927c4bcc8f416ffb3cabfe38": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "7f34e6470b374c258cc899d3d03666e3": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "FloatProgressModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_104e485bc4ba4793accff6b757d8487b",
            "max": 150000,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_c6b3d5719f7344a08e123505e2e65c1a",
            "value": 150000
          }
        },
        "7f8e663a518849f19f530e9a7e04f84e": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "85bc4b6579524db2bd47bbfc3c3ffe2f": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "8f9b723c723b4ce0a59f8d4514acd305": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "910f8c4e6ef041f4b23b2ef4be17590a": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HBoxModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_31d606615b7e4ee28c3fb10b5421ac89",
              "IPY_MODEL_5857deca493c4a289009903b8cca7eaa",
              "IPY_MODEL_132a9aafe9d247038231fe510e7e7283"
            ],
            "layout": "IPY_MODEL_ed956969e15c45a3818b4f175473a962"
          }
        },
        "94a205668ba74caeb8942547a2c8ac25": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "FloatProgressModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_74ba5812390a4f4d8fbaf462e80c4bac",
            "max": 466062,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_53808da8556d4f3da2cf7430fc950810",
            "value": 466062
          }
        },
        "986b60c69548453396e76b4af9aef8d8": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "ProgressStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "98ff1d8cf2cc4e96bd3bf894b82640c1": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "996c54b6fc0f421fa1d1cc31e27134e2": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_3470b025ac094eae8f3623897a51a713",
            "placeholder": "​",
            "style": "IPY_MODEL_4dfebf84c7e342cfa0af9c73476b5676",
            "value": "Downloading: 100%"
          }
        },
        "9bb6f10207534f0f8dc8dec1300b3e00": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_d42e08263a774255a7dcff046b8592ab",
            "placeholder": "​",
            "style": "IPY_MODEL_e6a0ff19e74d4ff89328b0d9968b83b4",
            "value": " 50000/50000 [00:18&lt;00:00, 2621.51ex/s]"
          }
        },
        "a7dbafc0087748179dc6366dd3db8699": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "af0e47a41c404036a01baae5b7dc473d": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "b115396ac40f4579a456902ea0c2c27c": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "b179ad0042d1435789b4869d2d0b5d2a": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "b2dffd1b7b984770a65817cc95c805d6": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "b2ee84b674d746aaa1d2b66538176125": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "b73121d550ad4697b8d89ad80139c7a4": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_12d400a620844b47b218a09780a87d84",
            "placeholder": "​",
            "style": "IPY_MODEL_07dde21456444417a82bad4df05736f2",
            "value": " 455k/455k [00:00&lt;00:00, 959kB/s]"
          }
        },
        "c12c22bcfebf4362a6963ced56d11524": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "FloatProgressModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_041969198b05460da479d33496de3ea3",
            "max": 231508,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_deef4bd3e8344c05ab6dc1f3a5bb4b10",
            "value": 231508
          }
        },
        "c5d1a0bb1ba34862957403f49dd0188b": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "FloatProgressModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_79b8ebb4962d4f498c966f382ee4f667",
            "max": 570,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_ee35e0788cb041fa8f56162f973335f2",
            "value": 570
          }
        },
        "c6b3d5719f7344a08e123505e2e65c1a": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "ProgressStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "cd1b47860cf94aca922f2362dc80aa32": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "FloatProgressModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_1d450c26c0ba4111bc7c60376c7cee50",
            "max": 28,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_986b60c69548453396e76b4af9aef8d8",
            "value": 28
          }
        },
        "d42e08263a774255a7dcff046b8592ab": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "dc67f5506eb1440f9dcc7934d46ed021": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "ProgressStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "deef4bd3e8344c05ab6dc1f3a5bb4b10": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "ProgressStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "df3678265f154c90bbaa0e93eaa0ce0b": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "e0632f7b600d48a19b0bca94d10b91fe": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "e1f34a29979d495188a6978a0bedcc93": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "e6a0ff19e74d4ff89328b0d9968b83b4": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "ec712bff1ad549e49411961602aae93f": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "ece16fbcf31f4ed9afbf824aff0115dd": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "ed956969e15c45a3818b4f175473a962": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "ee35e0788cb041fa8f56162f973335f2": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "ProgressStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "ee5e1617624a4345bee82d6619bf3cf3": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "f7f1b5a617e547fc926e79a1b4832cee": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_5d30bb5238244bc79fd823c44eac3679",
            "placeholder": "​",
            "style": "IPY_MODEL_df3678265f154c90bbaa0e93eaa0ce0b",
            "value": " 28.0/28.0 [00:00&lt;00:00, 623B/s]"
          }
        },
        "fe3eba48e9de4195b933f377dcc64872": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        }
      }
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}