sft/o1-reasoning-sft.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "2yievghzP1mE"
   },
   "source": [
    "# 安装必要的环境"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "cfQVpi4UNTJY",
    "outputId": "55a63297-b10e-44dd-ea96-511506cc4dea"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n",
      "\u001b[0m"
     ]
    }
   ],
   "source": [
    "!pip install accelerate datasets -q"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "Vz-ht4y_QBq7"
   },
   "source": [
    "# 0、加载训练好的基座模型（qwen2.5-1.5B）"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 241,
     "referenced_widgets": [
      "5f701e7d23f346859b2fd0f885f2c58f",
      "e30e11637be7473b873835e115517853",
      "0a575b3a5a5140b680ff8e20676f3dce",
      "f4c353c0fe944932bcf7879c7cd3dd2a",
      "7831b1d7c9054f3faa8f7e280f3676b9",
      "a4435caad91a4ab6ac25ebb860a1108e",
      "69b75a00c15f46568b6caf44e6b72384",
      "611af89de8ee4108adb8fccb5bb9cf32",
      "438b937befce4e2d9d49edca9d77d7d2",
      "39ae78eff3714fbfa1ef5ef006e45b5a",
      "d473f10ea00548428718b827ba7217db",
      "ffeb8b36b8dc4c0686940a4293ff0d2b",
      "6c57207ee15e46488f3b7d8a206e6fe6",
      "c453996f237f43f38d02ed0c3a808bc9",
      "5ce999cad3d84a00af2a479a86914bb1",
      "62d159e5720f496c82312d9efd4bfe79",
      "8142a04b596d484f87c2620cb1afab2f",
      "b6ee43871869434d9cc65504cba5cac7",
      "1c0ebb52daab43a7b316a8460535e897",
      "f8796e310a9548528122a3b20aab3582",
      "5e80ecdc0e75417285f8c45785a3d9ed",
      "7563d173e09d4c6eac618aaa1269a8fd",
      "a2b2494c19764c22908fa7247a1fc02c",
      "deb2379001af474cabbeca1df3c7dc95",
      "8ca7b55e3b8f40668eb4abd2892971df",
      "7dfab99d13504e76a3fbed258f1933b4",
      "2b05d0a1e27f4c4094ffb7801ad11076",
      "f7ea2b985bd44ab089b515a002cdcc44",
      "2c9c1b6e18fa4cd9979d5f13f41adf48",
      "a198338d2c6945699da654673bf9b745",
      "8164fbc2518641558417b26eaa2fb3be",
      "256ace0d05ae489fad26cdbac9fd3502",
      "1571ba54beca47a69ef597cb15df214b",
      "14637cdcc5d642b0989bf887100601a2",
      "cf98d52df6b9432a9877c8b5355c82e6",
      "e9ae9ba627aa4bf8a4eb64d68809f566",
      "d5ec0c9ab8d6453fb32e867102ee0eaf",
      "2434babf7137418b84dcf59553f33a3a",
      "197f61d70ab541aa9a1b1c1298435b2b",
      "e4c2626eabb84dc78273acfffcef82bd",
      "138e15d45660471baff2f09c93efe771",
      "e7aa821a05734fa8b2c81dff7f38c8af",
      "756f373c43b64c1ba4f9668d91d059ec",
      "76d6fcf7d45649a3947ef199beccac83",
      "28784f7543294c49b54a09c908b56c9e",
      "0c6887b9851644a684659752c7375885",
      "68d87d30b17c4970bcf48873fb6c9247",
      "d7e56c2c917648c1b8443247f8b49be3",
      "18d67eed7b8f407da269ec244cd04abb",
      "45f1f88bf6424096a522d364a1aa458b",
      "ddbc147f7ec34bbc9a1b0f0721f6451b",
      "9b977ee086ed49d38325d4fbb005d664",
      "09753c16b5c949d987236ee940ec5fa3",
      "bcbd9e82d1ed4ad6b51e8699a6087041",
      "5ebd1a93c8f744629cb6ba95289cedd8",
      "1116b93b89eb420490a12deacfb06c21",
      "2f401b1fcb8e4c969dac72bf215f6561",
      "c79df3e8fa824c8687d2bfaa57e7fe01",
      "415e8e498c2a4afc8e85d15a72f2fd73",
      "9bfa854836e249d79617969ac207754b",
      "6d3fa20d73e2496d81cf7fa17b8efc6c",
      "afdcb773c48f419a92b7231f3a07255f",
      "c4e994712aec44faa6204cdeb803c131",
      "58127f0348e94a9582de29fb9bcc7f92",
      "0f34d6fa253444e1a73c240cc5bdb62e",
      "01143789eb8f46379f9ce7b3e51ba846",
      "cdeaa1d0b6f54adcab53755eaeed3c71",
      "1f2f325f8f66426795de63fbed7b606b",
      "a820971e767f4c49a157e5ff2db82c6f",
      "24028c99b8a04fab8c122ab54412ea54",
      "938d14371d034e859cba003bab3c0cc5",
      "930dd8bebebc475795350db5405a591e",
      "cb748c092e4e4917ac1dbb64dbf2e14d",
      "6365c0f5fc35484dba06f88fbc8f0656",
      "aba50b35a10e4f6892742eed763a8890",
      "ef94fa10df1049eaa598e77851c05411",
      "b7f1116dd979421f986a3289bf98ad19"
     ]
    },
    "id": "Bvs0B3GEQNML",
    "outputId": "8df777dd-f137-4fd0-ff91-3463fd47efb2"
   },
   "outputs": [],
   "source": [
    "from transformers import AutoModelForCausalLM, AutoTokenizer\n",
    "import torch\n",
    "import os\n",
    "import numpy as np\n",
    "device = \"cuda\" # the device to load the model onto\n",
    "model_name = \"Qwen2___5-1___5B\"\n",
    "model_path = f'Qwen/{model_name}'\n",
    "\n",
    "# 获取可用GPU设备的数量和内存信息\n",
    "gpu_memory = {i: torch.cuda.get_device_properties(i).total_memory for i in range(torch.cuda.device_count())}\n",
    "# 加载模型\n",
    "try:\n",
    "    model = AutoModelForCausalLM.from_pretrained(\n",
    "        model_path,\n",
    "        torch_dtype=\"auto\",\n",
    "        device_map=\"balanced\",\n",
    "        max_memory={i: f\"{gpu_memory[i] // (1024 ** 3)}GB\" for i in range(len(gpu_memory))}\n",
    "    )\n",
    "except Exception as e:\n",
    "    print(f\"Failed to load model from {model_path}: {e}\")\n",
    "    raise\n",
    "# 加载分词器\n",
    "tokenizer = AutoTokenizer.from_pretrained(model_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{151643: AddedToken(\"<|endoftext|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
       " 151644: AddedToken(\"<|im_start|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
       " 151645: AddedToken(\"<|im_end|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
       " 151646: AddedToken(\"<|object_ref_start|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
       " 151647: AddedToken(\"<|object_ref_end|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
       " 151648: AddedToken(\"<|box_start|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
       " 151649: AddedToken(\"<|box_end|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
       " 151650: AddedToken(\"<|quad_start|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
       " 151651: AddedToken(\"<|quad_end|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
       " 151652: AddedToken(\"<|vision_start|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
       " 151653: AddedToken(\"<|vision_end|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
       " 151654: AddedToken(\"<|vision_pad|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
       " 151655: AddedToken(\"<|image_pad|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
       " 151656: AddedToken(\"<|video_pad|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
       " 151657: AddedToken(\"<tool_call>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),\n",
       " 151658: AddedToken(\"</tool_call>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),\n",
       " 151659: AddedToken(\"<|fim_prefix|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),\n",
       " 151660: AddedToken(\"<|fim_middle|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),\n",
       " 151661: AddedToken(\"<|fim_suffix|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),\n",
       " 151662: AddedToken(\"<|fim_pad|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),\n",
       " 151663: AddedToken(\"<|repo_name|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False),\n",
       " 151664: AddedToken(\"<|file_sep|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False)}"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tokenizer.added_tokens_decoder"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "bQJdfCrjkxLp",
    "outputId": "fdfde7b0-688b-44d7-ae7f-5a8866cee357"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Embedding(151936, 1536)"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.model.embed_tokens"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "Hugel2DfRE-D",
    "outputId": "7f199f47-003f-485c-afca-4526076be1a4"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'eos_token': '<|endoftext|>',\n",
       " 'pad_token': '<|endoftext|>',\n",
       " 'additional_special_tokens': ['<|im_start|>',\n",
       "  '<|im_end|>',\n",
       "  '<|object_ref_start|>',\n",
       "  '<|object_ref_end|>',\n",
       "  '<|box_start|>',\n",
       "  '<|box_end|>',\n",
       "  '<|quad_start|>',\n",
       "  '<|quad_end|>',\n",
       "  '<|vision_start|>',\n",
       "  '<|vision_end|>',\n",
       "  '<|vision_pad|>',\n",
       "  '<|image_pad|>',\n",
       "  '<|video_pad|>']}"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 查看分词器中的special token\n",
    "tokenizer.special_tokens_map"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "r7Pqr6bERhPy",
    "outputId": "9da6e7fe-efd7-4ce4-d518-06f475ac6678"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['<|endoftext|>',\n",
       " '<|im_start|>',\n",
       " '<|im_end|>',\n",
       " '<|object_ref_start|>',\n",
       " '<|object_ref_end|>',\n",
       " '<|box_start|>',\n",
       " '<|box_end|>',\n",
       " '<|quad_start|>',\n",
       " '<|quad_end|>',\n",
       " '<|vision_start|>',\n",
       " '<|vision_end|>',\n",
       " '<|vision_pad|>',\n",
       " '<|image_pad|>',\n",
       " '<|video_pad|>']"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tokenizer.all_special_tokens"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "fBdLWgUYRp4N",
    "outputId": "6e196bc4-0d18-49f5-99a8-64c2d8381c7b"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[151643,\n",
       " 151644,\n",
       " 151645,\n",
       " 151646,\n",
       " 151647,\n",
       " 151648,\n",
       " 151649,\n",
       " 151650,\n",
       " 151651,\n",
       " 151652,\n",
       " 151653,\n",
       " 151654,\n",
       " 151655,\n",
       " 151656]"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tokenizer.all_special_ids"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "0gywWK6iSBzP",
    "outputId": "d8769884-56e5-475f-d52f-e0e48161a6f0"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "GenerationConfig {\n",
       "  \"bos_token_id\": 151643,\n",
       "  \"eos_token_id\": 151643,\n",
       "  \"max_new_tokens\": 2048\n",
       "}"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.generation_config"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "5FbPvsSPnYbB"
   },
   "source": [
    "## 0.2 重设模型的generation_config文件"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "kcrQ2QFVnXuS",
    "outputId": "2de88642-d5fd-48f7-af3f-e4763cd946de"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "GenerationConfig {\n",
       "  \"bos_token_id\": 151643,\n",
       "  \"do_sample\": true,\n",
       "  \"eos_token_id\": [\n",
       "    151645,\n",
       "    151643\n",
       "  ],\n",
       "  \"max_new_tokens\": 2048,\n",
       "  \"pad_token_id\": 151643,\n",
       "  \"repetition_penalty\": 1.05,\n",
       "  \"temperature\": 0.7,\n",
       "  \"top_k\": 20,\n",
       "  \"top_p\": 0.8\n",
       "}"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.generation_config.do_sample = True\n",
    "model.generation_config.eos_token_id = [151645, 151643]\n",
    "model.generation_config.pad_token_id = 151643\n",
    "model.generation_config.temperature = 0.7\n",
    "model.generation_config.top_p = 0.8\n",
    "model.generation_config.top_k = 20\n",
    "model.generation_config.transformers_version = \"4.37.0\"\n",
    "model.generation_config.repetition_penalty = 1.05\n",
    "\n",
    "model.generation_config"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "v1VC87iYW6pW"
   },
   "source": [
    "# 1、 新增`reasoning`special token + 同步调整模型的`embedding`层\n",
    "- reasoning特殊标签格式：  **<|reasoning|>**\n",
    "- 模型的分词器、嵌入层和lm_head均适配新增一个特殊token：<|reasoning|>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "rMMLykkr_TE6",
    "outputId": "50c71858-84e8-4dd8-8077-425e3ac7d240"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Embedding(151936, 1536)"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 查看模型嵌入层\n",
    "model.model.embed_tokens"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "46Gt8uPuEiyn",
    "outputId": "5c786f37-2c53-444f-f63d-7d3a43588ab5"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "151643"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 查看模型的vocab_size\n",
    "tokenizer.vocab_size"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "T6-jG7geEpsX",
    "outputId": "abae71d6-6051-41e5-d3d2-1ad87c966f3d"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "151665"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(tokenizer)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "j_eLKGwLj56k",
    "outputId": "1a977d05-51a0-483c-9bb8-c0440a55b834"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "22"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(tokenizer) - tokenizer.vocab_size"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "csIQxiLoEvjP",
    "outputId": "2ae71970-3fc7-4ada-ebc9-5b5c85238ab6"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{151643: '<|endoftext|>',\n",
       " 151644: '<|im_start|>',\n",
       " 151645: '<|im_end|>',\n",
       " 151646: '<|object_ref_start|>',\n",
       " 151647: '<|object_ref_end|>',\n",
       " 151648: '<|box_start|>',\n",
       " 151649: '<|box_end|>',\n",
       " 151650: '<|quad_start|>',\n",
       " 151651: '<|quad_end|>',\n",
       " 151652: '<|vision_start|>',\n",
       " 151653: '<|vision_end|>',\n",
       " 151654: '<|vision_pad|>',\n",
       " 151655: '<|image_pad|>',\n",
       " 151656: '<|video_pad|>',\n",
       " 151657: '<tool_call>',\n",
       " 151658: '</tool_call>',\n",
       " 151659: '<|fim_prefix|>',\n",
       " 151660: '<|fim_middle|>',\n",
       " 151661: '<|fim_suffix|>',\n",
       " 151662: '<|fim_pad|>',\n",
       " 151663: '<|repo_name|>',\n",
       " 151664: '<|file_sep|>'}"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import numpy as np\n",
    "{k:v for k,v in zip([i for i in np.arange(tokenizer.vocab_size, len(tokenizer))], [tokenizer.decode(i) for i in np.arange(tokenizer.vocab_size, len(tokenizer))])}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "CwwTNQWUkFX5"
   },
   "source": [
    "可见，151643 - 151664 ，这22个元素是特殊token (有些甚至还是预留的适配多模态以及工具利用的特殊token标记)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "a1OzO3sgIBKV",
    "outputId": "828b4145-819d-4457-a738-b8d4544cb8b0"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([[ 0.0077,  0.0110, -0.0062,  ...,  0.0048,  0.0024,  0.0010],\n",
      "        [ 0.0077,  0.0110, -0.0062,  ...,  0.0048,  0.0024,  0.0010],\n",
      "        [ 0.0077,  0.0110, -0.0062,  ...,  0.0048,  0.0024,  0.0010],\n",
      "        ...,\n",
      "        [ 0.0077,  0.0110, -0.0062,  ...,  0.0048,  0.0024,  0.0010],\n",
      "        [ 0.0077,  0.0110, -0.0062,  ...,  0.0048,  0.0024,  0.0010],\n",
      "        [ 0.0077,  0.0110, -0.0062,  ...,  0.0048,  0.0024,  0.0010]],\n",
      "       device='cuda:0', dtype=torch.bfloat16)\n",
      "tensor size: torch.Size([21, 1536])\n",
      "所有张量都相同（在容差范围内）\n"
     ]
    }
   ],
   "source": [
    "# 因为我们导入的是基座模型（预训练模型），除了<|endoftext|> (<|endoftext|>嵌入在预训练过程中也会被训练) ，其他的特殊token应该都是随机初始化的，我们查看从第二行元素开始，嵌入是否都相同？\n",
    "special_tensorweight = model.model.embed_tokens.weight[tokenizer.vocab_size+1: len(tokenizer)].detach()\n",
    "print(special_tensorweight)\n",
    "print(f'tensor size: {special_tensorweight.shape}')\n",
    "# 选择第1个张量作为基准\n",
    "reference_tensor = special_tensorweight[0]\n",
    "\n",
    "# 定义一个容差值，用于比较浮点数是否相等\n",
    "tolerance = 1e-4\n",
    "\n",
    "all_same = all(torch.allclose(tensor, reference_tensor, atol=tolerance) for tensor in special_tensorweight)\n",
    "\n",
    "if all_same:\n",
    "  print(\"所有张量都相同（在容差范围内）\")\n",
    "else:\n",
    "  print(\"张量不都相同\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 35
    },
    "id": "9YnpO4n5gSFG",
    "outputId": "f396d421-0fb1-47e7-b0b6-edc4806449d5"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "''"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 那么 151665 - 151935 是什么token呢， 我们解码一下\n",
    "tokenizer.decode(torch.tensor(np.arange(len(tokenizer), model.model.embed_tokens.weight.detach().shape[0]).tolist()))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "Oe2AeO_ZkTkb"
   },
   "source": [
    "可以看到这些都是无用的token（空字符串），至于为什么需要pad这些token，找到的一篇答复：https://huggingface.co/Qwen/Qwen1.5-72B-Chat/discussions/1#662883f568adf59b07b176d2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "Grh7T_faKuOX",
    "outputId": "4eb10dad-96fb-4a05-e27e-faaa1eb8c3cd"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[ 0.0077,  0.0110, -0.0062,  ...,  0.0048,  0.0024,  0.0010],\n",
       "        [ 0.0077,  0.0110, -0.0062,  ...,  0.0048,  0.0024,  0.0010],\n",
       "        [ 0.0077,  0.0110, -0.0062,  ...,  0.0048,  0.0024,  0.0010],\n",
       "        ...,\n",
       "        [ 0.0077,  0.0110, -0.0062,  ...,  0.0048,  0.0024,  0.0010],\n",
       "        [ 0.0077,  0.0110, -0.0062,  ...,  0.0048,  0.0024,  0.0010],\n",
       "        [ 0.0077,  0.0110, -0.0062,  ...,  0.0048,  0.0024,  0.0010]],\n",
       "       device='cuda:0', dtype=torch.bfloat16)"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.model.embed_tokens.weight.detach()[len(tokenizer):, :]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "xe6cQdnLkd5K"
   },
   "source": [
    "可以看到，这些【151665 - 151935】的无用token也和上面**【151644 - 151664 】**的特殊token的嵌入相同"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "URVvFUY1qASc",
    "outputId": "fe4463e0-3db6-41c9-88d1-99e51ea1618f"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[ 0.0077,  0.0110, -0.0062,  ...,  0.0048,  0.0024,  0.0010],\n",
       "        [ 0.0077,  0.0110, -0.0062,  ...,  0.0048,  0.0024,  0.0010],\n",
       "        [ 0.0077,  0.0110, -0.0062,  ...,  0.0048,  0.0024,  0.0010],\n",
       "        ...,\n",
       "        [ 0.0077,  0.0110, -0.0062,  ...,  0.0048,  0.0024,  0.0010],\n",
       "        [ 0.0077,  0.0110, -0.0062,  ...,  0.0048,  0.0024,  0.0010],\n",
       "        [ 0.0077,  0.0110, -0.0062,  ...,  0.0048,  0.0024,  0.0010]],\n",
       "       device='cuda:0', dtype=torch.bfloat16)"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.lm_head.weight[tokenizer.vocab_size+1:].detach()\n",
    "# 即 【151644 - 151935】的token嵌入都相同"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "zgGps1xgoly6",
    "outputId": "2d455b9a-5292-4216-c51a-a9eb453f9dc0"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([[ 0.0077,  0.0110, -0.0062,  ...,  0.0048,  0.0024,  0.0010],\n",
      "        [ 0.0077,  0.0110, -0.0062,  ...,  0.0048,  0.0024,  0.0010],\n",
      "        [ 0.0077,  0.0110, -0.0062,  ...,  0.0048,  0.0024,  0.0010],\n",
      "        ...,\n",
      "        [ 0.0077,  0.0110, -0.0062,  ...,  0.0048,  0.0024,  0.0010],\n",
      "        [ 0.0077,  0.0110, -0.0062,  ...,  0.0048,  0.0024,  0.0010],\n",
      "        [ 0.0077,  0.0110, -0.0062,  ...,  0.0048,  0.0024,  0.0010]],\n",
      "       device='cuda:0', dtype=torch.bfloat16)\n",
      "所有张量都相同（在容差范围内）\n"
     ]
    }
   ],
   "source": [
    "# 再查看最后的`lm_head`层中151644之后的张量是否都相同\n",
    "print(model.lm_head.weight[tokenizer.vocab_size+1:].detach())\n",
    "special_token_lmhead_weight = model.lm_head.weight[tokenizer.vocab_size+1:].detach()  #除了<|endoftext|>，这个特殊token在预训练阶段也会被训练\n",
    "# 选择一个张量作为基准\n",
    "reference_tensor = special_token_lmhead_weight[0]\n",
    "\n",
    "# 定义一个容差值，用于比较浮点数是否相等\n",
    "tolerance = 1e-4\n",
    "\n",
    "all_same = all(torch.allclose(tensor, reference_tensor, atol=tolerance) for tensor in special_token_lmhead_weight)\n",
    "\n",
    "if all_same:\n",
    "  print(\"所有张量都相同（在容差范围内）\")\n",
    "else:\n",
    "  print(\"张量不都相同\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "0qU_Du0F2TPo"
   },
   "source": [
    "## 1.1、增加`<|reasoning|>`这个special token"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "DEj_xQark1xB",
    "outputId": "8b82cf14-9307-43e2-e47a-97c7c2655c6e"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "before add new special token, the Vocabulary Size is: 151665\n",
      "after add new special token, the Vocabulary Size is: 151666\n"
     ]
    }
   ],
   "source": [
    "print(f\"before add new special token, the Vocabulary Size is: {len(tokenizer)}\")\n",
    "new_special_token = \"<|reasoning|>\"\n",
    "tokenizer.add_special_tokens({\"additional_special_tokens\": [new_special_token]})\n",
    "print(f\"after add new special token, the Vocabulary Size is: {len(tokenizer)}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "ZfVgGem8lB3t",
    "outputId": "ccbfc127-ca42-49f5-b6b5-6c7b7e7735d5"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "the id of <|reasoning|> is: 151665\n"
     ]
    }
   ],
   "source": [
    "# check the id of th new token\n",
    "new_token_id = tokenizer.convert_tokens_to_ids(new_special_token)\n",
    "print(f\"the id of {new_special_token} is: {new_token_id}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "_XgW3YPR2Mzf"
   },
   "source": [
    "## 1.2、新增 `<|reasoning|>` 该special token后，还需要同步调整模型的`embedding`层(可以不用执行)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "KT8nIJWklR94",
    "outputId": "300eb0aa-456b-4c39-b203-24025672b9c1"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Qwen2ForCausalLM(\n",
       "  (model): Qwen2Model(\n",
       "    (embed_tokens): Embedding(151666, 1536)\n",
       "    (layers): ModuleList(\n",
       "      (0-27): 28 x Qwen2DecoderLayer(\n",
       "        (self_attn): Qwen2SdpaAttention(\n",
       "          (q_proj): Linear(in_features=1536, out_features=1536, bias=True)\n",
       "          (k_proj): Linear(in_features=1536, out_features=256, bias=True)\n",
       "          (v_proj): Linear(in_features=1536, out_features=256, bias=True)\n",
       "          (o_proj): Linear(in_features=1536, out_features=1536, bias=False)\n",
       "          (rotary_emb): Qwen2RotaryEmbedding()\n",
       "        )\n",
       "        (mlp): Qwen2MLP(\n",
       "          (gate_proj): Linear(in_features=1536, out_features=8960, bias=False)\n",
       "          (up_proj): Linear(in_features=1536, out_features=8960, bias=False)\n",
       "          (down_proj): Linear(in_features=8960, out_features=1536, bias=False)\n",
       "          (act_fn): SiLU()\n",
       "        )\n",
       "        (input_layernorm): Qwen2RMSNorm((1536,), eps=1e-06)\n",
       "        (post_attention_layernorm): Qwen2RMSNorm((1536,), eps=1e-06)\n",
       "      )\n",
       "    )\n",
       "    (norm): Qwen2RMSNorm((1536,), eps=1e-06)\n",
       "  )\n",
       "  (lm_head): Linear(in_features=1536, out_features=151666, bias=False)\n",
       ")"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.resize_token_embeddings(len(tokenizer))\n",
    "model.to(device)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "ur0UT0T1uOJw"
   },
   "source": [
    "可以看到 嵌入层`embed_tokens`和最后的输出层`lm_head`都新增加了一个维度，变为了`151666`，我们接下来分别查看下新增加的这个维度的嵌入和最后的分类头的映射weight"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "p6u91XT6n0_N",
    "outputId": "9f0b7498-093f-43e1-f0ff-24c1b51ed27d"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[ 0.0077,  0.0110, -0.0062,  ...,  0.0048,  0.0024,  0.0010],\n",
       "        [ 0.0077,  0.0110, -0.0062,  ...,  0.0048,  0.0024,  0.0010],\n",
       "        [ 0.0077,  0.0110, -0.0062,  ...,  0.0048,  0.0024,  0.0010],\n",
       "        ...,\n",
       "        [ 0.0077,  0.0110, -0.0062,  ...,  0.0048,  0.0024,  0.0010],\n",
       "        [ 0.0077,  0.0110, -0.0062,  ...,  0.0048,  0.0024,  0.0010],\n",
       "        [ 0.0077,  0.0110, -0.0062,  ...,  0.0048,  0.0024,  0.0010]],\n",
       "       device='cuda:0', dtype=torch.bfloat16, grad_fn=<SliceBackward0>)"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.model.embed_tokens.weight[-22:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "8mS2qjMtuxif",
    "outputId": "13570b92-aae9-410b-959b-ff5842478854"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[ 0.0077,  0.0110, -0.0062,  ...,  0.0048,  0.0024,  0.0010],\n",
       "        [ 0.0077,  0.0110, -0.0062,  ...,  0.0048,  0.0024,  0.0010],\n",
       "        [ 0.0077,  0.0110, -0.0062,  ...,  0.0048,  0.0024,  0.0010],\n",
       "        ...,\n",
       "        [ 0.0077,  0.0110, -0.0062,  ...,  0.0048,  0.0024,  0.0010],\n",
       "        [ 0.0077,  0.0110, -0.0062,  ...,  0.0048,  0.0024,  0.0010],\n",
       "        [ 0.0077,  0.0110, -0.0062,  ...,  0.0048,  0.0024,  0.0010]],\n",
       "       device='cuda:0', dtype=torch.bfloat16, grad_fn=<SliceBackward0>)"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.lm_head.weight[-22:]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "M8n70oP4w7wH"
   },
   "source": [
    "- 看样子，qwen2系列的小模型的确是使用了参数共享tie_weights方法（embedding层和最后的lm_head层权值共享或者说权重绑定）以压缩参数量\n",
    "- 没记错的话，qwen系列是只有小模型使用了参数共享方法，大些体量的模型并没有使用共享方法\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "4I22BRIMboBU",
    "outputId": "f1d4c6ac-84f4-454c-d817-2ffd08b2737f"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 测试编解码码效果\n",
    "input_text = \"this is a test with a new special token: <|reasoning|>. let's task it\"\n",
    "# 编码\n",
    "input_ids = tokenizer.encode(input_text, return_tensors=\"pt\")\n",
    "# 再解码回去\n",
    "tokenizer.decode(input_ids[0]) == input_text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\"this is a test with a new special token: <|reasoning|>. let's task it\""
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tokenizer.decode(input_ids[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[151665]])"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tokenizer.encode(\"<|reasoning|>\", return_tensors=\"pt\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Embedding(151936, 1536)"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.model.embed_tokens"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "VhyXQrG51q45"
   },
   "source": [
    "## 以上我们就完成了对模型的**分词器**、**嵌入层**和**lm_head**均适配新增一个特殊token：**<|reasoning|>**"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "uOrzDK3fS5Yg"
   },
   "source": [
    "# 2、定义模型训练过程中的超参"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "id": "qnG99jDPS2sY"
   },
   "outputs": [],
   "source": [
    "from dataclasses import dataclass\n",
    "@dataclass\n",
    "class modelConfig:\n",
    "    max_length:int = 3000\n",
    "    batch_size:int = 1\n",
    "    gradient_accumulation_steps:int = 8\n",
    "    log_iter:int = 400\n",
    "    max_lr:float = 2e-5\n",
    "    min_lr:float = 2e-6\n",
    "    warmup_steps:int = 1000"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "yOBGBrZqSPbS"
   },
   "source": [
    "# 3、导入训练数据并开启训练"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "import subprocess\n",
    "import os\n",
    "\n",
    "result = subprocess.run('bash -c \"source /etc/network_turbo && env | grep proxy\"', shell=True, capture_output=True, text=True)\n",
    "output = result.stdout\n",
    "for line in output.splitlines():\n",
    "    if '=' in line:\n",
    "        var, value = line.split('=', 1)\n",
    "        os.environ[var] = value"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 113,
     "referenced_widgets": [
      "4d95e63b84494846a31ddc20a92d06de",
      "8978e7bba2364d1d802f2a7c7560297b",
      "3460c7f3c7274fc4ba06c8b6675bb37a",
      "00cfba85032949a98b31f4bffbb89274",
      "f9c966f2b73e4f3da9163dd14ed16363",
      "851e5249a3474421a1c527c3f35545b5",
      "65073b78db274217b2f9800549cdcd2d",
      "fdca97d2f6e8424eb842a161e5fd5835",
      "fa9b41c17c67430b99b3bf763b8e463d",
      "01737b33fe7e4aef9bdfa46fb5352daf",
      "4d206ca710734371a9ca172f1fbd0f79",
      "e91a61c0dc984ec09b9632af8a282611",
      "fcf3e96c5cbf46abb84161f29d0f4199",
      "9a33cd22c9cc4c34a2e60a87c37d8eaa",
      "38b6585254aa4332b1cd6b6c64432263",
      "6336f105491b4bd8ae01c0e3b6940c36",
      "c1e063e9f8174863b4acd5a553b5a71d",
      "0855ea4d4a684f53acb099b20bbd2606",
      "006b088e27734f77a03883739f69042b",
      "1c6b9b798563409cad0bd52e2ba423a9",
      "95de1ad519ec479797c56a59ae13826a",
      "87336531a7d449ea8e73b1707e921ccf",
      "2c3d43d5b1d34d3b963bf2ed582620ba",
      "a4ac3a557f2347b2b828f0c966a4d4fe",
      "bb94ba3e0f714a8e9b9b61c134e2f74c",
      "d2c8b1dc82a5473ab6f38bcc0430c981",
      "82a94b011a904f2884f6f8206064ed89",
      "33113930b3a84366bb5f5b60e40f234e",
      "1f47d0fcdac84c0986ac056b480c0dc7",
      "f46cdc40f11349108d92539de4185b33",
      "55bb9e5806eb422197e7d19e62305a2a",
      "58f28acb0be04593accf297e5e06bed0",
      "df94c5dc68864f8db8213f00e42af793"
     ]
    },
    "id": "6BpJtzNJSMM4",
    "outputId": "510a3832-a2ed-444c-dd03-4512d054c4e3"
   },
   "outputs": [],
   "source": [
    "import datasets\n",
    "reasoning_data = datasets.load_dataset('KingNish/reasoning-base-20k')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 49,
     "referenced_widgets": [
      "20a0b51901a644cd8318eccf8219b577",
      "95aece1615ac4f4fa568a77b4a0d0e1a",
      "6ece5bd8907b43248feb86065e8a2ea5",
      "9a9f863626b54cfebb48efe95142ebb2",
      "6a26f4f9406946628786a62cebd5a867",
      "8e216fa2d97d4b638e046076ebf2ec2f",
      "9128a37a06ce4b269b2e60a29710c181",
      "7b27d475ea6544d19df5d4c1291539b5",
      "137dc35b0c1e4b9ab04d0d8ec5ce7dac",
      "ea3f8d8660a942fba64236fbc2cbd787",
      "c7ba627ffac54eb89180843c8ad721bb"
     ]
    },
    "id": "qJzavJHmRhgP",
    "outputId": "30e8718b-3ea3-457e-f602-0e2ae4297394"
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f29457ea563349fd85c9be7f9745040d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Saving the dataset (0/1 shards):   0%|          | 0/19944 [00:00<?, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# prompt: save the dataset into local file\n",
    "reasoning_data.save_to_disk(\"./reasoning_data\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "3IuPJQhn_tD1",
    "outputId": "db310eea-7957-4c5d-86e8-252ac4643a1b"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Dataset({\n",
       "    features: ['user', 'reasoning', 'assistant', 'template', 'conversations'],\n",
       "    num_rows: 19944\n",
       "})"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "reasoning_data['train']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "hz45cipCLKC5",
    "outputId": "fffd3e7a-9129-49a2-8ca5-5cff17ca0663"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 全部是单轮对话\n",
    "max([len(i) for i in reasoning_data['train']['conversations']]) == 3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 49,
     "referenced_widgets": [
      "c58771d16f554bc68b783da40043281f",
      "b83f73dd875742459448f70c4d98eb0c",
      "6731988b1bb74f8ebe1a35c0196bee7b",
      "9dd40131eecd42a79519b897749b39e6",
      "f8b66aed4c76404ea165db0c0dc091bf",
      "0b2c49f831304163acc0d2f4cf0089b0",
      "514e06329caa4fbb963a382a3e894054",
      "ca3c78a681834bc4a6fb26dd1e9c1f8d",
      "07e127b8f0244ee78c3b5014cf311e91",
      "7b48926e24d448d38810001834020b55",
      "b9e2cc6fde514d5dbb26c936d1cdfed2"
     ]
    },
    "id": "CKY2mGXz9UzF",
    "outputId": "7b6f4772-629d-4df0-d1f0-232ee87cc1b8"
   },
   "outputs": [],
   "source": [
    "reasoning_data['train'] = reasoning_data['train'].map(lambda x: {\n",
    "    **x,\n",
    "    'user_template': \"\".join([\"<|im_start|>user\\n\", x['user'], \"<|im_end|>\\n\"]),\n",
    "    'reasoning_template': \"\".join([\"<|im_start|><|reasoning|>\\n\", x['reasoning'], \"<|im_end|>\\n\"]),\n",
    "    'assistant_template': \"\".join([\"<|im_start|>assistant\\n\", x['assistant'], \"<|im_end|>\\n\"]),\n",
    "    'template_new': \"\".join([       \"<|im_start|>system\\nYou are a helpful assistant<|im_end|>\\n\", #添加系统提示词\n",
    "                        \"<|im_start|>user\\n\", x['user'], \"<|im_end|>\\n\",\n",
    "                        \"<|im_start|><|reasoning|>\\n\", x['reasoning'], \"<|im_end|>\\n\",\n",
    "                        \"<|im_start|>assistant\\n\", x['assistant'], \"<|im_end|>\\n\"])\n",
    "})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "rLTnedY3Bnf3",
    "outputId": "24f969ec-9128-44db-da94-648623372faa"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Dataset({\n",
       "    features: ['user', 'reasoning', 'assistant', 'template', 'conversations', 'user_template', 'reasoning_template', 'assistant_template', 'template_new'],\n",
       "    num_rows: 19944\n",
       "})"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "reasoning_data['train']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "6w7YYl0LTIeQ",
    "outputId": "28333479-bdbd-46fd-fb06-71238bd4bbc0"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<|im_start|>system\n",
      "You are a helpful assistant<|im_end|>\n",
      "<|im_start|>user\n",
      "Prove that the difference between two consecutive cubes cannot be divisible by 5, using the fact that the only possible remainders when a cube is divided by 5 are 0, 1, and -1.<|im_end|>\n",
      "<|im_start|><|reasoning|>\n",
      "Let's break down the problem step by step and reason through it.\n",
      "\n",
      "**Understanding the Problem**\n",
      "\n",
      "The problem asks us to prove that the difference between two consecutive cubes cannot be divisible by 5. To approach this problem, we need to understand what consecutive cubes are and how we can represent them mathematically.\n",
      "\n",
      "Consecutive cubes are cubes of consecutive integers. For example, if we take the integer 2, its cube is 2^3 = 8. The next consecutive integer is 3, and its cube is 3^3 = 27. So, the difference between these two consecutive cubes is 27 - 8 = 19.\n",
      "\n",
      "**Representing Consecutive Cubes Mathematically**\n",
      "\n",
      "Let's represent the two consecutive cubes as n^3 and (n+1)^3, where n is an integer. The difference between these two cubes can be represented as:\n",
      "\n",
      "(n+1)^3 - n^3\n",
      "\n",
      "**Expanding the Difference**\n",
      "\n",
      "To simplify the expression, let's expand the difference using the binomial theorem:\n",
      "\n",
      "(n+1)^3 = n^3 + 3n^2 + 3n + 1\n",
      "\n",
      "So, the difference becomes:\n",
      "\n",
      "(n^3 + 3n^2 + 3n + 1) - n^3\n",
      "= 3n^2 + 3n + 1\n",
      "\n",
      "**Understanding the Remainders**\n",
      "\n",
      "Now, let's consider the possible remainders when a cube is divided by 5. We are given that the only possible remainders are 0, 1, and -1.\n",
      "\n",
      "When n^3 is divided by 5, the possible remainders are 0, 1, and -1. This means that n^3 can be represented as 5k, 5k+1, or 5k-1, where k is an integer.\n",
      "\n",
      "**Analyzing the Possible Remainders of the Difference**\n",
      "\n",
      "Now, let's analyze the possible remainders of the difference 3n^2 + 3n + 1 when divided by 5.\n",
      "\n",
      "If n^3 has a remainder of 0 when divided by 5, then n^3 = 5k for some integer k. In this case, the difference becomes:\n",
      "\n",
      "3(5k)^2 + 3(5k) + 1\n",
      "= 75k^2 + 15k + 1\n",
      "\n",
      "When divided by 5, this expression has a remainder of 1.\n",
      "\n",
      "If n^3 has a remainder of 1 when divided by 5, then n^3 = 5k+1 for some integer k. In this case, the difference becomes:\n",
      "\n",
      "3(5k+1)^2 + 3(5k+1) + 1\n",
      "= 75k^2 + 30k + 7\n",
      "\n",
      "When divided by 5, this expression has a remainder of 2.\n",
      "\n",
      "If n^3 has a remainder of -1 when divided by 5, then n^3 = 5k-1 for some integer k. In this case, the difference becomes:\n",
      "\n",
      "3(5k-1)^2 + 3(5k-1) + 1\n",
      "= 75k^2 - 30k + 1\n",
      "\n",
      "When divided by 5, this expression has a remainder of 1.\n",
      "\n",
      "**Conclusion**\n",
      "\n",
      "In all cases, the remainder of the difference 3n^2 + 3n + 1 when divided by 5 is not 0. Therefore, we can conclude that the difference between two consecutive cubes cannot be divisible by 5.\n",
      "\n",
      "**Self-Reflection**\n",
      "\n",
      "In this problem, we used the fact that the only possible remainders when a cube is divided by 5 are 0, 1, and -1. We then analyzed the possible remainders of the difference between two consecutive cubes and showed that in all cases, the remainder is not 0.\n",
      "\n",
      "This problem requires a good understanding of modular arithmetic and the properties of cubes. The key insight is to recognize that the difference between two consecutive cubes can be represented as 3n^2 + 3n + 1, and then to analyze the possible remainders of this expression when divided by 5.\n",
      "\n",
      "Overall, this problem requires careful reasoning and attention to detail. By breaking down the problem step by step and analyzing the possible remainders, we were able to prove that the difference between two consecutive cubes cannot be divisible by 5.<|im_end|>\n",
      "<|im_start|>assistant\n",
      "Let the two consecutive cubes be $n^3$ and $(n+1)^3$. Their difference is:\n",
      "\n",
      "$$(n+1)^3 - n^3 = 3n^2 + 3n + 1.$$\n",
      "\n",
      "When $n^3$ is divided by 5, the possible remainders are 0, 1, and -1. Therefore, when $(n+1)^3$ is divided by 5, the possible remainders are 1, 2, and 0, respectively.\n",
      "\n",
      "Thus, the possible remainders when the difference between the cubes is divided by 5 are:\n",
      "\n",
      "* 1 - 0 = 1\n",
      "* 2 - 1 = 1\n",
      "* 0 - (-1) = 1\n",
      "\n",
      "Since none of these remainders is 0, the difference between two consecutive cubes cannot be divisible by 5.<|im_end|>\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# 探查数据\n",
    "print(reasoning_data['train']['template_new'][0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "id": "Moy0m06heW4q"
   },
   "outputs": [],
   "source": [
    "# 确保编解码前后的文本对齐\n",
    "assert tokenizer.decode(tokenizer.encode(reasoning_data['train']['template_new'][0], return_tensors=\"pt\")[0]) == reasoning_data['train']['template_new'][0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "b8VlUkxz8K00",
    "outputId": "8ece7507-1d82-4e6b-e9b7-5b4646df8c0d"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<|im_start|>system\n",
      "You are a helpful assistant<|im_end|>\n",
      "<|im_start|>user\n",
      "Prove that the difference between two consecutive cubes cannot be divisible by 5, using the fact that the only possible remainders when a cube is divided by 5 are 0, 1, and -1.<|im_end|>\n",
      "<|im_start|><|reasoning|>\n",
      "Let's break down the problem step by step and reason through it.\n",
      "\n",
      "**Understanding the Problem**\n",
      "\n",
      "The problem asks us to prove that the difference between two consecutive cubes cannot be divisible by 5. To approach this problem, we need to understand what consecutive cubes are and how we can represent them mathematically.\n",
      "\n",
      "Consecutive cubes are cubes of consecutive integers. For example, if we take the integer 2, its cube is 2^3 = 8. The next consecutive integer is 3, and its cube is 3^3 = 27. So, the difference between these two consecutive cubes is 27 - 8 = 19.\n",
      "\n",
      "**Representing Consecutive Cubes Mathematically**\n",
      "\n",
      "Let's represent the two consecutive cubes as n^3 and (n+1)^3, where n is an integer. The difference between these two cubes can be represented as:\n",
      "\n",
      "(n+1)^3 - n^3\n",
      "\n",
      "**Expanding the Difference**\n",
      "\n",
      "To simplify the expression, let's expand the difference using the binomial theorem:\n",
      "\n",
      "(n+1)^3 = n^3 + 3n^2 + 3n + 1\n",
      "\n",
      "So, the difference becomes:\n",
      "\n",
      "(n^3 + 3n^2 + 3n + 1) - n^3\n",
      "= 3n^2 + 3n + 1\n",
      "\n",
      "**Understanding the Remainders**\n",
      "\n",
      "Now, let's consider the possible remainders when a cube is divided by 5. We are given that the only possible remainders are 0, 1, and -1.\n",
      "\n",
      "When n^3 is divided by 5, the possible remainders are 0, 1, and -1. This means that n^3 can be represented as 5k, 5k+1, or 5k-1, where k is an integer.\n",
      "\n",
      "**Analyzing the Possible Remainders of the Difference**\n",
      "\n",
      "Now, let's analyze the possible remainders of the difference 3n^2 + 3n + 1 when divided by 5.\n",
      "\n",
      "If n^3 has a remainder of 0 when divided by 5, then n^3 = 5k for some integer k. In this case, the difference becomes:\n",
      "\n",
      "3(5k)^2 + 3(5k) + 1\n",
      "= 75k^2 + 15k + 1\n",
      "\n",
      "When divided by 5, this expression has a remainder of 1.\n",
      "\n",
      "If n^3 has a remainder of 1 when divided by 5, then n^3 = 5k+1 for some integer k. In this case, the difference becomes:\n",
      "\n",
      "3(5k+1)^2 + 3(5k+1) + 1\n",
      "= 75k^2 + 30k + 7\n",
      "\n",
      "When divided by 5, this expression has a remainder of 2.\n",
      "\n",
      "If n^3 has a remainder of -1 when divided by 5, then n^3 = 5k-1 for some integer k. In this case, the difference becomes:\n",
      "\n",
      "3(5k-1)^2 + 3(5k-1) + 1\n",
      "= 75k^2 - 30k + 1\n",
      "\n",
      "When divided by 5, this expression has a remainder of 1.\n",
      "\n",
      "**Conclusion**\n",
      "\n",
      "In all cases, the remainder of the difference 3n^2 + 3n + 1 when divided by 5 is not 0. Therefore, we can conclude that the difference between two consecutive cubes cannot be divisible by 5.\n",
      "\n",
      "**Self-Reflection**\n",
      "\n",
      "In this problem, we used the fact that the only possible remainders when a cube is divided by 5 are 0, 1, and -1. We then analyzed the possible remainders of the difference between two consecutive cubes and showed that in all cases, the remainder is not 0.\n",
      "\n",
      "This problem requires a good understanding of modular arithmetic and the properties of cubes. The key insight is to recognize that the difference between two consecutive cubes can be represented as 3n^2 + 3n + 1, and then to analyze the possible remainders of this expression when divided by 5.\n",
      "\n",
      "Overall, this problem requires careful reasoning and attention to detail. By breaking down the problem step by step and analyzing the possible remainders, we were able to prove that the difference between two consecutive cubes cannot be divisible by 5.<|im_end|>\n",
      "<|im_start|>assistant\n",
      "Let the two consecutive cubes be $n^3$ and $(n+1)^3$. Their difference is:\n",
      "\n",
      "$$(n+1)^3 - n^3 = 3n^2 + 3n + 1.$$\n",
      "\n",
      "When $n^3$ is divided by 5, the possible remainders are 0, 1, and -1. Therefore, when $(n+1)^3$ is divided by 5, the possible remainders are 1, 2, and 0, respectively.\n",
      "\n",
      "Thus, the possible remainders when the difference between the cubes is divided by 5 are:\n",
      "\n",
      "* 1 - 0 = 1\n",
      "* 2 - 1 = 1\n",
      "* 0 - (-1) = 1\n",
      "\n",
      "Since none of these remainders is 0, the difference between two consecutive cubes cannot be divisible by 5.<|im_end|>\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(tokenizer.decode(tokenizer.encode(reasoning_data['train']['template_new'][0], return_tensors=\"pt\", truncation=True, max_length=modelConfig.max_length)[0]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "V5qUvo8LSorW"
   },
   "source": [
    "## 3.1、训练数据转化成tokenid：str -> tokenid"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 49,
     "referenced_widgets": [
      "bd0aab37191745f09f540c2f9fa70f52",
      "002465c05b71462198afe7ffee5f23e6",
      "c5bd5af525c94d57ace64357c7ddb5f2",
      "fbb1c9b7489242619cf2dc3bc0476ddd",
      "8eaf3b02c5eb413a92308be974b31b49",
      "3bfcfec6942b4b32a46a9b2c43227d8d",
      "95ad4875749843a2b1cf08987c682df9",
      "7ad6a66d01684ca3bd845c5cb8e5825a",
      "2176b48b1f454b249d638f9f520e274b",
      "25843be9e67346f5a9faf475a9d396ad",
      "32767a4e19fd4bbe9887b6a54c916893"
     ]
    },
    "id": "0dxeb9UVTBG5",
    "outputId": "8baa550f-ff95-4c7d-e8db-3aff6798fd8c"
   },
   "outputs": [],
   "source": [
    "reasoning_data['train'] = reasoning_data['train'].map(lambda x: {\n",
    "    **x,\n",
    "    'truncated_token_id': tokenizer.encode(x['template_new'], truncation=True, max_length=modelConfig.max_length)\n",
    "})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 472
    },
    "id": "sP6Htq8opvXk",
    "outputId": "7b89a4e0-888f-4b06-c02a-164c11bcc235"
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAkQAAAHHCAYAAABeLEexAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjkuMCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy80BEi2AAAACXBIWXMAAA9hAAAPYQGoP6dpAAA+R0lEQVR4nO3dd3QV1f7+8ecAOSGUJLQ0Sug1NIELuTQlkVBEFLwCghSjKIKCIALqpekFpQliQa9XEEVFvAp8QZBAaGJE4RK6ERCIShIQSEJAWrJ/f7AyPw6hhpPGvF9rnbWY2XtmPrM5IQ8ze85xGGOMAAAAbKxQXhcAAACQ1whEAADA9ghEAADA9ghEAADA9ghEAADA9ghEAADA9ghEAADA9ghEAADA9ghEAADA9ghEQA4ZP368HA5Hrhzr7rvv1t13320tr1u3Tg6HQ19++WWuHL9///6qXLlyrhwru9LS0vT4448rICBADodDw4YNy9HjZf79//nnnzl6nDtd//79VaJEibwuAzZAIAJuwrx58+RwOKxX0aJFFRQUpIiICL355ps6deqUW45z5MgRjR8/XrGxsW7Znzvl59puxqRJkzRv3jwNGjRIH3/8sR599NEsfTJDzI1el4fPgiC3A/KtOnPmjMaPH69169bldSmwsSJ5XQBQkEycOFFVqlTRhQsXlJiYqHXr1mnYsGGaMWOGli5dqgYNGlh9X375ZY0ePfqW9n/kyBFNmDBBlStXVqNGjW56u1WrVt3ScbLjerX9+9//VkZGRo7XcDuio6PVokULjRs37pp9unXrpurVq1vLaWlpGjRokB588EF169bNWu/v75+jtdrNmTNnNGHCBEkqcGETdw4CEXALOnbsqKZNm1rLY8aMUXR0tO677z7df//92rt3r7y8vCRJRYoUUZEiOfsjdubMGRUrVkxOpzNHj3MjHh4eeXr8m3H06FHVrVv3un0aNGjgEmr//PNPDRo0SA0aNFCfPn1yukQAeYhbZsBtateunf75z3/q8OHD+uSTT6z1V5tDFBUVpVatWsnX11clSpRQrVq19OKLL0q6dFujWbNmkqQBAwZYt2fmzZsn6dL/nENCQrR161a1adNGxYoVs7a9cg5RpvT0dL344osKCAhQ8eLFdf/99+u3335z6VO5cmX1798/y7aX7/NGtV1tDtHp06c1YsQIVaxYUZ6enqpVq5amTZsmY4xLP4fDoSFDhmjx4sUKCQmRp6en6tWrp5UrV159wK9w9OhRRUZGyt/fX0WLFlXDhg310UcfWe2Zt4sOHjyo5cuXW7UfOnTopvZ/NdHR0WrdurWKFy8uX19fde3aVXv37r3hdocPH1b16tUVEhKipKQkSVJycrKGDRtmjVP16tX1+uuvu1xxO3TokBwOh6ZNm6b3339f1apVk6enp5o1a6affvop2+dxpZyoZdGiRapbt66KFi2qkJAQff311y7vl0OHDqlcuXKSpAkTJlh/P+PHj3fZzx9//KEHHnhAJUqUULly5fT8888rPT3dpc/nn3+uJk2aqGTJkvL29lb9+vU1a9Yst40P7mxcIQLc4NFHH9WLL76oVatW6Yknnrhqn927d+u+++5TgwYNNHHiRHl6emr//v3atGmTJKlOnTqaOHGixo4dq4EDB6p169aSpL///e/WPo4fP66OHTuqZ8+e6tOnzw1v3fzrX/+Sw+HQqFGjdPToUc2cOVPh4eGKjY21rmTdjJup7XLGGN1///1au3atIiMj1ahRI3377bcaOXKk/vjjD73xxhsu/b/77jt99dVXevrpp1WyZEm9+eab6t69u+Lj41WmTJlr1vXXX3/p7rvv1v79+zVkyBBVqVJFixYtUv/+/ZWcnKyhQ4eqTp06+vjjj/Xcc8+pQoUKGjFihCRZv4Rv1erVq9WxY0dVrVpV48eP119//aXZs2erZcuW+t///nfNyeUHDhxQu3btVLp0aUVFRals2bI6c+aM2rZtqz/++ENPPvmkKlWqpO+//15jxoxRQkKCZs6c6bKPTz/9VKdOndKTTz4ph8OhKVOmqFu3bvr1119v+ypdTtSyfPly9ejRQ/Xr19fkyZN18uRJRUZGqnz58tZ+ypUrp3fffTfLrcnLr9Slp6crIiJCzZs317Rp07R69WpNnz5d1apV06BBgyRd+s9Gr169FBYWptdff12StHfvXm3atElDhw69rbGBTRgANzR37lwjyfz000/X7OPj42MaN25sLY8bN85c/iP2xhtvGEnm2LFj19zHTz/9ZCSZuXPnZmlr27atkWTmzJlz1ba2bdtay2vXrjWSTPny5U1qaqq1/osvvjCSzKxZs6x1wcHBpl+/fjfc5/Vq69evnwkODraWFy9ebCSZV1991aXfQw89ZBwOh9m/f7+1TpJxOp0u67Zv324kmdmzZ2c51uVmzpxpJJlPPvnEWnf+/HkTGhpqSpQo4XLuwcHBpnPnztfd35WOHTtmJJlx48ZZ6xo1amT8/PzM8ePHXeotVKiQ6du3r7Uu8+//2LFjZu/evSYoKMg0a9bMnDhxwurzyiuvmOLFi5tffvnF5bijR482hQsXNvHx8cYYYw4ePGgkmTJlyrhsv2TJEiPJ/N///d91zyPz/bBo0aJr9smJWurXr28qVKhgTp06Za1bt26dkeTyfrnaOGfq16+fkWQmTpzosr5x48amSZMm1vLQoUONt7e3uXjx4nXHArgWbpkBblKiRInrPm3m6+srSVqyZEm2JyB7enpqwIABN92/b9++KlmypLX80EMPKTAwUN988022jn+zvvnmGxUuXFjPPvusy/oRI0bIGKMVK1a4rA8PD1e1atWs5QYNGsjb21u//vrrDY8TEBCgXr16Wes8PDz07LPPKi0tTevXr3fD2fx/CQkJio2NVf/+/VW6dGmXeu+9996rjuuuXbvUtm1bVa5cWatXr1apUqWstkWLFql169YqVaqU/vzzT+sVHh6u9PR0bdiwwWVfPXr0cNk+80rdjcbpZri7liNHjmjnzp3q27evy2Pzbdu2Vf369W+5vqeeespluXXr1i7n7evrq9OnTysqKuqW9w1IzCEC3CYtLc0lfFypR48eatmypR5//HH5+/urZ8+e+uKLL24pHJUvX/6WJlDXqFHDZdnhcKh69eq3NX/mZhw+fFhBQUFZxqNOnTpW++UqVaqUZR+lSpXSyZMnb3icGjVqqFAh13/KrnWc25W5v1q1amVpq1Onjv7880+dPn3aZX2XLl1UsmRJffvtt/L29nZp27dvn1auXKly5cq5vMLDwyVdmh91uSvHKTOQ3Gicboa7a8kcq8uf2st0tXXXU7Ro0Sy3OK98fzz99NOqWbOmOnbsqAoVKuixxx676XlogMQcIsAtfv/9d6WkpFz3H3ovLy9t2LBBa9eu1fLly7Vy5UotXLhQ7dq106pVq1S4cOEbHudW5v3crGt9eGR6evpN1eQO1zqOuWICdkHUvXt3ffTRR1qwYIGefPJJl7aMjAzde++9euGFF666bc2aNV2Wc3Kc8lMtV7qZ96Gfn59iY2P17bffasWKFVqxYoXmzp2rvn37ukyyB66FQAS4wccffyxJioiIuG6/QoUKKSwsTGFhYZoxY4YmTZqkl156SWvXrlV4eLjbP9l63759LsvGGO3fv99lwmqpUqWUnJycZdvDhw+ratWq1vKt1BYcHKzVq1fr1KlTLleJfv75Z6vdHYKDg7Vjxw5lZGS4XCVy93EuP54kxcXFZWn7+eefVbZsWRUvXtxl/dSpU1WkSBFrwvgjjzxitVWrVk1paWnWVZi85O5aMsdq//79WdquXOeu973T6VSXLl3UpUsXZWRk6Omnn9Z7772nf/7zn7d8VQr2wy0z4DZFR0frlVdeUZUqVdS7d+9r9jtx4kSWdZkfcHju3DlJsn6ZXi2gZMf8+fNd5jV9+eWXSkhIUMeOHa111apV0w8//KDz589b65YtW5bl8fxbqa1Tp05KT0/XW2+95bL+jTfekMPhcDn+7ejUqZMSExO1cOFCa93Fixc1e/ZslShRQm3btnXLcTIFBgaqUaNG+uijj1zGYdeuXVq1apU6deqUZRuHw6H3339fDz30kPr166elS5dabQ8//LBiYmL07bffZtkuOTlZFy9edGv91+PuWoKCghQSEqL58+crLS3NWr9+/Xrt3LnTpW+xYsWs42TX8ePHXZYLFSpkBf/Mny/gerhCBNyCFStW6Oeff9bFixeVlJSk6OhoRUVFKTg4WEuXLlXRokWvue3EiRO1YcMGde7cWcHBwTp69KjeeecdVahQQa1atZJ0KZz4+vpqzpw5KlmypIoXL67mzZurSpUq2aq3dOnSatWqlQYMGKCkpCTNnDlT1atXd/logMcff1xffvmlOnTooIcfflgHDhzQJ5984jLJ+VZr69Kli+655x699NJLOnTokBo2bKhVq1ZpyZIlGjZsWJZ9Z9fAgQP13nvvqX///tq6dasqV66sL7/8Ups2bdLMmTOvO6cru6ZOnaqOHTsqNDRUkZGR1mP3Pj4+WT47J1OhQoX0ySef6IEHHtDDDz+sb775Ru3atdPIkSO1dOlS3Xffferfv7+aNGmi06dPa+fOnfryyy916NAhlS1b1m21//e//7Wunl2uX79+OVLLpEmT1LVrV7Vs2VIDBgzQyZMn9dZbbykkJMQlJHl5ealu3bpauHChatasqdKlSyskJEQhISE3fazHH39cJ06cULt27VShQgUdPnxYs2fPVqNGjaw5ZcB15ekzbkABkfnYfebL6XSagIAAc++995pZs2a5PN6d6crH7tesWWO6du1qgoKCjNPpNEFBQaZXr15ZHnNesmSJqVu3rilSpIjLY+5t27Y19erVu2p913rs/rPPPjNjxowxfn5+xsvLy3Tu3NkcPnw4y/bTp0835cuXN56enqZly5Zmy5YtWfZ5vdqufOzeGGNOnTplnnvuORMUFGQ8PDxMjRo1zNSpU01GRoZLP0lm8ODBWWq61scBXCkpKckMGDDAlC1b1jidTlO/fv2rfjSAux67N8aY1atXm5YtWxovLy/j7e1tunTpYvbs2ePS5/LH7jOdOXPGtG3b1pQoUcL88MMPxphL4zRmzBhTvXp143Q6TdmyZc3f//53M23aNHP+/HljzP9/1H3q1KlZarxafVfKfD9c67Vx48Ycq+Xzzz83tWvXNp6eniYkJMQsXbrUdO/e3dSuXdul3/fff2+aNGlinE6ny3769etnihcvnuVYV/58ffnll6Z9+/bGz8/POJ1OU6lSJfPkk0+ahISE644NkMlhzB0waxEAUGA0atRI5cqV4xF55CvMIQIA5IgLFy5kmXu0bt06bd++nS9xRb7DFSIAQI44dOiQwsPD1adPHwUFBennn3/WnDlz5OPjo127dl33a1mA3MakagBAjihVqpSaNGmiDz74QMeOHVPx4sXVuXNnvfbaa4Qh5DtcIQIAALbHHCIAAGB7BCIAAGB7zCG6CRkZGTpy5IhKlizp9q9WAAAAOcMYo1OnTikoKCjLl0BfiUB0E44cOaKKFSvmdRkAACAbfvvtN1WoUOG6fQhENyHz4/9/++03eXt753E1AADgZqSmpqpixYo39TU+BKKbkHmbzNvbm0AEAEABczPTXZhUDQAAbI9ABAAAbI9ABAAAbI9ABAAAbI9ABAAAbI9ABAAAbI9ABAAAbI9ABAAAbI9ABAAAbI9ABAAAbI9ABAAAbI9ABAAAbI9ABAAAbI9ABAAAbI9ABAAAbK9IXhcA3Okqj16e7W0PvdbZjZUAAK6FK0QAAMD2CEQAAMD2CEQAAMD2CEQAAMD2CEQAAMD2CEQAAMD2CEQAAMD2CEQAAMD2CEQAAMD2CEQAAMD2CEQAAMD2CEQAAMD2CEQAAMD2CEQAAMD2CEQAAMD2iuR1AQCurfLo5dne9tBrnd1YCQDc2bhCBAAAbI9ABAAAbI9ABAAAbI9ABAAAbI9ABAAAbI9ABAAAbI9ABAAAbI9ABAAAbI9ABAAAbI9ABAAAbC9PA9HkyZPVrFkzlSxZUn5+fnrggQcUFxfn0ufs2bMaPHiwypQpoxIlSqh79+5KSkpy6RMfH6/OnTurWLFi8vPz08iRI3Xx4kWXPuvWrdNdd90lT09PVa9eXfPmzcvp0wMAAAVEnn6X2fr16zV48GA1a9ZMFy9e1Isvvqj27dtrz549Kl68uCTpueee0/Lly7Vo0SL5+PhoyJAh6tatmzZt2iRJSk9PV+fOnRUQEKDvv/9eCQkJ6tu3rzw8PDRp0iRJ0sGDB9W5c2c99dRTWrBggdasWaPHH39cgYGBioiIyLPzB3IS34MGADfPYYwxeV1EpmPHjsnPz0/r169XmzZtlJKSonLlyunTTz/VQw89JEn6+eefVadOHcXExKhFixZasWKF7rvvPh05ckT+/v6SpDlz5mjUqFE6duyYnE6nRo0apeXLl2vXrl3WsXr27Knk5GStXLnyhnWlpqbKx8dHKSkp8vb2zpmTxx3rdoJJXiEQAbgT3Mrv73w1hyglJUWSVLp0aUnS1q1bdeHCBYWHh1t9ateurUqVKikmJkaSFBMTo/r161thSJIiIiKUmpqq3bt3W30u30dmn8x9XOncuXNKTU11eQEAgDtXvglEGRkZGjZsmFq2bKmQkBBJUmJiopxOp3x9fV36+vv7KzEx0epzeRjKbM9su16f1NRU/fXXX1lqmTx5snx8fKxXxYoV3XKOAAAgf8o3gWjw4MHatWuXPv/887wuRWPGjFFKSor1+u233/K6JAAAkIPydFJ1piFDhmjZsmXasGGDKlSoYK0PCAjQ+fPnlZyc7HKVKCkpSQEBAVafH3/80WV/mU+hXd7nyifTkpKS5O3tLS8vryz1eHp6ytPT0y3nBgAA8r88vUJkjNGQIUP09ddfKzo6WlWqVHFpb9KkiTw8PLRmzRprXVxcnOLj4xUaGipJCg0N1c6dO3X06FGrT1RUlLy9vVW3bl2rz+X7yOyTuQ8AAGBveXqFaPDgwfr000+1ZMkSlSxZ0prz4+PjIy8vL/n4+CgyMlLDhw9X6dKl5e3trWeeeUahoaFq0aKFJKl9+/aqW7euHn30UU2ZMkWJiYl6+eWXNXjwYOsqz1NPPaW33npLL7zwgh577DFFR0friy++0PLlBe/pHwAA4H55eoXo3XffVUpKiu6++24FBgZar4ULF1p93njjDd13333q3r272rRpo4CAAH311VdWe+HChbVs2TIVLlxYoaGh6tOnj/r27auJEydafapUqaLly5crKipKDRs21PTp0/XBBx/wGUQAAEBSPvscovyKzyHC7eBziAAgbxTYzyECAADICwQiAABgewQiAABgewQiAABgewQiAABgewQiAABgewQiAABgewQiAABgewQiAABgewQiAABge3n65a5AQVEQv34DAHDzuEIEAABsj0AEAABsj0AEAABsj0AEAABsj0AEAABsj0AEAABsj0AEAABsj0AEAABsj0AEAABsj0AEAABsj6/uAJDF7XxVyaHXOruxEgDIHVwhAgAAtkcgAgAAtkcgAgAAtkcgAgAAtkcgAgAAtkcgAgAAtkcgAgAAtkcgAgAAtkcgAgAAtkcgAgAAtkcgAgAAtkcgAgAAtkcgAgAAtkcgAgAAtkcgAgAAtkcgAgAAtkcgAgAAtkcgAgAAtkcgAgAAtkcgAgAAtkcgAgAAtkcgAgAAtkcgAgAAtkcgAgAAtkcgAgAAtkcgAgAAtkcgAgAAtkcgAgAAtkcgAgAAtkcgAgAAtkcgAgAAtkcgAgAAtkcgAgAAtkcgAgAAtkcgAgAAtkcgAgAAtkcgAgAAtkcgAgAAtkcgAgAAtkcgAgAAtkcgAgAAtkcgAgAAtkcgAgAAtlckrwsAckvl0cvzugQAQD7FFSIAAGB7BCIAAGB7BCIAAGB7eRqINmzYoC5duigoKEgOh0OLFy92ae/fv78cDofLq0OHDi59Tpw4od69e8vb21u+vr6KjIxUWlqaS58dO3aodevWKlq0qCpWrKgpU6bk9KkBAIACJE8D0enTp9WwYUO9/fbb1+zToUMHJSQkWK/PPvvMpb13797avXu3oqKitGzZMm3YsEEDBw602lNTU9W+fXsFBwdr69atmjp1qsaPH6/3338/x84LAAAULHn6lFnHjh3VsWPH6/bx9PRUQEDAVdv27t2rlStX6qefflLTpk0lSbNnz1anTp00bdo0BQUFacGCBTp//rw+/PBDOZ1O1atXT7GxsZoxY4ZLcAIAAPaV7+cQrVu3Tn5+fqpVq5YGDRqk48ePW20xMTHy9fW1wpAkhYeHq1ChQtq8ebPVp02bNnI6nVafiIgIxcXF6eTJk1c95rlz55SamuryAgAAd658HYg6dOig+fPna82aNXr99de1fv16dezYUenp6ZKkxMRE+fn5uWxTpEgRlS5dWomJiVYff39/lz6Zy5l9rjR58mT5+PhYr4oVK7r71AAAQD6Srz+YsWfPntaf69evrwYNGqhatWpat26dwsLCcuy4Y8aM0fDhw63l1NRUQhEAAHewfH2F6EpVq1ZV2bJltX//fklSQECAjh496tLn4sWLOnHihDXvKCAgQElJSS59MpevNTfJ09NT3t7eLi8AAHDnKlCB6Pfff9fx48cVGBgoSQoNDVVycrK2bt1q9YmOjlZGRoaaN29u9dmwYYMuXLhg9YmKilKtWrVUqlSp3D0BAACQL+VpIEpLS1NsbKxiY2MlSQcPHlRsbKzi4+OVlpamkSNH6ocfftChQ4e0Zs0ade3aVdWrV1dERIQkqU6dOurQoYOeeOIJ/fjjj9q0aZOGDBminj17KigoSJL0yCOPyOl0KjIyUrt379bChQs1a9Ysl1tiAADA3vI0EG3ZskWNGzdW48aNJUnDhw9X48aNNXbsWBUuXFg7duzQ/fffr5o1ayoyMlJNmjTRxo0b5enpae1jwYIFql27tsLCwtSpUye1atXK5TOGfHx8tGrVKh08eFBNmjTRiBEjNHbsWB65BwAAFocxxuR1EfldamqqfHx8lJKSwnyiAoxvu88dh17rnNclAICkW/v9XaDmEAEAAOQEAhEAALA9AhEAALA9AhEAALA9AhEAALA9AhEAALA9AhEAALC9fP3lrgAKntv5vCc+wwhAXuEKEQAAsD0CEQAAsD0CEQAAsD0CEQAAsD0CEQAAsD0CEQAAsD0CEQAAsD0CEQAAsL1sBaJff/3V3XUAAADkmWwFourVq+uee+7RJ598orNnz7q7JgAAgFyVrUD0v//9Tw0aNNDw4cMVEBCgJ598Uj/++KO7awMAAMgV2QpEjRo10qxZs3TkyBF9+OGHSkhIUKtWrRQSEqIZM2bo2LFj7q4TAAAgx9zWpOoiRYqoW7duWrRokV5//XXt379fzz//vCpWrKi+ffsqISHBXXUCAADkmNsKRFu2bNHTTz+twMBAzZgxQ88//7wOHDigqKgoHTlyRF27dnVXnQAAADmmSHY2mjFjhubOnau4uDh16tRJ8+fPV6dOnVSo0KV8VaVKFc2bN0+VK1d2Z60AAAA5IluB6N1339Vjjz2m/v37KzAw8Kp9/Pz89J///Oe2igMAAMgN2QpE+/btu2Efp9Opfv36ZWf3AAAAuSpbc4jmzp2rRYsWZVm/aNEiffTRR7ddFAAAQG7KViCaPHmyypYtm2W9n5+fJk2adNtFAQAA5KZsBaL4+HhVqVIly/rg4GDFx8ffdlEAAAC5KVuByM/PTzt27Miyfvv27SpTpsxtFwUAAJCbshWIevXqpWeffVZr165Venq60tPTFR0draFDh6pnz57urhEAACBHZesps1deeUWHDh1SWFiYihS5tIuMjAz17duXOUQAAKDAyVYgcjqdWrhwoV555RVt375dXl5eql+/voKDg91dHwAAQI7LViDKVLNmTdWsWdNdtQAAAOSJbAWi9PR0zZs3T2vWrNHRo0eVkZHh0h4dHe2W4gAAAHJDtgLR0KFDNW/ePHXu3FkhISFyOBzurgsAACDXZCsQff755/riiy/UqVMnd9cDAACQ67L12L3T6VT16tXdXQsAAECeyFYgGjFihGbNmiVjjLvrAQAAyHXZumX23Xffae3atVqxYoXq1asnDw8Pl/avvvrKLcUBAADkhmwFIl9fXz344IPurgUAACBPZCsQzZ071911AIAqj16e7W0PvdbZjZXkDrudL5CfZWsOkSRdvHhRq1ev1nvvvadTp05Jko4cOaK0tDS3FQcAAJAbsnWF6PDhw+rQoYPi4+N17tw53XvvvSpZsqRef/11nTt3TnPmzHF3nQAAADkmW1eIhg4dqqZNm+rkyZPy8vKy1j/44INas2aN24oDAADIDdm6QrRx40Z9//33cjqdLusrV66sP/74wy2FAQAA5JZsXSHKyMhQenp6lvW///67SpYsedtFAQAA5KZsBaL27dtr5syZ1rLD4VBaWprGjRvH13kAAIACJ1u3zKZPn66IiAjVrVtXZ8+e1SOPPKJ9+/apbNmy+uyzz9xdIwAAQI7KViCqUKGCtm/frs8//1w7duxQWlqaIiMj1bt3b5dJ1gAAAAVBtgKRJBUpUkR9+vRxZy0AAAB5IluBaP78+ddt79u3b7aKAQAAyAvZCkRDhw51Wb5w4YLOnDkjp9OpYsWKEYgAAECBkq2nzE6ePOnySktLU1xcnFq1asWkagAAUOBk+7vMrlSjRg299tprWa4eAQAA5HduC0TSpYnWR44ccecuAQAAcly25hAtXbrUZdkYo4SEBL311ltq2bKlWwoDAADILdkKRA888IDLssPhULly5dSuXTtNnz7dHXUBAADkmmwFooyMDHfXAQAAkGfcOocIAACgIMrWFaLhw4ffdN8ZM2Zk5xAAAAC5JluBaNu2bdq2bZsuXLigWrVqSZJ++eUXFS5cWHfddZfVz+FwuKdKAACAHJStQNSlSxeVLFlSH330kUqVKiXp0oc1DhgwQK1bt9aIESPcWiQAAEBOytYcounTp2vy5MlWGJKkUqVK6dVXX+UpMwAAUOBkKxClpqbq2LFjWdYfO3ZMp06duu2iAAAAclO2AtGDDz6oAQMG6KuvvtLvv/+u33//Xf/9738VGRmpbt26ubtGAACAHJWtOURz5szR888/r0ceeUQXLly4tKMiRRQZGampU6e6tUAAAICclq1AVKxYMb3zzjuaOnWqDhw4IEmqVq2aihcv7tbiAAAAcsNtfTBjQkKCEhISVKNGDRUvXlzGGHfVBQAAkGuyFYiOHz+usLAw1axZU506dVJCQoIkKTIykkfuAQBAgZOtQPTcc8/Jw8ND8fHxKlasmLW+R48eWrly5U3vZ8OGDerSpYuCgoLkcDi0ePFil3ZjjMaOHavAwEB5eXkpPDxc+/btc+lz4sQJ9e7dW97e3vL19VVkZKTS0tJc+uzYsUOtW7dW0aJFVbFiRU2ZMuXWTxoAANyxshWIVq1apddff10VKlRwWV+jRg0dPnz4pvdz+vRpNWzYUG+//fZV26dMmaI333xTc+bM0ebNm1W8eHFFRETo7NmzVp/evXtr9+7dioqK0rJly7RhwwYNHDjQak9NTVX79u0VHBysrVu3aurUqRo/frzef//9WzxrAABwp8rWpOrTp0+7XBnKdOLECXl6et70fjp27KiOHTtetc0Yo5kzZ+rll19W165dJUnz58+Xv7+/Fi9erJ49e2rv3r1auXKlfvrpJzVt2lSSNHv2bHXq1EnTpk1TUFCQFixYoPPnz+vDDz+U0+lUvXr1FBsbqxkzZrgEJwAAYF/ZukLUunVrzZ8/31p2OBzKyMjQlClTdM8997ilsIMHDyoxMVHh4eHWOh8fHzVv3lwxMTGSpJiYGPn6+lphSJLCw8NVqFAhbd682erTpk0bOZ1Oq09ERITi4uJ08uTJqx773LlzSk1NdXkBAIA7V7auEE2ZMkVhYWHasmWLzp8/rxdeeEG7d+/WiRMntGnTJrcUlpiYKEny9/d3We/v72+1JSYmys/Pz6W9SJEiKl26tEufKlWqZNlHZtvlXz+SafLkyZowYYJbzgMAAOR/2bpCFBISol9++UWtWrVS165ddfr0aXXr1k3btm1TtWrV3F1jrhszZoxSUlKs12+//ZbXJQEAgBx0y1eILly4oA4dOmjOnDl66aWXcqImSVJAQIAkKSkpSYGBgdb6pKQkNWrUyOpz9OhRl+0uXryoEydOWNsHBAQoKSnJpU/mcmafK3l6et7SXCgAAFCw3XIg8vDw0I4dO3KiFhdVqlRRQECA1qxZYwWg1NRUbd68WYMGDZIkhYaGKjk5WVu3blWTJk0kSdHR0crIyFDz5s2tPi+99JIuXLggDw8PSVJUVJRq1ap11dtlAAqmyqOXZ3vbQ691dmMlAAqibN0y69Onj/7zn//c9sHT0tIUGxur2NhYSZcmUsfGxio+Pl4Oh0PDhg3Tq6++qqVLl2rnzp3q27evgoKC9MADD0iS6tSpow4dOuiJJ57Qjz/+qE2bNmnIkCHq2bOngoKCJEmPPPKInE6nIiMjtXv3bi1cuFCzZs3S8OHDb7t+AABwZ8jWpOqLFy/qww8/1OrVq9WkSZMs32E2Y8aMm9rPli1bXJ5Kywwp/fr107x58/TCCy/o9OnTGjhwoJKTk9WqVSutXLlSRYsWtbZZsGCBhgwZorCwMBUqVEjdu3fXm2++abX7+Pho1apVGjx4sJo0aaKyZctq7NixPHIPAAAsDnMLX0D266+/qnLlygoLC7v2Dh0ORUdHu6W4/CI1NVU+Pj5KSUmRt7d3XpeDbLqdWyq4s+XVLTNu8wE561Z+f9/SFaIaNWooISFBa9eulXTpqzrefPPNLI/GAwAAFCS3FIiuvJi0YsUKnT592q0FAdfDVR4AQE7I1qTqTLdwtw0AACDfuqVA5HA45HA4sqwDAAAoyG75lln//v2tDy08e/asnnrqqSxPmX311VfuqxAAACCH3VIg6tevn8tynz593FoMAABAXrilQDR37tycqgMAACDP3NakagAAgDsBgQgAANgegQgAANgegQgAANgegQgAANgegQgAANgegQgAANgegQgAANgegQgAANgegQgAANgegQgAANgegQgAANgegQgAANgegQgAANgegQgAANgegQgAANgegQgAANgegQgAANgegQgAANgegQgAANgegQgAANgegQgAANgegQgAANgegQgAANhekbwuAADyWuXRy7O97aHXOruxEgB5hStEAADA9ghEAADA9ghEAADA9ghEAADA9ghEAADA9ghEAADA9ghEAADA9ghEAADA9ghEAADA9ghEAADA9ghEAADA9ghEAADA9ghEAADA9ghEAADA9ghEAADA9ghEAADA9ghEAADA9ghEAADA9ghEAADA9ghEAADA9ghEAADA9ghEAADA9ghEAADA9ghEAADA9ghEAADA9ghEAADA9ghEAADA9ghEAADA9ghEAADA9ghEAADA9ghEAADA9ghEAADA9ghEAADA9ghEAADA9orkdQEAUJBVHr08r0sA4AZcIQIAALZHIAIAALaXrwPR+PHj5XA4XF61a9e22s+ePavBgwerTJkyKlGihLp3766kpCSXfcTHx6tz584qVqyY/Pz8NHLkSF28eDG3TwUAAORj+X4OUb169bR69WpruUiR/1/yc889p+XLl2vRokXy8fHRkCFD1K1bN23atEmSlJ6ers6dOysgIEDff/+9EhIS1LdvX3l4eGjSpEm5fi4AACB/yveBqEiRIgoICMiyPiUlRf/5z3/06aefql27dpKkuXPnqk6dOvrhhx/UokULrVq1Snv27NHq1avl7++vRo0a6ZVXXtGoUaM0fvx4OZ3O3D4dAACQD+XrW2aStG/fPgUFBalq1arq3bu34uPjJUlbt27VhQsXFB4ebvWtXbu2KlWqpJiYGElSTEyM6tevL39/f6tPRESEUlNTtXv37tw9EQAAkG/l6ytEzZs317x581SrVi0lJCRowoQJat26tXbt2qXExEQ5nU75+vq6bOPv76/ExERJUmJioksYymzPbLuWc+fO6dy5c9Zyamqqm84IEo8pAwDyn3wdiDp27Gj9uUGDBmrevLmCg4P1xRdfyMvLK8eOO3nyZE2YMCHH9g8AAPKXfH/L7HK+vr6qWbOm9u/fr4CAAJ0/f17JyckufZKSkqw5RwEBAVmeOstcvtq8pExjxoxRSkqK9frtt9/ceyIAACBfKVCBKC0tTQcOHFBgYKCaNGkiDw8PrVmzxmqPi4tTfHy8QkNDJUmhoaHauXOnjh49avWJioqSt7e36tate83jeHp6ytvb2+UFAADuXPn6ltnzzz+vLl26KDg4WEeOHNG4ceNUuHBh9erVSz4+PoqMjNTw4cNVunRpeXt765lnnlFoaKhatGghSWrfvr3q1q2rRx99VFOmTFFiYqJefvllDR48WJ6ennl8dgAAIL/I14Ho999/V69evXT8+HGVK1dOrVq10g8//KBy5cpJkt544w0VKlRI3bt317lz5xQREaF33nnH2r5w4cJatmyZBg0apNDQUBUvXlz9+vXTxIkT8+qUAABAPuQwxpi8LiK/S01NlY+Pj1JSUrh95gY8ZQbcvkOvdc7rEoB871Z+fxeoOUQAAAA5gUAEAABsj0AEAABsj0AEAABsj0AEAABsL18/dg8AuLrbfVqTp9QAV1whAgAAtkcgAgAAtkcgAgAAtkcgAgAAtkcgAgAAtkcgAgAAtkcgAgAAtkcgAgAAtkcgAgAAtkcgAgAAtkcgAgAAtkcgAgAAtkcgAgAAtkcgAgAAtkcgAgAAtkcgAgAAtkcgAgAAtkcgAgAAtkcgAgAAtkcgAgAAtkcgAgAAtkcgAgAAtkcgAgAAtkcgAgAAtkcgAgAAtkcgAgAAtkcgAgAAtkcgAgAAtkcgAgAAtkcgAgAAtkcgAgAAtkcgAgAAtkcgAgAAtkcgAgAAtkcgAgAAtkcgAgAAtkcgAgAAtkcgAgAAtkcgAgAAtkcgAgAAtkcgAgAAtkcgAgAAtkcgAgAAtlckrwtAwVR59PK8LgEAALfhChEAALA9AhEAALA9AhEAALA9AhEAALA9AhEAALA9AhEAALA9AhEAALA9AhEAALA9AhEAALA9AhEAALA9AhEAALA9AhEAALA9vtwVAGzodr6g+dBrnd1YCZA/cIUIAADYHoEIAADYHoEIAADYHoEIAADYHoEIAADYHoEIAADYnq0eu3/77bc1depUJSYmqmHDhpo9e7b+9re/5XVZeeZ2HrsFAOBOYptAtHDhQg0fPlxz5sxR8+bNNXPmTEVERCguLk5+fn55XR4AFBh8hhHuRA5jjMnrInJD8+bN1axZM7311luSpIyMDFWsWFHPPPOMRo8efd1tU1NT5ePjo5SUFHl7e+dGubmCK0QAChLCFG7Vrfz+tsUVovPnz2vr1q0aM2aMta5QoUIKDw9XTExMHlZ2+wg1AADcPlsEoj///FPp6eny9/d3We/v76+ff/45S/9z587p3Llz1nJKSoqkS0kzJ4SM+zZH9gsAd5JKzy3K6xJu2a4JEXly3Nv5vXI7NefVca8l8/f2zdwMs0UgulWTJ0/WhAkTsqyvWLFiHlQDACiofGbmdQW3Lq9qzsnjnjp1Sj4+PtftY4tAVLZsWRUuXFhJSUku65OSkhQQEJCl/5gxYzR8+HBrOSMjQydOnFCZMmXkcDhyvN7blZqaqooVK+q33367o+Y85TXGNWcwrjmDcc0ZjGvOyKlxNcbo1KlTCgoKumFfWwQip9OpJk2aaM2aNXrggQckXQo5a9as0ZAhQ7L09/T0lKenp8s6X1/fXKjUvby9vfmBzQGMa85gXHMG45ozGNeckRPjeqMrQ5lsEYgkafjw4erXr5+aNm2qv/3tb5o5c6ZOnz6tAQMG5HVpAAAgj9kmEPXo0UPHjh3T2LFjlZiYqEaNGmnlypVZJloDAAD7sU0gkqQhQ4Zc9RbZncbT01Pjxo3LctsPt4dxzRmMa85gXHMG45oz8sO42uaDGQEAAK6FL3cFAAC2RyACAAC2RyACAAC2RyACAAC2RyAqIMaPHy+Hw+Hyql27ttV+9uxZDR48WGXKlFGJEiXUvXv3LJ/MHR8fr86dO6tYsWLy8/PTyJEjdfHixdw+lTy1YcMGdenSRUFBQXI4HFq8eLFLuzFGY8eOVWBgoLy8vBQeHq59+/a59Dlx4oR69+4tb29v+fr6KjIyUmlpaS59duzYodatW6to0aKqWLGipkyZktOnlqduNK79+/fP8v7t0KGDSx/G1dXkyZPVrFkzlSxZUn5+fnrggQcUFxfn0sddP/fr1q3TXXfdJU9PT1WvXl3z5s3L6dPLMzczrnfffXeW9+tTTz3l0odxdfXuu++qQYMG1gcrhoaGasWKFVZ7gXivGhQI48aNM/Xq1TMJCQnW69ixY1b7U089ZSpWrGjWrFljtmzZYlq0aGH+/ve/W+0XL140ISEhJjw83Gzbts188803pmzZsmbMmDF5cTp55ptvvjEvvfSS+eqrr4wk8/XXX7u0v/baa8bHx8csXrzYbN++3dx///2mSpUq5q+//rL6dOjQwTRs2ND88MMPZuPGjaZ69eqmV69eVntKSorx9/c3vXv3Nrt27TKfffaZ8fLyMu+9915unWauu9G49uvXz3To0MHl/XvixAmXPoyrq4iICDN37lyza9cuExsbazp16mQqVapk0tLSrD7u+Ln/9ddfTbFixczw4cPNnj17zOzZs03hwoXNypUrc/V8c8vNjGvbtm3NE0884fJ+TUlJsdoZ16yWLl1qli9fbn755RcTFxdnXnzxRePh4WF27dpljCkY71UCUQExbtw407Bhw6u2JScnGw8PD7No0SJr3d69e40kExMTY4y59AurUKFCJjEx0erz7rvvGm9vb3Pu3LkcrT2/uvIXd0ZGhgkICDBTp0611iUnJxtPT0/z2WefGWOM2bNnj5FkfvrpJ6vPihUrjMPhMH/88Ycxxph33nnHlCpVymVcR40aZWrVqpXDZ5Q/XCsQde3a9ZrbMK43dvToUSPJrF+/3hjjvp/7F154wdSrV8/lWD169DARERE5fUr5wpXjasylQDR06NBrbsO43pxSpUqZDz74oMC8V7llVoDs27dPQUFBqlq1qnr37q34+HhJ0tatW3XhwgWFh4dbfWvXrq1KlSopJiZGkhQTE6P69eu7fDJ3RESEUlNTtXv37tw9kXzq4MGDSkxMdBlHHx8fNW/e3GUcfX191bRpU6tPeHi4ChUqpM2bN1t92rRpI6fTafWJiIhQXFycTp48mUtnk/+sW7dOfn5+qlWrlgYNGqTjx49bbYzrjaWkpEiSSpcuLcl9P/cxMTEu+8jsk7mPO92V45ppwYIFKlu2rEJCQjRmzBidOXPGamNcry89PV2ff/65Tp8+rdDQ0ALzXrXVJ1UXZM2bN9e8efNUq1YtJSQkaMKECWrdurV27dqlxMREOZ3OLF9A6+/vr8TERElSYmJilq8pyVzO7GN3meNwtXG6fBz9/Pxc2osUKaLSpUu79KlSpUqWfWS2lSpVKkfqz886dOigbt26qUqVKjpw4IBefPFFdezYUTExMSpcuDDjegMZGRkaNmyYWrZsqZCQEEly28/9tfqkpqbqr7/+kpeXV06cUr5wtXGVpEceeUTBwcEKCgrSjh07NGrUKMXFxemrr76SxLhey86dOxUaGqqzZ8+qRIkS+vrrr1W3bl3FxsYWiPcqgaiA6Nixo/XnBg0aqHnz5goODtYXX3xxR/5g4c7Ss2dP68/169dXgwYNVK1aNa1bt05hYWF5WFnBMHjwYO3atUvfffddXpdyR7nWuA4cOND6c/369RUYGKiwsDAdOHBA1apVy+0yC4xatWopNjZWKSkp+vLLL9WvXz+tX78+r8u6adwyK6B8fX1Vs2ZN7d+/XwEBATp//rySk5Nd+iQlJSkgIECSFBAQkGVGf+ZyZh+7yxyHq43T5eN49OhRl/aLFy/qxIkTjPUtqFq1qsqWLav9+/dLYlyvZ8iQIVq2bJnWrl2rChUqWOvd9XN/rT7e3t539H+2rjWuV9O8eXNJcnm/Mq5ZOZ1OVa9eXU2aNNHkyZPVsGFDzZo1q8C8VwlEBVRaWpoOHDigwMBANWnSRB4eHlqzZo3VHhcXp/j4eIWGhkqSQkNDtXPnTpdfOlFRUfL29lbdunVzvf78qEqVKgoICHAZx9TUVG3evNllHJOTk7V161arT3R0tDIyMqx/NENDQ7VhwwZduHDB6hMVFaVatWrd0bd1bsXvv/+u48ePKzAwUBLjejXGGA0ZMkRff/21oqOjs9wudNfPfWhoqMs+Mvtk7uNOc6NxvZrY2FhJcnm/Mq43lpGRoXPnzhWc96pbpmYjx40YMcKsW7fOHDx40GzatMmEh4ebsmXLmqNHjxpjLj3SWKlSJRMdHW22bNliQkNDTWhoqLV95iON7du3N7GxsWblypWmXLlytnvs/tSpU2bbtm1m27ZtRpKZMWOG2bZtmzl8+LAx5tJj976+vmbJkiVmx44dpmvXrld97L5x48Zm8+bN5rvvvjM1atRweTw8OTnZ+Pv7m0cffdTs2rXLfP7556ZYsWJ37OPhxlx/XE+dOmWef/55ExMTYw4ePGhWr15t7rrrLlOjRg1z9uxZax+Mq6tBgwYZHx8fs27dOpfHv8+cOWP1ccfPfeajzCNHjjR79+41b7/99h39ePiNxnX//v1m4sSJZsuWLebgwYNmyZIlpmrVqqZNmzbWPhjXrEaPHm3Wr19vDh48aHbs2GFGjx5tHA6HWbVqlTGmYLxXCUQFRI8ePUxgYKBxOp2mfPnypkePHmb//v1W+19//WWefvppU6pUKVOsWDHz4IMPmoSEBJd9HDp0yHTs2NF4eXmZsmXLmhEjRpgLFy7k9qnkqbVr1xpJWV79+vUzxlx69P6f//yn8ff3N56eniYsLMzExcW57OP48eOmV69epkSJEsbb29sMGDDAnDp1yqXP9u3bTatWrYynp6cpX768ee2113LrFPPE9cb1zJkzpn379qZcuXLGw8PDBAcHmyeeeMLl8VpjGNcrXW08JZm5c+dafdz1c7927VrTqFEj43Q6TdWqVV2Ocae50bjGx8ebNm3amNKlSxtPT09TvXp1M3LkSJfPITKGcb3SY489ZoKDg43T6TTlypUzYWFhVhgypmC8Vx3GGOOea00AAAAFE3OIAACA7RGIAACA7RGIAACA7RGIAACA7RGIAACA7RGIAACA7RGIAACA7RGIAOQbhw4dksPhsL4qAdLdd9+tYcOG5XUZwB2PQATArRwOx3Vf48ePz+sSs8gPoWPdunVyOBxZvgATQO4oktcFALizJCQkWH9euHChxo4dq7i4OGtdiRIl8qIsALgurhABcKuAgADr5ePjI4fDYS37+flpxowZqlChgjw9PdWoUSOtXLnymvtKT0/XY489ptq1ays+Pl6StGTJEt11110qWrSoqlatqgkTJujixYvWNg6HQx988IEefPBBFStWTDVq1NDSpUtv65y+++47tW7dWl5eXqpYsaKeffZZnT592mqvXLmyJk2apMcee0wlS5ZUpUqV9P7777vs4/vvv1ejRo1UtGhRNW3aVIsXL7ZuDx46dEj33HOPJKlUqVJyOBzq37+/tW1GRoZeeOEFlS5dWgEBAfnyKhtQ0BGIAOSaWbNmafr06Zo2bZp27NihiIgI3X///dq3b1+WvufOndM//vEPxcbGauPGjapUqZI2btyovn37aujQodqzZ4/ee+89zZs3T//6179ctp0wYYIefvhh7dixQ506dVLv3r114sSJbNV84MABdejQQd27d9eOHTu0cOFCfffddxoyZIhLv+nTp6tp06batm2bnn76aQ0aNMi6MpaamqouXbqofv36+t///qdXXnlFo0aNsratWLGi/vvf/0qS4uLilJCQoFmzZlntH330kYoXL67NmzdrypQpmjhxoqKiorJ1PgCuwW1fEwsAV5g7d67x8fGxloOCgsy//vUvlz7NmjUzTz/9tDHGmIMHDxpJZuPGjSYsLMy0atXKJCcnW33DwsLMpEmTXLb/+OOPTWBgoLUsybz88svWclpampFkVqxYcc0627Zta4YOHXrVtsjISDNw4ECXdRs3bjSFChUyf/31lzHGmODgYNOnTx+rPSMjw/j5+Zl3333XGGPMu+++a8qUKWP1N8aYf//730aS2bZtmzHm0rd4SzInT57MUlurVq1c1jVr1syMGjXqmucD4NYxhwhArkhNTdWRI0fUsmVLl/UtW7bU9u3bXdb16tVLFSpUUHR0tLy8vKz127dv16ZNm1yuCKWnp+vs2bM6c+aMihUrJklq0KCB1V68eHF5e3vr6NGj2ap7+/bt2rFjhxYsWGCtM8YoIyNDBw8eVJ06dbIcM/M2YeYx4+Li1KBBAxUtWtTq87e//e2ma7h835IUGBiY7fMBcHUEIgD5TqdOnfTJJ58oJiZG7dq1s9anpaVpwoQJ6tatW5ZtLg8bHh4eLm0Oh0MZGRnZqiUtLU1PPvmknn322SxtlSpVypFjXikn9w3gEgIRgFzh7e2toKAgbdq0SW3btrXWb9q0KcvVkkGDBikkJET333+/li9fbvW/6667FBcXp+rVq+da3XfddZf27NlzW8esVauWPvnkE507d06enp6SpJ9++smlj9PplHTpiheA3EcgApBrRo4cqXHjxqlatWpq1KiR5s6dq9jYWJfbUZmeeeYZpaen67777tOKFSvUqlUrjR07Vvfdd58qVaqkhx56SIUKFdL27du1a9cuvfrqq7dV27Fjx7J8IGRgYKBGjRqlFi1aaMiQIXr88cdVvHhx7dmzR1FRUXrrrbduat+PPPKIXnrpJQ0cOFCjR49WfHy8pk2bJunS1R5JCg4OlsPh0LJly9SpUyd5eXnxEQVALuIpMwC55tlnn9Xw4cM1YsQI1a9fXytXrtTSpUtVo0aNq/YfNmyYJkyYoE6dOun7779XRESEli1bplWrVqlZs2Zq0aKF3njjDQUHB992bZ9++qkaN27s8vr3v/+tBg0aaP369frll1/UunVrNW7cWGPHjlVQUNBN79vb21v/93//p9jYWDVq1EgvvfSSxo4dK+n/3+orX768JkyYoNGjR8vf3z/LU2wAcpbDGGPyuggAsJsFCxZowIABSklJcZk4DiBvcMsMAHLB/PnzVbVqVZUvX17bt2/XqFGj9PDDDxOGgHyCQAQAuSAxMVFjx45VYmKiAgMD9Y9//CPLB0oCyDvcMgMAALbHpGoAAGB7BCIAAGB7BCIAAGB7BCIAAGB7BCIAAGB7BCIAAGB7BCIAAGB7BCIAAGB7BCIAAGB7/w/dzxztRTvFMwAAAABJRU5ErkJggg==",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "\n",
    "token_lengths = [len(i) for i in reasoning_data['train']['truncated_token_id']]\n",
    "\n",
    "plt.hist(token_lengths, bins=30)\n",
    "plt.xlabel('Token Length')\n",
    "plt.ylabel('Frequency')\n",
    "plt.title('Distribution of Token Lengths')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "8xunbQypIhWV",
    "outputId": "234a930c-4099-4607-eab9-3d7cf8a7b5ad"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Dataset({\n",
       "    features: ['user', 'reasoning', 'assistant', 'template', 'conversations', 'user_template', 'reasoning_template', 'assistant_template', 'template_new', 'truncated_token_id'],\n",
       "    num_rows: 19944\n",
       "})"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "reasoning_data['train']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {
    "id": "tLGyqg_tTA7J"
   },
   "outputs": [],
   "source": [
    "assert tokenizer.decode(torch.tensor(reasoning_data['train'][0]['truncated_token_id'])) == reasoning_data['train'][0]['template_new']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "SJWF2tqkI3ep"
   },
   "source": [
    "## 3.2、使用设置的训练超参数"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {
    "id": "aco_71m1HxAR"
   },
   "outputs": [],
   "source": [
    "batch_size = modelConfig.batch_size\n",
    "gradient_accumulation_steps = modelConfig.gradient_accumulation_steps\n",
    "log_iter = modelConfig.log_iter\n",
    "max_lr = modelConfig.max_lr\n",
    "min_lr = modelConfig.min_lr\n",
    "warmup_steps = modelConfig.warmup_steps\n",
    "total_steps = len(reasoning_data['train'])//batch_size\n",
    "optimizer = torch.optim.AdamW(filter(lambda p:p.requires_grad, model.parameters()), lr=max_lr)\n",
    "trainable_parameters_num = sum(p.numel() for p in filter(lambda p:p.requires_grad, model.parameters()))  ##全参微调"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "TVGFoXjBJV_P"
   },
   "source": [
    "## 3.3、配置logging日志记录模型训练过程"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {
    "id": "iFId53UlJTmw"
   },
   "outputs": [],
   "source": [
    "##配置logging\n",
    "import time\n",
    "with open(f\"./{model_name}-SFT_log.txt\", \"a\") as my_file:\n",
    "  my_file.write(f'time:{time.strftime(\"%Y-%m-%d, %H:%M:%S\")}, batch_size:{batch_size}, trainable_parameters_num:{trainable_parameters_num}, warmup_steps:{warmup_steps}, max_lr:{max_lr}, min_lr:{min_lr}, gradient_accumulation_steps:{gradient_accumulation_steps}\\n')\n",
    "#定义一个日志记录函数\n",
    "def log_call(iters, iters_average_loss):\n",
    "  with open(f\"./{model_name}-SFT_log.txt\", \"a\") as my_file:\n",
    "    my_file.write(f'time:{time.strftime(\"%Y-%m-%d, %H:%M:%S\")}, iters:{iters+1}, iters_average_Loss:{iters_average_loss:.4f}\\n')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "88hgvlChJ65t"
   },
   "source": [
    "## 3.4、学习率设置：余弦衰减学习率"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {
    "id": "Lf02v3AcJy6G"
   },
   "outputs": [],
   "source": [
    "def linear_warmup(current_step, warmup_steps, max_lr):\n",
    "    if current_step < warmup_steps:\n",
    "        return max_lr * current_step / warmup_steps\n",
    "    else:\n",
    "        return max_lr\n",
    "\n",
    "def cosine_decay(current_step, warmup_steps, total_steps, max_lr, min_lr):\n",
    "    if current_step < warmup_steps:\n",
    "        return linear_warmup(current_step, warmup_steps, max_lr)\n",
    "    else:\n",
    "        progress = (current_step - warmup_steps) / (total_steps - warmup_steps)\n",
    "        decay = 0.5 * (1 + np.cos(np.pi * progress))\n",
    "        return (max_lr - min_lr) * decay + min_lr"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "zRTA4eZCKDhU"
   },
   "source": [
    "## 3.5、掩码设置\n",
    "- SFT和预训练的区别核心就是掩码掉“问题”部分的损失，而【只看“回答”部分的损失】，并仅基于回答部分的损失进行优化\n",
    "- 实现方式：【构造损失掩码】，仅针对每轮对话（含多轮）的模型“输出”部分（也就是回答部分，**本处默认包含reasoning部分**）进行损失计算"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {
    "id": "Iazl9HMfKBTt"
   },
   "outputs": [],
   "source": [
    "# 设置问题部分的掩码函数，用于执行仅针对回答部分（此处默认包含reasoning部分）才计算损失\n",
    "def return_answer_mask(input_ids):\n",
    "  assistant_answer_mask = torch.zeros_like(input_ids) #0初始化\n",
    "  for i in range(input_ids.shape[0]):\n",
    "        ## user部分的结尾\\n: \\n是<|im_end|>的下一个元素，所以有+1 【这个地方需要根据不同模型的不同聊天模版自定义更改】，关于聊天模版可阅读这篇文章：https://huggingface.co/blog/chat-templates\n",
    "        i_user_end_list = [i+1 for i in torch.where(input_ids[i]==tokenizer.encode('<|im_end|>')[0])[0].tolist()[1::3]]   #第1个im_end开始\n",
    "        ## assistant部分的结尾\\n：\\n是<|im_end|>的下一个元素，所以有+1 【这个地方需要根据不同模型的不同聊天模版自定义更改】\n",
    "        i_assistant_end_list = [i+1 for i in torch.where(input_ids[i]==tokenizer.encode('<|im_end|>')[0])[0].tolist()[3::3]] #第3个im_end开始\n",
    "\n",
    "        if len(i_user_end_list)==len(i_assistant_end_list):\n",
    "            for user_end, assistant_end in zip(i_user_end_list, i_assistant_end_list):\n",
    "                assistant_answer_mask[i][user_end+3:assistant_end-1]=1 #+3的操作，【这个地方需要根据不同模型的不同聊天模版自定义更改】\n",
    "        elif len(i_user_end_list)==len(i_assistant_end_list)+1==1:  ##单轮问答,且回答部分未结尾就被截断了\n",
    "            assistant_answer_mask[i][i_user_end_list[0]+3:]=1  ##会把右补的padding token也标记为1，所以后面还需要再结合padding mask以过滤padding\n",
    "        elif len(i_user_end_list)==len(i_assistant_end_list)+1:   ##兼顾多轮问答\n",
    "            assistant_answer_mask[i][i_user_end_list[-1]+3:]=1\n",
    "            for user_end, assistant_end in zip(i_user_end_list[:-1], i_assistant_end_list):\n",
    "                assistant_answer_mask[i][user_end+3:assistant_end-1]=1\n",
    "        else:\n",
    "            continue  ##跳出当前循环，继续下一次循环\n",
    "  return assistant_answer_mask"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "iCUNOzJdN0Xg"
   },
   "source": [
    "## 3.6、开启SFT微调训练"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "jHtm3OmCNv9c",
    "outputId": "aca586db-5169-4794-fb06-05a3d6ef49ea"
   },
   "outputs": [],
   "source": [
    "model.train()\n",
    "train_loss_list = []\n",
    "model.zero_grad() ##clear gradients at the start of training\n",
    "ignore_iters_count = 0\n",
    "for iters in range(len(reasoning_data['train'])//batch_size):\n",
    "    ## 获取批次数据\n",
    "    chosen_batch_inputids = reasoning_data['train'][iters*batch_size:(iters+1)*batch_size]['truncated_token_id']\n",
    "\n",
    "    ## 对该批次数据进行padding,以并行计算，首先计算该批次的最大token长度\n",
    "    chosen_max_dim = max([len(i) for i in chosen_batch_inputids])\n",
    "\n",
    "    ### 训练数据padding填充\n",
    "    chosen_batch_inputids_padding_list = []\n",
    "    for i in range(batch_size):\n",
    "        chosen_batch_inputids_padding_list.append(torch.nn.functional.pad(torch.tensor(chosen_batch_inputids[i]), (0, chosen_max_dim - len(chosen_batch_inputids[i])), mode='constant', value=model.generation_config.eos_token_id[-1]).tolist()) #右补\n",
    "    chosen_batch_inputids_tensor = torch.tensor(chosen_batch_inputids_padding_list)\n",
    "\n",
    "    ## 构建训练数据：x->y ,下一个单词预测\n",
    "    chosen_x = chosen_batch_inputids_tensor[:, :-1].to(device)\n",
    "    chosen_y = chosen_batch_inputids_tensor[:, 1:].to(device)\n",
    "\n",
    "    ## 构建掩码判别矩阵（paddding mask & answer_mask, padding mask用于执行对padding的token不计算损失，answer_mask用于执行仅针对回答部分才计算损失），总之，就是确认哪些tokens的logit需要\"忽视\"掉\n",
    "    ### 【padding mask】\n",
    "    chosen_padding_mask = torch.where(chosen_y==model.generation_config.eos_token_id[-1], 0, 1)\n",
    "    ### 【answer_mask】\n",
    "    chosen_assistant_answer_mask = return_answer_mask(chosen_x)\n",
    "    ### 【paddingmask & answermask】方便使用掩码判别矩阵对logit和y进行过滤->:我们只关注【回答】部分的损失，不关注问题部分的损失\n",
    "    chosen_assistant_answer_mask = (chosen_assistant_answer_mask&chosen_padding_mask)\n",
    "\n",
    "    if chosen_assistant_answer_mask.sum(dim=-1).min().item()==0:##如果该批次里有的问答数据在数据截取时，回答部分存在没有数据的情况（问题太长了，导致还未采集到回答部分的token就被硬截断了），那么该批次数据不再训练\n",
    "      #print(f'不处理第{iters+1}批次数据')\n",
    "      ignore_iters_count+=1\n",
    "      continue  #跳出当前循环\n",
    "\n",
    "    ## 执行训练数据的模型前向推理，计算logits\n",
    "    chosen_logits = model(chosen_x).logits\n",
    "    torch.cuda.empty_cache() #清除非必要的显存占用，但会导致速度变慢\n",
    "    torch.cuda.ipc_collect()\n",
    "\n",
    "    ## Compute Chosen_Answer_Loss，计算训练数据的回答部分的损失, batch_loss的shape_size:[batch_size]\n",
    "    batch_loss = torch.mul((torch.gather(torch.log(torch.softmax(chosen_logits, dim=-1)), dim=-1, index=chosen_y.unsqueeze(2))*(-1)).squeeze(2), chosen_assistant_answer_mask).sum(dim=-1) / chosen_assistant_answer_mask.sum(dim=-1)\n",
    "\n",
    "    ## Calculate the Final Loss, 只是新增了梯度积累的操作\n",
    "    loss = torch.nanmean(batch_loss)/(gradient_accumulation_steps)\n",
    "\n",
    "    loss.backward() ##反向传播计算梯度\n",
    "\n",
    "    # Compute the learning rate for the current step\n",
    "    lr = cosine_decay(iters, warmup_steps, total_steps, max_lr, min_lr)\n",
    "\n",
    "    # Update the learning rate for the AdamW optimizer\n",
    "    for param_group in optimizer.param_groups:\n",
    "        param_group['lr'] = lr\n",
    "\n",
    "    if (iters+1)%gradient_accumulation_steps==0 or (iters+1)==(len(reasoning_data['train'])//batch_size):\n",
    "        optimizer.step() #update weights after gradients accumulation\n",
    "        ##at last, clear gradients\n",
    "        optimizer.zero_grad() #clear gradients after updating, in this case equal to model.zero_grad()\n",
    "\n",
    "    train_loss_list.append(loss.item()*gradient_accumulation_steps)\n",
    "\n",
    "    if (iters+1)%log_iter==0 or (iters+1)==(len(reasoning_data['train'])//batch_size):\n",
    "      print(f'time:{time.strftime(\"%Y-%m-%d, %H:%M:%S\")}, iters:{iters+1}, last_{log_iter}_iters_average_train_Loss:{np.nanmean(train_loss_list[-log_iter:]):.4f}') ##避免空值影响\n",
    "      log_call(iters, np.nanmean(train_loss_list[-log_iter:]))\n",
    "    #过程中，保存模型\n",
    "    if ((iters+1)*(modelConfig.batch_size))%2000==0:\n",
    "        torch.save(model.state_dict(),f\"./{model_name}_SFT.pth\")\n",
    "\n",
    "print(\"Totally Completed!\")\n",
    "print(f'共计忽略{ignore_iters_count}个批次数据')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 472
    },
    "id": "AyczREciH03f",
    "outputId": "5a0af073-7741-4d45-f787-0ac3993223a6"
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAk0AAAHHCAYAAACiOWx7AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAA9hAAAPYQGoP6dpAAB750lEQVR4nO3deXhTZfo38G+WJum+r1Aoe1kLFKhFEZDK4gIIM4KDIujgio6iM8r8RhB0BtQZXkUZcUPQ0QH3BUcUiqBgZStlKfvSBbrve7Od94/knDZ0S9OkSdrv57pyCck5yZNDbe7cz/3cj0wQBAFERERE1Cq5swdARERE5A4YNBERERFZgUETERERkRUYNBERERFZgUETERERkRUYNBERERFZgUETERERkRUYNBERERFZgUETERERkRUYNBGRW1i0aBFiYmJsOvf555+HTCaz74CIqNth0EREHSKTyay67dmzx9lDdYpFixbBx8fH2cMgIjuQce85IuqI//znPxZ//+CDD7Bz5058+OGHFvfffPPNCA8Pt/l1dDodjEYj1Gp1u8/V6/XQ6/XQaDQ2v76tFi1ahM8++wxVVVWd/tpEZF9KZw+AiNzb3XffbfH33377DTt37mxy/7Vqamrg5eVl9et4eHjYND4AUCqVUCr5646IOobTc0TkcJMmTcKwYcNw5MgR3HjjjfDy8sJf//pXAMDXX3+NW2+9FVFRUVCr1ejXrx9eeOEFGAwGi+e4tqYpIyMDMpkM//znP/H222+jX79+UKvVGDt2LA4dOmRxbnM1TTKZDEuXLsVXX32FYcOGQa1WY+jQodixY0eT8e/ZswdjxoyBRqNBv3798NZbb9m9TurTTz9FfHw8PD09ERISgrvvvhtXr161OCYvLw+LFy9Gz549oVarERkZiVmzZiEjI0M65vDhw5g2bRpCQkLg6emJPn364L777rPbOIm6M371IqJOUVxcjBkzZmD+/Pm4++67pam6zZs3w8fHB8uWLYOPjw92796NFStWoKKiAq+88kqbz/vxxx+jsrISDz74IGQyGV5++WXMmTMHly5dajM7tW/fPnzxxRd45JFH4Ovri/Xr12Pu3LnIyspCcHAwAODo0aOYPn06IiMjsWrVKhgMBqxevRqhoaEdvyhmmzdvxuLFizF27FisWbMG+fn5eO2117B//34cPXoUAQEBAIC5c+ciPT0djz32GGJiYlBQUICdO3ciKytL+vvUqVMRGhqKZ599FgEBAcjIyMAXX3xht7ESdWsCEZEdPfroo8K1v1omTpwoABA2btzY5Piampom9z344IOCl5eXUFdXJ9137733Cr1795b+fvnyZQGAEBwcLJSUlEj3f/311wIA4dtvv5XuW7lyZZMxARBUKpVw4cIF6b5jx44JAITXX39duu/2228XvLy8hKtXr0r3nT9/XlAqlU2eszn33nuv4O3t3eLjWq1WCAsLE4YNGybU1tZK92/fvl0AIKxYsUIQBEEoLS0VAAivvPJKi8/15ZdfCgCEQ4cOtTkuImo/Ts8RUadQq9VYvHhxk/s9PT2lP1dWVqKoqAgTJkxATU0Nzpw50+bzzps3D4GBgdLfJ0yYAAC4dOlSm+cmJSWhX79+0t9HjBgBPz8/6VyDwYBdu3Zh9uzZiIqKko7r378/ZsyY0ebzW+Pw4cMoKCjAI488YlGofuuttyI2NhbfffcdANN1UqlU2LNnD0pLS5t9LjEjtX37duh0OruMj4gaMGgiok7Ro0cPqFSqJvenp6fjjjvugL+/P/z8/BAaGioVkZeXl7f5vL169bL4uxhAtRRYtHaueL54bkFBAWpra9G/f/8mxzV3ny0yMzMBAIMGDWryWGxsrPS4Wq3GSy+9hO+//x7h4eG48cYb8fLLLyMvL086fuLEiZg7dy5WrVqFkJAQzJo1C++//z7q6+vtMlai7o5BExF1isYZJVFZWRkmTpyIY8eOYfXq1fj222+xc+dOvPTSSwAAo9HY5vMqFIpm7xes6KbSkXOd4YknnsC5c+ewZs0aaDQaPPfccxg8eDCOHj0KwFTc/tlnnyElJQVLly7F1atXcd999yE+Pp4tD4jsgEETETnNnj17UFxcjM2bN+NPf/oTbrvtNiQlJVlMtzlTWFgYNBoNLly40OSx5u6zRe/evQEAZ8+ebfLY2bNnpcdF/fr1w1NPPYUff/wRJ0+ehFarxb/+9S+LY6677jr8/e9/x+HDh/HRRx8hPT0dW7dutct4ibozBk1E5DRipqdxZker1eLf//63s4ZkQaFQICkpCV999RVycnKk+y9cuIDvv//eLq8xZswYhIWFYePGjRbTaN9//z1Onz6NW2+9FYCpr1VdXZ3Fuf369YOvr690XmlpaZMs2ciRIwGAU3REdsCWA0TkNOPHj0dgYCDuvfdePP7445DJZPjwww9danrs+eefx48//ojrr78eDz/8MAwGA9544w0MGzYMaWlpVj2HTqfDiy++2OT+oKAgPPLII3jppZewePFiTJw4EXfddZfUciAmJgZPPvkkAODcuXOYMmUK7rzzTgwZMgRKpRJffvkl8vPzMX/+fADAli1b8O9//xt33HEH+vXrh8rKSrzzzjvw8/PDLbfcYrdrQtRdMWgiIqcJDg7G9u3b8dRTT+Fvf/sbAgMDcffdd2PKlCmYNm2as4cHAIiPj8f333+Pp59+Gs899xyio6OxevVqnD592qrVfYApe/bcc881ub9fv3545JFHsGjRInh5eWHt2rV45pln4O3tjTvuuAMvvfSStCIuOjoad911F5KTk/Hhhx9CqVQiNjYWn3zyCebOnQvAVAh+8OBBbN26Ffn5+fD398e4cePw0UcfoU+fPna7JkTdFfeeIyKywezZs5Geno7z5887eyhE1ElY00RE1Iba2lqLv58/fx7/+9//MGnSJOcMiIicgpkmIqI2REZGYtGiRejbty8yMzPx5ptvor6+HkePHsWAAQOcPTwi6iSsaSIiasP06dPx3//+F3l5eVCr1UhMTMQ//vEPBkxE3QwzTURERERWYE0TERERkRUYNBERERFZgTVNNjIajcjJyYGvry9kMpmzh0NERERWEAQBlZWViIqKglzevtwRgyYb5eTkIDo62tnDICIiIhtkZ2ejZ8+e7TqHQZONfH19AZguup+fn5NHQ0RERNaoqKhAdHS09DneHgyabCROyfn5+TFoIiIicjO2lNawEJyIiIjICgyaiIiIiKzAoImIiIjICgyaiIiIiKzAoImIiIjICgyaiIiIiKzAoImIiIjICgyaiIiIiKzAoImIiIjICgyaiIiIiKzAoImIiIjICgyaiIiIiKzAoMnFlNfocKmwCnU6g7OHQkRERI0waHIx01/7GTf9ay/O5lU6eyhERETUCIMmFxPkrQIAlFRrnTwSIiIiaoxBk4sRg6ZiBk1EREQuhUGTi2nINNU7eSRERETUGIMmF9MQNOmcPBIiIiJqjEGTiwlmpomIiMglMWhyMUHeagAsBCciInI1DJpcTJC3BwAWghMREbkapwdNGzZsQExMDDQaDRISEnDw4MEWj928eTNkMpnFTaPRWByzaNGiJsdMnz7d4piSkhIsWLAAfn5+CAgIwP3334+qqiqHvL/2EjNNpQyaiIiIXIrSmS++bds2LFu2DBs3bkRCQgJeffVVTJs2DWfPnkVYWFiz5/j5+eHs2bPS32UyWZNjpk+fjvfff1/6u1qttnh8wYIFyM3Nxc6dO6HT6bB48WI88MAD+Pjjj+30zmzHlgNERESuyamZpnXr1mHJkiVYvHgxhgwZgo0bN8LLywubNm1q8RyZTIaIiAjpFh4e3uQYtVptcUxgYKD02OnTp7Fjxw68++67SEhIwA033IDXX38dW7duRU5OjkPeZ3uIheCVdXpo9UYnj4aIiIhETguatFotjhw5gqSkpIbByOVISkpCSkpKi+dVVVWhd+/eiI6OxqxZs5Cent7kmD179iAsLAyDBg3Cww8/jOLiYumxlJQUBAQEYMyYMdJ9SUlJkMvlOHDggJ3ene38PT0gNyfPSmuYbSIiInIVTguaioqKYDAYmmSKwsPDkZeX1+w5gwYNwqZNm/D111/jP//5D4xGI8aPH48rV65Ix0yfPh0ffPABkpOT8dJLL2Hv3r2YMWMGDAbTBrh5eXlNpv6USiWCgoJafF0AqK+vR0VFhcXNEeRyGQK9uJUKERGRq3FqTVN7JSYmIjExUfr7+PHjMXjwYLz11lt44YUXAADz58+XHh8+fDhGjBiBfv36Yc+ePZgyZYrNr71mzRqsWrXK9sG3Q5C3CsXVWgZNRERELsRpmaaQkBAoFArk5+db3J+fn4+IiAirnsPDwwOjRo3ChQsXWjymb9++CAkJkY6JiIhAQUGBxTF6vR4lJSWtvu7y5ctRXl4u3bKzs60aoy1YDE5EROR6nBY0qVQqxMfHIzk5WbrPaDQiOTnZIpvUGoPBgBMnTiAyMrLFY65cuYLi4mLpmMTERJSVleHIkSPSMbt374bRaERCQkKLz6NWq+Hn52dxc5RgH/P0XBW7ghMREbkKp66eW7ZsGd555x1s2bIFp0+fxsMPP4zq6mosXrwYALBw4UIsX75cOn716tX48ccfcenSJaSmpuLuu+9GZmYm/vjHPwIwFYn/+c9/xm+//YaMjAwkJydj1qxZ6N+/P6ZNmwYAGDx4MKZPn44lS5bg4MGD2L9/P5YuXYr58+cjKiqq8y9CM1jTRERE5HqcWtM0b948FBYWYsWKFcjLy8PIkSOxY8cOqTg8KysLcnlDXFdaWoolS5YgLy8PgYGBiI+Px6+//oohQ4YAABQKBY4fP44tW7agrKwMUVFRmDp1Kl544QWLXk0fffQRli5diilTpkAul2Pu3LlYv3595775Vkj7z3H1HBERkcuQCYIgOHsQ7qiiogL+/v4oLy+3+1Td5v2X8fy3p3DL8Aj8e0G8XZ+biIioO+vI57fTt1GhpoJ8TFmx4ipmmoiIiFwFgyYXFMSaJiIiIpfDoMkFiS0H2BGciIjIdTBockFiy4HSGh2MRpacERERuQIGTS5IbDlgMAoor9U5eTREREQEMGhySSqlHL5qUzcIdgUnIiJyDQyaXFSQD+uaiIiIXAmDJhcl7T/HtgNEREQugUGTi5K6gnN6joiIyCUwaHJRQVLQxE17iYiIXAGDJhcVKAVNXD1HRETkChg0uahgZpqIiIhcCoMmFxXkbd5/jjVNRERELoFBk4tiITgREZFrYdDkosSaplIGTURERC6BQZOLEjNNxdVaCAL3nyMiInI2Bk0uSmw5UK83okZrcPJoiIiIiEGTi/JSKaBWmv55WNdERETkfAyaXJRMJmvU4JJBExERkbMxaHJhDJqIiIhcB4MmFxbUqBiciIiInItBkwtjV3AiIiLXwaDJhXH/OSIiItfBoMmFMdNERETkOhg0uTBx/zkWghMRETkfgyYXxkJwIiIi18GgyYUF+3D/OSIiIlfBoMmFBXox00REROQqGDS5MLEQvLJOD63e6OTREBERdW8MmlyYv6cHFHIZAKC0htkmIiIiZ2LQ5MLkchkCvTwAAMVVDJqIiIiciUGTixPrmphpIiIici4GTS6ObQeIiIhcA4MmFye2HSipYldwIiIiZ2LQ5OKCpK1UmGkiIiJyJgZNLi7IXNNUwpomIiIip2LQ5OKYaSIiInINDJpcXJCPadNethwgIiJyLqcHTRs2bEBMTAw0Gg0SEhJw8ODBFo/dvHkzZDKZxU2j0UiP63Q6PPPMMxg+fDi8vb0RFRWFhQsXIicnx+J5YmJimjzP2rVrHfYeOyKYmSYiIiKX4NSgadu2bVi2bBlWrlyJ1NRUxMXFYdq0aSgoKGjxHD8/P+Tm5kq3zMxM6bGamhqkpqbiueeeQ2pqKr744gucPXsWM2fObPI8q1evtniexx57zCHvsaPYp4mIiMg1KJ354uvWrcOSJUuwePFiAMDGjRvx3XffYdOmTXj22WebPUcmkyEiIqLZx/z9/bFz506L+9544w2MGzcOWVlZ6NWrl3S/r69vi8/jSsSWA6U1OhiNAuTmbVWIiIioczkt06TVanHkyBEkJSU1DEYuR1JSElJSUlo8r6qqCr1790Z0dDRmzZqF9PT0Vl+nvLwcMpkMAQEBFvevXbsWwcHBGDVqFF555RXo9foOvR9HETNNBqOA8lqdk0dDRETUfTkt01RUVASDwYDw8HCL+8PDw3HmzJlmzxk0aBA2bdqEESNGoLy8HP/85z8xfvx4pKeno2fPnk2Or6urwzPPPIO77roLfn5+0v2PP/44Ro8ejaCgIPz6669Yvnw5cnNzsW7duhbHW19fj/r6hgaTFRUV7X3LNlEp5fDVKFFZp0dxtRaB5honIiIi6lxOnZ5rr8TERCQmJkp/Hz9+PAYPHoy33noLL7zwgsWxOp0Od955JwRBwJtvvmnx2LJly6Q/jxgxAiqVCg8++CDWrFkDtVrd7GuvWbMGq1atsuO7sV6QtwqVdXrWNRERETmR06bnQkJCoFAokJ+fb3F/fn6+1bVGHh4eGDVqFC5cuGBxvxgwZWZmYufOnRZZpuYkJCRAr9cjIyOjxWOWL1+O8vJy6ZadnW3VGO1B2n+ObQeIiIicxmlBk0qlQnx8PJKTk6X7jEYjkpOTLbJJrTEYDDhx4gQiIyOl+8SA6fz589i1axeCg4PbfJ60tDTI5XKEhYW1eIxarYafn5/FrbOw7QAREZHzOXV6btmyZbj33nsxZswYjBs3Dq+++iqqq6ul1XQLFy5Ejx49sGbNGgCmNgHXXXcd+vfvj7KyMrzyyivIzMzEH//4RwCmgOl3v/sdUlNTsX37dhgMBuTl5QEAgoKCoFKpkJKSggMHDmDy5Mnw9fVFSkoKnnzySdx9990IDAx0zoVoQ0NXcG7aS0RE5CxODZrmzZuHwsJCrFixAnl5eRg5ciR27NghFYdnZWVBLm9IhpWWlmLJkiXIy8tDYGAg4uPj8euvv2LIkCEAgKtXr+Kbb74BAIwcOdLitX766SdMmjQJarUaW7duxfPPP4/6+nr06dMHTz75pEWdk6sJ8jbVWZVUc/UcERGRs8gEQRCcPQh3VFFRAX9/f5SXlzt8qu7tny/iH/87g9kjo/Dq/FEOfS0iIqKurCOf307fRoXaJmaailnTRERE5DQMmtwAC8GJiIicj0GTGxALwUsZNBERETkNgyY3IPVpqtaCJWhERETOwaDJDYhBU73eiBqtwcmjISIi6p4YNLkBL5UCaqXpn4p1TURERM7BoMkNyGQyFoMTERE5GYMmNxHIoImIiMipGDS5icbF4ERERNT5GDS5iWDuP0dERORUDJrcBLuCExEROReDJjcR5O0BgA0uiYiInIVBk5sQM00sBCciInIOBk1ugoXgREREzsWgyU0E+7DlABERkTMxaHITQezTRERE5FQMmtxEkJcpaKqs00OrNzp5NERERN0PgyY34e/pAYVcBgAorWG2iYiIqLMxaHITcrkMgV6mtgPFVQyaiIiIOhuDJjci1jUx00RERNT5GDS5kUAvth0gIiJyFgZNbkRqO1DF/eeIiIg6G4MmN8K2A0RERM7DoMmNSFupsKaJiIio0zFociNB5tVzzDQRERF1PgZNbiTIx5RpYssBIiKizsegyY0Es6aJiIjIaRg0uRH2aSIiInIeBk1upCFo0sFoFJw8GiIiou6FQZMbEZtbGowCymt1Th4NERFR98KgyY2olHL4apQA2BWciIioszFocjPBrGsiIiJyCgZNbkasa2LbASIios7FoMnNcCsVIiIi52DQ5GYagiZu2ktERNSZGDS5GWn/uWquniMiIupMDJrcTDAzTURERE7BoMnNBIqF4KxpIiIi6lROD5o2bNiAmJgYaDQaJCQk4ODBgy0eu3nzZshkMoubRqOxOEYQBKxYsQKRkZHw9PREUlISzp8/b3FMSUkJFixYAD8/PwQEBOD+++9HVVWVQ96fvXH/OSIiIudwatC0bds2LFu2DCtXrkRqairi4uIwbdo0FBQUtHiOn58fcnNzpVtmZqbF4y+//DLWr1+PjRs34sCBA/D29sa0adNQV1cnHbNgwQKkp6dj586d2L59O37++Wc88MADDnuf9iRtpcKgiYiIqFM5NWhat24dlixZgsWLF2PIkCHYuHEjvLy8sGnTphbPkclkiIiIkG7h4eHSY4Ig4NVXX8Xf/vY3zJo1CyNGjMAHH3yAnJwcfPXVVwCA06dPY8eOHXj33XeRkJCAG264Aa+//jq2bt2KnJwcR7/lDgtqND0nCNx/joiIqLM4LWjSarU4cuQIkpKSGgYjlyMpKQkpKSktnldVVYXevXsjOjoas2bNQnp6uvTY5cuXkZeXZ/Gc/v7+SEhIkJ4zJSUFAQEBGDNmjHRMUlIS5HI5Dhw4YM+36BBi0FSvN6JGa3DyaIiIiLoPpwVNRUVFMBgMFpkiAAgPD0deXl6z5wwaNAibNm3C119/jf/85z8wGo0YP348rly5AgDSea09Z15eHsLCwiweVyqVCAoKavF1AaC+vh4VFRUWN2fwUimgVpr+2VjXRERE1HmcXgjeHomJiVi4cCFGjhyJiRMn4osvvkBoaCjeeusth7/2mjVr4O/vL92io6Md/prNkclkUjE4V9ARERF1HqcFTSEhIVAoFMjPz7e4Pz8/HxEREVY9h4eHB0aNGoULFy4AgHRea88ZERHRpNBcr9ejpKSk1dddvnw5ysvLpVt2drZVY3SEIB8WgxMREXU2pwVNKpUK8fHxSE5Olu4zGo1ITk5GYmKiVc9hMBhw4sQJREZGAgD69OmDiIgIi+esqKjAgQMHpOdMTExEWVkZjhw5Ih2ze/duGI1GJCQktPhaarUafn5+FjdnCfRipomIiKizKZ354suWLcO9996LMWPGYNy4cXj11VdRXV2NxYsXAwAWLlyIHj16YM2aNQCA1atX47rrrkP//v1RVlaGV155BZmZmfjjH/8IwDR19cQTT+DFF1/EgAED0KdPHzz33HOIiorC7NmzAQCDBw/G9OnTsWTJEmzcuBE6nQ5Lly7F/PnzERUV5ZTr0F7sCk5ERNT5nBo0zZs3D4WFhVixYgXy8vIwcuRI7NixQyrkzsrKglzekAwrLS3FkiVLkJeXh8DAQMTHx+PXX3/FkCFDpGP+8pe/oLq6Gg888ADKyspwww03YMeOHRZNMD/66CMsXboUU6ZMgVwux9y5c7F+/frOe+MdJO4/x0wTERFR55EJbPZjk4qKCvj7+6O8vLzTp+o2/HQBr/xwFr+P74lXfh/Xqa9NRETkzjry+e1Wq+fIJMSHNU1ERESdjUGTGwr1NU3PFVaypomIiKizMGhyQ6E+pvosBk1ERESdh0GTGxIzTUVV9TAaWZJGRETUGRg0uaFgc02T3iigrFbn5NEQERF1Dwya3JCHQi5t3MspOiIios7BoMlNhfqwGJyIiKgzMWhyU9IKuqo6J4+EiIioe2DQ5KbYdoCIiKhzMWhyUwyaiIiIOheDJjfFmiYiIqLOxaDJTTXUNDFoIiIi6gwMmtwUp+eIiIg6F4MmN8WgiYiIqHMxaHJTYk1TaY0OWr3RyaMhIiLq+hg0uSl/Tw94KGQAgOJqZpuIiIgcjUGTm5LLZQjhCjoiIqJOw6DJjbGuiYiIqPMwaHJj7NVERETUeRg0uTFmmoiIiDoPgyY3xgaXREREnYdBkxtjpomIiKjzMGhyY6xpIiIi6jwMmtwYp+eIiIg6D4MmNyYGTQUV9RAEwcmjISIi6toYNLkxsbllrc6Aaq3ByaMhIiLq2hg0uTFvtRLeKgUA1jURERE5GoMmN8cVdERERJ2DQZObY9BERETUORg0ubmGoKnOySMhIiLq2hg0uTmpVxPbDhARETkUgyY3x+k5IiKizsGgyc0xaCIiIuocDJrcHLuCExERdQ4GTW4u1EcDgJkmIiIiR2PQ5ObETFNRlRZGI7dSISIichQGTW4u2EcFADAYBZTWaJ08GiIioq6LQZOb81DIEeRtCpxY10REROQ4Tg+aNmzYgJiYGGg0GiQkJODgwYNWnbd161bIZDLMnj3b4n6ZTNbs7ZVXXpGOiYmJafL42rVr7fm2OpXUq4l1TURERA7j1KBp27ZtWLZsGVauXInU1FTExcVh2rRpKCgoaPW8jIwMPP3005gwYUKTx3Jzcy1umzZtgkwmw9y5cy2OW716tcVxjz32mF3fW2di2wEiIiLHsyloys7OxpUrV6S/Hzx4EE888QTefvvtdj3PunXrsGTJEixevBhDhgzBxo0b4eXlhU2bNrV4jsFgwIIFC7Bq1Sr07du3yeMREREWt6+//hqTJ09ucqyvr6/Fcd7e3u0auyth0EREROR4NgVNf/jDH/DTTz8BAPLy8nDzzTfj4MGD+L//+z+sXr3aqufQarU4cuQIkpKSGgYjlyMpKQkpKSktnrd69WqEhYXh/vvvb/M18vPz8d133zV77Nq1axEcHIxRo0bhlVdegV6vt2rcrohBExERkeMpbTnp5MmTGDduHADgk08+wbBhw7B//378+OOPeOihh7BixYo2n6OoqAgGgwHh4eEW94eHh+PMmTPNnrNv3z689957SEtLs2qcW7Zsga+vL+bMmWNx/+OPP47Ro0cjKCgIv/76K5YvX47c3FysW7euxeeqr69HfX1DUFJRUWHVGDoD958jIiJyPJuCJp1OB7Xa9EG9a9cuzJw5EwAQGxuL3Nxc+42ukcrKStxzzz145513EBISYtU5mzZtwoIFC6DRaCzuX7ZsmfTnESNGQKVS4cEHH8SaNWuk93WtNWvWYNWqVba/AQdipomIiMjxbJqeGzp0KDZu3IhffvkFO3fuxPTp0wEAOTk5CA4Otuo5QkJCoFAokJ+fb3F/fn4+IiIimhx/8eJFZGRk4Pbbb4dSqYRSqcQHH3yAb775BkqlEhcvXrQ4/pdffsHZs2fxxz/+sc2xJCQkQK/XIyMjo8Vjli9fjvLycumWnZ1t1fvsDAyaiIiIHM+moOmll17CW2+9hUmTJuGuu+5CXFwcAOCbb76Rpu3aolKpEB8fj+TkZOk+o9GI5ORkJCYmNjk+NjYWJ06cQFpamnSbOXMmJk+ejLS0NERHR1sc/9577yE+Pl4aW2vS0tIgl8sRFhbW4jFqtRp+fn4WN1fB/eeIiIgcz6bpuUmTJqGoqAgVFRUIDAyU7n/ggQfg5eVl9fMsW7YM9957L8aMGYNx48bh1VdfRXV1NRYvXgwAWLhwIXr06IE1a9ZAo9Fg2LBhFucHBAQAQJP7Kyoq8Omnn+Jf//pXk9dMSUnBgQMHMHnyZPj6+iIlJQVPPvkk7r77bov34k7EmqayGh3q9QaolQonj4iIiKjrsSloqq2thSAIUpCRmZmJL7/8EoMHD8a0adOsfp558+ahsLAQK1asQF5eHkaOHIkdO3ZIxeFZWVmQy9ufDNu6dSsEQcBdd93V5DG1Wo2tW7fi+eefR319Pfr06YMnn3zSos7J3fh7esBDIYPOIKC4SouoAE9nD4mIiKjLkQmC0O5dXqdOnYo5c+bgoYceQllZGWJjY+Hh4YGioiKsW7cODz/8sCPG6lIqKirg7++P8vJyl5iqS1yTjNzyOnz96PWIiw5w9nCIiIhcUkc+v22qaUpNTZW6cX/22WcIDw9HZmYmPvjgA6xfv96Wp6QOYjE4ERGRY9kUNNXU1MDX1xcA8OOPP2LOnDmQy+W47rrrkJmZadcBknXYq4mIiMixbAqa+vfvj6+++grZ2dn44YcfMHXqVABAQUGBS0xVdUfMNBERETmWTUHTihUr8PTTTyMmJgbjxo2TWgT8+OOPGDVqlF0HSNZh0ERERORYNq2e+93vfocbbrgBubm5Fn2QpkyZgjvuuMNugyPrMWgiIiJyLJuCJgCIiIhAREQErly5AgDo2bOn1Y0tyf5Y00RERORYNk3PGY1GrF69Gv7+/ujduzd69+6NgIAAvPDCCzAajfYeI1mBmSYiIiLHsinT9H//93947733sHbtWlx//fUAgH379uH5559HXV0d/v73v9t1kNS2xkGTIAiQyWROHhEREVHXYlPQtGXLFrz77ruYOXOmdN+IESPQo0cPPPLIIwyanCDEPD1XqzOgWmuAj9rmmVciIiJqhk3TcyUlJYiNjW1yf2xsLEpKSjo8KGo/b7US3irTnnOcoiMiIrI/m4KmuLg4vPHGG03uf+ONNzBixIgOD4psw7omIiIix7FpDufll1/Grbfeil27dkk9mlJSUpCdnY3//e9/dh0gWS/UV42M4hoGTURERA5gU6Zp4sSJOHfuHO644w6UlZWhrKwMc+bMQXp6Oj788EN7j5Gs1JBpqnPySIiIiLoem6uFo6KimhR8Hzt2DO+99x7efvvtDg+M2o+9moiIiBzHpkwTuSbWNBERETkOg6YuhEETERGR4zBo6kKkoInTc0RERHbXrpqmOXPmtPp4WVlZR8ZCHRTqowHATBMREZEjtCto8vf3b/PxhQsXdmhAZDsx01RUpYXRKEAu51YqRERE9tKuoOn999931DjIDoJ9VAAAg1FAaY0WwebVdERERNRxrGnqQjwUcgR5mwIn1jURERHZF4OmLkbq1cS6JiIiIrti0NTFsO0AERGRYzBo6mIYNBERETkGg6YuhkETERGRYzBo6mK4/xwREZFjMGjqYphpIiIicgwGTV0MgyYiIiLHYNDUxXD/OSIiIsdg0NTFiDVNZTU61OsNTh4NERFR18GgqYvx9/SAh8K051xxldbJoyEiIuo6GDR1MXK5DCHsCk5E5Jb0BiOyS2qcPQxqAYOmLojF4ERE7unF705jwss/Yd/5ImcPhZrBoKkLYq8mIiL3dPxKGQDgmPm/5FoYNHVBzDQREbmn/ArT7+2rZbVOHgk1h0FTF8SgiYjI/RiNAgoq6wAAOQyaXBKDpi6IQRMRkfspqdFCZxAAMGhyVQyauiDWNBERuZ/8ijrpz1dLayEIghNHQ81xetC0YcMGxMTEQKPRICEhAQcPHrTqvK1bt0Imk2H27NkW9y9atAgymcziNn36dItjSkpKsGDBAvj5+SEgIAD3338/qqqq7PWWnI6ZJiIi99M4aKrWGlBRp3fiaKg5Tg2atm3bhmXLlmHlypVITU1FXFwcpk2bhoKCglbPy8jIwNNPP40JEyY0+/j06dORm5sr3f773/9aPL5gwQKkp6dj586d2L59O37++Wc88MADdntfztY4aOI3FSIi95BXbvlF92opp+hcjVODpnXr1mHJkiVYvHgxhgwZgo0bN8LLywubNm1q8RyDwYAFCxZg1apV6Nu3b7PHqNVqRERESLfAwEDpsdOnT2PHjh149913kZCQgBtuuAGvv/46tm7dipycHLu/R2cQm1vW6gyo1nIrFSIid9A40wSwrskVOS1o0mq1OHLkCJKSkhoGI5cjKSkJKSkpLZ63evVqhIWF4f7772/xmD179iAsLAyDBg3Cww8/jOLiYumxlJQUBAQEYMyYMdJ9SUlJkMvlOHDgQAfflWvwVivhrVIA4BQdEZG7aBI0lTNocjVKZ71wUVERDAYDwsPDLe4PDw/HmTNnmj1n3759eO+995CWltbi806fPh1z5sxBnz59cPHiRfz1r3/FjBkzkJKSAoVCgby8PISFhVmco1QqERQUhLy8vBaft76+HvX1DQFIRUWFFe/SeUJ91agurkFhZT36hHg7ezhERNSGPHPQ5KNWoqpez15NLshpQVN7VVZW4p577sE777yDkJCQFo+bP3++9Ofhw4djxIgR6NevH/bs2YMpU6bY/Ppr1qzBqlWrbD6/s4X6qpFhDpqIiMj1iY0tR0YHYN+FIuSU1bVxBnU2p03PhYSEQKFQID8/3+L+/Px8RERENDn+4sWLyMjIwO233w6lUgmlUokPPvgA33zzDZRKJS5evNjs6/Tt2xchISG4cOECACAiIqJJobler0dJSUmzrytavnw5ysvLpVt2dnZ733KnaigG5/90RETuQJyeG90rAABrmlyR04ImlUqF+Ph4JCcnS/cZjUYkJycjMTGxyfGxsbE4ceIE0tLSpNvMmTMxefJkpKWlITo6utnXuXLlCoqLixEZGQkASExMRFlZGY4cOSIds3v3bhiNRiQkJLQ4XrVaDT8/P4ubK2OvJiIi91GvN6CkWgsAGNXbtHiJQZPrcer03LJly3DvvfdizJgxGDduHF599VVUV1dj8eLFAICFCxeiR48eWLNmDTQaDYYNG2ZxfkBAAABI91dVVWHVqlWYO3cuIiIicPHiRfzlL39B//79MW3aNADA4MGDMX36dCxZsgQbN26ETqfD0qVLMX/+fERFRXXem3cw9moiInIfBeapOZVCjqFRpi/l+RV10BmM8FA4vaUimTk1aJo3bx4KCwuxYsUK5OXlYeTIkdixY4dUHJ6VlQW53PofFoVCgePHj2PLli0oKytDVFQUpk6dihdeeAFqtVo67qOPPsLSpUsxZcoUyOVyzJ07F+vXr7f7+3MmBk1ERO5DnJoL81MjxFsNlVIOrd6I/Io69Az0cvLoSOT0QvClS5di6dKlzT62Z8+eVs/dvHmzxd89PT3xww8/tPmaQUFB+Pjjj60doluSgiZOzxERuTyxCDzCTwO5XIYofw0yimuQU8agyZUw59dFhfpoADDTRETkDsR2A+H+pt/dUQGeAFjX5GoYNHVRYqapqEoLncHo5NEQEVFrCsSgydcyaGKvJtfCoKmLCvVVI9hbBYNRwM/nCp09HCIiaoWYaYrwN33hZdDkmhg0dVEKuQyzRvYAAHyeesXJoyEiotbklZszTX6mTFOPANN/OT3nWhg0dWG/i+8JANh1qgCl5v4fRETkegrM9adi0MSaJtfEoKkLGxLlhyGRftAajPj2eI6zh9MuJ66U42xepbOHQUTkcIIgSJmmiGuCpqultRAEwWljI0sMmrq4ueZs0+dH3GeKrrRai9+/9Svueuc3FrETUZdXUadHrc4AoFGmyd8UNFVrDaio0zttbGSJQVMXN2tkFJRyGY5dKcf5fPfI3JzMKUedzoiSai2yS2qcPRwiIocSV875aZTwVCkAAJ4qBYK9VQA4RedKGDR1cSE+akwaFAYA+MxNCsJP5VRIf75QUOXEkRAROZ7Uo8mcZRKxrsn1MGjqBsSC8C9Tr0LvBtNdp3IbBU2FDJrINWUUVeNCgXtkb53tcEYJ3t9/2S1+/ziD1A3c/9qgiSvoXI3Tt1Ehx7spNgyBXh4oqKzHvgtFUubJVTXONF0sqHbiSIiap9Ub8buNKajTGbD/2Zvg7+nh7CG5tL9+eQLn8qsgA7Do+j7OHo7LyW8j03S1rK7Tx0TNY6apG1Ap5VLPps9cvCC8TmfAxUbZJWaayBWdyatAUVU9qur1zDZZ4WqpKVPyavJ5lNfonDwa19PQo0ltcX8PTs+5HAZN3cTc0aYpuh9P5aO81nV/aZ3Nq4RRAOQy098vFVRxuS25nLTsMunPlwqZDW1Ndb0e1VrTyrCyGh3W7z7v5BG5HjHTFNFipolBk6tg0NRNDOvhh0HhvtDqjdjuwj2bxHqmMTFBkMuAynq91PSNyFWkZZVJf75UxKCpNUVVlv//fpCSgcu8Zhbamp5jpsl1MGjqJmQymVQQ7spTdGI906joAPQO9gbAFXTkeiwzTfz5bI0YNPUK8sLEgaHQGQSs+d9pJ4/KtYiF4E2DJo358Tr2rHMRDJq6kVmjoqCQy3A0q8yibsiVnDZnmoZE+aFfqA8AuOxYqXsqr9FZZJeYNWldoTlTHOqrxt9uHQyFXIYfT+Xj14tFTh6ZazAYBRRWNb96LsRbDZVCDqPQkI0i52LQ1I2E+WowcWAoANfsEG40Cg1BU6Qf+oUx00SuJ+1KGQDA29yEMKO4BgYj6+5aIgZNIT4qDAj3xR/G9QIAvLj9NK8bTJk4g1GAXAapmaVILpchUmo7wKDJFTBo6mbEgvAvj151uV9YWSU1qNYaoFbK0SfEG/3NmSYGTR1XXqvDxweyuHLJDsR6ppsGh0OllEOrN7LmpBWNM00A8ETSAPhqlDiVW4HP3aThriOJGaRQXzWUiqYfyeJ2KvwZcw0MmrqZKYPD4O/pgdzyOpdLj4tF4LERvlAq5Ogfxuk5e9nw0wX89csTeG//ZWcPxe0dzS4FAMT3CkBMsBcA/oy2Rpx6CvUxZUyCfdR47Kb+AIBXfjiL6vruva/atRv1Xosr6FwLg6ZuRuOhwO1xkQBcb4pOLAIfEuUHAOhnDpryK+pRUccMSUfsO28KkLmXX8cIgoBj5iLwkb0C0TfE9DPKtgMtK6zUAmjINAHAveNj0CvIC4WV9Xhr70VnDc0liJmmsBaCph6BzDS5EgZN3dDv4qMBADvS81wqGDnVqJ4JAPw0Hggz/6Llh5Ltymt0OJ1nuraFbN/QIZnFNSit0UGlkGNwpC/6hJrq7lgM3jIp09QoaFIrFVg+IxYA8PYvl7p1QCBtodJS0MStVFwKg6ZuKK6nP/qFeqNOZ8T/juc6eziSazNNAKQVdKxrst2hjBKI/UEZNHWM2GpgSJQf1EoF+oaYgqZLRfz5bElRo0LwxqYPi8C4mCDU6Yx4eccZZwzNJTRs1qtu9vGGXk0sBHcFDJq6IVPPJlO2yVUKMYur6pFXUQeZDBgU0RA0iXVNDJpsd+BysfTngkr+4u0IMWgaGR0AAOgrZpqYCW2WIAhNCsFFMpkMf7ttMADgq7Qci95X3UlLjS1FbHDpWhg0dVN3jOoBuQw4lFGKDBeYWjida9q/q3eQF3zUDftIsxi84w5cLpH+XFqjg1bPJnm2Omr+YB/VKwAApJqmnPI61Gi7d0Fzcypq9dCamzKG+DTNpIzoGYA5o037Yr64/VS33DJJ2kLFv4Wgybx6rrJe79JbYHUXDJq6qQh/DW4YYOrZ9IULZJtO5ZYDsJyaAxqm5y4y02STyjodTl41XVuZeT+/4mpO0dmiXm/AafMUsphpCvRWIdDLAwDrmpoj1jP5aZTQeCiaPeYv02Kh8ZDjcGYpvjvhOuUCnaVhs97mgyZPlQJB5v5NzDY5H4OmbkzcVuXz1KswOrlnk1TPFGkZNImZpsySGmZIbHA4sxRGwbSFRbiv6Zcy65pscyqnAlqDEUHeKvQK8pLu7xPCYvCWtDQ111iEvwYP3tgPALD2+zOo0xk6ZWyuoFZrQEWdKUPZUtAENGynwqDJ+Rg0dWNTh4TDV6PE1bJa/HqxuO0THOhUbtMicMBUHOmjVsJgFJBZzA+l9jpwyTQ1l9AnSPrgKqhg0GSLxvVMMjFtB6BvKNsOtETMNDU3NdfYgxP7ItxPjSultfjPb5mdMTSXIE7NeXoo4KdRtngcG1y6DgZN3ZjGQ4HZI031BOuTzzutnqBOZ8BF8wfOkEh/i8dkMhn6hXI7FVuJReAJfYOl9g2FVQyabHFtEbiImaaWWZNpAgAvlRJLJvQFAKd/getM+Y1WzjUOxK/V0OCSCzmcjUFTN/fI5H5QKeU4mFGCPecKnTKGc/mVMBgFBHmrml1224/F4Dap0epx4oqpnqlxponTc7ZpKWgSg/pL/PlswtqgCQBizatmM7pRRjmvjZVzoh5cQecyGDR1c5H+nlg0PgYA8PKOs06pbWq8SW9z37bYdsA2RzJLoTcK6BHgieggr4bpObYdaLfiqnpkFpu6qcddEzQ1np7rjqu/WlPUTGPLlvQ2b0lzpaTW5fbFdJS22g2I2HbAdTBoIjw8sR981Uqczq3At8dzOv31m2tq2ZjU4JLf5NulcT0TAGaaOuDYlTIApr5M/p4eFo/1CvKCTGZaEl5UpXXC6FyXlGlqo6YJMAUGHgoZtAYjcsu7R3AgdQNvod2AiFupuA4GTYRAbxUenGiqJ1i38xx0hs5dpXbt9inXkno1FVQ7fZWfO2moZzIFTWEMmmyWllUGoOnUHGCqDexp/lDjFJ0l8WctxIpMk0IuQ3SgKdskZvW6Omun58TVc3kVddB38u9nssSgiQAAi6/vgxAfNTKLa7DtUHanva7RKEiNLVvKNPUK8oJSLkOtzoDcCk4tWaNOZ8CxbLGeKRhAo0wTC8HbTWpq2UzQBAB9zE0uWQxuSdp3zopME9AwRdddgqb88ta3UBGFeKuhUshhFIB8fulxKgZNBADwVivx+JT+AIDXks+jVts5vVKyS2tQVa+HSimX9vG6lodCjpgQrqBrj9SsUmgNRoT7qaUPolAf07fVgop61t60g9Eo4JhUBB7Y7DENe9AxaBIZjAKKzUFTmBWZJgDoHWy6jt2lvUi+ub6wpc16RXK5DJHs1eQSGDSRZP7YXogO8kRhZT3e//Vyp7ymWM80KNwXSkXLP4792Rm8XRrqmYKl4nox01SvN6Kynlt+WOtycTUq6vRQK+WIjfRt9hiuoGuqtEYLo2DqRC92tG6LGOB3hxV0giBINU1tTc8BDb2arpYyaHImBk0kUSnleOrmQQCAjXsuorzG8fsctVXPJOoXZs408UPJKtfWMwGm7Rh8zfv6sa7JemI90/Ae/vBoIbAXp+eYaWog/owFe6ta/ULUWIyUaer603NljfaBDGtjeg5o3KuJQZMzMWgiCzPjohAb4YuKOj3e3HvR4a/X1so5UUMxOIOmttTrDThq/qAX65lEXEHXfi31Z2qsrznTlFVc0+kLKVyVVARuZT0TYFnT1NWnkMUi8CBvFdTK5vfla6wHp+dcAoMmsiCXy/DnaaZs0/v7L0ubSTpKS9unXKt/qGlahA0u23Ysuxz1eiNCfNTStJEoROrVxKDJWlLQ1CugxWMi/DTQeMihNwq4wukTAO1rbCnqGegFuQyo1Rm6fGAvBk3W1nuxV5NrYNBETdwUG4YxvQNRrzdi/e7zDnudkmotcs1BWWxE87UiIvGbfFGVFmU17IXTmgOXGqbmrm0WyrYD7VOnM0jNV1vLNMnlsoYpOgb2ANq/cg4wlQiIwUFmSdeeoiswB01t9WgSNQRNXEHsTE4PmjZs2ICYmBhoNBokJCTg4MGDVp23detWyGQyzJ49W7pPp9PhmWeewfDhw+Ht7Y2oqCgsXLgQOTmWDRtjYmIgk8ksbmvXrrXn23JrMpkMz8yIBQBsO5TtsGXU4odR72Av+Go8Wj3WW61ElPmXC7NNrTtw2VQEfl2foCaPcXqufU5eLYfeKCDERy1tZdGSvlIxOOuaAKDIhkwT0FDXlNHF68Pyys1F4L7tDZqYaXImpwZN27Ztw7Jly7By5UqkpqYiLi4O06ZNQ0FBQavnZWRk4Omnn8aECRMs7q+pqUFqaiqee+45pKam4osvvsDZs2cxc+bMJs+xevVq5ObmSrfHHnvMru/N3Y2NCcJNsWEwGAX868ezrR4rCAJOXi3Hqm/T8X9fnkCdzrp2BVI9UxtF4KJ+3E6lTTqDEUcySwGYNum9FoOm9mlcz9TahqoA2w5cq7AdW6g01qub9GoS2w2EW51pMh1XWa9HRZ3jF+lQ85TOfPF169ZhyZIlWLx4MQBg48aN+O6777Bp0yY8++yzzZ5jMBiwYMECrFq1Cr/88gvKysqkx/z9/bFz506L49944w2MGzcOWVlZ6NWrl3S/r68vIiIi7P+mupA/TxuEn84WYPvxXDw0sRzDevhbPF5UVY+vjl7FZ0eu4ExepXR/sLcKy6YOavP5rV05J+oX6oNfzhfhIr/Jt+j4lXLU6gwI8lZhgDnIbEycKuH+c9aRmlq2Us8k6su2AxZsKQQHgBgxaOri03NiY8u2ejSJvFRKBHp5oLRGh5yyWvhFtJ6dJ8dwWqZJq9XiyJEjSEpKahiMXI6kpCSkpKS0eN7q1asRFhaG+++/36rXKS8vh0wmQ0BAgMX9a9euRXBwMEaNGoVXXnkFen3rfWvq6+tRUVFhcevqBkf6YVZcFADglR9M2Sat3ogf0vOw5IPDuO4fyXjxu9M4k1cJlVKO8f1MmY2Ney9ZNaVn7co5ETfubZvYamBcTNN6JgAIM/+CZqbJOq1tn3ItdgW3ZEshONB9Glw2bKFi/fXhHnTO57RMU1FREQwGA8LDwy3uDw8Px5kzZ5o9Z9++fXjvvfeQlpZm1WvU1dXhmWeewV133QU/v4YP5scffxyjR49GUFAQfv31Vyxfvhy5ublYt25di8+1Zs0arFq1yqrX7UqW3TwI24/nYu+5Qjy5LQ17zxWipLqhEDsuOgC/j++J20dEwc9TiUXvH8Lec4VY+U06tiwe2+KURp3OINUmWRs0SRv3MmhqkdTUsm/TeiagIdNUxK1U2lRYWY+rZbWQyYARPf3bPL6PeXquoLIelXW6Nuv0ujpbp+ekBpddPPhsT2NLUZS/J05ercBVFoM7jVOn59qjsrIS99xzD9555x2EhIS0ebxOp8Odd94JQRDw5ptvWjy2bNky6c8jRoyASqXCgw8+iDVr1kCtbv5/8OXLl1ucV1FRgejoaBvfjfvoFeyFPyT0wgcpmfjy6FUApl+Cc0b3wO9G98SAcMtVb6tmDsXUV3/Gz+cKseNkHmYMj2z2eS8UVEFvFBDo5WF1elrMNGWX1qBOZ4DGo+3eJt2J3mDE4YyGTuDNET/Aiqu10BuMVjcd7I7EeqYBYT5WBUD+nh4I8VGhqEqLy0XVGNEzwLEDdGFavRFl5ua47Vk9B5j2mgSAijo9ymq0CPCyrpu4O9EZjCiuNgVN1q6eA1gM7gqcFjSFhIRAoVAgPz/f4v78/Pxma40uXryIjIwM3H777dJ9RqOpiZxSqcTZs2fRr18/AA0BU2ZmJnbv3m2RZWpOQkIC9Ho9MjIyMGhQ87U4arW6xYCqq/vTlAG4VFgNf08P/C6+JyYMCGnxwzYmxBsPTeyH9cnnsXr7Kdw4MBTe6qY/ZuLU3OBIvzYLbEUhPir4e3qgvFaHy0XVGGxlLVR3kZ5TgWqtAf6eHi22cAjyVkEuA4yCKXBqz7fc7iYt21RQb83UnKhviA+Kqkq6fdAkBgQeChn8PduXcfNSKRHup0Z+RT0yimswsgsGTQWV9RAE0/UJasf7E1dwcisV53Ha10yVSoX4+HgkJydL9xmNRiQnJyMxMbHJ8bGxsThx4gTS0tKk28yZMzF58mSkpaVJWR8xYDp//jx27dqF4ODmv3E3lpaWBrlcjrCwMPu9wS4k2EeN//wxARsWjMbk2LA2sxOPTOqH6CBP5JbXtdjnqb1F4ICpFYLYrJFTdE2J9UxjY4IglzcfiCrkMqkwl3VNrUtrY5Pe5ojF4N19sULDFirqFn8WW9M7qGvXNeVLjS017bo+zDQ5n1On55YtW4Z7770XY8aMwbhx4/Dqq6+iurpaWk23cOFC9OjRA2vWrIFGo8GwYcMszheLu8X7dTodfve73yE1NRXbt2+HwWBAXl4eACAoKAgqlQopKSk4cOAAJk+eDF9fX6SkpODJJ5/E3XffjcBA6385Uss0HgqsmjkU920+jPd+udzsNF57i8BF/cN8kJpV5va9mnQGI0qrtVJhtj2I9UzXtVDPJAr1VaOgsp5BUyuMRgHHs8sBtC/TJNY1dfdicFuLwEW9g71wMKOky7YdEFfOWbPnXGNR3ErF6ZwaNM2bNw+FhYVYsWIF8vLyMHLkSOzYsUMqDs/KyoJcbn0y7OrVq/jmm28AACNHjrR47KeffsKkSZOgVquxdetWPP/886ivr0efPn3w5JNPWtQrUcfdFBuOm4eEY+epfDz39Un8d8l10jSc0ShYvX3KtbpKMfhzX53EtsPZ+Pzh8Rjdq+PBusEo4GAb9Uwi9mpq28XCKlTW6+HpocDA8KatG1rSN5RdwYGOB00x5uAzo4tnmqyt5xSJ03N5FXWsSXQSpxeCL126FEuXLm32sT179rR67ubNmy3+HhMT0+Ymj6NHj8Zvv/3WniGSjVbePgS/nC/Eb5dK8HVaDmaP6gEAuFJai6p6PVQKuRQEWUvauNeNpz+Kq+rxeeoVCALw87lCuwRNp3MrUFmnh69a2WYgGubLXk1tEfszDe/p364PpsaZJkEQrK7X62qKbNhCpTGxGLyrZprybFg5B5h6XnkoZNAZBORX1rfZpZ7sj2EqOUzPQC88dtMAAMCL352WutieyjVNewyM8IFHO78piUHTpcIqGIzuuQv6l0evQmcwjf1MbmUbR1tH3DplTEwgFG3USDDT1Laj5v5Mo9oxNQeYPuwVchlqtAZpSXl31OFMUxfv1ZQv9WhqX9Akl8sQ6c+6Jmdi0EQO9ccJfdA3xBtFVfVY9+M5AO3fPqWxnoFeUCnlqNcb3XIFiSAI2HYoW/r7mTz7NElt2KS37YUP4rf/QvZqalHj7VPaQ6WUS1mS7jxFJ/5shfjYtvJN3EqlqEqLqvrWGw+7I2l6zr/9QSXrmpyLQRM5lFqpwOpZpkL9D1IykJ5TbtPKOZFCLpP2+HLHYvC07DKcL6iCypxhyyypQXUHPxSMFvVMrReBA0CoL7uCt6ZGq8dZczA70ortU67Vh3vQNco02bbQwd/TA0HepoCrK2abpG7gNlwfcQXdVQZNTsGgiRzuhgEhuHVEJIyCqQA6XVo513aX5ea488a9nxw2ZZluGxGJMF81BAE4l9+xKbpzBZUoq9HBS6Vosj9gc8QVOwUMmpp14ko5jIKpSFecCmkPaeNeN66766iOTs8BXbuuqUCsaWpHY0tRT7YdcCoGTdQpnrt1CLxVCqRmlSHXvNw2NrL5Boxtac8KuqNZpbjuH8l4aUfzW/N0phqtHt8eywUA3Dk2GrHmTFvjzY5tIbYaiO8daFWNWCj7NLUqtR37zTWnj7hxb5H7BfX2UlRl2mqpI0GTtHFvFwuaqur10pSjLc1lG3o1cSGHMzBook4R4a/BE0kDpb/3CvKCn417czWsoGv9QymzuBr3bzmMvIo6bPk1A3U6g02vZy/fHc9FVb0eMcFeSOgThMHmrt2ncztW1yQ2tbzOinomoOGDrEZr6PDUYFd0KKOhqN4Wfbv5xr012oagoCNBU1fduDfP/KXRR62ETzO7JbSFDS6di0ETdZpF18dIPW8G25hlAtDQFbywqsUWEyXVWix6/5C0uXCN1oC95wptfk3A1JDyp7MFqNfbFnyJU3O/HxMNmUwmZdo6soJOEAQcvGx9PRMAeKuV8FKZ9u1jtsmS0ShI+/eNs/J6Xkv8+cwuqbH5Z8WdFVWa/p/TeMjhrbJ9f0hp494uFjQVSCvnbAsoo7iVilMxaKJO46GQY92dI3Fd3yAsvr6Pzc/TL9QHMhlQVqOTgqLG6nQGPPDBYVwuqkaPAE/MHhkFANhxMs/m1wSAN/dcxOL3D+GZz463+9yLhVU4lFEKuQyYO7onACA2wjQ9dzqvos3+Yi25UlqLoiotPBQyDO9pfY1YQ68mBk2Nnc2vREWdHt4qhU0LFQBTdsVbpYBRMAVO3U1hlSkoCPVVd6hPlZhpyupi03N50so524rkxdVzlfV6qY0LdR4GTdSphvXwx9YHEq2eSmqOxkOBnoGmb1vX1jUZjQKe+uQYDmeWwlejxPuLx+Lu63oDAHadzodWb7TpNQVBkDJFX6Xl4NcLRe06/9PDVwAAkwaFSb8s+4X6QCmXobJOj5xy2+oTxKXxQyL9oFZa/62evZqaJ07Nje4daHO3ZZlMJnUGd+cmrLaSisBtbGwpEmuacsrrnD61bk8dWTkHmDY0DvQylTZwiq7zMWgityQVg19T1/TSjjP47kQuPBQyvHVPPAaG+2J0r0CE+apRWafH/ovtC3ZERzJLcaVROvy5r09aHYDpDUZ8nmoKmu4cEy3dr1LKpfqsMzbWNR0zB01x7SxabgiaWEzamDjVOTbGtqk5UZ9uvIKu0A5F4AAQ5K2San66UsauIyvnRJ1d1yQIAg5cKu7wSt+ugEETuaX+4jf5goYPpQ9TMvDWz5cAAC//bgTG9wsBYOqiO21oBABgxwnbpui+SrsKALh5SDiCvVW4WFiN9/Zdturcn84WorCyHiE+KkwZHGbxWKy5GNzWFXTHrpQBaP9KrzCxVxMbXEoEQZAyTR0NmvqGitupdL8VdPZoNwCYMna9u+AKOrEQPLwD16ehV5Pjv/RU1Onw+NY0zHv7N8x981eU13bvKUEGTeSWxAyNmGnadSofK79JBwA8dfNA3DGqp8Xx04eZgqadp/OhN7Rvik5nMOK746ZWAfdc1xt/vWUwAGB98nmrGsyJHcDnjO7ZpCWA2HbAlhV0eoMRJ66atqSxNdNU0I23+rhWdkkt8ivq4aGQYZQNTS0ba9i4txtmmirFbuAdC5qAhu1UulIxeH5lx2qagIaNex2daUrNKsUtr/2Cb4/lAAAq6/T49HB2G2d1bQyayC2JDS4vFlTh+JUyPPbfozAKwLwx0Vh6U/8mxyf0CUKglwdKqrVS92xr/XyuEKU1OoT4qDG+XzDmjO6BcTFBqNUZsPrb9FbPLaiow09nCwAAd47p2eTxjmSazuZXok5nhK9GiT7mDxdrcSuVpsSfi+E9/KHxsH3VF9DQ4LI7th2wV6YJaNhOpStlmvLFTJMNPZpEjt5KxWAU8Mbu8/j9xhRcKa1Fz0BPLBofAwB4f39Gu794diUMmsgtidNzV8tqcd/mw6jVGTBhQAhevGNYsyt2lAo5bh4SDqD9q+i+SjN9y7o9LhJKhRwymQyrZw+FQi7DD+n5UlDUnM9Tr8JgFDC6VwD6hzVtszDYnGm6VFjV7mLXY9nmLFPPAMjb2KT3WiwEb+qQWM9kY6uBxsSapuJqLcprutd0hhiId7QQHGjU4LKL1DQZjYK0YrVjQZMp03Q2r9LmlbctyS2vxR/e+Q3//PEcDEYBM+Oi8L8/TcCzM2IR6OWBq2W12HU6366v6U4YNJFbCvRWIdi8N1VRVT1iI3zx7wWjW+2IPWNYJABT0GQ0WveLpqpej52nTEHW7JE9pPtjI/xw3/UxAICVX6c3G/AIgiClsueNjW7yOGBa+h/o5QGj0P5tYY7ZuKks0Gh6jkGTRKxnGtfBeibA1AtL7MNzsZvVNRXZMdPU1RpcFldroTcKkMk6dn2u6xsMjYccZ/Iq8ct52xa3NGfHyTxMf/UXHLhcAm+VAv/6fRxemz8SfhoPaDwUWJBgWom8aX+G3V7T3TBoIrclrqCL8NPg/cVj4dtGh/Hx/YPhq1aioLIeR7NLrXqNH9PzUKczom+IN0Zc0wfpT0kDEe6nRlZJDd7cc7HJuYczS3GpqBpeKgVuHRHV7PPLZDIp29TeuqY0G1fOAQ19moqr6mGwMoDsygor63GpqBoyGTCmd8eDJqBRZ/BuVNckCIKUabJHTZNYCH6ltBa6LjAllG9uNxDsrbZqy6OWhPio8YdxpgBmffL5DmebarUG/PXLE3joP0dQXqvDiJ7++O7xCZgb39Mic39PYm8o5TIcvFyCk+Z6yu6GQRO5rfsn9MH1/YOx+b6xVm2sqlYqpNVr31u5iu7Lo6ZVc7NG9mgy7eejVuK524YAAN7ce7HJt2GxAPy2EZGtbpcgNrlsT11TVb0e5wpMx8e1o6mlKMhbBZkMMApotkFodyN2AR8U7gt/L9u297lW3264B11FnV5qxWGPTFO4rwZqpRwGo9AlehLlS40tO35tHpzYFyqlHIczS5Fyqdjm58kuqcHMN/bh4wNZ0vN+9tB4xIQ0rZMM99Pg1hGmjP2m/datHu5qGDSR25o2NAIf/fE6KeiwxnRxii49r81vZwWVddhvbmI5a2TzmaJbh0fihv4h0OqNWPlNuvSclXU6acVdS1NzInE7lfZkmk5eLYcgAFH+GoTZUBuhVMil6U3WNTUUgXe01UBjfbphMbj4s+SrUXa4mB4wtQtp2E7F/euapG7gHahnEoX7aTDf/LtlffJ5m55DEAQ89ekxnC+oQqivGv+5PwHLZwyGStlyaCDu5vDtsRwUdMM+bwyaqFuZODAUnh4KXCmtRXpO60HK9mO5MAqmmqHmvnUBpum11bOGQqWQY8/ZQvyQbiqQ3H48F7U6A/qGemN0r9Y3fh0c0TA9Z22aXapn6sDS+FBzr6bu+IvvWlJ/JjsUgYv6dcO2A/ZcOSfqFdR16prElXO2fNFpzkMT+8FDIcNvl0qkxqzt8VXaVRy8XAKNhxxfPDweNwwIafOckdEBiO8dCJ1BwEe/ZdkybLfGoIm6FU+VApMGhQIAvj+Z2+qxX5sbWs5uIcsk6hvqgwdu7AsAWP1tOmq0emlqbp55c97WDAj3gVwGlNborM76SPVMPQOsOr45XEFnUlmnwylzAG2PInBR40yTtQsP3F2RHVfOiWK6UNuBfHNfNHtkmgDTKrrfxZuyTa/vbl+2qbxWh79/dwYA8NhNAxAd5GX1uYvNi2A+OpDZpba4sQaDJup2xEaX359seYruclE1jl0ph0Iuw21xrQdNAPDo5P7oEeCJnPI6PLktDWnZZVDKZZgzumlvpmtpPBTSB+xpK+uabN0+pTH2ajJJzSqDUQCigzw71HDwWj0DPeGhkKFeb0ROufvX41hDamxpx0xT75Cuk2my5/Sc6JFJ/aCUy/DL+SKkZlm3wAUA/t/OcyiqqkffUG8smdC3Xa85fWgEovw1KKrSSo0vuwsGTdTt3BQbBpVCjkuF1TjfwjL/r8wF4BMGhFi1CshTpcCqmUMBQJqiuyk2zOppCrEzuDV70BVU1CGnvA5ymakRo62YaTI5ZKf95q6lVMjRy/ztvbtM0dmzR5MopgvVNImF4GF+9rs+0UFeuGOUqR3K61bWNp28Wo4PUjIAAC/MGtZqDVNzlAo5FjZqdmnvXlGujEETdTu+Gg9MMM/dN7eKThAEaa+5xr2Z2pI0JBxJjfaWa6sAvLHB7egMLk7NDQz3hXcrq/LaEsZeTQAaisDtOTUnErdTuVjYPVbQOaKmqbe5pimrpMbtpzkbVs/ZL9MEmDLdcplpn8sTV1pvBWA0Cnju65MwCqaVvdf3b7uOqTnzx0bD00OBU7kVOGBDPZW7YtBE3ZI4RbcjvWnQlJZdhsziGnh6KKQu4tZaeftQ+GmU6BfqjYkDQ60+LzbC+l5N4ia9HalnAphpAoB6vUEKQu1ZBC4aFmXKBB641D0+VBwRNEUFaKCUy6DVG6XpLXdUpzOg1NwdPtzXvkFTTIg3Zpm/4K1vo7bpk8PZOJpVBm+VAn+7dYjNrxngpcKc0abX3GTl5uVdAYMm6pZuHhIOhVyG07kVTWolvjZvmzJ1aHi7MznRQV74+S+T8c3SG6BsR/M6se3AxcIqqc9NS6TtUzpQzwQ0fLAVdeOg6cSVcmj1RoT4qKT94uxponnRwb4LRV2iOWNbpEJwOwZNSoVcKlJ25417954rBAAEe6sQYKdeYI09Ork/ZDJg56l8aWHDtUqrtXhph6n4+8mbB3Y44yUWhO88nY+sLjB9ag0GTdQtBXipkNg3GICpIFykNxix/bgpaGrP1Ny1z93eYKtHgCd81UroDEKrzRCNRkHKNNmyfUpjYcw0SVNzY3oHtbnK0RYjevgj2FuFqno9DmdYX6TrrqRMkx1rmgBItWHu/MH8n98yAQC/G9PTIT9r/cN8cJt554E3fmo+2/TyD2dQWqPDoHBf3GuuSerYa/pi4sBQCAKwxVwj1dUxaKJuq/EqOtG+C0UoqtIiyFtlVc8Se5HJZFK26Uxuy3VNl4qqUVmnh8ZDjoHhPh16TTEbUFmvR63W+mXD5bW6LrOFgj036W2OXC7DjeZp2j3nWt7YuSswGAUUm7vL2zPTBLh/Mfjlomr8cr4IMhmwwLz9iSMsndwfAPC/E3k4l2/5e+RoVim2mluhvDB7WIe2cWlMzDZtO5SNyrquvzk1gybqtqYODYdMZlq+L27RIE7N3T4i0m6/VKwl1TXltVzXJLYaGN7Dv13Tf83xUSuh8TA9R3uyTc98dhy3vb4Pe866dxBgMAo4nGnK/jiiCFwk9gXbe7bQYa/hCkprtDCYN6MNMnebtxd337j34wOmLNPEgaHoFWx9P6T2GhThixnmL4Nv7L4g3W8wF38LAjBndA+Ms+OXhBsHhKJfqDeq6vX47MgVuz2vq2LQRN1WmK8GY82bs+44mYcarR4/mAvDZ42ybWquIxo27m0502SvqTnAlN2SisGrrCuw1RmM+Pm86cP/k8PZHR6DM53Nq0RlnR7eKgUGm7N8jjBhQChkMtPKyNwu3K9JDLyDvFR2/8LR240bXNbpDPjksCmYuOc6x2WZREtvMmWbvj2eI63a/OhAJk5erYCfRonlMwbb9fXkcpm0tcrmXzPcfoVjWxg0UbcmraI7mYedp/JRozWgd7AXRtkhKGmvhum5ljNNaXZoatlYmHkVj7WZplM5FagxT+Ulny5w63S8uHXK6N6BHc7atSbIWyUFuV052+SIInBR40yTu/UE2n48F+W1OvQI8MSkQWFtn9BBQ6P8kTQ4HIIAbPjpAgor6/HKD2cBAH+eNsgh/z5zRveAn0aJzOIa7D7j3hnotjBoom5NDJoOZZZg0/4MAMCsuCiHFGq2ZVC4KWgqqKxHcTNduut0BqklQUfbDYjEgl1rezWJgQYA1OuN2Hkq3y7jcAZH9me61qSBpg/Ln9x8SrM1UjdwOxeBA6Zu7TIZUK01oKhKa/fnd6QPzQXgf0joBYW8c36vPD7FlG36Oi0Hyz5JQ2WdHsN7+OMPCY7JdHmplLgroRcAYNP+rt1+gEETdWtRAZ6Iiw6AIDTUCzljag4AvNVKaRribDNNLk/nVkBnEBDsrULPQE+7vGZ7ezWJK8BCfEw1K9+46RYKgiA4vAi8MbGuaf+F4jZbSrgrR/RoEqmVCkT5m37ms0rcp67pxJVyHMsug4dC1q5mtx01omcAJg0KhcEoSAXoL8we5tCgbWFiDBRyGX69WNzs76+ugkETdXvTh0ZIfx7ew1/and4ZYs2dwZvbg06cmhsZHWC3TFh7giZBEHA40xRo/HnaIADAvvNFKKl2r2/+gKm7dEFlPTwUMrvUh7VleKPWA0cyu2brAUcGTUBDXVNGkfvUNYltBmYMi3RIBq41j900QPrz/LG9HP5z3iPAE5PN04+7TrtvBrotDJqo2xNXmwDArJFtb87rSOIKuubqmuyxSe+12tOrKaO4BkVVWqgUcswa2QNDo/ygNwr434lcu42nsxw0Z5lG9AyAxkPh8NeTy2VSh/iu2nrAEfvONeZuK+jKa3X4+phpO6Z7Eh1fAH6t+N6BWJDQC/G9A/EX85ccRxvfz9T77nBG1+2Az6CJur2YEG9MHhSKMF81Zjtpak4kruJqbg+6Y1fs0wm8sdB27D93SAo0/KHxUGBmnCnAdPQUnSAIzdZ4dYRYm2XvTXpbI3YH33OmaxaDO7IQHGi0gq7EPTJNnx+5gjqdEYPCfTGmd6BTxvD3O4bj84fHI9DOLSBaIrYyOJxZCkMXXUXHoIkIwKZFY/Hb8imdnkK/lphpOpdfCX2jbTfKarS4XGT6hh3X099ur9ee6Tkp0DD/YrzNHDQdyihx6FL69/ZdRvyLu7D041S7rdY7ZK7NGhvTeR9mNw4IhVwGnM2vlPqCdSWOLAQH3KvBpSAI+I+5N9Pdib2dsrDEGWIjfOGtUqCyTt+kuWZXwaCJCKaeRfJOWtnSml5BXvD0UKBeb7T4cBCzTH1CvBHgZb9vjdL+c1X1bfZXERtBioFGjwBPjI0JhCAA2485ZopOEAR8fCALgGnp9m2v72tzF/e2FFTW4XJRNWQy0/YpnSXQWyVlCcV9yNqjoKIOv10qtvOo7MfxNU3uMz2XcrEYlwqr4a1S4A4nZ687k1Ihx2hzVu1QF52iY9BE5ELkchkGRYhTdA11TVI9kx2zTEBDVkBvFFBW23IWp7CyXsp0xfdqCDTEKbpvjztmii49pwKXiqqhVsrRI8ATmcU1mPPmfry//7LN/XrEFYCDwn3h74CNU1sjFsq2t5u63mDEXe/8hvlv/+aSgZNWb0Rpjennx9HTc2U1OpTXuHZ/MDHLdMfoHvBp5z6U7k6c8j7URfdadHrQtGHDBsTExECj0SAhIQEHDx606rytW7dCJpNh9uzZFvcLgoAVK1YgMjISnp6eSEpKwvnzlpsXlpSUYMGCBfDz80NAQADuv/9+VFW1vEkqUWca3MwedI4oAgcAD4Vc2vKioLLlruBHzKvmrg00bhkeCYVchuNXyqWgyp7EYGzK4DD87/EJmDY0HDqDgFXfnsIDHx5BWU37V+6JReCdWc8ksrX1wBepV3Gx0HR9v3XBNg/F1aYsk1IuQ4CnYwJRL5VSCsgyW2k7YDAK+ORQNv786TGnTIPmV9Thh3TT6rG7O6EDuKuRgqbLJW7XiNQaTg2atm3bhmXLlmHlypVITU1FXFwcpk2bhoKC1r+FZWRk4Omnn8aECROaPPbyyy9j/fr12LhxIw4cOABvb29MmzYNdXUNHwgLFixAeno6du7cie3bt+Pnn3/GAw88YPf3R2QLaQ868wo6QRAs2g3Ym7jaqbW6JvFb45hraoCCfdS4vr9pY2N7f5gLgiBN+90+Igr+Xh7YeHc8Vs0cCpVCjp2n8nHr+n3tXsJ/bW1WZxoW5Y8QH1PrAbF9Q1vq9Qa8ltzwxW/nqfwOb1VxNKsUnx+5gpNXy1Gns36z5pYUVZqC1xAftUOnuVuraxIEAT+dKcCM137GXz4/jk+PXMHDH6V2el+srQezYTAKGBsTKP2/3J2MjA6Ah0KGvIo6XCnterV7Tg2a1q1bhyVLlmDx4sUYMmQINm7cCC8vL2zatKnFcwwGAxYsWIBVq1ahb9++Fo8JgoBXX30Vf/vb3zBr1iyMGDECH3zwAXJycvDVV18BAE6fPo0dO3bg3XffRUJCAm644Qa8/vrr2Lp1K3JyXO8bHHU/sRGWK+iulNaiuFoLD4VM2p/OnsL8rAmazN2zmwk0Gq+is+c3y9SsMlwtq4W3SoHJsaZpLZlMhnvHx+CLR8YjJtgLV8tqcedbKXhzz0WrAonKOp0UjHZGJ/BryeUy3DigfRv4bjuUjatltQjzVcNXrURBZT2OmoNoW5RUazH/7d/w1KfHcNvr+zB05Q9IWrcXj/33KDb8dAG7z+Qjp6y2Xf+W4t6FIb6OXaUl1jVlXVPXdPxKGf7wzgEs3nwI5/Kr4O/pAV+NEseyy/DPH886dEyN6Q1G/PegqQavO2aZAMBTpcCwHqYygq5Y1+S0oEmr1eLIkSNISkpqGIxcjqSkJKSkpLR43urVqxEWFob777+/yWOXL19GXl6exXP6+/sjISFBes6UlBQEBARgzJgx0jFJSUmQy+U4cOBAi69bX1+PiooKixuRI8SaA6OrZbUor9VJm/QOjvRzSE+htjJN1fV6pOeYft7HNBNoTBsaDpVSjgsFVa1uNtxeYuZq6tCIJu97WA9/fPvYDbg9LgoGo4CXdpzBos2HcD6/EtX1+haf80hmKYyCaVuOCH+N3cbaHpNixbqmtoOmWq0Br5t3q3/spv7SuT+aN5a2xf9O5KJeb4SvRokALw8YjAIuFFTh22M5eOWHs7hv82GMX7sbcat+xIJ3f0OGFdOuUhG4g1ef9g6yzDRll9Tg8f8excw39iPlUjFUSjkevLEvfv7zZPzz93EAgLd/vmTz9jWCIODlHWdw97sHkHw6v81ActfpAuRV1CHYWyVt0dQddeW6JqdVqBUVFcFgMCA8PNzi/vDwcJw5c6bZc/bt24f33nsPaWlpzT6el5cnPce1zyk+lpeXh7Awy00TlUolgoKCpGOas2bNGqxatarV90RkD/6eHugR4ImrZbU4m1fZqAg8wCGv11avprTsMhiMAqL8NegR0HT7Fl+NB24aFIYd6Xn45lgOhkR1PBtmaNQ08/a4yGaP8dV4YP38kbi+XzCe/zYdP58rxM3mVWleKgXCfNUI9VUjzFeDUPOfxWvpjHom0Y0DQixaD0Q1c01FH/6WgcLKevQM9MS8sb0Q6K3Ct8dy8EN6Hp6dEWvTUnaxr9ZjN/XHkgl9UVBZj1O5FTiTW4kzeab/XiysQkWdHvsvFGPt92ew8Z74Vp/T0SvnRL1DTJmm07kVeHH7KXyQkgmtuTXHHaN64KmpA9Ez0BRYTRsagXsTe2NLSiae+uQYvv/TBIT7WR8oC4KA1dtP4X3znpT7LhQhrqc/nrx5ICYODG322osdwO8cGw210vFNU13VmN6BeBtdM9PkNmX9lZWVuOeee/DOO+8gJCSk019/+fLlWLZsmfT3iooKREd33l5C1L3ERvjialktzuRVOLSeCWi7V5P4i6+5LJNo5sgo7EjPw7fHcvDM9EEd7ktz8HIJCirr4e/pgRv6h7Z4nEwmw/xxvTCqVyD+78sTOJVbgRqtATVaAzKKa1rs6eOMqTlRgJcKI6MDkJpVhj1nC/EH80an16qs0+HNPRcBAH+aMgAqpRyTBoVBpZQjo7gG5wuqMNC8ybO1cspqcfByCWQy4HbzxtThfhqE+2mklX2AqY7q0OVS3P3eAexIz8PZvEppVWdzOitoEmua0nMqpOzn9f2DsXzGYGlKqLHltwzGoYxSnMqtwBNb0/CfPyZYvf/aup3npIDp9rgo7DqVj2NXyrHo/UMY3SsAT948EDf0D5F+1i8VVmHfBdM+b38Y1/y/aXch/q64UFCFkmqttNikK3Ba0BQSEgKFQoH8fMs9avLz8xER0TStefHiRWRkZOD222+X7jMaTd8wlEolzp49K52Xn5+PyMiGb6f5+fkYOXIkACAiIqJJobler0dJSUmzrytSq9VQq53b+JC6j9hIXySfKcDJq+U4cdX+ncAbaytoOmxFI8ibYsPgo1bialktUrNKEd/B/kfiqrkZwyKgUrZdRTAowhefPTwegGk6saCyHoWV9SiorDP/t176r5eHQmrM6SyTBoWZg6aCFoOm9/dnoLRGh76h3lKvHx+1Ejf0D8HuMwX44WReu4MmccpzXEwQIv1bznCplQrcMCAEM4ZF4PuTeXjjpwt4/a5RLR5fVGUqBHf09FxMiDeUchn0RgGxEb5Yfstg3DggpMUgXeOhwBt/GIXbXt+HlEvFeGP3BfwpaUCzxzb27z0XpGnRF2YNxT2JMSiqqsdbey/ig5RMpGaV4Z73DmJsTCCevHkgxvcLwUfmfmI3DQpDtHkasbsK8lahf5gPLhRU4XBGCaYO7TpTlU6raVKpVIiPj0dycrJ0n9FoRHJyMhITE5scHxsbixMnTiAtLU26zZw5E5MnT0ZaWhqio6PRp08fREREWDxnRUUFDhw4ID1nYmIiysrKcOTIEemY3bt3w2g0IiEhwYHvmMh64qqbHSfzUKcz1Z/0NU9N2JsUNDWzVYneYERqljloamW1mcZDgalDTNPi36R1bEGFzmDE99LUXPuDG2+1En1CvDGuTxBuGxGFxdf3wTPTY/HP38fhg/vGYeM98U7vndPQeqCo2dVdZTVavPPzJQDAk0kDoVQ0/KqeNtR0nX841f66pq/N/zazRlrXcHHpTf0BANuP5+BiYcttWaRu4A7ONPlpTCsoN/xhNL57fEKL02SN9Q31wYuzhwEAXks+12afqy2/ZuDlHabi8eUzYnFPYgwA08rA/7t1CH75y2Qsvj4GKqUchzJK8Yd3DmD+2yn47MgVAN23APxa4hT44S62QbVTV88tW7YM77zzDrZs2YLTp0/j4YcfRnV1NRYvXgwAWLhwIZYvXw4A0Gg0GDZsmMUtICAAvr6+GDZsGFQqFWQyGZ544gm8+OKL+Oabb3DixAksXLgQUVFRUj+nwYMHY/r06ViyZAkOHjyI/fv3Y+nSpZg/fz6iopz77ZNIJPZqqqgzFTXH9Qxw2FJucdPegoqmfZrE6S5fjRIDw1rPatxu3uz4uxO5FlvAtNf+C0UordEhxEeN6/oG2/w8rkxsPVCtNTTbeuCtny+hsl6P2Ahf3DrcsqYraXA45DLg5NUKXCm1fkuR8/mVOJVbAQ+FzGKT6tYMjfJH0uBwCAKw4acLLR7n6M16G0saEo5bR0RaPc0GAHNG98Tc0T1hFIA/bT2Kkurm+3t9cjgbK79JBwA8PmUAHpzYr8kxYX4arLx9KH7+82QsTOwNlUKO3y6VoLxWh+ggT9w4sOXp5O5EzEx3tbompwZN8+bNwz//+U+sWLECI0eORFpaGnbs2CEVcmdlZSE3t33bM/zlL3/BY489hgceeABjx45FVVUVduzYAY2moQDwo48+QmxsLKZMmYJbbrkFN9xwA95++227vjeijogJ9raYloqLtm8n8MZCfUz/b1TU6Zv07JH6M/UObDNou6F/CAK9PFBUpUVKB7pWf2vuzXTr8Ih2fTC6E7lcJn24Xtt6oLCyHpvNtTRPTR3U5LoH+6ilmpEf0y3LG1ojFoDfOCC0XRu4Pj7FlG36Oi2nxS1MOqumqSNWzxqKvqHeyK+ox9OfHmuyEu7bYzl49vPjAIA/3tAHT7YxjRfhr8HqWcOw58+TsCChF0J8VHh66qAu+zPbXmKm6cSVctRqO94LzFU4vSP40qVLkZmZifr6ehw4cMBiimzPnj3YvHlzi+du3rxZ6r8kkslkWL16NfLy8lBXV4ddu3Zh4MCBFscEBQXh448/RmVlJcrLy7Fp0yb4+PjY820RdYhSIcfA8IafSUetnAMAP0+lFKAVXTNFd9iKInCRh0KOW8xZEVun6Op0Bmk5vS1Tc+5kkrnw+trl8P/ecwG1OgPiogOQNDisuVMxzVwj8oOVrQcEQZCm5maObN91HdEzABMHhsJgFPDvny42ebxWa0CVuc2DKwdN3molNvxhNFRKOXafKcB7+y5Lj+06lY8nt6XBKAB3jeuF/7t1sNWLGaICPPH3O4bj8N9utnraszvoGeiJCD8N9MaG5rxdgdODJiJqXuNuwo5aOQeYvmiI0yqN2w4IgiBlmqxdoi82utyRnod6ffu/Xe49V4jKej2i/DUY3avlwvOuQGw9cC6/StruI6esFh/9ZioofnrqwBY/uMX6sUMZJShuphbtWseulCOrpAaeHgrcPCS8zeOvJWabPk+90mRKUAy0NR5yp9eKtWVwpB+eu20IAOClHWdw/EoZ9p0vwiMfp0JvFDB7ZBRenD2sw6s/yfR7RayD7EpTdAyaiFyU2Bk8yl+DsHb0l7FFcyvoMotrUFRVD5VCjhFWbhQ8NiYIkf4aVNbprWreeC1xddetIyIduh2HKwjwUmGUOTAUr9Xruy9AazAioU8QbujfcmuV6CAvDI3yg1EAkk+33bjx67SrAICpQ8PhpWp/YBPfOwjj+wVDbxSwca9ltkkMtEN81G4RbNyd0AszhkVAZxDw0IdHsOSDw9DqjZg2NBz//H0cp9fsqCvWNTFoInJRU4dEIMRHhd+PcXw/sLBmgqaD5l90I3r6W92JXC6X4bYR5im6du5FV6PVSwFAV5+aE00y1zXtOVuAzOJqfHo4GwDw9LS2e11ZO0VnMApSndisdk7NNfb4FFONzyeHriCvvGHRgDvUMzUmk8mwdu4I9Az0RE55HWp1Btw4MBTr7xplsUqROm6MufVIamZphxaHuBL+hBC5qF7BXjj8t5vx5M0D2z64g5rLNLWnnqmxmXGmuo7k0/mtbmlyrV2nC1CrM6B3sBeGN9OosCsS65r2XyjCKz+chd4oYOLAUKumQ8Wg6ZcLRVJNUXNSLhajqKoegV4emDDA9pVd1/UNxriYIGgNRrz1c0O2qTNXztmLv6cHXr9rFAK8PDBxYCjeuju+W3fwdpRBEb7w1ShRrTVIe2m6OwZNRNTsVirWNLVszrAefugT4o06nRE7T1m/ukucmrt9RJRbTPPYw9AoP6n1wPbjpmzQU1OtC5IHhvugd7AXtHpjq5v/ilNztwyPhEcHMymPmWubPj6QhYJKU7bJ3TJNolG9AnHwr0nYct84eKoYMDmCQi5DfG/T74+Dl7vGFB2DJiJqkmkqqqrHJfNGreIvPWvJZDJpeu3/7TqH7JK2ewmV1+qkD/7uMjUHmKYzJw5sWCE3bWg4Rli5UlImk7U5RVenM2DHSdNjM+1wXW/oH4KR0QGo1xvx7i+m1WdiIbi7BU0ArOo2Tx3T0OSSQRMRdRFhvqZCc3GqRcwyDQz3QYBX+/eNuvu6XugR4InM4hrMefNXnDLvE9aSH9PzoDUYMTDcp9U9zroisTu4TAYsu3lQu84Vu4P/dKag2c7ie86aViNG+mvsskmxTCbDn8y1Tf/5LRMl1dqGbuBuND1HnUf8uTt4ubRJbyx3xKCJiKQsQVGlGDSZvhXa+kEb5qvBF4+MR2yELwor6zHvrRT8erGoxeO/NU9N3T6i+2SZRDcPCceMYRH487RB7Q4YR0UHItRXjcp6fbPX95tjpqm5mXFRdluNOGlQKIb38EeN1oD39l1y2+k56hwjevpDpZCjqKoemS1soO1OGDQRkcX0nKk/U8eCJgAI99Ng24OJSOgThMp6PRZtOoTvjjft8F9cVY/9F0wf+M7eSNcZNB4KvHl3PB6Z1L/d58rlMqnv0g/XdAevrNNhl3k1YnsbWrZGJpNJe9Jt+TVTmn5l0ETN0XgopJYlB7tA6wEGTUSEEB/TFJzWYERueR1OmqfTxrSzCPxa/p4e2HLfOMwYFgGtwYil/03Fll8zLI75/mQeDEYBw3v4o4+DNiXuysS6pp2n8mEwNkx//JCeD63eiP5hPhgS6dfS6Ta5eXA4YiN8UVWvR7F5Hzd3Wj1HnUtscnmYQRMRdQVqpQIBXh4AGj58I/016BHg2eHn1ngo8MYfRuPu63pBEICV36Tjnz+cleobpFVzcZGtPQ21ILFvMHw1ShRV1eNoVsOO8uKquVlx9l+NKJc3ZJtEzDRRSxqaXJa2caTrY9BERAAaMgX/O2GaQhsTE2S3D1uFXIYXZg3DU+aeU2/8dAHPfH4cV0prpJT9rd2wnskeVEo5boo1rcATV9EVVjZMedpzaq6xGcMi0T/MtD+ir1ppdQNU6n7iewVBJgMuF1Vb9IJzRwyaiAhAQ6ZADGLGdXBq7loymQyPTRmAtXOGQy4DPjl8BXPf/BWCAIzpHWiXrFZ31dB6IB+CIOC74zkwCqY9C3sHO2bKUyGXYelkU7YpOsjLIa9BXYO/lwcGhZsWObj7FJ1r765IRJ1GDJrEVcHt7QRurfnjeiHIW4XH/nsU+RWmb53dqTeTI0wcGAqVUo6skhqcyauUtrCxR2+m1swaGQUBAgaGd682EdR+Y2ICcSavEocySjFjuPtOxTPTREQAGvafAwBfjdKhH4RTh0bgoz8mwN/TA94qBW5x41+irsBbrcSNA0wb/L77y2WkZpVBLoO0D6CjyGQy3DGqJ4ZGdY9tb8h2XaXJJTNNRATAspA3vnegw3d7HxMThD1PT0Kd3sAiYjuYOjQCu04X4PPUKwCA8f1CEOancfKoiEzEoCk9pwLV9Xp4q90z/GCmiYgAWAZN9ugebY1AbxUi/VnLZA9Jg8PROM51VAE4kS2iAjzRI8ATBqOAo1llzh6OzRg0ERGAhq1UgM4Lmsh+grxVGGfuh6NSyjF9WISTR0RkSWw94M5NLhk0EREAINzPlGlSKeRSB19yLzPjegAwrabz03g4eTREltrT5LK8RoefzhQ4ekjt5p6TikRkd/1CffDgxL7oE+zNnjtuav7YaET4qx228pGoI8QM9tGsMugMRngoLPM2JdVa/Jieh+9P5mH/hSIYBAEH/jrFIgvubAyaiAiAaSXU8hmDnT0M6gC5XIabYsOdPQyiZvUP9YG/pwfKa3VIz6nAyOgAFFTW4Yf0fHx/IhcHLpdYbAU0MNwHuWV1DJqIiIioe5HLZRgbE4hdpwvwevJ5VNbpcSizROoNBwBDo/xwy/BITB8WgX6hPs4bbAsYNBEREVGnGBMThF2nC5DcqF5pZHQAZgyLwIxhkegV7Nrd5Rk0ERERUaeYGReFz45cQaCXB2YMM2WUotxoCyUGTURERNQpogI8sWvZRGcPw2ZsOUBERERkBQZNRERERFZg0ERERERkBQZNRERERFZg0ERERERkBQZNRERERFZg0ERERERkBQZNRERERFZg0ERERERkBQZNRERERFZg0ERERERkBQZNRERERFZg0ERERERkBQZNRERERFZQOnsA7koQBABARUWFk0dCRERE1hI/t8XP8fZg0GSjyspKAEB0dLSTR0JERETtVVlZCX9//3adIxNsCbUIRqMROTk58PX1hUwms9vzVlRUIDo6GtnZ2fDz87Pb81LreN2dg9fdOXjdnYPX3Tmuve6CIKCyshJRUVGQy9tXpcRMk43kcjl69uzpsOf38/Pj/1ROwOvuHLzuzsHr7hy87s7R+Lq3N8MkYiE4ERERkRUYNBERERFZgUGTi1Gr1Vi5ciXUarWzh9Kt8Lo7B6+7c/C6Owevu3PY87qzEJyIiIjICsw0EREREVmBQRMRERGRFRg0EREREVmBQRMRERGRFRg0uZgNGzYgJiYGGo0GCQkJOHjwoLOH1KX8/PPPuP322xEVFQWZTIavvvrK4nFBELBixQpERkbC09MTSUlJOH/+vHMG20WsWbMGY8eOha+vL8LCwjB79mycPXvW4pi6ujo8+uijCA4Oho+PD+bOnYv8/HwnjbhrePPNNzFixAipoV9iYiK+//576XFe886xdu1ayGQyPPHEE9J9vPb29/zzz0Mmk1ncYmNjpcftdc0ZNLmQbdu2YdmyZVi5ciVSU1MRFxeHadOmoaCgwNlD6zKqq6sRFxeHDRs2NPv4yy+/jPXr12Pjxo04cOAAvL29MW3aNNTV1XXySLuOvXv34tFHH8Vvv/2GnTt3QqfTYerUqaiurpaOefLJJ/Htt9/i008/xd69e5GTk4M5c+Y4cdTur2fPnli7di2OHDmCw4cP46abbsKsWbOQnp4OgNe8Mxw6dAhvvfUWRowYYXE/r71jDB06FLm5udJt37590mN2u+YCuYxx48YJjz76qPR3g8EgREVFCWvWrHHiqLouAMKXX34p/d1oNAoRERHCK6+8It1XVlYmqNVq4b///a8TRtg1FRQUCACEvXv3CoJgusYeHh7Cp59+Kh1z+vRpAYCQkpLirGF2SYGBgcK7777La94JKisrhQEDBgg7d+4UJk6cKPzpT38SBIE/746ycuVKIS4urtnH7HnNmWlyEVqtFkeOHEFSUpJ0n1wuR1JSElJSUpw4su7j8uXLyMvLs/g38Pf3R0JCAv8N7Ki8vBwAEBQUBAA4cuQIdDqdxXWPjY1Fr169eN3txGAwYOvWraiurkZiYiKveSd49NFHceutt1pcY4A/7450/vx5REVFoW/fvliwYAGysrIA2Peac8NeF1FUVASDwYDw8HCL+8PDw3HmzBknjap7ycvLA4Bm/w3Ex6hjjEYjnnjiCVx//fUYNmwYANN1V6lUCAgIsDiW173jTpw4gcTERNTV1cHHxwdffvklhgwZgrS0NF5zB9q6dStSU1Nx6NChJo/x590xEhISsHnzZgwaNAi5ublYtWoVJkyYgJMnT9r1mjNoIqJO8+ijj+LkyZMWtQbkOIMGDUJaWhrKy8vx2Wef4d5778XevXudPawuLTs7G3/605+wc+dOaDQaZw+n25gxY4b05xEjRiAhIQG9e/fGJ598Ak9PT7u9DqfnXERISAgUCkWTav78/HxEREQ4aVTdi3id+W/gGEuXLsX27dvx008/oWfPntL9ERER0Gq1KCsrszie173jVCoV+vfvj/j4eKxZswZxcXF47bXXeM0d6MiRIygoKMDo0aOhVCqhVCqxd+9erF+/HkqlEuHh4bz2nSAgIAADBw7EhQsX7PrzzqDJRahUKsTHxyM5OVm6z2g0Ijk5GYmJiU4cWffRp08fREREWPwbVFRU4MCBA/w36ABBELB06VJ8+eWX2L17N/r06WPxeHx8PDw8PCyu+9mzZ5GVlcXrbmdGoxH19fW85g40ZcoUnDhxAmlpadJtzJgxWLBggfRnXnvHq6qqwsWLFxEZGWnfn/cOFKuTnW3dulVQq9XC5s2bhVOnTgkPPPCAEBAQIOTl5Tl7aF1GZWWlcPToUeHo0aMCAGHdunXC0aNHhczMTEEQBGHt2rVCQECA8PXXXwvHjx8XZs2aJfTp00eora118sjd18MPPyz4+/sLe/bsEXJzc6VbTU2NdMxDDz0k9OrVS9i9e7dw+PBhITExUUhMTHTiqN3fs88+K+zdu1e4fPmycPz4ceHZZ58VZDKZ8OOPPwqCwGvemRqvnhMEXntHeOqpp4Q9e/YIly9fFvbv3y8kJSUJISEhQkFBgSAI9rvmDJpczOuvvy706tVLUKlUwrhx44TffvvN2UPqUn766ScBQJPbvffeKwiCqe3Ac889J4SHhwtqtVqYMmWKcPbsWecO2s01d70BCO+//750TG1trfDII48IgYGBgpeXl3DHHXcIubm5zht0F3DfffcJvXv3FlQqlRAaGipMmTJFCpgEgde8M10bNPHa29+8efOEyMhIQaVSCT169BDmzZsnXLhwQXrcXtdcJgiCYIdMGBEREVGXxpomIiIiIiswaCIiIiKyAoMmIiIiIiswaCIiIiKyAoMmIiIiIiswaCIiIiKyAoMmIiIiIiswaCIislJMTAxeffVVZw+DiJyEQRMRuaRFixZh9uzZAIBJkybhiSee6LTX3rx5MwICAprcf+jQITzwwAOdNg4ici1KZw+AiKizaLVaqFQqm88PDQ2142iIyN0w00RELm3RokXYu3cvXnvtNchkMshkMmRkZAAATp48iRkzZsDHxwfh4eG45557UFRUJJ07adIkLF26FE888QRCQkIwbdo0AMC6deswfPhweHt7Izo6Go888giqqqoAAHv27MHixYtRXl4uvd7zzz8PoOn0XFZWFmbNmgUfHx/4+fnhzjvvRH5+vvT4888/j5EjR+LDDz9ETEwM/P39MX/+fFRWVkrHfPbZZxg+fDg8PT0RHByMpKQkVFdXO+hqElFHMGgiIpf22muvITExEUuWLEFubi5yc3MRHR2NsrIy3HTTTRg1ahQOHz6MHTt2ID8/H3feeafF+Vu2bIFKpcL+/fuxceNGAIBcLsf69euRnp6OLVu2YPfu3fjLX/4CABg/fjxeffVV+Pn5Sa/39NNPNxmX0WjErFmzUFJSgr1792Lnzp24dOkS5s2bZ3HcxYsX8dVXX2H79u3Yvn079u7di7Vr1wIAcnNzcdddd+G+++7D6dOnsWfPHsyZMwfcEpTINXF6johcmr+/P1QqFby8vBARESHd/8Ybb2DUqFH4xz/+Id23adMmREdH49y5cxg4cCAAYMCAAXj55ZctnrNxfVRMTAxefPFFPPTQQ/j3v/8NlUoFf39/yGQyi9e7VnJyMk6cOIHLly8jOjoaAPDBBx9g6NChOHToEMaOHQvAFFxt3rwZvr6+AIB77rkHycnJ+Pvf/47c3Fzo9XrMmTMHvXv3BgAMHz68A1eLiByJmSYickvHjh3DTz/9BB8fH+kWGxsLwJTdEcXHxzc5d9euXZgyZQp69OgBX19f3HPPPSguLkZNTY3Vr3/69GlER0dLARMADBkyBAEBATh9+rR0X0xMjBQwAUBkZCQKCgoAAHFxcZgyZQqGDx+O3//+93jnnXdQWlpq/UUgok7FoImI3FJVVRVuv/12pKWlWdzOnz+PG2+8UTrO29vb4ryMjAzcdtttGDFiBD7//HMcOXIEGzZsAGAqFLc3Dw8Pi7/LZDIYjUYAgEKhwM6dO/H9999jyJAheP311zFo0CBcvnzZ7uMgoo5j0ERELk+lUsFgMFjcN3r0aKSnpyMmJgb9+/e3uF0bKDV25MgRGI1G/Otf/8J1112HgQMHIicnp83Xu9bgwYORnZ2N7Oxs6b5Tp06hrKwMQ4YMsfq9yWQyXH/99Vi1ahWOHj0KlUqFL7/80urziajzMGgiIpcXExODAwcOICMjA0VFRTAajXj00UdRUlKCu+66C4cOHcLFixfxww8/YPHixa0GPP3794dOp8Prr7+OS5cu4cMPP5QKxBu/XlVVFZKTk1FUVNTstF1SUhKGDx+OBQsWIDU1FQcPHsTChQsxceJEjBkzxqr3deDAAfzjH//A4cOHkZWVhS+++AKFhYUYPHhw+y4QEXUKBk1E5PKefvppKBQKDBkyBKGhocjKykJUVBT2798Pg8GAqVOnYvjw4XjiiScQEBAAubzlX21xcXFYt24dXnrpJQwbNgwfffQR1qxZY3HM+PHj8dBDD2HevHkIDQ1tUkgOmDJEX3/9NQIDA3HjjTciKSkJffv2xbZt26x+X35+fvj5559xyy23YODAgfjb3/6Gf/3rX5gxY4b1F4eIOo1M4NpWIiIiojYx00RERERkBQZNRERERFZg0ERERERkBQZNRERERFZg0ERERERkBQZNRERERFZg0ERERERkBQZNRERERFZg0ERERERkBQZNRERERFZg0ERERERkBQZNRERERFb4/4ITHGf98qkAAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "interval = 400\n",
    "# Assuming train_loss_list contains the loss values from your training\n",
    "plt.plot([np.nanmean(train_loss_list[i*interval:(i+1)*interval]) for i in range(len(train_loss_list)//interval)])\n",
    "plt.xlabel('Iterations')\n",
    "plt.ylabel('Loss')\n",
    "plt.title('Training Loss')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "jyRpP_zeQcmR"
   },
   "source": [
    "## 3.7、保存模型"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "xvMdn1CUQIBd"
   },
   "outputs": [],
   "source": [
    "torch.save(model.state_dict(),f\"./{model_name}_SFT.pth\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "jPANW5-DQyAa"
   },
   "source": [
    "# 4、测试模型问答效果"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "W9ESxiAmXqOK"
   },
   "outputs": [],
   "source": [
    "from typing import List, Dict\n",
    "\n",
    "def new_apply_chat_template(history:List[Dict[str, str]], add_reasoning_generation_prompt:bool=True, add_assistant_generation_prompt:bool=False):\n",
    "  if add_reasoning_generation_prompt:\n",
    "    return \"\".join([f\"<|im_start|>{i['role']}\\n{i['content']}<|im_end|>\\n\" for i in history]) + \"<|im_start|><|reasoning|>\\n\"\n",
    "  if add_assistant_generation_prompt:\n",
    "    return \"\".join([f\"<|im_start|>{i['role']}\\n{i['content']}<|im_end|>\\n\" for i in history]) + \"<|im_start|>assistant\\n\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "ysqrsY-Men8h",
    "outputId": "64e3d86d-7e12-4bf4-c4b0-6290f4cf319f"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'eos_token': '<|endoftext|>',\n",
       " 'pad_token': '<|endoftext|>',\n",
       " 'additional_special_tokens': ['<|reasoning|>']}"
      ]
     },
     "execution_count": 46,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tokenizer.special_tokens_map"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 1000
    },
    "id": "T7mWVAFDQ2p5",
    "outputId": "00e9d8b0-d06a-4444-ec12-a7276441e0fe"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "User：\n",
      "Please find all prime numbers within 0-10\n",
      "Please find all prime numbers within 0-10\n",
      "\n",
      "\n",
      "reasoning:\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/markdown": [
       "To tackle this problem, let's break it down into manageable parts and reason through each step.\n",
       "\n",
       "**Understanding the Problem**\n",
       "\n",
       "The user is asking for all prime numbers within the range of 0 to 10. A prime number is a natural number greater than 1 that has no positive divisors other than 1 and itself. In other words, a prime number can only be divided evenly by 1 and itself.\n",
       "\n",
       "**Breaking Down the Problem**\n",
       "\n",
       "To find all prime numbers within the given range, we need to consider each number from 0 to 10 and determine if it meets the criteria of being a prime number.\n",
       "\n",
       "**Step 1: Identify the Range**\n",
       "\n",
       "The range is from 0 to 10. This means we need to check each number in this range to see if it's a prime number.\n",
       "\n",
       "**Step 2: Understand Prime Numbers**\n",
       "\n",
       "A prime number is a natural number greater than 1 that has no positive divisors other than 1 and itself. This means that a prime number cannot be divided evenly by any other number except 1 and itself.\n",
       "\n",
       "**Step 3: Check Each Number**\n",
       "\n",
       "Let's start checking each number from 0 to 10:\n",
       "\n",
       "* 0: Not a prime number (0 is not a natural number)\n",
       "* 1: Not a prime number (1 is not greater than 1)\n",
       "* 2: Prime number (only divisible by 1 and itself)\n",
       "* 3: Prime number (only divisible by 1 and itself)\n",
       "* 4: Not a prime number (divisible by 2)\n",
       "* 5: Prime number (only divisible by 1 and itself)\n",
       "* 6: Not a prime number (divisible by 2 and 3)\n",
       "* 7: Prime number (only divisible by 1 and itself)\n",
       "* 8: Not a prime number (divisible by 2)\n",
       "* 9: Not a prime number (divisible by 3)\n",
       "* 10: Not a prime number (divisible by 2 and 5)\n",
       "\n",
       "**Step 4: Collect the Prime Numbers**\n",
       "\n",
       "From the above checks, we can collect the prime numbers:\n",
       "\n",
       "* 2\n",
       "* 3\n",
       "* 5\n",
       "* 7\n",
       "\n",
       "**Constraints and Considerations**\n",
       "\n",
       "* We are only considering natural numbers greater than 1.\n",
       "* We are looking for prime numbers, which means they have no positive divisors other than 1 and themselves.\n",
       "\n",
       "**Implementation Details**\n",
       "\n",
       "We can implement this solution using a simple loop to iterate through the numbers from 0 to 10 and check if each number is prime.\n",
       "\n",
       "Here's some sample code in Python:\n",
       "```python\n",
       "def is_prime(n):\n",
       "    if n <= 1:\n",
       "        return False\n",
       "    for i in range(2, int(n**0.5) + 1):\n",
       "        if n % i == 0:\n",
       "            return False\n",
       "    return True\n",
       "\n",
       "primes = []\n",
       "for num in range(10):\n",
       "    if is_prime(num):\n",
       "        primes.append(num)\n",
       "\n",
       "print(primes)\n",
       "```\n",
       "This code defines a function `is_prime` that checks if a number is prime by iterating through all possible divisors up to the square root of the number. It then collects all prime numbers in a list and prints them out.\n",
       "\n",
       "**Self-Reflection**\n",
       "\n",
       "In this solution, we used a simple loop to iterate through the numbers from 0 to 10 and check if each number is prime. We also used a helper function `is_prime` to perform the actual primality test.\n",
       "\n",
       "One potential mistake to watch out for is forgetting to include 0 and 1 in our range, as they are not considered prime numbers. Additionally, we could improve the efficiency of our solution by using a more optimized primality test, such as the Sieve of Eratosthenes.\n",
       "\n",
       "Overall, this solution provides a clear and concise way to find all prime numbers within a given range, and it can be easily adapted to different programming languages and use cases."
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "------------\n",
      "\n",
      "\n",
      "assistant:\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/markdown": [
       "To find all prime numbers within 0-10, we can follow these steps:\n",
       "\n",
       "1. Initialize an empty list to store the prime numbers.\n",
       "2. Iterate through each number from 0 to 10.\n",
       "3. For each number, check if it is prime.\n",
       "4. If the number is prime, add it to the list of prime numbers.\n",
       "5. Print the list of prime numbers.\n",
       "\n",
       "Here is a sample implementation in Python:\n",
       "```python\n",
       "def is_prime(n):\n",
       "    if n <= 1:\n",
       "        return False\n",
       "    for i in range(2, int(n**0.5) + 1):\n",
       "        if n % i == 0:\n",
       "            return False\n",
       "    return True\n",
       "\n",
       "primes = []\n",
       "for num in range(10):\n",
       "    if is_prime(num):\n",
       "        primes.append(num)\n",
       "\n",
       "print(primes)\n",
       "```\n",
       "Output:\n",
       "```\n",
       "[2, 3, 5, 7]\n",
       "```\n",
       "Therefore, the prime numbers between 0 and 10 are 2, 3, 5, and 7."
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "------------\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "Interrupted by user",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-78-a678bbfd9ab5>\u001b[0m in \u001b[0;36m<cell line: 5>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      4\u001b[0m \u001b[0mhistory\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m{\u001b[0m\u001b[0;34m\"role\"\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0;34m\"system\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m\"content\"\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0;34m\"You are a helpful assistant\"\u001b[0m\u001b[0;34m}\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      5\u001b[0m \u001b[0;32mwhile\u001b[0m \u001b[0;32mTrue\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 6\u001b[0;31m     \u001b[0mquestion\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'User：'\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0;34m'\\n'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      7\u001b[0m     \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mquestion\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      8\u001b[0m     \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'\\n'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/ipykernel/kernelbase.py\u001b[0m in \u001b[0;36mraw_input\u001b[0;34m(self, prompt)\u001b[0m\n\u001b[1;32m    849\u001b[0m                 \u001b[0;34m\"raw_input was called, but this frontend does not support input requests.\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    850\u001b[0m             )\n\u001b[0;32m--> 851\u001b[0;31m         return self._input_request(str(prompt),\n\u001b[0m\u001b[1;32m    852\u001b[0m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_parent_ident\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    853\u001b[0m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_parent_header\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/ipykernel/kernelbase.py\u001b[0m in \u001b[0;36m_input_request\u001b[0;34m(self, prompt, ident, parent, password)\u001b[0m\n\u001b[1;32m    893\u001b[0m             \u001b[0;32mexcept\u001b[0m \u001b[0mKeyboardInterrupt\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    894\u001b[0m                 \u001b[0;31m# re-raise KeyboardInterrupt, to truncate traceback\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 895\u001b[0;31m                 \u001b[0;32mraise\u001b[0m \u001b[0mKeyboardInterrupt\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"Interrupted by user\"\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    896\u001b[0m             \u001b[0;32mexcept\u001b[0m \u001b[0mException\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    897\u001b[0m                 \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlog\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mwarning\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"Invalid Message:\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mexc_info\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: Interrupted by user"
     ]
    }
   ],
   "source": [
    "from IPython.display import Markdown, display\n",
    "\n",
    "history = []\n",
    "history.append({\"role\": \"system\", \"content\": \"You are a helpful assistant\"})\n",
    "while True:\n",
    "    question = input('User：' + '\\n')\n",
    "    print(question)\n",
    "    print('\\n')\n",
    "    history.append({\"role\": \"user\", \"content\": question})\n",
    "\n",
    "    input_text = new_apply_chat_template(\n",
    "            history,\n",
    "            add_reasoning_generation_prompt=True\n",
    "        )\n",
    "    model_inputs = tokenizer([input_text], return_tensors=\"pt\").to(device)\n",
    "\n",
    "    if model_inputs.input_ids.size()[1]>32000:\n",
    "        break\n",
    "\n",
    "    generated_ids = model.generate(\n",
    "        model_inputs.input_ids,\n",
    "        max_new_tokens=3000\n",
    "    )\n",
    "\n",
    "    if len(generated_ids)>32000:\n",
    "        break\n",
    "\n",
    "    generated_ids = [output_ids[len(input_ids):] for input_ids, output_ids in zip(model_inputs.input_ids, generated_ids)]\n",
    "\n",
    "    reasoning_response = tokenizer.batch_decode(generated_ids, skip_special_tokens=True)[0]\n",
    "    history.append({\"role\": \"<|reasoning|>\", \"content\": reasoning_response})\n",
    "    print('reasoning:\\n')\n",
    "    #print(response)\n",
    "    display(Markdown(reasoning_response))\n",
    "    print(\"------------\")\n",
    "    print('\\n')\n",
    "\n",
    "    input_text = new_apply_chat_template(\n",
    "            history,\n",
    "            add_assistant_generation_prompt=True\n",
    "        )\n",
    "    model_inputs = tokenizer([input_text], return_tensors=\"pt\").to(device)\n",
    "\n",
    "    if model_inputs.input_ids.size()[1]>32000:\n",
    "        break\n",
    "\n",
    "    generated_ids = model.generate(\n",
    "        model_inputs.input_ids,\n",
    "        max_new_tokens=3000\n",
    "    )\n",
    "\n",
    "    if len(generated_ids)>32000:\n",
    "        break\n",
    "\n",
    "    generated_ids = [output_ids[len(input_ids):] for input_ids, output_ids in zip(model_inputs.input_ids, generated_ids)]\n",
    "\n",
    "    assistant_response = tokenizer.batch_decode(generated_ids, skip_special_tokens=True)[0]\n",
    "    history.append({\"role\": \"assistant\", \"content\": assistant_response})\n",
    "    print('assistant:\\n')\n",
    "    display(Markdown(assistant_response))\n",
    "    print(\"------------\")\n",
    "\n",
    "print(\"超过模型字数上线，已退出\")"
   ]
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "gpuType": "A100",
   "machine_shape": "hm",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.8"
  },
  "widgets": {
   "application/vnd.jupyter.widget-state+json": {
    "002465c05b71462198afe7ffee5f23e6": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_3bfcfec6942b4b32a46a9b2c43227d8d",
      "placeholder": "​",
      "style": "IPY_MODEL_95ad4875749843a2b1cf08987c682df9",
      "value": "Map: 100%"
     }
    },
    "006b088e27734f77a03883739f69042b": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "00cfba85032949a98b31f4bffbb89274": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_01737b33fe7e4aef9bdfa46fb5352daf",
      "placeholder": "​",
      "style": "IPY_MODEL_4d206ca710734371a9ca172f1fbd0f79",
      "value": " 1.79k/1.79k [00:00&lt;00:00, 130kB/s]"
     }
    },
    "01143789eb8f46379f9ce7b3e51ba846": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "01737b33fe7e4aef9bdfa46fb5352daf": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "07e127b8f0244ee78c3b5014cf311e91": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "ProgressStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "ProgressStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "bar_color": null,
      "description_width": ""
     }
    },
    "0855ea4d4a684f53acb099b20bbd2606": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "09753c16b5c949d987236ee940ec5fa3": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "ProgressStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "ProgressStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "bar_color": null,
      "description_width": ""
     }
    },
    "0a575b3a5a5140b680ff8e20676f3dce": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "FloatProgressModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "FloatProgressModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "ProgressView",
      "bar_style": "success",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_611af89de8ee4108adb8fccb5bb9cf32",
      "max": 684,
      "min": 0,
      "orientation": "horizontal",
      "style": "IPY_MODEL_438b937befce4e2d9d49edca9d77d7d2",
      "value": 684
     }
    },
    "0b2c49f831304163acc0d2f4cf0089b0": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "0c6887b9851644a684659752c7375885": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_45f1f88bf6424096a522d364a1aa458b",
      "placeholder": "​",
      "style": "IPY_MODEL_ddbc147f7ec34bbc9a1b0f0721f6451b",
      "value": "vocab.json: 100%"
     }
    },
    "0f34d6fa253444e1a73c240cc5bdb62e": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "1116b93b89eb420490a12deacfb06c21": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HBoxModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HBoxModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HBoxView",
      "box_style": "",
      "children": [
       "IPY_MODEL_2f401b1fcb8e4c969dac72bf215f6561",
       "IPY_MODEL_c79df3e8fa824c8687d2bfaa57e7fe01",
       "IPY_MODEL_415e8e498c2a4afc8e85d15a72f2fd73"
      ],
      "layout": "IPY_MODEL_9bfa854836e249d79617969ac207754b"
     }
    },
    "137dc35b0c1e4b9ab04d0d8ec5ce7dac": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "ProgressStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "ProgressStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "bar_color": null,
      "description_width": ""
     }
    },
    "138e15d45660471baff2f09c93efe771": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "14637cdcc5d642b0989bf887100601a2": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HBoxModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HBoxModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HBoxView",
      "box_style": "",
      "children": [
       "IPY_MODEL_cf98d52df6b9432a9877c8b5355c82e6",
       "IPY_MODEL_e9ae9ba627aa4bf8a4eb64d68809f566",
       "IPY_MODEL_d5ec0c9ab8d6453fb32e867102ee0eaf"
      ],
      "layout": "IPY_MODEL_2434babf7137418b84dcf59553f33a3a"
     }
    },
    "1571ba54beca47a69ef597cb15df214b": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "18d67eed7b8f407da269ec244cd04abb": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "197f61d70ab541aa9a1b1c1298435b2b": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "1c0ebb52daab43a7b316a8460535e897": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "1c6b9b798563409cad0bd52e2ba423a9": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "ProgressStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "ProgressStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "bar_color": null,
      "description_width": ""
     }
    },
    "1f2f325f8f66426795de63fbed7b606b": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_930dd8bebebc475795350db5405a591e",
      "placeholder": "​",
      "style": "IPY_MODEL_cb748c092e4e4917ac1dbb64dbf2e14d",
      "value": "tokenizer.json: 100%"
     }
    },
    "1f47d0fcdac84c0986ac056b480c0dc7": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "20a0b51901a644cd8318eccf8219b577": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HBoxModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HBoxModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HBoxView",
      "box_style": "",
      "children": [
       "IPY_MODEL_95aece1615ac4f4fa568a77b4a0d0e1a",
       "IPY_MODEL_6ece5bd8907b43248feb86065e8a2ea5",
       "IPY_MODEL_9a9f863626b54cfebb48efe95142ebb2"
      ],
      "layout": "IPY_MODEL_6a26f4f9406946628786a62cebd5a867"
     }
    },
    "2176b48b1f454b249d638f9f520e274b": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "ProgressStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "ProgressStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "bar_color": null,
      "description_width": ""
     }
    },
    "24028c99b8a04fab8c122ab54412ea54": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_ef94fa10df1049eaa598e77851c05411",
      "placeholder": "​",
      "style": "IPY_MODEL_b7f1116dd979421f986a3289bf98ad19",
      "value": " 7.03M/7.03M [00:00&lt;00:00, 47.1MB/s]"
     }
    },
    "2434babf7137418b84dcf59553f33a3a": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "256ace0d05ae489fad26cdbac9fd3502": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "25843be9e67346f5a9faf475a9d396ad": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "28784f7543294c49b54a09c908b56c9e": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HBoxModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HBoxModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HBoxView",
      "box_style": "",
      "children": [
       "IPY_MODEL_0c6887b9851644a684659752c7375885",
       "IPY_MODEL_68d87d30b17c4970bcf48873fb6c9247",
       "IPY_MODEL_d7e56c2c917648c1b8443247f8b49be3"
      ],
      "layout": "IPY_MODEL_18d67eed7b8f407da269ec244cd04abb"
     }
    },
    "2b05d0a1e27f4c4094ffb7801ad11076": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "2c3d43d5b1d34d3b963bf2ed582620ba": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HBoxModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HBoxModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HBoxView",
      "box_style": "",
      "children": [
       "IPY_MODEL_a4ac3a557f2347b2b828f0c966a4d4fe",
       "IPY_MODEL_bb94ba3e0f714a8e9b9b61c134e2f74c",
       "IPY_MODEL_d2c8b1dc82a5473ab6f38bcc0430c981"
      ],
      "layout": "IPY_MODEL_82a94b011a904f2884f6f8206064ed89"
     }
    },
    "2c9c1b6e18fa4cd9979d5f13f41adf48": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "2f401b1fcb8e4c969dac72bf215f6561": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_6d3fa20d73e2496d81cf7fa17b8efc6c",
      "placeholder": "​",
      "style": "IPY_MODEL_afdcb773c48f419a92b7231f3a07255f",
      "value": "merges.txt: 100%"
     }
    },
    "32767a4e19fd4bbe9887b6a54c916893": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "33113930b3a84366bb5f5b60e40f234e": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "3460c7f3c7274fc4ba06c8b6675bb37a": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "FloatProgressModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "FloatProgressModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "ProgressView",
      "bar_style": "success",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_fdca97d2f6e8424eb842a161e5fd5835",
      "max": 1791,
      "min": 0,
      "orientation": "horizontal",
      "style": "IPY_MODEL_fa9b41c17c67430b99b3bf763b8e463d",
      "value": 1791
     }
    },
    "38b6585254aa4332b1cd6b6c64432263": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_95de1ad519ec479797c56a59ae13826a",
      "placeholder": "​",
      "style": "IPY_MODEL_87336531a7d449ea8e73b1707e921ccf",
      "value": " 307M/307M [00:01&lt;00:00, 245MB/s]"
     }
    },
    "39ae78eff3714fbfa1ef5ef006e45b5a": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "3bfcfec6942b4b32a46a9b2c43227d8d": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "415e8e498c2a4afc8e85d15a72f2fd73": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_0f34d6fa253444e1a73c240cc5bdb62e",
      "placeholder": "​",
      "style": "IPY_MODEL_01143789eb8f46379f9ce7b3e51ba846",
      "value": " 1.67M/1.67M [00:00&lt;00:00, 29.6MB/s]"
     }
    },
    "438b937befce4e2d9d49edca9d77d7d2": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "ProgressStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "ProgressStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "bar_color": null,
      "description_width": ""
     }
    },
    "45f1f88bf6424096a522d364a1aa458b": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "4d206ca710734371a9ca172f1fbd0f79": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "4d95e63b84494846a31ddc20a92d06de": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HBoxModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HBoxModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HBoxView",
      "box_style": "",
      "children": [
       "IPY_MODEL_8978e7bba2364d1d802f2a7c7560297b",
       "IPY_MODEL_3460c7f3c7274fc4ba06c8b6675bb37a",
       "IPY_MODEL_00cfba85032949a98b31f4bffbb89274"
      ],
      "layout": "IPY_MODEL_f9c966f2b73e4f3da9163dd14ed16363"
     }
    },
    "514e06329caa4fbb963a382a3e894054": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "55bb9e5806eb422197e7d19e62305a2a": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "ProgressStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "ProgressStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "bar_color": null,
      "description_width": ""
     }
    },
    "58127f0348e94a9582de29fb9bcc7f92": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "ProgressStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "ProgressStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "bar_color": null,
      "description_width": ""
     }
    },
    "58f28acb0be04593accf297e5e06bed0": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "5ce999cad3d84a00af2a479a86914bb1": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_5e80ecdc0e75417285f8c45785a3d9ed",
      "placeholder": "​",
      "style": "IPY_MODEL_7563d173e09d4c6eac618aaa1269a8fd",
      "value": " 3.09G/3.09G [00:13&lt;00:00, 181MB/s]"
     }
    },
    "5e80ecdc0e75417285f8c45785a3d9ed": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "5ebd1a93c8f744629cb6ba95289cedd8": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "5f701e7d23f346859b2fd0f885f2c58f": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HBoxModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HBoxModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HBoxView",
      "box_style": "",
      "children": [
       "IPY_MODEL_e30e11637be7473b873835e115517853",
       "IPY_MODEL_0a575b3a5a5140b680ff8e20676f3dce",
       "IPY_MODEL_f4c353c0fe944932bcf7879c7cd3dd2a"
      ],
      "layout": "IPY_MODEL_7831b1d7c9054f3faa8f7e280f3676b9"
     }
    },
    "611af89de8ee4108adb8fccb5bb9cf32": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "62d159e5720f496c82312d9efd4bfe79": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "6336f105491b4bd8ae01c0e3b6940c36": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "6365c0f5fc35484dba06f88fbc8f0656": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "65073b78db274217b2f9800549cdcd2d": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "6731988b1bb74f8ebe1a35c0196bee7b": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "FloatProgressModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "FloatProgressModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "ProgressView",
      "bar_style": "success",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_ca3c78a681834bc4a6fb26dd1e9c1f8d",
      "max": 19944,
      "min": 0,
      "orientation": "horizontal",
      "style": "IPY_MODEL_07e127b8f0244ee78c3b5014cf311e91",
      "value": 19944
     }
    },
    "68d87d30b17c4970bcf48873fb6c9247": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "FloatProgressModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "FloatProgressModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "ProgressView",
      "bar_style": "success",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_9b977ee086ed49d38325d4fbb005d664",
      "max": 2776833,
      "min": 0,
      "orientation": "horizontal",
      "style": "IPY_MODEL_09753c16b5c949d987236ee940ec5fa3",
      "value": 2776833
     }
    },
    "69b75a00c15f46568b6caf44e6b72384": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "6a26f4f9406946628786a62cebd5a867": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "6c57207ee15e46488f3b7d8a206e6fe6": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_8142a04b596d484f87c2620cb1afab2f",
      "placeholder": "​",
      "style": "IPY_MODEL_b6ee43871869434d9cc65504cba5cac7",
      "value": "model.safetensors: 100%"
     }
    },
    "6d3fa20d73e2496d81cf7fa17b8efc6c": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "6ece5bd8907b43248feb86065e8a2ea5": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "FloatProgressModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "FloatProgressModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "ProgressView",
      "bar_style": "success",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_7b27d475ea6544d19df5d4c1291539b5",
      "max": 19944,
      "min": 0,
      "orientation": "horizontal",
      "style": "IPY_MODEL_137dc35b0c1e4b9ab04d0d8ec5ce7dac",
      "value": 19944
     }
    },
    "7563d173e09d4c6eac618aaa1269a8fd": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "756f373c43b64c1ba4f9668d91d059ec": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "76d6fcf7d45649a3947ef199beccac83": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "7831b1d7c9054f3faa8f7e280f3676b9": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "7ad6a66d01684ca3bd845c5cb8e5825a": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "7b27d475ea6544d19df5d4c1291539b5": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "7b48926e24d448d38810001834020b55": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "7dfab99d13504e76a3fbed258f1933b4": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_256ace0d05ae489fad26cdbac9fd3502",
      "placeholder": "​",
      "style": "IPY_MODEL_1571ba54beca47a69ef597cb15df214b",
      "value": " 138/138 [00:00&lt;00:00, 12.1kB/s]"
     }
    },
    "8142a04b596d484f87c2620cb1afab2f": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "8164fbc2518641558417b26eaa2fb3be": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "ProgressStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "ProgressStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "bar_color": null,
      "description_width": ""
     }
    },
    "82a94b011a904f2884f6f8206064ed89": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "851e5249a3474421a1c527c3f35545b5": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "87336531a7d449ea8e73b1707e921ccf": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "8978e7bba2364d1d802f2a7c7560297b": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_851e5249a3474421a1c527c3f35545b5",
      "placeholder": "​",
      "style": "IPY_MODEL_65073b78db274217b2f9800549cdcd2d",
      "value": "README.md: 100%"
     }
    },
    "8ca7b55e3b8f40668eb4abd2892971df": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "FloatProgressModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "FloatProgressModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "ProgressView",
      "bar_style": "success",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_a198338d2c6945699da654673bf9b745",
      "max": 138,
      "min": 0,
      "orientation": "horizontal",
      "style": "IPY_MODEL_8164fbc2518641558417b26eaa2fb3be",
      "value": 138
     }
    },
    "8e216fa2d97d4b638e046076ebf2ec2f": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "8eaf3b02c5eb413a92308be974b31b49": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "9128a37a06ce4b269b2e60a29710c181": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "930dd8bebebc475795350db5405a591e": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "938d14371d034e859cba003bab3c0cc5": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "95ad4875749843a2b1cf08987c682df9": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "95aece1615ac4f4fa568a77b4a0d0e1a": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_8e216fa2d97d4b638e046076ebf2ec2f",
      "placeholder": "​",
      "style": "IPY_MODEL_9128a37a06ce4b269b2e60a29710c181",
      "value": "Saving the dataset (1/1 shards): 100%"
     }
    },
    "95de1ad519ec479797c56a59ae13826a": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "9a33cd22c9cc4c34a2e60a87c37d8eaa": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "FloatProgressModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "FloatProgressModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "ProgressView",
      "bar_style": "success",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_006b088e27734f77a03883739f69042b",
      "max": 307078005,
      "min": 0,
      "orientation": "horizontal",
      "style": "IPY_MODEL_1c6b9b798563409cad0bd52e2ba423a9",
      "value": 307078005
     }
    },
    "9a9f863626b54cfebb48efe95142ebb2": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_ea3f8d8660a942fba64236fbc2cbd787",
      "placeholder": "​",
      "style": "IPY_MODEL_c7ba627ffac54eb89180843c8ad721bb",
      "value": " 19944/19944 [00:00&lt;00:00, 66386.15 examples/s]"
     }
    },
    "9b977ee086ed49d38325d4fbb005d664": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "9bfa854836e249d79617969ac207754b": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "9dd40131eecd42a79519b897749b39e6": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_7b48926e24d448d38810001834020b55",
      "placeholder": "​",
      "style": "IPY_MODEL_b9e2cc6fde514d5dbb26c936d1cdfed2",
      "value": " 19944/19944 [00:04&lt;00:00, 4612.92 examples/s]"
     }
    },
    "a198338d2c6945699da654673bf9b745": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "a2b2494c19764c22908fa7247a1fc02c": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HBoxModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HBoxModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HBoxView",
      "box_style": "",
      "children": [
       "IPY_MODEL_deb2379001af474cabbeca1df3c7dc95",
       "IPY_MODEL_8ca7b55e3b8f40668eb4abd2892971df",
       "IPY_MODEL_7dfab99d13504e76a3fbed258f1933b4"
      ],
      "layout": "IPY_MODEL_2b05d0a1e27f4c4094ffb7801ad11076"
     }
    },
    "a4435caad91a4ab6ac25ebb860a1108e": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "a4ac3a557f2347b2b828f0c966a4d4fe": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_33113930b3a84366bb5f5b60e40f234e",
      "placeholder": "​",
      "style": "IPY_MODEL_1f47d0fcdac84c0986ac056b480c0dc7",
      "value": "Generating train split: 100%"
     }
    },
    "a820971e767f4c49a157e5ff2db82c6f": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "FloatProgressModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "FloatProgressModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "ProgressView",
      "bar_style": "success",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_6365c0f5fc35484dba06f88fbc8f0656",
      "max": 7031645,
      "min": 0,
      "orientation": "horizontal",
      "style": "IPY_MODEL_aba50b35a10e4f6892742eed763a8890",
      "value": 7031645
     }
    },
    "aba50b35a10e4f6892742eed763a8890": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "ProgressStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "ProgressStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "bar_color": null,
      "description_width": ""
     }
    },
    "afdcb773c48f419a92b7231f3a07255f": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "b6ee43871869434d9cc65504cba5cac7": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "b7f1116dd979421f986a3289bf98ad19": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "b83f73dd875742459448f70c4d98eb0c": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_0b2c49f831304163acc0d2f4cf0089b0",
      "placeholder": "​",
      "style": "IPY_MODEL_514e06329caa4fbb963a382a3e894054",
      "value": "Map: 100%"
     }
    },
    "b9e2cc6fde514d5dbb26c936d1cdfed2": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "bb94ba3e0f714a8e9b9b61c134e2f74c": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "FloatProgressModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "FloatProgressModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "ProgressView",
      "bar_style": "success",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_f46cdc40f11349108d92539de4185b33",
      "max": 19944,
      "min": 0,
      "orientation": "horizontal",
      "style": "IPY_MODEL_55bb9e5806eb422197e7d19e62305a2a",
      "value": 19944
     }
    },
    "bcbd9e82d1ed4ad6b51e8699a6087041": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "bd0aab37191745f09f540c2f9fa70f52": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HBoxModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HBoxModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HBoxView",
      "box_style": "",
      "children": [
       "IPY_MODEL_002465c05b71462198afe7ffee5f23e6",
       "IPY_MODEL_c5bd5af525c94d57ace64357c7ddb5f2",
       "IPY_MODEL_fbb1c9b7489242619cf2dc3bc0476ddd"
      ],
      "layout": "IPY_MODEL_8eaf3b02c5eb413a92308be974b31b49"
     }
    },
    "c1e063e9f8174863b4acd5a553b5a71d": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "c453996f237f43f38d02ed0c3a808bc9": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "FloatProgressModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "FloatProgressModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "ProgressView",
      "bar_style": "success",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_1c0ebb52daab43a7b316a8460535e897",
      "max": 3087467144,
      "min": 0,
      "orientation": "horizontal",
      "style": "IPY_MODEL_f8796e310a9548528122a3b20aab3582",
      "value": 3087467144
     }
    },
    "c4e994712aec44faa6204cdeb803c131": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "c58771d16f554bc68b783da40043281f": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HBoxModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HBoxModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HBoxView",
      "box_style": "",
      "children": [
       "IPY_MODEL_b83f73dd875742459448f70c4d98eb0c",
       "IPY_MODEL_6731988b1bb74f8ebe1a35c0196bee7b",
       "IPY_MODEL_9dd40131eecd42a79519b897749b39e6"
      ],
      "layout": "IPY_MODEL_f8b66aed4c76404ea165db0c0dc091bf"
     }
    },
    "c5bd5af525c94d57ace64357c7ddb5f2": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "FloatProgressModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "FloatProgressModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "ProgressView",
      "bar_style": "success",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_7ad6a66d01684ca3bd845c5cb8e5825a",
      "max": 19944,
      "min": 0,
      "orientation": "horizontal",
      "style": "IPY_MODEL_2176b48b1f454b249d638f9f520e274b",
      "value": 19944
     }
    },
    "c79df3e8fa824c8687d2bfaa57e7fe01": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "FloatProgressModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "FloatProgressModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "ProgressView",
      "bar_style": "success",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_c4e994712aec44faa6204cdeb803c131",
      "max": 1671839,
      "min": 0,
      "orientation": "horizontal",
      "style": "IPY_MODEL_58127f0348e94a9582de29fb9bcc7f92",
      "value": 1671839
     }
    },
    "c7ba627ffac54eb89180843c8ad721bb": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "ca3c78a681834bc4a6fb26dd1e9c1f8d": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "cb748c092e4e4917ac1dbb64dbf2e14d": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "cdeaa1d0b6f54adcab53755eaeed3c71": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HBoxModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HBoxModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HBoxView",
      "box_style": "",
      "children": [
       "IPY_MODEL_1f2f325f8f66426795de63fbed7b606b",
       "IPY_MODEL_a820971e767f4c49a157e5ff2db82c6f",
       "IPY_MODEL_24028c99b8a04fab8c122ab54412ea54"
      ],
      "layout": "IPY_MODEL_938d14371d034e859cba003bab3c0cc5"
     }
    },
    "cf98d52df6b9432a9877c8b5355c82e6": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_197f61d70ab541aa9a1b1c1298435b2b",
      "placeholder": "​",
      "style": "IPY_MODEL_e4c2626eabb84dc78273acfffcef82bd",
      "value": "tokenizer_config.json: 100%"
     }
    },
    "d2c8b1dc82a5473ab6f38bcc0430c981": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_58f28acb0be04593accf297e5e06bed0",
      "placeholder": "​",
      "style": "IPY_MODEL_df94c5dc68864f8db8213f00e42af793",
      "value": " 19944/19944 [00:03&lt;00:00, 6681.31 examples/s]"
     }
    },
    "d473f10ea00548428718b827ba7217db": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "d5ec0c9ab8d6453fb32e867102ee0eaf": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_756f373c43b64c1ba4f9668d91d059ec",
      "placeholder": "​",
      "style": "IPY_MODEL_76d6fcf7d45649a3947ef199beccac83",
      "value": " 7.23k/7.23k [00:00&lt;00:00, 638kB/s]"
     }
    },
    "d7e56c2c917648c1b8443247f8b49be3": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_bcbd9e82d1ed4ad6b51e8699a6087041",
      "placeholder": "​",
      "style": "IPY_MODEL_5ebd1a93c8f744629cb6ba95289cedd8",
      "value": " 2.78M/2.78M [00:00&lt;00:00, 28.9MB/s]"
     }
    },
    "ddbc147f7ec34bbc9a1b0f0721f6451b": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "deb2379001af474cabbeca1df3c7dc95": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_f7ea2b985bd44ab089b515a002cdcc44",
      "placeholder": "​",
      "style": "IPY_MODEL_2c9c1b6e18fa4cd9979d5f13f41adf48",
      "value": "generation_config.json: 100%"
     }
    },
    "df94c5dc68864f8db8213f00e42af793": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "e30e11637be7473b873835e115517853": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_a4435caad91a4ab6ac25ebb860a1108e",
      "placeholder": "​",
      "style": "IPY_MODEL_69b75a00c15f46568b6caf44e6b72384",
      "value": "config.json: 100%"
     }
    },
    "e4c2626eabb84dc78273acfffcef82bd": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "e7aa821a05734fa8b2c81dff7f38c8af": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "ProgressStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "ProgressStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "bar_color": null,
      "description_width": ""
     }
    },
    "e91a61c0dc984ec09b9632af8a282611": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HBoxModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HBoxModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HBoxView",
      "box_style": "",
      "children": [
       "IPY_MODEL_fcf3e96c5cbf46abb84161f29d0f4199",
       "IPY_MODEL_9a33cd22c9cc4c34a2e60a87c37d8eaa",
       "IPY_MODEL_38b6585254aa4332b1cd6b6c64432263"
      ],
      "layout": "IPY_MODEL_6336f105491b4bd8ae01c0e3b6940c36"
     }
    },
    "e9ae9ba627aa4bf8a4eb64d68809f566": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "FloatProgressModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "FloatProgressModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "ProgressView",
      "bar_style": "success",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_138e15d45660471baff2f09c93efe771",
      "max": 7228,
      "min": 0,
      "orientation": "horizontal",
      "style": "IPY_MODEL_e7aa821a05734fa8b2c81dff7f38c8af",
      "value": 7228
     }
    },
    "ea3f8d8660a942fba64236fbc2cbd787": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "ef94fa10df1049eaa598e77851c05411": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "f46cdc40f11349108d92539de4185b33": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "f4c353c0fe944932bcf7879c7cd3dd2a": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_39ae78eff3714fbfa1ef5ef006e45b5a",
      "placeholder": "​",
      "style": "IPY_MODEL_d473f10ea00548428718b827ba7217db",
      "value": " 684/684 [00:00&lt;00:00, 6.51kB/s]"
     }
    },
    "f7ea2b985bd44ab089b515a002cdcc44": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "f8796e310a9548528122a3b20aab3582": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "ProgressStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "ProgressStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "bar_color": null,
      "description_width": ""
     }
    },
    "f8b66aed4c76404ea165db0c0dc091bf": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "f9c966f2b73e4f3da9163dd14ed16363": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "fa9b41c17c67430b99b3bf763b8e463d": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "ProgressStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "ProgressStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "bar_color": null,
      "description_width": ""
     }
    },
    "fbb1c9b7489242619cf2dc3bc0476ddd": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_25843be9e67346f5a9faf475a9d396ad",
      "placeholder": "​",
      "style": "IPY_MODEL_32767a4e19fd4bbe9887b6a54c916893",
      "value": " 19944/19944 [01:17&lt;00:00, 276.43 examples/s]"
     }
    },
    "fcf3e96c5cbf46abb84161f29d0f4199": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_c1e063e9f8174863b4acd5a553b5a71d",
      "placeholder": "​",
      "style": "IPY_MODEL_0855ea4d4a684f53acb099b20bbd2606",
      "value": "combined_reasoning.json: 100%"
     }
    },
    "fdca97d2f6e8424eb842a161e5fd5835": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "ffeb8b36b8dc4c0686940a4293ff0d2b": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HBoxModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HBoxModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HBoxView",
      "box_style": "",
      "children": [
       "IPY_MODEL_6c57207ee15e46488f3b7d8a206e6fe6",
       "IPY_MODEL_c453996f237f43f38d02ed0c3a808bc9",
       "IPY_MODEL_5ce999cad3d84a00af2a479a86914bb1"
      ],
      "layout": "IPY_MODEL_62d159e5720f496c82312d9efd4bfe79"
     }
    }
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}