finetune-OPT.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "WE5GJ6s7y0Xo"
   },
   "source": [
    "## Fine-tune large models using 🤗 `peft` adapters, `transformers` & `bitsandbytes`\n",
    "\n",
    "In this tutorial we will cover how we can fine-tune large language models using the very recent `peft` library and `bitsandbytes` for loading large models in 8-bit.\n",
    "The fine-tuning method will rely on a recent method called \"Low Rank Adapters\" (LoRA), instead of fine-tuning the entire model you just have to fine-tune these adapters and load them properly inside the model.\n",
    "After fine-tuning the model you can also share your adapters on the 🤗 Hub and load them very easily. Let's get started!"
   ]
  },
  {
   "cell_type": "code",
   "source": [
    "from google.colab import drive\n",
    "drive.mount('/content/drive')"
   ],
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "WUjFZ4XnXLef",
    "outputId": "333b18e4-525c-40ef-c3f3-d4648f61a2fc"
   },
   "execution_count": null,
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "Mounted at /content/drive\n"
     ]
    }
   ]
  },
  {
   "cell_type": "code",
   "source": [
    "import os\n",
    "import numpy as np\n",
    "import torch\n",
    "os.chdir('/content/drive/My Drive/Colab Notebooks/compositional-reasoning-finetuning-2/')\n",
    "data_set = 'direct'"
   ],
   "metadata": {
    "id": "ippkXMtaXUAa"
   },
   "execution_count": null,
   "outputs": []
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "TfBzP8gWzkpv"
   },
   "source": [
    "### Install requirements\n",
    "\n",
    "First, run the cells below to install the requirements:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "otj46qRbtpnd",
    "outputId": "9f823b5e-da6a-4165-cb11-5be46947be2d"
   },
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "\u001B[2K     \u001B[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001B[0m \u001B[32m92.6/92.6 MB\u001B[0m \u001B[31m14.0 MB/s\u001B[0m eta \u001B[36m0:00:00\u001B[0m\n",
      "\u001B[2K     \u001B[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001B[0m \u001B[32m486.2/486.2 kB\u001B[0m \u001B[31m43.9 MB/s\u001B[0m eta \u001B[36m0:00:00\u001B[0m\n",
      "\u001B[2K     \u001B[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001B[0m \u001B[32m244.2/244.2 kB\u001B[0m \u001B[31m27.9 MB/s\u001B[0m eta \u001B[36m0:00:00\u001B[0m\n",
      "\u001B[2K     \u001B[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001B[0m \u001B[32m110.5/110.5 kB\u001B[0m \u001B[31m13.0 MB/s\u001B[0m eta \u001B[36m0:00:00\u001B[0m\n",
      "\u001B[2K     \u001B[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001B[0m \u001B[32m212.5/212.5 kB\u001B[0m \u001B[31m21.7 MB/s\u001B[0m eta \u001B[36m0:00:00\u001B[0m\n",
      "\u001B[2K     \u001B[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001B[0m \u001B[32m134.8/134.8 kB\u001B[0m \u001B[31m15.5 MB/s\u001B[0m eta \u001B[36m0:00:00\u001B[0m\n",
      "\u001B[2K     \u001B[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001B[0m \u001B[32m268.8/268.8 kB\u001B[0m \u001B[31m29.0 MB/s\u001B[0m eta \u001B[36m0:00:00\u001B[0m\n",
      "\u001B[2K     \u001B[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001B[0m \u001B[32m134.3/134.3 kB\u001B[0m \u001B[31m13.5 MB/s\u001B[0m eta \u001B[36m0:00:00\u001B[0m\n",
      "\u001B[?25h  Installing build dependencies ... \u001B[?25l\u001B[?25hdone\n",
      "  Getting requirements to build wheel ... \u001B[?25l\u001B[?25hdone\n",
      "  Preparing metadata (pyproject.toml) ... \u001B[?25l\u001B[?25hdone\n",
      "  Installing build dependencies ... \u001B[?25l\u001B[?25hdone\n",
      "  Getting requirements to build wheel ... \u001B[?25l\u001B[?25hdone\n",
      "  Preparing metadata (pyproject.toml) ... \u001B[?25l\u001B[?25hdone\n",
      "\u001B[2K     \u001B[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001B[0m \u001B[32m7.8/7.8 MB\u001B[0m \u001B[31m49.2 MB/s\u001B[0m eta \u001B[36m0:00:00\u001B[0m\n",
      "\u001B[2K     \u001B[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001B[0m \u001B[32m1.3/1.3 MB\u001B[0m \u001B[31m63.8 MB/s\u001B[0m eta \u001B[36m0:00:00\u001B[0m\n",
      "\u001B[?25h  Building wheel for transformers (pyproject.toml) ... \u001B[?25l\u001B[?25hdone\n",
      "  Building wheel for peft (pyproject.toml) ... \u001B[?25l\u001B[?25hdone\n"
     ]
    }
   ],
   "source": [
    "!pip install -q bitsandbytes datasets accelerate loralib\n",
    "!pip install -q git+https://github.com/huggingface/transformers.git@main git+https://github.com/huggingface/peft.git"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "FOtwYRI3zzXI"
   },
   "source": [
    "### Model loading\n",
    "\n",
    "Here let's load the `opt-6.7b` model, its weights in half-precision (float16) are about 13GB on the Hub! If we load them in 8-bit we would require around 7GB of memory instead."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "cg3fiQOvmI3Q"
   },
   "outputs": [],
   "source": [
    "import os\n",
    "os.environ[\"CUDA_VISIBLE_DEVICES\"]=\"0\"\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import bitsandbytes as bnb\n",
    "from transformers import AutoTokenizer, AutoConfig, AutoModelForCausalLM\n",
    "model_size = '125m'\n",
    "model = AutoModelForCausalLM.from_pretrained(\n",
    "    f\"facebook/opt-{model_size}\",\n",
    "    load_in_8bit=True,\n",
    "    device_map='auto',\n",
    ")\n",
    "\n",
    "tokenizer = AutoTokenizer.from_pretrained(f\"facebook/opt-{model_size}\")#,\n",
    "    # model_max_length=300,\n",
    "    # padding_side=\"left\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "9fTSZntA1iUG"
   },
   "source": [
    "### Post-processing on the model\n",
    "\n",
    "Finally, we need to apply some post-processing on the 8-bit model to enable training, let's freeze all our layers, and cast the layer-norm in `float32` for stability. We also cast the output of the last layer in `float32` for the same reasons."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "T-gy-LxM0yAi"
   },
   "outputs": [],
   "source": [
    "for param in model.parameters():\n",
    "  param.requires_grad = False  # freeze the model - train adapters later\n",
    "  if param.ndim == 1:\n",
    "    # cast the small parameters (e.g. layernorm) to fp32 for stability\n",
    "    param.data = param.data.to(torch.float32)\n",
    "\n",
    "model.gradient_checkpointing_enable()  # reduce number of stored activations\n",
    "model.enable_input_require_grads()\n",
    "\n",
    "class CastOutputToFloat(nn.Sequential):\n",
    "  def forward(self, x): return super().forward(x).to(torch.float32)\n",
    "model.lm_head = CastOutputToFloat(model.lm_head)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "KwOTr7B3NlM3"
   },
   "source": [
    "### Apply LoRA\n",
    "\n",
    "Here comes the magic with `peft`! Let's load a `PeftModel` and specify that we are going to use low-rank adapters (LoRA) using `get_peft_model` utility function from `peft`."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "4W1j6lxaNnxC"
   },
   "outputs": [],
   "source": [
    "def print_trainable_parameters(model):\n",
    "    \"\"\"\n",
    "    Prints the number of trainable parameters in the model.\n",
    "    \"\"\"\n",
    "    trainable_params = 0\n",
    "    all_param = 0\n",
    "    for _, param in model.named_parameters():\n",
    "        all_param += param.numel()\n",
    "        if param.requires_grad:\n",
    "            trainable_params += param.numel()\n",
    "    print(\n",
    "        f\"trainable params: {trainable_params} || all params: {all_param} || trainable%: {100 * trainable_params / all_param}\"\n",
    "    )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "4iwHGzKBN6wk",
    "outputId": "805005d6-f5b3-4e97-ef12-52d2e3b3248b"
   },
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "trainable params: 589824 || all params: 125829120 || trainable%: 0.46875\n"
     ]
    }
   ],
   "source": [
    "from peft import LoraConfig, get_peft_model\n",
    "\n",
    "config = LoraConfig(\n",
    "    r=16,\n",
    "    lora_alpha=32,\n",
    "    target_modules=[\"q_proj\", \"v_proj\"],\n",
    "    lora_dropout=0.05,\n",
    "    bias=\"none\",\n",
    "    task_type=\"CAUSAL_LM\"\n",
    ")\n",
    "\n",
    "model = get_peft_model(model, config)\n",
    "print_trainable_parameters(model)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "QdjWif4CVXR6"
   },
   "source": [
    "### Training"
   ]
  },
  {
   "cell_type": "code",
   "source": [
    "import transformers\n",
    "from datasets import load_dataset\n",
    "data = load_dataset(\"Abirate/english_quotes\")\n",
    "data = data.map(lambda samples: tokenizer(samples['quote']), batched=True)\n",
    "data['train'][0]"
   ],
   "metadata": {
    "id": "kslb6YDNU8vx"
   },
   "execution_count": null,
   "outputs": []
  },
  {
   "cell_type": "code",
   "source": [],
   "metadata": {
    "id": "fyw7ESlLs-K0"
   },
   "execution_count": null,
   "outputs": []
  },
  {
   "cell_type": "code",
   "source": [
    "if data_set == 'self-ask':\n",
    "  data_files = {\"train\": \"self_ask_train_300.json\", \"validation\": \"self_ask_dev_300.json\"}\n",
    "elif data_set == 'direct':\n",
    "  data_files = {\"train\": \"direct_train_130.json\", \"validation\": \"direct_dev_130.json\"}\n",
    "data = load_dataset('data/OPT/', data_files=data_files)\n",
    "data = data.map(lambda samples: {'input': samples['prompt']+samples['target']})\n",
    "print(data)\n",
    "print(data['train'][0])\n",
    "data = data.map(lambda samples: tokenizer(samples['input'], truncation=True, padding='max_length'), batched=True) # how to include the target?"
   ],
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 312,
     "referenced_widgets": [
      "62363de45e124f6f8312383ad51c80c2",
      "8462c2be6c244ee2b14e7501707050e0",
      "f7e67464c7b84577bb2361e65ce4a00c",
      "5db00c27391c4170b49c9000594f9b3d",
      "3e5ec4d67cdd48e78923609978cc8e5c",
      "4f7f7e90360946ada6f2c75605539422",
      "d6b7fed90999402596382c7de0a8249e",
      "9c860220bded4cdfb64bdf31606813f5",
      "4bde8b3db74f499886feb2a27ae6d22d",
      "596bdb8c9cd848ddb240389f060d094b",
      "f709eb51edf040959fae148fab8331cd",
      "a281f6c58b294f38989576054d04fc2c",
      "c7afb893b2d247c6ae31724da1ad409c",
      "11075f98f5e042148754dfa538922777",
      "9e227702a5854e7db485fe2c1b0bf854",
      "398fd823bdfb4a30905aa5a8647391c7",
      "de9ad0e9e3b84e8bbf552e72c2faded2",
      "aa1571bdfebc4cb08b96d73c0087ef72",
      "6ba40d1a265f4a5e981d3df233911e18",
      "8f742fec5b2841a28858b91e6d5da1da",
      "f0f28d7ffca64ee58262105ed6d7a1d4",
      "9cd731136ff64161abd0f3b6ab23a835",
      "b72a6b71410b421fbb6a550a93013fb1",
      "a3156a7a425c4dc4a9adefc553b198b3",
      "3d7d41c4f34e4e1b92fc70d77b4c60e6",
      "55d2e2fb2b3748b18aae77b24e267bff",
      "d2dbd101cd3040fab582701eaf0b8296",
      "2f9c8600f41f47a1ab12fd7d56642fea",
      "93e3c0c0e9cc465fbae3025679cf96ab",
      "e2523995baa54273ad1fbf42d73b0b8f",
      "67cc0e8aa1784dd6860fe8354b74ee95",
      "40c1e27634fd4db0905c21a4b453c1ba",
      "35b6a884c8dd443385f4bbfb2d3d0931"
     ]
    },
    "id": "bxPhJrgxYFqU",
    "outputId": "9715e766-1b90-4060-f431-c12294ed2836"
   },
   "execution_count": null,
   "outputs": [
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "WARNING:datasets.builder:Found cached dataset json (/root/.cache/huggingface/datasets/json/OPT-839fbe9de5cdb525/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)\n"
     ]
    },
    {
     "output_type": "display_data",
     "data": {
      "text/plain": [
       "  0%|          | 0/2 [00:00<?, ?it/s]"
      ],
      "application/vnd.jupyter.widget-view+json": {
       "version_major": 2,
       "version_minor": 0,
       "model_id": "62363de45e124f6f8312383ad51c80c2"
      }
     },
     "metadata": {}
    },
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "WARNING:datasets.arrow_dataset:Loading cached processed dataset at /root/.cache/huggingface/datasets/json/OPT-839fbe9de5cdb525/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-7d5824ca8cc90d4b.arrow\n",
      "WARNING:datasets.arrow_dataset:Loading cached processed dataset at /root/.cache/huggingface/datasets/json/OPT-839fbe9de5cdb525/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-32a9c0d016a68051.arrow\n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "DatasetDict({\n",
      "    train: Dataset({\n",
      "        features: ['num_target_tokens', 'num_tokens', 'prompt', 'num_prompt_tokens', 'target', 'input'],\n",
      "        num_rows: 105479\n",
      "    })\n",
      "    validation: Dataset({\n",
      "        features: ['num_target_tokens', 'num_tokens', 'prompt', 'num_prompt_tokens', 'target', 'input'],\n",
      "        num_rows: 8657\n",
      "    })\n",
      "})\n",
      "{'num_target_tokens': 3, 'num_tokens': 84, 'prompt': \"Facts:\\nFact #0: Egan was the wife of the state of Alaska's first governor, William Allen Egan, and the mother of former Juneau Mayor and Alaska State Senator Dennis Egan.\\nFact #1: William Allen Egan (October 8, 1914 – May 6, 1984) was an American Democratic politician.\\n\\nQuestion: When was Neva Egan's husband born?\\nAnswer:\", 'num_prompt_tokens': 81, 'target': 'October 8, 1914', 'input': \"Facts:\\nFact #0: Egan was the wife of the state of Alaska's first governor, William Allen Egan, and the mother of former Juneau Mayor and Alaska State Senator Dennis Egan.\\nFact #1: William Allen Egan (October 8, 1914 – May 6, 1984) was an American Democratic politician.\\n\\nQuestion: When was Neva Egan's husband born?\\nAnswer:October 8, 1914\"}\n"
     ]
    },
    {
     "output_type": "display_data",
     "data": {
      "text/plain": [
       "Map:   0%|          | 0/105479 [00:00<?, ? examples/s]"
      ],
      "application/vnd.jupyter.widget-view+json": {
       "version_major": 2,
       "version_minor": 0,
       "model_id": "a281f6c58b294f38989576054d04fc2c"
      }
     },
     "metadata": {}
    },
    {
     "output_type": "display_data",
     "data": {
      "text/plain": [
       "Map:   0%|          | 0/8657 [00:00<?, ? examples/s]"
      ],
      "application/vnd.jupyter.widget-view+json": {
       "version_major": 2,
       "version_minor": 0,
       "model_id": "b72a6b71410b421fbb6a550a93013fb1"
      }
     },
     "metadata": {}
    }
   ]
  },
  {
   "cell_type": "code",
   "source": [
    "# print([len(x['input_ids']) for x in data['train'][:10]])\n",
    "print([(data['train'][i]['input_ids']) for i in range(10) ])\n",
    "print([len(data['train'][i]['input_ids']) for i in range(10) ])"
   ],
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "RMlT6zK2r_wK",
    "outputId": "c404cc71-6573-485e-97bd-0ec843f6e31c"
   },
   "execution_count": null,
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "[[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 597, 19170, 35, 50118, 37724, 849, 288, 35, 381, 3494, 21, 5, 1141, 9, 5, 194, 9, 7227, 18, 78, 2318, 6, 2897, 3823, 381, 3494, 6, 8, 5, 985, 9, 320, 502, 1180, 3129, 8, 7227, 331, 4336, 8093, 381, 3494, 4, 50118, 37724, 849, 134, 35, 2897, 3823, 381, 3494, 36, 20144, 290, 6, 34720, 126, 392, 231, 6, 11724, 43, 21, 41, 470, 1557, 8676, 4, 50118, 50118, 45641, 35, 520, 21, 17423, 102, 381, 3494, 18, 1623, 2421, 116, 50118, 33683, 35, 20144, 290, 6, 34720], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 597, 19170, 35, 50118, 37724, 849, 288, 35, 8445, 3548, 256, 3290, 354, 808, 368, 1640, 2421, 1105, 550, 6200, 43, 16, 10, 320, 2038, 5919, 869, 31, 2809, 4, 50118, 37724, 849, 134, 35, 22134, 26883, 2794, 1640, 290, 644, 504, 5220, 11, 10124, 5317, 126, 6056, 4, 35211, 43, 21, 10, 1859, 822, 736, 6, 2441, 9408, 6, 34174, 7243, 8, 822, 3436, 4, 50118, 50118, 45641, 35, 3394, 21, 2421, 78, 6, 8445, 3548, 256, 3290, 354, 808, 368, 50, 22134, 26883, 2794, 116, 50118, 33683, 35, 16750, 718, 26883, 2794], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 597, 19170, 35, 50118, 37724, 849, 288, 35, 19696, 16, 10, 3503, 2059, 822, 3660, 30, 3040, 384, 108, 12017, 8, 8996, 8718, 12860, 2507, 4, 50118, 37724, 849, 134, 35, 3040, 384, 108, 12017, 36, 5400, 753, 587, 14757, 11, 4290, 6, 1221, 43, 16, 41, 2059, 3331, 6, 736, 6, 2701, 8, 3436, 4, 50118, 50118, 45641, 35, 653, 16, 5, 317, 9, 3113, 9, 5, 736, 9, 822, 19696, 36, 32701, 6026, 26610, 50118, 33683, 35, 104, 9611, 2596, 6, 1221], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 597, 19170, 35, 50118, 37724, 849, 288, 35, 20, 822, 67, 3520, 1554, 1879, 2279, 260, 6, 2032, 2331, 20559, 337, 13444, 857, 8, 7275, 459, 11, 14125, 4502, 6, 150, 930, 21, 14092, 30, 208, 4, 83, 4, 3288, 330, 16160, 4, 50118, 37724, 849, 134, 35, 208, 4, 83, 4, 3288, 330, 16160, 16, 41, 1362, 822, 930, 736, 54, 2489, 5290, 31, 17494, 6, 12634, 17732, 4, 50118, 50118, 45641, 35, 4820, 21, 5, 17964, 9, 822, 2271, 35608, 1627, 2421, 116, 50118, 33683, 35, 347, 2457, 39089], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 597, 19170, 35, 50118, 37724, 849, 288, 35, 11013, 3849, 11582, 2242, 462, 3849, 14292, 3043, 1469, 6377, 16, 10, 11724, 3108, 5313, 822, 716, 15, 5, 5808, 9, 5, 276, 766, 30, 7483, 5472, 926, 230, 1535, 438, 225, 6527, 4, 50118, 37724, 849, 134, 35, 7483, 5472, 926, 230, 1535, 438, 225, 6527, 36, 1366, 830, 34224, 126, 504, 550, 954, 43, 21, 41, 3108, 3331, 6, 822, 2701, 6, 736, 8, 8083, 4, 50118, 50118, 45641, 35, 653, 16, 5, 1248, 9, 3113, 9, 5, 736, 9, 822, 11013, 3849, 11582, 2884, 462, 3849, 14292, 3043, 1469, 6377, 116, 50118, 33683, 35, 1366, 830, 34224], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 597, 19170, 35, 50118, 37724, 849, 288, 35, 85, 21, 5, 371, 8568, 4320, 7, 28, 1027, 30, 8938, 1949, 2454, 6, 5, 97, 80, 145, 15930, 10193, 1653, 8, 15930, 7175, 1653, 4, 50118, 37724, 849, 134, 35, 5441, 28477, 628, 11, 4525, 24, 16, 2164, 30, 4275, 9431, 4, 50118, 50118, 45641, 35, 520, 21, 5, 138, 14, 1027, 234, 329, 705, 398, 10202, 4790, 116, 50118, 33683, 35, 29411], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 597, 19170, 35, 50118, 37724, 849, 288, 35, 22, 45545, 34381, 113, 16, 10, 2214, 30, 5, 1089, 3152, 1971, 8248, 21706, 551, 31, 49, 15524, 2642, 22, 33499, 845, 50118, 37724, 849, 134, 35, 8248, 21706, 36, 14175, 2368, 1378, 4321, 6, 5736, 6, 5476, 2463, 6, 6452, 260, 6, 22009, 6, 18652, 5610, 6, 4869, 6, 8, 7799, 43, 58, 14663, 196, 88, 5, 2751, 8, 13065, 1631, 9, 7392, 11, 336, 4, 50118, 50118, 45641, 35, 653, 16, 5, 2354, 14, 5, 12576, 9, 2214, 24499, 34381, 36, 42677, 43, 2208, 116, 50118, 33683, 35, 10448, 8, 13065, 1631, 9, 7392], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 597, 19170, 35, 50118, 37724, 849, 288, 35, 35548, 18, 2376, 16, 10, 28234, 470, 1846, 4149, 822, 3660, 30, 2897, 230, 4, 7108, 2279, 8, 1982, 30, 26477, 3884, 10734, 4, 50118, 37724, 849, 134, 35, 91, 21, 2421, 11, 4386, 6, 4367, 8, 962, 11, 1287, 1422, 6, 886, 4, 50118, 50118, 45641, 35, 653, 16, 5, 317, 9, 3113, 9, 5, 736, 9, 822, 35548, 108, 104, 2376, 36, 35163, 26610, 50118, 33683, 35, 37164, 39710, 6, 4367], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 597, 19170, 35, 50118, 37724, 849, 288, 35, 20, 4046, 28722, 20547, 16, 10, 35313, 1089, 8454, 4149, 822, 3660, 30, 3884, 20994, 234, 2160, 8, 8996, 32906, 497, 1399, 8, 3066, 26743, 384, 108, 725, 1742, 4, 50118, 37724, 849, 134, 35, 91, 21, 2421, 11, 391, 15485, 6598, 6, 928, 8, 962, 11, 12130, 13247, 6, 11799, 4, 50118, 50118, 45641, 35, 4820, 21, 5, 317, 9, 744, 9, 5, 736, 9, 822, 20, 4046, 28722, 20547, 116, 50118, 33683, 35, 771, 368, 13247], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 597, 19170, 35, 50118, 37724, 849, 288, 35, 344, 2407, 525, 6103, 4654, 1951, 1640, 2421, 291, 644, 12910, 43, 16, 10, 37650, 811, 24702, 4, 50118, 37724, 849, 134, 35, 11450, 20964, 2070, 7794, 1640, 2421, 601, 494, 10206, 11, 21115, 43, 16, 10, 11145, 3390, 6, 3250, 6, 8, 1421, 4, 50118, 50118, 45641, 35, 1832, 344, 2407, 525, 6103, 4654, 1951, 8, 11450, 20964, 2070, 7794, 33, 5, 276, 26241, 116, 50118, 33683, 35, 2362]]\n",
      "[200, 200, 200, 200, 200, 200, 200, 200, 200, 200]\n"
     ]
    }
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "AQ_HCYruWIHU",
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 1000
    },
    "outputId": "049d9414-768c-4f2c-e99a-0334e5549fa7"
   },
   "outputs": [
    {
     "metadata": {
      "tags": null
     },
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.10/dist-packages/transformers/optimization.py:411: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use the PyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning\n",
      "  warnings.warn(\n",
      "You're using a GPT2TokenizerFast tokenizer. Please note that with a fast tokenizer, using the `__call__` method is faster than using a method to encode the text followed by a call to the `pad` method to get a padded encoding.\n",
      "/usr/local/lib/python3.10/dist-packages/bitsandbytes/autograd/_functions.py:322: UserWarning: MatMul8bitLt: inputs will be cast from torch.float32 to float16 during quantization\n",
      "  warnings.warn(f\"MatMul8bitLt: inputs will be cast from {A.dtype} to float16 during quantization\")\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='2758' max='6594' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [2758/6594 25:29 < 35:28, 1.80 it/s, Epoch 0.84/2]\n",
       "    </div>\n",
       "    <table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       " <tr style=\"text-align: left;\">\n",
       "      <th>Step</th>\n",
       "      <th>Training Loss</th>\n",
       "      <th>Validation Loss</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>100</td>\n",
       "      <td>2.285200</td>\n",
       "      <td>2.190836</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>200</td>\n",
       "      <td>2.157900</td>\n",
       "      <td>2.123133</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>300</td>\n",
       "      <td>2.108000</td>\n",
       "      <td>2.055159</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>400</td>\n",
       "      <td>2.091800</td>\n",
       "      <td>2.045769</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>500</td>\n",
       "      <td>2.052600</td>\n",
       "      <td>2.050960</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>600</td>\n",
       "      <td>2.048500</td>\n",
       "      <td>2.038487</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>700</td>\n",
       "      <td>2.045500</td>\n",
       "      <td>1.966473</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>800</td>\n",
       "      <td>2.045800</td>\n",
       "      <td>2.022928</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>900</td>\n",
       "      <td>2.012600</td>\n",
       "      <td>1.944468</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1000</td>\n",
       "      <td>2.010300</td>\n",
       "      <td>1.931143</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1100</td>\n",
       "      <td>1.995900</td>\n",
       "      <td>1.927398</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1200</td>\n",
       "      <td>1.976100</td>\n",
       "      <td>1.995954</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1300</td>\n",
       "      <td>1.973200</td>\n",
       "      <td>1.981379</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1400</td>\n",
       "      <td>1.980100</td>\n",
       "      <td>1.913454</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1500</td>\n",
       "      <td>1.982900</td>\n",
       "      <td>1.909531</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1600</td>\n",
       "      <td>1.961900</td>\n",
       "      <td>1.903912</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1700</td>\n",
       "      <td>1.962900</td>\n",
       "      <td>1.897762</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1800</td>\n",
       "      <td>1.970500</td>\n",
       "      <td>1.907364</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1900</td>\n",
       "      <td>1.964100</td>\n",
       "      <td>1.905682</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2000</td>\n",
       "      <td>1.957200</td>\n",
       "      <td>1.887821</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2100</td>\n",
       "      <td>1.951200</td>\n",
       "      <td>1.886769</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2200</td>\n",
       "      <td>1.954400</td>\n",
       "      <td>1.884847</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2300</td>\n",
       "      <td>1.962300</td>\n",
       "      <td>1.885511</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2400</td>\n",
       "      <td>1.960900</td>\n",
       "      <td>1.878089</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2500</td>\n",
       "      <td>1.942000</td>\n",
       "      <td>1.881112</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2600</td>\n",
       "      <td>1.948100</td>\n",
       "      <td>1.878115</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2700</td>\n",
       "      <td>1.938300</td>\n",
       "      <td>1.871996</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table><p>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "metadata": {
      "tags": null
     },
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.10/dist-packages/bitsandbytes/autograd/_functions.py:322: UserWarning: MatMul8bitLt: inputs will be cast from torch.float32 to float16 during quantization\n",
      "  warnings.warn(f\"MatMul8bitLt: inputs will be cast from {A.dtype} to float16 during quantization\")\n",
      "/usr/local/lib/python3.10/dist-packages/bitsandbytes/autograd/_functions.py:322: UserWarning: MatMul8bitLt: inputs will be cast from torch.float32 to float16 during quantization\n",
      "  warnings.warn(f\"MatMul8bitLt: inputs will be cast from {A.dtype} to float16 during quantization\")\n",
      "/usr/local/lib/python3.10/dist-packages/bitsandbytes/autograd/_functions.py:322: UserWarning: MatMul8bitLt: inputs will be cast from torch.float32 to float16 during quantization\n",
      "  warnings.warn(f\"MatMul8bitLt: inputs will be cast from {A.dtype} to float16 during quantization\")\n",
      "/usr/local/lib/python3.10/dist-packages/bitsandbytes/autograd/_functions.py:322: UserWarning: MatMul8bitLt: inputs will be cast from torch.float32 to float16 during quantization\n",
      "  warnings.warn(f\"MatMul8bitLt: inputs will be cast from {A.dtype} to float16 during quantization\")\n",
      "/usr/local/lib/python3.10/dist-packages/bitsandbytes/autograd/_functions.py:322: UserWarning: MatMul8bitLt: inputs will be cast from torch.float32 to float16 during quantization\n",
      "  warnings.warn(f\"MatMul8bitLt: inputs will be cast from {A.dtype} to float16 during quantization\")\n"
     ]
    },
    {
     "output_type": "display_data",
     "data": {
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ],
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='6594' max='6594' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [6594/6594 1:01:02, Epoch 2/2]\n",
       "    </div>\n",
       "    <table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       " <tr style=\"text-align: left;\">\n",
       "      <th>Step</th>\n",
       "      <th>Training Loss</th>\n",
       "      <th>Validation Loss</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>100</td>\n",
       "      <td>2.285200</td>\n",
       "      <td>2.190836</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>200</td>\n",
       "      <td>2.157900</td>\n",
       "      <td>2.123133</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>300</td>\n",
       "      <td>2.108000</td>\n",
       "      <td>2.055159</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>400</td>\n",
       "      <td>2.091800</td>\n",
       "      <td>2.045769</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>500</td>\n",
       "      <td>2.052600</td>\n",
       "      <td>2.050960</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>600</td>\n",
       "      <td>2.048500</td>\n",
       "      <td>2.038487</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>700</td>\n",
       "      <td>2.045500</td>\n",
       "      <td>1.966473</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>800</td>\n",
       "      <td>2.045800</td>\n",
       "      <td>2.022928</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>900</td>\n",
       "      <td>2.012600</td>\n",
       "      <td>1.944468</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1000</td>\n",
       "      <td>2.010300</td>\n",
       "      <td>1.931143</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1100</td>\n",
       "      <td>1.995900</td>\n",
       "      <td>1.927398</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1200</td>\n",
       "      <td>1.976100</td>\n",
       "      <td>1.995954</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1300</td>\n",
       "      <td>1.973200</td>\n",
       "      <td>1.981379</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1400</td>\n",
       "      <td>1.980100</td>\n",
       "      <td>1.913454</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1500</td>\n",
       "      <td>1.982900</td>\n",
       "      <td>1.909531</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1600</td>\n",
       "      <td>1.961900</td>\n",
       "      <td>1.903912</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1700</td>\n",
       "      <td>1.962900</td>\n",
       "      <td>1.897762</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1800</td>\n",
       "      <td>1.970500</td>\n",
       "      <td>1.907364</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1900</td>\n",
       "      <td>1.964100</td>\n",
       "      <td>1.905682</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2000</td>\n",
       "      <td>1.957200</td>\n",
       "      <td>1.887821</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2100</td>\n",
       "      <td>1.951200</td>\n",
       "      <td>1.886769</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2200</td>\n",
       "      <td>1.954400</td>\n",
       "      <td>1.884847</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2300</td>\n",
       "      <td>1.962300</td>\n",
       "      <td>1.885511</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2400</td>\n",
       "      <td>1.960900</td>\n",
       "      <td>1.878089</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2500</td>\n",
       "      <td>1.942000</td>\n",
       "      <td>1.881112</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2600</td>\n",
       "      <td>1.948100</td>\n",
       "      <td>1.878115</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2700</td>\n",
       "      <td>1.938300</td>\n",
       "      <td>1.871996</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2800</td>\n",
       "      <td>1.946600</td>\n",
       "      <td>1.869889</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2900</td>\n",
       "      <td>1.936000</td>\n",
       "      <td>1.872220</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3000</td>\n",
       "      <td>1.931200</td>\n",
       "      <td>1.865865</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3100</td>\n",
       "      <td>1.936600</td>\n",
       "      <td>1.863788</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3200</td>\n",
       "      <td>1.933700</td>\n",
       "      <td>1.862808</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3300</td>\n",
       "      <td>1.914200</td>\n",
       "      <td>1.862182</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3400</td>\n",
       "      <td>1.932600</td>\n",
       "      <td>1.864297</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3500</td>\n",
       "      <td>1.929000</td>\n",
       "      <td>1.859697</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3600</td>\n",
       "      <td>1.933600</td>\n",
       "      <td>1.872780</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3700</td>\n",
       "      <td>1.908100</td>\n",
       "      <td>1.867362</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3800</td>\n",
       "      <td>1.920700</td>\n",
       "      <td>1.858800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3900</td>\n",
       "      <td>1.936700</td>\n",
       "      <td>1.856356</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4000</td>\n",
       "      <td>1.909400</td>\n",
       "      <td>1.854302</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4100</td>\n",
       "      <td>1.907600</td>\n",
       "      <td>1.859947</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4200</td>\n",
       "      <td>1.918600</td>\n",
       "      <td>1.852673</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4300</td>\n",
       "      <td>1.910400</td>\n",
       "      <td>1.852979</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4400</td>\n",
       "      <td>1.914100</td>\n",
       "      <td>1.861060</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4500</td>\n",
       "      <td>1.905800</td>\n",
       "      <td>1.848926</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4600</td>\n",
       "      <td>1.921000</td>\n",
       "      <td>1.847949</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4700</td>\n",
       "      <td>1.928900</td>\n",
       "      <td>1.850216</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4800</td>\n",
       "      <td>1.914700</td>\n",
       "      <td>1.847487</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4900</td>\n",
       "      <td>1.916300</td>\n",
       "      <td>1.847257</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>5000</td>\n",
       "      <td>1.914400</td>\n",
       "      <td>1.845971</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>5100</td>\n",
       "      <td>1.908500</td>\n",
       "      <td>1.847957</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>5200</td>\n",
       "      <td>1.900200</td>\n",
       "      <td>1.847408</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>5300</td>\n",
       "      <td>1.908400</td>\n",
       "      <td>1.844503</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>5400</td>\n",
       "      <td>1.906700</td>\n",
       "      <td>1.845427</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>5500</td>\n",
       "      <td>1.916000</td>\n",
       "      <td>1.843157</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>5600</td>\n",
       "      <td>1.896300</td>\n",
       "      <td>1.844875</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>5700</td>\n",
       "      <td>1.914800</td>\n",
       "      <td>1.843387</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>5800</td>\n",
       "      <td>1.921100</td>\n",
       "      <td>1.841841</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>5900</td>\n",
       "      <td>1.902100</td>\n",
       "      <td>1.842484</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>6000</td>\n",
       "      <td>1.918900</td>\n",
       "      <td>1.842067</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>6100</td>\n",
       "      <td>1.896600</td>\n",
       "      <td>1.855163</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>6200</td>\n",
       "      <td>1.908300</td>\n",
       "      <td>1.840541</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>6300</td>\n",
       "      <td>1.905000</td>\n",
       "      <td>1.839985</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>6400</td>\n",
       "      <td>1.913300</td>\n",
       "      <td>1.843514</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>6500</td>\n",
       "      <td>1.911400</td>\n",
       "      <td>1.840075</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table><p>"
      ]
     },
     "metadata": {}
    },
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "/usr/local/lib/python3.10/dist-packages/bitsandbytes/autograd/_functions.py:322: UserWarning: MatMul8bitLt: inputs will be cast from torch.float32 to float16 during quantization\n",
      "  warnings.warn(f\"MatMul8bitLt: inputs will be cast from {A.dtype} to float16 during quantization\")\n",
      "/usr/local/lib/python3.10/dist-packages/bitsandbytes/autograd/_functions.py:322: UserWarning: MatMul8bitLt: inputs will be cast from torch.float32 to float16 during quantization\n",
      "  warnings.warn(f\"MatMul8bitLt: inputs will be cast from {A.dtype} to float16 during quantization\")\n",
      "/usr/local/lib/python3.10/dist-packages/bitsandbytes/autograd/_functions.py:322: UserWarning: MatMul8bitLt: inputs will be cast from torch.float32 to float16 during quantization\n",
      "  warnings.warn(f\"MatMul8bitLt: inputs will be cast from {A.dtype} to float16 during quantization\")\n",
      "/usr/local/lib/python3.10/dist-packages/bitsandbytes/autograd/_functions.py:322: UserWarning: MatMul8bitLt: inputs will be cast from torch.float32 to float16 during quantization\n",
      "  warnings.warn(f\"MatMul8bitLt: inputs will be cast from {A.dtype} to float16 during quantization\")\n",
      "/usr/local/lib/python3.10/dist-packages/bitsandbytes/autograd/_functions.py:322: UserWarning: MatMul8bitLt: inputs will be cast from torch.float32 to float16 during quantization\n",
      "  warnings.warn(f\"MatMul8bitLt: inputs will be cast from {A.dtype} to float16 during quantization\")\n",
      "/usr/local/lib/python3.10/dist-packages/bitsandbytes/autograd/_functions.py:322: UserWarning: MatMul8bitLt: inputs will be cast from torch.float32 to float16 during quantization\n",
      "  warnings.warn(f\"MatMul8bitLt: inputs will be cast from {A.dtype} to float16 during quantization\")\n",
      "/usr/local/lib/python3.10/dist-packages/bitsandbytes/autograd/_functions.py:322: UserWarning: MatMul8bitLt: inputs will be cast from torch.float32 to float16 during quantization\n",
      "  warnings.warn(f\"MatMul8bitLt: inputs will be cast from {A.dtype} to float16 during quantization\")\n",
      "/usr/local/lib/python3.10/dist-packages/bitsandbytes/autograd/_functions.py:322: UserWarning: MatMul8bitLt: inputs will be cast from torch.float32 to float16 during quantization\n",
      "  warnings.warn(f\"MatMul8bitLt: inputs will be cast from {A.dtype} to float16 during quantization\")\n"
     ]
    }
   ],
   "source": [
    "os.environ[\"PYTORCH_CUDA_ALLOC_CONF\"] = \"max_split_size_mb:512\"\n",
    "\n",
    "trainer = transformers.Trainer(\n",
    "    model=model,\n",
    "    train_dataset=data['train'],\n",
    "    eval_dataset=data['validation'],\n",
    "    args=transformers.TrainingArguments(\n",
    "        per_device_train_batch_size=32,\n",
    "        per_device_eval_batch_size=32,\n",
    "        evaluation_strategy=\"steps\",\n",
    "        eval_steps=100,\n",
    "        # warmup_steps=100,\n",
    "        # max_steps=100,\n",
    "        num_train_epochs=2,\n",
    "        learning_rate=2e-4,\n",
    "        fp16=True,\n",
    "        logging_steps=50,\n",
    "        output_dir='outputs',\n",
    "    ),\n",
    "    data_collator=transformers.DataCollatorForLanguageModeling(tokenizer, mlm=False)\n",
    ")\n",
    "model.config.use_cache = False  # silence the warnings. Please re-enable for inference!\n",
    "with torch.autocast(\"cuda\"):\n",
    "  trainer.train()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "Duak7T_B3VpJ"
   },
   "source": [
    "## Share adapters on the 🤗 Hub"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 145,
     "referenced_widgets": [
      "f1c9b3744c664e21aacf2cd220917058",
      "13e0c5018eb64383bd7fedb7f346fb16",
      "8e6c0b740532485ba3233ba20e16ade6",
      "0098e91e661942f28975858a37524d21",
      "653497ed2c4843d0b51724021c18f9e6",
      "6007657f9bdb4f98be8d60fa5cfa4154",
      "6d5dfab877704054b6a66b619c32c946",
      "dc3e506bec8a47c6a57312c437779bb6",
      "6e557f79ac8549948f8c86cbfa71d6cd",
      "7d90d18918a742b2a6848431ba093ab6",
      "aa436b151a2a499792ba59191d974c75",
      "b59e3b393e794b7491f55d3262f63d96",
      "64237cc139ec4322844e29cb9bb4629c",
      "6f1bd533ac984413aee792a7fc1235bf",
      "483239d349ad413791ff7e26cad6f6af",
      "b629ca6df71f4a5abeb545bd84bb5031",
      "bf4f2788a6e04ef6a5c49c2a08cf66c7",
      "03ea3b9bd96b427394c311c6a03a6edb",
      "779027195c30440dbe9de99fb115c9c6",
      "2f375cd9577d4444aba5e5b951643806",
      "288ea5c2df5f4712ad47aa39c5c7bb88",
      "fa5703441e324e10a26fb4ecd3f2b7e6",
      "c2d4c1a1482d436bab5f61247fc994f6",
      "9dc89055fb4f44959bbcb26aae068e05",
      "09146549434d48848f4087be4ad38852",
      "1c9ec3b458f74d98b95323cb7acf5d0d",
      "d637855805804229b358030385ded12a",
      "e9eb3a83dcc348169f679ccb4187cafb",
      "fecc475a8f8244a29ca44b995d2274c8",
      "f6c8be78a6cb4f079d21e6382cfe4418",
      "bbd22b8185074ca1b23ce05b6be0ce0d",
      "78d6c9442171435789aa8b303ba95e07"
     ]
    },
    "id": "DpYr24pR8T_0",
    "outputId": "34efadd7-1df7-44dc-feab-c3398a8d7167"
   },
   "outputs": [
    {
     "output_type": "display_data",
     "data": {
      "text/plain": [
       "VBox(children=(HTML(value='<center> <img\\nsrc=https://huggingface.co/front/assets/huggingface_logo-noborder.sv…"
      ],
      "application/vnd.jupyter.widget-view+json": {
       "version_major": 2,
       "version_minor": 0,
       "model_id": "f1c9b3744c664e21aacf2cd220917058"
      }
     },
     "metadata": {}
    }
   ],
   "source": [
    "from huggingface_hub import notebook_login\n",
    "\n",
    "notebook_login()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "VxB6UV5XAvvP",
    "outputId": "ce1f581c-f157-4204-bf36-aacf10e49ca7"
   },
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "CommitInfo(commit_url='https://huggingface.co/adam-wein/opt-125m-direct/commit/7bbf0965541b444137f62ce6aea404ef56c79f8f', commit_message='Upload model', commit_description='', oid='7bbf0965541b444137f62ce6aea404ef56c79f8f', pr_url=None, pr_revision=None, pr_num=None)"
      ]
     },
     "metadata": {},
     "execution_count": 17
    }
   ],
   "source": [
    "model.push_to_hub(f\"adam-wein/opt-{model_size}-{data_set}\", use_auth_token=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "S65GcxNGA9kz"
   },
   "source": [
    "## Load adapters from the Hub\n",
    "\n",
    "You can also directly load adapters from the Hub using the commands below:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "hsD1VKqeA62Z"
   },
   "outputs": [],
   "source": [
    "import torch\n",
    "from peft import PeftModel, PeftConfig\n",
    "from transformers import AutoModelForCausalLM, AutoTokenizer\n",
    "\n",
    "peft_model_id = f\"adam-wein/opt-{model_size}-{data_set}\"\n",
    "config = PeftConfig.from_pretrained(peft_model_id)\n",
    "model = AutoModelForCausalLM.from_pretrained(config.base_model_name_or_path, return_dict=True, load_in_8bit=True, device_map='auto', max_length=50)\n",
    "tokenizer = AutoTokenizer.from_pretrained(config.base_model_name_or_path, padding_side='left', model_max_length=500)\n",
    "\n",
    "# Load the Lora model\n",
    "model = PeftModel.from_pretrained(model, peft_model_id)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "MHYljmTjj5wX"
   },
   "source": [
    "## Inference\n",
    "\n",
    "You can then directly use the trained model or the model that you have loaded from the 🤗 Hub for inference as you would do it usually in `transformers`."
   ]
  },
  {
   "cell_type": "code",
   "source": [
    "print(data['validation'][0]['prompt'])\n",
    "print(data['validation'][0]['target'])"
   ],
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "EvwLto-teGWS",
    "outputId": "4c0bc776-c223-4429-a4d6-4a154d96ff87"
   },
   "execution_count": null,
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "Facts:\n",
      "Fact #0: Éric Deflandre( born 2 August 1973 in Rocourt) is a former Belgian football right fullback.\n",
      "Fact #1: Polly Swann( born 5 June 1988) is a British rower and a member of the Great Britain Rowing Team.\n",
      "\n",
      "Question: Who was born earlier, Polly Swann or Éric Deflandre?\n",
      "Answer:\n",
      "Éric Deflandre\n"
     ]
    }
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "MDqJWba-tpnv",
    "outputId": "86c25ee1-dcd1-41d6-955a-93a2ae0b0427"
   },
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "\n",
      "\n",
      " Facts:\n",
      "Fact #0: Éric Deflandre( born 2 August 1973 in Rocourt) is a former Belgian football right fullback.\n",
      "Fact #1: Polly Swann( born 5 June 1988) is a British rower and a member of the Great Britain Rowing Team.\n",
      "\n",
      "Question: Who was born earlier, Polly Swann or Éric Deflandre?\n",
      "Answer:Éric Def, who is a British, British, British and American rower.\n",
      "\n",
      "Question: Who, who, in the United, was born, born, born, born, born, is a British, British and born, is a\n"
     ]
    }
   ],
   "source": [
    "batch = tokenizer(\"Facts:\\nFact #0: Éric Deflandre( born 2 August 1973 in Rocourt) is a former Belgian football right fullback.\\nFact #1: Polly Swann( born 5 June 1988) is a British rower and a member of the Great Britain Rowing Team.\\n\\nQuestion: Who was born earlier, Polly Swann or Éric Deflandre?\\nAnswer:\", return_tensors='pt')\n",
    "\n",
    "with torch.cuda.amp.autocast():\n",
    "  output_tokens = model.generate(**batch, max_new_tokens=50)\n",
    "\n",
    "print('\\n\\n', tokenizer.decode(output_tokens[0], skip_special_tokens=True))"
   ]
  },
  {
   "cell_type": "code",
   "source": [
    "tokenizer_config_ = {\"truncation\": True, \"return_tensors\": \"pt\"}\n",
    "print(tokenizer(data['validation'][0]['prompt'], **tokenizer_config_))\n",
    "data['validation_for_inference'] = data['validation'].map(lambda samples: tokenizer(samples['prompt'], **tokenizer_config_), batched=False)\n",
    "# data['validation_for_inference'][0]"
   ],
   "metadata": {
    "id": "JCdGEPyJc6vK",
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "outputId": "9102045d-1507-42de-db6d-a046bccd2319"
   },
   "execution_count": null,
   "outputs": [
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "WARNING:datasets.arrow_dataset:Loading cached processed dataset at /root/.cache/huggingface/datasets/json/OPT-7b60f4243c4ac549/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-348a08090208ac88.arrow\n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "{'input_ids': tensor([[    2,   597, 19170,    35, 50118, 37724,   849,   288,    35, 24986,\n",
      "          4063,  7858,  1245,   241,  1640,  2421,   132,   830, 14757,    11,\n",
      "         16809, 14103,    43,    16,    10,   320, 11731,  1037,   235, 24907,\n",
      "             4, 50118, 37724,   849,   134,    35, 36189,  3323,  2279,  1640,\n",
      "          2421,   195,   502, 11151,    43,    16,    10,  1089,  3236,   254,\n",
      "             8,    10,   919,     9,     5,  2860,  1444,   248,  6932,  2711,\n",
      "             4, 50118, 50118, 45641,    35,  3394,    21,  2421,   656,     6,\n",
      "         36189,  3323,  2279,    50, 24986,  4063,  7858,  1245,   241,   116,\n",
      "         50118, 33683,    35]]), 'attention_mask': tensor([[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n",
      "         1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n",
      "         1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n",
      "         1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]])}\n"
     ]
    }
   ]
  },
  {
   "cell_type": "code",
   "source": [
    "with torch.cuda.amp.autocast():\n",
    "  data['validation_for_inference'].map(lambda samples:\n",
    "  {'input_ids': model.generate(inputs=torch.tensor(np.array(samples['input_ids'])),\n",
    "                                max_new_tokens=10)})"
   ],
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 528,
     "referenced_widgets": [
      "50b07294170b4501b855bc589080022c",
      "c595188878274d269330d3ab2687c29f",
      "b3686ca7a89047a396e54c9838ef97e6",
      "a6689c4f31f1485f888fa7b85f0026ab",
      "7308ac7e81794d1ca0883d78201ca801",
      "c6dfd6c0e707466c9a4012233b4d1161",
      "210eae0956014ec0943dfe3cec7eb63f",
      "fd3d5aebc9a3470cb3c47109d9c6b989",
      "53894ce1b11941a79f1dcb06d64e6d56",
      "eaccf75741294a928b17db180c7a6067",
      "f600fe53ce314f42940c0cf95934f9d8"
     ]
    },
    "id": "d43l1AXO-oWz",
    "outputId": "e8c953fa-4bed-492a-bad9-5e26aa1dfcd7"
   },
   "execution_count": null,
   "outputs": [
    {
     "output_type": "display_data",
     "data": {
      "text/plain": [
       "Map:   0%|          | 0/8657 [00:00<?, ? examples/s]"
      ],
      "application/vnd.jupyter.widget-view+json": {
       "version_major": 2,
       "version_minor": 0,
       "model_id": "50b07294170b4501b855bc589080022c"
      }
     },
     "metadata": {}
    },
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "/usr/local/lib/python3.10/dist-packages/transformers/generation/utils.py:1270: UserWarning: You have modified the pretrained model configuration to control generation. This is a deprecated strategy to control generation and will be removed soon, in a future version. Please use a generation configuration file (see https://huggingface.co/docs/transformers/main_classes/text_generation )\n",
      "  warnings.warn(\n",
      "/usr/local/lib/python3.10/dist-packages/transformers/generation/utils.py:1468: UserWarning: You are calling .generate() with the `input_ids` being on a device type different than your model's device. `input_ids` is on cpu, whereas the model is on cuda. You may experience unexpected behaviors or slower generation. Please make sure that you have put `input_ids` to the correct device by calling for example input_ids = input_ids.to('cuda') before running `.generate()`.\n",
      "  warnings.warn(\n",
      "/usr/local/lib/python3.10/dist-packages/bitsandbytes/autograd/_functions.py:322: UserWarning: MatMul8bitLt: inputs will be cast from torch.float32 to float16 during quantization\n",
      "  warnings.warn(f\"MatMul8bitLt: inputs will be cast from {A.dtype} to float16 during quantization\")\n"
     ]
    },
    {
     "output_type": "error",
     "ename": "KeyboardInterrupt",
     "evalue": "ignored",
     "traceback": [
      "\u001B[0;31m---------------------------------------------------------------------------\u001B[0m",
      "\u001B[0;31mKeyboardInterrupt\u001B[0m                         Traceback (most recent call last)",
      "\u001B[0;32m<ipython-input-45-23cf14d1105c>\u001B[0m in \u001B[0;36m<cell line: 1>\u001B[0;34m()\u001B[0m\n\u001B[1;32m      1\u001B[0m \u001B[0;32mwith\u001B[0m \u001B[0mtorch\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0mcuda\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0mamp\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0mautocast\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m:\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0;32m----> 2\u001B[0;31m   data['validation_for_inference'].map(lambda samples:\n\u001B[0m\u001B[1;32m      3\u001B[0m   {'input_ids': model.generate(inputs=torch.tensor(np.array(samples['input_ids'])),\n\u001B[1;32m      4\u001B[0m                                 max_new_tokens=10)})\n",
      "\u001B[0;32m/usr/local/lib/python3.10/dist-packages/datasets/arrow_dataset.py\u001B[0m in \u001B[0;36mwrapper\u001B[0;34m(*args, **kwargs)\u001B[0m\n\u001B[1;32m    578\u001B[0m             \u001B[0mself\u001B[0m\u001B[0;34m:\u001B[0m \u001B[0;34m\"Dataset\"\u001B[0m \u001B[0;34m=\u001B[0m \u001B[0mkwargs\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0mpop\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0;34m\"self\"\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m    579\u001B[0m         \u001B[0;31m# apply actual function\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0;32m--> 580\u001B[0;31m         \u001B[0mout\u001B[0m\u001B[0;34m:\u001B[0m \u001B[0mUnion\u001B[0m\u001B[0;34m[\u001B[0m\u001B[0;34m\"Dataset\"\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0;34m\"DatasetDict\"\u001B[0m\u001B[0;34m]\u001B[0m \u001B[0;34m=\u001B[0m \u001B[0mfunc\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0mself\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0;34m*\u001B[0m\u001B[0margs\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0;34m**\u001B[0m\u001B[0mkwargs\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0m\u001B[1;32m    581\u001B[0m         \u001B[0mdatasets\u001B[0m\u001B[0;34m:\u001B[0m \u001B[0mList\u001B[0m\u001B[0;34m[\u001B[0m\u001B[0;34m\"Dataset\"\u001B[0m\u001B[0;34m]\u001B[0m \u001B[0;34m=\u001B[0m \u001B[0mlist\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0mout\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0mvalues\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m)\u001B[0m \u001B[0;32mif\u001B[0m \u001B[0misinstance\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0mout\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0mdict\u001B[0m\u001B[0;34m)\u001B[0m \u001B[0;32melse\u001B[0m \u001B[0;34m[\u001B[0m\u001B[0mout\u001B[0m\u001B[0;34m]\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m    582\u001B[0m         \u001B[0;32mfor\u001B[0m \u001B[0mdataset\u001B[0m \u001B[0;32min\u001B[0m \u001B[0mdatasets\u001B[0m\u001B[0;34m:\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n",
      "\u001B[0;32m/usr/local/lib/python3.10/dist-packages/datasets/arrow_dataset.py\u001B[0m in \u001B[0;36mwrapper\u001B[0;34m(*args, **kwargs)\u001B[0m\n\u001B[1;32m    543\u001B[0m         }\n\u001B[1;32m    544\u001B[0m         \u001B[0;31m# apply actual function\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0;32m--> 545\u001B[0;31m         \u001B[0mout\u001B[0m\u001B[0;34m:\u001B[0m \u001B[0mUnion\u001B[0m\u001B[0;34m[\u001B[0m\u001B[0;34m\"Dataset\"\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0;34m\"DatasetDict\"\u001B[0m\u001B[0;34m]\u001B[0m \u001B[0;34m=\u001B[0m \u001B[0mfunc\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0mself\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0;34m*\u001B[0m\u001B[0margs\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0;34m**\u001B[0m\u001B[0mkwargs\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0m\u001B[1;32m    546\u001B[0m         \u001B[0mdatasets\u001B[0m\u001B[0;34m:\u001B[0m \u001B[0mList\u001B[0m\u001B[0;34m[\u001B[0m\u001B[0;34m\"Dataset\"\u001B[0m\u001B[0;34m]\u001B[0m \u001B[0;34m=\u001B[0m \u001B[0mlist\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0mout\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0mvalues\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m)\u001B[0m \u001B[0;32mif\u001B[0m \u001B[0misinstance\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0mout\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0mdict\u001B[0m\u001B[0;34m)\u001B[0m \u001B[0;32melse\u001B[0m \u001B[0;34m[\u001B[0m\u001B[0mout\u001B[0m\u001B[0;34m]\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m    547\u001B[0m         \u001B[0;31m# re-apply format to the output\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n",
      "\u001B[0;32m/usr/local/lib/python3.10/dist-packages/datasets/arrow_dataset.py\u001B[0m in \u001B[0;36mmap\u001B[0;34m(self, function, with_indices, with_rank, input_columns, batched, batch_size, drop_last_batch, remove_columns, keep_in_memory, load_from_cache_file, cache_file_name, writer_batch_size, features, disable_nullable, fn_kwargs, num_proc, suffix_template, new_fingerprint, desc)\u001B[0m\n\u001B[1;32m   3085\u001B[0m                     \u001B[0mdesc\u001B[0m\u001B[0;34m=\u001B[0m\u001B[0mdesc\u001B[0m \u001B[0;32mor\u001B[0m \u001B[0;34m\"Map\"\u001B[0m\u001B[0;34m,\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m   3086\u001B[0m                 ) as pbar:\n\u001B[0;32m-> 3087\u001B[0;31m                     \u001B[0;32mfor\u001B[0m \u001B[0mrank\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0mdone\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0mcontent\u001B[0m \u001B[0;32min\u001B[0m \u001B[0mDataset\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0m_map_single\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0;34m**\u001B[0m\u001B[0mdataset_kwargs\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m:\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0m\u001B[1;32m   3088\u001B[0m                         \u001B[0;32mif\u001B[0m \u001B[0mdone\u001B[0m\u001B[0;34m:\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m   3089\u001B[0m                             \u001B[0mshards_done\u001B[0m \u001B[0;34m+=\u001B[0m \u001B[0;36m1\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n",
      "\u001B[0;32m/usr/local/lib/python3.10/dist-packages/datasets/arrow_dataset.py\u001B[0m in \u001B[0;36m_map_single\u001B[0;34m(shard, function, with_indices, with_rank, input_columns, batched, batch_size, drop_last_batch, remove_columns, keep_in_memory, cache_file_name, writer_batch_size, features, disable_nullable, fn_kwargs, new_fingerprint, rank, offset)\u001B[0m\n\u001B[1;32m   3439\u001B[0m                     \u001B[0m_time\u001B[0m \u001B[0;34m=\u001B[0m \u001B[0mtime\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0mtime\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m   3440\u001B[0m                     \u001B[0;32mfor\u001B[0m \u001B[0mi\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0mexample\u001B[0m \u001B[0;32min\u001B[0m \u001B[0mshard_iterable\u001B[0m\u001B[0;34m:\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0;32m-> 3441\u001B[0;31m                         \u001B[0mexample\u001B[0m \u001B[0;34m=\u001B[0m \u001B[0mapply_function_on_filtered_inputs\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0mexample\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0mi\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0moffset\u001B[0m\u001B[0;34m=\u001B[0m\u001B[0moffset\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0m\u001B[1;32m   3442\u001B[0m                         \u001B[0;32mif\u001B[0m \u001B[0mupdate_data\u001B[0m\u001B[0;34m:\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m   3443\u001B[0m                             \u001B[0;32mif\u001B[0m \u001B[0mi\u001B[0m \u001B[0;34m==\u001B[0m \u001B[0;36m0\u001B[0m\u001B[0;34m:\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n",
      "\u001B[0;32m/usr/local/lib/python3.10/dist-packages/datasets/arrow_dataset.py\u001B[0m in \u001B[0;36mapply_function_on_filtered_inputs\u001B[0;34m(pa_inputs, indices, check_same_num_examples, offset)\u001B[0m\n\u001B[1;32m   3342\u001B[0m             \u001B[0;32mif\u001B[0m \u001B[0mwith_rank\u001B[0m\u001B[0;34m:\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m   3343\u001B[0m                 \u001B[0madditional_args\u001B[0m \u001B[0;34m+=\u001B[0m \u001B[0;34m(\u001B[0m\u001B[0mrank\u001B[0m\u001B[0;34m,\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0;32m-> 3344\u001B[0;31m             \u001B[0mprocessed_inputs\u001B[0m \u001B[0;34m=\u001B[0m \u001B[0mfunction\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0;34m*\u001B[0m\u001B[0mfn_args\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0;34m*\u001B[0m\u001B[0madditional_args\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0;34m**\u001B[0m\u001B[0mfn_kwargs\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0m\u001B[1;32m   3345\u001B[0m             \u001B[0;32mif\u001B[0m \u001B[0misinstance\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0mprocessed_inputs\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0mLazyDict\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m:\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m   3346\u001B[0m                 processed_inputs = {\n",
      "\u001B[0;32m<ipython-input-45-23cf14d1105c>\u001B[0m in \u001B[0;36m<lambda>\u001B[0;34m(samples)\u001B[0m\n\u001B[1;32m      1\u001B[0m \u001B[0;32mwith\u001B[0m \u001B[0mtorch\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0mcuda\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0mamp\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0mautocast\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m:\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m      2\u001B[0m   data['validation_for_inference'].map(lambda samples:\n\u001B[0;32m----> 3\u001B[0;31m   {'input_ids': model.generate(inputs=torch.tensor(np.array(samples['input_ids'])),\n\u001B[0m\u001B[1;32m      4\u001B[0m                                 max_new_tokens=10)})\n",
      "\u001B[0;32m/usr/local/lib/python3.10/dist-packages/peft/peft_model.py\u001B[0m in \u001B[0;36mgenerate\u001B[0;34m(self, **kwargs)\u001B[0m\n\u001B[1;32m   1000\u001B[0m             \u001B[0mself\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0mbase_model\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0mgeneration_config\u001B[0m \u001B[0;34m=\u001B[0m \u001B[0mself\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0mgeneration_config\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m   1001\u001B[0m         \u001B[0;32mtry\u001B[0m\u001B[0;34m:\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0;32m-> 1002\u001B[0;31m             \u001B[0moutputs\u001B[0m \u001B[0;34m=\u001B[0m \u001B[0mself\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0mbase_model\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0mgenerate\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0;34m**\u001B[0m\u001B[0mkwargs\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0m\u001B[1;32m   1003\u001B[0m         \u001B[0;32mexcept\u001B[0m\u001B[0;34m:\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m   1004\u001B[0m             \u001B[0mself\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0mbase_model\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0mprepare_inputs_for_generation\u001B[0m \u001B[0;34m=\u001B[0m \u001B[0mself\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0mbase_model_prepare_inputs_for_generation\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n",
      "\u001B[0;32m/usr/local/lib/python3.10/dist-packages/torch/utils/_contextlib.py\u001B[0m in \u001B[0;36mdecorate_context\u001B[0;34m(*args, **kwargs)\u001B[0m\n\u001B[1;32m    113\u001B[0m     \u001B[0;32mdef\u001B[0m \u001B[0mdecorate_context\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0;34m*\u001B[0m\u001B[0margs\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0;34m**\u001B[0m\u001B[0mkwargs\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m:\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m    114\u001B[0m         \u001B[0;32mwith\u001B[0m \u001B[0mctx_factory\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m:\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0;32m--> 115\u001B[0;31m             \u001B[0;32mreturn\u001B[0m \u001B[0mfunc\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0;34m*\u001B[0m\u001B[0margs\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0;34m**\u001B[0m\u001B[0mkwargs\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0m\u001B[1;32m    116\u001B[0m \u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m    117\u001B[0m     \u001B[0;32mreturn\u001B[0m \u001B[0mdecorate_context\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n",
      "\u001B[0;32m/usr/local/lib/python3.10/dist-packages/transformers/generation/utils.py\u001B[0m in \u001B[0;36mgenerate\u001B[0;34m(self, inputs, generation_config, logits_processor, stopping_criteria, prefix_allowed_tokens_fn, synced_gpus, assistant_model, streamer, **kwargs)\u001B[0m\n\u001B[1;32m   1536\u001B[0m \u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m   1537\u001B[0m             \u001B[0;31m# 11. run greedy search\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0;32m-> 1538\u001B[0;31m             return self.greedy_search(\n\u001B[0m\u001B[1;32m   1539\u001B[0m                 \u001B[0minput_ids\u001B[0m\u001B[0;34m,\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m   1540\u001B[0m                 \u001B[0mlogits_processor\u001B[0m\u001B[0;34m=\u001B[0m\u001B[0mlogits_processor\u001B[0m\u001B[0;34m,\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n",
      "\u001B[0;32m/usr/local/lib/python3.10/dist-packages/transformers/generation/utils.py\u001B[0m in \u001B[0;36mgreedy_search\u001B[0;34m(self, input_ids, logits_processor, stopping_criteria, max_length, pad_token_id, eos_token_id, output_attentions, output_hidden_states, output_scores, return_dict_in_generate, synced_gpus, streamer, **model_kwargs)\u001B[0m\n\u001B[1;32m   2429\u001B[0m \u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m   2430\u001B[0m             \u001B[0;31m# forward pass to get next token\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0;32m-> 2431\u001B[0;31m             outputs = self(\n\u001B[0m\u001B[1;32m   2432\u001B[0m                 \u001B[0;34m**\u001B[0m\u001B[0mmodel_inputs\u001B[0m\u001B[0;34m,\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m   2433\u001B[0m                 \u001B[0mreturn_dict\u001B[0m\u001B[0;34m=\u001B[0m\u001B[0;32mTrue\u001B[0m\u001B[0;34m,\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n",
      "\u001B[0;32m/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py\u001B[0m in \u001B[0;36m_call_impl\u001B[0;34m(self, *args, **kwargs)\u001B[0m\n\u001B[1;32m   1499\u001B[0m                 \u001B[0;32mor\u001B[0m \u001B[0m_global_backward_pre_hooks\u001B[0m \u001B[0;32mor\u001B[0m \u001B[0m_global_backward_hooks\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m   1500\u001B[0m                 or _global_forward_hooks or _global_forward_pre_hooks):\n\u001B[0;32m-> 1501\u001B[0;31m             \u001B[0;32mreturn\u001B[0m \u001B[0mforward_call\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0;34m*\u001B[0m\u001B[0margs\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0;34m**\u001B[0m\u001B[0mkwargs\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0m\u001B[1;32m   1502\u001B[0m         \u001B[0;31m# Do not call functions when jit is used\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m   1503\u001B[0m         \u001B[0mfull_backward_hooks\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0mnon_full_backward_hooks\u001B[0m \u001B[0;34m=\u001B[0m \u001B[0;34m[\u001B[0m\u001B[0;34m]\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0;34m[\u001B[0m\u001B[0;34m]\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n",
      "\u001B[0;32m/usr/local/lib/python3.10/dist-packages/accelerate/hooks.py\u001B[0m in \u001B[0;36mnew_forward\u001B[0;34m(*args, **kwargs)\u001B[0m\n\u001B[1;32m    163\u001B[0m                 \u001B[0moutput\u001B[0m \u001B[0;34m=\u001B[0m \u001B[0mold_forward\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0;34m*\u001B[0m\u001B[0margs\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0;34m**\u001B[0m\u001B[0mkwargs\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m    164\u001B[0m         \u001B[0;32melse\u001B[0m\u001B[0;34m:\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0;32m--> 165\u001B[0;31m             \u001B[0moutput\u001B[0m \u001B[0;34m=\u001B[0m \u001B[0mold_forward\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0;34m*\u001B[0m\u001B[0margs\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0;34m**\u001B[0m\u001B[0mkwargs\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0m\u001B[1;32m    166\u001B[0m         \u001B[0;32mreturn\u001B[0m \u001B[0mmodule\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0m_hf_hook\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0mpost_forward\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0mmodule\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0moutput\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m    167\u001B[0m \u001B[0;34m\u001B[0m\u001B[0m\n",
      "\u001B[0;32m/usr/local/lib/python3.10/dist-packages/transformers/models/opt/modeling_opt.py\u001B[0m in \u001B[0;36mforward\u001B[0;34m(self, input_ids, attention_mask, head_mask, past_key_values, inputs_embeds, labels, use_cache, output_attentions, output_hidden_states, return_dict)\u001B[0m\n\u001B[1;32m    942\u001B[0m \u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m    943\u001B[0m         \u001B[0;31m# decoder outputs consists of (dec_features, layer_state, dec_hidden, dec_attn)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0;32m--> 944\u001B[0;31m         outputs = self.model.decoder(\n\u001B[0m\u001B[1;32m    945\u001B[0m             \u001B[0minput_ids\u001B[0m\u001B[0;34m=\u001B[0m\u001B[0minput_ids\u001B[0m\u001B[0;34m,\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m    946\u001B[0m             \u001B[0mattention_mask\u001B[0m\u001B[0;34m=\u001B[0m\u001B[0mattention_mask\u001B[0m\u001B[0;34m,\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n",
      "\u001B[0;32m/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py\u001B[0m in \u001B[0;36m_call_impl\u001B[0;34m(self, *args, **kwargs)\u001B[0m\n\u001B[1;32m   1499\u001B[0m                 \u001B[0;32mor\u001B[0m \u001B[0m_global_backward_pre_hooks\u001B[0m \u001B[0;32mor\u001B[0m \u001B[0m_global_backward_hooks\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m   1500\u001B[0m                 or _global_forward_hooks or _global_forward_pre_hooks):\n\u001B[0;32m-> 1501\u001B[0;31m             \u001B[0;32mreturn\u001B[0m \u001B[0mforward_call\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0;34m*\u001B[0m\u001B[0margs\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0;34m**\u001B[0m\u001B[0mkwargs\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0m\u001B[1;32m   1502\u001B[0m         \u001B[0;31m# Do not call functions when jit is used\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m   1503\u001B[0m         \u001B[0mfull_backward_hooks\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0mnon_full_backward_hooks\u001B[0m \u001B[0;34m=\u001B[0m \u001B[0;34m[\u001B[0m\u001B[0;34m]\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0;34m[\u001B[0m\u001B[0;34m]\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n",
      "\u001B[0;32m/usr/local/lib/python3.10/dist-packages/accelerate/hooks.py\u001B[0m in \u001B[0;36mnew_forward\u001B[0;34m(*args, **kwargs)\u001B[0m\n\u001B[1;32m    163\u001B[0m                 \u001B[0moutput\u001B[0m \u001B[0;34m=\u001B[0m \u001B[0mold_forward\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0;34m*\u001B[0m\u001B[0margs\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0;34m**\u001B[0m\u001B[0mkwargs\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m    164\u001B[0m         \u001B[0;32melse\u001B[0m\u001B[0;34m:\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0;32m--> 165\u001B[0;31m             \u001B[0moutput\u001B[0m \u001B[0;34m=\u001B[0m \u001B[0mold_forward\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0;34m*\u001B[0m\u001B[0margs\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0;34m**\u001B[0m\u001B[0mkwargs\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0m\u001B[1;32m    166\u001B[0m         \u001B[0;32mreturn\u001B[0m \u001B[0mmodule\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0m_hf_hook\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0mpost_forward\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0mmodule\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0moutput\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m    167\u001B[0m \u001B[0;34m\u001B[0m\u001B[0m\n",
      "\u001B[0;32m/usr/local/lib/python3.10/dist-packages/transformers/models/opt/modeling_opt.py\u001B[0m in \u001B[0;36mforward\u001B[0;34m(self, input_ids, attention_mask, head_mask, past_key_values, inputs_embeds, use_cache, output_attentions, output_hidden_states, return_dict)\u001B[0m\n\u001B[1;32m    708\u001B[0m                 )\n\u001B[1;32m    709\u001B[0m             \u001B[0;32melse\u001B[0m\u001B[0;34m:\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0;32m--> 710\u001B[0;31m                 layer_outputs = decoder_layer(\n\u001B[0m\u001B[1;32m    711\u001B[0m                     \u001B[0mhidden_states\u001B[0m\u001B[0;34m,\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m    712\u001B[0m                     \u001B[0mattention_mask\u001B[0m\u001B[0;34m=\u001B[0m\u001B[0mcausal_attention_mask\u001B[0m\u001B[0;34m,\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n",
      "\u001B[0;32m/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py\u001B[0m in \u001B[0;36m_call_impl\u001B[0;34m(self, *args, **kwargs)\u001B[0m\n\u001B[1;32m   1499\u001B[0m                 \u001B[0;32mor\u001B[0m \u001B[0m_global_backward_pre_hooks\u001B[0m \u001B[0;32mor\u001B[0m \u001B[0m_global_backward_hooks\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m   1500\u001B[0m                 or _global_forward_hooks or _global_forward_pre_hooks):\n\u001B[0;32m-> 1501\u001B[0;31m             \u001B[0;32mreturn\u001B[0m \u001B[0mforward_call\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0;34m*\u001B[0m\u001B[0margs\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0;34m**\u001B[0m\u001B[0mkwargs\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0m\u001B[1;32m   1502\u001B[0m         \u001B[0;31m# Do not call functions when jit is used\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m   1503\u001B[0m         \u001B[0mfull_backward_hooks\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0mnon_full_backward_hooks\u001B[0m \u001B[0;34m=\u001B[0m \u001B[0;34m[\u001B[0m\u001B[0;34m]\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0;34m[\u001B[0m\u001B[0;34m]\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n",
      "\u001B[0;32m/usr/local/lib/python3.10/dist-packages/accelerate/hooks.py\u001B[0m in \u001B[0;36mnew_forward\u001B[0;34m(*args, **kwargs)\u001B[0m\n\u001B[1;32m    163\u001B[0m                 \u001B[0moutput\u001B[0m \u001B[0;34m=\u001B[0m \u001B[0mold_forward\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0;34m*\u001B[0m\u001B[0margs\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0;34m**\u001B[0m\u001B[0mkwargs\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m    164\u001B[0m         \u001B[0;32melse\u001B[0m\u001B[0;34m:\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0;32m--> 165\u001B[0;31m             \u001B[0moutput\u001B[0m \u001B[0;34m=\u001B[0m \u001B[0mold_forward\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0;34m*\u001B[0m\u001B[0margs\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0;34m**\u001B[0m\u001B[0mkwargs\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0m\u001B[1;32m    166\u001B[0m         \u001B[0;32mreturn\u001B[0m \u001B[0mmodule\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0m_hf_hook\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0mpost_forward\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0mmodule\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0moutput\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m    167\u001B[0m \u001B[0;34m\u001B[0m\u001B[0m\n",
      "\u001B[0;32m/usr/local/lib/python3.10/dist-packages/transformers/models/opt/modeling_opt.py\u001B[0m in \u001B[0;36mforward\u001B[0;34m(self, hidden_states, attention_mask, layer_head_mask, past_key_value, output_attentions, use_cache)\u001B[0m\n\u001B[1;32m    328\u001B[0m \u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m    329\u001B[0m         \u001B[0;31m# Self Attention\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0;32m--> 330\u001B[0;31m         hidden_states, self_attn_weights, present_key_value = self.self_attn(\n\u001B[0m\u001B[1;32m    331\u001B[0m             \u001B[0mhidden_states\u001B[0m\u001B[0;34m=\u001B[0m\u001B[0mhidden_states\u001B[0m\u001B[0;34m,\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m    332\u001B[0m             \u001B[0mpast_key_value\u001B[0m\u001B[0;34m=\u001B[0m\u001B[0mpast_key_value\u001B[0m\u001B[0;34m,\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n",
      "\u001B[0;32m/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py\u001B[0m in \u001B[0;36m_call_impl\u001B[0;34m(self, *args, **kwargs)\u001B[0m\n\u001B[1;32m   1499\u001B[0m                 \u001B[0;32mor\u001B[0m \u001B[0m_global_backward_pre_hooks\u001B[0m \u001B[0;32mor\u001B[0m \u001B[0m_global_backward_hooks\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m   1500\u001B[0m                 or _global_forward_hooks or _global_forward_pre_hooks):\n\u001B[0;32m-> 1501\u001B[0;31m             \u001B[0;32mreturn\u001B[0m \u001B[0mforward_call\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0;34m*\u001B[0m\u001B[0margs\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0;34m**\u001B[0m\u001B[0mkwargs\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0m\u001B[1;32m   1502\u001B[0m         \u001B[0;31m# Do not call functions when jit is used\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m   1503\u001B[0m         \u001B[0mfull_backward_hooks\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0mnon_full_backward_hooks\u001B[0m \u001B[0;34m=\u001B[0m \u001B[0;34m[\u001B[0m\u001B[0;34m]\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0;34m[\u001B[0m\u001B[0;34m]\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n",
      "\u001B[0;32m/usr/local/lib/python3.10/dist-packages/accelerate/hooks.py\u001B[0m in \u001B[0;36mnew_forward\u001B[0;34m(*args, **kwargs)\u001B[0m\n\u001B[1;32m    163\u001B[0m                 \u001B[0moutput\u001B[0m \u001B[0;34m=\u001B[0m \u001B[0mold_forward\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0;34m*\u001B[0m\u001B[0margs\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0;34m**\u001B[0m\u001B[0mkwargs\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m    164\u001B[0m         \u001B[0;32melse\u001B[0m\u001B[0;34m:\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0;32m--> 165\u001B[0;31m             \u001B[0moutput\u001B[0m \u001B[0;34m=\u001B[0m \u001B[0mold_forward\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0;34m*\u001B[0m\u001B[0margs\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0;34m**\u001B[0m\u001B[0mkwargs\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0m\u001B[1;32m    166\u001B[0m         \u001B[0;32mreturn\u001B[0m \u001B[0mmodule\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0m_hf_hook\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0mpost_forward\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0mmodule\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0moutput\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m    167\u001B[0m \u001B[0;34m\u001B[0m\u001B[0m\n",
      "\u001B[0;32m/usr/local/lib/python3.10/dist-packages/transformers/models/opt/modeling_opt.py\u001B[0m in \u001B[0;36mforward\u001B[0;34m(self, hidden_states, key_value_states, past_key_value, attention_mask, layer_head_mask, output_attentions)\u001B[0m\n\u001B[1;32m    183\u001B[0m         \u001B[0;32melif\u001B[0m \u001B[0mpast_key_value\u001B[0m \u001B[0;32mis\u001B[0m \u001B[0;32mnot\u001B[0m \u001B[0;32mNone\u001B[0m\u001B[0;34m:\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m    184\u001B[0m             \u001B[0;31m# reuse k, v, self_attention\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0;32m--> 185\u001B[0;31m             \u001B[0mkey_states\u001B[0m \u001B[0;34m=\u001B[0m \u001B[0mself\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0m_shape\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0mself\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0mk_proj\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0mhidden_states\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0;34m-\u001B[0m\u001B[0;36m1\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0mbsz\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0m\u001B[1;32m    186\u001B[0m             \u001B[0mvalue_states\u001B[0m \u001B[0;34m=\u001B[0m \u001B[0mself\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0m_shape\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0mself\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0mv_proj\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0mhidden_states\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0;34m-\u001B[0m\u001B[0;36m1\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0mbsz\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m    187\u001B[0m             \u001B[0mkey_states\u001B[0m \u001B[0;34m=\u001B[0m \u001B[0mtorch\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0mcat\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0;34m[\u001B[0m\u001B[0mpast_key_value\u001B[0m\u001B[0;34m[\u001B[0m\u001B[0;36m0\u001B[0m\u001B[0;34m]\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0mkey_states\u001B[0m\u001B[0;34m]\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0mdim\u001B[0m\u001B[0;34m=\u001B[0m\u001B[0;36m2\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n",
      "\u001B[0;32m/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py\u001B[0m in \u001B[0;36m_call_impl\u001B[0;34m(self, *args, **kwargs)\u001B[0m\n\u001B[1;32m   1499\u001B[0m                 \u001B[0;32mor\u001B[0m \u001B[0m_global_backward_pre_hooks\u001B[0m \u001B[0;32mor\u001B[0m \u001B[0m_global_backward_hooks\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m   1500\u001B[0m                 or _global_forward_hooks or _global_forward_pre_hooks):\n\u001B[0;32m-> 1501\u001B[0;31m             \u001B[0;32mreturn\u001B[0m \u001B[0mforward_call\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0;34m*\u001B[0m\u001B[0margs\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0;34m**\u001B[0m\u001B[0mkwargs\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0m\u001B[1;32m   1502\u001B[0m         \u001B[0;31m# Do not call functions when jit is used\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m   1503\u001B[0m         \u001B[0mfull_backward_hooks\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0mnon_full_backward_hooks\u001B[0m \u001B[0;34m=\u001B[0m \u001B[0;34m[\u001B[0m\u001B[0;34m]\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0;34m[\u001B[0m\u001B[0;34m]\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n",
      "\u001B[0;32m/usr/local/lib/python3.10/dist-packages/accelerate/hooks.py\u001B[0m in \u001B[0;36mnew_forward\u001B[0;34m(*args, **kwargs)\u001B[0m\n\u001B[1;32m    163\u001B[0m                 \u001B[0moutput\u001B[0m \u001B[0;34m=\u001B[0m \u001B[0mold_forward\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0;34m*\u001B[0m\u001B[0margs\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0;34m**\u001B[0m\u001B[0mkwargs\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m    164\u001B[0m         \u001B[0;32melse\u001B[0m\u001B[0;34m:\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0;32m--> 165\u001B[0;31m             \u001B[0moutput\u001B[0m \u001B[0;34m=\u001B[0m \u001B[0mold_forward\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0;34m*\u001B[0m\u001B[0margs\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0;34m**\u001B[0m\u001B[0mkwargs\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0m\u001B[1;32m    166\u001B[0m         \u001B[0;32mreturn\u001B[0m \u001B[0mmodule\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0m_hf_hook\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0mpost_forward\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0mmodule\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0moutput\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m    167\u001B[0m \u001B[0;34m\u001B[0m\u001B[0m\n",
      "\u001B[0;32m/usr/local/lib/python3.10/dist-packages/bitsandbytes/nn/modules.py\u001B[0m in \u001B[0;36mforward\u001B[0;34m(self, x)\u001B[0m\n\u001B[1;32m    439\u001B[0m             \u001B[0mself\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0mbias\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0mdata\u001B[0m \u001B[0;34m=\u001B[0m \u001B[0mself\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0mbias\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0mdata\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0mto\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0mx\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0mdtype\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m    440\u001B[0m \u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0;32m--> 441\u001B[0;31m         \u001B[0mout\u001B[0m \u001B[0;34m=\u001B[0m \u001B[0mbnb\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0mmatmul\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0mx\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0mself\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0mweight\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0mbias\u001B[0m\u001B[0;34m=\u001B[0m\u001B[0mself\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0mbias\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0mstate\u001B[0m\u001B[0;34m=\u001B[0m\u001B[0mself\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0mstate\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0m\u001B[1;32m    442\u001B[0m \u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m    443\u001B[0m         \u001B[0;32mif\u001B[0m \u001B[0;32mnot\u001B[0m \u001B[0mself\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0mstate\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0mhas_fp16_weights\u001B[0m\u001B[0;34m:\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n",
      "\u001B[0;32m/usr/local/lib/python3.10/dist-packages/bitsandbytes/autograd/_functions.py\u001B[0m in \u001B[0;36mmatmul\u001B[0;34m(A, B, out, state, threshold, bias)\u001B[0m\n\u001B[1;32m    561\u001B[0m     \u001B[0;32mif\u001B[0m \u001B[0mthreshold\u001B[0m \u001B[0;34m>\u001B[0m \u001B[0;36m0.0\u001B[0m\u001B[0;34m:\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m    562\u001B[0m         \u001B[0mstate\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0mthreshold\u001B[0m \u001B[0;34m=\u001B[0m \u001B[0mthreshold\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0;32m--> 563\u001B[0;31m     \u001B[0;32mreturn\u001B[0m \u001B[0mMatMul8bitLt\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0mapply\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0mA\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0mB\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0mout\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0mbias\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0mstate\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0m\u001B[1;32m    564\u001B[0m \u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m    565\u001B[0m \u001B[0;34m\u001B[0m\u001B[0m\n",
      "\u001B[0;32m/usr/local/lib/python3.10/dist-packages/torch/autograd/function.py\u001B[0m in \u001B[0;36mapply\u001B[0;34m(cls, *args, **kwargs)\u001B[0m\n\u001B[1;32m    504\u001B[0m             \u001B[0;31m# See NOTE: [functorch vjp and autograd interaction]\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m    505\u001B[0m             \u001B[0margs\u001B[0m \u001B[0;34m=\u001B[0m \u001B[0m_functorch\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0mutils\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0munwrap_dead_wrappers\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0margs\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0;32m--> 506\u001B[0;31m             \u001B[0;32mreturn\u001B[0m \u001B[0msuper\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0mapply\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0;34m*\u001B[0m\u001B[0margs\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0;34m**\u001B[0m\u001B[0mkwargs\u001B[0m\u001B[0;34m)\u001B[0m  \u001B[0;31m# type: ignore[misc]\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0m\u001B[1;32m    507\u001B[0m \u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m    508\u001B[0m         \u001B[0;32mif\u001B[0m \u001B[0mcls\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0msetup_context\u001B[0m \u001B[0;34m==\u001B[0m \u001B[0m_SingleLevelFunction\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0msetup_context\u001B[0m\u001B[0;34m:\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n",
      "\u001B[0;31mKeyboardInterrupt\u001B[0m: "
     ]
    }
   ]
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "machine_shape": "hm",
   "provenance": [],
   "gpuType": "T4",
   "toc_visible": true
  },
  "kernelspec": {
   "display_name": "Python 3",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.4"
  },
  "widgets": {
   "application/vnd.jupyter.widget-state+json": {
    "f1c9b3744c664e21aacf2cd220917058": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "VBoxModel",
     "model_module_version": "1.5.0",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "VBoxModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "VBoxView",
      "box_style": "",
      "children": [
       "IPY_MODEL_288ea5c2df5f4712ad47aa39c5c7bb88",
       "IPY_MODEL_fa5703441e324e10a26fb4ecd3f2b7e6",
       "IPY_MODEL_c2d4c1a1482d436bab5f61247fc994f6",
       "IPY_MODEL_9dc89055fb4f44959bbcb26aae068e05"
      ],
      "layout": "IPY_MODEL_6d5dfab877704054b6a66b619c32c946"
     }
    },
    "13e0c5018eb64383bd7fedb7f346fb16": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HTMLModel",
     "model_module_version": "1.5.0",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_dc3e506bec8a47c6a57312c437779bb6",
      "placeholder": "​",
      "style": "IPY_MODEL_6e557f79ac8549948f8c86cbfa71d6cd",
      "value": "<center> <img\nsrc=https://huggingface.co/front/assets/huggingface_logo-noborder.svg\nalt='Hugging Face'> <br> Copy a token from <a\nhref=\"https://huggingface.co/settings/tokens\" target=\"_blank\">your Hugging Face\ntokens page</a> and paste it below. <br> Immediately click login after copying\nyour token or it might be stored in plain text in this notebook file. </center>"
     }
    },
    "8e6c0b740532485ba3233ba20e16ade6": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "PasswordModel",
     "model_module_version": "1.5.0",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "PasswordModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "PasswordView",
      "continuous_update": true,
      "description": "Token:",
      "description_tooltip": null,
      "disabled": false,
      "layout": "IPY_MODEL_7d90d18918a742b2a6848431ba093ab6",
      "placeholder": "​",
      "style": "IPY_MODEL_aa436b151a2a499792ba59191d974c75",
      "value": ""
     }
    },
    "0098e91e661942f28975858a37524d21": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "CheckboxModel",
     "model_module_version": "1.5.0",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "CheckboxModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "CheckboxView",
      "description": "Add token as git credential?",
      "description_tooltip": null,
      "disabled": false,
      "indent": true,
      "layout": "IPY_MODEL_b59e3b393e794b7491f55d3262f63d96",
      "style": "IPY_MODEL_64237cc139ec4322844e29cb9bb4629c",
      "value": true
     }
    },
    "653497ed2c4843d0b51724021c18f9e6": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "ButtonModel",
     "model_module_version": "1.5.0",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "ButtonModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "ButtonView",
      "button_style": "",
      "description": "Login",
      "disabled": false,
      "icon": "",
      "layout": "IPY_MODEL_6f1bd533ac984413aee792a7fc1235bf",
      "style": "IPY_MODEL_483239d349ad413791ff7e26cad6f6af",
      "tooltip": ""
     }
    },
    "6007657f9bdb4f98be8d60fa5cfa4154": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HTMLModel",
     "model_module_version": "1.5.0",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_b629ca6df71f4a5abeb545bd84bb5031",
      "placeholder": "​",
      "style": "IPY_MODEL_bf4f2788a6e04ef6a5c49c2a08cf66c7",
      "value": "\n<b>Pro Tip:</b> If you don't already have one, you can create a dedicated\n'notebooks' token with 'write' access, that you can then easily reuse for all\nnotebooks. </center>"
     }
    },
    "6d5dfab877704054b6a66b619c32c946": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "model_module_version": "1.2.0",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": "center",
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": "flex",
      "flex": null,
      "flex_flow": "column",
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": "50%"
     }
    },
    "dc3e506bec8a47c6a57312c437779bb6": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "model_module_version": "1.2.0",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "6e557f79ac8549948f8c86cbfa71d6cd": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "DescriptionStyleModel",
     "model_module_version": "1.5.0",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "7d90d18918a742b2a6848431ba093ab6": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "model_module_version": "1.2.0",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "aa436b151a2a499792ba59191d974c75": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "DescriptionStyleModel",
     "model_module_version": "1.5.0",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "b59e3b393e794b7491f55d3262f63d96": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "model_module_version": "1.2.0",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "64237cc139ec4322844e29cb9bb4629c": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "DescriptionStyleModel",
     "model_module_version": "1.5.0",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "6f1bd533ac984413aee792a7fc1235bf": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "model_module_version": "1.2.0",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "483239d349ad413791ff7e26cad6f6af": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "ButtonStyleModel",
     "model_module_version": "1.5.0",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "ButtonStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "button_color": null,
      "font_weight": ""
     }
    },
    "b629ca6df71f4a5abeb545bd84bb5031": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "model_module_version": "1.2.0",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "bf4f2788a6e04ef6a5c49c2a08cf66c7": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "DescriptionStyleModel",
     "model_module_version": "1.5.0",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "03ea3b9bd96b427394c311c6a03a6edb": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "LabelModel",
     "model_module_version": "1.5.0",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "LabelModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "LabelView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_779027195c30440dbe9de99fb115c9c6",
      "placeholder": "​",
      "style": "IPY_MODEL_2f375cd9577d4444aba5e5b951643806",
      "value": "Connecting..."
     }
    },
    "779027195c30440dbe9de99fb115c9c6": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "model_module_version": "1.2.0",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "2f375cd9577d4444aba5e5b951643806": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "DescriptionStyleModel",
     "model_module_version": "1.5.0",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "288ea5c2df5f4712ad47aa39c5c7bb88": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "LabelModel",
     "model_module_version": "1.5.0",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "LabelModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "LabelView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_09146549434d48848f4087be4ad38852",
      "placeholder": "​",
      "style": "IPY_MODEL_1c9ec3b458f74d98b95323cb7acf5d0d",
      "value": "Token is valid (permission: write)."
     }
    },
    "fa5703441e324e10a26fb4ecd3f2b7e6": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "LabelModel",
     "model_module_version": "1.5.0",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "LabelModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "LabelView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_d637855805804229b358030385ded12a",
      "placeholder": "​",
      "style": "IPY_MODEL_e9eb3a83dcc348169f679ccb4187cafb",
      "value": "Your token has been saved in your configured git credential helpers (store)."
     }
    },
    "c2d4c1a1482d436bab5f61247fc994f6": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "LabelModel",
     "model_module_version": "1.5.0",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "LabelModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "LabelView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_fecc475a8f8244a29ca44b995d2274c8",
      "placeholder": "​",
      "style": "IPY_MODEL_f6c8be78a6cb4f079d21e6382cfe4418",
      "value": "Your token has been saved to /root/.cache/huggingface/token"
     }
    },
    "9dc89055fb4f44959bbcb26aae068e05": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "LabelModel",
     "model_module_version": "1.5.0",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "LabelModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "LabelView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_bbd22b8185074ca1b23ce05b6be0ce0d",
      "placeholder": "​",
      "style": "IPY_MODEL_78d6c9442171435789aa8b303ba95e07",
      "value": "Login successful"
     }
    },
    "09146549434d48848f4087be4ad38852": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "model_module_version": "1.2.0",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "1c9ec3b458f74d98b95323cb7acf5d0d": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "DescriptionStyleModel",
     "model_module_version": "1.5.0",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "d637855805804229b358030385ded12a": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "model_module_version": "1.2.0",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "e9eb3a83dcc348169f679ccb4187cafb": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "DescriptionStyleModel",
     "model_module_version": "1.5.0",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "fecc475a8f8244a29ca44b995d2274c8": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "model_module_version": "1.2.0",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "f6c8be78a6cb4f079d21e6382cfe4418": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "DescriptionStyleModel",
     "model_module_version": "1.5.0",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "bbd22b8185074ca1b23ce05b6be0ce0d": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "model_module_version": "1.2.0",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "78d6c9442171435789aa8b303ba95e07": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "DescriptionStyleModel",
     "model_module_version": "1.5.0",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "50b07294170b4501b855bc589080022c": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HBoxModel",
     "model_module_version": "1.5.0",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HBoxModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HBoxView",
      "box_style": "",
      "children": [
       "IPY_MODEL_c595188878274d269330d3ab2687c29f",
       "IPY_MODEL_b3686ca7a89047a396e54c9838ef97e6",
       "IPY_MODEL_a6689c4f31f1485f888fa7b85f0026ab"
      ],
      "layout": "IPY_MODEL_7308ac7e81794d1ca0883d78201ca801"
     }
    },
    "c595188878274d269330d3ab2687c29f": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HTMLModel",
     "model_module_version": "1.5.0",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_c6dfd6c0e707466c9a4012233b4d1161",
      "placeholder": "​",
      "style": "IPY_MODEL_210eae0956014ec0943dfe3cec7eb63f",
      "value": "Map:   1%"
     }
    },
    "b3686ca7a89047a396e54c9838ef97e6": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "FloatProgressModel",
     "model_module_version": "1.5.0",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "FloatProgressModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "ProgressView",
      "bar_style": "danger",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_fd3d5aebc9a3470cb3c47109d9c6b989",
      "max": 8657,
      "min": 0,
      "orientation": "horizontal",
      "style": "IPY_MODEL_53894ce1b11941a79f1dcb06d64e6d56",
      "value": 55
     }
    },
    "a6689c4f31f1485f888fa7b85f0026ab": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HTMLModel",
     "model_module_version": "1.5.0",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_eaccf75741294a928b17db180c7a6067",
      "placeholder": "​",
      "style": "IPY_MODEL_f600fe53ce314f42940c0cf95934f9d8",
      "value": " 55/8657 [00:41&lt;1:46:56,  1.34 examples/s]"
     }
    },
    "7308ac7e81794d1ca0883d78201ca801": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "model_module_version": "1.2.0",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "c6dfd6c0e707466c9a4012233b4d1161": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "model_module_version": "1.2.0",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "210eae0956014ec0943dfe3cec7eb63f": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "DescriptionStyleModel",
     "model_module_version": "1.5.0",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "fd3d5aebc9a3470cb3c47109d9c6b989": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "model_module_version": "1.2.0",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "53894ce1b11941a79f1dcb06d64e6d56": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "ProgressStyleModel",
     "model_module_version": "1.5.0",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "ProgressStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "bar_color": null,
      "description_width": ""
     }
    },
    "eaccf75741294a928b17db180c7a6067": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "model_module_version": "1.2.0",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "f600fe53ce314f42940c0cf95934f9d8": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "DescriptionStyleModel",
     "model_module_version": "1.5.0",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "62363de45e124f6f8312383ad51c80c2": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HBoxModel",
     "model_module_version": "1.5.0",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HBoxModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HBoxView",
      "box_style": "",
      "children": [
       "IPY_MODEL_8462c2be6c244ee2b14e7501707050e0",
       "IPY_MODEL_f7e67464c7b84577bb2361e65ce4a00c",
       "IPY_MODEL_5db00c27391c4170b49c9000594f9b3d"
      ],
      "layout": "IPY_MODEL_3e5ec4d67cdd48e78923609978cc8e5c"
     }
    },
    "8462c2be6c244ee2b14e7501707050e0": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HTMLModel",
     "model_module_version": "1.5.0",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_4f7f7e90360946ada6f2c75605539422",
      "placeholder": "​",
      "style": "IPY_MODEL_d6b7fed90999402596382c7de0a8249e",
      "value": "100%"
     }
    },
    "f7e67464c7b84577bb2361e65ce4a00c": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "FloatProgressModel",
     "model_module_version": "1.5.0",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "FloatProgressModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "ProgressView",
      "bar_style": "success",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_9c860220bded4cdfb64bdf31606813f5",
      "max": 2,
      "min": 0,
      "orientation": "horizontal",
      "style": "IPY_MODEL_4bde8b3db74f499886feb2a27ae6d22d",
      "value": 2
     }
    },
    "5db00c27391c4170b49c9000594f9b3d": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HTMLModel",
     "model_module_version": "1.5.0",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_596bdb8c9cd848ddb240389f060d094b",
      "placeholder": "​",
      "style": "IPY_MODEL_f709eb51edf040959fae148fab8331cd",
      "value": " 2/2 [00:00&lt;00:00, 88.15it/s]"
     }
    },
    "3e5ec4d67cdd48e78923609978cc8e5c": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "model_module_version": "1.2.0",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "4f7f7e90360946ada6f2c75605539422": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "model_module_version": "1.2.0",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "d6b7fed90999402596382c7de0a8249e": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "DescriptionStyleModel",
     "model_module_version": "1.5.0",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "9c860220bded4cdfb64bdf31606813f5": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "model_module_version": "1.2.0",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "4bde8b3db74f499886feb2a27ae6d22d": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "ProgressStyleModel",
     "model_module_version": "1.5.0",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "ProgressStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "bar_color": null,
      "description_width": ""
     }
    },
    "596bdb8c9cd848ddb240389f060d094b": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "model_module_version": "1.2.0",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "f709eb51edf040959fae148fab8331cd": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "DescriptionStyleModel",
     "model_module_version": "1.5.0",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "a281f6c58b294f38989576054d04fc2c": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HBoxModel",
     "model_module_version": "1.5.0",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HBoxModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HBoxView",
      "box_style": "",
      "children": [
       "IPY_MODEL_c7afb893b2d247c6ae31724da1ad409c",
       "IPY_MODEL_11075f98f5e042148754dfa538922777",
       "IPY_MODEL_9e227702a5854e7db485fe2c1b0bf854"
      ],
      "layout": "IPY_MODEL_398fd823bdfb4a30905aa5a8647391c7"
     }
    },
    "c7afb893b2d247c6ae31724da1ad409c": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HTMLModel",
     "model_module_version": "1.5.0",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_de9ad0e9e3b84e8bbf552e72c2faded2",
      "placeholder": "​",
      "style": "IPY_MODEL_aa1571bdfebc4cb08b96d73c0087ef72",
      "value": "Map: 100%"
     }
    },
    "11075f98f5e042148754dfa538922777": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "FloatProgressModel",
     "model_module_version": "1.5.0",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "FloatProgressModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "ProgressView",
      "bar_style": "",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_6ba40d1a265f4a5e981d3df233911e18",
      "max": 105479,
      "min": 0,
      "orientation": "horizontal",
      "style": "IPY_MODEL_8f742fec5b2841a28858b91e6d5da1da",
      "value": 105479
     }
    },
    "9e227702a5854e7db485fe2c1b0bf854": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HTMLModel",
     "model_module_version": "1.5.0",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_f0f28d7ffca64ee58262105ed6d7a1d4",
      "placeholder": "​",
      "style": "IPY_MODEL_9cd731136ff64161abd0f3b6ab23a835",
      "value": " 105000/105479 [00:20&lt;00:00, 5400.52 examples/s]"
     }
    },
    "398fd823bdfb4a30905aa5a8647391c7": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "model_module_version": "1.2.0",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": "hidden",
      "width": null
     }
    },
    "de9ad0e9e3b84e8bbf552e72c2faded2": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "model_module_version": "1.2.0",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "aa1571bdfebc4cb08b96d73c0087ef72": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "DescriptionStyleModel",
     "model_module_version": "1.5.0",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "6ba40d1a265f4a5e981d3df233911e18": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "model_module_version": "1.2.0",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "8f742fec5b2841a28858b91e6d5da1da": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "ProgressStyleModel",
     "model_module_version": "1.5.0",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "ProgressStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "bar_color": null,
      "description_width": ""
     }
    },
    "f0f28d7ffca64ee58262105ed6d7a1d4": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "model_module_version": "1.2.0",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "9cd731136ff64161abd0f3b6ab23a835": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "DescriptionStyleModel",
     "model_module_version": "1.5.0",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "b72a6b71410b421fbb6a550a93013fb1": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HBoxModel",
     "model_module_version": "1.5.0",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HBoxModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HBoxView",
      "box_style": "",
      "children": [
       "IPY_MODEL_a3156a7a425c4dc4a9adefc553b198b3",
       "IPY_MODEL_3d7d41c4f34e4e1b92fc70d77b4c60e6",
       "IPY_MODEL_55d2e2fb2b3748b18aae77b24e267bff"
      ],
      "layout": "IPY_MODEL_d2dbd101cd3040fab582701eaf0b8296"
     }
    },
    "a3156a7a425c4dc4a9adefc553b198b3": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HTMLModel",
     "model_module_version": "1.5.0",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_2f9c8600f41f47a1ab12fd7d56642fea",
      "placeholder": "​",
      "style": "IPY_MODEL_93e3c0c0e9cc465fbae3025679cf96ab",
      "value": "Map: 100%"
     }
    },
    "3d7d41c4f34e4e1b92fc70d77b4c60e6": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "FloatProgressModel",
     "model_module_version": "1.5.0",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "FloatProgressModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "ProgressView",
      "bar_style": "",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_e2523995baa54273ad1fbf42d73b0b8f",
      "max": 8657,
      "min": 0,
      "orientation": "horizontal",
      "style": "IPY_MODEL_67cc0e8aa1784dd6860fe8354b74ee95",
      "value": 8657
     }
    },
    "55d2e2fb2b3748b18aae77b24e267bff": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HTMLModel",
     "model_module_version": "1.5.0",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_40c1e27634fd4db0905c21a4b453c1ba",
      "placeholder": "​",
      "style": "IPY_MODEL_35b6a884c8dd443385f4bbfb2d3d0931",
      "value": " 8657/8657 [00:01&lt;00:00, 5408.62 examples/s]"
     }
    },
    "d2dbd101cd3040fab582701eaf0b8296": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "model_module_version": "1.2.0",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": "hidden",
      "width": null
     }
    },
    "2f9c8600f41f47a1ab12fd7d56642fea": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "model_module_version": "1.2.0",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "93e3c0c0e9cc465fbae3025679cf96ab": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "DescriptionStyleModel",
     "model_module_version": "1.5.0",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "e2523995baa54273ad1fbf42d73b0b8f": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "model_module_version": "1.2.0",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "67cc0e8aa1784dd6860fe8354b74ee95": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "ProgressStyleModel",
     "model_module_version": "1.5.0",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "ProgressStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "bar_color": null,
      "description_width": ""
     }
    },
    "40c1e27634fd4db0905c21a4b453c1ba": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "model_module_version": "1.2.0",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "35b6a884c8dd443385f4bbfb2d3d0931": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "DescriptionStyleModel",
     "model_module_version": "1.5.0",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    }
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}