lora_trainer.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "78437d00-0dfd-420f-9d1d-accf9a2cde82",
   "metadata": {},
   "source": [
    "The goal of this notebook is to distill down the essential compenents of training HF models in a way that is easy to port to a data pipeline and apply to new model training scenarios."
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3ff4105b-66b9-43ed-81d8-9769e56f1df8",
   "metadata": {},
   "source": [
    "# Imports"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "5186aef3-01b0-4a1f-ad6e-9c99d46ca824",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import pandas as pd\n",
    "from rich import print\n",
    "import torch\n",
    "from transformers import AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig\n",
    "from datetime import datetime;\n",
    "pd.set_option('display.max_colwidth', None)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f4f647f4-e46c-4b7d-bb28-4e11e6d632ab",
   "metadata": {},
   "source": [
    "# Define variables"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "88eb322b-c949-4461-9d04-7e4d396d3e72",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'2023_11_11'"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "if True:\n",
    "    date_str = datetime.now().strftime(\"%Y_%m_%d\")\n",
    "date_str"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "b85d9c78-ef0c-4d91-a9df-bd9c799da50f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'work/work_for_2023_11_11/best_checkpoint'"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "work_dir = \"work\"\n",
    "seed = 42 #reproducability  over training runs\n",
    "clobber_project_dir = False #wipe the project director clean\n",
    "project_dir = f\"work_for_{date_str}\" #project name, also used as wandb project name\n",
    "\n",
    "#creds\n",
    "hf_token = os.environ[\"HF_TOKEN\"]\n",
    "wandb_key = os.environ[\"WANDB_KEY\"]\n",
    "hf_account_name = os.environ[\"HF_ACCOUNT_NAME\"]\n",
    "\n",
    "#HF touchpoints\n",
    "hf_dataset_name = \"knkarthick/dialogsum\"\n",
    "source_hf_model_name = \"meta-llama/Llama-2-7b-hf\"\n",
    "\n",
    "checkpoint_dir = \"best_checkpoint\"\n",
    "quantization_dir = 'quantized_8bit'\n",
    "merged_dir = 'merged_model'\n",
    "quantized_train_dir = 'quant_trained_model_new'\n",
    "\n",
    "train_test_split_ratio = 0.025\n",
    "\n",
    "#LoRA hparams\n",
    "lora_r = 16  # rank\n",
    "lora_alpha = lora_r * 2\n",
    "lora_dropout = 0.05\n",
    "\n",
    "#training hparams\n",
    "epochs = 1\n",
    "max_steps = 10\n",
    "per_device_train_batch_size=16\n",
    "gradient_accumulation_steps=16\n",
    "early_stopping_patience = 3\n",
    "learning_rate = 7e-4\n",
    "logging_steps = 5\n",
    "device = \"cuda\"\n",
    "max_training_sample_length = 512 #2048\n",
    "\n",
    "#quantization params\n",
    "quantization_bits = 4\n",
    "\n",
    "destination_hf_model_name = f\"{hf_account_name}/llama2-7b-dialogsum-qlora-gptq\"\n",
    "project_path = os.path.join(work_dir,project_dir, checkpoint_dir)\n",
    "quant_path = os.path.join(project_dir, quantization_dir)\n",
    "quantized_train_path = os.path.join(project_dir, quantized_train_dir)\n",
    "merged_path = os.path.join(project_path, merged_dir)\n",
    "\n",
    "project_path"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "35d0fc97-fbe3-4097-9751-3feafcbf218b",
   "metadata": {},
   "source": [
    "## State of working directory"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "07ffbb7b-ca53-4226-a31f-407e141bc587",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">work_for_2023_11_11\n",
       "</pre>\n"
      ],
      "text/plain": [
       "work_for_2023_11_11\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "total 0\n",
      "0 drwxr-xr-x 1 root root 4096 Nov 11 18:56 .\n",
      "0 drwxr-xr-x 1 root root 4096 Nov 11 18:57 ..\n"
     ]
    }
   ],
   "source": [
    "print(project_dir)\n",
    "!mkdir -p $project_dir\n",
    "\n",
    "#Set clobber_project_dir to True to reset working directory\n",
    "if clobber_project_dir:\n",
    "    !rm -rf ./$project_dir\n",
    "\n",
    "!mkdir -p $project_dir\n",
    "\n",
    "!ls -latrs ./$project_dir"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a77d175a-1860-4ae4-a387-97981d2b3466",
   "metadata": {},
   "source": [
    "## Initiate wandb"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "256352b0-c7c7-45a3-9e5a-2e25fcae01cf",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[34m\u001b[1mwandb\u001b[0m: Currently logged in as: \u001b[33mjason-anderson-professional\u001b[0m. Use \u001b[1m`wandb login --relogin`\u001b[0m to force relogin\n",
      "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[33mWARNING\u001b[0m If you're specifying your api key in code, ensure this code is not shared publicly.\n",
      "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[33mWARNING\u001b[0m Consider setting the WANDB_API_KEY environment variable, or running `wandb login` from the command line.\n",
      "\u001b[34m\u001b[1mwandb\u001b[0m: Appending key for api.wandb.ai to your netrc file: /root/.netrc\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "Tracking run with wandb version 0.16.0"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "Run data is saved locally in <code>/app/jupyter/lora_gptq/wandb/run-20231111_185800-urbauepf</code>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "Syncing run <strong><a href='https://wandb.ai/jason-anderson-professional/work_for_2023_11_11/runs/urbauepf' target=\"_blank\">rich-tree-2</a></strong> to <a href='https://wandb.ai/jason-anderson-professional/work_for_2023_11_11' target=\"_blank\">Weights & Biases</a> (<a href='https://wandb.me/run' target=\"_blank\">docs</a>)<br/>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       " View project at <a href='https://wandb.ai/jason-anderson-professional/work_for_2023_11_11' target=\"_blank\">https://wandb.ai/jason-anderson-professional/work_for_2023_11_11</a>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       " View run at <a href='https://wandb.ai/jason-anderson-professional/work_for_2023_11_11/runs/urbauepf' target=\"_blank\">https://wandb.ai/jason-anderson-professional/work_for_2023_11_11/runs/urbauepf</a>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import wandb\n",
    "\n",
    "wandb.login(key = wandb_key)\n",
    "run = wandb.init(project=project_dir, job_type=\"training\", anonymous=\"allow\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b98c44be-3cb1-45d4-ba26-5fa6a1e39e46",
   "metadata": {},
   "source": [
    "# Process data for training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "274adcda-c2a1-4524-b964-75b69186daa3",
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import seaborn as sns\n",
    "from matplotlib.ticker import MaxNLocator\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "def analyze_generation_params(text, model):\n",
    "    print(text,\"-\"*60)\n",
    "    results = []\n",
    "    for temp in np.linspace(.5,1.5,num = 5):\n",
    "        print(f\"temp {temp}:\",end=\"\")\n",
    "        results.append(generate(text, model, prompt=False, max_new_tokens=185,temp=temp))\n",
    "    \n",
    "    print(\"-\"*60)\n",
    "    results = []\n",
    "    for top_p in np.linspace(0,1,num = 11):\n",
    "        print(f\"top_p {top_p}:\",end=\"\")\n",
    "        results.append(generate(text, model, prompt=False, max_new_tokens=185,temp=.7, top_p=top_p))\n",
    "    \n",
    "    \n",
    "    print(\"-\"*60)\n",
    "    results = []\n",
    "    for top_k in np.linspace(0,100,num = 11,dtype=int):\n",
    "        print(f\"top_k {top_k}:\",end=\"\")\n",
    "        results.append(generate(text,  model, prompt=False, max_new_tokens=185, top_k=int(top_k)))\n",
    "    \n",
    "    print(\"-\"*60)\n",
    "    results = []\n",
    "    for do_sample in [True,False]:\n",
    "        print(f\"do_sample {do_sample}:\",end=\"\")\n",
    "        results.append(generate(text,  model, prompt=False, max_new_tokens=185, do_sample = do_sample, top_k=None))\n",
    "    \n",
    "    \n",
    "    print(\"-\"*60)\n",
    "    results = []\n",
    "    for repetition_penalty in np.linspace(1.,2.,num=11):\n",
    "        print(f\"repetition_penalty {repetition_penalty}:\",end=\"\")\n",
    "        results.append(generate(text,  model, prompt=False, max_new_tokens=48,repetition_penalty=repetition_penalty))\n",
    "    \n",
    "    print(\"-\"*60)\n",
    "    results = []\n",
    "    for typical_p in np.linspace(.1,.9,num=9):\n",
    "        print(f\"typical_p {typical_p}:\",end=\"\")\n",
    "        results.append(generate(text,  model, prompt=False, max_new_tokens=48,typical_p=typical_p))\n",
    "\n",
    "\n",
    "def generate(text, model, prompt=False, temp=0.7,top_k= 40, top_p = 0.1, do_sample = True, repetition_penalty = 1.23, typical_p = 1, guidance_scale = 1, max_new_tokens = 24,):\n",
    "    from transformers import TextStreamer\n",
    "    if prompt:\n",
    "        print(text,\"-\"*20)\n",
    "    streamer = TextStreamer(tokenizer, skip_prompt=True, skip_special_tokens=False)\n",
    "    inputs = tokenizer(text, return_tensors=\"pt\").to('cuda')\n",
    "\n",
    "    # Call the generate method of the model with the given inputs and additional generation configurations.\n",
    "    outputs = model.generate(**inputs,  # \"inputs\" likely includes input_ids or prompts for generation.\n",
    "                           streamer=streamer,  # \"streamer\" not a standard parameter in HF documentation, likely custom for model-specific streaming.\n",
    "                           max_new_tokens=max_new_tokens,  # Sets the maximum number of new tokens to generate; range varies based on model and computational limits.\n",
    "                           temperature=temp,  # Controls randomness: lower->more deterministic, higher->more random; typically in range [0.5, 1.5].\n",
    "                           do_sample=do_sample,  # Enables sampling; when True, picks tokens based on probability distribution, rather than just most likely.\n",
    "                           top_p=top_p,  # Nucleus sampling: selects top p% probability tokens for sampling; range [0, 1].\n",
    "                           top_k=top_k,  # Top-k sampling: chooses from top k probability tokens; if k=0, it's the same as using no top-k.\n",
    "                           repetition_penalty=repetition_penalty,  # Penalizes repeated tokens; >1 discourages, <1 encourages repetition; typically close to 1.\n",
    "                           typical_p=typical_p,  # Typical sampling; selects tokens whose cumulative probability is above this threshold, range [0, 1], usually close to 1.\n",
    "                           guidance_scale=guidance_scale,  # Affects the scale of guidance in models that support it, like CTRL; standard range not well-defined.\n",
    "                           #seed=seed,  # Sets a seed for reproducibility; commented out, so not in use.\n",
    "                          )\n",
    "    generated_tokens = outputs[0].tolist()[len(inputs[0]):]\n",
    "    result = tokenizer.decode(generated_tokens)\n",
    "    return result\n",
    "\n",
    "def set_plot_appearance(fontsize_title=14, \n",
    "                        fontsize_label=12, \n",
    "                        fontsize_ticks=8, \n",
    "                        theme='dark'):\n",
    "    \"\"\"\n",
    "    Configure the plot's appearance with font sizes and theme for dark backgrounds.\n",
    "    \n",
    "    Parameters:\n",
    "    fontsize_title: Font size for the title.\n",
    "    fontsize_label: Font size for the x and y labels.\n",
    "    fontsize_ticks: Font size for the x and y tick labels.\n",
    "    theme: Seaborn theme for the plot's aesthetic style.\n",
    "    \"\"\"\n",
    "    sns.set_theme(style=theme, palette='pastel')\n",
    "    \n",
    "    plt.rcParams.update({\n",
    "        'figure.facecolor': '#1a1a1a',  # Very dark grey background for the figure\n",
    "        'axes.facecolor': '#1a1a1a',    # Very dark grey background for the plots\n",
    "        'grid.color': 'gray',           # Lighter grey grid lines (less contrast)\n",
    "        'text.color': 'white',          # White text for better contrast on dark bg\n",
    "        'axes.labelcolor': 'white',     # White labels for axes\n",
    "        'xtick.color': 'white',         # White x-tick labels\n",
    "        'ytick.color': 'white',         # White y-tick labels\n",
    "        'axes.labelsize': fontsize_label,\n",
    "        'axes.titlesize': fontsize_title,\n",
    "        'xtick.labelsize': fontsize_ticks,\n",
    "        'ytick.labelsize': fontsize_ticks,\n",
    "        'legend.title_fontsize': fontsize_ticks + 2,\n",
    "        'legend.fontsize': fontsize_ticks,\n",
    "        'axes.edgecolor': 'lightgrey',  # Light grey color for the axes' spines\n",
    "    })\n",
    "# Possible modern themes: 'darkgrid', 'whitegrid', 'dark', 'white', and 'ticks'.\n",
    "# Example usage: set_plot_appearance(theme='ticks')\n",
    "\n",
    "def plot_text_length_histogram(df, bucket_size=64, percentile=95):\n",
    "    \"\"\"\n",
    "    Plot a histogram of text lengths and indicate the percentile.\n",
    "\n",
    "    Parameters:\n",
    "    df: the pandas dataframe to render from\n",
    "    bucket_size: the number of characters to bucket the histogram with\n",
    "    percentile: draws a vertical line at this percentile to visualize whether an acceptable fraction of the data lenghts are being used for training set\n",
    "    \"\"\"\n",
    "    set_plot_appearance()  # Apply appearance settings to the plot\n",
    "    text_lengths = df['text'].apply(len)  # Calculate text lengths.\n",
    "    bins = range(0, max(text_lengths) + bucket_size, bucket_size)  # Define histogram bins.\n",
    "    percentile_value = np.percentile(text_lengths, percentile)  # Calculate percentile value.\n",
    "\n",
    "    fig, ax = plt.subplots()  # Create figure and axis objects.\n",
    "    ax.hist(text_lengths, bins=bins, edgecolor='black', alpha=0.7)  # Plot histogram.\n",
    "    ax.axvline(x=percentile_value, color='red', linestyle='--')  # Draw percentile line.\n",
    "    \n",
    "    # Annotate the percentile line.\n",
    "    ax.text(percentile_value, ax.get_ylim()[1]*0.9, f' {percentile}th Percentile: {int(round(percentile_value, -1))}',\n",
    "            color='red', ha='left')\n",
    "\n",
    "    ax.set_xlabel('Text Length')  # Set x-axis label.\n",
    "    ax.set_ylabel('Frequency')  # Set y-axis label.\n",
    "    ax.set_title('Histogram of Text Lengths')  # Set title.\n",
    "\n",
    "    ax.xaxis.set_major_locator(MaxNLocator(integer=True))  # Set major locator for x-axis.\n",
    "    plt.show()  # Display the plot.\n",
    "\n",
    "def visualize_train_test_category_splits(train_df, test_df, top_x=5, normalize=True):\n",
    "    \"\"\"\n",
    "    Visualize the frequency of top categories in training and test datasets.\n",
    "    \"\"\"\n",
    "    set_plot_appearance()  # Apply appearance settings to the plot\n",
    "\n",
    "    # Get top x categories from both datasets.\n",
    "    train_topic_norm = train_df['topic'].value_counts(normalize=normalize).head(top_x)\n",
    "    test_topic_norm = test_df['topic'].value_counts(normalize=normalize).head(top_x)\n",
    "    \n",
    "    # Prepare the data for plotting.\n",
    "    train_topics = pd.DataFrame({'topic': train_topic_norm.index, 'frequency': train_topic_norm.values, 'dataset': 'Training'})\n",
    "    test_topics = pd.DataFrame({'topic': test_topic_norm.index, 'frequency': test_topic_norm.values, 'dataset': 'Test'})\n",
    "    combined_topics = pd.concat([train_topics, test_topics])\n",
    "\n",
    "    sns.barplot(x='frequency', y='topic', hue='dataset', data=combined_topics)  # Create bar plot.\n",
    "    plt.title('Top {} Topics Frequency Comparison'.format(top_x))  # Set title.\n",
    "    plt.xlabel('Normalized Frequency')  # Set x-axis label.\n",
    "    plt.ylabel('Topic')  # Set y-axis label.\n",
    "    plt.tight_layout()  # Adjust layout.\n",
    "    plt.show()  # Display the plot.\n",
    "\n",
    "\n",
    "\n",
    "def get_data_stuff(dataset_name, split_ratio=0.2):    \n",
    "    # %%\n",
    "    from datasets import load_dataset\n",
    "    from datasets import Dataset\n",
    "    import pandas as pd\n",
    "    pd.set_option('display.max_colwidth', None)\n",
    "    \n",
    "    \n",
    "    #load the dataset\n",
    "    dataset = load_dataset(dataset_name)\n",
    "    \n",
    "    # Shuffle the dataset (setting seed for reproducibility)\n",
    "    shuffled_ds = dataset['train']#.shuffle(seed=seed)\n",
    "    #shuffled_ds = shuffled_ds.select(range(1000))\n",
    "\n",
    "    \n",
    "    # Split the dataset into training and test sets with a test size of 20%\n",
    "    train_test_split = shuffled_ds.train_test_split(test_size=train_test_split_ratio, seed = seed)\n",
    "    \n",
    "    # The train/test datasets are now accessible as follows:\n",
    "    train_ds = train_test_split['train']\n",
    "    raw_train_df = pd.DataFrame(train_ds)\n",
    "    \n",
    "    test_ds = train_test_split['test']\n",
    "    raw_test_df = pd.DataFrame(test_ds)\n",
    "    \n",
    "    prepared_train_df = prepare_dataset(raw_train_df, \"train\")\n",
    "    prepared_test_df = prepare_dataset(raw_test_df, \"test\")\n",
    "\n",
    "    prepared_train_dataset = Dataset.from_pandas(prepared_train_df)\n",
    "    prepared_test_dataset = Dataset.from_pandas(prepared_test_df)\n",
    "    \n",
    "    print(f\"length of downloaded ds: {len(shuffled_ds)}\")\n",
    "    print(f\"length of train_ds: {len(prepared_train_dataset)}\")\n",
    "    print(f\"length of test_ds: {len(prepared_test_dataset)}\")\n",
    "    estimated_training_steps = epochs * len(prepared_train_dataset) / per_device_train_batch_size / gradient_accumulation_steps\n",
    "    print(f\"estimated training steps will be {estimated_training_steps}\")\n",
    "    return prepared_train_df, prepared_test_df, prepared_train_dataset, prepared_test_dataset\n",
    "\n",
    "\n",
    "def prepare_dataset(df, split=\"train\"):\n",
    "    text_col = []\n",
    "    instruction = \"\"\"Write a concise summary of the below input text.Ensure that responses covers the key points of the text.Only provide full sentence responses.\"\"\"  # change instuction according to the task\n",
    "    if split == \"train\":\n",
    "        for _, row in df.iterrows():\n",
    "            input_q = row[\"dialogue\"]\n",
    "            output = row[\"summary\"]\n",
    "            text = (\n",
    "                \"### Instruction: \\n\"\n",
    "                + instruction\n",
    "                + \"\\n### Input: \\n\"\n",
    "                + input_q\n",
    "                + \"\\n### Response :\\n\"\n",
    "                + output\n",
    "                + \"\\n### End\"\n",
    "            )  # keeping output column in training dataset\n",
    "            text_col.append(text)\n",
    "        df.loc[:, \"text\"] = text_col\n",
    "    else:\n",
    "        for _, row in df.iterrows():\n",
    "            input_q = row[\"dialogue\"]\n",
    "            text = (\n",
    "                \"### Instruction: \\n\"\n",
    "                + instruction\n",
    "                + \"\\n### Input: \\n\"\n",
    "                + input_q\n",
    "                + \"\\n### Response :\\n\"\n",
    "            )  # not keeping output column in test dataset\n",
    "            text_col.append(text)\n",
    "        df.loc[:, \"text\"] = text_col\n",
    "    return df\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "2a764bbf-2d24-410d-9356-68f2153af13e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">length of downloaded ds: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">12460</span>\n",
       "</pre>\n"
      ],
      "text/plain": [
       "length of downloaded ds: \u001b[1;36m12460\u001b[0m\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">length of train_ds: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">12148</span>\n",
       "</pre>\n"
      ],
      "text/plain": [
       "length of train_ds: \u001b[1;36m12148\u001b[0m\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">length of test_ds: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">312</span>\n",
       "</pre>\n"
      ],
      "text/plain": [
       "length of test_ds: \u001b[1;36m312\u001b[0m\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">estimated training steps will be <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">47.453125</span>\n",
       "</pre>\n"
      ],
      "text/plain": [
       "estimated training steps will be \u001b[1;36m47.453125\u001b[0m\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "train_df, test_df, train_dataset, test_dataset = get_data_stuff(hf_dataset_name, \n",
    "                                                                split_ratio=train_test_split_ratio,\n",
    "                                                               )\n",
    "text = test_df['text'][1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "46b9d045-f3df-45b3-beea-92336c089d1c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">### Instruction: \n",
       "Write a concise summary of the below input text.Ensure that responses covers the key points of the text.Only \n",
       "provide full sentence responses.\n",
       "### Input: \n",
       "#Person1#: Mister Ewing said we should show up at the conference center at <span style=\"color: #00ff00; text-decoration-color: #00ff00; font-weight: bold\">4:00</span> o'clock, right?\n",
       "#Person2#: Yes, he especially asked us not to be late. Some of the people from our east york branch office are \n",
       "coming, and he wants to make a good impression on them. How are you getting there?\n",
       "#Person1#: I was thinking of taking my car, but I think I'm just going to take the underground, because there is \n",
       "construction on the highway. What about you?\n",
       "#Person2#: I'll be taking the underground as well. Why don't we go together? I've been to the conference center \n",
       "only once, and I'm not sure if I can find my way around there.\n",
       "### Response :\n",
       "\n",
       "</pre>\n"
      ],
      "text/plain": [
       "### Instruction: \n",
       "Write a concise summary of the below input text.Ensure that responses covers the key points of the text.Only \n",
       "provide full sentence responses.\n",
       "### Input: \n",
       "#Person1#: Mister Ewing said we should show up at the conference center at \u001b[1;92m4:00\u001b[0m o'clock, right?\n",
       "#Person2#: Yes, he especially asked us not to be late. Some of the people from our east york branch office are \n",
       "coming, and he wants to make a good impression on them. How are you getting there?\n",
       "#Person1#: I was thinking of taking my car, but I think I'm just going to take the underground, because there is \n",
       "construction on the highway. What about you?\n",
       "#Person2#: I'll be taking the underground as well. Why don't we go together? I've been to the conference center \n",
       "only once, and I'm not sure if I can find my way around there.\n",
       "### Response :\n",
       "\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "print(text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "fe3b1874-38b3-406c-a9c3-ce34e42f1f64",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAm8AAAHPCAYAAAAFwj37AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/SrBM8AAAACXBIWXMAAA9hAAAPYQGoP6dpAABf2klEQVR4nO3dd1yVdf/H8RdLRRQEI+8QJ5a5R5qjcCVQSWYOUnPPTE3DkWU5bstZYq7KrMzUFNTU1ATcGGKuNNySI8GNCioo6/eHP84tgizB46Xv5+PRI841vtfne3E45+33WhYuLi4piIiIiIghWJq7ABERERHJPoU3EREREQNReBMRERExEIU3EREREQNReBMRERExEIU3EREREQNReBMRERExEIU3EREREQNReBMRERExEIU3EXnkuLq6EhkZiZ+fn7lLEXks+fn5ERkZiaurq7lLkVywNncBIkYQGRmZo+VLliyZT5Xcn5+fHz4+Pvedn52ajNDPh8XX15chQ4bcd/66devo2bPnQ6xIsmJlZUWbNm3w9vamWrVqFCtWjPj4eP755x+2bNnCwoULc/weF3kUKbyJZMOXX36ZblqvXr1wcHDIcJ45zZ07l2vXruVq3Ueln+fOnaNRo0bExsY+tG3ez5o1azh8+HC66cePHzdDNXI/JUuW5Mcff6RKlSpcuHCBkJAQoqKisLW1pVq1avTv35++ffvyyiuvcPLkSXOXa3YTJkxg5syZnDt3ztylSC4ovIlkw9SpU9NN8/HxwcHBIcN55vTdd99x5syZXK37qPQzMTGRiIiIh7a9zKxevZpVq1aZuwzJhJ2dHYsWLaJChQrMnj2bKVOmcPv27TTLlC1bltGjR2NnZ2emKh8tFy5c4MKFC+YuQ3JJ57yJ5DFHR0fGjh3L9u3b+eeff9i3bx/ffPMNFStWTLds6nknpUuXpl+/fmzbto2IiAi2b9/O4MGDsbZ+dP99lZ/9zOycNzs7Oz744AOCg4M5fvw4hw4dIjAwkGHDhqVpp2rVqsyZM4c///yTf/75h/3797NmzRref//9PN0PAQEBREZGUrBgQYYPH84ff/zByZMn8fX1NS1TqlQppkyZYqplz549+Pn53fews6enJ2vWrOH48eP89ddfTJ48GQcHB8LCwggLC8tw+xnJ7LwmT09PlixZwoEDB4iIiGDDhg307dsXS8u0Xws+Pj5ERkbi4+NDo0aNWLlyJcePHyc8PJxp06bh6OiY4bYrV67MjBkz2LVrl6nPCxYswMPDA4AOHToQGRlJv379Mlz/pZdeIjIykkmTJmU4/27vvvsuFSpUYNmyZXz++efpghvAyZMn6d69O0ePHk0zvU6dOsyfP5/w8HAiIiLYsmULQ4YMoVChQunaiIyMJCAggP/85z/MnDmTv//+myNHjjB//nxKly4NQIUKFfj+++8JDw/nyJEjzJkzh6eeeipNO3e/v5977jnmz5/PwYMHOXr0KIsWLaJatWrptl2tWjU+++wzNmzYwKFDhzh+/Djr16+nf//+GX5OpL5X7O3t+eyzz9i5cyenTp0ynVpxv/fG66+/ztKlS9m3bx8RERHs3r2bxYsX8/rrr6fbhoeHBwEBAaZ6goOD6dOnD1ZWVvftb9myZZk7dy4HDhzg2LFjLF68mMqVK6drWzL36H4ziBiQk5MTq1atoly5coSGhrJy5UpKly5NixYteOWVV+jYsSM7d+5Mt97YsWOpW7cuv/32Gzdu3MDDw4Nhw4ZRuXJl+vTpk6MamjdvTpEiRbh9+zbHjh1j27ZtJCQk5FUXAfP1s3jx4ixbtoxnn32W8PBw5s+fj6WlJW5ubrz33nt8++23xMTEUKVKFVauXElycjKBgYGcOXMGBwcHnn32Wd555x2mT5+ep/sD7ox4Vq5cmU2bNhETE8O///4LQK1atVi4cCGFCxdm/fr1nDhxglKlSvHWW2/RtGlTWrZsyenTp03ttG3blq+++oqYmBiWLVtGTEwMzZs3Z/HixdjY2OTJ73LEiBEMHDiQs2fP8vvvvxMbG8uLL77IqFGjqF27Nn379k23joeHB6+88grr169n165d1KtXj3bt2lGmTBneeuutNMu+/vrrzJw5EwsLC4KDg/nnn38oXrw4tWrVon379gQHB7NixQpGjRpFhw4d+Prrr9Ntr2PHjgAsXLgwy/60b98eIFsXuNy9/7y9vZk1axa3b99m1apVXLp0icaNG+Pr60uTJk1o27Ytt27dSrN+sWLF+PXXX7l48SIBAQGUL18eDw8P3Nzc6NGjB8uXL+fvv/9myZIlVK9enRYtWlCsWLEMz0ctXbo0K1asML2XXV1d8fb2Zvny5fj4+LB3717Tsu+88w7Nmzdnx44dbNy4EVtbWxo0aMDHH39MjRo1Mvz7KVCgAP7+/tjZ2REUFERiYiIXL168777p0qULEyZM4Ny5c6xbt44rV67g7OxMzZo1efXVV1m7dq1p2T59+jB69GiuXLnCihUruHnzJp6enowePZoXX3yRXr16pWvf1dWV3377jSNHjrBkyRLKlCnDq6++ir+/P02aNOHSpUuZ//LEROFNJA+NHDmScuXKMWPGDCZOnGia3qxZM37++Wf8/Pxwd3cnJSUlzXq1a9fGw8ODs2fPAjBp0iR++eUXWrRoweuvv57mQzMrn3/+eZrX586dw9fXly1btjxAz9IyVz8nTJjAs88+y/Tp09ONyDz11FPcuHEDgDZt2lCoUCG6d+9OUFBQmuXuN1J0P97e3lSoUCHd9FmzZqX5Yi9RogTNmzfn6tWrpmnW1tbMnj0bS0tLWrRowYEDB0zz6taty9KlS/nvf/9Lt27dAChSpAjjxo3jxo0btGjRgn/++Qe4s58WL17Mf/7zH1MozC13d3cGDhzIpk2b6N27N3FxcaZ5EyZMoEuXLhn+Ljw8PGjbti27du0CwNLSkiVLltCwYUNq167Nnj17gDu/h2nTppGYmMhbb72Vps8AzzzzDABxcXH8+uuvdO3alfr166cZUSxWrBivvvoq4eHh7N+/P9P+lCxZEhcXF6Kiojhx4kS290ORIkWYPHkyiYmJtGzZkkOHDgEwceJEZs2axZtvvkm/fv2YNm1amvUqV67MnDlzGDt2rGna+PHj6dq1K8uXL2fq1Kl8//33pnnz58/nlVdeoWrVqoSHh6dpq379+un+hgICAli0aBGTJ082jVICTJ8+nY8//pjk5OQ0bXzxxRd06NCBOnXqmH43qUqUKMHBgwdp1aoV8fHxWe6TDh06cOvWLTw9Pbl8+XKaeXf/3ZQpU4aPP/6Yixcv8vrrrxMVFQX873362muv0aZNG5YtW5amjYYNG/L5558ze/Zs07Rhw4YxePBg3n77bWbNmpVljXKHDpuK5BEbGxvefPNNoqOj033gb9y4kS1btlCuXDnq1q2bbt3vv//eFGjgzuhAajjJ7ArSu4WFhdG3b1/q1q1L+fLleemll5gyZQoODg78+OOPVK9ePfedu4u5+uns7Mxrr73GiRMnMrx44tKlSyQlJaWZltEX1pUrVzLdzr1atGjBkCFD0v1XsGDBNMt98cUXaYIb3BkFLV26NN988026ELNz504CAwNp1qwZRYoUAeDVV1/F3t6exYsXm4Ib3DkHMDuHD7Oje/fuAAwfPjxNcIM7ISQ5OZlWrVqlW2/FihVpwkFycjIBAQEA1KhRwzS9Xbt22NnZ8e2336brM5Dm9//zzz8D/xtlS5UavhctWpRlf55++ul07WaHl5cXDg4OLFmyxBTcAFJSUvjss89ISEigXbt26da7fv16ut/FihUrgDvvrbuDG8DKlSsBMjw0ePXq1XSjwFu2bCEkJITKlSunOXwaFRWVLrgBzJs3D7gTyjPy+eefZyu4pUpMTCQxMTHd9Lv/bt566y1sbGz49ttvTcEN4Pbt24wfPx7I+O/51KlT6UZZf/nlFyDte0iyppE3kTxSoUIFbG1tCQ0NzfDDMjQ0lMaNG1OlShX+/PPPNPN27NiRbvndu3eTkJBA1apVs7X9JUuWpHl98uRJpk2bxtmzZ5k6dSoffPCB6Yv7QZirnzVq1MDS0pLQ0NAMv1zu9ttvv9GrVy++//57Vq1axdatW9mxY0eurqzr169fti5Y+Ouvv9JNq127NgDly5dPcw5cqqeffhorKyvKly/P/v37TV/w9+43+N9+elC1a9fmxo0bpkON94qPj8fNzS3d9IxGwFIDk4ODg2larVq1ALI10nvo0CF2797N66+/zieffEJMTAxw5zDozZs3Wb58edYdyqXU99v27dvTzYuKiuL06dO4ublhZ2dnGtEFOHHiRLr3feqJ/3eHwHvn/ec//0k378CBA9y8eTPd9B07duDu7k7VqlX5+++/gTv/aOrevTstW7akQoUK2NnZpTk/MaP24+LiMqzpflauXMmnn37Khg0bWLFiBaGhofz5559cv349zXJVqlQBMt53u3btIi4uzrTMvf29dzQ+o/eQZE3hTSSPpI6e3O+8jdQP8aJFi6abl9E6ycnJXLlyJcPlcyIgIIDPP/88w5Gw3DBXP1PnZyeA7d27l7Zt2zJw4EBatWplCip79+5l/PjxhIaGZtlGTmV0LlGxYsWAOyNJmSlcuDDwvz5mtp8eVLFixbCxscn0Hnap9dzt3i9wwBSi7w4ROfk9ASxYsAA/Pz9at27NvHnzqFWrFpUrV8bf3z9bt4rJLBxlJvV9fL9zwM6fP4+bmxtFixZNE94yqil1P2S2jzK6qOB+2079/dvb25umzZkzB09PTyIiIli1ahWXL18mISEBBwcHevXqRYECBdK1c++hz6x88803XLlyhS5dutC3b1/69etHQkICGzZsYMyYMaZD9qm/48zqz+j3kdH+SR0tv/dCGcmcwptIHkn9YLr3yrJUzs7OQMYf/k899VS6W2NYWlri6Oj4wCfxJicnExMTk2f/sjVXP1NHZbL7Jf3nn3/SuXNnChUqRK1atfDw8KBLly7Mnz+fZs2apblIIL+k7quuXbuyfv36LJdP3WcZ7dvU/XRvKEodybCyskp32DijQHz9+nVSUlIyvKIxL9z9e8rOLWtWrVrF6NGj6dixI/PmzaNDhw5A9i5UgDtXgJ49e5aSJUtSrly5bJ/3lvq7SX2/3iv1cGx+3mvwfttO/f2n7ssaNWrg6enJpk2b6NKlS5rDp7Vr187w4gAg3ShXdixZsoQlS5bg6OjIiy++SKtWrWjZsiXlypWjefPmJCcnm/aJs7Nzhlc6P/XUUxkGNck7iroieeT48ePExcVRs2bNDG8z0KBBA4AMzwOqV69eumkvvPACNjY26U5yzikXFxdKlCiR63u/3ctc/dy3bx9JSUk0bNgwR7dQiY+PZ/v27fz3v/9l5syZ2Nra0qhRo2yv/yBSrxZ84YUXsrX8wYMHAXjxxRfTzUvdT/dKPc/u3lBrYWGR4XlWe/bswcnJiXLlymWrppxK7XPjxo2ztXx8fDxLly6lSpUqNGzYkJYtW3L06NF0J99nJvW8qUGDBmW5bOo+TH2/pb5f7+bi4kKZMmU4efJkmlG3vFalSpUMRzlT/05SayxTpgwAGzZsSHfeW0bvlbxw5coVAgMDTbf2qVixouk9k/q3ndG+q127Nra2thn+/UveUXgTySMJCQmsXLmS4sWLM3DgwDTzmjRpQtOmTTlx4kSGt9Do2bOn6So8uPMF8+GHHwLg7++f5badnZ0zHJGyt7c3XVTw66+/5qQ792Wufl66dIm1a9dSrly5DM8fK168uOn+Ui+88EK6CwrgfyMa997+Ib+k3qakd+/eGQZXa2vrNIezAwMDiYmJoX379pQvXz7NcsOHD89wG/v27QPSnyDep08f05f+3X744QfgztM0Mrry1tnZOcOra7MrICCA69ev07dv3wzPe8rofbpgwQLgzhWVRYsWzdaFCnf75ptvOH78OO3atWPEiBEZHkIsVaoU33//Pc899xxwZ19fu3YNHx8f07RUH3/8MTY2NqYLMvJLsWLF0t13sHHjxri7u3Po0CHT+W6po1v3BrXnnnuOAQMG5Fk9GYUxa2tr0+H/1HP9fv31VxISEujTpw8lSpQwLWtjY8PHH38MZO9zS3JPh01F8tDnn39O/fr1GTx4MHXq1GHv3r2mezfdvHmTDz74IMNDGXv27CE4OJhVq1Zx8+ZNPDw8qFChAmvWrMnWbUIqVKjAL7/8wq5duzhx4gSXL1/GxcWFpk2b4uTkxLZt2zK8l5bR+vnRRx9RsWJFBg0aRLNmzfjjjz+wsLCgfPnyNGrUiJo1axITE8N7771Hw4YN2bFjB6dPn+bWrVtUq1YNd3d3Tp48ye+//55n+yIzt2/fpk+fPixYsIDly5ezbds2Dh8+TEpKCiVLlqRevXpcuXLFNEoVGxvLqFGjmDZtGmvWrGHVqlWm+7zFx8dneB7ZkiVL6NevH0OHDqVKlSqcOnWK6tWr8/zzzxMaGkrDhg3TLL9582b8/Pz44IMP2LZtG5s3b+bMmTM4OjpSrlw5XnzxRSZPnpzrx39dvnyZQYMGMXv2bFavXk1wcDARERE4OTlRq1Yt/v3333TPhD127BhhYWHUr1/fNBKXEzdu3KBjx478+OOPDBw4EB8fH7Zu3crZs2extbWlSpUq1K1bl8TERMaNGwfcOWw6fPhwZs2aZXqKxuXLl3F3d6dGjRrs2bMnT/9mMhIWFkaXLl1Mt1pJ/RuKi4tj2LBhpuX27t3Lnj17aNmyJU8//TR79uyhZMmSeHp6smHDBry9vfOknu+//57r16+zZ88ezpw5g42NDe7u7lSsWJHVq1ebQuSpU6cYP348o0ePZv369fz2229p/p7XrVuX7jYhkrcU3kTyUHR0NN7e3gwePBgvLy9efPFFYmNjCQwMZOrUqRw5ciTD9UaPHo23tzcdO3bExcWFCxcu8MUXXzBz5sxsbffUqVP4+/ubbqaZepL1oUOHWLFiBYsWLcrwNgNG6+eVK1d44403ePfdd/H29qZbt27cunWLf//9l1mzZpmu3Js/fz6xsbHUqlWL+vXrY2FhQWRkJNOnT2fOnDkP9Xycffv24eHhQb9+/WjWrBl16tTh9u3bphuhpt5KIlVAQAAxMTEMGjSItm3bEhsbS1BQEJ9//jmBgYHp2r906RI+Pj6MGjWKxo0bk5iYSGhoKG+88cZ9DyN+8cUX7Nixgx49evDyyy9jb2/PlStX+Pfff5k6deoDj9KuW7eON954gwEDBlC/fn08PDyIjo7mwIED9x1VCwgIoH79+qabw+ZUZGQkr7/+Oq1bt+aNN96gcePGFCtWjFu3bnHixAm+/vprfv755zS3tli9ejUXLlxgwIABvPbaa9ja2nLmzBn8/PzS3ccvP5w+fZqPPvqITz75hK5du2JlZcX27dsZP368adQN7py32rVrVz7++GOaNGlCjRo1OHHiBOPGjWPjxo15Ft4mTpxIkyZNqFmzJs2bNycuLo6TJ08yYsQI06HpVHPmzOHEiRP06dOH1q1bY2Njwz///MPYsWPT3S5F8p6Fi4tLzs9oFJE84efnh4+PD/Xq1cuzc9IeRU9KP/Nb6o1s69evb+ZK8t5nn31G9+7d8fHx4Y8//jB3OfnK1dWVHTt24O/vzwcffGDucsSAdM6biIiYlZOTE+3ateP48eOPfXATyQs6bCoiImbxyiuvUK1aNVq0aEGRIkUyfHKGiKSn8CYiImbh7e2Nj48PZ8+eZcKECdl6koWI6Jw3EREREUPROW8iIiIiBqLDppKpZcuWYWNjQ3R0tLlLEREReaw5OTmRkJCQ5fOQFd4kUzY2Nqa71ouIiEj+ye73rcKbZCp1xK1bt27mLUREROQxN2/evGwtp3PeRERERAxE4U1ERETEQBTeRERERAxE57xJlqysrKhataq5yxARETMrUKAARYsWxdLywcZ+rl69yoULF/KoKmNJSUkhOjqa+Pj4XLeh8CZZcnFxITAw0NxliIiIGV2/fp1jx45x8+ZNUlIe7P7+KSkpnDt3jqSkpDyqzlgSExP54YcfOHLkSK7WV3iTLFlYWLAjIobYuCfzj0xE5ImXkkyB6yfYszOMbaFhJCUn57opKwsLbAtYcvLkSW7dupWHRRqDtbU1np6e9OjRg9GjR+dqBE7hTbIlNi6JqzcTzV2GiIiYgTUJFI67ydY/wog6e/6B2rKyBHtba6Kioh7o0KGRBQUF8fzzz+Pk5ERUVFSO19cFCyIiIpIpi///f1JS7kfc5H8SE+8MhlhYWGSxZMYU3kREREQMROFNREREzCogIICxY8eauwzDUHgTERERw2jQoAGRkZHY29s/1O36+voSFBT0ULd5PwpvIiIiIgai8CYiIiIPTaFChRgyZAj79+9nz5499O3bN838Nm3asHbtWo4cOcLevXuZOXMmxYsXB8DV1ZWlS5cCcOjQISIjI/Hz8wOgSZMm/Prrrxw8eJDw8HB++uknypQpY2rXxsaGzz77jD179hAREcGOHTsYMGCAab69vT1Tpkxh//79HD58GH9/fypXrgyAj48PQ4YMoUqVKkRGRhIZGYmPj0++7qfM6FYhIiIi8tD0f7c31apV49133yUqKooRI0ZQrVo1Dh48CNy5D9qUKVOIiIjgqaeeYvTo0fj5+dGlSxeioqLo1asXc+fOxd3dndjYWNPtRgoXLsycOXM4dOgQdnZ2DB06lLlz5+Lp6UlKSgo9evTA09OTd999l8jISFxcXHBxcTHV9e233xIfH0+nTp2IjY2lU6dOLFmyBHd3d1atWkXFihVp0qQJ7du3ByA2Nvbh77z/p/AmIiIiD4WtbSG8X3+VKVOmEBoaSnx8PIMHD2bXrl2mZZYsWWL6+fTp03z66af8/vvvFC5cmJs3b3L16lUALl26RExMjGnZtWvXptmWr68v4eHhPPfccxw5coSSJUty4sQJ/vzzTwAiIyNNy9atW5eaNWtSo0YNbt++DcC4cePw8vKiRYsWLFy4kBs3bpCUlMTFixfzfL/klMKbiIiIPBQlXVwoUKBAmsdCXb16lYiICNPratWqMWTIECpXroyDg4PpOaolS5bk2LFj9227XLlyDB06lFq1auHk5JRmvSNHjuDv78/ixYsJCQlh06ZNrF+/nq1btwJQuXJl7OzsCA8PT9NmoUKF0hx6fVQovImIiMgjwdbWlkWLFrF582YGDBjA5cuXKVmyJL/88gsFChTIdN158+Zx5swZhg8fzrlz57C0tGTTpk3Y2NgAEB4eTv369WnWrBkvv/wy33zzDdu2baNPnz7Y2dlx4cIF2rZtm67da9eu5UtfH4TCm4iIiDwUkVFRJCQkULFiRf744w8AHBwcKF++PGFhYVSoUAEnJycmTJhgemxUjRo10rSRkJAAgJWVlWmao6MjFSpUYNiwYabDonXr1k23/evXr7Nq1SpWrVrFmjVrWLRoEcWKFePvv//G2dmZxMREzpw5k2HtCQkJptE8c3s0qhAREZHHXlxcPKvXrqNXr140aNCAihUr4ufnR/L/P+g+MjKSW7du0b17d0qXLo2HhweDBw9O08aZM2dITk6mefPmODk5UbhwYa5evUp0dDSdOnWibNmyvPTSS4wePTrNen369OHNN9/Ezc2N8uXL4+3tzfnz57l27RohISHs3r2bH374gUaNGuHq6kqdOnX48MMPqV69OgD//vsvpUuXpkqVKjg6OmY5EpifFN5ERETkoZn1zRzCw8OZM2cOixcv5s8//2T//v0AREdH88EHH+Dt7c2mTZsYMGAA48aNS7P+uXPn+PLLL/noo4/Yt28fn3/+OSkpKbz33ntUq1aNDRs2MGbMGD777LM0612/fp333nuP33//nTVr1lCqVCk6d+5MSkoKAJ07dyYsLIypU6cSEhLC7NmzKVmyJJcuXQLuXBCxefNm/P39CQ8Pp1WrVvm/s+7DwsXFJcVsW5dH3rx58yhdujT/JjzN1ZuJ5i5HRETMwIYEbK8fZ9bsbzh/4cGutrSyBHtba44ePWq6zceTpmTJkvj6+jJ16tQ0V73OmzcPgG7dumW6vkbeRERERAxE4e0BNGjQ4KE+58zDw4MxY8Y8tO2JiIjIo0dXmxpIcHAwwcHB5i5DREREzEjhLZsKFSqEn58fzz//PImJiVy8eJEZM2ZgZWXF+PHjqVu3LlZWVgwePNh04mWbNm149913ATh79qzp3jM+Pj60bduWGzduULZsWaKjoxk0aBBnzpzJcp6Xlxc9e/akQYMGfPbZZ+zYsSPDbXfq1Im+ffty48YN1q1bx7BhwyhZsmSu+1/U1irrhURE5LFkmZxM8g2wtLxzztqDsLK0yJuinmAKb9nUpEkT7O3tadq0KQDFihWjUqVKVKhQgaFDh/Lxxx/TuXNnPvzwQ9555x0qVqzIJ598wmuvvca5c+d4//33mTJlCp07dwagTp06eHp6cvz4cfr168fkyZPp2LFjlvPultm2fX198fLy4uLFiwwZMuSB+p6SkkI9N/sHakNERIwrPj6eXbssKVLQinjbB48OycnJJCUl5UFlTyaFt2w6ePAgzz77LOPHjycsLIwNGzYAcPLkSfbu3QvA7t276du3LwANGzZk8+bNnDt3DoCffvqJwYMHm27wt3v3bo4fPw7AwoULGT58eLbm3e1+237ppZfYvHmz6flrixYtwtfXN9d9j4qKYtCgQbleX0REjM3Z2ZnWrVtz8uRJ081zH0RSUpLpZruScwpv2XT69GmaNGnCSy+9hLu7OyNHjmT06NHcunXLtExSUhLW1hnv0tT7yOSlh7XtpKSkdM97ExGRJ0fJkiW5ffs2t27dSnN7DxcXF5ycnPJ8e9HR0XkSEh9XCm/Z9Mwzz3D16lWCg4PZvHkzr776Ki4uLvddPjQ0lPfff58SJUpw/vx5OnfuzLZt20x3ka5duzZubm5ERETQoUMHQkNDszUvO0JDQ+nfvz/Fixfn8uXLtG/f/sE6LyIicg8XFxe2hoRgW6hQnrcdFx9PI3d3Bbj7UHjLpueff56PPvoICwsLrKysWLZsGYcOHbrv8keOHOGzzz5jwYIFwP8uWEi1e/duRo4cSdmyZbly5Uqaw5KZzcuOw4cPM336dFauXMn169fZvHnzI/lgXRERMS4nJydsCxViR0QMsXF5d/5aUVsr6rnZ4+TklGl4S71Vl42NDW5ubhw+fBiAiIgI+vXrl61teXh48NJLL2V5G64SJUrw9ddf07p16+x1Ip8pvGXTpk2b2LRpU7rpnp6epp+PHDlC/fr1Ta+XLVvGsmXLMmwvNjaWnj175miev78//v7+AGzfvj3TbS9dupSffvoJgJ49e7J79+7MuiciIpIrsXFJZnkCT+p3oKurK0FBQWm+E1NZWVllemFEdm/Bdf78+UcmuIFu0vvY+vjjjwkKCmLjxo00b96ckSNHmrskERGRfBcWFsbHH3/M6tWrmTZtGs7OzgQEBPD777+zceNGPvvsMyws7tyuxMfHh++//x64c+P9DRs2MH78eIKDg9m4caPpofSurq4cPHjQtI3IyEgGDhzI6tWr2b59Oz4+PqZ5derUISgoiPXr1/Pll18SHBxMgwYN8rSPGnkzg7tH0HIyLycU1kRE5Enl6OiIt7c3AAULFqRr167cvHkTS0tLfvzxR9544w1WrVqVbr373YIrI7dv38bb2xs3NzfWrl3LsmXLsLS05Ouvv2bQoEGEhobSsGHDfDnvXCNvIiIi8li5exDEwsKCkSNHEhwcTGBgINWrV6dKlSoZrnfvLbjKlClz320sX74cuHOOXWJiIk8//TQVKlQgMTGR0NBQ4M4FhCdOnMirbplo5E1EREQeKzdv3jT93LdvX5566im8vb25desWo0ePptB9rpDN7i247l02OTkZK6uH9yQihTcRERHJtbx+fGJet+fg4MCFCxe4desWzs7OeHt7s3bt2jzdRqqIiAhsbGyoX78+YWFh1K9fn3LlyuX5dhTeREREJMeio6OJi4/Pl8cnxsXHEx0dnSdtzZ07lzlz5rBx40bOnz9PSEhInrSbkdu3b9OvXz/Gjx+PhYUFf//9N8ePHycmJiZPt6PwJiIiIjkWFRVFI3d3sz9h4cyZM1SuXNn0+u7bZsGdOlMvXrhXdm/Bde82SpYsmaadatWqmX4+ePAgHh4eANSoUYPGjRsTERGRrb5kl8KbiIiI5EpUVJSegnCPFi1a0Lt3b+DOeXODBg1K80ixvKDwJiIiIpJH8uqWX5nRrUJEREREDEThTURERMRAFN5EREREDETnvImIiEiuuLi4mP1q0yeRwpuIiIjkmIuLCyFbt1LI1jbP246Pi8O9UaNMA1xQUBAANjY2uLm5cfjwYeDOjXL79euX7W01aNCAggULsnnz5geq+WFSeBMREZEcc3JyopCtLTf2BZJ840qetWtp54hdDS+cnJwyDW+p92RzdXUlKCgozT3acqJBgwY4ODgovImIiMiTIfnGFZJiLpq7DJPGjRszePBgChUqRFJSEuPHjyc0NJRy5crh5+eHnZ0dFhYWBAUFsWbNGjp37oyVlRUNGjRg7dq1TJs2zdxdyJLCm4iIiDwWSpcuzZAhQ+jYsSPXr1+nbNmyLF++nPr169OtWzfWr1/PzJkzAShWrBhXr17l559/xsHBgdGjR5u5+uxTeBMREZHHQtOmTU2BLVVycjIlS5Zkx44dfPLJJ9jZ2bF9+/Z8fcZpflN4ExERkcfG1q1bGTBgQLrpJ06cYNeuXTRq1Iju3bvTq1cvunTpYoYKH5zCm4iIiOSapZ3jI9Peli1b8PX1pVKlShw6dAiAmjVr8tdff1GuXDlOnjzJ0qVL2bt3LytXrgTg+vXruLq65kntD4vCm4iIiORYdHQ08XFx2NXwyvO24+PiiI6OzvF6J0+eZMCAAUyaNAlbW1tsbGwIDw9nwIABtGjRgtatW5OQkIClpSUfffQRAL///jtt2rQhKChIFyyIiIjI4ysqKgr3Ro3MfpPeM2fOULlyZdPrkJCQDM9nmzlzpulihbv9+++/eHnlfQDNTwpvIiIikitRUVF6EoIZ6NmmIiIiIgai8CYiIiJiIDpsKlmysrKiatWq5i5DRESykF8PdE9JSQHA2lqxIS+k7sfU/Zrj9fOyGHk8ubi4EBgYaO4yREQkC3Hx8TRyd8/zABcdHU1iYiKenp4EBQWRmJiYp+0/SaytrfH09CQxMTFXV9SCwptkg4WFBTsiYoiNSzJ3KSIich9Fba2o52af5QPdcyM+Pp4ffviBHj168Pzzz+dp20+ixMREfvjhB+Lj43O1vsKbZEtsXBJXb+pfWiIiT6ojR44wevRonJycsLCwMHc5hpWSknLnHnm5DG6g8CYiIiLZFB8fr1uDPAJ0tamIiIiIgSi8iYiIiBiIwpuIiIiIgSi8iYiIiBiIwpuIiIiIgSi8iYiIiBiIwpuIiIiIgSi8iYiIiBiIwpuIiIiIgSi8iYiIiBiIwpuIiIiIgSi8iYiIiBiIwpuIiIiIgSi8iYiIiBjIYxvegoKCsLOzu+98V1dXDh48mK22hg4dyltvvZXlclWqVKFly5bZrjEn5s+fj5ubW760LSIiIsZhbe4C8ounp2eetfXFF19ka7kqVarg5eXFqlWrcrwNKysrkpKS7ju/S5cuOW5TREREHj+PbXiLjIykUqVKlC1blnHjxlG4cGFu3brFmDFj2LVrl2m5Tz/9lEaNGmFlZcXo0aMJCQlJ15afnx8HDhxg7ty5+Pr68uyzz2Jra0uZMmW4ePEiffr0wcrKiqFDh1K0aFGCgoLYs2cPI0aMoEaNGowcOZIiRYpgZWXFjBkzWL16Na6urgQFBbFgwQIaNWrEmjVr6Nu3L02bNuXixYsA+Pr6Ym9vz5gxYwgLC6Nnz54cOHAAZ2dnxo0bh6urK4UKFSIoKIjJkyfTqFEj3n33XTp27EiRIkUIDw9n5MiRLFy4kLZt29KgQQOGDBmSq/1ZwsGGooWscvfLEBGRfFe44J2DaRUqVDBzJY+/6OhooqKizLb9xza8AdjY2DB37lyGDRvGli1bqFu3Lt999x0vvfQSAA4ODhw7doxx48ZRu3ZtfvzxRxo2bMiNGzcybbdWrVq89tprXLlyhdmzZ9OpUydmzpzJF198gZeXFz179gTA3t6eyZMn07lzZy5cuICjoyOBgYGm8Ojg4MDRo0cZP348cOdQbuvWrfn2228BaNeuHd27d0+3/WnTpjFjxgzCwsKwsrLip59+wtvbm/Xr1zN79mwKFChAw4YN2bdvH+7u7ixcuJBGjRqxcePGXO/LaqWK5HpdERF5eGbNmmXuEh578XFxuDdqZLYA91iHNzc3N5KTk9myZQsAO3fu5NKlS1SpUoWzZ8+SkJCAv78/AHv27OH8+fNUrVqVHTt2ZNru5s2buXLlCgC7d+/m+eefz3C5OnXqULp0aRYsWJCurlOnTnH79m2WLVtmmu7v78+UKVP49ttvadiwIVeuXOHw4cNp1rW1teXll1/G2dnZNK1w4cK4ubmxevVqDh48SN26dXF3d2fmzJmMHj0aCwsLGjZsyLhx47K559K7sS+Q5BtXcr2+iIjI48DSzhG7Gl44OTkpvD0sKSkpDzQf4NatW6afk5KSsLbOeDdaWFhw9OhR3nzzzXTzXF1diYuLS7O93bt3Y2lpSc2aNfHx8WHJkiUZtgnwxhtvpKkjVUhICO7u7tSvX5/x48dz+PBh2rRpw7Vr10yHY3Mj+cYVkmJyv76IiIjkjcf2alOAiIgILC0tcXd3B+6MhDk7O3PgwAHgzmHVtm3bAlCzZk1KlChhmpcbsbGx2Nvbm17v2rWLUqVKmbYPdy5qsLGxuW8bS5YsoUePHrzyyiusWLEi3fybN28SGhpK//79TdNKlCjBM888A9wJb61ateLatWvExcUREhLC0KFD2bZtW677JSIiIo+Oxzq83b59m169ejF06FCCg4MZM2YMffr04ebNmwBcu3aNihUrEhwczNSpUxkwYECW57tlZtu2bRQoUIDg4GAmTpzItWvX6NKlCwMHDiQ4OJhNmzbx0UcfYWl5/92+bNkyWrZsSUhICNeuXctwmQEDBlC2bFk2bNjA+vXrmTt3Lo6OjgDs27ePokWLmsJaSEgIpUqVUngTERF5TFi4uLhkfZzQYIoXL87OnTtxc3PL1mFQub958+ZRunRpXC7v1WFTERF54lnZO1O0YXu8vLwIDw/P07bnzZsHQLdu3TJd7rEbeatRowa//fYbU6dOVXATERGRx85jd8HCvn37aNiwobnLEBEREckXj93Im4iIiMjjTOFNRERExEAU3kREREQMROFNRERExEAU3kREREQMROFNRERExEAU3kREREQMROFNRERExEAU3kREREQMROFNRERExEAU3kREREQMROFNRERExEAU3kREREQMxNrcBYgxWNo5mrsEERERs3sUvg8V3iRLKSkp2NXwMncZIiIij4T4uDiio6PNtn2FN8lSVFQUgwYNMncZIiIij4To6GiioqLMtn2FN8lSUlIS4eHh5i5DRERE0AULIiIiIoai8CYiIiJiIApvIiIiIgai8CYiIiJiIApvIiIiIgai8CYiIiJiIApvIiIiIgai+7xJlqysrKhataq5yxCRx5C5b3YqYkQKb5IlFxcXAgMDzV2GiDyG4uLjaeTurgAnkgMKb5IlCwsLdkTEEBuXZO5SROQxUtTWinpu9jg5OSm8ieSAwptkS2xcEldvJpq7DBERkSeeLlgQERERMRCFNxEREREDUXgTERERMRCFNxEREREDUXgTERERMRCFNxEREREDUXgTERERMRCFNxEREREDUXgTERERMRCFNxEREREDUXgTERERMRCFNxEREREDUXgTERERMRCFNxEREREDUXgTERERMZBHOrz9/fffuLq6ZrlcUFAQdnZ2AISFhVGlSpX8Li3P3a9uV1dXOnfunO12IiMjsbe3z7RNERERMa5HOrxll6enJzdu3DB3GfmiVKlSOQpvIiIi8nizNncBd/Pw8GDkyJEkJCSwefPmNPM+/fRT6tevj42NDbGxsQwfPpyIiAjgzmhTpUqViImJMS1fvXp1ZsyYQePGjU3TVq5cybRp09i0aVOatosWLcqoUaOoXbs2ycnJ7N+/nyFDhvDyyy8zfPhwChYsiI2NDXPmzGHx4sUAdOjQgT59+pCQkIClpSXDhg1j7969hIWF0bNnTw4cOADA2rVrGTduHNu3b6dPnz68+eab2NjYkJCQwKhRo9i9e3em+2TixImULFmSoKAgIiMj6d69e6b74n66d+/Om2++SY8ePYiOjs78F5GBEg42FC1kleP1RETup3DBO+MHFSpUMHMlIjkTHR1NVFSU2bb/yIS34sWL4+fnx1tvvcWxY8d45513cHJyMs2fNWsW48aNA6Bly5aMHTuWTp063be9/fv3c+XKFRo1asTWrVupUqUKxYsXTxfcAMaOHUt8fDzNmzcnJSXFtN2///6bVq1akZycTLFixQgMDGTLli2cPXuWUaNG0bhxYy5cuIC1tTUFChTIso9Lly5lzpw5ANSuXRs/P7804TIjI0aMYOzYsXh6euZqX1hYWDB69GhKlSpF+/btiY+Pz7LOjFQrVSRX64mIZGXWrFnmLkEkR+Lj4nBv1MhsAe6RCW+1a9fm0KFDHDt2DIBffvnFFFAAGjVqRI8ePbCzs8PS0pJixYpl2eb3339P9+7d2bp1K926deOnn37KcLnmzZvj7e1NSkoKgGlkytHRkS+++ILy5cuTlJSEo6MjFStW5OzZs2zbto3p06cTHBzMpk2b+Oeff7Ksp2rVqrz//vs4OjqSlJREhQoVKFSoUI4DVU72xeTJkwkPD6d3796m/uXGjX2BJN+4kuv1RUREHgeWdo7Y1fDCyclJ4e1edwcNFxcXPvvsM1q0aMGpU6eoVKkSy5Yty7KNtWvXMnLkSKpUqYKnp2eaMJgdEydOZOPGjfTu3RuAdevWUbBgQQB69+5N9erVadCgAfPnz2fy5MmsWrWKxMRELC3/dyph6vI2NjbMnTuXdu3asW/fPooUKcKRI0coUKBAjsJbTvdFWFgY7u7uPP3005w/fz5H/b9b8o0rJMVczPX6IiIikjcemQsWdu/eTaVKlXBzcwOgffv2puBjb29PYmKiKXx069YtW20mJSXx888/M2/ePNatW5fmnLi7BQUF8e6772JhYQFgOmzq4ODAmTNnAKhXrx6VK1cGwMrKirJly7J//36+/fZb1qxZQ61atQA4efIktWvXBqBmzZqm/qSeNxcZGQlAjx49stWH2NhYihYtanqd032xbNkypk2bhr+/P6VKlcrWNkVEROTR9ciMvEVHR+Pr68v3339PQkICmzZtMh2+PHz4MCtXrmTTpk1cuXKFwMDAbLf7yy+/MGLECH788cf7LjNmzBjGjBnDhg0bSExM5K+//mL48OGMHz+eCRMmMHjwYA4cOMDevXuBO+Htyy+/pFixYiQlJXH58mV8fX2BO4cpp02bRqdOndi9ezdHjx4F4Pr160yePJk1a9YQHR3NypUrs1X/oUOHOHr0KBs2bOD06dN07949x/tizZo1xMfH88svv9C1a9csL24QERGRR5eFi4tL7k+EMoAWLVrQpUsX3n77bXOXYkjz5s2jdOnSuFzeq8OmIiLyxLOyd6Zow/Z4eXkRHh6ep23PmzcPyPqo2iMz8pYfFixYQPny5enVq5e5SxERERHJE491eMvsViIiIiIiRvTIXLAgIiIiIllTeBMRERExEIU3EREREQNReBMRERExEIU3EREREQNReBMRERExEIU3EREREQNReBMRERExEIU3EREREQNReBMRERExEIU3EREREQNReBMRERExkMf6wfSSdyztHM1dgoiIiNk9Ct+HCm+SpZSUFOxqeJm7DBERkUdCfFwc0dHRZtu+wptkKSoqikGDBpm7DBERkUdCdHQ0UVFRZtu+wptkKSkpifDwcHOXISIiIuiCBRERERFDUXgTERERMRCFNxEREREDUXgTERERMRCFNxEREREDUXgTERERMRCFNxERERED0X3eJEtWVlZUrVrV3GWIPBLMfXNOERGFN8mSi4sLgYGB5i5D5JEQFx9PI3d3BTgRMRuFN8mShYUFOyJiiI1LMncpImZV1NaKem72ODk5KbyJiNkovEm2xMYlcfVmornLEBEReeLl6oKFN998Ez8/v/vOnzp1Km+88UauixIRERGRjOUqvPXu3Zvbt2/fd358fDy9e/fOdVEiIiIikrFchTc3NzfCw8PvO//gwYO4ubnluigRERERyViuwpuFhQX29vb3ne/g4ICNjU2uixIRERGRjOUqvIWHh9OqVasMA1qBAgVo1apVpiNzIiIiIpI7uQpvs2bNomLFigQEBODh4UHp0qUpXbo0Hh4eLF26lIoVKzJr1qy8rlVERETkiZerW4Vs2rSJoUOHMnbsWL7//nvTdAsLC65fv86wYcPYsGFDnhUpIiIiInfk+j5v/v7+rF27lkaNGlGmTBkATp06xZYtW7hx40aeFSgiIiIi//NAN+m9fv06a9euzataRERERCQL2QpvLi4uAKbHwaS+zooeHyMiIiKSt7IV3nbs2EFKSgpubm4kJCSYXmeldOnSD1ygiIiIiPxPtsLbkCFDSElJISEhIc1rEREREXm4shXe/P39M30tIiIiIg9Hru7zdq/ixYtTvHjxvGhKRERERDKR66tNn332WYYNG0bjxo2xtbUFIC4uji1btvDll19y5MiRPCtSRERERO7IVXh78cUXWbBgAZaWlgQGBvLPP/8Adx5Y7+npSdOmTXnnnXf4888/87RYc4uMjKRSpUrExMSYu5Rs8fPz48CBA8ydOxdfX18cHBwYPXq0ucsSERGRB5Cr8DZmzBguXbpE27Zt090OxMXFhWXLljF69GhatGiRJ0WKiIiIyB25Cm/PPfccU6ZMyfA+blFRUcyfP58hQ4Y8cHHmEBkZyVdffcUrr7xC4cKFmTp1Kr/++qtpfteuXfHy8qJ48eL4+fmZLt6oXr0648aNo3Dhwty6dYsxY8awa9cuXF1dCQoK4vvvv6d58+YULVqUUaNGsXHjRgBq1KjByJEjKVKkCFZWVsyYMYPVq1enqyt1vdq1a5OcnMz+/fsZMmQIL7/8MsOHD6dgwYLY2NgwZ84cFi9enOf7pYSDDUULWeV5uyJGUrjgndOEK1SoYOZKRMScoqOjzXov21yFt8jISAoWLHjf+QUKFDD0DXpTUlLw8vKidOnSrF27lp07d3LmzBkAbt++jbe3N25ubqxdu5Zly5ZhaWnJ3LlzGTZsGFu2bKFu3bp89913vPTSSwA4ODhw6NAhvvzyS5o0acJ///tfNm7ciL29PZMnT6Zz585cuHABR0dHAgMD2bVrF+fOnUtT09ixY4mPj6d58+akpKTg5OQEwN9//02rVq1ITk6mWLFiBAYGsmXLFs6ePZun+6RaqSJ52p6Ikc2aNcvcJYiIGcXHxeHeqJHZsk6uwpufnx9jxoxhw4YNHDhwIM28KlWq0L17d0OfW7Vo0SIATp8+zY4dO6hfvz5Lly4FYPny5QBERESQmJjI008/jYODA8nJyWzZsgWAnTt3cunSJapUqcLZs2eJi4szPUZs9+7dpmfB1qlTh9KlS7NgwYI023dzc0sX3po3b463t7fp/nrR0dEAODo68sUXX1C+fHmSkpJwdHSkYsWKeR7ebuwLJPnGlTxtU0RExGgs7Ryxq+GFk5OTscJb7dq1uXjxIr///ju7du3i5MmTAJQrV44XXniBI0eO8MILL/DCCy+Y1klJSTFsoLv7hsS3bt0y/ZycnIyVVcaHEu9e5/bt26afk5KSsLa+s9stLCw4evQob775Zq5rmzhxIhs3bqR3794ArFu3LtNR0dxKvnGFpJiLed6uiIiI5Eyuwlv37t1NP9etW5e6deummf/888/z/PPPp5lmpPD29ttvM3XqVFxdXalXr16WdUdERGBpaYm7uzshISHUqVMHZ2dnDhw4YDq8mZFdu3ZRqlQp03pwZ+Ty6NGjpqdZpAoKCuLdd99l5MiRpsOm0dHRODg4mA7p1qtXj8qVKz9g70VERORRlqvwVqpUqbyu45FiZWVFYGAghQsX5tNPPzWFo/tJSEigV69ejBs3jlGjRnHr1i369OnDzZs3Mw1v165do0uXLowaNYpRo0ZhbW1NZGQkPXv2TLfsmDFjTIeqExMT+euvvxg+fDjjx49nwoQJDB48mAMHDrB3794H7r+IiIg8uixcXFz0kNK7GO1ebvlt3rx5lC5dGpfLe3XYVEREnnhW9s4UbdgeLy8vwsPD87TtefPmAdCtW7dMl8v1Exbgzghcs2bNKFmyJHAn+GzcuJF///33QZoVERERkfvIdXgbNWoUPXv2xNIy7eNRk5OTmTt3LuPGjXvg4swhNYiKiIiIPIpyFd769u1L7969WbNmDd9++y3Hjh0D7jzvtHfv3vTu3Ztz587x3Xff5WmxIiIiIk+6XIW3jh07mq5+vNvevXt57733KFiwIJ06dVJ4ExEREcljllkvkp6rq6vphrQZ2bJlC66urrkuSkREREQylqvwdvny5UzvJ1a5cmXTEwBEREREJO9kO7zVq1fPdM+y1atX06FDB/r374+tra1pGVtbW9577z06dOjAqlWr8r5aERERkSdcts95CwgI4P3332fFihVMnjyZKlWqMGLECIYOHcr58+cBKFGiBNbW1oSGhjJlypR8K1pERETkSZXt8GZhYWH6OT4+nrfffhtPT88093nbvHkzGzZsIDg4OO8rFREREZEHu0lvUFAQQUFBeVWLiIiIiGQhRxcspKToSVoiIiIi5pSjkbcZM2YwY8aMbC2bkpJCmTJlclWUiIiIiGQsR+EtJCSEf/75J79qEREREZEs5Ci8BQQEsGLFinwqRURERESykqub9IqIiIiIeTzQ1aby5LC0czR3CSIiImb3KHwfKrxJllJSUrCr4WXuMkRERB4J8XFxZn0MaLbDW6lSpfKzDnmERUVFMWjQIHOXISIi8kiIjo4mKirKbNvXyJtkKSkpifDwcHOXISIiIuiCBRERERFDUXgTERERMRCFNxEREREDUXgTERERMRCFNxEREREDUXgTERERMRDdKkSyZGVlRdWqVc1dhkieMPf9mUREHpTCm2TJxcWFwMBAc5chkifi4uNp5O6uACcihqXwJlmysLBgR0QMsXFJ5i5F5IEUtbWinps9Tk5OCm8iYlgKb5ItsXFJXL2ZaO4yREREnni6YEFERETEQBTeRERERAxE4U1ERETEQBTeRERERAxE4U1ERETEQBTeRERERAxE4U1ERETEQBTeRERERAxE4U1ERETEQBTeRERERAxE4U1ERETEQBTeRERERAxE4U1ERETEQBTeRERERAxE4U1ERETEQBTeRERERAxE4S2PrF27lgYNGjxwO2XKlGHdunUEBgbi4+OTB5VBxYoVCQsLy5O2RERExLyszV2AEVlZWZGUlJQv7bRo0YJ9+/bx4YcfPnD7IiIi8vhReLtL48aN+eijj7CysuLatWt89NFHHDt2jAYNGvD555+zZ88eqlevzvTp0zl37hzjx4/HysqKffv2YW39v13p7OzMuHHjcHV1pVChQgQFBTF58mQAwsLCWLVqFQ0bNuTEiRMMHDjQtF7btm3p3bs3VlZW1KpVi/79+5OQkMDEiRMpXrw4KSkpfPnllwQGBmZaL4Cvry+tW7cmNjaWTZs2PfC+KWpr9cBtiJib3sci8jhQePt/xYsXZ9asWbRt25bDhw/z1ltvMWfOHJo2bQrAs88+y8cff8zQoUOxsbEhNDQUX19fQkJCaNSoEW+//baprWnTpjFjxgzCwsKwsrLip59+wtvbm9WrVwPg6OiIt7d3uhqWLl1K6dKlcXBwYPTo0QD89ttvLFmyhAULFlCuXDl+++03wsPDiY+Pv2+9r7zyCt7e3rz66qtcv36d6dOnP9C+SUlJoZ6b/QO1IfKoiIuPJzo62txliIjkmsLb/6tduzaHDh3i8OHDAPz66698/vnnPPPMMwCcOnXKdN5YhQoVSExMJCQkBICtW7dy8uRJAGxtbXn55ZdxdnY2tV24cGHc3NxMr/39/bNVk52dHdWqVaNVq1YAnDhxgj///JN69eoRGxt733pffvllfvvtN65fvw7AggULePHFF3O5ZyAqKopBgwblen2RR0l0dDRRUVHmLkNEJNcU3rLp5s2bmc5PSUkBwMLCAoA33niDW7du5aqt7Gwnv9e5W1JSEuHh4Q/UhoiIiOQNXW36/3bv3k2lSpWoWLEiAC1btuTcuXOcPXs23bLHjx/H2tqahg0bAuDu7k65cuWAO8EsNDSU/v37m5YvUaKEaQQvJ27cuMHff/9tOiRbtmxZXnzxRcLCwjKtNyQkBG9vb+zs7ADo1KlTjrctIiIijyaNvP2/6OhoBgwYwFdffWW6AKBv374ZLpuQkEC/fv0YP348lpaW7Nu3jwMHDpjmDxgwgNGjR7NhwwZSUlKIi4vjww8/zDAIZmXgwIFMnDiR7t27k5KSwtChQ02HfO5X78aNG6lZsyaBgYF5dsGCiIiIPBosXFxcHuyYmjzW5s2bB0C3bt3MWoeIiMjjLrvfuTpsKiIiImIgCm8iIiIiBqLwJiIiImIgCm8iIiIiBqLwJiIiImIgCm8iIiIiBqLwJiIiImIgCm8iIiIiBqLwJiIiImIgCm8iIiIiBqLwJiIiImIgCm8iIiIiBqLwJiIiImIgCm8iIiIiBqLwJiIiImIgCm8iIiIiBqLwJiIiImIgCm8iIiIiBqLwJiIiImIgCm8iIiIiBqLwJiIiImIgCm8iIiIiBqLwJiIiImIgCm8iIiIiBqLwJiIiImIgCm8iIiIiBqLwJiIiImIgCm8iIiIiBqLwJiIiImIgCm8iIiIiBqLwJiIiImIgCm8iIiIiBqLwJiIiImIgCm8iIiIiBqLwJiIiImIgCm8iIiIiBqLwJiIiImIgCm8iIiIiBqLwJiIiImIgCm8iIiIiBqLwJiIiImIgCm8iIiIiBqLwJiIiImIgCm8iIiIiBqLwlg1BQUHY2dlluZyPjw9ubm55vv3q1avz9ddf53m7IiIiYjwKb9ng6enJjRs3slyuXbt2VKhQIcftW1hYYGFhcd/5+/fvp1+/fjluV0RERB4/1uYuwAgiIyOpVKkSMTExhIWFsXTpUho1aoSzszOLFy/mq6++okOHDtSoUYMxY8YwZMgQJk6cyMaNG+nbty9vvPEG1tbWXLp0iQ8//JDIyEh8fX15/vnnsbOzw8XFhenTp9OqVSu6du1q2m5oaCi9evXCwcGBsWPH4unpCUDjxo0ZPHgwhQoVIikpifHjxxMaGsrMmTNZv349K1asoGvXrowZM4bKlSsTFxeHv78/X375JTt27Mhx/62srKhatWqe7U8REREji46OJioqymzbV3jLBXt7e1q2bImjoyOhoaEsWbKEX375hdatWzN37lwCAwMBaNWqFW5ubrRs2ZLk5GTatGnDhAkT6NKlCwAvvPACXl5eXLp0iUKFCvHf//4XZ2dnLl68SMOGDbl27RoHDx6kQYMGpm2XLl2aIUOG0LFjR65fv07ZsmVZvnw59evXJyQkBHd3d1asWIG7uzv79++nQYMGhIaGUrlyZXbv3p2r/rq4uJj6JCIi8qSLj4vDvVEjswU4hbdcWLFiBQBXrlzh9OnTlCpVinPnzqVb7tVXX6VGjRqsW7cOAEvLtEepN27cyKVLlwCIj49n7dq1tGnThm+++QYfHx+WLFmSrs2mTZuaAluq5ORkSpYsSUhICL6+vlhaWvLcc88xadIk3N3dSUpKYu/evSQmJuaqvxYWFtzYF0jyjSu5Wl9ERORxYWnniF0NL5ycnBTejOTWrVumn5OSkrC2zng3WlhYMHPmTBYuXJjh/Js3b6Z5vXjxYqZOncr8+fNp3rw5Y8aMyXC9rVu3MmDAgAzn3b59m9atW7N//362bdvG+++/T1JSEtu2bctGz+4v+cYVkmIuPlAbIiIi8uB0wUIeun79Ovb29qbX69ato3PnzhQrVgwAa2trqlSpct/19+7dC8CoUaMICQnh6tWr6ZbZsmUL7u7uVKpUyTStZs2app9DQkIYOnQoISEhXLt2jYSEBLy9vR84vImIiMijQeEtDy1cuJCBAwcSFBREs2bN+PXXX/H39ycgIIDg4GCCgoJ4+eWXM21jyZIldO7cOcNDpgAnT55kwIABTJo0ieDgYDZv3kyvXr1M80NCQihVqhQhISEAbNu2jcKFC3PgwIG866iIiIiYjYWLi0uKuYuQR9e8efMoXbo0Lpf36rCpiIg88azsnSnasD1eXl6Eh4fnadvz5s0DoFu3bpkup5E3EREREQNReBMRERExEIU3EREREQNReBMRERExEIU3EREREQNReBMRERExEIU3EREREQNReBMRERExEIU3EREREQNReBMRERExEIU3EREREQNReBMRERExEIU3EREREQNReBMRERExEGtzFyDGYGnnaO4SREREzO5R+D5UeJMspaSkYFfDy9xliIiIPBLi4+KIjo422/YV3iRLUVFRDBo0yNxliIiIPBKio6OJiooy2/YV3iRLSUlJhIeHm7sMERERQRcsiIiIiBiKwpuIiIiIgSi8iYiIiBiIwpuIiIiIgSi8iYiIiBiIwpuIiIiIgSi8iYiIiBiIwpuIiIiIgSi8iYiIiBiIwpuIiIiIgSi8iYiIiBiIwpuIiIiIgSi8iYiIiBiIwpuIiIiIgSi8iYiIiBiIwpuIiIiIgSi8iYiIiBiIwpuIiIiIgSi8iYiIiBiIwpuIiIiIgSi8iYiIiBiIwpuIiIiIgSi8iYiIiBiIwpuIiIiIgSi8iYiIiBiIwpuIiIiIgSi8iYiIiBiIwls2BAUFYWdnl+v1S5QowfLly02vfX19KViwYLbW9fHxwc3NzfTaw8ODMWPG5LoWERERMTaFt2zw9PTkxo0buV7//PnztG7d2vR6yJAh2Q5v7dq1o0KFCqbXwcHBCm8iIiJPMGtzF2AEkZGRVKpUiZiYGMLCwli6dCmNGjXC2dmZxYsX89VXX1GnTh0mTpxI8+bNTesFBATw3XffcfDgQYKCgqhcuTITJ04EYPny5SQnJ9OhQwfc3d3p2bMnBQoUwNLSksmTJxMcHEyHDh2oUaMGY8aMYciQIUycOJGnnnoKLy8vevbsCcC7776Lj48PycnJHDp0iI8//pjY2Fh8fX159tlnsbW1pUyZMly8eJE+ffpw9erVHPffysqKqlWr5sm+FJEnT3R0NFFRUeYuQ+SxofCWC/b29rRs2RJHR0dCQ0NZsmQJu3btomDBglSvXp39+/dTunRp3Nzc2LBhA88884xp3REjRtC5c2dat25NTEwMAFu2bGHFihUAuLq68ttvv1GvXj1++eUXWrduzdy5cwkMDATuHEZN1bRpU9q3b0/Lli2JiYlh0qRJfPzxx3z00UcA1KpVi9dee40rV64we/ZsOnXqxMyZM3PcXxcXF9P2RURyKj4uDvdGjRTgRPKIwlsupAatK1eucPr0aUqVKsW5c+dYsmQJb7/9Nvv376ddu3b8+uuvJCUlZdleqVKlmDFjBs888wxJSUkUK1aMUqVKERERkel67u7urFq1yhQC58+fz7fffmuav3nzZq5cuQLA7t27ef7553PVXwsLC27sCyT5xpVcrS8iTy5LO0fsanjh5OSk8CaSRxTecuHWrVumn5OSkrC2vrMbAwICCA4OZty4cbRt25auXbtmq73Zs2czYcIE1qxZA0B4eDiFChXKtzpzI/nGFZJiLj5wTSIiIvJgdMFCHjp//jx//fUXY8aM4dKlSxw9ejTD5WJjYylatKjpdbFixTh9+jQArVu3xtHR0TTv+vXr2NvbZ9hOSEgIb7zxBkWKFAGgU6dObN26Na+6IyIiIo8ghbc85u/vT+fOnfH397/vMt9++y2LFy8mKCiI4sWLM2rUKObMmUNgYCBVq1blzJkzpmUXLlzIwIEDCQoKolmzZmna2bRpE0uWLGHVqlWsX7+eIkWKMGHChHzrm4iIiJifhYuLS4q5i5BH17x58yhdujQul/fqsKmI5JiVvTNFG7bHy8uL8PBwc5cj8kibN28eAN26dct0OY28iYiIiBiIwpuIiIiIgSi8iYiIiBiIwpuIiIiIgSi8iYiIiBiIwpuIiIiIgSi8iYiIiBiIwpuIiIiIgSi8iYiIiBiIwpuIiIiIgSi8iYiIiBiIwpuIiIiIgSi8iYiIiBiIwpuIiIiIgVibuwAxBks7R3OXICIGpM8Okbyn8CZZSklJwa6Gl7nLEBGDio+LIzo62txliDw2FN4kS1FRUQwaNMjcZYiIQUVHRxMVFWXuMkQeGwpvkqWkpCTCw8PNXYaIiIigCxZEREREDEUjb5IpJycnrKysmDdvnrlLEREReaw5OzuTlJSU5XIKb5KphIQEc5cgIiLyREhKSsrW966Fi4tLykOoR0RERETygM55ExERETEQhTcRERERA1F4ExERETEQhTcRERERA1F4ExERETEQhTcRERERA1F4ExERETEQhTcRERERA1F4ExERETEQhTcRERERA1F4ExERETEQhTcRERERA1F4e4KUK1eOlStXEhISwpo1a3juuecyXK59+/Zs27aNP/74g8mTJ2Ntbf3A8x6W/OzjSy+9xOrVq9m0aRMbN25k5MiRWFhYPJR+3S2/f4+p/P39OXjwYL71437yu3/PP/88AQEBbN68mc2bN/Paa6/le5/ulZ99tLCwYPTo0WzatIng4GACAgIoW7bsw+hWGg/aR1dXVwICAjh06BBBQUHZXu9hyc/+PS6fNVn9DlOZ67MG8r+P+fV5o/D2BJk0aRILFy7E3d2d2bNn4+fnl26ZUqVKMWzYMN566y1eeuklnJ2deeeddx5o3sOUn328evUq7733Hk2bNuW1116jTp06tGvX7qH2D/K3j6n69OnDyZMnH0Z30snP/hUqVIgffviByZMn06RJE5o1a8aOHTseav8gf/vo6elJnTp18PDwwMPDg23btjFixIiH2j948D5ev36dyZMnM2DAgByt97DkZ/8el8+azPqYypyfNZC/fczPzxuFtydE8eLFqV69OsuWLQNgzZo1uLi4pPsXeYsWLQgODubixYsA/Pzzz7Rq1eqB5j0s+d3HAwcOcPr0aQBu3brFgQMHcHV1zf+O3SW/+wjw3HPP4eXlxaxZs/K9P/fK7/699dZb7Nmzh507dwKQnJxMdHR0/nfsLvndx5SUFAoWLEjBggUBKFKkCGfPns3/jt0lL/p49epVdu7cyc2bN9O1b+7Pm/zu3+PyWZNZH8G8nzWQ/33Mz88bhbcnhIuLCxcuXCApKck0LTIykpIlS6ZZrmTJkpw5c8b0+t9//zUtk9t5D0t+9/Fuzs7OtGjRgvXr1+d1NzKV3320trZmypQpfPjhh2m28bDkd/+ee+45bt++zU8//URQUBBfffUVTk5O+dmldPK7j8HBwWzfvp2//vqLvXv38vLLLzNlypT87FI6edHHzJj78ya/+3c3I3/WZMbcnzWQ/33Mz88bhTeRHCpSpAjz5s3j66+/Zv/+/eYuJ0/5+vqydu1ajh8/bu5S8oWVlRXu7u58+OGHeHp6cvbsWSZOnGjusvJUjRo1qFixIi+88AK1a9dm27Ztj10fnxT6rDG2/Py8UXh7QkRFRfH0009jZWVlmlayZEkiIyPTLBcZGZlmeL5UqVKmZXI772HJ7z4C2NnZsXDhQoKCgpgzZ05+deW+8ruP9evXp0ePHoSFhbFixQqKFi1KWFjYQxudehjv09DQUM6dOwfA8uXLqV27dr71JyP53ce2bdvyxx9/EBMTQ0pKCgEBATRs2DA/u5ROXvQxM+b+vMnv/sHj8VmTGXN/1sDDeZ/m1+eNwtsT4vLly/z999+0adMGuHMM/+zZs+lOFF27di0eHh44OzsD0LlzZ1auXPlA8x6W/O5j4cKFWbhwIZs2beKrr756SL1KK7/72Lp1a+rVq0f9+vVp1aoVsbGx1K9f/6GdF5bf/fvtt9+oUaMGRYoUAaBZs2YP/Sq3/O7j6dOneemll7CxsQGgefPmHDly5GF0zSQv+pgZc3/e5Hf/HpfPmsyY+7MG8r+P+fl5Y+Hi4pKSJy3JI8/NzQ0/Pz8cHR2JjY3F19eXw4cPM2XKFIKCgggODgagY8eO9O/fH4Dt27czYsQIEhMTH2je49DH999/H19fX44ePWra3urVq5k+ffpj08e7ubq6EhQUROXKlR9i7/K/f23atOG9994jOTmZc+fO8eGHHxIVFfXY9LFAgQJ89tlnvPjiiyQkJHDx4kVGjBhhOgHeKH0sVKgQ27Zto0CBAhQtWpTLly+zdOlS02Enc3/e5Gf/HpfPmqx+h6nM9VnzMPqYX583Cm8iIiIiBqLDpiIiIiIGovAmIiIiYiAKbyIiIiIGovAmIiIiYiAKbyIiIiIGovAmIiIiYiAKbyIiIiIGovAmIpJHAgICCAgIML12dXUlMjISHx+fh1qHn58fYWFhD3WbIvLwKLyJyEPj4+NDZGQkERER/Oc//0k3PyAggA0bNpihsidTQEAAkZGRGf7n5uZm7vJE5D6szV2AiDx5ChUqRP/+/fn000/NXUq+OnPmDOXLlychIcHcpdxXVFQUEyZMSDf9/PnzZqhGRLJD4U1EHrrw8HA6duzIzJkz8zUkFCpUiPj4+HxrPztu3bpl1u1nJSYmhuXLl2d7eVtbW+Li4vKxIhHJig6bishDN2PGDKysrEwPes6MlZUVgwcP5o8//uCff/4hLCyMESNGUKBAgTTLhYWF8dNPP9G4cWPWrl1LREQEnTp1okGDBkRGRvLGG2/wwQcfsGvXLo4cOcKcOXMoWrQoBQoUYOzYsezbt4+jR48yderUdG37+Pjg7+/Pvn37+Oeff9i0aRNdunTJsvZ7z3lLrSWj/+49R61p06YsX76cY8eOceTIEebPn89zzz2XbhteXl5s2LCBiIgINmzYwKuvvpplXdnl5+fH0aNHKVOmDPPnz+fIkSPMnDkTAAsLC3r16sXGjRuJiIjgr7/+YtKkSTg4OKRrZ9CgQezatYvjx48TEBDAc889R1hYGH5+fqZlfH19iYyMTLdu6qF2V1fXNNOzs39S6//Pf/7D999/z9GjR9m/fz+ffvoplpZpv/4sLCzo2bMn69evJyIigv3797NgwQKqV68OwNKlS00PKb/X1q1bWbhwYTb2qEje0MibiDx0p0+fZunSpXTs2JFZs2ZlOvr2xRdf4OPjw+rVq5kzZw61atVi4MCBVKhQgV69eqVZ1s3NjVmzZrFgwQIWLVpERESEad6AAQOIj49n1qxZlC1blh49epCQkEBycjIODg5MnTqV2rVr8/bbb3P69GmmTZtmWrdLly4cPXqUoKAgEhMT8fDwYMKECVhYWPDTTz9lu9/Hjh1j4MCBaabZ29szevRoLl26ZJrWpk0bpk2bxubNm/n888+xtbWlS5cu/Prrr3h5eXHmzBkAGjVqxHfffcfRo0eZOHEijo6OTJ06lbNnz2a7JisrKxwdHdNMu3XrFjdv3jTNX7hwITt37mTcuHGmUbdJkybh4+PDkiVL+OGHHyhVqhTdu3enSpUqtGrVisTERACGDRvG4MGD2bBhAxs2bKBatWosWrQoXUDOiezuHwBLS0sWLlzI3r17GTduHO7u7rz77rucOnWK+fPnm5b78ssvefvtt9mwYQO//PIL1tbWvPjii9SuXZv9+/ezbNkyvvjiCypWrMiRI0dM69WoUQM3Nze++uqrXPdHJKcU3kTELKZPn07btm157733GD16dIbLVK5cGR8fHxYuXMjw4cMB+Omnn7h06RL9+vWjYcOGhIaGmpYvV64cHTt2ZMuWLaZpDRo0AO6EkDZt2phCRfHixXnzzTfTjKL99NNPlC1blvbt26cJb23btk1z+HXevHksWLCAPn365Ci8Xbp0Kd0hynnz5nH79m0++OADAAoXLsx///tfFi1axIcffmhaLiAggK1btzJw4EDT9JEjR3Lx4kXeeustYmNjAdi+fTuLFy/m33//zVZNzz77LOHh4Wmm+fv7m+opVKgQq1evZuLEiab5devW5Z133qF///6sWLHCND00NJRFixbh7e3NihUrcHJyol+/fqxfv56uXbualvvwww95//33s1XfvXKyf+DOYd7ffvvN9Pv8+eefWbduHe3btzeFt4YNG/L2228zd+7cNO/Fb7/91vTz6tWrGTduHK1bt05zjmDr1q25ceMGa9euzVV/RHJDh01FxCxOnz7NsmXLeOedd3j66aczXKZZs2YAzJkzJ8301C/VV155Jc30U6dOpQlud1u6dKkpuAHs3bsXS0tLlixZkma5vXv34uLigpWVlWna3cGtaNGiODo6EhYWRtmyZSlatGhWXb2vwYMH4+HhwQcffMCxY8eAO6NpxYoVY+XKlTg6Opr+S0pKYu/evbz00ksAPP3001StWpWAgABTcAMICQlJMzKUldOnT9O+ffs0/82ePTvNMnePUAF4e3tz7do1tm7dmqbG/fv3c/36dRo2bAiAu7s7BQsW5Icffkiz/nfffZf9nXSP7O6fzOrfsWMHpUuXNr1+/fXXSU5OTnMY916xsbEEBQXRqlUr0zRLS0tatmzJunXrdB6gPFQaeRMRs/nqq69o06YN/fv3z3D0zdXVlaSkJE6ePJlm+sWLF7l69Wq686AyG22KiopK8zo18Nw7PSYmBisrK+zt7bly5QoAderUYejQobzwwgsULlw4zfJFixZNE56yq0mTJvj6+jJjxow0ozblypUDSHO/uHvrA0x9P3HiRLplIiIiqFatWrbquHnzJiEhIfedn5CQkO4wbLly5XBwcODvv//OcJ2nnnoq0xqjo6NN+zansrt/UsXFxREdHZ1m2rVr19IcKi5Tpgznz5/n6tWrmW576dKlvPnmm9SrV48dO3bg7u7O008/zbJly3LRE5HcU3gTEbM5ffo0y5cv55133mHWrFn3XS4lJSVb7WV2ZWlSUlKOpqcqU6YMS5YsISIigrFjxxIVFUVCQgLNmjWjT58+6U58z45SpUoxc+ZMtm7dyqRJk9LMS21v4MCBXLx4Md26d48ePgy3b99Ot/8tLS25ePFiuvP3Ul2+fDnH27nf7/je/ZvT/ZOcnJzjWu5n8+bNXLhwgdatW7Njxw7atGnD+fPnMw2/IvlB4U1EzOqrr76idevWGV55eubMGaysrChXrhzHjx83TX/qqacoVqxYmhPT84uHhweFChWiW7duaUbpUg8N5lShQoWYO3cu165do3///ulCy6lTp4A758dlFgpS+546EnW3/L7B7qlTp3B3d2fnzp2ZBua7azx9+rRpupOTU7qLJK5duwbcuYDj7tGze0dXs7t/cuLUqVM0adKEYsWKZTr6lpyczIoVK2jXrh3jx4/Hy8uLRYsW5WlAFMkOnfMmImZ16tQp0+jbvee+bdy4EYDevXunmd6nTx+Ah/I0htSROQsLC9O0okWL5vqRVxMnTqR8+fL06tXLFFjutnnzZmJiYhg4cCDW1un/fe3k5ATAhQsXCA8Pp127dmnOu3N3d6dixYq5qi27fvvtN6ytrRk8eHC6eamHnOHO+Xe3b9+mR48eaZa59/cJ/wtl9erVM02ztbWlXbt2aZbL7v7JibVr12JpaWm6SCMzS5cuxdHRkUmTJlGkSBEdMhWz0MibiJjd9OnTadOmDRUqVODw4cOm6QcPHsTf359OnTphb29PWFgYNWvWxMfHh99//z3Nlab5ZevWrdy6dct0hamdnR0dO3bk8uXLGT7iKzOvvPIK7dq1Y82aNVSqVIlKlSqZ5t24cYPAwECuX7/ORx99xPTp01m3bh2rVq3i8uXLlCxZkldeeYWdO3fyySefADBhwgTmz5/Pr7/+ypIlSyhWrBjdu3fn8OHD2NnZ5el+uFtYWBg///wzAwcOpHLlymzdupWEhATKly9PixYtGD16NGvWrCE6Oppvv/2WgQMHMn/+fDZs2EDVqlVp2rRpukOrW7Zs4cyZM3z55Zd8/fXXJCcn8/bbb3P58uU0o2852T/ZFRoaytKlS+nVqxflypVj8+bNWFpa8uKLLxIaGsq8efNMyx44cIBDhw7xxhtvcPTo0XRX6oo8DApvImJ2J0+eZPny5RmOZg0dOpRTp07h4+PDq6++ysWLF5kxYwZTp059KLVFRETQt29fhg8fzqeffsrFixeZP38+ly9fzvTqxIwUL14cgBYtWtCiRYs08/79918CAwMBWLFiBefPn6d///68++67FChQgHPnzvHnn3+muTp28+bNptpGjBjBqVOn8PX1xcvLy3SLlPwyYsQI9u/fT6dOnRgxYgSJiYn8+++/LF++nJ07d5qWmzRpEvHx8XTu3JmGDRuyd+9eOnbsmO4K0MTERHr27MmECRMYNmwYFy9eNB1evnc/Z3f/5MQHH3zAwYMH6dChA5988gmxsbHs27ePXbt2pVt26dKlfPrppxp1E7OxcHFxyd6ZwCIiInkkLCyM7du3Z+tQ5aOmZ8+ejBkzhnr16qW7WlnkYdA5byIiIjnQvn17wsLCFNzEbHTYVEREJAu2trZ4enrSsGFDKleuTLdu3cxdkjzBFN5ERESyULx4cWbPns3Vq1eZPn36fR9SL/Iw6Jw3EREREQPROW8iIiIiBqLwJiIiImIgCm8iIiIiBqLwJiIiImIgCm8iIiIiBqLwJiIiImIgCm8iIiIiBqLwJiIiImIg/wdahu3HQUso7wAAAABJRU5ErkJggg==",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "#to enhance training, consider rolling the shuffle dice until you get good consistency between training and test set categories.\n",
    "visualize_train_test_category_splits(train_df, test_df, top_x=5, normalize=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "3b8f0497-7b16-406e-b3dc-3f0c265fdc69",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAk8AAAHLCAYAAADRDnw9AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/SrBM8AAAACXBIWXMAAA9hAAAPYQGoP6dpAABVNUlEQVR4nO3deXgUVaL+8W91Zw8JaULYw3JlREXB4aoswqiDKBIBlUVRwOWHC4Ogoiw6LnNFRVCEQdHhijPKIpdVEZAdRRgWUZRNREARSNgCIfvSS/3+IOkhK2m6052G9/M8eejUqa461V10vznn1CmjQYMGJiIiIiJSKZZAV0BEREQkmCg8iYiIiHhA4UlERETEAwpPIiIiIh5QeBIRERHxgMKTiIiIiAcUnkREREQ8oPAkIiIi4gGFJxEREREPKDyJ+MjEiRNJTk6mUaNGga6KVKBnz54sX76cvXv3kpyczP/8z/8Eukrigb59+5KcnEzfvn0DXRW5hCk8iZyjUaNGJCcnM3PmzHLXad++PcnJybz55ps+3+/EiRN9tk0p7b//+7957733qFGjBtOnT2fChAl89dVXZa47fPhwkpOTK/0zfPjwKqlzcnIy8+bNu6DnrVu3rgpqVPUu9JhF/CUk0BUQuViMHTuW9957j2PHjgW6KlKOzp07Y7FYePrpp/nuu+8qXHfTpk1MmDCh2LKWLVvStWtXNm7cyKZNm0qtLyKXBoUnER85ceIEJ06cCHQ1pAJ169YFqFTA3bRpU6lA1LdvX7p27cqmTZt45513qqSOIlL9qdtOxEfKG/PUrVs35s+fz/bt2zlw4ADff/89//d//0e3bt2As1/IW7ZscT8+tyuoffv27u1ERkby7LPPsm7dOg4cOMCuXbuYPn061113XZn1sdlsjBs3ju3bt7N//36WLl1K165dyxwzcm63YfPmzZk2bRq7du0qdjxdu3ZlypQpbNiwgf3797Nnzx4WLlzoPo5zldzeJ598wk8//cTu3buZMmUKNpsNONuNNmfOHH7++Wd2797NW2+9RWRkpEev+3XXXcf06dPZtWsXBw4cYN26dTz77LNERES41ynqar3vvvsA2LJli/s19tUYtSuvvJL333+fbdu28dtvv7FlyxbGjBnjPlY4+55899137N27l6ZNmxZ7fsmyojoDdOjQodh54evxPqGhoTz22GMsX76cffv2sXfvXhYuXEiXLl1KrVt0nicmJvLII4+wbt06fv31V7Zs2cIzzzyDYRilnhMREcFf//pXtm7dyoEDB1izZg3333+/+xiLujw9PeY//elPLFq0iP3797Nr1y4mTZpU7PUu0qFDB2bMmMH333/Pr7/+yo8//sjChQt54IEHvH3p5BKllieRKjRw4EDGjh3LsWPHWL58OWlpaSQkJHDttdfStWtXvvzyS3bv3s20adMYNGgQu3fvZvny5e7nHz58GIDw8HDmzp1LmzZt2LFjB9OmTSMhIYEePXpw0003MWTIEJYsWeJ+XlRUFAsWLKBFixZs3bqVzZs306BBA95///0Kx8E0bdqUxYsX8/PPPzN37lxsNht2ux2A559/noKCArZu3crx48eJj4/ntttu48MPP+TFF1/kX//6V6ntJSYmsmjRInbs2MHs2bNp1aoVd911Fw0aNOCNN95g9uzZfPPNN8yaNYv27dtz//33Y7FYePbZZyv1+t55551MmTKFgoICvvjiC1JTU7npppsYPnw4N998M7179yY/P5/Dhw8zYcIEunbtSsuWLZk2bRrp6ekAZGRkVGpfFenSpQv/+Mc/ME2TFStWkJKSwuWXX84jjzzCzTffzJ133kl6ejppaWk89dRTzJ49m/fee4+77roLh8MBwNtvv039+vV5+umnOXjwIA6HgwkTJvDss89y+PBh5s6d697f7t27va5zkbCwMGbNmkWHDh3YtWsXs2fPJjQ0lM6dO/Pxxx/z17/+lY8//rjU81566SXatWvH6tWr+frrr+natSvPPfccYWFhjBs3zr2exWJh+vTp3Hjjjfz000989tlnxMXF8fLLL5dq2St6nypzzF26dKFz586sXr2a7777jrZt29KnTx+aNGnC3Xff7V6v6DjS09NZuXIlJ06coFatWlx11VX06tWLWbNm+eiVlEuJwpNIGZo2bVruAODExMRKb6dfv37k5+dz2223cerUqWJlRX8h7969mw8//NAdnsrqDho8eDBt2rRhwYIFDBs2zL38o48+YvHixYwbN46vvvqK7OxsAIYMGUKLFi2YOXMmo0aNcq8/d+5c5syZU259b7jhBt55551SY30ABgwYwKFDh4ote+WVV1i0aBEjRoxg9uzZ5OXlFStv3749L7/8Mh999JF72fTp091faH/5y19YuXIlACEhISxbtoxevXoxduxYUlNTy60nQI0aNRg/fjwOh4MePXqwZ88eAN58802mTJlCz549GTx4MJMmTeLIkSO88847JCYm0rJlSz788EOOHDlS4fYry2azMXnyZNLS0ujZs6e75QSgR48efPDBBzz33HO89NJLAPz73//m/fffZ+jQoYwcOZI33niDBx98kK5du/LZZ5+5B0oX1bkoSFRVN+EzzzxDhw4dmDhxIm+//bZ7eXR0NHPnzuXll19m2bJlHD9+vNjzrr76am699VZ3V/WkSZPYsGEDDz/8MO+88447dPft25cbb7yRNWvW8NBDD+FyuQD48MMPWbZsWbFtenLMXbp0oXfv3u6xaxaLhTlz5tChQwfatGnDtm3bALj33nuxWCz07duXn376qdg2ymqlEqkMdduJlKFZs2Y8++yzZf542mXicDjcrQvnSktLq/Q2+vTpQ0FBAWPHji22fPfu3cybN4+4uDi6du3qXn7PPfeQn5/PW2+9VWz9DRs28PXXX5e7n+PHjzN58uQyy0oGJ4CcnBzmzZtHzZo1ufbaa0uV//bbb8WCE8CiRYvcdS8KTnD2dVq6dCmhoaFcfvnl5daxyO23307NmjWZM2eOOzgBmKbJa6+9ht1up0+fPufdjrd69+5NbGwsY8eOLRacAL744gt27NhBz549iy1/++23+eGHH3jiiSd4+OGHeemllzh06BCjR4+u8vqeyzAMBgwYwG+//VYsOAFkZ2czadIkwsPDueOOO0o9d9KkScXG+KWlpbFy5UpiYmK47LLL3Mt79eoFwLhx49zBCWDfvn0sWLDgguv++eefFxv073K53MGzdevWpdYvGeyL6ixyIdTyJFKGr776iv79+5dZ1r59e+bPn1+p7SxatIiXXnqJNWvW8Pnnn7Nx40a+/fZbsrKyKl2XGjVq0LRpU3755ReOHj1aqnzjxo3079+fli1bsmDBAmrUqEHjxo3Zu3dvma03W7du5eabby5zXz/99JO7xaCk+Ph4nnzySW655RYaNWpUamxS0WDsc50baooUfeGW1fVU1LpR1rZKuvrqq4Gyr3JLSUnh0KFDXHbZZURHR7tb5KpCmzZtAPjjH/9IkyZNSpWHh4cTHx+PzWZzf1k7HA6GDBnCypUree2113A4HAwdOtSj88IXLrvsMmw2G8ePHy+zpTU+Ph6A5s2blyrbuXNnqWVF52dsbKx72ZVXXkl2dnaZ7/fWrVvL/X92Pjt27Ch3/zVr1nQv++KLL0hKSuKLL77g888/Z8OGDWzZskXBSbyi8CRShf7xj3+QlpbGwIEDefzxxxk8eDB2u501a9bwt7/9zT2mqSIxMTEA5XZjFYWRGjVqFFu/ZDdhkYq6w8ori4uL48svv6RRo0Z8++23rF+/noyMDJxOp/vy/fDw8FLPKysMFLXCZWZmlipzOp3A2QHM51N0vCdPniyz/Pjx41x22WXExMRUaXiKi4sD4OGHH65wvaioqGJf2L///js//fQTN9xwAzt27Djv1AlVoajuV1xxBVdccUW560VFRZVaVtb7V/TeWq1W97KYmBhSUlLK3G55711lVHRuWSz/6VRZsmQJDz/8MI899hgDBgzg4YcfxuVysXHjRl599VWfjh+TS4fCk0gVmzNnDnPmzMFms3HDDTdw11130aNHD5o1a8att95arCujLEVfUrVr1y6zPCEhAfjPl0nR+kWtBiWVtx042+VVlvvuu49GjRoxfvx4/v73vxcrGzJkSLEuQ38pOt6i4y+pTp06QNlf8r5UtP0///nP7N27t9LPe/zxx7nhhhs4ffo0bdq04cEHH+STTz6pqmqWqeg1XLp0KY899liV7CMzM7Pcc7G8987XVq5cycqVK4mOjub666/njjvuoF+/fsycOZObbrrJJxcNyKVFY55E/CQtLY0VK1YwePBgNmzYQIsWLWjWrBmAO0Cd+xd7kaysLA4ePEjTpk2pV69eqfIOHToA/+kGy8rK4tChQzRt2rTML63ypjaoSNFl9StWrChV1rZtW4+35wu7du0CKDadQ5EGDRrQpEkTDh48WKWtTgA//PADcHbahcpq2bIlo0aNYv/+/XTu3Jnff/+dl156qcyxXk6ns8zzwhf27dtHRkYGrVq1IiSkav6W3rNnD9HR0bRs2bJUWXnnYlUdc3Z2Nl9//TWjRo1i7ty51KlThz/+8Y8+349c/BSeRKpQWV/sISEh7u6SokGsZ86cweVyUb9+/TK3M2/ePMLCwnj++eeLLb/yyivp06cP6enpxaY4+OyzzwgPD+e5554rVZ9bbrnF4+MoujLthhtuKLb8rrvuonPnzh5vzxdWrFhBeno6ffv2LRU6XnjhBUJDQ/1yi485c+aQmZnJqFGjygw/ERER7nFRcHa+rvfffx+Av/zlL5w4cYInn3ySkJAQpkyZUqr788yZM+WeF95yOp1Mnz6dxMREXnrppTIDVIsWLcptOaqMhQsXAjBy5Mhic0Bddtll9O7du8zn+PKY27ZtW6wbr0hRC2x+fr5P9iOXFnXbiVShjz76iKysLLZt28aRI0cIDQ2lU6dOtGjRgiVLlrivzsrJyeHHH3+kXbt2TJ48mV9//RXTNJk/fz7Jycl88MEHdO7cmd69e9O8eXM2bNhA7dq16dGjByEhITz11FPFWlimTJlCt27dGDhwIC1atODbb7+lfv363HnnnaxcuZLbbrvtvN2F51qwYAF/+ctfGDNmDB06dODIkSNcddVVdOzYkaVLl5KUlOTz1+58srKyGDlyJFOmTGHJkiV88cUXnDp1ik6dOtG6dWu2bdvGBx98UOX1OH36NEOGDGHq1KmsWrWKr7/+mv379xMWFkZiYiLt2rXju+++cw+MfvXVV2nevHmx8Tbbtm1j4sSJjBw5khdffNE9rQGcndqgR48efPTRR+zatQuXy8XKlSvLHIxfUp06dcq9X+Lp06cZM2YMEyZM4JprrmHQoEF07tyZLVu2kJqaSv369bniiito2bIl3bt3L3cM3fnMmTOHXr16ceutt7Jy5UrWrl1LXFwcPXv2ZP369WWei94cc0ljxoyhbt26fPvttxw5cgTTNLn++utp06YN33//Pd9+++0FHZdc2hSeRKrQm2++yc0338y1117LrbfeSm5uLgcPHmT06NHMnj272LpPPfUUf/vb3+jcuTN33303FouFb7/9luTkZPLz8+nbty9DhgyhR48ePProo+Tm5rJp0ybeffddtm7dWmxb2dnZ3HPPPTz//PPcfvvttGrVil9++YUhQ4bQpEkTbrvtNo/GAh09epTevXvz17/+lY4dOxISEsLOnTu5//77adCgQUDCE5wdDFzUcnPHHXcQGRnJkSNHmDhxIlOmTPFbq8KaNWu4/fbbeeKJJ+jUqROdOnUiJyeHo0ePMmfOHHfrS7du3bj//vtZt24dU6dOLbaNyZMn06lTJ/es3atXrwbg5ZdfBuDGG2+kS5cuWK1Wjh49WqkgERsbW+7UGocPH2bMmDEUFBTQv39/+vXrR+/evenWrRthYWGkpqbyyy+/MGPGjAsKLUVcLhcDBgzgueeeo2fPngwaNIjff/+dV199lTNnznDbbbeVGvztzTGX9N5773HHHXdwzTXXcPPNN2O32zly5AivvfYan3zyiUd/RIgUMRo0aFD2CFERuShNnjyZXr16cdNNN7F///5AV0cuYSNHjuSpp56if//+fPXVV4GujkilacyTyEWq6Gqzc7Vr146ePXuyf/9+BSfxm7LOxT/84Q888sgjnDlzpsy5ukSqM3XbiVykZsyYQV5eHrt37yYnJ4fLL7+cm2++GafTyYsvvhjo6sklZOzYsSQmJvLDDz+Qnp5O06ZNufXWWwkNDeXZZ58tc/ZvkepM3XYiF6lBgwZx991306RJE2rUqEFGRgZbt27lvffec19eL+IPd999NwMGDOAPf/iDe9LS7du3M3Xq1ApvVC1SXSk8iYiIiHhAY55EREREPKDwJCIiIuIBDRivpAULFhAaGsrp06cDXRURERGppFq1amG32+nVq5fPtqnwVEmhoaFVdn8pEU8ZQKLTCcBhqxUNXBQRKVtVfHcrPFVSUYvTQw89FNiKiACRLhf7jx0DIKlePXLLuHeXiIjAxx9/7PNt6hNXRERExAMKTyIiIiIeULedSBByGgZzIyPdj0VExH8UnkSCUIFh8IzNFuhqiIhcktRtJyIiIuIBtTyJBCPTJNI8O0FBrmGAuu5ERPxGLU8iQSjSNNl/7Bj7jx1zhygREfEPhScRERERDyg8iYiIiHhA4UlERETEAwpPIiIiIh5QeBIRERHxgMKTiIiIiAc0z5NIEHIZBksiItyPRUTEfxSexC+cTifmOfMRGYaB1WoNYI2CW75h8HitWoGuhojIJUnddlLlnE4n+XYnZkiE+yff7sTpdAa6aiIiIh5Ty5NUOdM0CY2I4tbejxNrq01GWiqr50/FdOQFumoiIiIeU3gSv4m11caWUD/Q1bgoRLpc7D92DIDm9eqRa1EjsoiIv+gTV0RERMQDCk8iIiIiHlB4EhEREfGAwpOIeG1eaiprTpwIdDWCTvv8fJJTUmifn+9eNjEtjc3HjwewViJyPhowLnKRqu108kJGBp3z84l2udgfGsp7NWqwJDKyzPV75OYyKCuLKx0O7MC+kBDGx8by7/BwAOo6nTyQnc2KyEh2h4b6tK6bjx8n8ZypK1ItFg6EhPC/0dEsL6e+weTB7GxyDYO5UVGBrorbtQUF9M3J4Y92O1fa7YQCDRs0KHNdT8+lIrNTU/lTQQH/iorixbi4UuX3ZWfzRHY2iQ4HR61WPoqO5l81avjg6ESqlsKTyEWohsvF56mp1Ha5+Cg6mhNWK91zc5malsYQ0+TzEl/iwzMyeCYri6UREcyNiiIEuMJup945gaau08mzWVkcCQnxeXgC2BUSwtTCL866Tif9c3L4KC2N0S4XM6Kjfb4/fxqYnc1pi6VUeNocFsZ/1a9PQQDq9Oe8PPrl5LAnNJRDViuXlTPvmqfnUpE7cnP5b7u93P33z85mXHo6SyMi+N/oaNoWFPBaRgaRpsn7MTE+OUaRqqLwJBKEXIbBmsIWobJuz9I/J4dmTid94+PdLUfTo6JYnJrKyxkZLI2MxF74vDYFBTyTlcWrsbF8GMC/+o9ZrSw854t4flQU/z5xgkezsrwOT+GmSQFgVrNb2ZiGQf75V6sS06OjeT8mhjzD4LUzZ7gsJ6fM9Tw5l4qEmyYvZ2Twfo0ajMjMLLXNCNNkVGYmq8PDeaxwpvxPo6MxgKezspgVHU26pt+Qakxnp0gQyjcMBsbHMzA+nvwyAkHb/HxSLRb3lx2c/aJeHBlJXZeLdueMsRmUlcUJi4Vp0dFgmkS5XKW21z4/n2WpqQBMPHOG5JQUklNS6FviC/cPdjvzUlPZf/Qo3x07xuAyvjgr66TVyr6QkGLdefWcTiakpfHjsWP8mpLC2hMnuLdEHYrGEfXIzWVkRgbfHTvG/qNHiSm8PdAfCwqYfuoUu48eZd/Ro6w6cYL/l5VVbBuX2e387+nT7Dp6lAMpKXx58iRd8opP6to3J4fklBSuy8/nlfR0dhw7xr6jR5l2+jS1zqnz5uPHucLhoENBgft1m1f4WpY15qkshmkyKCuLtSdOcCAlhR+PHWPcmTPULPFexbhcXGa3E1PGe1hSqtVKXiXCpCfnUpHBWVlYgH+UE8Y75OdTy+XikxKh+JPoaKJNk855mkBXqje1PIlchMKgzC/GomWt7HbWF95YuGNBAd+FhfH/srN5KiuLWi4Xxy0WJsfE8HHhl9u+kBDeiolhRGYmM6Oi2BIWBsB3hf8C1HS5mHX6NMsiIlgcGUlSbi4vZmbyc2goXxXuyxMhpkkDp5O0whaI2k4ni0+exDQMPo6O5pTFwi35+bxz5gwxLhfTSnxRP52ZiR2YWqMGYYUtT53y8vjk9GlOFI6vOWm10tzh4Na8PD4qfP7ldjufp6ZyzGplSo0a5FgsdM/N5Z+nT/OozVZqDNZrGRmkGwbvxMSQ6HAwKDub14HBhS0qr8TG8lp6OtkWC5ML93HSw1aVcenp9M3JYU5UFP+MjibR6eTh7Gxa2u3cVbs2jsL39Y68PCaeOcMzcXE+G1/lybkE0MDh4MmsLIbHxZUbzq4u7M7bXqL7d0doKM7C8oU+qb1I1VB4ErkIHQgJoVN+Pg0dDpJD/vPf/IaCs6Nr6hW2TNR0uYh3ubi+oIAb8/N5JyaGFKuVvjk5vJ6ejgOYGR1NqtXK2vBwRmRm8n1YWLHutSL1XS6GxcWxoLBsdlQUW44fp19OTqXCUwhgK2yxqedy8WRWFnUKx9kAjMrMxALcmpDgDlQzoqOZkpbG8MxMZkZHF/uyDjdNutWp415mMU3GpadzwmrltoQEMs4NMOfctPrV9HSSrVaSEhIoKHzuJ1FRfJ6ayl8zMkqFpzTDoF98PBTtB3gkO5sYl4tMi4UVkZGMzMzktMVS5ut2Ptfn5/NATg5D4uKKjS/aGBbGp6dPc2dubrnjjnyhsudSkVcyMtgVEsIXFQwmr+N04gBOlbg5uN0wSLNYqFuJljORQFK3nUgQinS52FfY7RRZxhfN7KgonMA/0tK4rqCAJg4HT2Zm0jU3Fzg75gQguvDfWi4XI+LimFqjBosjIxlYqxZ7Q0J4yoNutyzDYME5X5h2w+DHsDAaV/IG0Dfn57Pr+HF2HT/O6pMnuTM3l/mRkbwRGwumSbfcXFZHRIBpYnM63T/rwsOpaZpcXVB82PW8qKhiYepqu50mTifToqOLBydwB584l4sbCwpYEhlJtMv1n/24XHwdEcF/OZ3FBtEDzIqOdj8fYEtYGCFAIx/d+PrOvDzSDYNvwsOLHfeO0FCyDIMO5xz33KgoGjZo4NOr+ip7LsHZ7rhueXm8UrNmhduMgFLjpIrkG0axbYpUR2p5EglSURV8wewJDeVJm40309NZVDi+5rjFwt9q1uTN9HRyCr+4cgvXLwCWnNM6ZBoGX0RGMiIzkwYOBykh5/+oOGq1FgsRAOmGwZWVbEXYFhrK+NhYTCDXMNgXEuIOOfFOJ3GmSf+cHPqXM7C5don9HC7RqtG0MMzsreBKwaYOBxZgZGYmI8sJjvEuF8fO2XZyif0UDXQuOR7pQjVzOKhpmuwsZ+6nksfta5U9l6ymyavp6SyIjGT7Od25ZckDQss5f8NNs1JjsUQCSeFJ5CK1NDKSlRERXGW3YwV2hoa6Byb/WhiGzlgs5AIZFkupq/ZOFYaAONMkpRL7K6+dpbJfg6ctFtafMyj5XEXtRAsiI5lXTqvKTyUC3oV8ARft54PoaNaV09V4sERY8va4K1OnkxYLQ222MstP+eGqtMqcS71zc7nM4WB0XByNHI5iz69hmjRyOEi1WMizWDhhtRLC2VB8btddqGliKxxzJ1KdKTxJQLicTswSH7CGYWAt8cUk3rEbRrFWgE6FX3hFIcU0DH4KDaW13U6oaRbrSqlb2FJT9OUcyI6UUxYLmYaBBcoNWOdTFHpa2O3lbuP3wnUchnHB+ymLN6/d71YrnfLz2RoWFtAWmfOdSw0dDsLA3Tp1rj65ufTJzeURm63YJKut7XbWnvN/vnVhOKuKecREfKlaxftXX32VzZs3k5ycTMuWLd3LmzVrxqJFi1i/fj1Lly7l8ssv97pMAic3OxO7vQBLeDRmSIT7J9/uxOmjcSJSWjOHgwE5OawKD3e3FgB8ERlJCNDnnO6wcNPk7txc9oaEcLzwy62oeyY2AIN5XYbBlxERdMvNpUUZEy/WqsR5szM0lN+tVgZlZ5c+hsIupFNWKxvDwuifnU2dMrZZmf2UJdcwLrgbb3Hh+/N0Gd2IVtMsdiyeTFXgjbLOpUWRkTxis5X6AVgTHs4jNhs/FIavf4eHk2YYDMzOLrbdgdnZ5BgGay7g6kwRf6pWLU9Lly7lgw8+4LPPPiu2fNy4ccyaNYu5c+eSlJTExIkTSUpK8qpMAqcgP4+wiCi69H6UWFsCABlpqayePxXTofldfOWrEydYEhFBstVKY6eTgdnZnLFYGF3iNhkzo6PpV3h13X85HCRbrfTKzaWR08lDhZfbA/weEsIZw2BATg5ZhkGOxcIPoaEcrsR4KF94IzaWDgUFLElN5dOoKH4JCSHO5eIau52O+flcXb9+hc83DYPna9bk49OnWXnyJHOiojhhsdDc4eByh4MH4uMBeKFmTT5LTWXNyZPMiorikNVKgsvFfxcUUN/ppEudOh7XfUdoKANzcngqM5PfQkI4VWLepIpsDg9nRlQUQ7OyuMpu55vwcOyGwX85HCTl5vJKzZosLRyo78lUBQ0dDnoXDvpuXRhIiy4QOGK1uq+ahMqdSwdCQzlQVotRWhqHrFZWnHMxQZ5h8FZsLG+kpzP19Gm+Dg+nbUEBvXJzeTMmhjPqtpNqrlqFpy1btpRaFh8fT6tWrejXrx9wNmC99tprNG3alMzMzAsqO3jwoN+OScoXE1cbW0LFX3hy4X4KDeXe3FxqO52ctlhYHBnJ2zExpS4PzzMM+sbH82JGBvfl5BBpmvwUGsrAWrWKjftxGAZP22w8n5HBm+nphALPxMX5LTylWq0k1a7NM5mZ3JGXx8DCOaB+CQk5e0VeJayLiKBPfDzDs7J4vHAix9+t1rNXzBXaFxpKt4QEhmdm0jcnB5vLxSmLhV2hoUy8wNuGTIyJoZHTyeCsLGJMk41hYZUOTwCj4+LYERpK/5wcRmdm4uDsgPiFUVFsPc/g7PI0djpLDYov+n1jWFix8FTZc8kTn0RHYwcez86mS14eKVYrr8TGnp2sVaSaq1bhqSwNGjTgxIkTxbpzkpOTadiwIRkZGRdUpvAkwc40DDYWfmmWd8uRIeUMMC7LKauVZyqx/qqICFaV0aXSp3btMtevzDYB2tWtW6n1TlmtvBgXx4sVrLMpPLzcG9wCfBcezv3nCS6HQkJ4+jx1nxsVVWbrTln7T7VaebCwZet865b3mn0aHc2n5wkW5dWpLOd7nc7lyblUUkX7qMwxiVRH1T48iUhpeYZRbmAREZGqVe3DU0pKCnXq1MFqtbpbkRo2bEhycjKZmZkXVCYiIiJyoar9qLxTp06xc+dOevXqBUBSUhJHjx7l4MGDF1wmIiIicqGqVcvTuHHj6Ny5MwkJCcyaNYusrCw6duzI6NGjmThxIkOHDiUzM5Phw4e7n3OhZSLBLNLlYsuJEwC0rVOHXF2dJCLiN9UqPI0aNarM5QcOHKBHjx4+LRMJdvG6eaqISEDoz1URERERDyg8iYiIiHhA4UlERETEAwpPIiIiIh5QeBIRERHxQLW62k5EKsc0DH4svAlrebdnERGRqqHwJBKE8gyDpISEQFdDROSSpG47EREREQ8oPImIiIh4QOFJJAhFuFxsPn6czcePE6GZxkVE/EpjnkSCkAEkOp3uxyIi4j9qeRIRERHxgMKTiIiIiAcUnkREREQ8oPAkIiIi4gGFJxEREREP6Go7kSBkAntDQtyPRUTEfxSeRIJQnsXCn+vUCXQ1REQuSeq2ExEREfGAwpOIiIiIBxSeRIJQhMvF2hMnWHvihG7PIiLiZxrzJBKEDKCFw+F+LCIi/qOWJxEREREPKDyJiIiIeEDhSURERMQDCk8iIiIiHtCAcak2XE4nZuEg6CKGYWC1WgNUIxERkdIUnqRayM3OxG4vICw8GtP4z/VjBXk5hIMCVAkmcLjwNdHtWURE/EvhSaqFgvw8wiKi6NL7UWJtCQBkpKWyev5UTEdegGtX/eRZLLSrWzfQ1RARuSQpPEm1EhNXG1tC/UBXQ0REpFwaMC4iIiLiAYUnkSAUYZosPXmSpSdPEmFq1JOIiD+p204kCBmmybV2u/sxhm7SIiLiL2p5EhEREfGAwpOIiIiIBxSeRERERDyg8CQiIiLiAYUnEREREQ/oajuRIHXKor99REQCQeFJJAjlWiy0qlcv0NUQEbkk6U9XEREREQ8oPImIiIh4QOFJJAhFmCbzUlOZl5qq27OIiPiZxjyJBCHDNOlQUOB+rNuziIj4j1qeRERERDygliep1lxOJ6bD4f7dMAysVmsAayQiIpc6hSeptnKzM7HbCwgLj8Ys7JYqyMshHBSgREQkYBSepNoqyM8jLCKKLr0fJdaWQEZaKqvnT8V05AW6aiIicglTeJJqLyauNraE+oGuhoiICKDwJBK0cnSFnYhIQCg8iQShXIuFP9RXa5yISCBoqgIRERERDyg8iYiIiHhA4UkkCIWbJtNPnWL6qVOE6/YsIiJ+pTFPIkHIYpp0zs93P9btWURE/EctTyIiIiIeUHgSERER8UDQhKc///nPLF++nJUrV7JmzRr69OkDQHx8PDNnzmTDhg2sWbOGtm3bup9TUZmIiIjIhQiaMU+TJ0+mT58+7Nmzh0aNGrFu3Tq+/PJLXnjhBbZt20b//v1p3bo1H330Ee3atcPhcFRYJiIiInIhgqblCSA2NhaAmJgY0tLSKCgooHv37syYMQOA7du3c+zYMdq3bw9QYZmIiIjIhQialqfBgwczbdo0cnJyqFmzJo8++ig1atQgJCSEkydPutc7cuQIDRo0wGazlVsmIiIicqGCIjxZrVaGDRvGoEGD2LJlC61bt+Zf//oXt912W6CrJhIQuRYLDfWHgIhIQARFt13Lli2pV68eW7ZsAc52wR09epQrr7wSp9NJQkKCe91GjRqRkpJCWlpauWUiIiIiFyoowlNKSgp16tShefPmADRt2pQmTZpw4MABlixZwoABAwBo3bo19erVY9OmTQAVlomIiIhciKDotktNTWXkyJH84x//wDRNDMPgxRdfJCUlhddff53JkyezYcMGCgoKGDZsmPtquorKRIJZuGkyOS0NgGE2G/maYVxExG+CIjwBLFq0iEWLFpVanpqayv3331/mcyoqEwlmFtPkzrw8AJ7W7VlERPwqKLrtRERERKoLhScRERERDyg8iYiIiHhA4UlERETEAwpPIiIiIh5QeBIRERHxQNBMVSAi/5FrGDSvV8/9WERE/EfhSSQYGYZCk4hIgKjbTkRERMQDCk8iQSjMNJmYlsbEtDTCTDPQ1RERuaQoPIkEIatp0jc3l765uVgVnkRE/ErhSURERMQDCk8iIiIiHlB4EhEREfGAwpOIiIiIBxSeRERERDyg8CQiIiLiAc0wLhKEcg2Da+rWdT8WERH/UXiSoOJyOjEdjmLLDMPAarUGqEYBYhicvtSOWUSkmlB4kqCRm52J3V5AWHg05jmtLQV5OYTDpRegREQkIBSeJGgU5OcRFhFFl96PEmtLACAjLZXV86diOvICXDv/CjNNXklPB+B/atakQF13IiJ+o/AkPud0OjHPuWWIw+HAavXdLURi4mpjS6jvs+0FI6tp8lBODgCvxcaCwpOIiN8oPIlPOZ1O8u1OQiOizllqxzQpFqhERESClcKT+JRpmoRGRHFr78eJtdUGIPm3vWxevSDANRMREfENhSepErG2/3StpZ8+GeDaiIiI+I4myRQRERHxgMKTiIiIiAcUnkREREQ8oDFPIkEozzBoW6eO+7GIiPiPwpNIEDINgyMh+u8rIhII6rYTERER8YD+dBUJQqGmyaiMDADGxcZiV9ediIjfqOVJJAiFmCaDs7MZnJ1NiGZuFxHxK4UnEREREQ8oPImIiIh4QOFJRERExANehac6hfPMyMXJ6XTicDiK/TidzkBXS0REJKC8Ck9bt27l008/pVevXkRGRvqqTlINOJ1O8u1OzJCIYj/5dqcClIiIXNK8mqrg7bff5q677mLSpEmMHTuW5cuXs3DhQtatW4epK4CCmmmahEZEcWvvx4m11QYgIy2V1fOnYjryAlw7ERGRwPEqPL377ru8++67tGzZknvuuYeePXty9913k5qayqJFi1i4cCE7duzwVV0lAGJttbEl1A90NaSEPMPgloQE92MREfEfnwwY3717N2PGjOG6666jX79+rFmzhr59+7JkyRK++uorhg4dSoMGDXyxKxHh7O1ZfgkN5ZfQUEyFJxERv/L51XZbtmxh7dq1bNu2DcMwaNasGcOHD2fTpk1MnTpVg8xFREQkqPns9iwdOnTg7rvvplu3bsTExPDzzz8zZswYFi5ciNPppG/fvgwdOpTJkydz3333+Wq3IpekUNNkaGYmAO/GxOj2LCIifuRVeLrqqqu4++67ueuuu6hbty4nTpxg9uzZzJ8/n59//rnYulOnTiU/P5+XXnrJqwqLyNnbszyblQXABzVqKDyJiPiRV+FpxYoV5OXlsXz5cubPn88333xT4VV2v/zyC9u2bfNmlyIiIiIB5VV4evbZZ1myZAk5OTmVWn/jxo1s3LjRm12KlOJyOjEdjmLLDMPAarUGqEYiInIx8yo8zZ0711f1ELkgudmZ2O0FhIVHF7vqrCAvh3BQgBIREZ/zKjw98sgjdO7cmQceeKDM8hkzZrBq1SqmT5/uzW5EylWQn0dYRBRdej9KrO3svEeazFNERKqSV1MV9OvXj3379pVbvm/fvnKDlYgvxcSdnczTllDfPSO6iIhIVfAqPDVp0qTC8LR//36aNGnizS5EREREqhWvuu0KCgpIKLxFRFnq1KmDy+XyZhciUoZ8w6Bb7druxyIi4j9etTxt27aNvn37Eh0dXaosJiaGe++9V1MTiFQBl2GwPSyM7WFhuBSeRET8yquWp4kTJzJ//nxWrlzJtGnT+OWXXwBo0aIFgwYNok6dOjz55JM+qaiIiIhIdeBVePrhhx946KGHGDduHK+++qp7gkzDMDh06BAPP/ww33//vU8qKiL/EWqa/L/sbAA+io7WDOMiIn7k9b3t1q9fz4033sjVV19N06ZNATh48CA7d+70dtMiUo4Q0+SljAwAPomKUngSEfEjn9wY2DRNdu7cqcAkIiIiFz2fhKc//OEPNGnShJo1a2KU8Rfw/PnzfbEbERERkYDzKjw1adKEd999l2uvvbbM0ARnW6V8EZ7CwsJ4+eWXufnmm8nLy+Onn35i2LBhNGvWjEmTJlGrVi0yMjJ45pln3APXKyoTERERuRBehadx48ZxxRVX8Morr/Dtt99y5swZH1WrtBdeeAHTNOnYsSOAe36pcePGMWvWLObOnUtSUhITJ04kKSnpvGUiIiIiF8Kr8HTdddfx3nvv8a9//ctX9SlTZGQk9913H9ddd5172cmTJ4mPj6dVq1b069cPgKVLl/Laa6/RtGlTMjMzyy07ePBgldZXRERELl5ehae0tDQyCq/4qUpNmzblzJkzDB06lE6dOpGXl8c777xDeno6J06cwOl0utdNTk6mYcOGZGRklFum8CQiIiIXyqsZxmfMmME999yDxeLVZs7LarWSmJjIvn376NatGy+99BIffPABISE+Ge8uEnTyDYPe8fH0jo/X7VlERPzMq/Tx66+/YrVaWbVqFXPmzCElJaVYS0+RZcuWebMbkpOTcTqdLFy4EIDdu3dz6NAhGjVqRJ06dbBare79NmzYkOTkZDIzM8stk4ufy+nEdDiKLTMMA6vVGqAa+ZbLMNgUHh7oaoiIXJK8Ck8ffPCB+/FLL71U5jqmadK4cWNvdkNaWhobNmzg5ptvZu3atSQmJtK4cWO2bt3Kzp076dWrl3tQ+NGjR93dchWVycUrNzsTu72AsPBozHNaZQrycgiHiyZAiYhIYHgVnvr06eOrepzX6NGjmTBhgvuqu1GjRnHs2DFGjx7NxIkTGTp0KJmZmQwfPrzYc8ork4tXQX4eYRFRdOn9KLG2s1dlZqSlsnr+VExHXoBr5xshpskDOTkAzIqKwqGuOxERv/EqPG3evNlX9TivQ4cOlRnWDhw4QI8ePcp8TkVlcvGLiauNLaF+oKtRJUJNkzfS0wGYGxmp8CQi4kc+GXEdFhbGNddcQ3x8PFu3biUtLc0XmxURERGpdry+TO6RRx5h27ZtfPbZZ0ybNo2rrroKAJvNxs6dO7n33nu9rqSIiIhIdeFVeOrbty//8z//w9dff81zzz1X7BYtaWlp/Pvf/6Znz55eV1JERESkuvAqPD3++OOsWLGCJ598klWrVpUq37FjB5dffrk3uxARERGpVrwKT02bNuWrr74qt/zMmTPYbDZvdiEiIiJSrXgVnjIyMqhVq1a55ZdffjknT570ZhciIiIi1YpX4Wnt2rU88MADxMbGliq7/PLLuf/++1m5cqU3uxCRMhQYBgNr1WJgrVoUaJoCERG/8mqqgvHjx7NkyRLWrl3LqlWrME2TPn36cO+999KtWzdOnDjBxIkTfVVXESnkNAzWREQEuhoiIpckr1qejh8/TteuXfnqq6/o3r07hmHQq1cvunTpwqJFi+jevbvmfBIREZGLiteTZJ46dYoRI0YwYsQIatWqhcVi4dSpU5im6Yv6iUgZQkyTe3JzAVioGcZFRPzKJzOMFzl9+rQvNyci5Qg1TSaeOQPA4ogIhScRET/yKjw9/fTTlVpv0qRJ3uxGREREpNrwKjw9++yz5ZaZpolhGJimqfAkIiIiFw2vwlNiYmKpZYZh0KhRIx566CHatm3LgAEDvNmFiIiISLXi9Y2BSzJNk8OHDzNmzBh+++03xowZ4+tdiIiIiASMz8PTubZs2cKf//znqtyF+JnL6cThcLh/nE5noKskIiLiVz692q6kVq1a4XK5qnIX4ke52ZnY7QWEhUdjFl7dVZCXQzhgtVoDWzkRERE/8So89e7du8zlsbGxtGvXjjvuuINPP/3Um11INVKQn0dYRBRdej9KrC2BjLRUVs+fiunIC3TVLjkFhsHjhTfd1u1ZRET8y6vwVNGtV06fPs2UKVN0e5aLUExcbWwJ9QNdjUua0zBYEhkZ6GqIiFySvApP7dq1K7XMNE3S09PJzs72ZtMiIiIi1ZJX4Sk5OdlX9RARD1hNkzvyznaXLouIwKmuOxERv6nSAeNy8XM5nZgOh/t3h8OB1ar7Gla1MNNkauFNt5vXq0euwpOIiN94FZ4OHz7s8Q2ATdOkSZMm3uxWqomyrr4DO6aJbgwtIiIXLa8HjHft2pXLL7+cdevWceDAAQCaN2/On/70J/bu3cvy5ct9UlGpfkpefQeQ/NteNq9eEOCaiYiIVB2vwtPx48eJj4+nc+fO7uBUpHnz5sydO5fjx49ruoKL3LlX36WfPhng2pSvZBejYRian0pERDzm1QzjgwcP5uOPPy4VnAD279/Pxx9/zF/+8hdvdiHiE0VdjJbwaMyQCMyQCPLtTs2QLiIiHvOq5alevXrY7fZyy+12O/Xq1fNmFyI+oQk+RUTEV7xqedq7dy8PPvhgmQGpfv36PPjgg/z888/e7ELEp4q6GGNttQNdFRERCVJetTz97W9/49NPP2X9+vUsW7aMgwcPAtCsWTO6du2KYRgMHTrUF/UUkXPYDYNn4uLcj0VExH+8Ck9bt27lzjvvZMSIEdxxxx1EREQAkJeXx9dff82ECRPU8iRSBRyGwdyoqEBXQ0TkkuT1JJl79+5l0KBBGIZBfHw8AKdOndI8PyIiInJR8tkM46Zpkp+fT3Z2toKTSBWzmiY35+cD8HV4uG7PIiLiR14NGAdo1aoVM2fOZP/+/ezatYv27dsDYLPZ+Oc//+n+XUR8J8w0mX76NNNPnyZMf6yIiPiVV+Hpuuuu47PPPqNZs2YsWLAAi+U/m0tLSyMmJob+/ft7XUkRERGR6sKr8DRq1Cj279/PLbfcwptvvlmqfOPGjfzxj3/0ZhciIiIi1YpX4enaa69lzpw5FBQUlFl+7Ngx6tSp480uRERERKoVr8KT3W4v1lVXUr169cjOzvZmFyIiIiLVilfhadu2bSQlJZVZFhkZyb333svmzZu92YWIiIhIteJVeJowYQKtWrVi+vTp3HLLLQBcddVV9OvXj+XLlxMfH8+kSZN8UU8RERGRasGreZ5++OEHBg4cyNixY/n73/8OwMsvvwzA77//zoABA9izZ4/3tRSRYuyGwQs1a7ofi4iI/3gVnmrUqMF3333Hn/70J1q2bEmzZs2wWCwcPHiQHTt2+KqOIlKCwzD4JDo60NUQEbkkXXB4CgsLY/fu3bz55pt88MEH7N69m927d/uybiIiIiLVzgWHp4KCAk6ePFnuNAUiUnUspknbwv97W8LCcKnrTkTEb7waMD537lx69+5NaGior+ojIpUQbprMP3WK+adOEa7bs4iI+JVXY55+/vlnbr/9dr766ivmzp3L4cOHycvLK7XesmXLvNmNSJVwOZ2YDkexZYZhYLVaA1QjEREJBl6FpylTprgfjxgxosx1TNOkcePG3uxGxOdyszOx2wsIC4/GPKfLqyAvh3BQgBIRkXJ5HJ5Gjx7NokWL2LNnD3369KmKOolUuYL8PMIioujS+1FibQkAZKSlsnr+VExH6dZTERGRIh6HpyFDhvDzzz+zZ88eNm/ejM1mY/v27fTr149///vfVVFHkSoTE1cbW0L9QFdDRESCiFcDxosYutJHRERELhE+CU8iIiIilwqvBoyLSGA4DIMxsbHuxyIi4j8XFJ4SExO5+uqrAYgt/ABv1qwZ6enpZa6/a9euC6yeiJTFbhj8o0aNQFdDROSSdEHhacSIEaWmJnjjjTdKrWcYhqYqEBERkYuKx+Fp+PDhVVEPEfGAxTS5xm4HYGdoqG7PIiLiRx6Hp3nz5lVFPUTEA+GmyZepqQA0r1ePXIUnERG/0dV2IiIiIh5QeBIRERHxgMKTiIiIiAeCLjz17duX5ORkbr/9dgDi4+OZOXMmGzZsYM2aNbRt29a9bkVlIiIiIhciqMJTo0aNeOCBB/j+++/dy1544QW2bdtGx44dGT58OFOmTCEkJOS8ZSIiIiIXImjCk2EYvP3227z44ovk5+e7l3fv3p0ZM2YAsH37do4dO0b79u3PWyYiIiJyIYKmGeaxxx5j69at7Ny5073MZrMREhLCyZMn3cuOHDlCgwYNKiwTCXYOw2BC4Qzjuj2LiIh/BUV4atGiBUlJSdxzzz2BropItWA3DN4pvDWSiIj4V1CEp7Zt29KoUSM2bNgAQEJCAuPHj2fChAk4nU4SEhLcLUyNGjUiJSWFtLS0csukNKfTiWma7t8dDgdWq1nBM0RERC5NQTHmafr06bRp04Z27drRrl07tm3bxsiRI5k+fTpLlixhwIABALRu3Zp69eqxadMmgArL5D+cTif5didmSIT7h5BwTJNigUqqD8M0udxu53K7HUPvkYiIXwVFy1NFXn/9dSZPnsyGDRsoKChg2LBhOByO85bJf5imSWhEFLf2fpxYW20Akn/by+bVCwJcMylPhGnyVWGLqm7PIiLiX0EZnvr06eN+nJqayv3331/mehWVSWmxttrYEuoDkH765HnWFhERuTQFRbediIiISHWh8CQiIiLiAYUnEREREQ8E5ZgnkaricjoxS1xUYBgGVqs1QDUSEZHqRuFJpFBudiZ2ewFh4dGY51y9VpCXQzgoQImICKDwJOJWkJ9HWEQUXXo/SqwtAYCMtFRWz5+K6cgLcO2KcxgGH0RHux+LiIj/KDyJlBAT958pG6oru2HwWs2aga6GiMglSQPGRURERDyglieRIGSYJg2dTgCSrdZiY7RERKRqKTyJBKEI02TLiROAbs8iIuJv6rYTERER8YDCk4iIiIgHFJ5EREREPKAxTyLnUXLWcc04LiJyaVN4EqlAWbOOa8ZxEZFLm8KTSAVKzjpeXWccFxER/1F4EqmE6jbruNMw+Dgqyv1YRET8R+FJJAgVGAZ/jYsLdDVERC5JutpORERExANqeRIJRqZJLZcLgNMWC6jrTkTEbxSeRIJQpGmy8/hxQLdnERHxN3XbiYiIiHhA4UlERETEAwpPIiIiIh5QeBIRERHxgMKTiIiIiAcUnkREREQ8oKkKRDzkcjoxHY5iywzD8OuNgp2GwdzISPdjERHxH4UnEQ/kZmditxcQFh6NeU5oKcjLIRz8FqAKDINnbDa/7EtERIpTeBLxQEF+HmERUXTp/SixtgQAMtJSWT1/KqYjL8C1ExERf1B4ErkAMXG1sSXUD1wFTJNI0wQ4O7u4uu5ERPxGA8ZFglCkabL/2DH2HzvmDlEiIuIfCk8iIiIiHlB4EhEREfGAwpOIiIiIBxSeRERERDyg8CQiIiLiAYUnEREREQ9onieRIOQyDJZERLgfi4iI/yg8iQShfMPg8Vq1Al0NEZFLkrrtRERERDyg8CQiIiLiAYUnkSAU6XKRnJJCckoKkS5XoKsjInJJUXgSERER8YDCk4iIiIgHdLWdiA+4nE5Mh6PYMsMwsFqtAaqRiIhUFYUnES/lZmditxcQFh6Nec6cSwV5OYSDApSIyEVG4UnESwX5eYRFRNGl96PE2hIAyEhLZfX8qZiOvADXTkREfE3hScRHYuJqY0uoH+hqiIhIFVN4EglCLsNgTXi4+7GIiPiPwpNIEMo3DAbGxwe6GiIilyRNVSAiIiLiAYUnEREREQ8oPIkEoUiXi31Hj7Lv6FHdnkVExM805kkkSEWZZqCrICJySVLLk4iIiIgHFJ5EREREPBAU4Sk8PJyPPvqI9evXs2rVKmbPnk3Tpk0BiI+PZ+bMmWzYsIE1a9bQtm1b9/MqKhMRERG5EEERngBmzZpFp06d6NKlCytWrOCtt94C4IUXXmDbtm107NiR4cOHM2XKFEJCQs5bJiIiInIhgiI85efns3btWvfv27ZtIzExEYDu3bszY8YMALZv386xY8do3779ectERERELkRQNsMMGjSIFStWYLPZCAkJ4eTJk+6yI0eO0KBBgwrLRIKdaRhsDAtzPxYREf8JuvA0dOhQmjZtysiRI4mMjAx0dUQCIs8w6FO7dqCrISJySQqKbrsijz/+OHfccQf9+/cnLy+PtLQ0nE4nCQkJ7nUaNWpESkpKhWUi/uByOnE4HO4fp9MZ6CqJiIgPBE14euyxx7jrrrvo168fGRkZ7uVLlixhwIABALRu3Zp69eqxadOm85aJVKXc7Ezs9gIs4dGYIRGYIRHk250KUCIiF4Gg6LarX78+r7zyCgcPHmTevHnA2UHk3bt35/XXX2fy5Mls2LCBgoIChg0bhsPhAKiwTKQqFeTnERYRRZfejxJrSyAjLZXV86diOvJ8sv1Il4stJ04A0LZOHXItQfN3kIhI0AuK8HT06FEaNmxYZllqair333+/x2Ui/hATVxtbQn3gbDeeWSK8G4aB1Wq9oG3H6552IiIBERThSSTYFXXjhYVHF7s6riAvh3C44AAlIiL+p/Ak4gclu/EAn3fliYiIfyg8ifjRud14IiISnDTKVERERMQDankSCSBfDyIXEZGqp/AkEiDeDCI3DYMfQ0Pdj0VExH8UnkQCxJtB5HmGQdI5s+eLiIj/KDyJBJgGkYuIBBcNGBcRERHxgMKTSBCKcLnYfPw4m48fJ0IzjYuI+JW67USCkAEkFt5kWMPFRUT8Sy1PIiIiIh5QeBIRERHxgLrtRKoZTZwpIlK9KTyJVCPeTJwpIiL+ofAkUo14M3GmiIj4h8KTSDV0vokzTWBvSIj7sYiI+I/Ck0gQyrNY+HOdOoGuhojIJUlX24mIiIh4QOFJRERExAMKTyJBKMLlYu2JE6w9cUK3ZxER8TONeRIJQgbQonAuKN2eRUTEv9TyJCIiIuIBhScRERERD6jbTiQIlLxli0Y5iYgEjsKTSDVX1i1b8nOzA1wrEZFLl8KTSDVX8pYtGWmpbJz7QaCrJSJyyVJ4EgkS596yxQQOWSwYhqHbs4iI+JnCk0gQyrVYuL52bUJC9F9YRMTfdLWdiIiIiAcUnkREREQ8oPAkEoQiTBfLT59m6cmTRJga9SQi4k8aMCESjJxO/lg475PL4cBhGBiGgdVqDXDFREQufmp5EgkyZ+d9srt/N63hmCER5NudOJ3OANZMROTSoPAkEmQK8vMIC490/97j4ZHc2vtxQiOiMNWFJyJS5dRtJxLk4mrXoyAsPNDVEBG5ZCg8iVwkSt7/DtA4KBGRKqDwJHIRKOv+dwAFeTmEgwKUiIgPKTyJBKnMqBoYlrPDFkve/w4gIy2V1fOnYjryAllNEZGLjsKTSBDKsVh47rm3qVWnQbHl597/TkREqoauthMRERHxgFqeRC5iJQeRawC5iIj3FJ5EglCE6WL4JxMIDQ3nX395pcx1yhpErgHkIiLeU3gSCUIWE1r8vg8Ao5yJMUsOItcAchER31B4ErnIaRC5iIhvacC4iIiIiAcUnkREREQ8oG47kUuIbuEiIuI9hSeRS0R5t3DJy8ki1OooFqAUqEREyqfwJBKk8kPDMM4JQedT1i1cTiT/zpqFH2KEReieeCIilaTwJBKEciwWhj0/udTtWSrj3Kvv0k+f1D3xREQ8pPAkIprOQETEA7raTkRERMQDankSCULhpsmTn75HaHgEswaN9vn2S16VZ5pmqfFVGlQuIpcqhSeRIGQ1Ta7ZvwsAw+Xy6bZLXpXncjopKMgjIjIKNKhcREThSUSKK3lVXvJve9m8egG3lhhUvnLO+9jtuZjn3FuvMq1RTqez2HPgwlu2ytqWWsREpKopPIlImYoGkaefPlnsd6j8nFElQ5HL5cLuNAmLjP7Psgts2XI6neTbnYRGRBVbrhYxEalqCk8i4rHKzBlVVihyOeyEhoRwa+/H3M+rbMtWyRYl0zQJjYji1t6PE2ur7X5eZaZZ8GWLlVq/RC49F314atasGZMmTaJWrVpkZGTwzDPP8MsvvwS6WiIXhYrmjCorFBUtK/m8ktsqq2WrZKuWw+HAajWJtVU8zULJcFNW6xdcWIuVWr9ELk0XfXgaN24cs2bNYu7cuSQlJTFx4kSSkpICXa2AKvllUvQlJOILFXX3FS07n5ItW2XPhG7HNCnV6nPulYJldhOW0fpVVktXZcZhldf6dSHbgtL/N9WCJVI9GQ0aNLhovzXj4+P597//TcuWLXE6nQD88MMP3H333Rw8eNCjbX3xxRdYrVZOnqzch7+nSn4BVDnjnCm+TBMMiIyOxWI5+0HtdNjJy80mMjqm3GWVWac6PK861MHXdS/IzSKx8Jw+FRePw+GodnWvivcrIjIaw2Jxr1OQn1f8eU4HudkZhUHFcJ/bEZE1PHse5jmB5z+hxzRdlLohjmHx3bYKt3fedUQuQp7cbsoTCQkJOJ1OevTo4bNtXtQtTw0aNODEiRPu4ASQnJxMw4YNPQ5Pdrvdx7UrrqpOmvKdE9YKd52XnVFsDaMSyyqzTnV4XnWogy/rDnCoqEUi80y1rbuv36/83KzzPs/i/r9kus9tj5/Huf8ni7cElWb6cFuVXUdEKsvpdPr8O/yiDk++1KtXr0BXQURERKqBi/r2LCkpKdSpU6fYmIGGDRuSnJwcwFqJiIhIMLuow9OpU6fYuXOnu9UoKSmJo0ePetxlJyIiIlLkoh4wDnDZZZcxceJEbDYbmZmZDB8+nJ9//jnQ1RIREZEgddGHJxERERFfuqi77URERER8TeFJRERExAMKTyIiIiIeUHgSERER8YDCk4iIiIgHFJ5EREREPKDwJCIiIuIBhScRERERDyg8BVizZs1YtGgR69evZ+nSpVx++eWBrtIF+/Of/8zy5ctZuXIla9asoU+fPgDEx8czc+ZMNmzYwJo1a2jbtq37ORWVVSevvvoqmzdvJjk5mZYtW7qXh4WF8dprr7FhwwZWr17N5MmT3WUVvbfV9X0PDw/no48+Yv369axatYrZs2fTtGnTYuvceOONHDp0iEGDBrmXRUREMGXKFDZs2MD69etJSkqqVFmgffrpp6xatYqVK1eycOFCWrZsed7XIJjP5759+5KcnMztt98OXPixVPfjhNLHeu2117J48WJWrFjB119/zeDBg93rBuv5u3nzZr755htWrlzJypUr6dGjB3DxfS5BxccEpd9vqNpzOMS7wxFvjRs3jlmzZjF37lySkpKYOHFitfrP6YnJkyfTp08f9uzZQ6NGjVi3bh1ffvklL7zwAtu2baN///60bt2ajz76iHbt2uFwOCosq06WLl3KBx98wGeffVZs+QsvvIBpmnTs2BGAhIQEd1lF7211ft9nzZrF2rVrAXjooYd466233EE4JiaG559/3l1e5IknnqCgoICOHTuSmJjIkiVL2LhxI2lpaRWWBdoTTzxBRkYGAF27dmXSpEnceeedFb4GwXo+N2rUiAceeIDvv//evexCj6U6HyeUfazjx4/nrbfeYtWqVcTFxbFu3TpWr17Nvn37gvb8BRg8eDC7d+8utuxi/Fyq6JjKer+LnlNV57BangIoPj6eVq1asWDBAuDsF3SDBg1K/aUfTGJjY4GzX7JpaWkUFBTQvXt3ZsyYAcD27ds5duwY7du3B6iwrDrZsmULR48eLbYsMjKS++67j3HjxrmXnTx5Eqj4va3O73t+fn6xYLRt2zYSExPdv7/++uv8/e9/L/XF0aNHD/f7ePjwYTZt2kTXrl3PWxZoRcEJzp67pmme9zUIxvPZMAzefvttXnzxRfLz893LL/RYqutxQvnHapomNWvWBCAqKgq73c6ZM2eA4D1/y3Ixfi5VdEzlvd9QteewWp4CqEGDBpw4cQKn0+lelpycTMOGDTl48GDgKnaBBg8ezLRp08jJyaFmzZo8+uij1KhRg5CQEPeJDnDkyBEaNGiAzWYrtywYNG3alDNnzjB06FA6depEXl4e77zzDhs2bKjwvc3IyAia933QoEGsWLECgKSkJFwuF6tWraJbt27F1mvYsCFHjhxx/3748GEaNmx43rLq4O9//zsdOnQAYMCAAaXKz30NKjpnq/P5/Nhjj7F161Z27tzpXnahx1KdjxPKPlaA4cOH889//pORI0dSq1YtRo8e7T6GYD5/J02ahGEY/Pjjj7zxxhvUrVv3ovtcquiztrz3u6rPYYUn8Qmr1cqwYcMYNGgQW7ZsoXXr1vzrX//itttuC3TVqozVaiUxMZF9+/YxduxYWrZsyf/93/9xyy23BLpqPjF06FCaNm3KyJEjSUhI4KmnnqJ3796BrpbPPfXUUwD06dOHF154gYEDB7rLzn0NglWLFi1ISkrinnvuCXRVqlxFxzpkyBDGjh3L559/TuPGjVmwYAHbt29n3759Aaipb9xzzz2kpKQQEhLCyJEjmTRpEuPHj7/oPpfK+6x99NFHA3Zuq9sugFJSUqhTpw5Wq9W9rGHDhiQnJwewVhemZcuW1KtXjy1btgBnm0GPHj3KlVdeidPpLNU/nZKSQlpaWrllwSA5ORmn08nChQsB2L17N4cOHeLKK6+s8L0Nhvf98ccf54477qB///7k5eXRqlUr6tSpw8qVK9m8eTNJSUk8/fTTjBo1Cjj7WjRq1Mj9/MTERPfxVFRWncybN48OHTpgs9mA0q8BUOE5W13P57Zt29KoUSM2bNjA5s2badOmDePHj6d79+4XdCzV9Tih/GN9+umn6dq1K59//jkAhw4dYtu2bVx//fVA8J6/Ra+5w+Fg2rRptG3b9qL8XKromMp6vwcOHFjl57DCUwCdOnWKnTt30qtXL+Bst8jRo0cD3kR6IYr+4zVv3hw428zapEkTDhw4wJIlS9zdIa1bt6ZevXps2rQJoMKy6i4tLY0NGzZw8803A2c/VBs3bsy+ffsqfG+r+/v+2GOPcdddd9GvXz/3mKA1a9Zw7bXX0q5dO9q1a8fSpUuZNGmSewzCue9jYmIi7du3Z/ny5ectC6TY2Fjq1q3r/v3222/nzJkzpKWllfkaFAm283n69Om0adPG/d5t27aNkSNHMn369As+lup4nFD+sU6ePJmcnBxuvPFG4GyXzh//+Ef27t0LBOf5GxkZ6R5jCnDXXXexa9eui/JzqbxjWrZsWbnnNlTtOWw0aNDA9OExiocuu+wyJk6ciM1mIzMzk+HDh/Pzzz8HuloXpGfPngwdOhTTNDEMg/fee4/PP/+c2rVrM3nyZBo3bkxBQQEvvvgiGzduBKiwrDoZN24cnTt3JiEhgbS0NLKysujYsSONGzdmwoQJ2Gw2TNNk4sSJfPnll0DF7211fd/r16/Pd999x8GDB8nOzgbODiLv3r17sfUmTpzI7t27mTZtGnD2g/ydd96hVatWuFwuxo8fz+LFi89bFkgNGzZk6tSpREREYJomp06dYsyYMZw+fbrC1yDYz+d58+Yxbdo0VqxYccHHEgzHCcWPtVOnTrzwwguEhIQQEhLC7Nmz+d///V8gOM/fxo0b8+GHH2KxWDAMg0OHDvHyyy9z5MiRi+5zCajwmIqc+35D1Z7DCk8iIiIiHlC3nYiIiIgHFJ5EREREPKDwJCIiIuIBhScRERERDyg8iYiIiHhA4UlERETEAwpPIiIiIh5QeBIRqebmzZvHmjVrAl0NESmk8CQiPpGcnFypn/bt2/tkf3Xr1mX48OG0bNmyUuv37duX5ORkWrVq5ZP9+5qnxyMigRMS6AqIyMVh6NChxX7v3bs3N910U6nlvrqLfd26dXn22Wc5cuQIu3fv9sk2A+liOx6Ri5nCk4j4RNEdz4u0adOGm266qdRyEZFgp247EfEbwzAYNGgQa9eu5cCBA/z444+MGzeOmjVrutd59tlnOXz4MB07diz23HHjxvHbb79x1VVX0b59e5YtWwacvUlxUZdg3759va5jvXr1mDBhAj/++CO//vora9eu5d577y22Tvv27UlOTqZ79+4MGzaM7777jgMHDjBnzhyaNm1aapsPPvggGzduZP/+/SxZsoQbbriBefPmMW/ePPf2KnM8f/jDH5g3bx779+/nu+++Y/DgwV4fr4h4Ti1PIuI348aNo2/fvsyZM4d//vOfJCYm8vDDD9OyZUvuuusuHA4Hf//73+nSpQtvv/02nTt3Jjs7m5tuuon+/fszfvx4fvrpJ2rXrs1bb73FiBEjmDlzJlu2bAHgu+++86p+tWvXZvHixZimyccff8ypU6e45ZZbeOedd4iJiWHatGnF1h8yZAgul4t//OMfxMTE8Je//IV3332X7t27u9cZOHAgb7zxBps3b+bDDz8kMTGRf/7zn5w5c4ajR48CZ7syz3c8NWvWZNasWSxbtozFixeTlJTEiy++yM8//8xXX33l1XGLiGcUnkTEL66//noeeOABhgwZwueff+5evnHjRj799FPuvPNOPv/8cxwOB0899RTLli3jlVde4bXXXnO3BL333nsApKamsnbtWkaMGMH333/vs67BUaNGYbFYuPXWW0lLSwNgxowZTJkyheHDhzNz5kzy8vLc64eHh3Pbbbdht9sBSE9PZ8yYMbRo0YK9e/cSGhrKiBEj+OGHH+jbty9OpxOAPXv2MGnSJHd4qszx1K9fn2HDhrFgwQIAZs+ezZYtW+jXr5/Ck4ifqdtORPzizjvvJD09nW+++Qabzeb+2bFjB1lZWXTo0MG97t69e5kwYQIPPPAAs2bNwmaz8fTTT7vDR1Xp1q0bq1evBihWx3Xr1lGzZk2uvvrqYuvPmTPHHZwAd4tR48aNAWjdujW1atXi008/LVb3hQsXusNZZWVlZbmDE4DdbufHH39070tE/EctTyLiF82aNaNmzZrs3LmzzPLatWsX+/2DDz6gZ8+etGnThrFjx/rsKr3yxMfHExcXR//+/enfv3+l6piSklLs9/T0dADi4uIAaNiwIQAHDx4stp7T6eTIkSMe1a+olark/q688kqPtiMi3lN4EhG/sFgsnDx5stTUBUVOnTpV7PcmTZrQrFkzAK644gq/1A9gwYIF7oHcJf3000/Ffi+vJcwwDN9Wzs/7EpGKKTyJiF/8/vvvdOrUia1btxYbN1QWwzCYOHEimZmZTJs2jWHDhrF06VL3FWkApmn6tH6nTp0iMzMTi8XC+vXrfbLN5ORkAJo2bcrGjRvdy61WK40aNWLPnj3uZb4+HhGpOhrzJCJ+sXjxYkJCQnj66adLlVmtVmJjY92/P/bYY1x//fWMGjWK8ePHs3XrVsaOHYvNZnOvk5OTA1Dsed5wuVx8+eWXdOvWjRYtWpQqr1Wrlsfb3L59O6dPn+b+++/HarW6l99zzz3FjgV8fzwiUnXU8iQifrF582ZmzJjB0KFDueqqq/jmm2+w2+3813/9F0lJSbzyyissXbqU5s2bM2LECObMmcOqVasAeOaZZ1i5ciVjx47liSeeAM62ZJ05c4YBAwaQlZVFTk4OP/zwA4cPH66wHvfddx+33HJLqeXTpk3jjTfeoEOHDixZsoRPP/2UX375hbi4OK655ho6duxYasD4+djtdiZMmMDrr7/O3LlzWbx4MYmJifTt25fffvut2LoXejwi4n8KTyLiN6NHj2bHjh3079+f0aNH43A4OHz4MAsXLmTr1q1YLBYmTZpEWloar7zyivt5v/32G2PHjmXMmDEsXbqUxYsX43A4ePrpp3n++ed58803CQ0N5Zlnnjlv2HjwwQfLXD537lyOHj1KUlISzzzzDHfccQcDBw4kLS2NX375hTfeeOOCjvnjjz/GMAwef/xxXnrpJX766ScefvhhXn311WLdlxd6PCLif0aDBg3U0S4i4keGYbBz506+/PJLRo4cGejqiIiHNOZJRKQKhYeHl1rWp08fbDYbmzZtCkCNRMRb6rYTEalCbdq04W9/+xtLliwhLS2Na665hvvuu489e/awZMmSQFdPRC6AwpOISBU6fPgwKSkpPPLII8TFxXHmzBnmz5/PG2+8UWx2chEJHhrzJCIiIuIBjXkSERER8YDCk4iIiIgHFJ5EREREPKDwJCIiIuIBhScRERERDyg8iYiIiHhA4UlERETEAwpPIiIiIh74/xlXYDFxPekOAAAAAElFTkSuQmCC",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "#this just measures space-delimited words which isn't necessarily the same as toke count, but it should be close-ish\n",
    "plot_text_length_histogram(train_df, percentile = 96)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0851b1bc-71dd-4c6b-94ff-2193c98c4e4b",
   "metadata": {},
   "source": [
    "## Load source model/tokenizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "f3d05cc5-123a-4e12-82cd-57671df3a049",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ffba170e6b284389a0a3f9d2bb14ea88",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Loading checkpoint shards:   0%|          | 0/2 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 6.53 s, sys: 30 s, total: 36.5 s\n",
      "Wall time: 5min 23s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "import torch\n",
    "from transformers import AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig\n",
    "\n",
    "# Quantization config\n",
    "bnb_config = BitsAndBytesConfig(\n",
    "    load_in_4bit=True,\n",
    "    bnb_4bit_quant_type=\"nf4\",\n",
    "    bnb_4bit_compute_dtype=\"float16\",\n",
    ")\n",
    "\n",
    "# loading the model with quantization config\n",
    "model = AutoModelForCausalLM.from_pretrained(\n",
    "    source_hf_model_name,\n",
    "    quantization_config=bnb_config,\n",
    "    trust_remote_code=True,\n",
    "    device_map= \"auto\",\n",
    "    #token = hf_token,\n",
    ")\n",
    "\n",
    "model.config.use_cache = True  #surpress the download and use cache. # apparently needed because of https://github.com/huggingface/transformers/pull/24906\n",
    "model.config.pretraining_tp = 1  #disable tensor parallelism\n",
    "\n",
    "#fixme\n",
    "tokenizer = AutoTokenizer.from_pretrained(source_hf_model_name,\n",
    "     trust_remote_code=True, \n",
    "     return_token_type_ids=False,\n",
    "     token = hf_token,\n",
    " )\n",
    "tokenizer.pad_token = tokenizer.eos_token\n",
    "tokenizer.padding_side = \"right\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "d4c0253b-48f1-4f6f-824b-762c857d22a1",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">LlamaForCausalLM</span><span style=\"font-weight: bold\">(</span>\n",
       "  <span style=\"font-weight: bold\">(</span>model<span style=\"font-weight: bold\">)</span>: <span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">LlamaModel</span><span style=\"font-weight: bold\">(</span>\n",
       "    <span style=\"font-weight: bold\">(</span>embed_tokens<span style=\"font-weight: bold\">)</span>: <span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">Embedding</span><span style=\"font-weight: bold\">(</span><span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">32000</span>, <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">4096</span><span style=\"font-weight: bold\">)</span>\n",
       "    <span style=\"font-weight: bold\">(</span>layers<span style=\"font-weight: bold\">)</span>: <span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">ModuleList</span><span style=\"font-weight: bold\">(</span>\n",
       "      <span style=\"font-weight: bold\">(</span><span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0</span>-<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">31</span><span style=\"font-weight: bold\">)</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">32</span> x <span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">LlamaDecoderLayer</span><span style=\"font-weight: bold\">(</span>\n",
       "        <span style=\"font-weight: bold\">(</span>self_attn<span style=\"font-weight: bold\">)</span>: <span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">LlamaAttention</span><span style=\"font-weight: bold\">(</span>\n",
       "          <span style=\"font-weight: bold\">(</span>q_proj<span style=\"font-weight: bold\">)</span>: <span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">Linear4bit</span><span style=\"font-weight: bold\">(</span><span style=\"color: #808000; text-decoration-color: #808000\">in_features</span>=<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">4096</span>, <span style=\"color: #808000; text-decoration-color: #808000\">out_features</span>=<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">4096</span>, <span style=\"color: #808000; text-decoration-color: #808000\">bias</span>=<span style=\"color: #ff0000; text-decoration-color: #ff0000; font-style: italic\">False</span><span style=\"font-weight: bold\">)</span>\n",
       "          <span style=\"font-weight: bold\">(</span>k_proj<span style=\"font-weight: bold\">)</span>: <span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">Linear4bit</span><span style=\"font-weight: bold\">(</span><span style=\"color: #808000; text-decoration-color: #808000\">in_features</span>=<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">4096</span>, <span style=\"color: #808000; text-decoration-color: #808000\">out_features</span>=<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">4096</span>, <span style=\"color: #808000; text-decoration-color: #808000\">bias</span>=<span style=\"color: #ff0000; text-decoration-color: #ff0000; font-style: italic\">False</span><span style=\"font-weight: bold\">)</span>\n",
       "          <span style=\"font-weight: bold\">(</span>v_proj<span style=\"font-weight: bold\">)</span>: <span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">Linear4bit</span><span style=\"font-weight: bold\">(</span><span style=\"color: #808000; text-decoration-color: #808000\">in_features</span>=<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">4096</span>, <span style=\"color: #808000; text-decoration-color: #808000\">out_features</span>=<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">4096</span>, <span style=\"color: #808000; text-decoration-color: #808000\">bias</span>=<span style=\"color: #ff0000; text-decoration-color: #ff0000; font-style: italic\">False</span><span style=\"font-weight: bold\">)</span>\n",
       "          <span style=\"font-weight: bold\">(</span>o_proj<span style=\"font-weight: bold\">)</span>: <span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">Linear4bit</span><span style=\"font-weight: bold\">(</span><span style=\"color: #808000; text-decoration-color: #808000\">in_features</span>=<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">4096</span>, <span style=\"color: #808000; text-decoration-color: #808000\">out_features</span>=<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">4096</span>, <span style=\"color: #808000; text-decoration-color: #808000\">bias</span>=<span style=\"color: #ff0000; text-decoration-color: #ff0000; font-style: italic\">False</span><span style=\"font-weight: bold\">)</span>\n",
       "          <span style=\"font-weight: bold\">(</span>rotary_emb<span style=\"font-weight: bold\">)</span>: <span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">LlamaRotaryEmbedding</span><span style=\"font-weight: bold\">()</span>\n",
       "        <span style=\"font-weight: bold\">)</span>\n",
       "        <span style=\"font-weight: bold\">(</span>mlp<span style=\"font-weight: bold\">)</span>: <span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">LlamaMLP</span><span style=\"font-weight: bold\">(</span>\n",
       "          <span style=\"font-weight: bold\">(</span>gate_proj<span style=\"font-weight: bold\">)</span>: <span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">Linear4bit</span><span style=\"font-weight: bold\">(</span><span style=\"color: #808000; text-decoration-color: #808000\">in_features</span>=<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">4096</span>, <span style=\"color: #808000; text-decoration-color: #808000\">out_features</span>=<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">11008</span>, <span style=\"color: #808000; text-decoration-color: #808000\">bias</span>=<span style=\"color: #ff0000; text-decoration-color: #ff0000; font-style: italic\">False</span><span style=\"font-weight: bold\">)</span>\n",
       "          <span style=\"font-weight: bold\">(</span>up_proj<span style=\"font-weight: bold\">)</span>: <span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">Linear4bit</span><span style=\"font-weight: bold\">(</span><span style=\"color: #808000; text-decoration-color: #808000\">in_features</span>=<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">4096</span>, <span style=\"color: #808000; text-decoration-color: #808000\">out_features</span>=<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">11008</span>, <span style=\"color: #808000; text-decoration-color: #808000\">bias</span>=<span style=\"color: #ff0000; text-decoration-color: #ff0000; font-style: italic\">False</span><span style=\"font-weight: bold\">)</span>\n",
       "          <span style=\"font-weight: bold\">(</span>down_proj<span style=\"font-weight: bold\">)</span>: <span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">Linear4bit</span><span style=\"font-weight: bold\">(</span><span style=\"color: #808000; text-decoration-color: #808000\">in_features</span>=<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">11008</span>, <span style=\"color: #808000; text-decoration-color: #808000\">out_features</span>=<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">4096</span>, <span style=\"color: #808000; text-decoration-color: #808000\">bias</span>=<span style=\"color: #ff0000; text-decoration-color: #ff0000; font-style: italic\">False</span><span style=\"font-weight: bold\">)</span>\n",
       "          <span style=\"font-weight: bold\">(</span>act_fn<span style=\"font-weight: bold\">)</span>: <span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">SiLUActivation</span><span style=\"font-weight: bold\">()</span>\n",
       "        <span style=\"font-weight: bold\">)</span>\n",
       "        <span style=\"font-weight: bold\">(</span>input_layernorm<span style=\"font-weight: bold\">)</span>: <span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">LlamaRMSNorm</span><span style=\"font-weight: bold\">()</span>\n",
       "        <span style=\"font-weight: bold\">(</span>post_attention_layernorm<span style=\"font-weight: bold\">)</span>: <span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">LlamaRMSNorm</span><span style=\"font-weight: bold\">()</span>\n",
       "      <span style=\"font-weight: bold\">)</span>\n",
       "    <span style=\"font-weight: bold\">)</span>\n",
       "    <span style=\"font-weight: bold\">(</span>norm<span style=\"font-weight: bold\">)</span>: <span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">LlamaRMSNorm</span><span style=\"font-weight: bold\">()</span>\n",
       "  <span style=\"font-weight: bold\">)</span>\n",
       "  <span style=\"font-weight: bold\">(</span>lm_head<span style=\"font-weight: bold\">)</span>: <span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">Linear</span><span style=\"font-weight: bold\">(</span><span style=\"color: #808000; text-decoration-color: #808000\">in_features</span>=<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">4096</span>, <span style=\"color: #808000; text-decoration-color: #808000\">out_features</span>=<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">32000</span>, <span style=\"color: #808000; text-decoration-color: #808000\">bias</span>=<span style=\"color: #ff0000; text-decoration-color: #ff0000; font-style: italic\">False</span><span style=\"font-weight: bold\">)</span>\n",
       "<span style=\"font-weight: bold\">)</span>\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\u001b[1;35mLlamaForCausalLM\u001b[0m\u001b[1m(\u001b[0m\n",
       "  \u001b[1m(\u001b[0mmodel\u001b[1m)\u001b[0m: \u001b[1;35mLlamaModel\u001b[0m\u001b[1m(\u001b[0m\n",
       "    \u001b[1m(\u001b[0membed_tokens\u001b[1m)\u001b[0m: \u001b[1;35mEmbedding\u001b[0m\u001b[1m(\u001b[0m\u001b[1;36m32000\u001b[0m, \u001b[1;36m4096\u001b[0m\u001b[1m)\u001b[0m\n",
       "    \u001b[1m(\u001b[0mlayers\u001b[1m)\u001b[0m: \u001b[1;35mModuleList\u001b[0m\u001b[1m(\u001b[0m\n",
       "      \u001b[1m(\u001b[0m\u001b[1;36m0\u001b[0m-\u001b[1;36m31\u001b[0m\u001b[1m)\u001b[0m: \u001b[1;36m32\u001b[0m x \u001b[1;35mLlamaDecoderLayer\u001b[0m\u001b[1m(\u001b[0m\n",
       "        \u001b[1m(\u001b[0mself_attn\u001b[1m)\u001b[0m: \u001b[1;35mLlamaAttention\u001b[0m\u001b[1m(\u001b[0m\n",
       "          \u001b[1m(\u001b[0mq_proj\u001b[1m)\u001b[0m: \u001b[1;35mLinear4bit\u001b[0m\u001b[1m(\u001b[0m\u001b[33min_features\u001b[0m=\u001b[1;36m4096\u001b[0m, \u001b[33mout_features\u001b[0m=\u001b[1;36m4096\u001b[0m, \u001b[33mbias\u001b[0m=\u001b[3;91mFalse\u001b[0m\u001b[1m)\u001b[0m\n",
       "          \u001b[1m(\u001b[0mk_proj\u001b[1m)\u001b[0m: \u001b[1;35mLinear4bit\u001b[0m\u001b[1m(\u001b[0m\u001b[33min_features\u001b[0m=\u001b[1;36m4096\u001b[0m, \u001b[33mout_features\u001b[0m=\u001b[1;36m4096\u001b[0m, \u001b[33mbias\u001b[0m=\u001b[3;91mFalse\u001b[0m\u001b[1m)\u001b[0m\n",
       "          \u001b[1m(\u001b[0mv_proj\u001b[1m)\u001b[0m: \u001b[1;35mLinear4bit\u001b[0m\u001b[1m(\u001b[0m\u001b[33min_features\u001b[0m=\u001b[1;36m4096\u001b[0m, \u001b[33mout_features\u001b[0m=\u001b[1;36m4096\u001b[0m, \u001b[33mbias\u001b[0m=\u001b[3;91mFalse\u001b[0m\u001b[1m)\u001b[0m\n",
       "          \u001b[1m(\u001b[0mo_proj\u001b[1m)\u001b[0m: \u001b[1;35mLinear4bit\u001b[0m\u001b[1m(\u001b[0m\u001b[33min_features\u001b[0m=\u001b[1;36m4096\u001b[0m, \u001b[33mout_features\u001b[0m=\u001b[1;36m4096\u001b[0m, \u001b[33mbias\u001b[0m=\u001b[3;91mFalse\u001b[0m\u001b[1m)\u001b[0m\n",
       "          \u001b[1m(\u001b[0mrotary_emb\u001b[1m)\u001b[0m: \u001b[1;35mLlamaRotaryEmbedding\u001b[0m\u001b[1m(\u001b[0m\u001b[1m)\u001b[0m\n",
       "        \u001b[1m)\u001b[0m\n",
       "        \u001b[1m(\u001b[0mmlp\u001b[1m)\u001b[0m: \u001b[1;35mLlamaMLP\u001b[0m\u001b[1m(\u001b[0m\n",
       "          \u001b[1m(\u001b[0mgate_proj\u001b[1m)\u001b[0m: \u001b[1;35mLinear4bit\u001b[0m\u001b[1m(\u001b[0m\u001b[33min_features\u001b[0m=\u001b[1;36m4096\u001b[0m, \u001b[33mout_features\u001b[0m=\u001b[1;36m11008\u001b[0m, \u001b[33mbias\u001b[0m=\u001b[3;91mFalse\u001b[0m\u001b[1m)\u001b[0m\n",
       "          \u001b[1m(\u001b[0mup_proj\u001b[1m)\u001b[0m: \u001b[1;35mLinear4bit\u001b[0m\u001b[1m(\u001b[0m\u001b[33min_features\u001b[0m=\u001b[1;36m4096\u001b[0m, \u001b[33mout_features\u001b[0m=\u001b[1;36m11008\u001b[0m, \u001b[33mbias\u001b[0m=\u001b[3;91mFalse\u001b[0m\u001b[1m)\u001b[0m\n",
       "          \u001b[1m(\u001b[0mdown_proj\u001b[1m)\u001b[0m: \u001b[1;35mLinear4bit\u001b[0m\u001b[1m(\u001b[0m\u001b[33min_features\u001b[0m=\u001b[1;36m11008\u001b[0m, \u001b[33mout_features\u001b[0m=\u001b[1;36m4096\u001b[0m, \u001b[33mbias\u001b[0m=\u001b[3;91mFalse\u001b[0m\u001b[1m)\u001b[0m\n",
       "          \u001b[1m(\u001b[0mact_fn\u001b[1m)\u001b[0m: \u001b[1;35mSiLUActivation\u001b[0m\u001b[1m(\u001b[0m\u001b[1m)\u001b[0m\n",
       "        \u001b[1m)\u001b[0m\n",
       "        \u001b[1m(\u001b[0minput_layernorm\u001b[1m)\u001b[0m: \u001b[1;35mLlamaRMSNorm\u001b[0m\u001b[1m(\u001b[0m\u001b[1m)\u001b[0m\n",
       "        \u001b[1m(\u001b[0mpost_attention_layernorm\u001b[1m)\u001b[0m: \u001b[1;35mLlamaRMSNorm\u001b[0m\u001b[1m(\u001b[0m\u001b[1m)\u001b[0m\n",
       "      \u001b[1m)\u001b[0m\n",
       "    \u001b[1m)\u001b[0m\n",
       "    \u001b[1m(\u001b[0mnorm\u001b[1m)\u001b[0m: \u001b[1;35mLlamaRMSNorm\u001b[0m\u001b[1m(\u001b[0m\u001b[1m)\u001b[0m\n",
       "  \u001b[1m)\u001b[0m\n",
       "  \u001b[1m(\u001b[0mlm_head\u001b[1m)\u001b[0m: \u001b[1;35mLinear\u001b[0m\u001b[1m(\u001b[0m\u001b[33min_features\u001b[0m=\u001b[1;36m4096\u001b[0m, \u001b[33mout_features\u001b[0m=\u001b[1;36m32000\u001b[0m, \u001b[33mbias\u001b[0m=\u001b[3;91mFalse\u001b[0m\u001b[1m)\u001b[0m\n",
       "\u001b[1m)\u001b[0m\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from rich import print\n",
    "print(model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "40757681-3f73-4b95-9e14-9228c67b668c",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/app/venv/lib/python3.10/site-packages/trl/trainer/ppo_config.py:141: UserWarning: The `optimize_cuda_cache` arguement will be deprecated soon, please use `optimize_device_cache` instead.\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "3d511e5e247141d9a4683760f787698e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Map:   0%|          | 0/12148 [00:00<?, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6bf796a9a986455d8cf3bb88370bc39d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Map:   0%|          | 0/312 [00:00<?, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 29.2 s, sys: 1.12 s, total: 30.3 s\n",
      "Wall time: 20.8 s\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "transformers.models.llama.modeling_llama.LlamaForCausalLM"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%time\n",
    "import trl\n",
    "from peft import LoraConfig, get_peft_model\n",
    "from peft import prepare_model_for_kbit_training\n",
    "from trl import SFTTrainer\n",
    "from transformers import EarlyStoppingCallback\n",
    "import transformers\n",
    "\n",
    "\n",
    "peft_config = LoraConfig(\n",
    "    lora_alpha=lora_alpha,\n",
    "    lora_dropout=lora_dropout,\n",
    "    #target_modules=[ \"q_proj\",\"v_proj\",],  # we will only create adopters for q, v metrices of attention module\n",
    "    target_modules=[\"q_proj\", \"k_proj\", \"v_proj\", \"o_proj\",\"gate_proj\"],\n",
    "    r=lora_r,\n",
    "    bias=\"none\",\n",
    "    task_type=\"CAUSAL_LM\",\n",
    ")\n",
    "\n",
    "\n",
    "training_arguments = transformers.TrainingArguments(\n",
    "    output_dir=project_dir,\n",
    "    #load_from_checkpoint=str,\n",
    "    seed = seed,\n",
    "    per_device_train_batch_size=per_device_train_batch_size,\n",
    "    gradient_accumulation_steps=gradient_accumulation_steps,\n",
    "    optim=\"paged_adamw_8bit\",\n",
    "    learning_rate= learning_rate,\n",
    "    lr_scheduler_type=\"linear\",\n",
    "    save_strategy=\"steps\",\n",
    "    logging_steps=logging_steps,\n",
    "    num_train_epochs=epochs,\n",
    "    max_steps=max_steps,\n",
    "    fp16=True,\n",
    "    push_to_hub=False,\n",
    "    evaluation_strategy=\"steps\",  # Add this line\n",
    "    eval_steps=logging_steps,  # Add this line\n",
    "    metric_for_best_model=\"loss\",\n",
    "    greater_is_better=False,\n",
    "    load_best_model_at_end=True,\n",
    ")\n",
    "model.gradient_checkpointing_enable()\n",
    "model = prepare_model_for_kbit_training(model)\n",
    "\n",
    "trainer = SFTTrainer(\n",
    "    model=model,\n",
    "    train_dataset=train_dataset,\n",
    "    eval_dataset=test_dataset,\n",
    "    peft_config=peft_config,  # passing peft config\n",
    "    dataset_text_field=\"text\",  # mentioned the required column\n",
    "    args=training_arguments,  # training agruments\n",
    "    tokenizer=tokenizer,  # tokenizer\n",
    "    packing=False,\n",
    "    max_seq_length=max_training_sample_length,\n",
    "    callbacks=[EarlyStoppingCallback(early_stopping_patience=early_stopping_patience)],\n",
    ")\n",
    "type(model)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "91d1ced4-7675-4be0-957c-a7e232ec7af9",
   "metadata": {},
   "source": [
    "## Train lora"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "8f9bfc5b-da5b-4f93-9b5b-e9e48097f947",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "You're using a LlamaTokenizerFast tokenizer. Please note that with a fast tokenizer, using the `__call__` method is faster than using a method to encode the text followed by a call to the `pad` method to get a padded encoding.\n",
      "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...\n",
      "/app/venv/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='10' max='10' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [10/10 13:03, Epoch 0/1]\n",
       "    </div>\n",
       "    <table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       " <tr style=\"text-align: left;\">\n",
       "      <th>Step</th>\n",
       "      <th>Training Loss</th>\n",
       "      <th>Validation Loss</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>5</td>\n",
       "      <td>1.547600</td>\n",
       "      <td>1.279097</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>10</td>\n",
       "      <td>1.190100</td>\n",
       "      <td>1.256467</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table><p>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 5min 53s, sys: 8min 32s, total: 14min 25s\n",
      "Wall time: 14min 25s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "if True:\n",
    "    trainer.train()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "f12d4322-25cf-4a33-92a2-a23eab8439b1",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">done!\n",
       "</pre>\n"
      ],
      "text/plain": [
       "done!\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "print(\"done!\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1fae22eb-210a-4f8b-97f2-fa9fda59a0b7",
   "metadata": {},
   "source": [
    "## save peft model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "c40dedb5-c6af-47b8-936f-fbf5b16cec9a",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 85.5 ms, sys: 47.3 ms, total: 133 ms\n",
      "Wall time: 270 ms\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "import os\n",
    "trainer.model.save_pretrained(project_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "04d58d58-3ba8-44e8-b6fc-a51154a9af8a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'work_for_2023_11_11'"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "project_dir"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "13c0d830-d825-41f4-91b6-d7e424844dc1",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "total 94M\n",
      "  0 drwxr-xr-x 1 root root 4.0K Nov 11 19:18 ..\n",
      "  0 -rw-r--r-- 1 root root  464 Nov 11 19:18 README.md\n",
      "94M -rw-r--r-- 1 root root  94M Nov 11 19:18 adapter_model.bin\n",
      "  0 -rw-r--r-- 1 root root  492 Nov 11 19:18 adapter_config.json\n",
      "  0 drwxr-xr-x 1 root root 4.0K Nov 11 19:18 .\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n"
     ]
    }
   ],
   "source": [
    "!ls -lhatrs ./$project_path/"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "dfc7faf6-3e0c-4f5c-922d-cabd545bac5d",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "total 94M\n",
      "  0 drwxr-xr-x 1 root root 4.0K Nov 11 19:18 ..\n",
      "  0 -rw-r--r-- 1 root root  464 Nov 11 19:18 README.md\n",
      "94M -rw-r--r-- 1 root root  94M Nov 11 19:18 adapter_model.bin\n",
      "  0 -rw-r--r-- 1 root root  492 Nov 11 19:18 adapter_config.json\n",
      "  0 drwxr-xr-x 1 root root 4.0K Nov 11 19:18 .\n"
     ]
    }
   ],
   "source": [
    "!ls -latrsh ./$project_path"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "787bccd7-4a58-4cfa-a3fc-3a30d0a9b122",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---\n",
      "library_name: peft\n",
      "---\n",
      "## Training procedure\n",
      "\n",
      "\n",
      "The following `bitsandbytes` quantization config was used during training:\n",
      "- quant_method: bitsandbytes\n",
      "- load_in_8bit: False\n",
      "- load_in_4bit: True\n",
      "- llm_int8_threshold: 6.0\n",
      "- llm_int8_skip_modules: None\n",
      "- llm_int8_enable_fp32_cpu_offload: False\n",
      "- llm_int8_has_fp16_weight: False\n",
      "- bnb_4bit_quant_type: nf4\n",
      "- bnb_4bit_use_double_quant: False\n",
      "- bnb_4bit_compute_dtype: float16\n",
      "### Framework versions\n",
      "\n",
      "\n",
      "- PEFT 0.5.0\n"
     ]
    }
   ],
   "source": [
    "!cat ./$project_path/README.md\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "48cce7d8-03a6-4bd5-91b4-f9c8918726dd",
   "metadata": {},
   "source": [
    "## Load peft adapter from checkpoint on disk"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "5f030a0e-f519-42bc-b1fb-8c11b2bd59d0",
   "metadata": {},
   "outputs": [],
   "source": [
    "from peft import LoraConfig, get_peft_model\n",
    "lora_config = LoraConfig.from_pretrained(project_path)\n",
    "loaded_model = get_peft_model(model, lora_config)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "e869dacf-b6b2-4ad1-942d-b5336e4606c9",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">### Instruction: \n",
       "Write a concise summary of the below input text.Ensure that responses covers the key points of the text.Only \n",
       "provide full sentence responses.\n",
       "### Input: \n",
       "#Person1#: Mister Ewing said we should show up at the conference center at <span style=\"color: #00ff00; text-decoration-color: #00ff00; font-weight: bold\">4:00</span> o'clock, right?\n",
       "#Person2#: Yes, he especially asked us not to be late. Some of the people from our east york branch office are \n",
       "coming, and he wants to make a good impression on them. How are you getting there?\n",
       "#Person1#: I was thinking of taking my car, but I think I'm just going to take the underground, because there is \n",
       "construction on the highway. What about you?\n",
       "#Person2#: I'll be taking the underground as well. Why don't we go together? I've been to the conference center \n",
       "only once, and I'm not sure if I can find my way around there.\n",
       "### Response :\n",
       "\n",
       "</pre>\n"
      ],
      "text/plain": [
       "### Instruction: \n",
       "Write a concise summary of the below input text.Ensure that responses covers the key points of the text.Only \n",
       "provide full sentence responses.\n",
       "### Input: \n",
       "#Person1#: Mister Ewing said we should show up at the conference center at \u001b[1;92m4:00\u001b[0m o'clock, right?\n",
       "#Person2#: Yes, he especially asked us not to be late. Some of the people from our east york branch office are \n",
       "coming, and he wants to make a good impression on them. How are you getting there?\n",
       "#Person1#: I was thinking of taking my car, but I think I'm just going to take the underground, because there is \n",
       "construction on the highway. What about you?\n",
       "#Person2#: I'll be taking the underground as well. Why don't we go together? I've been to the conference center \n",
       "only once, and I'm not sure if I can find my way around there.\n",
       "### Response :\n",
       "\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The meeting will start at four in the afternoon according to Mr. Ewing who has requested for all employees to attend it. He also emphasized on being punctual since some members from his East York Branch Office would be attending this event. Personally, I am planning to use public transportation due to traffic\n"
     ]
    }
   ],
   "source": [
    "#text = test_df['text'][500]\n",
    "print(text)\n",
    "result = generate(text, loaded_model, max_new_tokens=64, temp=0.1)\n",
    "#del lmodel"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "cf2bb12c-2f64-44b6-94a5-c52e82ba7412",
   "metadata": {},
   "outputs": [],
   "source": [
    "#del  loaded_model\n",
    "import torch\n",
    "torch.cuda.empty_cache()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "974716d4-a8d1-4ea0-a43a-64d2a063f172",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'The meeting will start at four in the afternoon according to Mr. Ewing who has requested for all employees to attend it. He also emphasized on being punctual since some members from his East York Branch Office would be attending this event. Personally, I am planning to use public transportation due to traffic'"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "result"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1628a103-d702-4bea-b56d-f7d7f29b74a7",
   "metadata": {},
   "source": [
    "## load the orignal mode"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "f68debd1-1547-4210-b56c-4bf51ce35621",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">work/work_for_2023_11_11/best_checkpoint\n",
       "</pre>\n"
      ],
      "text/plain": [
       "work/work_for_2023_11_11/best_checkpoint\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "total 94M\n",
      "  0 drwxr-xr-x 1 root root 4.0K Nov 11 19:18 ..\n",
      "  0 -rw-r--r-- 1 root root  464 Nov 11 19:18 README.md\n",
      "94M -rw-r--r-- 1 root root  94M Nov 11 19:18 adapter_model.bin\n",
      "  0 -rw-r--r-- 1 root root  492 Nov 11 19:18 adapter_config.json\n",
      "  0 drwxr-xr-x 1 root root 4.0K Nov 11 19:18 .\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{\n",
      "  \"auto_mapping\": null,\n",
      "  \"base_model_name_or_path\": \"meta-llama/Llama-2-7b-hf\",\n",
      "  \"bias\": \"none\",\n",
      "  \"fan_in_fan_out\": false,\n",
      "  \"inference_mode\": true,\n",
      "  \"init_lora_weights\": true,\n",
      "  \"layers_pattern\": null,\n",
      "  \"layers_to_transform\": null,\n",
      "  \"lora_alpha\": 32,\n",
      "  \"lora_dropout\": 0.05,\n",
      "  \"modules_to_save\": null,\n",
      "  \"peft_type\": \"LORA\",\n",
      "  \"r\": 16,\n",
      "  \"revision\": null,\n",
      "  \"target_modules\": [\n",
      "    \"q_proj\",\n",
      "    \"k_proj\",\n",
      "    \"v_proj\",\n",
      "    \"o_proj\",\n",
      "    \"gate_proj\"\n",
      "  ],\n",
      "  \"task_type\": \"CAUSAL_LM\"\n",
      "}"
     ]
    }
   ],
   "source": [
    "print(project_path)\n",
    "!ls -latrsh $project_path\n",
    "!cat $project_path/adapter_config.json"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "b3bcd0ff-edd5-42c7-8b7f-63e91fea44c4",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "cd0b4572dd2a422e8106d769c0ae6adc",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Loading checkpoint shards:   0%|          | 0/2 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 42.4 s, sys: 38.5 s, total: 1min 20s\n",
      "Wall time: 4min 21s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "from peft import AutoPeftModelForCausalLM\n",
    "from transformers import  AutoTokenizer\n",
    "\n",
    "tokenizer = AutoTokenizer.from_pretrained(\n",
    "    source_hf_model_name, \n",
    "    trust_remote_code=True, \n",
    "    return_token_type_ids=False,\n",
    "    token = hf_token,\n",
    ")\n",
    "#tokenizer = AutoTokenizer.from_pretrained(source_hf_model_name)\n",
    "\n",
    "persisted_model = AutoPeftModelForCausalLM.from_pretrained(\n",
    "    project_path, # path to the project or model directory\n",
    "    low_cpu_mem_usage=True, # reduces CPU memory usage if True\n",
    "    return_dict=True, # ensures output is returned as a dictionary\n",
    "    torch_dtype=torch.float16, # sets the data type for PyTorch tensors\n",
    "    device_map=device, # specifies the device placement for the model\n",
    "    #token=hf_token, # token for Hugging Face authentication\n",
    "\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "e5d2a7b5-7e32-443e-8eee-01a45e50c125",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">### Instruction: \n",
       "Write a concise summary of the below input text.Ensure that responses covers the key points of the text.Only \n",
       "provide full sentence responses.\n",
       "### Input: \n",
       "#Person1#: Mister Ewing said we should show up at the conference center at <span style=\"color: #00ff00; text-decoration-color: #00ff00; font-weight: bold\">4:00</span> o'clock, right?\n",
       "#Person2#: Yes, he especially asked us not to be late. Some of the people from our east york branch office are \n",
       "coming, and he wants to make a good impression on them. How are you getting there?\n",
       "#Person1#: I was thinking of taking my car, but I think I'm just going to take the underground, because there is \n",
       "construction on the highway. What about you?\n",
       "#Person2#: I'll be taking the underground as well. Why don't we go together? I've been to the conference center \n",
       "only once, and I'm not sure if I can find my way around there.\n",
       "### Response :\n",
       "\n",
       "</pre>\n"
      ],
      "text/plain": [
       "### Instruction: \n",
       "Write a concise summary of the below input text.Ensure that responses covers the key points of the text.Only \n",
       "provide full sentence responses.\n",
       "### Input: \n",
       "#Person1#: Mister Ewing said we should show up at the conference center at \u001b[1;92m4:00\u001b[0m o'clock, right?\n",
       "#Person2#: Yes, he especially asked us not to be late. Some of the people from our east york branch office are \n",
       "coming, and he wants to make a good impression on them. How are you getting there?\n",
       "#Person1#: I was thinking of taking my car, but I think I'm just going to take the underground, because there is \n",
       "construction on the highway. What about you?\n",
       "#Person2#: I'll be taking the underground as well. Why don't we go together? I've been to the conference center \n",
       "only once, and I'm not sure if I can find my way around there.\n",
       "### Response :\n",
       "\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Ms. #Person1# will drive her own car while Mr. #Person2# takes public transportation. They both want to arrive early for their meeting with Mr. Ewing who expects some guests from his East York Branch Office.\n",
      "### End\n",
      "</s>\n"
     ]
    }
   ],
   "source": [
    "#text = test_df['text'][500]\n",
    "print(text)\n",
    "result = generate(text, persisted_model, max_new_tokens=96, temp=0.1)\n",
    "#del persisted_model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "1c5eca4a-dbf3-463c-97e1-b2df66da15ad",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'Ms. #Person1# will drive her own car while Mr. #Person2# takes public transportation. They both want to arrive early for their meeting with Mr. Ewing who expects some guests from his East York Branch Office.\\n### End\\n</s>'"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "788fd3c3-140e-4f8d-8594-dfd01c8d6acd",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "c1af269a-c9c8-4a7b-898b-16ded1a5d4f5",
   "metadata": {},
   "source": [
    "## Merge the lora and original model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "251ae65f-46f8-45d6-94da-61edc884dccc",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unloading and merging model: 100%|██████████| 454/454 [00:18<00:00, 25.11it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 21.1 s, sys: 5.52 s, total: 26.7 s\n",
      "Wall time: 36.5 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "# Merges the parts of the model that were possibly loaded on different devices and unloads them from memory\n",
    "# ^^ from gpt4.  i suspect an halucination, verify.\n",
    "merged_model = persisted_model.merge_and_unload(progressbar=True)\n",
    "\n",
    "merged_model.save_pretrained(merged_path, safe_serialization=True)\n",
    "tokenizer.save_pretrained(merged_path)\n",
    "\n",
    "del persisted_model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "59d50514-acd2-48a5-abc3-ec86c29673de",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">work/work_for_2023_11_11/best_checkpoint/merged_model\n",
       "</pre>\n"
      ],
      "text/plain": [
       "work/work_for_2023_11_11/best_checkpoint/merged_model\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "total 13G\n",
      "   0 drwxr-xr-x 1 root root 4.0K Nov 11 19:23 ..\n",
      "4.0K -rw-r--r-- 1 root root  656 Nov 11 19:23 config.json\n",
      "   0 -rw-r--r-- 1 root root  183 Nov 11 19:23 generation_config.json\n",
      "9.3G -rw-r--r-- 1 root root 9.3G Nov 11 19:23 model-00001-of-00002.safetensors\n",
      "3.3G -rw-r--r-- 1 root root 3.3G Nov 11 19:23 model-00002-of-00002.safetensors\n",
      " 24K -rw-r--r-- 1 root root  24K Nov 11 19:23 model.safetensors.index.json\n",
      "4.0K -rw-r--r-- 1 root root  900 Nov 11 19:23 tokenizer_config.json\n",
      "   0 -rw-r--r-- 1 root root  414 Nov 11 19:23 special_tokens_map.json\n",
      "492K -rw-r--r-- 1 root root 489K Nov 11 19:23 tokenizer.model\n",
      "   0 drwxr-xr-x 1 root root 4.0K Nov 11 19:23 .\n",
      "1.8M -rw-r--r-- 1 root root 1.8M Nov 11 19:23 tokenizer.json\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n"
     ]
    }
   ],
   "source": [
    "print(merged_path)\n",
    "!ls -lathrs $merged_path"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "54c3c607-e405-4d95-bad2-102931f38686",
   "metadata": {},
   "source": [
    "## Load merged model from disk to be quantized"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d4e6b94e-4273-4cd2-8703-d268dfb5a001",
   "metadata": {},
   "source": [
    "### if dead again, try starting here"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1a8de00f-b0e1-4056-bd95-d74f910f1b11",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "e278c014-3120-443d-9713-405b44f4003c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "84f3017c4de24535ba66fdb7f9aa267f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Loading checkpoint shards:   0%|          | 0/2 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 22.7 s, sys: 3min 5s, total: 3min 28s\n",
      "Wall time: 1min 10s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "#qqqq\n",
    "tokenizer = AutoTokenizer.from_pretrained(merged_path)\n",
    "merged_model_from_disk = AutoModelForCausalLM.from_pretrained( merged_path, \n",
    "                                                              device_map=device,\n",
    "                                                             )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "51f0bcf5-3eb4-4858-9f27-634daeb486a6",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">### Instruction: \n",
       "Write a concise summary of the below input text.Ensure that responses covers the key points of the text.Only \n",
       "provide full sentence responses.\n",
       "### Input: \n",
       "#Person1#: Mister Ewing said we should show up at the conference center at <span style=\"color: #00ff00; text-decoration-color: #00ff00; font-weight: bold\">4:00</span> o'clock, right?\n",
       "#Person2#: Yes, he especially asked us not to be late. Some of the people from our east york branch office are \n",
       "coming, and he wants to make a good impression on them. How are you getting there?\n",
       "#Person1#: I was thinking of taking my car, but I think I'm just going to take the underground, because there is \n",
       "construction on the highway. What about you?\n",
       "#Person2#: I'll be taking the underground as well. Why don't we go together? I've been to the conference center \n",
       "only once, and I'm not sure if I can find my way around there.\n",
       "### Response :\n",
       "\n",
       "</pre>\n"
      ],
      "text/plain": [
       "### Instruction: \n",
       "Write a concise summary of the below input text.Ensure that responses covers the key points of the text.Only \n",
       "provide full sentence responses.\n",
       "### Input: \n",
       "#Person1#: Mister Ewing said we should show up at the conference center at \u001b[1;92m4:00\u001b[0m o'clock, right?\n",
       "#Person2#: Yes, he especially asked us not to be late. Some of the people from our east york branch office are \n",
       "coming, and he wants to make a good impression on them. How are you getting there?\n",
       "#Person1#: I was thinking of taking my car, but I think I'm just going to take the underground, because there is \n",
       "construction on the highway. What about you?\n",
       "#Person2#: I'll be taking the underground as well. Why don't we go together? I've been to the conference center \n",
       "only once, and I'm not sure if I can find my way around there.\n",
       "### Response :\n",
       "\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Ms. #Person1# will drive her own car while Mr. #Person2# takes public transportation. They both want to arrive early for their meeting with Mr. Ewing who expects some guests from his East York Branch Office.\n",
      "### End\n",
      "</s>\n"
     ]
    }
   ],
   "source": [
    "print(text)\n",
    "result = generate(text, merged_model_from_disk,max_new_tokens=96, temp = 0.01)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cfa68f3a-6725-4cae-8464-5d2973f3e086",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}