lora_gptq_trainer.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "78437d00-0dfd-420f-9d1d-accf9a2cde82",
   "metadata": {},
   "source": [
    "The goal of this notebook is to distill down the essential compenents of training HF models in a way that is easy to port to a data pipeline and apply to new model training scenarios."
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3ff4105b-66b9-43ed-81d8-9769e56f1df8",
   "metadata": {},
   "source": [
    "# Imports"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "5186aef3-01b0-4a1f-ad6e-9c99d46ca824",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import pandas as pd\n",
    "from rich import print\n",
    "import torch\n",
    "from transformers import AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig\n",
    "from datetime import datetime;\n",
    "pd.set_option('display.max_colwidth', None)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f4f647f4-e46c-4b7d-bb28-4e11e6d632ab",
   "metadata": {},
   "source": [
    "# Define variables"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "88eb322b-c949-4461-9d04-7e4d396d3e72",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'2023_11_10'"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "if True:\n",
    "    date_str = datetime.now().strftime(\"%Y_%m_%d\")\n",
    "date_str"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "b85d9c78-ef0c-4d91-a9df-bd9c799da50f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'work_for_2023_11_10'"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\n",
    "seed = 42 #reproducability  over training runs\n",
    "clobber_project_dir = False #wipe the project director clean\n",
    "project_dir = f\"work_for_{date_str}\" #project name, also used as wandb project name\n",
    "\n",
    "#creds\n",
    "hf_token = os.environ[\"HF_TOKEN\"]\n",
    "wandb_key = os.environ[\"WANDB_KEY\"]\n",
    "hf_account_name = os.environ[\"HF_ACCOUNT_NAME\"]\n",
    "\n",
    "#HF touchpoints\n",
    "hf_dataset_name = \"knkarthick/dialogsum\"\n",
    "source_hf_model_name = \"meta-llama/Llama-2-7b-hf\"\n",
    "\n",
    "checkpoint_dir = \"best_checkpoint\"\n",
    "quantization_dir = 'quantized_8bit'\n",
    "merged_dir = 'merged_model'\n",
    "quantized_train_dir = 'quant_trained_model_new'\n",
    "\n",
    "train_test_split_ratio = 0.025\n",
    "\n",
    "#LoRA hparams\n",
    "lora_r = 16  # rank\n",
    "lora_alpha = lora_r * 2\n",
    "lora_dropout = 0.05\n",
    "\n",
    "#training hparams\n",
    "epochs = 1\n",
    "#max_steps = 500\n",
    "per_device_train_batch_size=16\n",
    "gradient_accumulation_steps=16\n",
    "early_stopping_patience = 3\n",
    "learning_rate = 7e-4\n",
    "logging_steps = 5\n",
    "device = \"cuda\"\n",
    "max_training_sample_length = 512 #2048\n",
    "\n",
    "#quantization params\n",
    "quantization_bits = 4\n",
    "\n",
    "destination_hf_model_name = f\"{hf_account_name}/llama2-7b-dialogsum-qlora-gptq\"\n",
    "project_path = os.path.join(project_dir, checkpoint_dir)\n",
    "quant_path = os.path.join(project_dir, quantization_dir)\n",
    "quantized_train_path = os.path.join(project_dir, quantized_train_dir)\n",
    "merged_path = os.path.join(project_path, merged_dir)\n",
    "\n",
    "\n",
    "project_dir"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "35d0fc97-fbe3-4097-9751-3feafcbf218b",
   "metadata": {},
   "source": [
    "## State of working directory"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "07ffbb7b-ca53-4226-a31f-407e141bc587",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">work_for_2023_11_10\n",
       "</pre>\n"
      ],
      "text/plain": [
       "work_for_2023_11_10\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "total 0\n",
      "0 drwxr-xr-x 1 root root 4096 Nov 10 20:46 .\n",
      "0 drwxr-xr-x 1 root root 4096 Nov 10 20:54 runs\n",
      "0 drwxr-xr-x 1 root root 4096 Nov 10 22:10 best_checkpoint\n",
      "0 drwxr-xr-x 1 root root 4096 Nov 10 22:14 ..\n"
     ]
    }
   ],
   "source": [
    "print(project_dir)\n",
    "!mkdir -p $project_dir\n",
    "\n",
    "#Set clobber_project_dir to True to reset working directory\n",
    "if clobber_project_dir:\n",
    "    !rm -rf ./$project_dir\n",
    "\n",
    "!mkdir -p $project_dir\n",
    "\n",
    "!ls -latrs ./$project_dir"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a77d175a-1860-4ae4-a387-97981d2b3466",
   "metadata": {},
   "source": [
    "## Initiate wandb"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "256352b0-c7c7-45a3-9e5a-2e25fcae01cf",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[34m\u001b[1mwandb\u001b[0m: Currently logged in as: \u001b[33mjason-anderson-professional\u001b[0m. Use \u001b[1m`wandb login --relogin`\u001b[0m to force relogin\n",
      "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[33mWARNING\u001b[0m If you're specifying your api key in code, ensure this code is not shared publicly.\n",
      "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[33mWARNING\u001b[0m Consider setting the WANDB_API_KEY environment variable, or running `wandb login` from the command line.\n",
      "\u001b[34m\u001b[1mwandb\u001b[0m: Appending key for api.wandb.ai to your netrc file: /root/.netrc\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "Tracking run with wandb version 0.16.0"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "Run data is saved locally in <code>/app/jupyter/lora_gptq/wandb/run-20231110_221618-w32yr6nn</code>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "Syncing run <strong><a href='https://wandb.ai/jason-anderson-professional/work_for_2023_11_10/runs/w32yr6nn' target=\"_blank\">fallen-armadillo-15</a></strong> to <a href='https://wandb.ai/jason-anderson-professional/work_for_2023_11_10' target=\"_blank\">Weights & Biases</a> (<a href='https://wandb.me/run' target=\"_blank\">docs</a>)<br/>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       " View project at <a href='https://wandb.ai/jason-anderson-professional/work_for_2023_11_10' target=\"_blank\">https://wandb.ai/jason-anderson-professional/work_for_2023_11_10</a>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       " View run at <a href='https://wandb.ai/jason-anderson-professional/work_for_2023_11_10/runs/w32yr6nn' target=\"_blank\">https://wandb.ai/jason-anderson-professional/work_for_2023_11_10/runs/w32yr6nn</a>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import wandb\n",
    "\n",
    "wandb.login(key = wandb_key)\n",
    "run = wandb.init(project=project_dir, job_type=\"training\", anonymous=\"allow\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b98c44be-3cb1-45d4-ba26-5fa6a1e39e46",
   "metadata": {},
   "source": [
    "# Process data for training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "274adcda-c2a1-4524-b964-75b69186daa3",
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import seaborn as sns\n",
    "from matplotlib.ticker import MaxNLocator\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "def analyze_generation_params(text, model):\n",
    "    print(text,\"-\"*60)\n",
    "    results = []\n",
    "    for temp in np.linspace(.5,1.5,num = 5):\n",
    "        print(f\"temp {temp}:\",end=\"\")\n",
    "        results.append(generate(text, model, prompt=False, max_new_tokens=185,temp=temp))\n",
    "    \n",
    "    print(\"-\"*60)\n",
    "    results = []\n",
    "    for top_p in np.linspace(0,1,num = 11):\n",
    "        print(f\"top_p {top_p}:\",end=\"\")\n",
    "        results.append(generate(text, model, prompt=False, max_new_tokens=185,temp=.7, top_p=top_p))\n",
    "    \n",
    "    \n",
    "    print(\"-\"*60)\n",
    "    results = []\n",
    "    for top_k in np.linspace(0,100,num = 11,dtype=int):\n",
    "        print(f\"top_k {top_k}:\",end=\"\")\n",
    "        results.append(generate(text,  model, prompt=False, max_new_tokens=185, top_k=int(top_k)))\n",
    "    \n",
    "    print(\"-\"*60)\n",
    "    results = []\n",
    "    for do_sample in [True,False]:\n",
    "        print(f\"do_sample {do_sample}:\",end=\"\")\n",
    "        results.append(generate(text,  model, prompt=False, max_new_tokens=185, do_sample = do_sample, top_k=None))\n",
    "    \n",
    "    \n",
    "    print(\"-\"*60)\n",
    "    results = []\n",
    "    for repetition_penalty in np.linspace(1.,2.,num=11):\n",
    "        print(f\"repetition_penalty {repetition_penalty}:\",end=\"\")\n",
    "        results.append(generate(text,  model, prompt=False, max_new_tokens=48,repetition_penalty=repetition_penalty))\n",
    "    \n",
    "    print(\"-\"*60)\n",
    "    results = []\n",
    "    for typical_p in np.linspace(.1,.9,num=9):\n",
    "        print(f\"typical_p {typical_p}:\",end=\"\")\n",
    "        results.append(generate(text,  model, prompt=False, max_new_tokens=48,typical_p=typical_p))\n",
    "\n",
    "\n",
    "def generate(text, model, prompt=False, temp=0.7,top_k= 40, top_p = 0.1, do_sample = True, repetition_penalty = 1.23, typical_p = 1, guidance_scale = 1, max_new_tokens = 24,):\n",
    "    from transformers import TextStreamer\n",
    "    if prompt:\n",
    "        print(text,\"-\"*20)\n",
    "    streamer = TextStreamer(tokenizer, skip_prompt=True, skip_special_tokens=False)\n",
    "    inputs = tokenizer(text, return_tensors=\"pt\").to('cuda')\n",
    "\n",
    "    # Call the generate method of the model with the given inputs and additional generation configurations.\n",
    "    outputs = model.generate(**inputs,  # \"inputs\" likely includes input_ids or prompts for generation.\n",
    "                           streamer=streamer,  # \"streamer\" not a standard parameter in HF documentation, likely custom for model-specific streaming.\n",
    "                           max_new_tokens=max_new_tokens,  # Sets the maximum number of new tokens to generate; range varies based on model and computational limits.\n",
    "                           temperature=temp,  # Controls randomness: lower->more deterministic, higher->more random; typically in range [0.5, 1.5].\n",
    "                           do_sample=do_sample,  # Enables sampling; when True, picks tokens based on probability distribution, rather than just most likely.\n",
    "                           top_p=top_p,  # Nucleus sampling: selects top p% probability tokens for sampling; range [0, 1].\n",
    "                           top_k=top_k,  # Top-k sampling: chooses from top k probability tokens; if k=0, it's the same as using no top-k.\n",
    "                           repetition_penalty=repetition_penalty,  # Penalizes repeated tokens; >1 discourages, <1 encourages repetition; typically close to 1.\n",
    "                           typical_p=typical_p,  # Typical sampling; selects tokens whose cumulative probability is above this threshold, range [0, 1], usually close to 1.\n",
    "                           guidance_scale=guidance_scale,  # Affects the scale of guidance in models that support it, like CTRL; standard range not well-defined.\n",
    "                           #seed=seed,  # Sets a seed for reproducibility; commented out, so not in use.\n",
    "                          )\n",
    "    generated_tokens = outputs[0].tolist()[len(inputs[0]):]\n",
    "    result = tokenizer.decode(generated_tokens)\n",
    "    return result\n",
    "\n",
    "def set_plot_appearance(fontsize_title=14, \n",
    "                        fontsize_label=12, \n",
    "                        fontsize_ticks=8, \n",
    "                        theme='dark'):\n",
    "    \"\"\"\n",
    "    Configure the plot's appearance with font sizes and theme for dark backgrounds.\n",
    "    \n",
    "    Parameters:\n",
    "    fontsize_title: Font size for the title.\n",
    "    fontsize_label: Font size for the x and y labels.\n",
    "    fontsize_ticks: Font size for the x and y tick labels.\n",
    "    theme: Seaborn theme for the plot's aesthetic style.\n",
    "    \"\"\"\n",
    "    sns.set_theme(style=theme, palette='pastel')\n",
    "    \n",
    "    plt.rcParams.update({\n",
    "        'figure.facecolor': '#1a1a1a',  # Very dark grey background for the figure\n",
    "        'axes.facecolor': '#1a1a1a',    # Very dark grey background for the plots\n",
    "        'grid.color': 'gray',           # Lighter grey grid lines (less contrast)\n",
    "        'text.color': 'white',          # White text for better contrast on dark bg\n",
    "        'axes.labelcolor': 'white',     # White labels for axes\n",
    "        'xtick.color': 'white',         # White x-tick labels\n",
    "        'ytick.color': 'white',         # White y-tick labels\n",
    "        'axes.labelsize': fontsize_label,\n",
    "        'axes.titlesize': fontsize_title,\n",
    "        'xtick.labelsize': fontsize_ticks,\n",
    "        'ytick.labelsize': fontsize_ticks,\n",
    "        'legend.title_fontsize': fontsize_ticks + 2,\n",
    "        'legend.fontsize': fontsize_ticks,\n",
    "        'axes.edgecolor': 'lightgrey',  # Light grey color for the axes' spines\n",
    "    })\n",
    "# Possible modern themes: 'darkgrid', 'whitegrid', 'dark', 'white', and 'ticks'.\n",
    "# Example usage: set_plot_appearance(theme='ticks')\n",
    "\n",
    "def plot_text_length_histogram(df, bucket_size=64, percentile=95):\n",
    "    \"\"\"\n",
    "    Plot a histogram of text lengths and indicate the percentile.\n",
    "\n",
    "    Parameters:\n",
    "    df: the pandas dataframe to render from\n",
    "    bucket_size: the number of characters to bucket the histogram with\n",
    "    percentile: draws a vertical line at this percentile to visualize whether an acceptable fraction of the data lenghts are being used for training set\n",
    "    \"\"\"\n",
    "    set_plot_appearance()  # Apply appearance settings to the plot\n",
    "    text_lengths = df['text'].apply(len)  # Calculate text lengths.\n",
    "    bins = range(0, max(text_lengths) + bucket_size, bucket_size)  # Define histogram bins.\n",
    "    percentile_value = np.percentile(text_lengths, percentile)  # Calculate percentile value.\n",
    "\n",
    "    fig, ax = plt.subplots()  # Create figure and axis objects.\n",
    "    ax.hist(text_lengths, bins=bins, edgecolor='black', alpha=0.7)  # Plot histogram.\n",
    "    ax.axvline(x=percentile_value, color='red', linestyle='--')  # Draw percentile line.\n",
    "    \n",
    "    # Annotate the percentile line.\n",
    "    ax.text(percentile_value, ax.get_ylim()[1]*0.9, f' {percentile}th Percentile: {int(round(percentile_value, -1))}',\n",
    "            color='red', ha='left')\n",
    "\n",
    "    ax.set_xlabel('Text Length')  # Set x-axis label.\n",
    "    ax.set_ylabel('Frequency')  # Set y-axis label.\n",
    "    ax.set_title('Histogram of Text Lengths')  # Set title.\n",
    "\n",
    "    ax.xaxis.set_major_locator(MaxNLocator(integer=True))  # Set major locator for x-axis.\n",
    "    plt.show()  # Display the plot.\n",
    "\n",
    "def visualize_train_test_category_splits(train_df, test_df, top_x=5, normalize=True):\n",
    "    \"\"\"\n",
    "    Visualize the frequency of top categories in training and test datasets.\n",
    "    \"\"\"\n",
    "    set_plot_appearance()  # Apply appearance settings to the plot\n",
    "\n",
    "    # Get top x categories from both datasets.\n",
    "    train_topic_norm = train_df['topic'].value_counts(normalize=normalize).head(top_x)\n",
    "    test_topic_norm = test_df['topic'].value_counts(normalize=normalize).head(top_x)\n",
    "    \n",
    "    # Prepare the data for plotting.\n",
    "    train_topics = pd.DataFrame({'topic': train_topic_norm.index, 'frequency': train_topic_norm.values, 'dataset': 'Training'})\n",
    "    test_topics = pd.DataFrame({'topic': test_topic_norm.index, 'frequency': test_topic_norm.values, 'dataset': 'Test'})\n",
    "    combined_topics = pd.concat([train_topics, test_topics])\n",
    "\n",
    "    sns.barplot(x='frequency', y='topic', hue='dataset', data=combined_topics)  # Create bar plot.\n",
    "    plt.title('Top {} Topics Frequency Comparison'.format(top_x))  # Set title.\n",
    "    plt.xlabel('Normalized Frequency')  # Set x-axis label.\n",
    "    plt.ylabel('Topic')  # Set y-axis label.\n",
    "    plt.tight_layout()  # Adjust layout.\n",
    "    plt.show()  # Display the plot.\n",
    "\n",
    "\n",
    "\n",
    "def get_data_stuff(dataset_name, split_ratio=0.2):    \n",
    "    # %%\n",
    "    from datasets import load_dataset\n",
    "    from datasets import Dataset\n",
    "    import pandas as pd\n",
    "    pd.set_option('display.max_colwidth', None)\n",
    "    \n",
    "    \n",
    "    #load the dataset\n",
    "    dataset = load_dataset(dataset_name)\n",
    "    \n",
    "    # Shuffle the dataset (setting seed for reproducibility)\n",
    "    shuffled_ds = dataset['train']#.shuffle(seed=seed)\n",
    "    #shuffled_ds = shuffled_ds.select(range(1000))\n",
    "\n",
    "    \n",
    "    # Split the dataset into training and test sets with a test size of 20%\n",
    "    train_test_split = shuffled_ds.train_test_split(test_size=train_test_split_ratio, seed = seed)\n",
    "    \n",
    "    # The train/test datasets are now accessible as follows:\n",
    "    train_ds = train_test_split['train']\n",
    "    raw_train_df = pd.DataFrame(train_ds)\n",
    "    \n",
    "    test_ds = train_test_split['test']\n",
    "    raw_test_df = pd.DataFrame(test_ds)\n",
    "    \n",
    "    prepared_train_df = prepare_dataset(raw_train_df, \"train\")\n",
    "    prepared_test_df = prepare_dataset(raw_test_df, \"test\")\n",
    "\n",
    "    prepared_train_dataset = Dataset.from_pandas(prepared_train_df)\n",
    "    prepared_test_dataset = Dataset.from_pandas(prepared_test_df)\n",
    "    \n",
    "    print(f\"length of downloaded ds: {len(shuffled_ds)}\")\n",
    "    print(f\"length of train_ds: {len(prepared_train_dataset)}\")\n",
    "    print(f\"length of test_ds: {len(prepared_test_dataset)}\")\n",
    "    estimated_training_steps = epochs * len(prepared_train_dataset) / per_device_train_batch_size / gradient_accumulation_steps\n",
    "    print(f\"estimated training steps will be {estimated_training_steps}\")\n",
    "    return prepared_train_df, prepared_test_df, prepared_train_dataset, prepared_test_dataset\n",
    "\n",
    "\n",
    "def prepare_dataset(df, split=\"train\"):\n",
    "    text_col = []\n",
    "    instruction = \"\"\"Write a concise summary of the below input text.Ensure that responses covers the key points of the text.Only provide full sentence responses.\"\"\"  # change instuction according to the task\n",
    "    if split == \"train\":\n",
    "        for _, row in df.iterrows():\n",
    "            input_q = row[\"dialogue\"]\n",
    "            output = row[\"summary\"]\n",
    "            text = (\n",
    "                \"### Instruction: \\n\"\n",
    "                + instruction\n",
    "                + \"\\n### Input: \\n\"\n",
    "                + input_q\n",
    "                + \"\\n### Response :\\n\"\n",
    "                + output\n",
    "                + \"\\n### End\"\n",
    "            )  # keeping output column in training dataset\n",
    "            text_col.append(text)\n",
    "        df.loc[:, \"text\"] = text_col\n",
    "    else:\n",
    "        for _, row in df.iterrows():\n",
    "            input_q = row[\"dialogue\"]\n",
    "            text = (\n",
    "                \"### Instruction: \\n\"\n",
    "                + instruction\n",
    "                + \"\\n### Input: \\n\"\n",
    "                + input_q\n",
    "                + \"\\n### Response :\\n\"\n",
    "            )  # not keeping output column in test dataset\n",
    "            text_col.append(text)\n",
    "        df.loc[:, \"text\"] = text_col\n",
    "    return df\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "2a764bbf-2d24-410d-9356-68f2153af13e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">length of downloaded ds: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">12460</span>\n",
       "</pre>\n"
      ],
      "text/plain": [
       "length of downloaded ds: \u001b[1;36m12460\u001b[0m\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">length of train_ds: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">12148</span>\n",
       "</pre>\n"
      ],
      "text/plain": [
       "length of train_ds: \u001b[1;36m12148\u001b[0m\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">length of test_ds: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">312</span>\n",
       "</pre>\n"
      ],
      "text/plain": [
       "length of test_ds: \u001b[1;36m312\u001b[0m\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">estimated training steps will be <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">47.453125</span>\n",
       "</pre>\n"
      ],
      "text/plain": [
       "estimated training steps will be \u001b[1;36m47.453125\u001b[0m\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "train_df, test_df, train_dataset, test_dataset = get_data_stuff(hf_dataset_name, \n",
    "                                                                split_ratio=train_test_split_ratio,\n",
    "                                                               )\n",
    "text = test_df['text'][1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "46b9d045-f3df-45b3-beea-92336c089d1c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">### Instruction: \n",
       "Write a concise summary of the below input text.Ensure that responses covers the key points of the text.Only \n",
       "provide full sentence responses.\n",
       "### Input: \n",
       "#Person1#: Mister Ewing said we should show up at the conference center at <span style=\"color: #00ff00; text-decoration-color: #00ff00; font-weight: bold\">4:00</span> o'clock, right?\n",
       "#Person2#: Yes, he especially asked us not to be late. Some of the people from our east york branch office are \n",
       "coming, and he wants to make a good impression on them. How are you getting there?\n",
       "#Person1#: I was thinking of taking my car, but I think I'm just going to take the underground, because there is \n",
       "construction on the highway. What about you?\n",
       "#Person2#: I'll be taking the underground as well. Why don't we go together? I've been to the conference center \n",
       "only once, and I'm not sure if I can find my way around there.\n",
       "### Response :\n",
       "\n",
       "</pre>\n"
      ],
      "text/plain": [
       "### Instruction: \n",
       "Write a concise summary of the below input text.Ensure that responses covers the key points of the text.Only \n",
       "provide full sentence responses.\n",
       "### Input: \n",
       "#Person1#: Mister Ewing said we should show up at the conference center at \u001b[1;92m4:00\u001b[0m o'clock, right?\n",
       "#Person2#: Yes, he especially asked us not to be late. Some of the people from our east york branch office are \n",
       "coming, and he wants to make a good impression on them. How are you getting there?\n",
       "#Person1#: I was thinking of taking my car, but I think I'm just going to take the underground, because there is \n",
       "construction on the highway. What about you?\n",
       "#Person2#: I'll be taking the underground as well. Why don't we go together? I've been to the conference center \n",
       "only once, and I'm not sure if I can find my way around there.\n",
       "### Response :\n",
       "\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "print(text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "fe3b1874-38b3-406c-a9c3-ce34e42f1f64",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAm8AAAHPCAYAAAAFwj37AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/SrBM8AAAACXBIWXMAAA9hAAAPYQGoP6dpAABf2klEQVR4nO3dd1yVdf/H8RdLRRQEI+8QJ5a5R5qjcCVQSWYOUnPPTE3DkWU5bstZYq7KrMzUFNTU1ATcGGKuNNySI8GNCioo6/eHP84tgizB46Xv5+PRI841vtfne3E45+33WhYuLi4piIiIiIghWJq7ABERERHJPoU3EREREQNReBMRERExEIU3EREREQNReBMRERExEIU3EREREQNReBMRERExEIU3EREREQNReBMRERExEIU3EXnkuLq6EhkZiZ+fn7lLEXks+fn5ERkZiaurq7lLkVywNncBIkYQGRmZo+VLliyZT5Xcn5+fHz4+Pvedn52ajNDPh8XX15chQ4bcd/66devo2bPnQ6xIsmJlZUWbNm3w9vamWrVqFCtWjPj4eP755x+2bNnCwoULc/weF3kUKbyJZMOXX36ZblqvXr1wcHDIcJ45zZ07l2vXruVq3Ueln+fOnaNRo0bExsY+tG3ez5o1azh8+HC66cePHzdDNXI/JUuW5Mcff6RKlSpcuHCBkJAQoqKisLW1pVq1avTv35++ffvyyiuvcPLkSXOXa3YTJkxg5syZnDt3ztylSC4ovIlkw9SpU9NN8/HxwcHBIcN55vTdd99x5syZXK37qPQzMTGRiIiIh7a9zKxevZpVq1aZuwzJhJ2dHYsWLaJChQrMnj2bKVOmcPv27TTLlC1bltGjR2NnZ2emKh8tFy5c4MKFC+YuQ3JJ57yJ5DFHR0fGjh3L9u3b+eeff9i3bx/ffPMNFStWTLds6nknpUuXpl+/fmzbto2IiAi2b9/O4MGDsbZ+dP99lZ/9zOycNzs7Oz744AOCg4M5fvw4hw4dIjAwkGHDhqVpp2rVqsyZM4c///yTf/75h/3797NmzRref//9PN0PAQEBREZGUrBgQYYPH84ff/zByZMn8fX1NS1TqlQppkyZYqplz549+Pn53fews6enJ2vWrOH48eP89ddfTJ48GQcHB8LCwggLC8tw+xnJ7LwmT09PlixZwoEDB4iIiGDDhg307dsXS8u0Xws+Pj5ERkbi4+NDo0aNWLlyJcePHyc8PJxp06bh6OiY4bYrV67MjBkz2LVrl6nPCxYswMPDA4AOHToQGRlJv379Mlz/pZdeIjIykkmTJmU4/27vvvsuFSpUYNmyZXz++efpghvAyZMn6d69O0ePHk0zvU6dOsyfP5/w8HAiIiLYsmULQ4YMoVChQunaiIyMJCAggP/85z/MnDmTv//+myNHjjB//nxKly4NQIUKFfj+++8JDw/nyJEjzJkzh6eeeipNO3e/v5977jnmz5/PwYMHOXr0KIsWLaJatWrptl2tWjU+++wzNmzYwKFDhzh+/Djr16+nf//+GX5OpL5X7O3t+eyzz9i5cyenTp0ynVpxv/fG66+/ztKlS9m3bx8RERHs3r2bxYsX8/rrr6fbhoeHBwEBAaZ6goOD6dOnD1ZWVvftb9myZZk7dy4HDhzg2LFjLF68mMqVK6drWzL36H4ziBiQk5MTq1atoly5coSGhrJy5UpKly5NixYteOWVV+jYsSM7d+5Mt97YsWOpW7cuv/32Gzdu3MDDw4Nhw4ZRuXJl+vTpk6MamjdvTpEiRbh9+zbHjh1j27ZtJCQk5FUXAfP1s3jx4ixbtoxnn32W8PBw5s+fj6WlJW5ubrz33nt8++23xMTEUKVKFVauXElycjKBgYGcOXMGBwcHnn32Wd555x2mT5+ep/sD7ox4Vq5cmU2bNhETE8O///4LQK1atVi4cCGFCxdm/fr1nDhxglKlSvHWW2/RtGlTWrZsyenTp03ttG3blq+++oqYmBiWLVtGTEwMzZs3Z/HixdjY2OTJ73LEiBEMHDiQs2fP8vvvvxMbG8uLL77IqFGjqF27Nn379k23joeHB6+88grr169n165d1KtXj3bt2lGmTBneeuutNMu+/vrrzJw5EwsLC4KDg/nnn38oXrw4tWrVon379gQHB7NixQpGjRpFhw4d+Prrr9Ntr2PHjgAsXLgwy/60b98eIFsXuNy9/7y9vZk1axa3b99m1apVXLp0icaNG+Pr60uTJk1o27Ytt27dSrN+sWLF+PXXX7l48SIBAQGUL18eDw8P3Nzc6NGjB8uXL+fvv/9myZIlVK9enRYtWlCsWLEMz0ctXbo0K1asML2XXV1d8fb2Zvny5fj4+LB3717Tsu+88w7Nmzdnx44dbNy4EVtbWxo0aMDHH39MjRo1Mvz7KVCgAP7+/tjZ2REUFERiYiIXL168777p0qULEyZM4Ny5c6xbt44rV67g7OxMzZo1efXVV1m7dq1p2T59+jB69GiuXLnCihUruHnzJp6enowePZoXX3yRXr16pWvf1dWV3377jSNHjrBkyRLKlCnDq6++ir+/P02aNOHSpUuZ//LEROFNJA+NHDmScuXKMWPGDCZOnGia3qxZM37++Wf8/Pxwd3cnJSUlzXq1a9fGw8ODs2fPAjBp0iR++eUXWrRoweuvv57mQzMrn3/+eZrX586dw9fXly1btjxAz9IyVz8nTJjAs88+y/Tp09ONyDz11FPcuHEDgDZt2lCoUCG6d+9OUFBQmuXuN1J0P97e3lSoUCHd9FmzZqX5Yi9RogTNmzfn6tWrpmnW1tbMnj0bS0tLWrRowYEDB0zz6taty9KlS/nvf/9Lt27dAChSpAjjxo3jxo0btGjRgn/++Qe4s58WL17Mf/7zH1MozC13d3cGDhzIpk2b6N27N3FxcaZ5EyZMoEuXLhn+Ljw8PGjbti27du0CwNLSkiVLltCwYUNq167Nnj17gDu/h2nTppGYmMhbb72Vps8AzzzzDABxcXH8+uuvdO3alfr166cZUSxWrBivvvoq4eHh7N+/P9P+lCxZEhcXF6Kiojhx4kS290ORIkWYPHkyiYmJtGzZkkOHDgEwceJEZs2axZtvvkm/fv2YNm1amvUqV67MnDlzGDt2rGna+PHj6dq1K8uXL2fq1Kl8//33pnnz58/nlVdeoWrVqoSHh6dpq379+un+hgICAli0aBGTJ082jVICTJ8+nY8//pjk5OQ0bXzxxRd06NCBOnXqmH43qUqUKMHBgwdp1aoV8fHxWe6TDh06cOvWLTw9Pbl8+XKaeXf/3ZQpU4aPP/6Yixcv8vrrrxMVFQX873362muv0aZNG5YtW5amjYYNG/L5558ze/Zs07Rhw4YxePBg3n77bWbNmpVljXKHDpuK5BEbGxvefPNNoqOj033gb9y4kS1btlCuXDnq1q2bbt3vv//eFGjgzuhAajjJ7ArSu4WFhdG3b1/q1q1L+fLleemll5gyZQoODg78+OOPVK9ePfedu4u5+uns7Mxrr73GiRMnMrx44tKlSyQlJaWZltEX1pUrVzLdzr1atGjBkCFD0v1XsGDBNMt98cUXaYIb3BkFLV26NN988026ELNz504CAwNp1qwZRYoUAeDVV1/F3t6exYsXm4Ib3DkHMDuHD7Oje/fuAAwfPjxNcIM7ISQ5OZlWrVqlW2/FihVpwkFycjIBAQEA1KhRwzS9Xbt22NnZ8e2336brM5Dm9//zzz8D/xtlS5UavhctWpRlf55++ul07WaHl5cXDg4OLFmyxBTcAFJSUvjss89ISEigXbt26da7fv16ut/FihUrgDvvrbuDG8DKlSsBMjw0ePXq1XSjwFu2bCEkJITKlSunOXwaFRWVLrgBzJs3D7gTyjPy+eefZyu4pUpMTCQxMTHd9Lv/bt566y1sbGz49ttvTcEN4Pbt24wfPx7I+O/51KlT6UZZf/nlFyDte0iyppE3kTxSoUIFbG1tCQ0NzfDDMjQ0lMaNG1OlShX+/PPPNPN27NiRbvndu3eTkJBA1apVs7X9JUuWpHl98uRJpk2bxtmzZ5k6dSoffPCB6Yv7QZirnzVq1MDS0pLQ0NAMv1zu9ttvv9GrVy++//57Vq1axdatW9mxY0eurqzr169fti5Y+Ouvv9JNq127NgDly5dPcw5cqqeffhorKyvKly/P/v37TV/w9+43+N9+elC1a9fmxo0bpkON94qPj8fNzS3d9IxGwFIDk4ODg2larVq1ALI10nvo0CF2797N66+/zieffEJMTAxw5zDozZs3Wb58edYdyqXU99v27dvTzYuKiuL06dO4ublhZ2dnGtEFOHHiRLr3feqJ/3eHwHvn/ec//0k378CBA9y8eTPd9B07duDu7k7VqlX5+++/gTv/aOrevTstW7akQoUK2NnZpTk/MaP24+LiMqzpflauXMmnn37Khg0bWLFiBaGhofz5559cv349zXJVqlQBMt53u3btIi4uzrTMvf29dzQ+o/eQZE3hTSSPpI6e3O+8jdQP8aJFi6abl9E6ycnJXLlyJcPlcyIgIIDPP/88w5Gw3DBXP1PnZyeA7d27l7Zt2zJw4EBatWplCip79+5l/PjxhIaGZtlGTmV0LlGxYsWAOyNJmSlcuDDwvz5mtp8eVLFixbCxscn0Hnap9dzt3i9wwBSi7w4ROfk9ASxYsAA/Pz9at27NvHnzqFWrFpUrV8bf3z9bt4rJLBxlJvV9fL9zwM6fP4+bmxtFixZNE94yqil1P2S2jzK6qOB+2079/dvb25umzZkzB09PTyIiIli1ahWXL18mISEBBwcHevXqRYECBdK1c++hz6x88803XLlyhS5dutC3b1/69etHQkICGzZsYMyYMaZD9qm/48zqz+j3kdH+SR0tv/dCGcmcwptIHkn9YLr3yrJUzs7OQMYf/k899VS6W2NYWlri6Oj4wCfxJicnExMTk2f/sjVXP1NHZbL7Jf3nn3/SuXNnChUqRK1atfDw8KBLly7Mnz+fZs2apblIIL+k7quuXbuyfv36LJdP3WcZ7dvU/XRvKEodybCyskp32DijQHz9+nVSUlIyvKIxL9z9e8rOLWtWrVrF6NGj6dixI/PmzaNDhw5A9i5UgDtXgJ49e5aSJUtSrly5bJ/3lvq7SX2/3iv1cGx+3mvwfttO/f2n7ssaNWrg6enJpk2b6NKlS5rDp7Vr187w4gAg3ShXdixZsoQlS5bg6OjIiy++SKtWrWjZsiXlypWjefPmJCcnm/aJs7Nzhlc6P/XUUxkGNck7iroieeT48ePExcVRs2bNDG8z0KBBA4AMzwOqV69eumkvvPACNjY26U5yzikXFxdKlCiR63u/3ctc/dy3bx9JSUk0bNgwR7dQiY+PZ/v27fz3v/9l5syZ2Nra0qhRo2yv/yBSrxZ84YUXsrX8wYMHAXjxxRfTzUvdT/dKPc/u3lBrYWGR4XlWe/bswcnJiXLlymWrppxK7XPjxo2ztXx8fDxLly6lSpUqNGzYkJYtW3L06NF0J99nJvW8qUGDBmW5bOo+TH2/pb5f7+bi4kKZMmU4efJkmlG3vFalSpUMRzlT/05SayxTpgwAGzZsSHfeW0bvlbxw5coVAgMDTbf2qVixouk9k/q3ndG+q127Nra2thn+/UveUXgTySMJCQmsXLmS4sWLM3DgwDTzmjRpQtOmTTlx4kSGt9Do2bOn6So8uPMF8+GHHwLg7++f5badnZ0zHJGyt7c3XVTw66+/5qQ792Wufl66dIm1a9dSrly5DM8fK168uOn+Ui+88EK6CwrgfyMa997+Ib+k3qakd+/eGQZXa2vrNIezAwMDiYmJoX379pQvXz7NcsOHD89wG/v27QPSnyDep08f05f+3X744QfgztM0Mrry1tnZOcOra7MrICCA69ev07dv3wzPe8rofbpgwQLgzhWVRYsWzdaFCnf75ptvOH78OO3atWPEiBEZHkIsVaoU33//Pc899xxwZ19fu3YNHx8f07RUH3/8MTY2NqYLMvJLsWLF0t13sHHjxri7u3Po0CHT+W6po1v3BrXnnnuOAQMG5Fk9GYUxa2tr0+H/1HP9fv31VxISEujTpw8lSpQwLWtjY8PHH38MZO9zS3JPh01F8tDnn39O/fr1GTx4MHXq1GHv3r2mezfdvHmTDz74IMNDGXv27CE4OJhVq1Zx8+ZNPDw8qFChAmvWrMnWbUIqVKjAL7/8wq5duzhx4gSXL1/GxcWFpk2b4uTkxLZt2zK8l5bR+vnRRx9RsWJFBg0aRLNmzfjjjz+wsLCgfPnyNGrUiJo1axITE8N7771Hw4YN2bFjB6dPn+bWrVtUq1YNd3d3Tp48ye+//55n+yIzt2/fpk+fPixYsIDly5ezbds2Dh8+TEpKCiVLlqRevXpcuXLFNEoVGxvLqFGjmDZtGmvWrGHVqlWm+7zFx8dneB7ZkiVL6NevH0OHDqVKlSqcOnWK6tWr8/zzzxMaGkrDhg3TLL9582b8/Pz44IMP2LZtG5s3b+bMmTM4OjpSrlw5XnzxRSZPnpzrx39dvnyZQYMGMXv2bFavXk1wcDARERE4OTlRq1Yt/v3333TPhD127BhhYWHUr1/fNBKXEzdu3KBjx478+OOPDBw4EB8fH7Zu3crZs2extbWlSpUq1K1bl8TERMaNGwfcOWw6fPhwZs2aZXqKxuXLl3F3d6dGjRrs2bMnT/9mMhIWFkaXLl1Mt1pJ/RuKi4tj2LBhpuX27t3Lnj17aNmyJU8//TR79uyhZMmSeHp6smHDBry9vfOknu+//57r16+zZ88ezpw5g42NDe7u7lSsWJHVq1ebQuSpU6cYP348o0ePZv369fz2229p/p7XrVuX7jYhkrcU3kTyUHR0NN7e3gwePBgvLy9efPFFYmNjCQwMZOrUqRw5ciTD9UaPHo23tzcdO3bExcWFCxcu8MUXXzBz5sxsbffUqVP4+/ubbqaZepL1oUOHWLFiBYsWLcrwNgNG6+eVK1d44403ePfdd/H29qZbt27cunWLf//9l1mzZpmu3Js/fz6xsbHUqlWL+vXrY2FhQWRkJNOnT2fOnDkP9Xycffv24eHhQb9+/WjWrBl16tTh9u3bphuhpt5KIlVAQAAxMTEMGjSItm3bEhsbS1BQEJ9//jmBgYHp2r906RI+Pj6MGjWKxo0bk5iYSGhoKG+88cZ9DyN+8cUX7Nixgx49evDyyy9jb2/PlStX+Pfff5k6deoDj9KuW7eON954gwEDBlC/fn08PDyIjo7mwIED9x1VCwgIoH79+qabw+ZUZGQkr7/+Oq1bt+aNN96gcePGFCtWjFu3bnHixAm+/vprfv755zS3tli9ejUXLlxgwIABvPbaa9ja2nLmzBn8/PzS3ccvP5w+fZqPPvqITz75hK5du2JlZcX27dsZP368adQN7py32rVrVz7++GOaNGlCjRo1OHHiBOPGjWPjxo15Ft4mTpxIkyZNqFmzJs2bNycuLo6TJ08yYsQI06HpVHPmzOHEiRP06dOH1q1bY2Njwz///MPYsWPT3S5F8p6Fi4tLzs9oFJE84efnh4+PD/Xq1cuzc9IeRU9KP/Nb6o1s69evb+ZK8t5nn31G9+7d8fHx4Y8//jB3OfnK1dWVHTt24O/vzwcffGDucsSAdM6biIiYlZOTE+3ateP48eOPfXATyQs6bCoiImbxyiuvUK1aNVq0aEGRIkUyfHKGiKSn8CYiImbh7e2Nj48PZ8+eZcKECdl6koWI6Jw3EREREUPROW8iIiIiBqLDppKpZcuWYWNjQ3R0tLlLEREReaw5OTmRkJCQ5fOQFd4kUzY2Nqa71ouIiEj+ye73rcKbZCp1xK1bt27mLUREROQxN2/evGwtp3PeRERERAxE4U1ERETEQBTeRERERAxE57xJlqysrKhataq5yxARETMrUKAARYsWxdLywcZ+rl69yoULF/KoKmNJSUkhOjqa+Pj4XLeh8CZZcnFxITAw0NxliIiIGV2/fp1jx45x8+ZNUlIe7P7+KSkpnDt3jqSkpDyqzlgSExP54YcfOHLkSK7WV3iTLFlYWLAjIobYuCfzj0xE5ImXkkyB6yfYszOMbaFhJCUn57opKwsLbAtYcvLkSW7dupWHRRqDtbU1np6e9OjRg9GjR+dqBE7hTbIlNi6JqzcTzV2GiIiYgTUJFI67ydY/wog6e/6B2rKyBHtba6Kioh7o0KGRBQUF8fzzz+Pk5ERUVFSO19cFCyIiIpIpi///f1JS7kfc5H8SE+8MhlhYWGSxZMYU3kREREQMROFNREREzCogIICxY8eauwzDUHgTERERw2jQoAGRkZHY29s/1O36+voSFBT0ULd5PwpvIiIiIgai8CYiIiIPTaFChRgyZAj79+9nz5499O3bN838Nm3asHbtWo4cOcLevXuZOXMmxYsXB8DV1ZWlS5cCcOjQISIjI/Hz8wOgSZMm/Prrrxw8eJDw8HB++uknypQpY2rXxsaGzz77jD179hAREcGOHTsYMGCAab69vT1Tpkxh//79HD58GH9/fypXrgyAj48PQ4YMoUqVKkRGRhIZGYmPj0++7qfM6FYhIiIi8tD0f7c31apV49133yUqKooRI0ZQrVo1Dh48CNy5D9qUKVOIiIjgqaeeYvTo0fj5+dGlSxeioqLo1asXc+fOxd3dndjYWNPtRgoXLsycOXM4dOgQdnZ2DB06lLlz5+Lp6UlKSgo9evTA09OTd999l8jISFxcXHBxcTHV9e233xIfH0+nTp2IjY2lU6dOLFmyBHd3d1atWkXFihVp0qQJ7du3ByA2Nvbh77z/p/AmIiIiD4WtbSG8X3+VKVOmEBoaSnx8PIMHD2bXrl2mZZYsWWL6+fTp03z66af8/vvvFC5cmJs3b3L16lUALl26RExMjGnZtWvXptmWr68v4eHhPPfccxw5coSSJUty4sQJ/vzzTwAiIyNNy9atW5eaNWtSo0YNbt++DcC4cePw8vKiRYsWLFy4kBs3bpCUlMTFixfzfL/klMKbiIiIPBQlXVwoUKBAmsdCXb16lYiICNPratWqMWTIECpXroyDg4PpOaolS5bk2LFj9227XLlyDB06lFq1auHk5JRmvSNHjuDv78/ixYsJCQlh06ZNrF+/nq1btwJQuXJl7OzsCA8PT9NmoUKF0hx6fVQovImIiMgjwdbWlkWLFrF582YGDBjA5cuXKVmyJL/88gsFChTIdN158+Zx5swZhg8fzrlz57C0tGTTpk3Y2NgAEB4eTv369WnWrBkvv/wy33zzDdu2baNPnz7Y2dlx4cIF2rZtm67da9eu5UtfH4TCm4iIiDwUkVFRJCQkULFiRf744w8AHBwcKF++PGFhYVSoUAEnJycmTJhgemxUjRo10rSRkJAAgJWVlWmao6MjFSpUYNiwYabDonXr1k23/evXr7Nq1SpWrVrFmjVrWLRoEcWKFePvv//G2dmZxMREzpw5k2HtCQkJptE8c3s0qhAREZHHXlxcPKvXrqNXr140aNCAihUr4ufnR/L/P+g+MjKSW7du0b17d0qXLo2HhweDBw9O08aZM2dITk6mefPmODk5UbhwYa5evUp0dDSdOnWibNmyvPTSS4wePTrNen369OHNN9/Ezc2N8uXL4+3tzfnz57l27RohISHs3r2bH374gUaNGuHq6kqdOnX48MMPqV69OgD//vsvpUuXpkqVKjg6OmY5EpifFN5ERETkoZn1zRzCw8OZM2cOixcv5s8//2T//v0AREdH88EHH+Dt7c2mTZsYMGAA48aNS7P+uXPn+PLLL/noo4/Yt28fn3/+OSkpKbz33ntUq1aNDRs2MGbMGD777LM0612/fp333nuP33//nTVr1lCqVCk6d+5MSkoKAJ07dyYsLIypU6cSEhLC7NmzKVmyJJcuXQLuXBCxefNm/P39CQ8Pp1WrVvm/s+7DwsXFJcVsW5dH3rx58yhdujT/JjzN1ZuJ5i5HRETMwIYEbK8fZ9bsbzh/4cGutrSyBHtba44ePWq6zceTpmTJkvj6+jJ16tQ0V73OmzcPgG7dumW6vkbeRERERAxE4e0BNGjQ4KE+58zDw4MxY8Y8tO2JiIjIo0dXmxpIcHAwwcHB5i5DREREzEjhLZsKFSqEn58fzz//PImJiVy8eJEZM2ZgZWXF+PHjqVu3LlZWVgwePNh04mWbNm149913ATh79qzp3jM+Pj60bduWGzduULZsWaKjoxk0aBBnzpzJcp6Xlxc9e/akQYMGfPbZZ+zYsSPDbXfq1Im+ffty48YN1q1bx7BhwyhZsmSu+1/U1irrhURE5LFkmZxM8g2wtLxzztqDsLK0yJuinmAKb9nUpEkT7O3tadq0KQDFihWjUqVKVKhQgaFDh/Lxxx/TuXNnPvzwQ9555x0qVqzIJ598wmuvvca5c+d4//33mTJlCp07dwagTp06eHp6cvz4cfr168fkyZPp2LFjlvPultm2fX198fLy4uLFiwwZMuSB+p6SkkI9N/sHakNERIwrPj6eXbssKVLQinjbB48OycnJJCUl5UFlTyaFt2w6ePAgzz77LOPHjycsLIwNGzYAcPLkSfbu3QvA7t276du3LwANGzZk8+bNnDt3DoCffvqJwYMHm27wt3v3bo4fPw7AwoULGT58eLbm3e1+237ppZfYvHmz6flrixYtwtfXN9d9j4qKYtCgQbleX0REjM3Z2ZnWrVtz8uRJ081zH0RSUpLpZruScwpv2XT69GmaNGnCSy+9hLu7OyNHjmT06NHcunXLtExSUhLW1hnv0tT7yOSlh7XtpKSkdM97ExGRJ0fJkiW5ffs2t27dSnN7DxcXF5ycnPJ8e9HR0XkSEh9XCm/Z9Mwzz3D16lWCg4PZvHkzr776Ki4uLvddPjQ0lPfff58SJUpw/vx5OnfuzLZt20x3ka5duzZubm5ERETQoUMHQkNDszUvO0JDQ+nfvz/Fixfn8uXLtG/f/sE6LyIicg8XFxe2hoRgW6hQnrcdFx9PI3d3Bbj7UHjLpueff56PPvoICwsLrKysWLZsGYcOHbrv8keOHOGzzz5jwYIFwP8uWEi1e/duRo4cSdmyZbly5Uqaw5KZzcuOw4cPM336dFauXMn169fZvHnzI/lgXRERMS4nJydsCxViR0QMsXF5d/5aUVsr6rnZ4+TklGl4S71Vl42NDW5ubhw+fBiAiIgI+vXrl61teXh48NJLL2V5G64SJUrw9ddf07p16+x1Ip8pvGXTpk2b2LRpU7rpnp6epp+PHDlC/fr1Ta+XLVvGsmXLMmwvNjaWnj175miev78//v7+AGzfvj3TbS9dupSffvoJgJ49e7J79+7MuiciIpIrsXFJZnkCT+p3oKurK0FBQWm+E1NZWVllemFEdm/Bdf78+UcmuIFu0vvY+vjjjwkKCmLjxo00b96ckSNHmrskERGRfBcWFsbHH3/M6tWrmTZtGs7OzgQEBPD777+zceNGPvvsMyws7tyuxMfHh++//x64c+P9DRs2MH78eIKDg9m4caPpofSurq4cPHjQtI3IyEgGDhzI6tWr2b59Oz4+PqZ5derUISgoiPXr1/Pll18SHBxMgwYN8rSPGnkzg7tH0HIyLycU1kRE5Enl6OiIt7c3AAULFqRr167cvHkTS0tLfvzxR9544w1WrVqVbr373YIrI7dv38bb2xs3NzfWrl3LsmXLsLS05Ouvv2bQoEGEhobSsGHDfDnvXCNvIiIi8li5exDEwsKCkSNHEhwcTGBgINWrV6dKlSoZrnfvLbjKlClz320sX74cuHOOXWJiIk8//TQVKlQgMTGR0NBQ4M4FhCdOnMirbplo5E1EREQeKzdv3jT93LdvX5566im8vb25desWo0ePptB9rpDN7i247l02OTkZK6uH9yQihTcRERHJtbx+fGJet+fg4MCFCxe4desWzs7OeHt7s3bt2jzdRqqIiAhsbGyoX78+YWFh1K9fn3LlyuX5dhTeREREJMeio6OJi4/Pl8cnxsXHEx0dnSdtzZ07lzlz5rBx40bOnz9PSEhInrSbkdu3b9OvXz/Gjx+PhYUFf//9N8ePHycmJiZPt6PwJiIiIjkWFRVFI3d3sz9h4cyZM1SuXNn0+u7bZsGdOlMvXrhXdm/Bde82SpYsmaadatWqmX4+ePAgHh4eANSoUYPGjRsTERGRrb5kl8KbiIiI5EpUVJSegnCPFi1a0Lt3b+DOeXODBg1K80ixvKDwJiIiIpJH8uqWX5nRrUJEREREDEThTURERMRAFN5EREREDETnvImIiEiuuLi4mP1q0yeRwpuIiIjkmIuLCyFbt1LI1jbP246Pi8O9UaNMA1xQUBAANjY2uLm5cfjwYeDOjXL79euX7W01aNCAggULsnnz5geq+WFSeBMREZEcc3JyopCtLTf2BZJ840qetWtp54hdDS+cnJwyDW+p92RzdXUlKCgozT3acqJBgwY4ODgovImIiMiTIfnGFZJiLpq7DJPGjRszePBgChUqRFJSEuPHjyc0NJRy5crh5+eHnZ0dFhYWBAUFsWbNGjp37oyVlRUNGjRg7dq1TJs2zdxdyJLCm4iIiDwWSpcuzZAhQ+jYsSPXr1+nbNmyLF++nPr169OtWzfWr1/PzJkzAShWrBhXr17l559/xsHBgdGjR5u5+uxTeBMREZHHQtOmTU2BLVVycjIlS5Zkx44dfPLJJ9jZ2bF9+/Z8fcZpflN4ExERkcfG1q1bGTBgQLrpJ06cYNeuXTRq1Iju3bvTq1cvunTpYoYKH5zCm4iIiOSapZ3jI9Peli1b8PX1pVKlShw6dAiAmjVr8tdff1GuXDlOnjzJ0qVL2bt3LytXrgTg+vXruLq65kntD4vCm4iIiORYdHQ08XFx2NXwyvO24+PiiI6OzvF6J0+eZMCAAUyaNAlbW1tsbGwIDw9nwIABtGjRgtatW5OQkIClpSUfffQRAL///jtt2rQhKChIFyyIiIjI4ysqKgr3Ro3MfpPeM2fOULlyZdPrkJCQDM9nmzlzpulihbv9+++/eHnlfQDNTwpvIiIikitRUVF6EoIZ6NmmIiIiIgai8CYiIiJiIDpsKlmysrKiatWq5i5DRESykF8PdE9JSQHA2lqxIS+k7sfU/Zrj9fOyGHk8ubi4EBgYaO4yREQkC3Hx8TRyd8/zABcdHU1iYiKenp4EBQWRmJiYp+0/SaytrfH09CQxMTFXV9SCwptkg4WFBTsiYoiNSzJ3KSIich9Fba2o52af5QPdcyM+Pp4ffviBHj168Pzzz+dp20+ixMREfvjhB+Lj43O1vsKbZEtsXBJXb+pfWiIiT6ojR44wevRonJycsLCwMHc5hpWSknLnHnm5DG6g8CYiIiLZFB8fr1uDPAJ0tamIiIiIgSi8iYiIiBiIwpuIiIiIgSi8iYiIiBiIwpuIiIiIgSi8iYiIiBiIwpuIiIiIgSi8iYiIiBiIwpuIiIiIgSi8iYiIiBiIwpuIiIiIgSi8iYiIiBiIwpuIiIiIgSi8iYiIiBjIYxvegoKCsLOzu+98V1dXDh48mK22hg4dyltvvZXlclWqVKFly5bZrjEn5s+fj5ubW760LSIiIsZhbe4C8ounp2eetfXFF19ka7kqVarg5eXFqlWrcrwNKysrkpKS7ju/S5cuOW5TREREHj+PbXiLjIykUqVKlC1blnHjxlG4cGFu3brFmDFj2LVrl2m5Tz/9lEaNGmFlZcXo0aMJCQlJ15afnx8HDhxg7ty5+Pr68uyzz2Jra0uZMmW4ePEiffr0wcrKiqFDh1K0aFGCgoLYs2cPI0aMoEaNGowcOZIiRYpgZWXFjBkzWL16Na6urgQFBbFgwQIaNWrEmjVr6Nu3L02bNuXixYsA+Pr6Ym9vz5gxYwgLC6Nnz54cOHAAZ2dnxo0bh6urK4UKFSIoKIjJkyfTqFEj3n33XTp27EiRIkUIDw9n5MiRLFy4kLZt29KgQQOGDBmSq/1ZwsGGooWscvfLEBGRfFe44J2DaRUqVDBzJY+/6OhooqKizLb9xza8AdjY2DB37lyGDRvGli1bqFu3Lt999x0vvfQSAA4ODhw7doxx48ZRu3ZtfvzxRxo2bMiNGzcybbdWrVq89tprXLlyhdmzZ9OpUydmzpzJF198gZeXFz179gTA3t6eyZMn07lzZy5cuICjoyOBgYGm8Ojg4MDRo0cZP348cOdQbuvWrfn2228BaNeuHd27d0+3/WnTpjFjxgzCwsKwsrLip59+wtvbm/Xr1zN79mwKFChAw4YN2bdvH+7u7ixcuJBGjRqxcePGXO/LaqWK5HpdERF5eGbNmmXuEh578XFxuDdqZLYA91iHNzc3N5KTk9myZQsAO3fu5NKlS1SpUoWzZ8+SkJCAv78/AHv27OH8+fNUrVqVHTt2ZNru5s2buXLlCgC7d+/m+eefz3C5OnXqULp0aRYsWJCurlOnTnH79m2WLVtmmu7v78+UKVP49ttvadiwIVeuXOHw4cNp1rW1teXll1/G2dnZNK1w4cK4ubmxevVqDh48SN26dXF3d2fmzJmMHj0aCwsLGjZsyLhx47K559K7sS+Q5BtXcr2+iIjI48DSzhG7Gl44OTkpvD0sKSkpDzQf4NatW6afk5KSsLbOeDdaWFhw9OhR3nzzzXTzXF1diYuLS7O93bt3Y2lpSc2aNfHx8WHJkiUZtgnwxhtvpKkjVUhICO7u7tSvX5/x48dz+PBh2rRpw7Vr10yHY3Mj+cYVkmJyv76IiIjkjcf2alOAiIgILC0tcXd3B+6MhDk7O3PgwAHgzmHVtm3bAlCzZk1KlChhmpcbsbGx2Nvbm17v2rWLUqVKmbYPdy5qsLGxuW8bS5YsoUePHrzyyiusWLEi3fybN28SGhpK//79TdNKlCjBM888A9wJb61ateLatWvExcUREhLC0KFD2bZtW677JSIiIo+Oxzq83b59m169ejF06FCCg4MZM2YMffr04ebNmwBcu3aNihUrEhwczNSpUxkwYECW57tlZtu2bRQoUIDg4GAmTpzItWvX6NKlCwMHDiQ4OJhNmzbx0UcfYWl5/92+bNkyWrZsSUhICNeuXctwmQEDBlC2bFk2bNjA+vXrmTt3Lo6OjgDs27ePokWLmsJaSEgIpUqVUngTERF5TFi4uLhkfZzQYIoXL87OnTtxc3PL1mFQub958+ZRunRpXC7v1WFTERF54lnZO1O0YXu8vLwIDw/P07bnzZsHQLdu3TJd7rEbeatRowa//fYbU6dOVXATERGRx85jd8HCvn37aNiwobnLEBEREckXj93Im4iIiMjjTOFNRERExEAU3kREREQMROFNRERExEAU3kREREQMROFNRERExEAU3kREREQMROFNRERExEAU3kREREQMROFNRERExEAU3kREREQMROFNRERExEAU3kREREQMxNrcBYgxWNo5mrsEERERs3sUvg8V3iRLKSkp2NXwMncZIiIij4T4uDiio6PNtn2FN8lSVFQUgwYNMncZIiIij4To6GiioqLMtn2FN8lSUlIS4eHh5i5DRERE0AULIiIiIoai8CYiIiJiIApvIiIiIgai8CYiIiJiIApvIiIiIgai8CYiIiJiIApvIiIiIgai+7xJlqysrKhataq5yxCRx5C5b3YqYkQKb5IlFxcXAgMDzV2GiDyG4uLjaeTurgAnkgMKb5IlCwsLdkTEEBuXZO5SROQxUtTWinpu9jg5OSm8ieSAwptkS2xcEldvJpq7DBERkSeeLlgQERERMRCFNxEREREDUXgTERERMRCFNxEREREDUXgTERERMRCFNxEREREDUXgTERERMRCFNxEREREDUXgTERERMRCFNxEREREDUXgTERERMRCFNxEREREDUXgTERERMRCFNxEREREDUXgTERERMZBHOrz9/fffuLq6ZrlcUFAQdnZ2AISFhVGlSpX8Li3P3a9uV1dXOnfunO12IiMjsbe3z7RNERERMa5HOrxll6enJzdu3DB3GfmiVKlSOQpvIiIi8nizNncBd/Pw8GDkyJEkJCSwefPmNPM+/fRT6tevj42NDbGxsQwfPpyIiAjgzmhTpUqViImJMS1fvXp1ZsyYQePGjU3TVq5cybRp09i0aVOatosWLcqoUaOoXbs2ycnJ7N+/nyFDhvDyyy8zfPhwChYsiI2NDXPmzGHx4sUAdOjQgT59+pCQkIClpSXDhg1j7969hIWF0bNnTw4cOADA2rVrGTduHNu3b6dPnz68+eab2NjYkJCQwKhRo9i9e3em+2TixImULFmSoKAgIiMj6d69e6b74n66d+/Om2++SY8ePYiOjs78F5GBEg42FC1kleP1RETup3DBO+MHFSpUMHMlIjkTHR1NVFSU2bb/yIS34sWL4+fnx1tvvcWxY8d45513cHJyMs2fNWsW48aNA6Bly5aMHTuWTp063be9/fv3c+XKFRo1asTWrVupUqUKxYsXTxfcAMaOHUt8fDzNmzcnJSXFtN2///6bVq1akZycTLFixQgMDGTLli2cPXuWUaNG0bhxYy5cuIC1tTUFChTIso9Lly5lzpw5ANSuXRs/P7804TIjI0aMYOzYsXh6euZqX1hYWDB69GhKlSpF+/btiY+Pz7LOjFQrVSRX64mIZGXWrFnmLkEkR+Lj4nBv1MhsAe6RCW+1a9fm0KFDHDt2DIBffvnFFFAAGjVqRI8ePbCzs8PS0pJixYpl2eb3339P9+7d2bp1K926deOnn37KcLnmzZvj7e1NSkoKgGlkytHRkS+++ILy5cuTlJSEo6MjFStW5OzZs2zbto3p06cTHBzMpk2b+Oeff7Ksp2rVqrz//vs4OjqSlJREhQoVKFSoUI4DVU72xeTJkwkPD6d3796m/uXGjX2BJN+4kuv1RUREHgeWdo7Y1fDCyclJ4e1edwcNFxcXPvvsM1q0aMGpU6eoVKkSy5Yty7KNtWvXMnLkSKpUqYKnp2eaMJgdEydOZOPGjfTu3RuAdevWUbBgQQB69+5N9erVadCgAfPnz2fy5MmsWrWKxMRELC3/dyph6vI2NjbMnTuXdu3asW/fPooUKcKRI0coUKBAjsJbTvdFWFgY7u7uPP3005w/fz5H/b9b8o0rJMVczPX6IiIikjcemQsWdu/eTaVKlXBzcwOgffv2puBjb29PYmKiKXx069YtW20mJSXx888/M2/ePNatW5fmnLi7BQUF8e6772JhYQFgOmzq4ODAmTNnAKhXrx6VK1cGwMrKirJly7J//36+/fZb1qxZQ61atQA4efIktWvXBqBmzZqm/qSeNxcZGQlAjx49stWH2NhYihYtanqd032xbNkypk2bhr+/P6VKlcrWNkVEROTR9ciMvEVHR+Pr68v3339PQkICmzZtMh2+PHz4MCtXrmTTpk1cuXKFwMDAbLf7yy+/MGLECH788cf7LjNmzBjGjBnDhg0bSExM5K+//mL48OGMHz+eCRMmMHjwYA4cOMDevXuBO+Htyy+/pFixYiQlJXH58mV8fX2BO4cpp02bRqdOndi9ezdHjx4F4Pr160yePJk1a9YQHR3NypUrs1X/oUOHOHr0KBs2bOD06dN07949x/tizZo1xMfH88svv9C1a9csL24QERGRR5eFi4tL7k+EMoAWLVrQpUsX3n77bXOXYkjz5s2jdOnSuFzeq8OmIiLyxLOyd6Zow/Z4eXkRHh6ep23PmzcPyPqo2iMz8pYfFixYQPny5enVq5e5SxERERHJE491eMvsViIiIiIiRvTIXLAgIiIiIllTeBMRERExEIU3EREREQNReBMRERExEIU3EREREQNReBMRERExEIU3EREREQNReBMRERExEIU3EREREQNReBMRERExEIU3EREREQNReBMRERExkMf6wfSSdyztHM1dgoiIiNk9Ct+HCm+SpZSUFOxqeJm7DBERkUdCfFwc0dHRZtu+wptkKSoqikGDBpm7DBERkUdCdHQ0UVFRZtu+wptkKSkpifDwcHOXISIiIuiCBRERERFDUXgTERERMRCFNxEREREDUXgTERERMRCFNxEREREDUXgTERERMRCFNxERERED0X3eJEtWVlZUrVrV3GWIPBLMfXNOERGFN8mSi4sLgYGB5i5D5JEQFx9PI3d3BTgRMRuFN8mShYUFOyJiiI1LMncpImZV1NaKem72ODk5KbyJiNkovEm2xMYlcfVmornLEBEReeLl6oKFN998Ez8/v/vOnzp1Km+88UauixIRERGRjOUqvPXu3Zvbt2/fd358fDy9e/fOdVEiIiIikrFchTc3NzfCw8PvO//gwYO4ubnluigRERERyViuwpuFhQX29vb3ne/g4ICNjU2uixIRERGRjOUqvIWHh9OqVasMA1qBAgVo1apVpiNzIiIiIpI7uQpvs2bNomLFigQEBODh4UHp0qUpXbo0Hh4eLF26lIoVKzJr1qy8rlVERETkiZerW4Vs2rSJoUOHMnbsWL7//nvTdAsLC65fv86wYcPYsGFDnhUpIiIiInfk+j5v/v7+rF27lkaNGlGmTBkATp06xZYtW7hx40aeFSgiIiIi//NAN+m9fv06a9euzataRERERCQL2QpvLi4uAKbHwaS+zooeHyMiIiKSt7IV3nbs2EFKSgpubm4kJCSYXmeldOnSD1ygiIiIiPxPtsLbkCFDSElJISEhIc1rEREREXm4shXe/P39M30tIiIiIg9Hru7zdq/ixYtTvHjxvGhKRERERDKR66tNn332WYYNG0bjxo2xtbUFIC4uji1btvDll19y5MiRPCtSRERERO7IVXh78cUXWbBgAZaWlgQGBvLPP/8Adx5Y7+npSdOmTXnnnXf4888/87RYc4uMjKRSpUrExMSYu5Rs8fPz48CBA8ydOxdfX18cHBwYPXq0ucsSERGRB5Cr8DZmzBguXbpE27Zt090OxMXFhWXLljF69GhatGiRJ0WKiIiIyB25Cm/PPfccU6ZMyfA+blFRUcyfP58hQ4Y8cHHmEBkZyVdffcUrr7xC4cKFmTp1Kr/++qtpfteuXfHy8qJ48eL4+fmZLt6oXr0648aNo3Dhwty6dYsxY8awa9cuXF1dCQoK4vvvv6d58+YULVqUUaNGsXHjRgBq1KjByJEjKVKkCFZWVsyYMYPVq1enqyt1vdq1a5OcnMz+/fsZMmQIL7/8MsOHD6dgwYLY2NgwZ84cFi9enOf7pYSDDUULWeV5uyJGUrjgndOEK1SoYOZKRMScoqOjzXov21yFt8jISAoWLHjf+QUKFDD0DXpTUlLw8vKidOnSrF27lp07d3LmzBkAbt++jbe3N25ubqxdu5Zly5ZhaWnJ3LlzGTZsGFu2bKFu3bp89913vPTSSwA4ODhw6NAhvvzyS5o0acJ///tfNm7ciL29PZMnT6Zz585cuHABR0dHAgMD2bVrF+fOnUtT09ixY4mPj6d58+akpKTg5OQEwN9//02rVq1ITk6mWLFiBAYGsmXLFs6ePZun+6RaqSJ52p6Ikc2aNcvcJYiIGcXHxeHeqJHZsk6uwpufnx9jxoxhw4YNHDhwIM28KlWq0L17d0OfW7Vo0SIATp8+zY4dO6hfvz5Lly4FYPny5QBERESQmJjI008/jYODA8nJyWzZsgWAnTt3cunSJapUqcLZs2eJi4szPUZs9+7dpmfB1qlTh9KlS7NgwYI023dzc0sX3po3b463t7fp/nrR0dEAODo68sUXX1C+fHmSkpJwdHSkYsWKeR7ebuwLJPnGlTxtU0RExGgs7Ryxq+GFk5OTscJb7dq1uXjxIr///ju7du3i5MmTAJQrV44XXniBI0eO8MILL/DCCy+Y1klJSTFsoLv7hsS3bt0y/ZycnIyVVcaHEu9e5/bt26afk5KSsLa+s9stLCw4evQob775Zq5rmzhxIhs3bqR3794ArFu3LtNR0dxKvnGFpJiLed6uiIiI5Eyuwlv37t1NP9etW5e6deummf/888/z/PPPp5lmpPD29ttvM3XqVFxdXalXr16WdUdERGBpaYm7uzshISHUqVMHZ2dnDhw4YDq8mZFdu3ZRqlQp03pwZ+Ty6NGjpqdZpAoKCuLdd99l5MiRpsOm0dHRODg4mA7p1qtXj8qVKz9g70VERORRlqvwVqpUqbyu45FiZWVFYGAghQsX5tNPPzWFo/tJSEigV69ejBs3jlGjRnHr1i369OnDzZs3Mw1v165do0uXLowaNYpRo0ZhbW1NZGQkPXv2TLfsmDFjTIeqExMT+euvvxg+fDjjx49nwoQJDB48mAMHDrB3794H7r+IiIg8uixcXFz0kNK7GO1ebvlt3rx5lC5dGpfLe3XYVEREnnhW9s4UbdgeLy8vwsPD87TtefPmAdCtW7dMl8v1Exbgzghcs2bNKFmyJHAn+GzcuJF///33QZoVERERkfvIdXgbNWoUPXv2xNIy7eNRk5OTmTt3LuPGjXvg4swhNYiKiIiIPIpyFd769u1L7969WbNmDd9++y3Hjh0D7jzvtHfv3vTu3Ztz587x3Xff5WmxIiIiIk+6XIW3jh07mq5+vNvevXt57733KFiwIJ06dVJ4ExEREcljllkvkp6rq6vphrQZ2bJlC66urrkuSkREREQylqvwdvny5UzvJ1a5cmXTEwBEREREJO9kO7zVq1fPdM+y1atX06FDB/r374+tra1pGVtbW9577z06dOjAqlWr8r5aERERkSdcts95CwgI4P3332fFihVMnjyZKlWqMGLECIYOHcr58+cBKFGiBNbW1oSGhjJlypR8K1pERETkSZXt8GZhYWH6OT4+nrfffhtPT88093nbvHkzGzZsIDg4OO8rFREREZEHu0lvUFAQQUFBeVWLiIiIiGQhRxcspKToSVoiIiIi5pSjkbcZM2YwY8aMbC2bkpJCmTJlclWUiIiIiGQsR+EtJCSEf/75J79qEREREZEs5Ci8BQQEsGLFinwqRURERESykqub9IqIiIiIeTzQ1aby5LC0czR3CSIiImb3KHwfKrxJllJSUrCr4WXuMkRERB4J8XFxZn0MaLbDW6lSpfKzDnmERUVFMWjQIHOXISIi8kiIjo4mKirKbNvXyJtkKSkpifDwcHOXISIiIuiCBRERERFDUXgTERERMRCFNxEREREDUXgTERERMRCFNxEREREDUXgTERERMRDdKkSyZGVlRdWqVc1dhkieMPf9mUREHpTCm2TJxcWFwMBAc5chkifi4uNp5O6uACcihqXwJlmysLBgR0QMsXFJ5i5F5IEUtbWinps9Tk5OCm8iYlgKb5ItsXFJXL2ZaO4yREREnni6YEFERETEQBTeRERERAxE4U1ERETEQBTeRERERAxE4U1ERETEQBTeRERERAxE4U1ERETEQBTeRERERAxE4U1ERETEQBTeRERERAxE4U1ERETEQBTeRERERAxE4U1ERETEQBTeRERERAxE4U1ERETEQBTeRERERAxE4S2PrF27lgYNGjxwO2XKlGHdunUEBgbi4+OTB5VBxYoVCQsLy5O2RERExLyszV2AEVlZWZGUlJQv7bRo0YJ9+/bx4YcfPnD7IiIi8vhReLtL48aN+eijj7CysuLatWt89NFHHDt2jAYNGvD555+zZ88eqlevzvTp0zl37hzjx4/HysqKffv2YW39v13p7OzMuHHjcHV1pVChQgQFBTF58mQAwsLCWLVqFQ0bNuTEiRMMHDjQtF7btm3p3bs3VlZW1KpVi/79+5OQkMDEiRMpXrw4KSkpfPnllwQGBmZaL4Cvry+tW7cmNjaWTZs2PfC+KWpr9cBtiJib3sci8jhQePt/xYsXZ9asWbRt25bDhw/z1ltvMWfOHJo2bQrAs88+y8cff8zQoUOxsbEhNDQUX19fQkJCaNSoEW+//baprWnTpjFjxgzCwsKwsrLip59+wtvbm9WrVwPg6OiIt7d3uhqWLl1K6dKlcXBwYPTo0QD89ttvLFmyhAULFlCuXDl+++03wsPDiY+Pv2+9r7zyCt7e3rz66qtcv36d6dOnP9C+SUlJoZ6b/QO1IfKoiIuPJzo62txliIjkmsLb/6tduzaHDh3i8OHDAPz66698/vnnPPPMMwCcOnXKdN5YhQoVSExMJCQkBICtW7dy8uRJAGxtbXn55ZdxdnY2tV24cGHc3NxMr/39/bNVk52dHdWqVaNVq1YAnDhxgj///JN69eoRGxt733pffvllfvvtN65fvw7AggULePHFF3O5ZyAqKopBgwblen2RR0l0dDRRUVHmLkNEJNcU3rLp5s2bmc5PSUkBwMLCAoA33niDW7du5aqt7Gwnv9e5W1JSEuHh4Q/UhoiIiOQNXW36/3bv3k2lSpWoWLEiAC1btuTcuXOcPXs23bLHjx/H2tqahg0bAuDu7k65cuWAO8EsNDSU/v37m5YvUaKEaQQvJ27cuMHff/9tOiRbtmxZXnzxRcLCwjKtNyQkBG9vb+zs7ADo1KlTjrctIiIijyaNvP2/6OhoBgwYwFdffWW6AKBv374ZLpuQkEC/fv0YP348lpaW7Nu3jwMHDpjmDxgwgNGjR7NhwwZSUlKIi4vjww8/zDAIZmXgwIFMnDiR7t27k5KSwtChQ02HfO5X78aNG6lZsyaBgYF5dsGCiIiIPBosXFxcHuyYmjzW5s2bB0C3bt3MWoeIiMjjLrvfuTpsKiIiImIgCm8iIiIiBqLwJiIiImIgCm8iIiIiBqLwJiIiImIgCm8iIiIiBqLwJiIiImIgCm8iIiIiBqLwJiIiImIgCm8iIiIiBqLwJiIiImIgCm8iIiIiBqLwJiIiImIgCm8iIiIiBqLwJiIiImIgCm8iIiIiBqLwJiIiImIgCm8iIiIiBqLwJiIiImIgCm8iIiIiBqLwJiIiImIgCm8iIiIiBqLwJiIiImIgCm8iIiIiBqLwJiIiImIgCm8iIiIiBqLwJiIiImIgCm8iIiIiBqLwJiIiImIgCm8iIiIiBqLwJiIiImIgCm8iIiIiBqLwJiIiImIgCm8iIiIiBqLwJiIiImIgCm8iIiIiBqLwJiIiImIgCm8iIiIiBqLwJiIiImIgCm8iIiIiBqLwJiIiImIgCm8iIiIiBqLwJiIiImIgCm8iIiIiBqLwlg1BQUHY2dlluZyPjw9ubm55vv3q1avz9ddf53m7IiIiYjwKb9ng6enJjRs3slyuXbt2VKhQIcftW1hYYGFhcd/5+/fvp1+/fjluV0RERB4/1uYuwAgiIyOpVKkSMTExhIWFsXTpUho1aoSzszOLFy/mq6++okOHDtSoUYMxY8YwZMgQJk6cyMaNG+nbty9vvPEG1tbWXLp0iQ8//JDIyEh8fX15/vnnsbOzw8XFhenTp9OqVSu6du1q2m5oaCi9evXCwcGBsWPH4unpCUDjxo0ZPHgwhQoVIikpifHjxxMaGsrMmTNZv349K1asoGvXrowZM4bKlSsTFxeHv78/X375JTt27Mhx/62srKhatWqe7U8REREji46OJioqymzbV3jLBXt7e1q2bImjoyOhoaEsWbKEX375hdatWzN37lwCAwMBaNWqFW5ubrRs2ZLk5GTatGnDhAkT6NKlCwAvvPACXl5eXLp0iUKFCvHf//4XZ2dnLl68SMOGDbl27RoHDx6kQYMGpm2XLl2aIUOG0LFjR65fv07ZsmVZvnw59evXJyQkBHd3d1asWIG7uzv79++nQYMGhIaGUrlyZXbv3p2r/rq4uJj6JCIi8qSLj4vDvVEjswU4hbdcWLFiBQBXrlzh9OnTlCpVinPnzqVb7tVXX6VGjRqsW7cOAEvLtEepN27cyKVLlwCIj49n7dq1tGnThm+++QYfHx+WLFmSrs2mTZuaAluq5ORkSpYsSUhICL6+vlhaWvLcc88xadIk3N3dSUpKYu/evSQmJuaqvxYWFtzYF0jyjSu5Wl9ERORxYWnniF0NL5ycnBTejOTWrVumn5OSkrC2zng3WlhYMHPmTBYuXJjh/Js3b6Z5vXjxYqZOncr8+fNp3rw5Y8aMyXC9rVu3MmDAgAzn3b59m9atW7N//362bdvG+++/T1JSEtu2bctGz+4v+cYVkmIuPlAbIiIi8uB0wUIeun79Ovb29qbX69ato3PnzhQrVgwAa2trqlSpct/19+7dC8CoUaMICQnh6tWr6ZbZsmUL7u7uVKpUyTStZs2app9DQkIYOnQoISEhXLt2jYSEBLy9vR84vImIiMijQeEtDy1cuJCBAwcSFBREs2bN+PXXX/H39ycgIIDg4GCCgoJ4+eWXM21jyZIldO7cOcNDpgAnT55kwIABTJo0ieDgYDZv3kyvXr1M80NCQihVqhQhISEAbNu2jcKFC3PgwIG866iIiIiYjYWLi0uKuYuQR9e8efMoXbo0Lpf36rCpiIg88azsnSnasD1eXl6Eh4fnadvz5s0DoFu3bpkup5E3EREREQNReBMRERExEIU3EREREQNReBMRERExEIU3EREREQNReBMRERExEIU3EREREQNReBMRERExEIU3EREREQNReBMRERExEIU3EREREQNReBMRERExEIU3EREREQNReBMRERExEGtzFyDGYGnnaO4SREREzO5R+D5UeJMspaSkYFfDy9xliIiIPBLi4+KIjo422/YV3iRLUVFRDBo0yNxliIiIPBKio6OJiooy2/YV3iRLSUlJhIeHm7sMERERQRcsiIiIiBiKwpuIiIiIgSi8iYiIiBiIwpuIiIiIgSi8iYiIiBiIwpuIiIiIgSi8iYiIiBiIwpuIiIiIgSi8iYiIiBiIwpuIiIiIgSi8iYiIiBiIwpuIiIiIgSi8iYiIiBiIwpuIiIiIgSi8iYiIiBiIwpuIiIiIgSi8iYiIiBiIwpuIiIiIgSi8iYiIiBiIwpuIiIiIgSi8iYiIiBiIwpuIiIiIgSi8iYiIiBiIwpuIiIiIgSi8iYiIiBiIwpuIiIiIgSi8iYiIiBiIwls2BAUFYWdnl+v1S5QowfLly02vfX19KViwYLbW9fHxwc3NzfTaw8ODMWPG5LoWERERMTaFt2zw9PTkxo0buV7//PnztG7d2vR6yJAh2Q5v7dq1o0KFCqbXwcHBCm8iIiJPMGtzF2AEkZGRVKpUiZiYGMLCwli6dCmNGjXC2dmZxYsX89VXX1GnTh0mTpxI8+bNTesFBATw3XffcfDgQYKCgqhcuTITJ04EYPny5SQnJ9OhQwfc3d3p2bMnBQoUwNLSksmTJxMcHEyHDh2oUaMGY8aMYciQIUycOJGnnnoKLy8vevbsCcC7776Lj48PycnJHDp0iI8//pjY2Fh8fX159tlnsbW1pUyZMly8eJE+ffpw9erVHPffysqKqlWr5sm+FJEnT3R0NFFRUeYuQ+SxofCWC/b29rRs2RJHR0dCQ0NZsmQJu3btomDBglSvXp39+/dTunRp3Nzc2LBhA88884xp3REjRtC5c2dat25NTEwMAFu2bGHFihUAuLq68ttvv1GvXj1++eUXWrduzdy5cwkMDATuHEZN1bRpU9q3b0/Lli2JiYlh0qRJfPzxx3z00UcA1KpVi9dee40rV64we/ZsOnXqxMyZM3PcXxcXF9P2RURyKj4uDvdGjRTgRPKIwlsupAatK1eucPr0aUqVKsW5c+dYsmQJb7/9Nvv376ddu3b8+uuvJCUlZdleqVKlmDFjBs888wxJSUkUK1aMUqVKERERkel67u7urFq1yhQC58+fz7fffmuav3nzZq5cuQLA7t27ef7553PVXwsLC27sCyT5xpVcrS8iTy5LO0fsanjh5OSk8CaSRxTecuHWrVumn5OSkrC2vrMbAwICCA4OZty4cbRt25auXbtmq73Zs2czYcIE1qxZA0B4eDiFChXKtzpzI/nGFZJiLj5wTSIiIvJgdMFCHjp//jx//fUXY8aM4dKlSxw9ejTD5WJjYylatKjpdbFixTh9+jQArVu3xtHR0TTv+vXr2NvbZ9hOSEgIb7zxBkWKFAGgU6dObN26Na+6IyIiIo8ghbc85u/vT+fOnfH397/vMt9++y2LFy8mKCiI4sWLM2rUKObMmUNgYCBVq1blzJkzpmUXLlzIwIEDCQoKolmzZmna2bRpE0uWLGHVqlWsX7+eIkWKMGHChHzrm4iIiJifhYuLS4q5i5BH17x58yhdujQul/fqsKmI5JiVvTNFG7bHy8uL8PBwc5cj8kibN28eAN26dct0OY28iYiIiBiIwpuIiIiIgSi8iYiIiBiIwpuIiIiIgSi8iYiIiBiIwpuIiIiIgSi8iYiIiBiIwpuIiIiIgSi8iYiIiBiIwpuIiIiIgSi8iYiIiBiIwpuIiIiIgSi8iYiIiBiIwpuIiIiIgVibuwAxBks7R3OXICIGpM8Okbyn8CZZSklJwa6Gl7nLEBGDio+LIzo62txliDw2FN4kS1FRUQwaNMjcZYiIQUVHRxMVFWXuMkQeGwpvkqWkpCTCw8PNXYaIiIigCxZEREREDEUjb5IpJycnrKysmDdvnrlLEREReaw5OzuTlJSU5XIKb5KphIQEc5cgIiLyREhKSsrW966Fi4tLykOoR0RERETygM55ExERETEQhTcRERERA1F4ExERETEQhTcRERERA1F4ExERETEQhTcRERERA1F4ExERETEQhTcRERERA1F4ExERETEQhTcRERERA1F4ExERETEQhTcRERERA1F4e4KUK1eOlStXEhISwpo1a3juuecyXK59+/Zs27aNP/74g8mTJ2Ntbf3A8x6W/OzjSy+9xOrVq9m0aRMbN25k5MiRWFhYPJR+3S2/f4+p/P39OXjwYL71437yu3/PP/88AQEBbN68mc2bN/Paa6/le5/ulZ99tLCwYPTo0WzatIng4GACAgIoW7bsw+hWGg/aR1dXVwICAjh06BBBQUHZXu9hyc/+PS6fNVn9DlOZ67MG8r+P+fV5o/D2BJk0aRILFy7E3d2d2bNn4+fnl26ZUqVKMWzYMN566y1eeuklnJ2deeeddx5o3sOUn328evUq7733Hk2bNuW1116jTp06tGvX7qH2D/K3j6n69OnDyZMnH0Z30snP/hUqVIgffviByZMn06RJE5o1a8aOHTseav8gf/vo6elJnTp18PDwwMPDg23btjFixIiH2j948D5ev36dyZMnM2DAgByt97DkZ/8el8+azPqYypyfNZC/fczPzxuFtydE8eLFqV69OsuWLQNgzZo1uLi4pPsXeYsWLQgODubixYsA/Pzzz7Rq1eqB5j0s+d3HAwcOcPr0aQBu3brFgQMHcHV1zf+O3SW/+wjw3HPP4eXlxaxZs/K9P/fK7/699dZb7Nmzh507dwKQnJxMdHR0/nfsLvndx5SUFAoWLEjBggUBKFKkCGfPns3/jt0lL/p49epVdu7cyc2bN9O1b+7Pm/zu3+PyWZNZH8G8nzWQ/33Mz88bhbcnhIuLCxcuXCApKck0LTIykpIlS6ZZrmTJkpw5c8b0+t9//zUtk9t5D0t+9/Fuzs7OtGjRgvXr1+d1NzKV3320trZmypQpfPjhh2m28bDkd/+ee+45bt++zU8//URQUBBfffUVTk5O+dmldPK7j8HBwWzfvp2//vqLvXv38vLLLzNlypT87FI6edHHzJj78ya/+3c3I3/WZMbcnzWQ/33Mz88bhTeRHCpSpAjz5s3j66+/Zv/+/eYuJ0/5+vqydu1ajh8/bu5S8oWVlRXu7u58+OGHeHp6cvbsWSZOnGjusvJUjRo1qFixIi+88AK1a9dm27Ztj10fnxT6rDG2/Py8UXh7QkRFRfH0009jZWVlmlayZEkiIyPTLBcZGZlmeL5UqVKmZXI772HJ7z4C2NnZsXDhQoKCgpgzZ05+deW+8ruP9evXp0ePHoSFhbFixQqKFi1KWFjYQxudehjv09DQUM6dOwfA8uXLqV27dr71JyP53ce2bdvyxx9/EBMTQ0pKCgEBATRs2DA/u5ROXvQxM+b+vMnv/sHj8VmTGXN/1sDDeZ/m1+eNwtsT4vLly/z999+0adMGuHMM/+zZs+lOFF27di0eHh44OzsD0LlzZ1auXPlA8x6W/O5j4cKFWbhwIZs2beKrr756SL1KK7/72Lp1a+rVq0f9+vVp1aoVsbGx1K9f/6GdF5bf/fvtt9+oUaMGRYoUAaBZs2YP/Sq3/O7j6dOneemll7CxsQGgefPmHDly5GF0zSQv+pgZc3/e5Hf/HpfPmsyY+7MG8r+P+fl5Y+Hi4pKSJy3JI8/NzQ0/Pz8cHR2JjY3F19eXw4cPM2XKFIKCgggODgagY8eO9O/fH4Dt27czYsQIEhMTH2je49DH999/H19fX44ePWra3urVq5k+ffpj08e7ubq6EhQUROXKlR9i7/K/f23atOG9994jOTmZc+fO8eGHHxIVFfXY9LFAgQJ89tlnvPjiiyQkJHDx4kVGjBhhOgHeKH0sVKgQ27Zto0CBAhQtWpTLly+zdOlS02Enc3/e5Gf/HpfPmqx+h6nM9VnzMPqYX583Cm8iIiIiBqLDpiIiIiIGovAmIiIiYiAKbyIiIiIGovAmIiIiYiAKbyIiIiIGovAmIiIiYiAKbyIiIiIGovAmIpJHAgICCAgIML12dXUlMjISHx+fh1qHn58fYWFhD3WbIvLwKLyJyEPj4+NDZGQkERER/Oc//0k3PyAggA0bNpihsidTQEAAkZGRGf7n5uZm7vJE5D6szV2AiDx5ChUqRP/+/fn000/NXUq+OnPmDOXLlychIcHcpdxXVFQUEyZMSDf9/PnzZqhGRLJD4U1EHrrw8HA6duzIzJkz8zUkFCpUiPj4+HxrPztu3bpl1u1nJSYmhuXLl2d7eVtbW+Li4vKxIhHJig6bishDN2PGDKysrEwPes6MlZUVgwcP5o8//uCff/4hLCyMESNGUKBAgTTLhYWF8dNPP9G4cWPWrl1LREQEnTp1okGDBkRGRvLGG2/wwQcfsGvXLo4cOcKcOXMoWrQoBQoUYOzYsezbt4+jR48yderUdG37+Pjg7+/Pvn37+Oeff9i0aRNdunTJsvZ7z3lLrSWj/+49R61p06YsX76cY8eOceTIEebPn89zzz2XbhteXl5s2LCBiIgINmzYwKuvvpplXdnl5+fH0aNHKVOmDPPnz+fIkSPMnDkTAAsLC3r16sXGjRuJiIjgr7/+YtKkSTg4OKRrZ9CgQezatYvjx48TEBDAc889R1hYGH5+fqZlfH19iYyMTLdu6qF2V1fXNNOzs39S6//Pf/7D999/z9GjR9m/fz+ffvoplpZpv/4sLCzo2bMn69evJyIigv3797NgwQKqV68OwNKlS00PKb/X1q1bWbhwYTb2qEje0MibiDx0p0+fZunSpXTs2JFZs2ZlOvr2xRdf4OPjw+rVq5kzZw61atVi4MCBVKhQgV69eqVZ1s3NjVmzZrFgwQIWLVpERESEad6AAQOIj49n1qxZlC1blh49epCQkEBycjIODg5MnTqV2rVr8/bbb3P69GmmTZtmWrdLly4cPXqUoKAgEhMT8fDwYMKECVhYWPDTTz9lu9/Hjh1j4MCBaabZ29szevRoLl26ZJrWpk0bpk2bxubNm/n888+xtbWlS5cu/Prrr3h5eXHmzBkAGjVqxHfffcfRo0eZOHEijo6OTJ06lbNnz2a7JisrKxwdHdNMu3XrFjdv3jTNX7hwITt37mTcuHGmUbdJkybh4+PDkiVL+OGHHyhVqhTdu3enSpUqtGrVisTERACGDRvG4MGD2bBhAxs2bKBatWosWrQoXUDOiezuHwBLS0sWLlzI3r17GTduHO7u7rz77rucOnWK+fPnm5b78ssvefvtt9mwYQO//PIL1tbWvPjii9SuXZv9+/ezbNkyvvjiCypWrMiRI0dM69WoUQM3Nze++uqrXPdHJKcU3kTELKZPn07btm157733GD16dIbLVK5cGR8fHxYuXMjw4cMB+Omnn7h06RL9+vWjYcOGhIaGmpYvV64cHTt2ZMuWLaZpDRo0AO6EkDZt2phCRfHixXnzzTfTjKL99NNPlC1blvbt26cJb23btk1z+HXevHksWLCAPn365Ci8Xbp0Kd0hynnz5nH79m0++OADAAoXLsx///tfFi1axIcffmhaLiAggK1btzJw4EDT9JEjR3Lx4kXeeustYmNjAdi+fTuLFy/m33//zVZNzz77LOHh4Wmm+fv7m+opVKgQq1evZuLEiab5devW5Z133qF///6sWLHCND00NJRFixbh7e3NihUrcHJyol+/fqxfv56uXbualvvwww95//33s1XfvXKyf+DOYd7ffvvN9Pv8+eefWbduHe3btzeFt4YNG/L2228zd+7cNO/Fb7/91vTz6tWrGTduHK1bt05zjmDr1q25ceMGa9euzVV/RHJDh01FxCxOnz7NsmXLeOedd3j66aczXKZZs2YAzJkzJ8301C/VV155Jc30U6dOpQlud1u6dKkpuAHs3bsXS0tLlixZkma5vXv34uLigpWVlWna3cGtaNGiODo6EhYWRtmyZSlatGhWXb2vwYMH4+HhwQcffMCxY8eAO6NpxYoVY+XKlTg6Opr+S0pKYu/evbz00ksAPP3001StWpWAgABTcAMICQlJMzKUldOnT9O+ffs0/82ePTvNMnePUAF4e3tz7do1tm7dmqbG/fv3c/36dRo2bAiAu7s7BQsW5Icffkiz/nfffZf9nXSP7O6fzOrfsWMHpUuXNr1+/fXXSU5OTnMY916xsbEEBQXRqlUr0zRLS0tatmzJunXrdB6gPFQaeRMRs/nqq69o06YN/fv3z3D0zdXVlaSkJE6ePJlm+sWLF7l69Wq686AyG22KiopK8zo18Nw7PSYmBisrK+zt7bly5QoAderUYejQobzwwgsULlw4zfJFixZNE56yq0mTJvj6+jJjxow0ozblypUDSHO/uHvrA0x9P3HiRLplIiIiqFatWrbquHnzJiEhIfedn5CQkO4wbLly5XBwcODvv//OcJ2nnnoq0xqjo6NN+zansrt/UsXFxREdHZ1m2rVr19IcKi5Tpgznz5/n6tWrmW576dKlvPnmm9SrV48dO3bg7u7O008/zbJly3LRE5HcU3gTEbM5ffo0y5cv55133mHWrFn3XS4lJSVb7WV2ZWlSUlKOpqcqU6YMS5YsISIigrFjxxIVFUVCQgLNmjWjT58+6U58z45SpUoxc+ZMtm7dyqRJk9LMS21v4MCBXLx4Md26d48ePgy3b99Ot/8tLS25ePFiuvP3Ul2+fDnH27nf7/je/ZvT/ZOcnJzjWu5n8+bNXLhwgdatW7Njxw7atGnD+fPnMw2/IvlB4U1EzOqrr76idevWGV55eubMGaysrChXrhzHjx83TX/qqacoVqxYmhPT84uHhweFChWiW7duaUbpUg8N5lShQoWYO3cu165do3///ulCy6lTp4A758dlFgpS+546EnW3/L7B7qlTp3B3d2fnzp2ZBua7azx9+rRpupOTU7qLJK5duwbcuYDj7tGze0dXs7t/cuLUqVM0adKEYsWKZTr6lpyczIoVK2jXrh3jx4/Hy8uLRYsW5WlAFMkOnfMmImZ16tQp0+jbvee+bdy4EYDevXunmd6nTx+Ah/I0htSROQsLC9O0okWL5vqRVxMnTqR8+fL06tXLFFjutnnzZmJiYhg4cCDW1un/fe3k5ATAhQsXCA8Pp127dmnOu3N3d6dixYq5qi27fvvtN6ytrRk8eHC6eamHnOHO+Xe3b9+mR48eaZa59/cJ/wtl9erVM02ztbWlXbt2aZbL7v7JibVr12JpaWm6SCMzS5cuxdHRkUmTJlGkSBEdMhWz0MibiJjd9OnTadOmDRUqVODw4cOm6QcPHsTf359OnTphb29PWFgYNWvWxMfHh99//z3Nlab5ZevWrdy6dct0hamdnR0dO3bk8uXLGT7iKzOvvPIK7dq1Y82aNVSqVIlKlSqZ5t24cYPAwECuX7/ORx99xPTp01m3bh2rVq3i8uXLlCxZkldeeYWdO3fyySefADBhwgTmz5/Pr7/+ypIlSyhWrBjdu3fn8OHD2NnZ5el+uFtYWBg///wzAwcOpHLlymzdupWEhATKly9PixYtGD16NGvWrCE6Oppvv/2WgQMHMn/+fDZs2EDVqlVp2rRpukOrW7Zs4cyZM3z55Zd8/fXXJCcn8/bbb3P58uU0o2852T/ZFRoaytKlS+nVqxflypVj8+bNWFpa8uKLLxIaGsq8efNMyx44cIBDhw7xxhtvcPTo0XRX6oo8DApvImJ2J0+eZPny5RmOZg0dOpRTp07h4+PDq6++ysWLF5kxYwZTp059KLVFRETQt29fhg8fzqeffsrFixeZP38+ly9fzvTqxIwUL14cgBYtWtCiRYs08/79918CAwMBWLFiBefPn6d///68++67FChQgHPnzvHnn3+muTp28+bNptpGjBjBqVOn8PX1xcvLy3SLlPwyYsQI9u/fT6dOnRgxYgSJiYn8+++/LF++nJ07d5qWmzRpEvHx8XTu3JmGDRuyd+9eOnbsmO4K0MTERHr27MmECRMYNmwYFy9eNB1evnc/Z3f/5MQHH3zAwYMH6dChA5988gmxsbHs27ePXbt2pVt26dKlfPrppxp1E7OxcHFxyd6ZwCIiInkkLCyM7du3Z+tQ5aOmZ8+ejBkzhnr16qW7WlnkYdA5byIiIjnQvn17wsLCFNzEbHTYVEREJAu2trZ4enrSsGFDKleuTLdu3cxdkjzBFN5ERESyULx4cWbPns3Vq1eZPn36fR9SL/Iw6Jw3EREREQPROW8iIiIiBqLwJiIiImIgCm8iIiIiBqLwJiIiImIgCm8iIiIiBqLwJiIiImIgCm8iIiIiBqLwJiIiImIg/wdahu3HQUso7wAAAABJRU5ErkJggg==",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "#to enhance training, consider rolling the shuffle dice until you get good consistency between training and test set categories.\n",
    "visualize_train_test_category_splits(train_df, test_df, top_x=5, normalize=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "3b8f0497-7b16-406e-b3dc-3f0c265fdc69",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAk8AAAHLCAYAAADRDnw9AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/SrBM8AAAACXBIWXMAAA9hAAAPYQGoP6dpAABVNUlEQVR4nO3deXgUVaL+8W91Zw8JaULYw3JlREXB4aoswqiDKBIBlUVRwOWHC4Ogoiw6LnNFRVCEQdHhijPKIpdVEZAdRRgWUZRNREARSNgCIfvSS/3+IOkhK2m6052G9/M8eejUqa461V10vznn1CmjQYMGJiIiIiJSKZZAV0BEREQkmCg8iYiIiHhA4UlERETEAwpPIiIiIh5QeBIRERHxgMKTiIiIiAcUnkREREQ8oPAkIiIi4gGFJxEREREPKDyJ+MjEiRNJTk6mUaNGga6KVKBnz54sX76cvXv3kpyczP/8z/8Eukrigb59+5KcnEzfvn0DXRW5hCk8iZyjUaNGJCcnM3PmzHLXad++PcnJybz55ps+3+/EiRN9tk0p7b//+7957733qFGjBtOnT2fChAl89dVXZa47fPhwkpOTK/0zfPjwKqlzcnIy8+bNu6DnrVu3rgpqVPUu9JhF/CUk0BUQuViMHTuW9957j2PHjgW6KlKOzp07Y7FYePrpp/nuu+8qXHfTpk1MmDCh2LKWLVvStWtXNm7cyKZNm0qtLyKXBoUnER85ceIEJ06cCHQ1pAJ169YFqFTA3bRpU6lA1LdvX7p27cqmTZt45513qqSOIlL9qdtOxEfKG/PUrVs35s+fz/bt2zlw4ADff/89//d//0e3bt2As1/IW7ZscT8+tyuoffv27u1ERkby7LPPsm7dOg4cOMCuXbuYPn061113XZn1sdlsjBs3ju3bt7N//36WLl1K165dyxwzcm63YfPmzZk2bRq7du0qdjxdu3ZlypQpbNiwgf3797Nnzx4WLlzoPo5zldzeJ598wk8//cTu3buZMmUKNpsNONuNNmfOHH7++Wd2797NW2+9RWRkpEev+3XXXcf06dPZtWsXBw4cYN26dTz77LNERES41ynqar3vvvsA2LJli/s19tUYtSuvvJL333+fbdu28dtvv7FlyxbGjBnjPlY4+55899137N27l6ZNmxZ7fsmyojoDdOjQodh54evxPqGhoTz22GMsX76cffv2sXfvXhYuXEiXLl1KrVt0nicmJvLII4+wbt06fv31V7Zs2cIzzzyDYRilnhMREcFf//pXtm7dyoEDB1izZg3333+/+xiLujw9PeY//elPLFq0iP3797Nr1y4mTZpU7PUu0qFDB2bMmMH333/Pr7/+yo8//sjChQt54IEHvH3p5BKllieRKjRw4EDGjh3LsWPHWL58OWlpaSQkJHDttdfStWtXvvzyS3bv3s20adMYNGgQu3fvZvny5e7nHz58GIDw8HDmzp1LmzZt2LFjB9OmTSMhIYEePXpw0003MWTIEJYsWeJ+XlRUFAsWLKBFixZs3bqVzZs306BBA95///0Kx8E0bdqUxYsX8/PPPzN37lxsNht2ux2A559/noKCArZu3crx48eJj4/ntttu48MPP+TFF1/kX//6V6ntJSYmsmjRInbs2MHs2bNp1aoVd911Fw0aNOCNN95g9uzZfPPNN8yaNYv27dtz//33Y7FYePbZZyv1+t55551MmTKFgoICvvjiC1JTU7npppsYPnw4N998M7179yY/P5/Dhw8zYcIEunbtSsuWLZk2bRrp6ekAZGRkVGpfFenSpQv/+Mc/ME2TFStWkJKSwuWXX84jjzzCzTffzJ133kl6ejppaWk89dRTzJ49m/fee4+77roLh8MBwNtvv039+vV5+umnOXjwIA6HgwkTJvDss89y+PBh5s6d697f7t27va5zkbCwMGbNmkWHDh3YtWsXs2fPJjQ0lM6dO/Pxxx/z17/+lY8//rjU81566SXatWvH6tWr+frrr+natSvPPfccYWFhjBs3zr2exWJh+vTp3Hjjjfz000989tlnxMXF8fLLL5dq2St6nypzzF26dKFz586sXr2a7777jrZt29KnTx+aNGnC3Xff7V6v6DjS09NZuXIlJ06coFatWlx11VX06tWLWbNm+eiVlEuJwpNIGZo2bVruAODExMRKb6dfv37k5+dz2223cerUqWJlRX8h7969mw8//NAdnsrqDho8eDBt2rRhwYIFDBs2zL38o48+YvHixYwbN46vvvqK7OxsAIYMGUKLFi2YOXMmo0aNcq8/d+5c5syZU259b7jhBt55551SY30ABgwYwKFDh4ote+WVV1i0aBEjRoxg9uzZ5OXlFStv3749L7/8Mh999JF72fTp091faH/5y19YuXIlACEhISxbtoxevXoxduxYUlNTy60nQI0aNRg/fjwOh4MePXqwZ88eAN58802mTJlCz549GTx4MJMmTeLIkSO88847JCYm0rJlSz788EOOHDlS4fYry2azMXnyZNLS0ujZs6e75QSgR48efPDBBzz33HO89NJLAPz73//m/fffZ+jQoYwcOZI33niDBx98kK5du/LZZ5+5B0oX1bkoSFRVN+EzzzxDhw4dmDhxIm+//bZ7eXR0NHPnzuXll19m2bJlHD9+vNjzrr76am699VZ3V/WkSZPYsGEDDz/8MO+88447dPft25cbb7yRNWvW8NBDD+FyuQD48MMPWbZsWbFtenLMXbp0oXfv3u6xaxaLhTlz5tChQwfatGnDtm3bALj33nuxWCz07duXn376qdg2ymqlEqkMdduJlKFZs2Y8++yzZf542mXicDjcrQvnSktLq/Q2+vTpQ0FBAWPHji22fPfu3cybN4+4uDi6du3qXn7PPfeQn5/PW2+9VWz9DRs28PXXX5e7n+PHjzN58uQyy0oGJ4CcnBzmzZtHzZo1ufbaa0uV//bbb8WCE8CiRYvcdS8KTnD2dVq6dCmhoaFcfvnl5daxyO23307NmjWZM2eOOzgBmKbJa6+9ht1up0+fPufdjrd69+5NbGwsY8eOLRacAL744gt27NhBz549iy1/++23+eGHH3jiiSd4+OGHeemllzh06BCjR4+u8vqeyzAMBgwYwG+//VYsOAFkZ2czadIkwsPDueOOO0o9d9KkScXG+KWlpbFy5UpiYmK47LLL3Mt79eoFwLhx49zBCWDfvn0sWLDgguv++eefFxv073K53MGzdevWpdYvGeyL6ixyIdTyJFKGr776iv79+5dZ1r59e+bPn1+p7SxatIiXXnqJNWvW8Pnnn7Nx40a+/fZbsrKyKl2XGjVq0LRpU3755ReOHj1aqnzjxo3079+fli1bsmDBAmrUqEHjxo3Zu3dvma03W7du5eabby5zXz/99JO7xaCk+Ph4nnzySW655RYaNWpUamxS0WDsc50baooUfeGW1fVU1LpR1rZKuvrqq4Gyr3JLSUnh0KFDXHbZZURHR7tb5KpCmzZtAPjjH/9IkyZNSpWHh4cTHx+PzWZzf1k7HA6GDBnCypUree2113A4HAwdOtSj88IXLrvsMmw2G8ePHy+zpTU+Ph6A5s2blyrbuXNnqWVF52dsbKx72ZVXXkl2dnaZ7/fWrVvL/X92Pjt27Ch3/zVr1nQv++KLL0hKSuKLL77g888/Z8OGDWzZskXBSbyi8CRShf7xj3+QlpbGwIEDefzxxxk8eDB2u501a9bwt7/9zT2mqSIxMTEA5XZjFYWRGjVqFFu/ZDdhkYq6w8ori4uL48svv6RRo0Z8++23rF+/noyMDJxOp/vy/fDw8FLPKysMFLXCZWZmlipzOp3A2QHM51N0vCdPniyz/Pjx41x22WXExMRUaXiKi4sD4OGHH65wvaioqGJf2L///js//fQTN9xwAzt27Djv1AlVoajuV1xxBVdccUW560VFRZVaVtb7V/TeWq1W97KYmBhSUlLK3G55711lVHRuWSz/6VRZsmQJDz/8MI899hgDBgzg4YcfxuVysXHjRl599VWfjh+TS4fCk0gVmzNnDnPmzMFms3HDDTdw11130aNHD5o1a8att95arCujLEVfUrVr1y6zPCEhAfjPl0nR+kWtBiWVtx042+VVlvvuu49GjRoxfvx4/v73vxcrGzJkSLEuQ38pOt6i4y+pTp06QNlf8r5UtP0///nP7N27t9LPe/zxx7nhhhs4ffo0bdq04cEHH+STTz6pqmqWqeg1XLp0KY899liV7CMzM7Pcc7G8987XVq5cycqVK4mOjub666/njjvuoF+/fsycOZObbrrJJxcNyKVFY55E/CQtLY0VK1YwePBgNmzYQIsWLWjWrBmAO0Cd+xd7kaysLA4ePEjTpk2pV69eqfIOHToA/+kGy8rK4tChQzRt2rTML63ypjaoSNFl9StWrChV1rZtW4+35wu7du0CKDadQ5EGDRrQpEkTDh48WKWtTgA//PADcHbahcpq2bIlo0aNYv/+/XTu3Jnff/+dl156qcyxXk6ns8zzwhf27dtHRkYGrVq1IiSkav6W3rNnD9HR0bRs2bJUWXnnYlUdc3Z2Nl9//TWjRo1i7ty51KlThz/+8Y8+349c/BSeRKpQWV/sISEh7u6SokGsZ86cweVyUb9+/TK3M2/ePMLCwnj++eeLLb/yyivp06cP6enpxaY4+OyzzwgPD+e5554rVZ9bbrnF4+MoujLthhtuKLb8rrvuonPnzh5vzxdWrFhBeno6ffv2LRU6XnjhBUJDQ/1yi485c+aQmZnJqFGjygw/ERER7nFRcHa+rvfffx+Av/zlL5w4cYInn3ySkJAQpkyZUqr788yZM+WeF95yOp1Mnz6dxMREXnrppTIDVIsWLcptOaqMhQsXAjBy5Mhic0Bddtll9O7du8zn+PKY27ZtW6wbr0hRC2x+fr5P9iOXFnXbiVShjz76iKysLLZt28aRI0cIDQ2lU6dOtGjRgiVLlrivzsrJyeHHH3+kXbt2TJ48mV9//RXTNJk/fz7Jycl88MEHdO7cmd69e9O8eXM2bNhA7dq16dGjByEhITz11FPFWlimTJlCt27dGDhwIC1atODbb7+lfv363HnnnaxcuZLbbrvtvN2F51qwYAF/+ctfGDNmDB06dODIkSNcddVVdOzYkaVLl5KUlOTz1+58srKyGDlyJFOmTGHJkiV88cUXnDp1ik6dOtG6dWu2bdvGBx98UOX1OH36NEOGDGHq1KmsWrWKr7/+mv379xMWFkZiYiLt2rXju+++cw+MfvXVV2nevHmx8Tbbtm1j4sSJjBw5khdffNE9rQGcndqgR48efPTRR+zatQuXy8XKlSvLHIxfUp06dcq9X+Lp06cZM2YMEyZM4JprrmHQoEF07tyZLVu2kJqaSv369bniiito2bIl3bt3L3cM3fnMmTOHXr16ceutt7Jy5UrWrl1LXFwcPXv2ZP369WWei94cc0ljxoyhbt26fPvttxw5cgTTNLn++utp06YN33//Pd9+++0FHZdc2hSeRKrQm2++yc0338y1117LrbfeSm5uLgcPHmT06NHMnj272LpPPfUUf/vb3+jcuTN33303FouFb7/9luTkZPLz8+nbty9DhgyhR48ePProo+Tm5rJp0ybeffddtm7dWmxb2dnZ3HPPPTz//PPcfvvttGrVil9++YUhQ4bQpEkTbrvtNo/GAh09epTevXvz17/+lY4dOxISEsLOnTu5//77adCgQUDCE5wdDFzUcnPHHXcQGRnJkSNHmDhxIlOmTPFbq8KaNWu4/fbbeeKJJ+jUqROdOnUiJyeHo0ePMmfOHHfrS7du3bj//vtZt24dU6dOLbaNyZMn06lTJ/es3atXrwbg5ZdfBuDGG2+kS5cuWK1Wjh49WqkgERsbW+7UGocPH2bMmDEUFBTQv39/+vXrR+/evenWrRthYWGkpqbyyy+/MGPGjAsKLUVcLhcDBgzgueeeo2fPngwaNIjff/+dV199lTNnznDbbbeVGvztzTGX9N5773HHHXdwzTXXcPPNN2O32zly5AivvfYan3zyiUd/RIgUMRo0aFD2CFERuShNnjyZXr16cdNNN7F///5AV0cuYSNHjuSpp56if//+fPXVV4GujkilacyTyEWq6Gqzc7Vr146ePXuyf/9+BSfxm7LOxT/84Q888sgjnDlzpsy5ukSqM3XbiVykZsyYQV5eHrt37yYnJ4fLL7+cm2++GafTyYsvvhjo6sklZOzYsSQmJvLDDz+Qnp5O06ZNufXWWwkNDeXZZ58tc/ZvkepM3XYiF6lBgwZx991306RJE2rUqEFGRgZbt27lvffec19eL+IPd999NwMGDOAPf/iDe9LS7du3M3Xq1ApvVC1SXSk8iYiIiHhAY55EREREPKDwJCIiIuIBDRivpAULFhAaGsrp06cDXRURERGppFq1amG32+nVq5fPtqnwVEmhoaFVdn8pEU8ZQKLTCcBhqxUNXBQRKVtVfHcrPFVSUYvTQw89FNiKiACRLhf7jx0DIKlePXLLuHeXiIjAxx9/7PNt6hNXRERExAMKTyIiIiIeULedSBByGgZzIyPdj0VExH8UnkSCUIFh8IzNFuhqiIhcktRtJyIiIuIBtTyJBCPTJNI8O0FBrmGAuu5ERPxGLU8iQSjSNNl/7Bj7jx1zhygREfEPhScRERERDyg8iYiIiHhA4UlERETEAwpPIiIiIh5QeBIRERHxgMKTiIiIiAc0z5NIEHIZBksiItyPRUTEfxSexC+cTifmOfMRGYaB1WoNYI2CW75h8HitWoGuhojIJUnddlLlnE4n+XYnZkiE+yff7sTpdAa6aiIiIh5Ty5NUOdM0CY2I4tbejxNrq01GWiqr50/FdOQFumoiIiIeU3gSv4m11caWUD/Q1bgoRLpc7D92DIDm9eqRa1EjsoiIv+gTV0RERMQDCk8iIiIiHlB4EhEREfGAwpOIeG1eaiprTpwIdDWCTvv8fJJTUmifn+9eNjEtjc3HjwewViJyPhowLnKRqu108kJGBp3z84l2udgfGsp7NWqwJDKyzPV75OYyKCuLKx0O7MC+kBDGx8by7/BwAOo6nTyQnc2KyEh2h4b6tK6bjx8n8ZypK1ItFg6EhPC/0dEsL6e+weTB7GxyDYO5UVGBrorbtQUF9M3J4Y92O1fa7YQCDRs0KHNdT8+lIrNTU/lTQQH/iorixbi4UuX3ZWfzRHY2iQ4HR61WPoqO5l81avjg6ESqlsKTyEWohsvF56mp1Ha5+Cg6mhNWK91zc5malsYQ0+TzEl/iwzMyeCYri6UREcyNiiIEuMJup945gaau08mzWVkcCQnxeXgC2BUSwtTCL866Tif9c3L4KC2N0S4XM6Kjfb4/fxqYnc1pi6VUeNocFsZ/1a9PQQDq9Oe8PPrl5LAnNJRDViuXlTPvmqfnUpE7cnP5b7u93P33z85mXHo6SyMi+N/oaNoWFPBaRgaRpsn7MTE+OUaRqqLwJBKEXIbBmsIWobJuz9I/J4dmTid94+PdLUfTo6JYnJrKyxkZLI2MxF74vDYFBTyTlcWrsbF8GMC/+o9ZrSw854t4flQU/z5xgkezsrwOT+GmSQFgVrNb2ZiGQf75V6sS06OjeT8mhjzD4LUzZ7gsJ6fM9Tw5l4qEmyYvZ2Twfo0ajMjMLLXNCNNkVGYmq8PDeaxwpvxPo6MxgKezspgVHU26pt+Qakxnp0gQyjcMBsbHMzA+nvwyAkHb/HxSLRb3lx2c/aJeHBlJXZeLdueMsRmUlcUJi4Vp0dFgmkS5XKW21z4/n2WpqQBMPHOG5JQUklNS6FviC/cPdjvzUlPZf/Qo3x07xuAyvjgr66TVyr6QkGLdefWcTiakpfHjsWP8mpLC2hMnuLdEHYrGEfXIzWVkRgbfHTvG/qNHiSm8PdAfCwqYfuoUu48eZd/Ro6w6cYL/l5VVbBuX2e387+nT7Dp6lAMpKXx58iRd8opP6to3J4fklBSuy8/nlfR0dhw7xr6jR5l2+jS1zqnz5uPHucLhoENBgft1m1f4WpY15qkshmkyKCuLtSdOcCAlhR+PHWPcmTPULPFexbhcXGa3E1PGe1hSqtVKXiXCpCfnUpHBWVlYgH+UE8Y75OdTy+XikxKh+JPoaKJNk855mkBXqje1PIlchMKgzC/GomWt7HbWF95YuGNBAd+FhfH/srN5KiuLWi4Xxy0WJsfE8HHhl9u+kBDeiolhRGYmM6Oi2BIWBsB3hf8C1HS5mHX6NMsiIlgcGUlSbi4vZmbyc2goXxXuyxMhpkkDp5O0whaI2k4ni0+exDQMPo6O5pTFwi35+bxz5gwxLhfTSnxRP52ZiR2YWqMGYYUtT53y8vjk9GlOFI6vOWm10tzh4Na8PD4qfP7ldjufp6ZyzGplSo0a5FgsdM/N5Z+nT/OozVZqDNZrGRmkGwbvxMSQ6HAwKDub14HBhS0qr8TG8lp6OtkWC5ML93HSw1aVcenp9M3JYU5UFP+MjibR6eTh7Gxa2u3cVbs2jsL39Y68PCaeOcMzcXE+G1/lybkE0MDh4MmsLIbHxZUbzq4u7M7bXqL7d0doKM7C8oU+qb1I1VB4ErkIHQgJoVN+Pg0dDpJD/vPf/IaCs6Nr6hW2TNR0uYh3ubi+oIAb8/N5JyaGFKuVvjk5vJ6ejgOYGR1NqtXK2vBwRmRm8n1YWLHutSL1XS6GxcWxoLBsdlQUW44fp19OTqXCUwhgK2yxqedy8WRWFnUKx9kAjMrMxALcmpDgDlQzoqOZkpbG8MxMZkZHF/uyDjdNutWp415mMU3GpadzwmrltoQEMs4NMOfctPrV9HSSrVaSEhIoKHzuJ1FRfJ6ayl8zMkqFpzTDoF98PBTtB3gkO5sYl4tMi4UVkZGMzMzktMVS5ut2Ptfn5/NATg5D4uKKjS/aGBbGp6dPc2dubrnjjnyhsudSkVcyMtgVEsIXFQwmr+N04gBOlbg5uN0wSLNYqFuJljORQFK3nUgQinS52FfY7RRZxhfN7KgonMA/0tK4rqCAJg4HT2Zm0jU3Fzg75gQguvDfWi4XI+LimFqjBosjIxlYqxZ7Q0J4yoNutyzDYME5X5h2w+DHsDAaV/IG0Dfn57Pr+HF2HT/O6pMnuTM3l/mRkbwRGwumSbfcXFZHRIBpYnM63T/rwsOpaZpcXVB82PW8qKhiYepqu50mTifToqOLBydwB584l4sbCwpYEhlJtMv1n/24XHwdEcF/OZ3FBtEDzIqOdj8fYEtYGCFAIx/d+PrOvDzSDYNvwsOLHfeO0FCyDIMO5xz33KgoGjZo4NOr+ip7LsHZ7rhueXm8UrNmhduMgFLjpIrkG0axbYpUR2p5EglSURV8wewJDeVJm40309NZVDi+5rjFwt9q1uTN9HRyCr+4cgvXLwCWnNM6ZBoGX0RGMiIzkwYOBykh5/+oOGq1FgsRAOmGwZWVbEXYFhrK+NhYTCDXMNgXEuIOOfFOJ3GmSf+cHPqXM7C5don9HC7RqtG0MMzsreBKwaYOBxZgZGYmI8sJjvEuF8fO2XZyif0UDXQuOR7pQjVzOKhpmuwsZ+6nksfta5U9l6ymyavp6SyIjGT7Od25ZckDQss5f8NNs1JjsUQCSeFJ5CK1NDKSlRERXGW3YwV2hoa6Byb/WhiGzlgs5AIZFkupq/ZOFYaAONMkpRL7K6+dpbJfg6ctFtafMyj5XEXtRAsiI5lXTqvKTyUC3oV8ARft54PoaNaV09V4sERY8va4K1OnkxYLQ222MstP+eGqtMqcS71zc7nM4WB0XByNHI5iz69hmjRyOEi1WMizWDhhtRLC2VB8btddqGliKxxzJ1KdKTxJQLicTswSH7CGYWAt8cUk3rEbRrFWgE6FX3hFIcU0DH4KDaW13U6oaRbrSqlb2FJT9OUcyI6UUxYLmYaBBcoNWOdTFHpa2O3lbuP3wnUchnHB+ymLN6/d71YrnfLz2RoWFtAWmfOdSw0dDsLA3Tp1rj65ufTJzeURm63YJKut7XbWnvN/vnVhOKuKecREfKlaxftXX32VzZs3k5ycTMuWLd3LmzVrxqJFi1i/fj1Lly7l8ssv97pMAic3OxO7vQBLeDRmSIT7J9/uxOmjcSJSWjOHgwE5OawKD3e3FgB8ERlJCNDnnO6wcNPk7txc9oaEcLzwy62oeyY2AIN5XYbBlxERdMvNpUUZEy/WqsR5szM0lN+tVgZlZ5c+hsIupFNWKxvDwuifnU2dMrZZmf2UJdcwLrgbb3Hh+/N0Gd2IVtMsdiyeTFXgjbLOpUWRkTxis5X6AVgTHs4jNhs/FIavf4eHk2YYDMzOLrbdgdnZ5BgGay7g6kwRf6pWLU9Lly7lgw8+4LPPPiu2fNy4ccyaNYu5c+eSlJTExIkTSUpK8qpMAqcgP4+wiCi69H6UWFsCABlpqayePxXTofldfOWrEydYEhFBstVKY6eTgdnZnLFYGF3iNhkzo6PpV3h13X85HCRbrfTKzaWR08lDhZfbA/weEsIZw2BATg5ZhkGOxcIPoaEcrsR4KF94IzaWDgUFLElN5dOoKH4JCSHO5eIau52O+flcXb9+hc83DYPna9bk49OnWXnyJHOiojhhsdDc4eByh4MH4uMBeKFmTT5LTWXNyZPMiorikNVKgsvFfxcUUN/ppEudOh7XfUdoKANzcngqM5PfQkI4VWLepIpsDg9nRlQUQ7OyuMpu55vwcOyGwX85HCTl5vJKzZosLRyo78lUBQ0dDnoXDvpuXRhIiy4QOGK1uq+ahMqdSwdCQzlQVotRWhqHrFZWnHMxQZ5h8FZsLG+kpzP19Gm+Dg+nbUEBvXJzeTMmhjPqtpNqrlqFpy1btpRaFh8fT6tWrejXrx9wNmC99tprNG3alMzMzAsqO3jwoN+OScoXE1cbW0LFX3hy4X4KDeXe3FxqO52ctlhYHBnJ2zExpS4PzzMM+sbH82JGBvfl5BBpmvwUGsrAWrWKjftxGAZP22w8n5HBm+nphALPxMX5LTylWq0k1a7NM5mZ3JGXx8DCOaB+CQk5e0VeJayLiKBPfDzDs7J4vHAix9+t1rNXzBXaFxpKt4QEhmdm0jcnB5vLxSmLhV2hoUy8wNuGTIyJoZHTyeCsLGJMk41hYZUOTwCj4+LYERpK/5wcRmdm4uDsgPiFUVFsPc/g7PI0djpLDYov+n1jWFix8FTZc8kTn0RHYwcez86mS14eKVYrr8TGnp2sVaSaq1bhqSwNGjTgxIkTxbpzkpOTadiwIRkZGRdUpvAkwc40DDYWfmmWd8uRIeUMMC7LKauVZyqx/qqICFaV0aXSp3btMtevzDYB2tWtW6n1TlmtvBgXx4sVrLMpPLzcG9wCfBcezv3nCS6HQkJ4+jx1nxsVVWbrTln7T7VaebCwZet865b3mn0aHc2n5wkW5dWpLOd7nc7lyblUUkX7qMwxiVRH1T48iUhpeYZRbmAREZGqVe3DU0pKCnXq1MFqtbpbkRo2bEhycjKZmZkXVCYiIiJyoar9qLxTp06xc+dOevXqBUBSUhJHjx7l4MGDF1wmIiIicqGqVcvTuHHj6Ny5MwkJCcyaNYusrCw6duzI6NGjmThxIkOHDiUzM5Phw4e7n3OhZSLBLNLlYsuJEwC0rVOHXF2dJCLiN9UqPI0aNarM5QcOHKBHjx4+LRMJdvG6eaqISEDoz1URERERDyg8iYiIiHhA4UlERETEAwpPIiIiIh5QeBIRERHxQLW62k5EKsc0DH4svAlrebdnERGRqqHwJBKE8gyDpISEQFdDROSSpG47EREREQ8oPImIiIh4QOFJJAhFuFxsPn6czcePE6GZxkVE/EpjnkSCkAEkOp3uxyIi4j9qeRIRERHxgMKTiIiIiAcUnkREREQ8oPAkIiIi4gGFJxEREREP6Go7kSBkAntDQtyPRUTEfxSeRIJQnsXCn+vUCXQ1REQuSeq2ExEREfGAwpOIiIiIBxSeRIJQhMvF2hMnWHvihG7PIiLiZxrzJBKEDKCFw+F+LCIi/qOWJxEREREPKDyJiIiIeEDhSURERMQDCk8iIiIiHtCAcak2XE4nZuEg6CKGYWC1WgNUIxERkdIUnqRayM3OxG4vICw8GtP4z/VjBXk5hIMCVAkmcLjwNdHtWURE/EvhSaqFgvw8wiKi6NL7UWJtCQBkpKWyev5UTEdegGtX/eRZLLSrWzfQ1RARuSQpPEm1EhNXG1tC/UBXQ0REpFwaMC4iIiLiAYUnkSAUYZosPXmSpSdPEmFq1JOIiD+p204kCBmmybV2u/sxhm7SIiLiL2p5EhEREfGAwpOIiIiIBxSeRERERDyg8CQiIiLiAYUnEREREQ/oajuRIHXKor99REQCQeFJJAjlWiy0qlcv0NUQEbkk6U9XEREREQ8oPImIiIh4QOFJJAhFmCbzUlOZl5qq27OIiPiZxjyJBCHDNOlQUOB+rNuziIj4j1qeRERERDygliep1lxOJ6bD4f7dMAysVmsAayQiIpc6hSeptnKzM7HbCwgLj8Ys7JYqyMshHBSgREQkYBSepNoqyM8jLCKKLr0fJdaWQEZaKqvnT8V05AW6aiIicglTeJJqLyauNraE+oGuhoiICKDwJBK0cnSFnYhIQCg8iQShXIuFP9RXa5yISCBoqgIRERERDyg8iYiIiHhA4UkkCIWbJtNPnWL6qVOE6/YsIiJ+pTFPIkHIYpp0zs93P9btWURE/EctTyIiIiIeUHgSERER8UDQhKc///nPLF++nJUrV7JmzRr69OkDQHx8PDNnzmTDhg2sWbOGtm3bup9TUZmIiIjIhQiaMU+TJ0+mT58+7Nmzh0aNGrFu3Tq+/PJLXnjhBbZt20b//v1p3bo1H330Ee3atcPhcFRYJiIiInIhgqblCSA2NhaAmJgY0tLSKCgooHv37syYMQOA7du3c+zYMdq3bw9QYZmIiIjIhQialqfBgwczbdo0cnJyqFmzJo8++ig1atQgJCSEkydPutc7cuQIDRo0wGazlVsmIiIicqGCIjxZrVaGDRvGoEGD2LJlC61bt+Zf//oXt912W6CrJhIQuRYLDfWHgIhIQARFt13Lli2pV68eW7ZsAc52wR09epQrr7wSp9NJQkKCe91GjRqRkpJCWlpauWUiIiIiFyoowlNKSgp16tShefPmADRt2pQmTZpw4MABlixZwoABAwBo3bo19erVY9OmTQAVlomIiIhciKDotktNTWXkyJH84x//wDRNDMPgxRdfJCUlhddff53JkyezYcMGCgoKGDZsmPtquorKRIJZuGkyOS0NgGE2G/maYVxExG+CIjwBLFq0iEWLFpVanpqayv3331/mcyoqEwlmFtPkzrw8AJ7W7VlERPwqKLrtRERERKoLhScRERERDyg8iYiIiHhA4UlERETEAwpPIiIiIh5QeBIRERHxQNBMVSAi/5FrGDSvV8/9WERE/EfhSSQYGYZCk4hIgKjbTkRERMQDCk8iQSjMNJmYlsbEtDTCTDPQ1RERuaQoPIkEIatp0jc3l765uVgVnkRE/ErhSURERMQDCk8iIiIiHlB4EhEREfGAwpOIiIiIBxSeRERERDyg8CQiIiLiAc0wLhKEcg2Da+rWdT8WERH/UXiSoOJyOjEdjmLLDMPAarUGqEYBYhicvtSOWUSkmlB4kqCRm52J3V5AWHg05jmtLQV5OYTDpRegREQkIBSeJGgU5OcRFhFFl96PEmtLACAjLZXV86diOvICXDv/CjNNXklPB+B/atakQF13IiJ+o/AkPud0OjHPuWWIw+HAavXdLURi4mpjS6jvs+0FI6tp8lBODgCvxcaCwpOIiN8oPIlPOZ1O8u1OQiOizllqxzQpFqhERESClcKT+JRpmoRGRHFr78eJtdUGIPm3vWxevSDANRMREfENhSepErG2/3StpZ8+GeDaiIiI+I4myRQRERHxgMKTiIiIiAcUnkREREQ8oDFPIkEozzBoW6eO+7GIiPiPwpNIEDINgyMh+u8rIhII6rYTERER8YD+dBUJQqGmyaiMDADGxcZiV9ediIjfqOVJJAiFmCaDs7MZnJ1NiGZuFxHxK4UnEREREQ8oPImIiIh4QOFJRERExANehac6hfPMyMXJ6XTicDiK/TidzkBXS0REJKC8Ck9bt27l008/pVevXkRGRvqqTlINOJ1O8u1OzJCIYj/5dqcClIiIXNK8mqrg7bff5q677mLSpEmMHTuW5cuXs3DhQtatW4epK4CCmmmahEZEcWvvx4m11QYgIy2V1fOnYjryAlw7ERGRwPEqPL377ru8++67tGzZknvuuYeePXty9913k5qayqJFi1i4cCE7duzwVV0lAGJttbEl1A90NaSEPMPgloQE92MREfEfnwwY3717N2PGjOG6666jX79+rFmzhr59+7JkyRK++uorhg4dSoMGDXyxKxHh7O1ZfgkN5ZfQUEyFJxERv/L51XZbtmxh7dq1bNu2DcMwaNasGcOHD2fTpk1MnTpVg8xFREQkqPns9iwdOnTg7rvvplu3bsTExPDzzz8zZswYFi5ciNPppG/fvgwdOpTJkydz3333+Wq3IpekUNNkaGYmAO/GxOj2LCIifuRVeLrqqqu4++67ueuuu6hbty4nTpxg9uzZzJ8/n59//rnYulOnTiU/P5+XXnrJqwqLyNnbszyblQXABzVqKDyJiPiRV+FpxYoV5OXlsXz5cubPn88333xT4VV2v/zyC9u2bfNmlyIiIiIB5VV4evbZZ1myZAk5OTmVWn/jxo1s3LjRm12KlOJyOjEdjmLLDMPAarUGqEYiInIx8yo8zZ0711f1ELkgudmZ2O0FhIVHF7vqrCAvh3BQgBIREZ/zKjw98sgjdO7cmQceeKDM8hkzZrBq1SqmT5/uzW5EylWQn0dYRBRdej9KrO3svEeazFNERKqSV1MV9OvXj3379pVbvm/fvnKDlYgvxcSdnczTllDfPSO6iIhIVfAqPDVp0qTC8LR//36aNGnizS5EREREqhWvuu0KCgpIKLxFRFnq1KmDy+XyZhciUoZ8w6Bb7druxyIi4j9etTxt27aNvn37Eh0dXaosJiaGe++9V1MTiFQBl2GwPSyM7WFhuBSeRET8yquWp4kTJzJ//nxWrlzJtGnT+OWXXwBo0aIFgwYNok6dOjz55JM+qaiIiIhIdeBVePrhhx946KGHGDduHK+++qp7gkzDMDh06BAPP/ww33//vU8qKiL/EWqa/L/sbAA+io7WDOMiIn7k9b3t1q9fz4033sjVV19N06ZNATh48CA7d+70dtMiUo4Q0+SljAwAPomKUngSEfEjn9wY2DRNdu7cqcAkIiIiFz2fhKc//OEPNGnShJo1a2KU8Rfw/PnzfbEbERERkYDzKjw1adKEd999l2uvvbbM0ARnW6V8EZ7CwsJ4+eWXufnmm8nLy+Onn35i2LBhNGvWjEmTJlGrVi0yMjJ45pln3APXKyoTERERuRBehadx48ZxxRVX8Morr/Dtt99y5swZH1WrtBdeeAHTNOnYsSOAe36pcePGMWvWLObOnUtSUhITJ04kKSnpvGUiIiIiF8Kr8HTdddfx3nvv8a9//ctX9SlTZGQk9913H9ddd5172cmTJ4mPj6dVq1b069cPgKVLl/Laa6/RtGlTMjMzyy07ePBgldZXRERELl5ehae0tDQyCq/4qUpNmzblzJkzDB06lE6dOpGXl8c777xDeno6J06cwOl0utdNTk6mYcOGZGRklFum8CQiIiIXyqsZxmfMmME999yDxeLVZs7LarWSmJjIvn376NatGy+99BIffPABISE+Ge8uEnTyDYPe8fH0jo/X7VlERPzMq/Tx66+/YrVaWbVqFXPmzCElJaVYS0+RZcuWebMbkpOTcTqdLFy4EIDdu3dz6NAhGjVqRJ06dbBare79NmzYkOTkZDIzM8stk4ufy+nEdDiKLTMMA6vVGqAa+ZbLMNgUHh7oaoiIXJK8Ck8ffPCB+/FLL71U5jqmadK4cWNvdkNaWhobNmzg5ptvZu3atSQmJtK4cWO2bt3Kzp076dWrl3tQ+NGjR93dchWVycUrNzsTu72AsPBozHNaZQrycgiHiyZAiYhIYHgVnvr06eOrepzX6NGjmTBhgvuqu1GjRnHs2DFGjx7NxIkTGTp0KJmZmQwfPrzYc8ork4tXQX4eYRFRdOn9KLG2s1dlZqSlsnr+VExHXoBr5xshpskDOTkAzIqKwqGuOxERv/EqPG3evNlX9TivQ4cOlRnWDhw4QI8ePcp8TkVlcvGLiauNLaF+oKtRJUJNkzfS0wGYGxmp8CQi4kc+GXEdFhbGNddcQ3x8PFu3biUtLc0XmxURERGpdry+TO6RRx5h27ZtfPbZZ0ybNo2rrroKAJvNxs6dO7n33nu9rqSIiIhIdeFVeOrbty//8z//w9dff81zzz1X7BYtaWlp/Pvf/6Znz55eV1JERESkuvAqPD3++OOsWLGCJ598klWrVpUq37FjB5dffrk3uxARERGpVrwKT02bNuWrr74qt/zMmTPYbDZvdiEiIiJSrXgVnjIyMqhVq1a55ZdffjknT570ZhciIiIi1YpX4Wnt2rU88MADxMbGliq7/PLLuf/++1m5cqU3uxCRMhQYBgNr1WJgrVoUaJoCERG/8mqqgvHjx7NkyRLWrl3LqlWrME2TPn36cO+999KtWzdOnDjBxIkTfVVXESnkNAzWREQEuhoiIpckr1qejh8/TteuXfnqq6/o3r07hmHQq1cvunTpwqJFi+jevbvmfBIREZGLiteTZJ46dYoRI0YwYsQIatWqhcVi4dSpU5im6Yv6iUgZQkyTe3JzAVioGcZFRPzKJzOMFzl9+rQvNyci5Qg1TSaeOQPA4ogIhScRET/yKjw9/fTTlVpv0qRJ3uxGREREpNrwKjw9++yz5ZaZpolhGJimqfAkIiIiFw2vwlNiYmKpZYZh0KhRIx566CHatm3LgAEDvNmFiIiISLXi9Y2BSzJNk8OHDzNmzBh+++03xowZ4+tdiIiIiASMz8PTubZs2cKf//znqtyF+JnL6cThcLh/nE5noKskIiLiVz692q6kVq1a4XK5qnIX4ke52ZnY7QWEhUdjFl7dVZCXQzhgtVoDWzkRERE/8So89e7du8zlsbGxtGvXjjvuuINPP/3Um11INVKQn0dYRBRdej9KrC2BjLRUVs+fiunIC3TVLjkFhsHjhTfd1u1ZRET8y6vwVNGtV06fPs2UKVN0e5aLUExcbWwJ9QNdjUua0zBYEhkZ6GqIiFySvApP7dq1K7XMNE3S09PJzs72ZtMiIiIi1ZJX4Sk5OdlX9RARD1hNkzvyznaXLouIwKmuOxERv6nSAeNy8XM5nZgOh/t3h8OB1ar7Gla1MNNkauFNt5vXq0euwpOIiN94FZ4OHz7s8Q2ATdOkSZMm3uxWqomyrr4DO6aJbgwtIiIXLa8HjHft2pXLL7+cdevWceDAAQCaN2/On/70J/bu3cvy5ct9UlGpfkpefQeQ/NteNq9eEOCaiYiIVB2vwtPx48eJj4+nc+fO7uBUpHnz5sydO5fjx49ruoKL3LlX36WfPhng2pSvZBejYRian0pERDzm1QzjgwcP5uOPPy4VnAD279/Pxx9/zF/+8hdvdiHiE0VdjJbwaMyQCMyQCPLtTs2QLiIiHvOq5alevXrY7fZyy+12O/Xq1fNmFyI+oQk+RUTEV7xqedq7dy8PPvhgmQGpfv36PPjgg/z888/e7ELEp4q6GGNttQNdFRERCVJetTz97W9/49NPP2X9+vUsW7aMgwcPAtCsWTO6du2KYRgMHTrUF/UUkXPYDYNn4uLcj0VExH+8Ck9bt27lzjvvZMSIEdxxxx1EREQAkJeXx9dff82ECRPU8iRSBRyGwdyoqEBXQ0TkkuT1JJl79+5l0KBBGIZBfHw8AKdOndI8PyIiInJR8tkM46Zpkp+fT3Z2toKTSBWzmiY35+cD8HV4uG7PIiLiR14NGAdo1aoVM2fOZP/+/ezatYv27dsDYLPZ+Oc//+n+XUR8J8w0mX76NNNPnyZMf6yIiPiVV+Hpuuuu47PPPqNZs2YsWLAAi+U/m0tLSyMmJob+/ft7XUkRERGR6sKr8DRq1Cj279/PLbfcwptvvlmqfOPGjfzxj3/0ZhciIiIi1YpX4enaa69lzpw5FBQUlFl+7Ngx6tSp480uRERERKoVr8KT3W4v1lVXUr169cjOzvZmFyIiIiLVilfhadu2bSQlJZVZFhkZyb333svmzZu92YWIiIhIteJVeJowYQKtWrVi+vTp3HLLLQBcddVV9OvXj+XLlxMfH8+kSZN8UU8RERGRasGreZ5++OEHBg4cyNixY/n73/8OwMsvvwzA77//zoABA9izZ4/3tRSRYuyGwQs1a7ofi4iI/3gVnmrUqMF3333Hn/70J1q2bEmzZs2wWCwcPHiQHTt2+KqOIlKCwzD4JDo60NUQEbkkXXB4CgsLY/fu3bz55pt88MEH7N69m927d/uybiIiIiLVzgWHp4KCAk6ePFnuNAUiUnUspknbwv97W8LCcKnrTkTEb7waMD537lx69+5NaGior+ojIpUQbprMP3WK+adOEa7bs4iI+JVXY55+/vlnbr/9dr766ivmzp3L4cOHycvLK7XesmXLvNmNSJVwOZ2YDkexZYZhYLVaA1QjEREJBl6FpylTprgfjxgxosx1TNOkcePG3uxGxOdyszOx2wsIC4/GPKfLqyAvh3BQgBIRkXJ5HJ5Gjx7NokWL2LNnD3369KmKOolUuYL8PMIioujS+1FibQkAZKSlsnr+VExH6dZTERGRIh6HpyFDhvDzzz+zZ88eNm/ejM1mY/v27fTr149///vfVVFHkSoTE1cbW0L9QFdDRESCiFcDxosYutJHRERELhE+CU8iIiIilwqvBoyLSGA4DIMxsbHuxyIi4j8XFJ4SExO5+uqrAYgt/ABv1qwZ6enpZa6/a9euC6yeiJTFbhj8o0aNQFdDROSSdEHhacSIEaWmJnjjjTdKrWcYhqYqEBERkYuKx+Fp+PDhVVEPEfGAxTS5xm4HYGdoqG7PIiLiRx6Hp3nz5lVFPUTEA+GmyZepqQA0r1ePXIUnERG/0dV2IiIiIh5QeBIRERHxgMKTiIiIiAeCLjz17duX5ORkbr/9dgDi4+OZOXMmGzZsYM2aNbRt29a9bkVlIiIiIhciqMJTo0aNeOCBB/j+++/dy1544QW2bdtGx44dGT58OFOmTCEkJOS8ZSIiIiIXImjCk2EYvP3227z44ovk5+e7l3fv3p0ZM2YAsH37do4dO0b79u3PWyYiIiJyIYKmGeaxxx5j69at7Ny5073MZrMREhLCyZMn3cuOHDlCgwYNKiwTCXYOw2BC4Qzjuj2LiIh/BUV4atGiBUlJSdxzzz2BropItWA3DN4pvDWSiIj4V1CEp7Zt29KoUSM2bNgAQEJCAuPHj2fChAk4nU4SEhLcLUyNGjUiJSWFtLS0csukNKfTiWma7t8dDgdWq1nBM0RERC5NQTHmafr06bRp04Z27drRrl07tm3bxsiRI5k+fTpLlixhwIABALRu3Zp69eqxadMmgArL5D+cTif5didmSIT7h5BwTJNigUqqD8M0udxu53K7HUPvkYiIXwVFy1NFXn/9dSZPnsyGDRsoKChg2LBhOByO85bJf5imSWhEFLf2fpxYW20Akn/by+bVCwJcMylPhGnyVWGLqm7PIiLiX0EZnvr06eN+nJqayv3331/mehWVSWmxttrYEuoDkH765HnWFhERuTQFRbediIiISHWh8CQiIiLiAYUnEREREQ8E5ZgnkaricjoxS1xUYBgGVqs1QDUSEZHqRuFJpFBudiZ2ewFh4dGY51y9VpCXQzgoQImICKDwJOJWkJ9HWEQUXXo/SqwtAYCMtFRWz5+K6cgLcO2KcxgGH0RHux+LiIj/KDyJlBAT958pG6oru2HwWs2aga6GiMglSQPGRURERDyglieRIGSYJg2dTgCSrdZiY7RERKRqKTyJBKEI02TLiROAbs8iIuJv6rYTERER8YDCk4iIiIgHFJ5EREREPKAxTyLnUXLWcc04LiJyaVN4EqlAWbOOa8ZxEZFLm8KTSAVKzjpeXWccFxER/1F4EqmE6jbruNMw+Dgqyv1YRET8R+FJJAgVGAZ/jYsLdDVERC5JutpORERExANqeRIJRqZJLZcLgNMWC6jrTkTEbxSeRIJQpGmy8/hxQLdnERHxN3XbiYiIiHhA4UlERETEAwpPIiIiIh5QeBIRERHxgMKTiIiIiAcUnkREREQ8oKkKRDzkcjoxHY5iywzD8OuNgp2GwdzISPdjERHxH4UnEQ/kZmditxcQFh6NeU5oKcjLIRz8FqAKDINnbDa/7EtERIpTeBLxQEF+HmERUXTp/SixtgQAMtJSWT1/KqYjL8C1ExERf1B4ErkAMXG1sSXUD1wFTJNI0wQ4O7u4uu5ERPxGA8ZFglCkabL/2DH2HzvmDlEiIuIfCk8iIiIiHlB4EhEREfGAwpOIiIiIBxSeRERERDyg8CQiIiLiAYUnEREREQ9onieRIOQyDJZERLgfi4iI/yg8iQShfMPg8Vq1Al0NEZFLkrrtRERERDyg8CQiIiLiAYUnkSAU6XKRnJJCckoKkS5XoKsjInJJUXgSERER8YDCk4iIiIgHdLWdiA+4nE5Mh6PYMsMwsFqtAaqRiIhUFYUnES/lZmditxcQFh6Nec6cSwV5OYSDApSIyEVG4UnESwX5eYRFRNGl96PE2hIAyEhLZfX8qZiOvADXTkREfE3hScRHYuJqY0uoH+hqiIhIFVN4EglCLsNgTXi4+7GIiPiPwpNIEMo3DAbGxwe6GiIilyRNVSAiIiLiAYUnEREREQ8oPIkEoUiXi31Hj7Lv6FHdnkVExM805kkkSEWZZqCrICJySVLLk4iIiIgHFJ5EREREPBAU4Sk8PJyPPvqI9evXs2rVKmbPnk3Tpk0BiI+PZ+bMmWzYsIE1a9bQtm1b9/MqKhMRERG5EEERngBmzZpFp06d6NKlCytWrOCtt94C4IUXXmDbtm107NiR4cOHM2XKFEJCQs5bJiIiInIhgiI85efns3btWvfv27ZtIzExEYDu3bszY8YMALZv386xY8do3779ectERERELkRQNsMMGjSIFStWYLPZCAkJ4eTJk+6yI0eO0KBBgwrLRIKdaRhsDAtzPxYREf8JuvA0dOhQmjZtysiRI4mMjAx0dUQCIs8w6FO7dqCrISJySQqKbrsijz/+OHfccQf9+/cnLy+PtLQ0nE4nCQkJ7nUaNWpESkpKhWUi/uByOnE4HO4fp9MZ6CqJiIgPBE14euyxx7jrrrvo168fGRkZ7uVLlixhwIABALRu3Zp69eqxadOm85aJVKXc7Ezs9gIs4dGYIRGYIRHk250KUCIiF4Gg6LarX78+r7zyCgcPHmTevHnA2UHk3bt35/XXX2fy5Mls2LCBgoIChg0bhsPhAKiwTKQqFeTnERYRRZfejxJrSyAjLZXV86diOvJ8sv1Il4stJ04A0LZOHXItQfN3kIhI0AuK8HT06FEaNmxYZllqair333+/x2Ui/hATVxtbQn3gbDeeWSK8G4aB1Wq9oG3H6552IiIBERThSSTYFXXjhYVHF7s6riAvh3C44AAlIiL+p/Ak4gclu/EAn3fliYiIfyg8ifjRud14IiISnDTKVERERMQDankSCSBfDyIXEZGqp/AkEiDeDCI3DYMfQ0Pdj0VExH8UnkQCxJtB5HmGQdI5s+eLiIj/KDyJBJgGkYuIBBcNGBcRERHxgMKTSBCKcLnYfPw4m48fJ0IzjYuI+JW67USCkAEkFt5kWMPFRUT8Sy1PIiIiIh5QeBIRERHxgLrtRKoZTZwpIlK9KTyJVCPeTJwpIiL+ofAkUo14M3GmiIj4h8KTSDV0vokzTWBvSIj7sYiI+I/Ck0gQyrNY+HOdOoGuhojIJUlX24mIiIh4QOFJRERExAMKTyJBKMLlYu2JE6w9cUK3ZxER8TONeRIJQgbQonAuKN2eRUTEv9TyJCIiIuIBhScRERERD6jbTiQIlLxli0Y5iYgEjsKTSDVX1i1b8nOzA1wrEZFLl8KTSDVX8pYtGWmpbJz7QaCrJSJyyVJ4EgkS596yxQQOWSwYhqHbs4iI+JnCk0gQyrVYuL52bUJC9F9YRMTfdLWdiIiIiAcUnkREREQ8oPAkEoQiTBfLT59m6cmTRJga9SQi4k8aMCESjJxO/lg475PL4cBhGBiGgdVqDXDFREQufmp5EgkyZ+d9srt/N63hmCER5NudOJ3OANZMROTSoPAkEmQK8vMIC490/97j4ZHc2vtxQiOiMNWFJyJS5dRtJxLk4mrXoyAsPNDVEBG5ZCg8iVwkSt7/DtA4KBGRKqDwJHIRKOv+dwAFeTmEgwKUiIgPKTyJBKnMqBoYlrPDFkve/w4gIy2V1fOnYjryAllNEZGLjsKTSBDKsVh47rm3qVWnQbHl597/TkREqoauthMRERHxgFqeRC5iJQeRawC5iIj3FJ5EglCE6WL4JxMIDQ3nX395pcx1yhpErgHkIiLeU3gSCUIWE1r8vg8Ao5yJMUsOItcAchER31B4ErnIaRC5iIhvacC4iIiIiAcUnkREREQ8oG47kUuIbuEiIuI9hSeRS0R5t3DJy8ki1OooFqAUqEREyqfwJBKk8kPDMM4JQedT1i1cTiT/zpqFH2KEReieeCIilaTwJBKEciwWhj0/udTtWSrj3Kvv0k+f1D3xREQ8pPAkIprOQETEA7raTkRERMQDankSCULhpsmTn75HaHgEswaN9vn2S16VZ5pmqfFVGlQuIpcqhSeRIGQ1Ta7ZvwsAw+Xy6bZLXpXncjopKMgjIjIKNKhcREThSUSKK3lVXvJve9m8egG3lhhUvnLO+9jtuZjn3FuvMq1RTqez2HPgwlu2ytqWWsREpKopPIlImYoGkaefPlnsd6j8nFElQ5HL5cLuNAmLjP7Psgts2XI6neTbnYRGRBVbrhYxEalqCk8i4rHKzBlVVihyOeyEhoRwa+/H3M+rbMtWyRYl0zQJjYji1t6PE2ur7X5eZaZZ8GWLlVq/RC49F314atasGZMmTaJWrVpkZGTwzDPP8MsvvwS6WiIXhYrmjCorFBUtK/m8ktsqq2WrZKuWw+HAajWJtVU8zULJcFNW6xdcWIuVWr9ELk0XfXgaN24cs2bNYu7cuSQlJTFx4kSSkpICXa2AKvllUvQlJOILFXX3FS07n5ItW2XPhG7HNCnV6nPulYJldhOW0fpVVktXZcZhldf6dSHbgtL/N9WCJVI9GQ0aNLhovzXj4+P597//TcuWLXE6nQD88MMP3H333Rw8eNCjbX3xxRdYrVZOnqzch7+nSn4BVDnjnCm+TBMMiIyOxWI5+0HtdNjJy80mMjqm3GWVWac6PK861MHXdS/IzSKx8Jw+FRePw+GodnWvivcrIjIaw2Jxr1OQn1f8eU4HudkZhUHFcJ/bEZE1PHse5jmB5z+hxzRdlLohjmHx3bYKt3fedUQuQp7cbsoTCQkJOJ1OevTo4bNtXtQtTw0aNODEiRPu4ASQnJxMw4YNPQ5Pdrvdx7UrrqpOmvKdE9YKd52XnVFsDaMSyyqzTnV4XnWogy/rDnCoqEUi80y1rbuv36/83KzzPs/i/r9kus9tj5/Huf8ni7cElWb6cFuVXUdEKsvpdPr8O/yiDk++1KtXr0BXQURERKqBi/r2LCkpKdSpU6fYmIGGDRuSnJwcwFqJiIhIMLuow9OpU6fYuXOnu9UoKSmJo0ePetxlJyIiIlLkoh4wDnDZZZcxceJEbDYbmZmZDB8+nJ9//jnQ1RIREZEgddGHJxERERFfuqi77URERER8TeFJRERExAMKTyIiIiIeUHgSERER8YDCk4iIiIgHFJ5EREREPKDwJCIiIuIBhScRERERDyg8BVizZs1YtGgR69evZ+nSpVx++eWBrtIF+/Of/8zy5ctZuXIla9asoU+fPgDEx8czc+ZMNmzYwJo1a2jbtq37ORWVVSevvvoqmzdvJjk5mZYtW7qXh4WF8dprr7FhwwZWr17N5MmT3WUVvbfV9X0PDw/no48+Yv369axatYrZs2fTtGnTYuvceOONHDp0iEGDBrmXRUREMGXKFDZs2MD69etJSkqqVFmgffrpp6xatYqVK1eycOFCWrZsed7XIJjP5759+5KcnMztt98OXPixVPfjhNLHeu2117J48WJWrFjB119/zeDBg93rBuv5u3nzZr755htWrlzJypUr6dGjB3DxfS5BxccEpd9vqNpzOMS7wxFvjRs3jlmzZjF37lySkpKYOHFitfrP6YnJkyfTp08f9uzZQ6NGjVi3bh1ffvklL7zwAtu2baN///60bt2ajz76iHbt2uFwOCosq06WLl3KBx98wGeffVZs+QsvvIBpmnTs2BGAhIQEd1lF7211ft9nzZrF2rVrAXjooYd466233EE4JiaG559/3l1e5IknnqCgoICOHTuSmJjIkiVL2LhxI2lpaRWWBdoTTzxBRkYGAF27dmXSpEnceeedFb4GwXo+N2rUiAceeIDvv//evexCj6U6HyeUfazjx4/nrbfeYtWqVcTFxbFu3TpWr17Nvn37gvb8BRg8eDC7d+8utuxi/Fyq6JjKer+LnlNV57BangIoPj6eVq1asWDBAuDsF3SDBg1K/aUfTGJjY4GzX7JpaWkUFBTQvXt3ZsyYAcD27ds5duwY7du3B6iwrDrZsmULR48eLbYsMjKS++67j3HjxrmXnTx5Eqj4va3O73t+fn6xYLRt2zYSExPdv7/++uv8/e9/L/XF0aNHD/f7ePjwYTZt2kTXrl3PWxZoRcEJzp67pmme9zUIxvPZMAzefvttXnzxRfLz893LL/RYqutxQvnHapomNWvWBCAqKgq73c6ZM2eA4D1/y3Ixfi5VdEzlvd9QteewWp4CqEGDBpw4cQKn0+lelpycTMOGDTl48GDgKnaBBg8ezLRp08jJyaFmzZo8+uij1KhRg5CQEPeJDnDkyBEaNGiAzWYrtywYNG3alDNnzjB06FA6depEXl4e77zzDhs2bKjwvc3IyAia933QoEGsWLECgKSkJFwuF6tWraJbt27F1mvYsCFHjhxx/3748GEaNmx43rLq4O9//zsdOnQAYMCAAaXKz30NKjpnq/P5/Nhjj7F161Z27tzpXnahx1KdjxPKPlaA4cOH889//pORI0dSq1YtRo8e7T6GYD5/J02ahGEY/Pjjj7zxxhvUrVv3ovtcquiztrz3u6rPYYUn8Qmr1cqwYcMYNGgQW7ZsoXXr1vzrX//itttuC3TVqozVaiUxMZF9+/YxduxYWrZsyf/93/9xyy23BLpqPjF06FCaNm3KyJEjSUhI4KmnnqJ3796BrpbPPfXUUwD06dOHF154gYEDB7rLzn0NglWLFi1ISkrinnvuCXRVqlxFxzpkyBDGjh3L559/TuPGjVmwYAHbt29n3759Aaipb9xzzz2kpKQQEhLCyJEjmTRpEuPHj7/oPpfK+6x99NFHA3Zuq9sugFJSUqhTpw5Wq9W9rGHDhiQnJwewVhemZcuW1KtXjy1btgBnm0GPHj3KlVdeidPpLNU/nZKSQlpaWrllwSA5ORmn08nChQsB2L17N4cOHeLKK6+s8L0Nhvf98ccf54477qB///7k5eXRqlUr6tSpw8qVK9m8eTNJSUk8/fTTjBo1Cjj7WjRq1Mj9/MTERPfxVFRWncybN48OHTpgs9mA0q8BUOE5W13P57Zt29KoUSM2bNjA5s2badOmDePHj6d79+4XdCzV9Tih/GN9+umn6dq1K59//jkAhw4dYtu2bVx//fVA8J6/Ra+5w+Fg2rRptG3b9qL8XKromMp6vwcOHFjl57DCUwCdOnWKnTt30qtXL+Bst8jRo0cD3kR6IYr+4zVv3hw428zapEkTDhw4wJIlS9zdIa1bt6ZevXps2rQJoMKy6i4tLY0NGzZw8803A2c/VBs3bsy+ffsqfG+r+/v+2GOPcdddd9GvXz/3mKA1a9Zw7bXX0q5dO9q1a8fSpUuZNGmSewzCue9jYmIi7du3Z/ny5ectC6TY2Fjq1q3r/v3222/nzJkzpKWllfkaFAm283n69Om0adPG/d5t27aNkSNHMn369As+lup4nFD+sU6ePJmcnBxuvPFG4GyXzh//+Ef27t0LBOf5GxkZ6R5jCnDXXXexa9eui/JzqbxjWrZsWbnnNlTtOWw0aNDA9OExiocuu+wyJk6ciM1mIzMzk+HDh/Pzzz8HuloXpGfPngwdOhTTNDEMg/fee4/PP/+c2rVrM3nyZBo3bkxBQQEvvvgiGzduBKiwrDoZN24cnTt3JiEhgbS0NLKysujYsSONGzdmwoQJ2Gw2TNNk4sSJfPnll0DF7211fd/r16/Pd999x8GDB8nOzgbODiLv3r17sfUmTpzI7t27mTZtGnD2g/ydd96hVatWuFwuxo8fz+LFi89bFkgNGzZk6tSpREREYJomp06dYsyYMZw+fbrC1yDYz+d58+Yxbdo0VqxYccHHEgzHCcWPtVOnTrzwwguEhIQQEhLC7Nmz+d///V8gOM/fxo0b8+GHH2KxWDAMg0OHDvHyyy9z5MiRi+5zCajwmIqc+35D1Z7DCk8iIiIiHlC3nYiIiIgHFJ5EREREPKDwJCIiIuIBhScRERERDyg8iYiIiHhA4UlERETEAwpPIiIiIh5QeBIRqebmzZvHmjVrAl0NESmk8CQiPpGcnFypn/bt2/tkf3Xr1mX48OG0bNmyUuv37duX5ORkWrVq5ZP9+5qnxyMigRMS6AqIyMVh6NChxX7v3bs3N910U6nlvrqLfd26dXn22Wc5cuQIu3fv9sk2A+liOx6Ri5nCk4j4RNEdz4u0adOGm266qdRyEZFgp247EfEbwzAYNGgQa9eu5cCBA/z444+MGzeOmjVrutd59tlnOXz4MB07diz23HHjxvHbb79x1VVX0b59e5YtWwacvUlxUZdg3759va5jvXr1mDBhAj/++CO//vora9eu5d577y22Tvv27UlOTqZ79+4MGzaM7777jgMHDjBnzhyaNm1aapsPPvggGzduZP/+/SxZsoQbbriBefPmMW/ePPf2KnM8f/jDH5g3bx779+/nu+++Y/DgwV4fr4h4Ti1PIuI348aNo2/fvsyZM4d//vOfJCYm8vDDD9OyZUvuuusuHA4Hf//73+nSpQtvv/02nTt3Jjs7m5tuuon+/fszfvx4fvrpJ2rXrs1bb73FiBEjmDlzJlu2bAHgu+++86p+tWvXZvHixZimyccff8ypU6e45ZZbeOedd4iJiWHatGnF1h8yZAgul4t//OMfxMTE8Je//IV3332X7t27u9cZOHAgb7zxBps3b+bDDz8kMTGRf/7zn5w5c4ajR48CZ7syz3c8NWvWZNasWSxbtozFixeTlJTEiy++yM8//8xXX33l1XGLiGcUnkTEL66//noeeOABhgwZwueff+5evnHjRj799FPuvPNOPv/8cxwOB0899RTLli3jlVde4bXXXnO3BL333nsApKamsnbtWkaMGMH333/vs67BUaNGYbFYuPXWW0lLSwNgxowZTJkyheHDhzNz5kzy8vLc64eHh3Pbbbdht9sBSE9PZ8yYMbRo0YK9e/cSGhrKiBEj+OGHH+jbty9OpxOAPXv2MGnSJHd4qszx1K9fn2HDhrFgwQIAZs+ezZYtW+jXr5/Ck4ifqdtORPzizjvvJD09nW+++Qabzeb+2bFjB1lZWXTo0MG97t69e5kwYQIPPPAAs2bNwmaz8fTTT7vDR1Xp1q0bq1evBihWx3Xr1lGzZk2uvvrqYuvPmTPHHZwAd4tR48aNAWjdujW1atXi008/LVb3hQsXusNZZWVlZbmDE4DdbufHH39070tE/EctTyLiF82aNaNmzZrs3LmzzPLatWsX+/2DDz6gZ8+etGnThrFjx/rsKr3yxMfHExcXR//+/enfv3+l6piSklLs9/T0dADi4uIAaNiwIQAHDx4stp7T6eTIkSMe1a+olark/q688kqPtiMi3lN4EhG/sFgsnDx5stTUBUVOnTpV7PcmTZrQrFkzAK644gq/1A9gwYIF7oHcJf3000/Ffi+vJcwwDN9Wzs/7EpGKKTyJiF/8/vvvdOrUia1btxYbN1QWwzCYOHEimZmZTJs2jWHDhrF06VL3FWkApmn6tH6nTp0iMzMTi8XC+vXrfbLN5ORkAJo2bcrGjRvdy61WK40aNWLPnj3uZb4+HhGpOhrzJCJ+sXjxYkJCQnj66adLlVmtVmJjY92/P/bYY1x//fWMGjWK8ePHs3XrVsaOHYvNZnOvk5OTA1Dsed5wuVx8+eWXdOvWjRYtWpQqr1Wrlsfb3L59O6dPn+b+++/HarW6l99zzz3FjgV8fzwiUnXU8iQifrF582ZmzJjB0KFDueqqq/jmm2+w2+3813/9F0lJSbzyyissXbqU5s2bM2LECObMmcOqVasAeOaZZ1i5ciVjx47liSeeAM62ZJ05c4YBAwaQlZVFTk4OP/zwA4cPH66wHvfddx+33HJLqeXTpk3jjTfeoEOHDixZsoRPP/2UX375hbi4OK655ho6duxYasD4+djtdiZMmMDrr7/O3LlzWbx4MYmJifTt25fffvut2LoXejwi4n8KTyLiN6NHj2bHjh3079+f0aNH43A4OHz4MAsXLmTr1q1YLBYmTZpEWloar7zyivt5v/32G2PHjmXMmDEsXbqUxYsX43A4ePrpp3n++ed58803CQ0N5Zlnnjlv2HjwwQfLXD537lyOHj1KUlISzzzzDHfccQcDBw4kLS2NX375hTfeeOOCjvnjjz/GMAwef/xxXnrpJX766ScefvhhXn311WLdlxd6PCLif0aDBg3U0S4i4keGYbBz506+/PJLRo4cGejqiIiHNOZJRKQKhYeHl1rWp08fbDYbmzZtCkCNRMRb6rYTEalCbdq04W9/+xtLliwhLS2Na665hvvuu489e/awZMmSQFdPRC6AwpOISBU6fPgwKSkpPPLII8TFxXHmzBnmz5/PG2+8UWx2chEJHhrzJCIiIuIBjXkSERER8YDCk4iIiIgHFJ5EREREPKDwJCIiIuIBhScRERERDyg8iYiIiHhA4UlERETEAwpPIiIiIh74/xlXYDFxPekOAAAAAElFTkSuQmCC",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "#this just measures space-delimited words which isn't necessarily the same as toke count, but it should be close-ish\n",
    "plot_text_length_histogram(train_df, percentile = 96)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0851b1bc-71dd-4c6b-94ff-2193c98c4e4b",
   "metadata": {},
   "source": [
    "## Load source model/tokenizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "f3d05cc5-123a-4e12-82cd-57671df3a049",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6d600b5f5e8d4c7686f9136c457e685e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Loading checkpoint shards:   0%|          | 0/2 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 6.72 s, sys: 31.1 s, total: 37.8 s\n",
      "Wall time: 3min 28s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "import torch\n",
    "from transformers import AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig\n",
    "\n",
    "# Quantization config\n",
    "bnb_config = BitsAndBytesConfig(\n",
    "    load_in_4bit=True,\n",
    "    bnb_4bit_quant_type=\"nf4\",\n",
    "    bnb_4bit_compute_dtype=\"float16\",\n",
    ")\n",
    "\n",
    "# loading the model with quantization config\n",
    "model = AutoModelForCausalLM.from_pretrained(\n",
    "    source_hf_model_name,\n",
    "    quantization_config=bnb_config,\n",
    "    trust_remote_code=True,\n",
    "    device_map= \"auto\",\n",
    "    #token = hf_token,\n",
    ")\n",
    "\n",
    "model.config.use_cache = True  #surpress the download and use cache. # apparently needed because of https://github.com/huggingface/transformers/pull/24906\n",
    "model.config.pretraining_tp = 1  #disable tensor parallelism\n",
    "\n",
    "#fixme\n",
    "tokenizer = AutoTokenizer.from_pretrained(source_hf_model_name,\n",
    "     trust_remote_code=True, \n",
    "     return_token_type_ids=False,\n",
    "     token = hf_token,\n",
    " )\n",
    "tokenizer.pad_token = tokenizer.eos_token\n",
    "tokenizer.padding_side = \"right\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "d4c0253b-48f1-4f6f-824b-762c857d22a1",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">LlamaForCausalLM</span><span style=\"font-weight: bold\">(</span>\n",
       "  <span style=\"font-weight: bold\">(</span>model<span style=\"font-weight: bold\">)</span>: <span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">LlamaModel</span><span style=\"font-weight: bold\">(</span>\n",
       "    <span style=\"font-weight: bold\">(</span>embed_tokens<span style=\"font-weight: bold\">)</span>: <span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">Embedding</span><span style=\"font-weight: bold\">(</span><span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">32000</span>, <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">4096</span><span style=\"font-weight: bold\">)</span>\n",
       "    <span style=\"font-weight: bold\">(</span>layers<span style=\"font-weight: bold\">)</span>: <span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">ModuleList</span><span style=\"font-weight: bold\">(</span>\n",
       "      <span style=\"font-weight: bold\">(</span><span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0</span>-<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">31</span><span style=\"font-weight: bold\">)</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">32</span> x <span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">LlamaDecoderLayer</span><span style=\"font-weight: bold\">(</span>\n",
       "        <span style=\"font-weight: bold\">(</span>self_attn<span style=\"font-weight: bold\">)</span>: <span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">LlamaAttention</span><span style=\"font-weight: bold\">(</span>\n",
       "          <span style=\"font-weight: bold\">(</span>q_proj<span style=\"font-weight: bold\">)</span>: <span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">Linear4bit</span><span style=\"font-weight: bold\">(</span><span style=\"color: #808000; text-decoration-color: #808000\">in_features</span>=<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">4096</span>, <span style=\"color: #808000; text-decoration-color: #808000\">out_features</span>=<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">4096</span>, <span style=\"color: #808000; text-decoration-color: #808000\">bias</span>=<span style=\"color: #ff0000; text-decoration-color: #ff0000; font-style: italic\">False</span><span style=\"font-weight: bold\">)</span>\n",
       "          <span style=\"font-weight: bold\">(</span>k_proj<span style=\"font-weight: bold\">)</span>: <span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">Linear4bit</span><span style=\"font-weight: bold\">(</span><span style=\"color: #808000; text-decoration-color: #808000\">in_features</span>=<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">4096</span>, <span style=\"color: #808000; text-decoration-color: #808000\">out_features</span>=<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">4096</span>, <span style=\"color: #808000; text-decoration-color: #808000\">bias</span>=<span style=\"color: #ff0000; text-decoration-color: #ff0000; font-style: italic\">False</span><span style=\"font-weight: bold\">)</span>\n",
       "          <span style=\"font-weight: bold\">(</span>v_proj<span style=\"font-weight: bold\">)</span>: <span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">Linear4bit</span><span style=\"font-weight: bold\">(</span><span style=\"color: #808000; text-decoration-color: #808000\">in_features</span>=<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">4096</span>, <span style=\"color: #808000; text-decoration-color: #808000\">out_features</span>=<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">4096</span>, <span style=\"color: #808000; text-decoration-color: #808000\">bias</span>=<span style=\"color: #ff0000; text-decoration-color: #ff0000; font-style: italic\">False</span><span style=\"font-weight: bold\">)</span>\n",
       "          <span style=\"font-weight: bold\">(</span>o_proj<span style=\"font-weight: bold\">)</span>: <span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">Linear4bit</span><span style=\"font-weight: bold\">(</span><span style=\"color: #808000; text-decoration-color: #808000\">in_features</span>=<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">4096</span>, <span style=\"color: #808000; text-decoration-color: #808000\">out_features</span>=<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">4096</span>, <span style=\"color: #808000; text-decoration-color: #808000\">bias</span>=<span style=\"color: #ff0000; text-decoration-color: #ff0000; font-style: italic\">False</span><span style=\"font-weight: bold\">)</span>\n",
       "          <span style=\"font-weight: bold\">(</span>rotary_emb<span style=\"font-weight: bold\">)</span>: <span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">LlamaRotaryEmbedding</span><span style=\"font-weight: bold\">()</span>\n",
       "        <span style=\"font-weight: bold\">)</span>\n",
       "        <span style=\"font-weight: bold\">(</span>mlp<span style=\"font-weight: bold\">)</span>: <span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">LlamaMLP</span><span style=\"font-weight: bold\">(</span>\n",
       "          <span style=\"font-weight: bold\">(</span>gate_proj<span style=\"font-weight: bold\">)</span>: <span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">Linear4bit</span><span style=\"font-weight: bold\">(</span><span style=\"color: #808000; text-decoration-color: #808000\">in_features</span>=<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">4096</span>, <span style=\"color: #808000; text-decoration-color: #808000\">out_features</span>=<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">11008</span>, <span style=\"color: #808000; text-decoration-color: #808000\">bias</span>=<span style=\"color: #ff0000; text-decoration-color: #ff0000; font-style: italic\">False</span><span style=\"font-weight: bold\">)</span>\n",
       "          <span style=\"font-weight: bold\">(</span>up_proj<span style=\"font-weight: bold\">)</span>: <span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">Linear4bit</span><span style=\"font-weight: bold\">(</span><span style=\"color: #808000; text-decoration-color: #808000\">in_features</span>=<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">4096</span>, <span style=\"color: #808000; text-decoration-color: #808000\">out_features</span>=<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">11008</span>, <span style=\"color: #808000; text-decoration-color: #808000\">bias</span>=<span style=\"color: #ff0000; text-decoration-color: #ff0000; font-style: italic\">False</span><span style=\"font-weight: bold\">)</span>\n",
       "          <span style=\"font-weight: bold\">(</span>down_proj<span style=\"font-weight: bold\">)</span>: <span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">Linear4bit</span><span style=\"font-weight: bold\">(</span><span style=\"color: #808000; text-decoration-color: #808000\">in_features</span>=<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">11008</span>, <span style=\"color: #808000; text-decoration-color: #808000\">out_features</span>=<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">4096</span>, <span style=\"color: #808000; text-decoration-color: #808000\">bias</span>=<span style=\"color: #ff0000; text-decoration-color: #ff0000; font-style: italic\">False</span><span style=\"font-weight: bold\">)</span>\n",
       "          <span style=\"font-weight: bold\">(</span>act_fn<span style=\"font-weight: bold\">)</span>: <span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">SiLUActivation</span><span style=\"font-weight: bold\">()</span>\n",
       "        <span style=\"font-weight: bold\">)</span>\n",
       "        <span style=\"font-weight: bold\">(</span>input_layernorm<span style=\"font-weight: bold\">)</span>: <span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">LlamaRMSNorm</span><span style=\"font-weight: bold\">()</span>\n",
       "        <span style=\"font-weight: bold\">(</span>post_attention_layernorm<span style=\"font-weight: bold\">)</span>: <span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">LlamaRMSNorm</span><span style=\"font-weight: bold\">()</span>\n",
       "      <span style=\"font-weight: bold\">)</span>\n",
       "    <span style=\"font-weight: bold\">)</span>\n",
       "    <span style=\"font-weight: bold\">(</span>norm<span style=\"font-weight: bold\">)</span>: <span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">LlamaRMSNorm</span><span style=\"font-weight: bold\">()</span>\n",
       "  <span style=\"font-weight: bold\">)</span>\n",
       "  <span style=\"font-weight: bold\">(</span>lm_head<span style=\"font-weight: bold\">)</span>: <span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">Linear</span><span style=\"font-weight: bold\">(</span><span style=\"color: #808000; text-decoration-color: #808000\">in_features</span>=<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">4096</span>, <span style=\"color: #808000; text-decoration-color: #808000\">out_features</span>=<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">32000</span>, <span style=\"color: #808000; text-decoration-color: #808000\">bias</span>=<span style=\"color: #ff0000; text-decoration-color: #ff0000; font-style: italic\">False</span><span style=\"font-weight: bold\">)</span>\n",
       "<span style=\"font-weight: bold\">)</span>\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\u001b[1;35mLlamaForCausalLM\u001b[0m\u001b[1m(\u001b[0m\n",
       "  \u001b[1m(\u001b[0mmodel\u001b[1m)\u001b[0m: \u001b[1;35mLlamaModel\u001b[0m\u001b[1m(\u001b[0m\n",
       "    \u001b[1m(\u001b[0membed_tokens\u001b[1m)\u001b[0m: \u001b[1;35mEmbedding\u001b[0m\u001b[1m(\u001b[0m\u001b[1;36m32000\u001b[0m, \u001b[1;36m4096\u001b[0m\u001b[1m)\u001b[0m\n",
       "    \u001b[1m(\u001b[0mlayers\u001b[1m)\u001b[0m: \u001b[1;35mModuleList\u001b[0m\u001b[1m(\u001b[0m\n",
       "      \u001b[1m(\u001b[0m\u001b[1;36m0\u001b[0m-\u001b[1;36m31\u001b[0m\u001b[1m)\u001b[0m: \u001b[1;36m32\u001b[0m x \u001b[1;35mLlamaDecoderLayer\u001b[0m\u001b[1m(\u001b[0m\n",
       "        \u001b[1m(\u001b[0mself_attn\u001b[1m)\u001b[0m: \u001b[1;35mLlamaAttention\u001b[0m\u001b[1m(\u001b[0m\n",
       "          \u001b[1m(\u001b[0mq_proj\u001b[1m)\u001b[0m: \u001b[1;35mLinear4bit\u001b[0m\u001b[1m(\u001b[0m\u001b[33min_features\u001b[0m=\u001b[1;36m4096\u001b[0m, \u001b[33mout_features\u001b[0m=\u001b[1;36m4096\u001b[0m, \u001b[33mbias\u001b[0m=\u001b[3;91mFalse\u001b[0m\u001b[1m)\u001b[0m\n",
       "          \u001b[1m(\u001b[0mk_proj\u001b[1m)\u001b[0m: \u001b[1;35mLinear4bit\u001b[0m\u001b[1m(\u001b[0m\u001b[33min_features\u001b[0m=\u001b[1;36m4096\u001b[0m, \u001b[33mout_features\u001b[0m=\u001b[1;36m4096\u001b[0m, \u001b[33mbias\u001b[0m=\u001b[3;91mFalse\u001b[0m\u001b[1m)\u001b[0m\n",
       "          \u001b[1m(\u001b[0mv_proj\u001b[1m)\u001b[0m: \u001b[1;35mLinear4bit\u001b[0m\u001b[1m(\u001b[0m\u001b[33min_features\u001b[0m=\u001b[1;36m4096\u001b[0m, \u001b[33mout_features\u001b[0m=\u001b[1;36m4096\u001b[0m, \u001b[33mbias\u001b[0m=\u001b[3;91mFalse\u001b[0m\u001b[1m)\u001b[0m\n",
       "          \u001b[1m(\u001b[0mo_proj\u001b[1m)\u001b[0m: \u001b[1;35mLinear4bit\u001b[0m\u001b[1m(\u001b[0m\u001b[33min_features\u001b[0m=\u001b[1;36m4096\u001b[0m, \u001b[33mout_features\u001b[0m=\u001b[1;36m4096\u001b[0m, \u001b[33mbias\u001b[0m=\u001b[3;91mFalse\u001b[0m\u001b[1m)\u001b[0m\n",
       "          \u001b[1m(\u001b[0mrotary_emb\u001b[1m)\u001b[0m: \u001b[1;35mLlamaRotaryEmbedding\u001b[0m\u001b[1m(\u001b[0m\u001b[1m)\u001b[0m\n",
       "        \u001b[1m)\u001b[0m\n",
       "        \u001b[1m(\u001b[0mmlp\u001b[1m)\u001b[0m: \u001b[1;35mLlamaMLP\u001b[0m\u001b[1m(\u001b[0m\n",
       "          \u001b[1m(\u001b[0mgate_proj\u001b[1m)\u001b[0m: \u001b[1;35mLinear4bit\u001b[0m\u001b[1m(\u001b[0m\u001b[33min_features\u001b[0m=\u001b[1;36m4096\u001b[0m, \u001b[33mout_features\u001b[0m=\u001b[1;36m11008\u001b[0m, \u001b[33mbias\u001b[0m=\u001b[3;91mFalse\u001b[0m\u001b[1m)\u001b[0m\n",
       "          \u001b[1m(\u001b[0mup_proj\u001b[1m)\u001b[0m: \u001b[1;35mLinear4bit\u001b[0m\u001b[1m(\u001b[0m\u001b[33min_features\u001b[0m=\u001b[1;36m4096\u001b[0m, \u001b[33mout_features\u001b[0m=\u001b[1;36m11008\u001b[0m, \u001b[33mbias\u001b[0m=\u001b[3;91mFalse\u001b[0m\u001b[1m)\u001b[0m\n",
       "          \u001b[1m(\u001b[0mdown_proj\u001b[1m)\u001b[0m: \u001b[1;35mLinear4bit\u001b[0m\u001b[1m(\u001b[0m\u001b[33min_features\u001b[0m=\u001b[1;36m11008\u001b[0m, \u001b[33mout_features\u001b[0m=\u001b[1;36m4096\u001b[0m, \u001b[33mbias\u001b[0m=\u001b[3;91mFalse\u001b[0m\u001b[1m)\u001b[0m\n",
       "          \u001b[1m(\u001b[0mact_fn\u001b[1m)\u001b[0m: \u001b[1;35mSiLUActivation\u001b[0m\u001b[1m(\u001b[0m\u001b[1m)\u001b[0m\n",
       "        \u001b[1m)\u001b[0m\n",
       "        \u001b[1m(\u001b[0minput_layernorm\u001b[1m)\u001b[0m: \u001b[1;35mLlamaRMSNorm\u001b[0m\u001b[1m(\u001b[0m\u001b[1m)\u001b[0m\n",
       "        \u001b[1m(\u001b[0mpost_attention_layernorm\u001b[1m)\u001b[0m: \u001b[1;35mLlamaRMSNorm\u001b[0m\u001b[1m(\u001b[0m\u001b[1m)\u001b[0m\n",
       "      \u001b[1m)\u001b[0m\n",
       "    \u001b[1m)\u001b[0m\n",
       "    \u001b[1m(\u001b[0mnorm\u001b[1m)\u001b[0m: \u001b[1;35mLlamaRMSNorm\u001b[0m\u001b[1m(\u001b[0m\u001b[1m)\u001b[0m\n",
       "  \u001b[1m)\u001b[0m\n",
       "  \u001b[1m(\u001b[0mlm_head\u001b[1m)\u001b[0m: \u001b[1;35mLinear\u001b[0m\u001b[1m(\u001b[0m\u001b[33min_features\u001b[0m=\u001b[1;36m4096\u001b[0m, \u001b[33mout_features\u001b[0m=\u001b[1;36m32000\u001b[0m, \u001b[33mbias\u001b[0m=\u001b[3;91mFalse\u001b[0m\u001b[1m)\u001b[0m\n",
       "\u001b[1m)\u001b[0m\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from rich import print\n",
    "print(model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "40757681-3f73-4b95-9e14-9228c67b668c",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/app/venv/lib/python3.10/site-packages/trl/trainer/ppo_config.py:141: UserWarning: The `optimize_cuda_cache` arguement will be deprecated soon, please use `optimize_device_cache` instead.\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a95d9128519f4cd0b61dd65ffec46555",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Map:   0%|          | 0/12148 [00:00<?, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "67a20c67538747b2a44053bbdb9085bd",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Map:   0%|          | 0/312 [00:00<?, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 28.8 s, sys: 1.21 s, total: 30 s\n",
      "Wall time: 20.8 s\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "transformers.models.llama.modeling_llama.LlamaForCausalLM"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%time\n",
    "import trl\n",
    "from peft import LoraConfig, get_peft_model\n",
    "from peft import prepare_model_for_kbit_training\n",
    "from trl import SFTTrainer\n",
    "from transformers import EarlyStoppingCallback\n",
    "import transformers\n",
    "\n",
    "\n",
    "peft_config = LoraConfig(\n",
    "    lora_alpha=lora_alpha,\n",
    "    lora_dropout=lora_dropout,\n",
    "    #target_modules=[ \"q_proj\",\"v_proj\",],  # we will only create adopters for q, v metrices of attention module\n",
    "    target_modules=[\"q_proj\", \"k_proj\", \"v_proj\", \"o_proj\",\"gate_proj\"],\n",
    "    r=lora_r,\n",
    "    bias=\"none\",\n",
    "    task_type=\"CAUSAL_LM\",\n",
    ")\n",
    "\n",
    "\n",
    "training_arguments = transformers.TrainingArguments(\n",
    "    output_dir=project_dir,\n",
    "    #load_from_checkpoint=str,\n",
    "    seed = seed,\n",
    "    per_device_train_batch_size=per_device_train_batch_size,\n",
    "    gradient_accumulation_steps=gradient_accumulation_steps,\n",
    "    optim=\"paged_adamw_8bit\",\n",
    "    learning_rate= learning_rate,\n",
    "    lr_scheduler_type=\"linear\",\n",
    "    save_strategy=\"steps\",\n",
    "    logging_steps=logging_steps,\n",
    "    num_train_epochs=epochs,\n",
    "    #max_steps=max_steps,\n",
    "    fp16=True,\n",
    "    push_to_hub=False,\n",
    "    evaluation_strategy=\"steps\",  # Add this line\n",
    "    eval_steps=logging_steps,  # Add this line\n",
    "    metric_for_best_model=\"loss\",\n",
    "    greater_is_better=False,\n",
    "    load_best_model_at_end=True,\n",
    ")\n",
    "model.gradient_checkpointing_enable()\n",
    "model = prepare_model_for_kbit_training(model)\n",
    "\n",
    "trainer = SFTTrainer(\n",
    "    model=model,\n",
    "    train_dataset=train_dataset,\n",
    "    eval_dataset=test_dataset,\n",
    "    peft_config=peft_config,  # passing peft config\n",
    "    dataset_text_field=\"text\",  # mentioned the required column\n",
    "    args=training_arguments,  # training agruments\n",
    "    tokenizer=tokenizer,  # tokenizer\n",
    "    packing=False,\n",
    "    max_seq_length=max_training_sample_length,\n",
    "    callbacks=[EarlyStoppingCallback(early_stopping_patience=early_stopping_patience)],\n",
    ")\n",
    "type(model)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "91d1ced4-7675-4be0-957c-a7e232ec7af9",
   "metadata": {},
   "source": [
    "## Train lora"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "8f9bfc5b-da5b-4f93-9b5b-e9e48097f947",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "You're using a LlamaTokenizerFast tokenizer. Please note that with a fast tokenizer, using the `__call__` method is faster than using a method to encode the text followed by a call to the `pad` method to get a padded encoding.\n",
      "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...\n",
      "/app/venv/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='47' max='47' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [47/47 1:03:13, Epoch 0/1]\n",
       "    </div>\n",
       "    <table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       " <tr style=\"text-align: left;\">\n",
       "      <th>Step</th>\n",
       "      <th>Training Loss</th>\n",
       "      <th>Validation Loss</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>5</td>\n",
       "      <td>1.541100</td>\n",
       "      <td>1.286439</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>10</td>\n",
       "      <td>1.179800</td>\n",
       "      <td>1.220081</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>15</td>\n",
       "      <td>1.140400</td>\n",
       "      <td>1.211979</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>20</td>\n",
       "      <td>1.140100</td>\n",
       "      <td>1.207712</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>25</td>\n",
       "      <td>1.124500</td>\n",
       "      <td>1.204040</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>30</td>\n",
       "      <td>1.128600</td>\n",
       "      <td>1.201542</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>35</td>\n",
       "      <td>1.116200</td>\n",
       "      <td>1.199799</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>40</td>\n",
       "      <td>1.121100</td>\n",
       "      <td>1.198511</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>45</td>\n",
       "      <td>1.118000</td>\n",
       "      <td>1.197850</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table><p>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 26min 47s, sys: 37min 49s, total: 1h 4min 36s\n",
      "Wall time: 1h 4min 35s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "if True:\n",
    "    trainer.train()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "f12d4322-25cf-4a33-92a2-a23eab8439b1",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">done!\n",
       "</pre>\n"
      ],
      "text/plain": [
       "done!\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "print(\"done!\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1fae22eb-210a-4f8b-97f2-fa9fda59a0b7",
   "metadata": {},
   "source": [
    "## save peft model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "c40dedb5-c6af-47b8-936f-fbf5b16cec9a",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 68.3 ms, sys: 45.5 ms, total: 114 ms\n",
      "Wall time: 244 ms\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "import os\n",
    "trainer.model.save_pretrained(project_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "04d58d58-3ba8-44e8-b6fc-a51154a9af8a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'work_for_2023_11_10'"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "project_dir"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "13c0d830-d825-41f4-91b6-d7e424844dc1",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "total 94M\n",
      "   0 drwxr-xr-x 1 root root 4.0K Nov 10 20:46 ..\n",
      "   0 drwxr-xr-x 1 root root 4.0K Nov 10 20:46 .\n",
      "4.0K -rw-r--r-- 1 root root  853 Nov 10 21:59 README.md\n",
      " 94M -rw-r--r-- 1 root root  94M Nov 10 21:59 adapter_model.bin\n",
      "   0 -rw-r--r-- 1 root root  492 Nov 10 21:59 adapter_config.json\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n"
     ]
    }
   ],
   "source": [
    "!ls -lhatrs ./$project_path/"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "dfc7faf6-3e0c-4f5c-922d-cabd545bac5d",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "total 94M\n",
      "   0 drwxr-xr-x 1 root root 4.0K Nov 10 20:46 ..\n",
      "   0 drwxr-xr-x 1 root root 4.0K Nov 10 20:46 .\n",
      "4.0K -rw-r--r-- 1 root root  853 Nov 10 21:59 README.md\n",
      " 94M -rw-r--r-- 1 root root  94M Nov 10 21:59 adapter_model.bin\n",
      "   0 -rw-r--r-- 1 root root  492 Nov 10 21:59 adapter_config.json\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n"
     ]
    }
   ],
   "source": [
    "!ls -latrsh ./$project_path"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "787bccd7-4a58-4cfa-a3fc-3a30d0a9b122",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---\n",
      "library_name: peft\n",
      "---\n",
      "## Training procedure\n",
      "\n",
      "\n",
      "The following `bitsandbytes` quantization config was used during training:\n",
      "- quant_method: bitsandbytes\n",
      "- load_in_8bit: False\n",
      "- load_in_4bit: True\n",
      "- llm_int8_threshold: 6.0\n",
      "- llm_int8_skip_modules: None\n",
      "- llm_int8_enable_fp32_cpu_offload: False\n",
      "- llm_int8_has_fp16_weight: False\n",
      "- bnb_4bit_quant_type: nf4\n",
      "- bnb_4bit_use_double_quant: False\n",
      "- bnb_4bit_compute_dtype: float16\n",
      "\n",
      "The following `bitsandbytes` quantization config was used during training:\n",
      "- quant_method: bitsandbytes\n",
      "- load_in_8bit: False\n",
      "- load_in_4bit: True\n",
      "- llm_int8_threshold: 6.0\n",
      "- llm_int8_skip_modules: None\n",
      "- llm_int8_enable_fp32_cpu_offload: False\n",
      "- llm_int8_has_fp16_weight: False\n",
      "- bnb_4bit_quant_type: nf4\n",
      "- bnb_4bit_use_double_quant: False\n",
      "- bnb_4bit_compute_dtype: float16\n",
      "### Framework versions\n",
      "\n",
      "- PEFT 0.5.0\n",
      "\n",
      "- PEFT 0.5.0\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n"
     ]
    }
   ],
   "source": [
    "!cat ./$project_path/README.md\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "48cce7d8-03a6-4bd5-91b4-f9c8918726dd",
   "metadata": {},
   "source": [
    "## Load peft adapter from checkpoint on disk"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "5f030a0e-f519-42bc-b1fb-8c11b2bd59d0",
   "metadata": {},
   "outputs": [],
   "source": [
    "from peft import LoraConfig, get_peft_model\n",
    "lora_config = LoraConfig.from_pretrained(project_path)\n",
    "loaded_model = get_peft_model(model, lora_config)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "e869dacf-b6b2-4ad1-942d-b5336e4606c9",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">### Instruction: \n",
       "Write a concise summary of the below input text.Ensure that responses covers the key points of the text.Only \n",
       "provide full sentence responses.\n",
       "### Input: \n",
       "#Person1#: Mister Ewing said we should show up at the conference center at <span style=\"color: #00ff00; text-decoration-color: #00ff00; font-weight: bold\">4:00</span> o'clock, right?\n",
       "#Person2#: Yes, he especially asked us not to be late. Some of the people from our east york branch office are \n",
       "coming, and he wants to make a good impression on them. How are you getting there?\n",
       "#Person1#: I was thinking of taking my car, but I think I'm just going to take the underground, because there is \n",
       "construction on the highway. What about you?\n",
       "#Person2#: I'll be taking the underground as well. Why don't we go together? I've been to the conference center \n",
       "only once, and I'm not sure if I can find my way around there.\n",
       "### Response :\n",
       "\n",
       "</pre>\n"
      ],
      "text/plain": [
       "### Instruction: \n",
       "Write a concise summary of the below input text.Ensure that responses covers the key points of the text.Only \n",
       "provide full sentence responses.\n",
       "### Input: \n",
       "#Person1#: Mister Ewing said we should show up at the conference center at \u001b[1;92m4:00\u001b[0m o'clock, right?\n",
       "#Person2#: Yes, he especially asked us not to be late. Some of the people from our east york branch office are \n",
       "coming, and he wants to make a good impression on them. How are you getting there?\n",
       "#Person1#: I was thinking of taking my car, but I think I'm just going to take the underground, because there is \n",
       "construction on the highway. What about you?\n",
       "#Person2#: I'll be taking the underground as well. Why don't we go together? I've been to the conference center \n",
       "only once, and I'm not sure if I can find my way around there.\n",
       "### Response :\n",
       "\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The meeting will start at four in the afternoon according to Mr. Ewing who has requested for all employees to attend it. He also emphasized on being punctual since some members from his East York Branch Office would be attending this event. Personally, I am planning to use public transportation due to traffic\n"
     ]
    }
   ],
   "source": [
    "#text = test_df['text'][500]\n",
    "print(text)\n",
    "result = generate(text, loaded_model, max_new_tokens=64, temp=0.1)\n",
    "#del lmodel"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "cf2bb12c-2f64-44b6-94a5-c52e82ba7412",
   "metadata": {},
   "outputs": [],
   "source": [
    "#del  loaded_model\n",
    "import torch\n",
    "torch.cuda.empty_cache()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "974716d4-a8d1-4ea0-a43a-64d2a063f172",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'The meeting will start at four in the afternoon according to Mr. Ewing who has requested for all employees to attend it. He also emphasized on being punctual since some members from his East York Branch Office would be attending this event. Personally, I am planning to use public transportation due to traffic'"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "result"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1628a103-d702-4bea-b56d-f7d7f29b74a7",
   "metadata": {},
   "source": [
    "## load the orignal mode"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "f68debd1-1547-4210-b56c-4bf51ce35621",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">work_for_2023_11_10/best_checkpoint\n",
       "</pre>\n"
      ],
      "text/plain": [
       "work_for_2023_11_10/best_checkpoint\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "total 94M\n",
      "   0 drwxr-xr-x 1 root root 4.0K Nov 10 20:46 ..\n",
      "   0 drwxr-xr-x 1 root root 4.0K Nov 10 20:46 .\n",
      "4.0K -rw-r--r-- 1 root root  853 Nov 10 21:59 README.md\n",
      " 94M -rw-r--r-- 1 root root  94M Nov 10 21:59 adapter_model.bin\n",
      "   0 -rw-r--r-- 1 root root  492 Nov 10 21:59 adapter_config.json\n",
      "{\n",
      "  \"auto_mapping\": null,\n",
      "  \"base_model_name_or_path\": \"meta-llama/Llama-2-7b-hf\",\n",
      "  \"bias\": \"none\",\n",
      "  \"fan_in_fan_out\": false,\n",
      "  \"inference_mode\": true,\n",
      "  \"init_lora_weights\": true,\n",
      "  \"layers_pattern\": null,\n",
      "  \"layers_to_transform\": null,\n",
      "  \"lora_alpha\": 32,\n",
      "  \"lora_dropout\": 0.05,\n",
      "  \"modules_to_save\": null,\n",
      "  \"peft_type\": \"LORA\",\n",
      "  \"r\": 16,\n",
      "  \"revision\": null,\n",
      "  \"target_modules\": [\n",
      "    \"q_proj\",\n",
      "    \"k_proj\",\n",
      "    \"v_proj\",\n",
      "    \"o_proj\",\n",
      "    \"gate_proj\"\n",
      "  ],\n",
      "  \"task_type\": \"CAUSAL_LM\"\n",
      "}"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n"
     ]
    }
   ],
   "source": [
    "print(project_path)\n",
    "!ls -latrsh $project_path\n",
    "!cat $project_path/adapter_config.json"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "b3bcd0ff-edd5-42c7-8b7f-63e91fea44c4",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f801a16c974b4845b4d19914f38e59a8",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Loading checkpoint shards:   0%|          | 0/2 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 39.8 s, sys: 30.1 s, total: 1min 9s\n",
      "Wall time: 3min 54s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "from peft import AutoPeftModelForCausalLM\n",
    "from transformers import  AutoTokenizer\n",
    "\n",
    "tokenizer = AutoTokenizer.from_pretrained(\n",
    "    source_hf_model_name, \n",
    "    trust_remote_code=True, \n",
    "    return_token_type_ids=False,\n",
    "    token = hf_token,\n",
    ")\n",
    "#tokenizer = AutoTokenizer.from_pretrained(source_hf_model_name)\n",
    "\n",
    "persisted_model = AutoPeftModelForCausalLM.from_pretrained(\n",
    "    project_path, # path to the project or model directory\n",
    "    low_cpu_mem_usage=True, # reduces CPU memory usage if True\n",
    "    return_dict=True, # ensures output is returned as a dictionary\n",
    "    torch_dtype=torch.float16, # sets the data type for PyTorch tensors\n",
    "    device_map=device, # specifies the device placement for the model\n",
    "    #token=hf_token, # token for Hugging Face authentication\n",
    "\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "e5d2a7b5-7e32-443e-8eee-01a45e50c125",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">### Instruction: \n",
       "Write a concise summary of the below input text.Ensure that responses covers the key points of the text.Only \n",
       "provide full sentence responses.\n",
       "### Input: \n",
       "#Person1#: Mister Ewing said we should show up at the conference center at <span style=\"color: #00ff00; text-decoration-color: #00ff00; font-weight: bold\">4:00</span> o'clock, right?\n",
       "#Person2#: Yes, he especially asked us not to be late. Some of the people from our east york branch office are \n",
       "coming, and he wants to make a good impression on them. How are you getting there?\n",
       "#Person1#: I was thinking of taking my car, but I think I'm just going to take the underground, because there is \n",
       "construction on the highway. What about you?\n",
       "#Person2#: I'll be taking the underground as well. Why don't we go together? I've been to the conference center \n",
       "only once, and I'm not sure if I can find my way around there.\n",
       "### Response :\n",
       "\n",
       "</pre>\n"
      ],
      "text/plain": [
       "### Instruction: \n",
       "Write a concise summary of the below input text.Ensure that responses covers the key points of the text.Only \n",
       "provide full sentence responses.\n",
       "### Input: \n",
       "#Person1#: Mister Ewing said we should show up at the conference center at \u001b[1;92m4:00\u001b[0m o'clock, right?\n",
       "#Person2#: Yes, he especially asked us not to be late. Some of the people from our east york branch office are \n",
       "coming, and he wants to make a good impression on them. How are you getting there?\n",
       "#Person1#: I was thinking of taking my car, but I think I'm just going to take the underground, because there is \n",
       "construction on the highway. What about you?\n",
       "#Person2#: I'll be taking the underground as well. Why don't we go together? I've been to the conference center \n",
       "only once, and I'm not sure if I can find my way around there.\n",
       "### Response :\n",
       "\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Mister Ewing asks #Person1# and #Person2# to arrive early for today's meeting with some representatives from their East York Branch Office. They will both use public transportation since they have no idea how to get to the Conference Center by themselves.\n",
      "### End of Section.</s>\n"
     ]
    }
   ],
   "source": [
    "#text = test_df['text'][500]\n",
    "print(text)\n",
    "result = generate(text, persisted_model, max_new_tokens=96, temp=0.1)\n",
    "#del persisted_model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "1c5eca4a-dbf3-463c-97e1-b2df66da15ad",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'The conversation between Person1 (Jenny) and Person2(Dr.Tyler), who are both students from university. In which they discuss about her schedule card with six subjects including chemistry, calculus, modern american novels and romatic poems as well as two seminars.</s>'"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "788fd3c3-140e-4f8d-8594-dfd01c8d6acd",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "c1af269a-c9c8-4a7b-898b-16ded1a5d4f5",
   "metadata": {},
   "source": [
    "## Merge the lora and original model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "251ae65f-46f8-45d6-94da-61edc884dccc",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unloading and merging model: 100%|██████████| 454/454 [00:18<00:00, 25.09it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 20.5 s, sys: 5.8 s, total: 26.3 s\n",
      "Wall time: 34.1 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "# Merges the parts of the model that were possibly loaded on different devices and unloads them from memory\n",
    "# ^^ from gpt4.  i suspect an halucination, verify.\n",
    "merged_model = persisted_model.merge_and_unload(progressbar=True)\n",
    "\n",
    "merged_model.save_pretrained(merged_path, safe_serialization=True)\n",
    "tokenizer.save_pretrained(merged_path)\n",
    "\n",
    "del persisted_model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "59d50514-acd2-48a5-abc3-ec86c29673de",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">work_for_2023_11_10/best_checkpoint/merged_model\n",
       "</pre>\n"
      ],
      "text/plain": [
       "work_for_2023_11_10/best_checkpoint/merged_model\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "total 13G\n",
      "   0 drwxr-xr-x 1 root root 4.0K Nov 10 22:10 ..\n",
      "4.0K -rw-r--r-- 1 root root  656 Nov 10 22:10 config.json\n",
      "   0 -rw-r--r-- 1 root root  183 Nov 10 22:10 generation_config.json\n",
      "9.3G -rw-r--r-- 1 root root 9.3G Nov 10 22:10 model-00001-of-00002.safetensors\n",
      "3.3G -rw-r--r-- 1 root root 3.3G Nov 10 22:10 model-00002-of-00002.safetensors\n",
      " 24K -rw-r--r-- 1 root root  24K Nov 10 22:10 model.safetensors.index.json\n",
      "4.0K -rw-r--r-- 1 root root  900 Nov 10 22:10 tokenizer_config.json\n",
      "   0 -rw-r--r-- 1 root root  414 Nov 10 22:10 special_tokens_map.json\n",
      "492K -rw-r--r-- 1 root root 489K Nov 10 22:10 tokenizer.model\n",
      "   0 drwxr-xr-x 1 root root 4.0K Nov 10 22:10 .\n",
      "1.8M -rw-r--r-- 1 root root 1.8M Nov 10 22:10 tokenizer.json\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n"
     ]
    }
   ],
   "source": [
    "print(merged_path)\n",
    "!ls -lathrs $merged_path"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "54c3c607-e405-4d95-bad2-102931f38686",
   "metadata": {},
   "source": [
    "## Load merged model from disk to be quantized"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d4e6b94e-4273-4cd2-8703-d268dfb5a001",
   "metadata": {},
   "source": [
    "### if dead again, try starting here"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1a8de00f-b0e1-4056-bd95-d74f910f1b11",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "e278c014-3120-443d-9713-405b44f4003c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "688896de383349e0bcc471c5dda6486c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Loading checkpoint shards:   0%|          | 0/2 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 19.5 s, sys: 2min 49s, total: 3min 9s\n",
      "Wall time: 1min 45s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "#qqqq\n",
    "tokenizer = AutoTokenizer.from_pretrained(merged_dir)\n",
    "merged_model_from_disk = AutoModelForCausalLM.from_pretrained( merged_dir, \n",
    "                                                              device_map=device,\n",
    "                                                             )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "51f0bcf5-3eb4-4858-9f27-634daeb486a6",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">### Instruction: \n",
       "Write a concise summary of the below input text.Ensure that responses covers the key points of the text.Only \n",
       "provide full sentence responses.\n",
       "### Input: \n",
       "#Person1#: Mister Ewing said we should show up at the conference center at <span style=\"color: #00ff00; text-decoration-color: #00ff00; font-weight: bold\">4:00</span> o'clock, right?\n",
       "#Person2#: Yes, he especially asked us not to be late. Some of the people from our east york branch office are \n",
       "coming, and he wants to make a good impression on them. How are you getting there?\n",
       "#Person1#: I was thinking of taking my car, but I think I'm just going to take the underground, because there is \n",
       "construction on the highway. What about you?\n",
       "#Person2#: I'll be taking the underground as well. Why don't we go together? I've been to the conference center \n",
       "only once, and I'm not sure if I can find my way around there.\n",
       "### Response :\n",
       "\n",
       "</pre>\n"
      ],
      "text/plain": [
       "### Instruction: \n",
       "Write a concise summary of the below input text.Ensure that responses covers the key points of the text.Only \n",
       "provide full sentence responses.\n",
       "### Input: \n",
       "#Person1#: Mister Ewing said we should show up at the conference center at \u001b[1;92m4:00\u001b[0m o'clock, right?\n",
       "#Person2#: Yes, he especially asked us not to be late. Some of the people from our east york branch office are \n",
       "coming, and he wants to make a good impression on them. How are you getting there?\n",
       "#Person1#: I was thinking of taking my car, but I think I'm just going to take the underground, because there is \n",
       "construction on the highway. What about you?\n",
       "#Person2#: I'll be taking the underground as well. Why don't we go together? I've been to the conference center \n",
       "only once, and I'm not sure if I can find my way around there.\n",
       "### Response :\n",
       "\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The meeting will start at four in the afternoon according to Mr. Ewing who has requested for everyone to arrive early so they do not miss out anything important during the event. He also mentioned that some members from their East York Branch Office would attend this meeting which means it’s very crucial since these individuals have never met before hence making an impactful first impression matters most when dealing with such situations where one needs to impress others by being punctual while ensuring everything goes\n"
     ]
    }
   ],
   "source": [
    "print(text)\n",
    "result = generate(text, merged_model_from_disk,max_new_tokens=96, temp = 0.01)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "9469bf85-7598-404e-9629-7a04ee4788c1",
   "metadata": {},
   "outputs": [],
   "source": [
    "#del merged_model_from_disk"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7891dd5c-bd1a-4df5-85f5-956af0f6e2e4",
   "metadata": {},
   "source": [
    "## Quantize it"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "93335f9b-1e43-4e1e-ad10-5dd7154ee036",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "4910"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "2a0fdab5-15b8-43b2-8d00-b0bac153833c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'work_for_2023_11_10/best_checkpoint/merged_model'"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "merged_path\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "f7373ac2-d526-4190-ae7a-cdf7a98e0dee",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "959d51c2020d4d8cb5db35343ca3e012",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Loading checkpoint shards:   0%|          | 0/2 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e719908a47b244a3adf58898ffecf2ec",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Quantizing model.layers blocks :   0%|          | 0/32 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Quantizing layers inside the block:   0%|          | 0/7 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Quantizing layers inside the block:   0%|          | 0/7 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Quantizing layers inside the block:   0%|          | 0/7 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Quantizing layers inside the block:   0%|          | 0/7 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Quantizing layers inside the block:   0%|          | 0/7 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Quantizing layers inside the block:   0%|          | 0/7 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Quantizing layers inside the block:   0%|          | 0/7 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Quantizing layers inside the block:   0%|          | 0/7 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Quantizing layers inside the block:   0%|          | 0/7 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Quantizing layers inside the block:   0%|          | 0/7 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Quantizing layers inside the block:   0%|          | 0/7 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Quantizing layers inside the block:   0%|          | 0/7 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Quantizing layers inside the block:   0%|          | 0/7 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Quantizing layers inside the block:   0%|          | 0/7 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Quantizing layers inside the block:   0%|          | 0/7 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Quantizing layers inside the block:   0%|          | 0/7 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Quantizing layers inside the block:   0%|          | 0/7 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Quantizing layers inside the block:   0%|          | 0/7 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Quantizing layers inside the block:   0%|          | 0/7 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Quantizing layers inside the block:   0%|          | 0/7 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Quantizing layers inside the block:   0%|          | 0/7 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Quantizing layers inside the block:   0%|          | 0/7 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Quantizing layers inside the block:   0%|          | 0/7 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Quantizing layers inside the block:   0%|          | 0/7 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Quantizing layers inside the block:   0%|          | 0/7 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Quantizing layers inside the block:   0%|          | 0/7 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Quantizing layers inside the block:   0%|          | 0/7 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Quantizing layers inside the block:   0%|          | 0/7 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Quantizing layers inside the block:   0%|          | 0/7 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Quantizing layers inside the block:   0%|          | 0/7 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Quantizing layers inside the block:   0%|          | 0/7 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Quantizing layers inside the block:   0%|          | 0/7 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 35min 36s, sys: 2min 26s, total: 38min 2s\n",
      "Wall time: 18min 51s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "from transformers import GPTQConfig\n",
    "\n",
    "quantization_config = GPTQConfig(\n",
    "    bits=quantization_bits,\n",
    "    dataset=[\"c4\"],\n",
    "    desc_act=False,\n",
    ")\n",
    "\n",
    "quant_model = AutoModelForCausalLM.from_pretrained(\n",
    "    merged_path, \n",
    "    quantization_config=quantization_config, \n",
    "    device_map=device\n",
    ")\n",
    "tokenizer = AutoTokenizer.from_pretrained(merged_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "9e35e993-b47b-42c8-b683-4c243f693d4f",
   "metadata": {},
   "outputs": [],
   "source": [
    "#del persisted_model"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5297fcab-e846-42d4-a32f-e233dd673d30",
   "metadata": {},
   "source": [
    "## Save quantized model to disk"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "acc207ed-e2a3-4af9-920c-7c447a2bd2ec",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'work_for_2023_11_10/quantized_8bit'"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "quant_path"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "d8431321-8378-42e4-b14b-81ab9fbd31ee",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">saving to work_for_2023_11_10/quantized_8bit\n",
       "</pre>\n"
      ],
      "text/plain": [
       "saving to work_for_2023_11_10/quantized_8bit\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 1.2 s, sys: 781 ms, total: 1.98 s\n",
      "Wall time: 4.24 s\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "('work_for_2023_11_10/quantized_8bit/tokenizer_config.json',\n",
       " 'work_for_2023_11_10/quantized_8bit/special_tokens_map.json',\n",
       " 'work_for_2023_11_10/quantized_8bit/tokenizer.model',\n",
       " 'work_for_2023_11_10/quantized_8bit/added_tokens.json',\n",
       " 'work_for_2023_11_10/quantized_8bit/tokenizer.json')"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "quant_path = os.path.join(project_dir, quantization_dir)\n",
    "print(f\"saving to {quant_path}\")\n",
    "\n",
    "# Save the quantized model\n",
    "quant_model.save_pretrained(quant_path, safe_serialization=True)\n",
    "tokenizer.save_pretrained(quant_path)\n",
    "#del persisted_model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "c63e7173-bce1-42d7-8b38-cd1684247665",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">done\n",
       "</pre>\n"
      ],
      "text/plain": [
       "done\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "print('done')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "9d01927a-4ef2-4718-8f03-34caa6c19172",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">work_for_2023_11_10/quantized_8bit\n",
       "</pre>\n"
      ],
      "text/plain": [
       "work_for_2023_11_10/quantized_8bit\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "total 3.7G\n",
      "   0 drwxr-xr-x 1 root root 4.0K Nov 10 22:41 ..\n",
      "4.0K -rw-r--r-- 1 root root 1.2K Nov 10 22:41 config.json\n",
      "   0 -rw-r--r-- 1 root root  183 Nov 10 22:41 generation_config.json\n",
      "3.7G -rw-r--r-- 1 root root 3.7G Nov 10 22:41 model.safetensors\n",
      "4.0K -rw-r--r-- 1 root root  900 Nov 10 22:41 tokenizer_config.json\n",
      "   0 -rw-r--r-- 1 root root  414 Nov 10 22:41 special_tokens_map.json\n",
      "492K -rw-r--r-- 1 root root 489K Nov 10 22:41 tokenizer.model\n",
      "   0 drwxr-xr-x 1 root root 4.0K Nov 10 22:41 .\n",
      "1.8M -rw-r--r-- 1 root root 1.8M Nov 10 22:41 tokenizer.json\n"
     ]
    }
   ],
   "source": [
    "print(quant_path)\n",
    "!ls -latrsh $quant_path"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "bad76bea-cf16-49eb-aa88-41f7a4aa8b95",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "3d21bcd6-7817-48d2-965a-4fdd56b22d28",
   "metadata": {},
   "source": [
    "## Load quantized model from disk"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "d47bde57-c8ba-4667-95ad-5d00ddf8b2dd",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">work_for_2023_11_10/quantized_8bit\n",
       "</pre>\n"
      ],
      "text/plain": [
       "work_for_2023_11_10/quantized_8bit\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 6.47 s, sys: 8.58 s, total: 15 s\n",
      "Wall time: 1min\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "from transformers import AutoModelForCausalLM, AutoTokenizer\n",
    "\n",
    "print(quant_path)\n",
    "quant_model_from_disk = AutoModelForCausalLM.from_pretrained( quant_path, device_map=device)\n",
    "tokenizer = AutoTokenizer.from_pretrained(quant_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "355987cd-30e6-4190-b6f2-c9c580869f80",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">### Instruction: \n",
       "Write a concise summary of the below input text.Ensure that responses covers the key points of the text.Only \n",
       "provide full sentence responses.\n",
       "### Input: \n",
       "#Person1#: Mister Ewing said we should show up at the conference center at <span style=\"color: #00ff00; text-decoration-color: #00ff00; font-weight: bold\">4:00</span> o'clock, right?\n",
       "#Person2#: Yes, he especially asked us not to be late. Some of the people from our east york branch office are \n",
       "coming, and he wants to make a good impression on them. How are you getting there?\n",
       "#Person1#: I was thinking of taking my car, but I think I'm just going to take the underground, because there is \n",
       "construction on the highway. What about you?\n",
       "#Person2#: I'll be taking the underground as well. Why don't we go together? I've been to the conference center \n",
       "only once, and I'm not sure if I can find my way around there.\n",
       "### Response :\n",
       " \n",
       " ==============================\n",
       "</pre>\n"
      ],
      "text/plain": [
       "### Instruction: \n",
       "Write a concise summary of the below input text.Ensure that responses covers the key points of the text.Only \n",
       "provide full sentence responses.\n",
       "### Input: \n",
       "#Person1#: Mister Ewing said we should show up at the conference center at \u001b[1;92m4:00\u001b[0m o'clock, right?\n",
       "#Person2#: Yes, he especially asked us not to be late. Some of the people from our east york branch office are \n",
       "coming, and he wants to make a good impression on them. How are you getting there?\n",
       "#Person1#: I was thinking of taking my car, but I think I'm just going to take the underground, because there is \n",
       "construction on the highway. What about you?\n",
       "#Person2#: I'll be taking the underground as well. Why don't we go together? I've been to the conference center \n",
       "only once, and I'm not sure if I can find my way around there.\n",
       "### Response :\n",
       " \n",
       " ==============================\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Mister Ewing asks #Person1# and #Person2# not to be late for the meeting in the conference center. They will both use public transportation.\n",
      "### End\n",
      "</s>\n"
     ]
    }
   ],
   "source": [
    "text = test_df['text'][1]\n",
    "print(text,\"\\n\",\"=\"*30)\n",
    "result = generate(text, quant_model_from_disk,max_new_tokens=96, temp = 0.01)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "e2e5cefb-746c-4e92-854b-16f3ff2bfdea",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'Mister Ewing asks #Person1# and #Person2# not to be late for the meeting in the conference center. They will both use public transportation.\\n### End\\n</s>'"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "result"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "bd2bf0f3-489f-41ac-b217-f0e0231c5b79",
   "metadata": {},
   "source": [
    "## Push model to HuggingFace hub"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "81ccf35c-da18-49a5-a314-3af228d0fe6d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "24e01154e1484e1885ead7e261462dfc",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "pytorch_model.bin:   0%|          | 0.00/3.90G [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 21.4 s, sys: 6.61 s, total: 28 s\n",
      "Wall time: 58min 25s\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "CommitInfo(commit_url='https://huggingface.co/andersonjas/llama2-7b-dialogsum-qlora-gptq/commit/ef4c03b44b0f94ae2bf2cece9dff714594a04183', commit_message='Upload LlamaForCausalLM', commit_description='', oid='ef4c03b44b0f94ae2bf2cece9dff714594a04183', pr_url=None, pr_revision=None, pr_num=None)"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "tokenizer.push_to_hub(destination_hf_model_name, token=hf_token)\n",
    "quant_model_from_disk.push_to_hub(destination_hf_model_name, token = hf_token)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "5556e7c9-97cc-4831-a765-33a7bf9ec4fb",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d219e918b12943bc8588933edf4cd348",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading (…)lve/main/config.json:   0%|          | 0.00/1.20k [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d9c63fb40fb846e7af5493d124093439",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading pytorch_model.bin:   0%|          | 0.00/3.90G [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 12.3 s, sys: 12.6 s, total: 24.9 s\n",
      "Wall time: 2min 15s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "# %%\n",
    "#from transformers import GPTQConfig\n",
    "\n",
    "# quantization_config = GPTQConfig(\n",
    "#     bits=4,\n",
    "#     dataset=[\"c4\"],\n",
    "#     desc_act=False,\n",
    "# )\n",
    "#quantization_config_loading = GPTQConfig(bits=4, disable_exllama=True)\n",
    "from_disk_quant_model = AutoModelForCausalLM.from_pretrained(\n",
    "    destination_hf_model_name, \n",
    "    #quantization_config=quantization_config_loading, \n",
    "    device_map=device,\n",
    ")\n",
    "tokenizer = AutoTokenizer.from_pretrained(destination_hf_model_name)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9ab71ee7-6115-4358-b186-b2d856aeb216",
   "metadata": {},
   "source": [
    "## Test the model by pulling from hugging face hub"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "7eb3d1fe-666e-4705-9761-f4fc3b2db399",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">### Instruction: \n",
       "Write a concise summary of the below input text.Ensure that responses covers the key points of the text.Only \n",
       "provide full sentence responses.\n",
       "### Input: \n",
       "#Person1#: Mister Ewing said we should show up at the conference center at <span style=\"color: #00ff00; text-decoration-color: #00ff00; font-weight: bold\">4:00</span> o'clock, right?\n",
       "#Person2#: Yes, he especially asked us not to be late. Some of the people from our east york branch office are \n",
       "coming, and he wants to make a good impression on them. How are you getting there?\n",
       "#Person1#: I was thinking of taking my car, but I think I'm just going to take the underground, because there is \n",
       "construction on the highway. What about you?\n",
       "#Person2#: I'll be taking the underground as well. Why don't we go together? I've been to the conference center \n",
       "only once, and I'm not sure if I can find my way around there.\n",
       "### Response :\n",
       "\n",
       "</pre>\n"
      ],
      "text/plain": [
       "### Instruction: \n",
       "Write a concise summary of the below input text.Ensure that responses covers the key points of the text.Only \n",
       "provide full sentence responses.\n",
       "### Input: \n",
       "#Person1#: Mister Ewing said we should show up at the conference center at \u001b[1;92m4:00\u001b[0m o'clock, right?\n",
       "#Person2#: Yes, he especially asked us not to be late. Some of the people from our east york branch office are \n",
       "coming, and he wants to make a good impression on them. How are you getting there?\n",
       "#Person1#: I was thinking of taking my car, but I think I'm just going to take the underground, because there is \n",
       "construction on the highway. What about you?\n",
       "#Person2#: I'll be taking the underground as well. Why don't we go together? I've been to the conference center \n",
       "only once, and I'm not sure if I can find my way around there.\n",
       "### Response :\n",
       "\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Mister Ewing asks #Person1# and #Person2# not to be late for the meeting in the conference center. They will both use public transportation.\n",
      "### End\n",
      "</s>\n"
     ]
    }
   ],
   "source": [
    "#text = test_df['text'][500]\n",
    "text = test_df['text'][1]\n",
    "print(text)\n",
    "result = generate(text, from_disk_quant_model, max_new_tokens=85,temp=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d40f01a4-fb35-4e03-af5c-c4dd93566285",
   "metadata": {},
   "outputs": [],
   "source": [
    "result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "id": "9a3fe8f6-478f-4c18-8c5e-cda334ce9282",
   "metadata": {},
   "outputs": [],
   "source": [
    "def wipe(model):\n",
    "    del model\n",
    "    import torch\n",
    "    torch.cuda.empty_cache()\n",
    "    torch.cuda.empty_cache()\n",
    "    import gc\n",
    "    gc.collect()\n",
    "\n",
    "#wipe(quant_model_from_disk)\n",
    "#wipe(from_disk_quant_model)\n",
    "#del quant_model_from_disk\n",
    "#del from_disk_quant_model\n",
    "#del quant_model\n",
    "#del merged_model\n",
    "#del persisted_model\n",
    "#del prepared_model\n",
    "#del peft_model\n",
    "torch.cuda.empty_cache()\n",
    "import gc"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0534db9f-f3d6-41bc-b6a1-307ee1afedcb",
   "metadata": {},
   "source": [
    "# Train a GPTQ model from scratch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "id": "e8e3f643-1b54-483c-9484-629be675e024",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "You passed `quantization_config` to `from_pretrained` but the model you're loading already has a `quantization_config` attribute and has already quantized weights. However, loading attributes (e.g. disable_exllama, use_cuda_fp16, max_input_length) will be overwritten with the one you passed to `from_pretrained`. The rest will be ignored.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "trainable params: 16,777,216 || all params: 279,187,456 || trainable%: 6.009301506726721\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "8f05f2c074bb417da5c017ab84a3f357",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Map:   0%|          | 0/12148 [00:00<?, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/app/venv/lib/python3.10/site-packages/transformers/optimization.py:411: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use the PyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning\n",
      "  warnings.warn(\n",
      "You're using a LlamaTokenizerFast tokenizer. Please note that with a fast tokenizer, using the `__call__` method is faster than using a method to encode the text followed by a call to the `pad` method to get a padded encoding.\n",
      "/app/venv/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='300' max='300' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [300/300 18:24, Epoch 0/1]\n",
       "    </div>\n",
       "    <table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       " <tr style=\"text-align: left;\">\n",
       "      <th>Step</th>\n",
       "      <th>Training Loss</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>25</td>\n",
       "      <td>1.356200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>50</td>\n",
       "      <td>1.158900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>75</td>\n",
       "      <td>1.136200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>100</td>\n",
       "      <td>1.133700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>125</td>\n",
       "      <td>1.107700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>150</td>\n",
       "      <td>1.099700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>175</td>\n",
       "      <td>1.132100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>200</td>\n",
       "      <td>1.109000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>225</td>\n",
       "      <td>1.111000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>250</td>\n",
       "      <td>1.120500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>275</td>\n",
       "      <td>1.088000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>300</td>\n",
       "      <td>1.135900</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table><p>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from huggingface_hub import login\n",
    "from dotenv import load_dotenv\n",
    "\n",
    "import torch\n",
    "from transformers import AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig\n",
    "from peft import LoraConfig, get_peft_model\n",
    "from peft import prepare_model_for_kbit_training\n",
    "from transformers import GPTQConfig\n",
    "from transformers import Trainer, TrainingArguments #, DataCollatorForLanguageModeling\n",
    "from trl import SFTTrainer\n",
    "\n",
    "quant_model_id = \"TheBloke/Llama-2-7B-GPTQ\"\n",
    "\n",
    "tokenizer = AutoTokenizer.from_pretrained(quant_model_id)\n",
    "tokenizer.pad_token = tokenizer.eos_token\n",
    "tokenizer.padding_side = \"right\"\n",
    "\n",
    "\n",
    "quantization_config_loading = GPTQConfig(bits=4, \n",
    "                                         disable_exllama=True,\n",
    "                                        )\n",
    "config = LoraConfig(\n",
    "                            r=16,\n",
    "                            lora_alpha=32,\n",
    "                            target_modules=[\"k_proj\",\"o_proj\",\"q_proj\",\"v_proj\"],\n",
    "                            lora_dropout=0.05,\n",
    "                            bias=\"none\",\n",
    "                            task_type=\"CAUSAL_LM\"\n",
    "                        )\n",
    "training_args=TrainingArguments(\n",
    "                                per_device_train_batch_size=4, # Batch size per device during training\n",
    "                                gradient_accumulation_steps=4, # Number of steps to accumulate gradients before updating model parameters\n",
    "                                warmup_steps=2, # Number of steps for the warm-up phase\n",
    "                                max_steps=300, # Total number of training steps to perform\n",
    "                                learning_rate=2e-4, # Initial learning rate\n",
    "                                fp16=True, # Enable mixed precision training to improve performance\n",
    "                                logging_steps=25, # Frequency of logging training information\n",
    "                                output_dir=quantized_train_path, # Directory for saving output (like model checkpoints)\n",
    "                                optim=\"adamw_hf\", # Optimizer to be used for training, here it's a variant of AdamW from Hugging Face\n",
    "                                save_strategy=\"epoch\", # Strategy for saving model checkpoints (here, at the end of each epoch)\n",
    "                                report_to=\"none\", # Disable reporting to any external services\n",
    "                            )\n",
    "\n",
    "\n",
    "quant_model = AutoModelForCausalLM.from_pretrained(quant_model_id, \n",
    "                                             quantization_config=quantization_config_loading, \n",
    "                                             device_map=\"auto\",\n",
    "                                            )\n",
    "\n",
    "\n",
    "quant_model.config.use_cache = False\n",
    "quant_model.config.pretraining_tp = 1\n",
    "quant_model.gradient_checkpointing_enable()\n",
    "\n",
    "\n",
    "# lora_config = LoraConfig(\n",
    "#     r=8,\n",
    "#     lora_alpha=32,\n",
    "#     target_modules=[\"k_proj\",\"o_proj\",\"q_proj\",\"v_proj\"],\n",
    "#     lora_dropout=0.05,\n",
    "#     bias=\"none\",\n",
    "#     task_type=\"CAUSAL_LM\"\n",
    "# )\n",
    "\n",
    "prepared_model = prepare_model_for_kbit_training(quant_model)\n",
    "peft_model = get_peft_model(prepared_model, config)\n",
    "peft_model.print_trainable_parameters()\n",
    "peft_model.config.use_cache = False # silence the warnings. Please re-enable for inference!\n",
    "\n",
    "# tokenizer.pad_token = tokenizer.eos_token\n",
    "# tokenizer.padding_side = \"right\"\n",
    "\n",
    "# training_args=TrainingArguments(\n",
    "#         per_device_train_batch_size=4,\n",
    "#         gradient_accumulation_steps=4,\n",
    "#         warmup_steps=2,\n",
    "#         max_steps=100,\n",
    "#         learning_rate=2e-4,\n",
    "#         fp16=True, #use mixed precision training\n",
    "#         logging_steps=25,\n",
    "#         output_dir=quantized_train_path,\n",
    "#         optim=\"adamw_hf\",\n",
    "#         save_strategy=\"epoch\",\n",
    "#         report_to=\"none\")\n",
    "\n",
    "\n",
    "trainer = SFTTrainer(\n",
    "    model=peft_model,\n",
    "    args=training_args,\n",
    "    train_dataset=train_dataset,\n",
    "    #peft_config=lora_config,\n",
    "    dataset_text_field=\"text\",\n",
    "    tokenizer=tokenizer,\n",
    "    packing=False,\n",
    "    max_seq_length=512)\n",
    "\n",
    "train_result = trainer.train()\n",
    "#Also I noticed that you pass a PeftModel to SFTTrainer together with a peft_config argument, this might lead to create a nested peft model which can lead to some bugs."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "id": "b2eccb11-1481-4b7e-87df-b56b310c2c36",
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "checkpoint_name =\"final_checkpoints_gptqsummarizer_7b_peft\"\n",
    "output_dir = os.path.join(quantized_train_path, checkpoint_name)\n",
    "trainer.model.save_pretrained(output_dir)\n",
    "from peft import AutoPeftModelForCausalLM\n",
    "# To perform inference on the test dataset example load the model from the checkpoint\n",
    "persisted_model = AutoPeftModelForCausalLM.from_pretrained(\n",
    "    output_dir,\n",
    "    low_cpu_mem_usage=True,\n",
    "    return_dict=True,\n",
    "    torch_dtype=torch.float16,\n",
    "    device_map=\"cuda\",\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "id": "8b744471-42de-4ac5-8ea6-3db56e1d9a8f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">work_for_2023_11_10/quant_trained_model_new/final_checkpoints_gptqsummarizer_7b_peft\n",
       "</pre>\n"
      ],
      "text/plain": [
       "work_for_2023_11_10/quant_trained_model_new/final_checkpoints_gptqsummarizer_7b_peft\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "total 65M\n",
      "  0 drwxr-xr-x 1 root root 4.0K Nov 11 03:48 ..\n",
      "  0 -rw-r--r-- 1 root root  525 Nov 11 03:48 README.md\n",
      "65M -rw-r--r-- 1 root root  65M Nov 11 03:48 adapter_model.bin\n",
      "  0 drwxr-xr-x 1 root root 4.0K Nov 11 03:48 .\n",
      "  0 -rw-r--r-- 1 root root  475 Nov 11 03:48 adapter_config.json\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n"
     ]
    }
   ],
   "source": [
    "print(output_dir)\n",
    "!ls -latrhs $output_dir"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "id": "85b5ce4a-bed3-4d79-9e7f-afdebf330750",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">==============================\n",
       "</pre>\n"
      ],
      "text/plain": [
       "==============================\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/app/venv/lib/python3.10/site-packages/transformers/generation/utils.py:1421: UserWarning: You have modified the pretrained model configuration to control generation. This is a deprecated strategy to control generation and will be removed soon, in a future version. Please use and modify the model generation configuration (see https://huggingface.co/docs/transformers/generation_strategies#default-text-generation-configuration )\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Mister Ewing asks #Person1# and #Person2# to arrive early for the meeting so they will have time to meet some colleagues in their East York Branch Office. They decide to travel by subway instead of driving due to traffic congestion.\n",
      "### EndResponse ?\n",
      "</s>\n"
     ]
    }
   ],
   "source": [
    "text = test_df['text'][1]\n",
    "print(\"=\"*30)\n",
    "result = generate(text, persisted_model, max_new_tokens=96, temp=0.01)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "id": "7fe02b5c-6a32-419f-8daa-f82f2d74d4a9",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">### Instruction: \n",
       "Write a concise summary of the below input text.Ensure that responses covers the key points of the text.Only \n",
       "provide full sentence responses.\n",
       "### Input: \n",
       "#Person1#: Mister Ewing said we should show up at the conference center at <span style=\"color: #00ff00; text-decoration-color: #00ff00; font-weight: bold\">4:00</span> o'clock, right?\n",
       "#Person2#: Yes, he especially asked us not to be late. Some of the people from our east york branch office are \n",
       "coming, and he wants to make a good impression on them. How are you getting there?\n",
       "#Person1#: I was thinking of taking my car, but I think I'm just going to take the underground, because there is \n",
       "construction on the highway. What about you?\n",
       "#Person2#: I'll be taking the underground as well. Why don't we go together? I've been to the conference center \n",
       "only once, and I'm not sure if I can find my way around there.\n",
       "### Response :\n",
       " ------------------------------------------------------------\n",
       "</pre>\n"
      ],
      "text/plain": [
       "### Instruction: \n",
       "Write a concise summary of the below input text.Ensure that responses covers the key points of the text.Only \n",
       "provide full sentence responses.\n",
       "### Input: \n",
       "#Person1#: Mister Ewing said we should show up at the conference center at \u001b[1;92m4:00\u001b[0m o'clock, right?\n",
       "#Person2#: Yes, he especially asked us not to be late. Some of the people from our east york branch office are \n",
       "coming, and he wants to make a good impression on them. How are you getting there?\n",
       "#Person1#: I was thinking of taking my car, but I think I'm just going to take the underground, because there is \n",
       "construction on the highway. What about you?\n",
       "#Person2#: I'll be taking the underground as well. Why don't we go together? I've been to the conference center \n",
       "only once, and I'm not sure if I can find my way around there.\n",
       "### Response :\n",
       " ------------------------------------------------------------\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">temp <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.5</span>:</pre>\n"
      ],
      "text/plain": [
       "temp \u001b[1;36m0.5\u001b[0m:"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Mister Ewing asks #Person1# and #Person2# to arrive early for the meeting so they will have time to meet some colleagues in their East York Branch Office. They decide to travel by subway instead of driving due to traffic congestion.\n",
      "### EndResponse ?\n",
      "</s>\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">temp <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.75</span>:</pre>\n"
      ],
      "text/plain": [
       "temp \u001b[1;36m0.75\u001b[0m:"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Mister Ewing asks #Person1# and #Person2# to arrive early for the meeting so they will have time to meet some colleagues in their East York Branch Office. They decide to travel by subway instead of driving due to traffic congestion.\n",
      "### EndResponse ?\n",
      "</s>\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">temp <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">1.0</span>:</pre>\n"
      ],
      "text/plain": [
       "temp \u001b[1;36m1.0\u001b[0m:"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Mister Ewing asks #Person1# and #Person2# to arrive early for the meeting so they decide to travel by subway instead of driving their cars. They will also go together since it would help each other in finding directions.\n",
      "### EndResponse ?\n",
      "</s>\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">temp <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">1.25</span>:</pre>\n"
      ],
      "text/plain": [
       "temp \u001b[1;36m1.25\u001b[0m:"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Mister Ewing asks #Person1# and #Person2# to arrive early for the meeting so they decide to travel by subway instead of driving their cars. They will also go together since it would help each other in finding directions.\n",
      "### EndResponse ?\n",
      "</s>\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">temp <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">1.5</span>:</pre>\n"
      ],
      "text/plain": [
       "temp \u001b[1;36m1.5\u001b[0m:"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Mister Ewing asks #Person1# and #Person2# to arrive early for the meeting so they will have time to meet some colleagues in their East York Branch Office. They decide to travel by subway instead of driving due to traffic congestion.\n",
      "### EndResponse ?\n",
      "</s>\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">------------------------------------------------------------\n",
       "</pre>\n"
      ],
      "text/plain": [
       "------------------------------------------------------------\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">top_p <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.0</span>:</pre>\n"
      ],
      "text/plain": [
       "top_p \u001b[1;36m0.0\u001b[0m:"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Mister Ewing asks #Person1# and #Person2# to arrive early for the meeting so they will have time to meet some colleagues in their East York Branch Office. They decide to travel by subway instead of driving due to traffic congestion.\n",
      "### EndResponse ?\n",
      "</s>\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">top_p <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.1</span>:</pre>\n"
      ],
      "text/plain": [
       "top_p \u001b[1;36m0.1\u001b[0m:"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Mister Ewing asks #Person1# and #Person2# to arrive early for the meeting so they decide to travel by subway instead of driving their cars. They will also go together since it would help each other in finding directions.\n",
      "### EndResponse ?\n",
      "</s>\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">top_p <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.2</span>:</pre>\n"
      ],
      "text/plain": [
       "top_p \u001b[1;36m0.2\u001b[0m:"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Mister Ewing asks #Person1# and #Person2# to arrive early for the meeting so they decide to travel by subway instead of driving their cars. They will also get off at the same station.\n",
      "### EndResponse ?\n",
      "</s>\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">top_p <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.30000000000000004</span>:</pre>\n"
      ],
      "text/plain": [
       "top_p \u001b[1;36m0.30000000000000004\u001b[0m:"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Mister Ewing asks #Person1# and #Person2# to arrive early for an important meeting with some representatives from their East York Branch Office. They will both use the subway since they have no idea how to get to the place by themselves.\n",
      "### EndResponse ?\n",
      "</s>\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">top_p <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.4</span>:</pre>\n"
      ],
      "text/plain": [
       "top_p \u001b[1;36m0.4\u001b[0m:"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Mister Ewing asks #Person1# and #Person2# to arrive early for an important meeting in the afternoon so they decide to take the subway instead of driving their cars.\n",
      "### EndResponse ?\n",
      "</s>\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">top_p <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.5</span>:</pre>\n"
      ],
      "text/plain": [
       "top_p \u001b[1;36m0.5\u001b[0m:"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Mister Ewing asks #Person1# and #Person2# to arrive at the conference center by 4 pm so they decide to take an underground train instead of driving their cars.\n",
      "### EndResponse !\n",
      "</s>\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">top_p <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.6000000000000001</span>:</pre>\n"
      ],
      "text/plain": [
       "top_p \u001b[1;36m0.6000000000000001\u001b[0m:"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Mister Ewing asks #Person1# and #Person2# to attend the meeting at 4 pm so they will take the subway instead of driving their cars. They decide to travel by train for convenience.\n",
      "### EndResponse !\n",
      "</s>\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">top_p <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.7000000000000001</span>:</pre>\n"
      ],
      "text/plain": [
       "top_p \u001b[1;36m0.7000000000000001\u001b[0m:"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Ms. Liu will get to the meeting by subway with Mr. Ewing who asks their colleagues in East York Branch Office to come. They decide to go together since they have never visited it before.\n",
      "### EndResponse\n",
      "</s>\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">top_p <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.8</span>:</pre>\n"
      ],
      "text/plain": [
       "top_p \u001b[1;36m0.8\u001b[0m:"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "#Person1# will take #Person2# by underground to attend the meeting with Mr. Ewing who asks them not to be late.\n",
      "### EndResponse ]\n",
      "### Person3#! You were supposed to meet me half an hour ago! Where have you gone this time?\n",
      "#Person2#: Sorry. There seems to be some problem in the traffic today. It took longer than expected for me to get here. But what do you mean 'this time?' Have I missed something before?\n",
      "### Response :\n",
      "Mr. Ewing tells #Person1# and #Person2# they mustn't miss his appointment and reminds him it happened twice previously.\n",
      "### EndResponse ]\n",
      "### Person3#, could you please give your presentation first? The clock shows five minutes past nine already. Thank you very much for waiting so patiently.\n",
      "##\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">top_p <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.9</span>:</pre>\n"
      ],
      "text/plain": [
       "top_p \u001b[1;36m0.9\u001b[0m:"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Mister Ewing asks #Person1# and #Person2# to come early for the meeting today so they decide to take public transportation.\n",
      "### EndResponse ?\n",
      "</s>\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">top_p <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">1.0</span>:</pre>\n"
      ],
      "text/plain": [
       "top_p \u001b[1;36m1.0\u001b[0m:"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Mister Ewing asks #Person1# and #Person2# to arrive early at the meeting place so they will take an underground train there with each other for convenience.\n",
      "### EndResponse ?\n",
      "</s>\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">------------------------------------------------------------\n",
       "</pre>\n"
      ],
      "text/plain": [
       "------------------------------------------------------------\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">top_k <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0</span>:</pre>\n"
      ],
      "text/plain": [
       "top_k \u001b[1;36m0\u001b[0m:"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Mister Ewing asks #Person1# and #Person2# to arrive early for the meeting so they decide to travel by subway instead of driving their cars. They will also go together since it would help each other in finding directions.\n",
      "### EndResponse ?\n",
      "</s>\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">top_k <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">10</span>:</pre>\n"
      ],
      "text/plain": [
       "top_k \u001b[1;36m10\u001b[0m:"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Mister Ewing asks #Person1# and #Person2# to arrive early for the meeting so they will have time to meet some colleagues in their East York Branch Office. They decide to travel by subway instead of driving due to traffic congestion.\n",
      "### EndResponse ?\n",
      "</s>\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">top_k <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">20</span>:</pre>\n"
      ],
      "text/plain": [
       "top_k \u001b[1;36m20\u001b[0m:"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Mister Ewing asks #Person1# and #Person2# to arrive early for the meeting so they decide to travel by subway instead of driving their cars. They will also go together since it would help each other in finding directions.\n",
      "### EndResponse ?\n",
      "</s>\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">top_k <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">30</span>:</pre>\n"
      ],
      "text/plain": [
       "top_k \u001b[1;36m30\u001b[0m:"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Mister Ewing asks #Person1# and #Person2# to arrive early for the meeting so they will have time to meet some colleagues in their East York Branch Office. They decide to travel by subway instead of driving due to traffic congestion.\n",
      "### EndResponse ?\n",
      "</s>\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">top_k <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">40</span>:</pre>\n"
      ],
      "text/plain": [
       "top_k \u001b[1;36m40\u001b[0m:"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Mister Ewing asks #Person1# and #Person2# to arrive early for the meeting so they will have time to meet some colleagues in their East York Branch Office. They decide to travel by subway instead of driving due to traffic congestion.\n",
      "### EndResponse ?\n",
      "</s>\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">top_k <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">50</span>:</pre>\n"
      ],
      "text/plain": [
       "top_k \u001b[1;36m50\u001b[0m:"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Mister Ewing asks #Person1# and #Person2# to arrive early for the meeting so they will have time to meet some colleagues in their East York Branch Office. They decide to travel by subway instead of driving due to traffic congestion.\n",
      "### EndResponse ?\n",
      "</s>\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">top_k <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">60</span>:</pre>\n"
      ],
      "text/plain": [
       "top_k \u001b[1;36m60\u001b[0m:"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Mister Ewing asks #Person1# and #Person2# to arrive early for the meeting so they will have time to meet some colleagues in their East York Branch Office. They decide to travel by subway instead of driving due to traffic congestion.\n",
      "### EndResponse ?\n",
      "</s>\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">top_k <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">70</span>:</pre>\n"
      ],
      "text/plain": [
       "top_k \u001b[1;36m70\u001b[0m:"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Mister Ewing asks #Person1# and #Person2# to arrive early for the meeting so they decide to travel by subway instead of driving their cars. They will also go together since it would help each other in finding directions.\n",
      "### EndResponse ?\n",
      "</s>\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">top_k <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">80</span>:</pre>\n"
      ],
      "text/plain": [
       "top_k \u001b[1;36m80\u001b[0m:"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Mister Ewing asks #Person1# and #Person2# to arrive early for the meeting so they will have time to meet some colleagues in their East York Branch Office. They decide to travel by subway instead of driving due to traffic congestion.\n",
      "### EndResponse ?\n",
      "</s>\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">top_k <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">90</span>:</pre>\n"
      ],
      "text/plain": [
       "top_k \u001b[1;36m90\u001b[0m:"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Mister Ewing asks #Person1# and #Person2# to arrive early for the meeting so they will have time to meet some colleagues in their East York Branch Office. They decide to travel by subway instead of driving due to traffic congestion.\n",
      "### EndResponse ?\n",
      "</s>\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">top_k <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">100</span>:</pre>\n"
      ],
      "text/plain": [
       "top_k \u001b[1;36m100\u001b[0m:"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Mister Ewing asks #Person1# and #Person2# to arrive early for the meeting so they will have time to meet some colleagues in their East York Branch Office. They decide to travel by subway instead of driving due to traffic congestion.\n",
      "### EndResponse ?\n",
      "</s>\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">------------------------------------------------------------\n",
       "</pre>\n"
      ],
      "text/plain": [
       "------------------------------------------------------------\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">do_sample <span style=\"color: #00ff00; text-decoration-color: #00ff00; font-style: italic\">True</span>:</pre>\n"
      ],
      "text/plain": [
       "do_sample \u001b[3;92mTrue\u001b[0m:"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Mister Ewing asks #Person1# and #Person2# to arrive early for the meeting so they decide to travel by subway instead of driving their cars. They will also go together since it would help each other in finding directions.\n",
      "### EndResponse ?\n",
      "</s>\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">do_sample <span style=\"color: #ff0000; text-decoration-color: #ff0000; font-style: italic\">False</span>:</pre>\n"
      ],
      "text/plain": [
       "do_sample \u001b[3;91mFalse\u001b[0m:"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/app/venv/lib/python3.10/site-packages/transformers/generation/configuration_utils.py:362: UserWarning: `do_sample` is set to `False`. However, `temperature` is set to `0.7` -- this flag is only used in sample-based generation modes. You should set `do_sample=True` or unset `temperature`.\n",
      "  warnings.warn(\n",
      "/app/venv/lib/python3.10/site-packages/transformers/generation/configuration_utils.py:367: UserWarning: `do_sample` is set to `False`. However, `top_p` is set to `0.1` -- this flag is only used in sample-based generation modes. You should set `do_sample=True` or unset `top_p`.\n",
      "  warnings.warn(\n",
      "/app/venv/lib/python3.10/site-packages/transformers/generation/configuration_utils.py:377: UserWarning: `do_sample` is set to `False`. However, `top_k` is set to `None` -- this flag is only used in sample-based generation modes. You should set `do_sample=True` or unset `top_k`.\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Mister Ewing asks #Person1# and #Person2# to arrive early for the meeting so they will have time to meet some colleagues in their East York Branch Office. They decide to travel by subway instead of driving due to traffic congestion.\n",
      "### EndResponse ?\n",
      "</s>\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">------------------------------------------------------------\n",
       "</pre>\n"
      ],
      "text/plain": [
       "------------------------------------------------------------\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">repetition_penalty <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">1.0</span>:</pre>\n"
      ],
      "text/plain": [
       "repetition_penalty \u001b[1;36m1.0\u001b[0m:"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "#Person1# and #Person2# will take the underground to the conference center.\n",
      "### End Response\n",
      "###\n",
      "###\n",
      "###\n",
      "###\n",
      "###\n",
      "###\n",
      "###\n",
      "###\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">repetition_penalty <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">1.1</span>:</pre>\n"
      ],
      "text/plain": [
       "repetition_penalty \u001b[1;36m1.1\u001b[0m:"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "#Person1# and #Person2# will take the underground to the conference center. They decide to go together.\n",
      "### EndResponse ?\n",
      "###</s>\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">repetition_penalty <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">1.2</span>:</pre>\n"
      ],
      "text/plain": [
       "repetition_penalty \u001b[1;36m1.2\u001b[0m:"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "#Person1# will take #Person2# to the conference center by subway since they have to arrive early for their meeting with Mr. Ewing.\n",
      "### EndResponse ?\n",
      "</s>\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">repetition_penalty <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">1.3</span>:</pre>\n"
      ],
      "text/plain": [
       "repetition_penalty \u001b[1;36m1.3\u001b[0m:"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Ms. Smith will drive her own car while Mr. Brown takes an underground train with #Person1#, who thinks it would be better for him to use his car due to traffic congestion. They decide they could travel by\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">repetition_penalty <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">1.4</span>:</pre>\n"
      ],
      "text/plain": [
       "repetition_penalty \u001b[1;36m1.4\u001b[0m:"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Ms #Person1#, Mr. Ewings asks everyone in their company who will attend his meeting with some colleagues form East York Branch Office shows up punctually so they decide to travel by subway instead of driving cars due\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">repetition_penalty <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">1.5</span>:</pre>\n"
      ],
      "text/plain": [
       "repetition_penalty \u001b[1;36m1.5\u001b[0m:"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Ms #person1#, Mr Wong will also use an Undergroud train for attending their meeting with some colleagues in East York Branch Office today since they have traffic jam problems so do she does too. They decide to travel\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">repetition_penalty <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">1.6</span>:</pre>\n"
      ],
      "text/plain": [
       "repetition_penalty \u001b[1;36m1.6\u001b[0m:"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Ms #person3#, Mr Wong will also attend an important meeting with his colleagues in East York Branch Office so they decide it would better for both parties by using public transportation instead driving their own cars due traffic congestion\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">repetition_penalty <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">1.7000000000000002</span>:</pre>\n"
      ],
      "text/plain": [
       "repetition_penalty \u001b[1;36m1.7000000000000002\u001b[0m:"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Ms #person3#, Mr Wong will also attend an important meeting with his colleagues in East York Branch Office so they decide it would better for both parties by using public transportation instead driving their own cars due traffic congestion\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">repetition_penalty <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">1.8</span>:</pre>\n"
      ],
      "text/plain": [
       "repetition_penalty \u001b[1;36m1.8\u001b[0m:"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Ms #person3#, Mr Wong will also attend an important meeting with his colleagues in East York Branch Office so they decide it would better for both parties by using public transportation instead driving their own cars due traffic congestion\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">repetition_penalty <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">1.9</span>:</pre>\n"
      ],
      "text/plain": [
       "repetition_penalty \u001b[1;36m1.9\u001b[0m:"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Ms #person3#, Mr Wong will also attend an important meeting with his colleagues in East York Branch Office so they decide it would better for both parties by using public transportation instead driving their own cars due traffic congestion\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">repetition_penalty <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">2.0</span>:</pre>\n"
      ],
      "text/plain": [
       "repetition_penalty \u001b[1;36m2.0\u001b[0m:"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Ms #person3#, Mr Wong will also attend an important meeting with his colleagues in East York Branch Office so they decide it would better for both parties by using public transportation instead driving their own cars due traffic congestion\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">------------------------------------------------------------\n",
       "</pre>\n"
      ],
      "text/plain": [
       "------------------------------------------------------------\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">typical_p <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.1</span>:</pre>\n"
      ],
      "text/plain": [
       "typical_p \u001b[1;36m0.1\u001b[0m:"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Mister Ewing asks #Person1# and #Person2# to arrive early for the meeting so they will have time to meet some colleagues in their East York Branch Office. They decide to travel by subway instead of driving due\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">typical_p <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.2</span>:</pre>\n"
      ],
      "text/plain": [
       "typical_p \u001b[1;36m0.2\u001b[0m:"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Mister Ewing asks #Person1# and #Person2# to arrive early for the meeting so they will have time to meet some colleagues in their East York Branch Office. They decide to travel by subway instead of driving due\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">typical_p <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.30000000000000004</span>:</pre>\n"
      ],
      "text/plain": [
       "typical_p \u001b[1;36m0.30000000000000004\u001b[0m:"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Mister Ewing asks #Person1# and #Person2# to arrive early for the meeting so they will have time to meet some colleagues in their East York Branch Office. They decide to travel by subway instead of driving due\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">typical_p <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.4</span>:</pre>\n"
      ],
      "text/plain": [
       "typical_p \u001b[1;36m0.4\u001b[0m:"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Mister Ewing asks #Person1# and #Person2# to arrive early for the meeting so they decide to travel by subway instead of driving their cars. They will also go together since it would help each other in finding directions\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">typical_p <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.5</span>:</pre>\n"
      ],
      "text/plain": [
       "typical_p \u001b[1;36m0.5\u001b[0m:"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Mister Ewing asks #Person1# and #Person2# to arrive early for the meeting so they decide to travel by subway instead of driving their cars. They will also go together since it would help each other in finding directions\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">typical_p <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.6</span>:</pre>\n"
      ],
      "text/plain": [
       "typical_p \u001b[1;36m0.6\u001b[0m:"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Mister Ewing asks #Person1# and #Person2# to arrive early for the meeting so they will have time to meet some colleagues in their East York Branch Office. They decide to travel by subway instead of driving due\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">typical_p <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.7000000000000001</span>:</pre>\n"
      ],
      "text/plain": [
       "typical_p \u001b[1;36m0.7000000000000001\u001b[0m:"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Mister Ewing asks #Person1# and #Person2# to arrive early for the meeting so they will have time to meet some colleagues in their East York Branch Office. They decide to travel by subway instead of driving due\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">typical_p <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.8</span>:</pre>\n"
      ],
      "text/plain": [
       "typical_p \u001b[1;36m0.8\u001b[0m:"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Mister Ewing asks #Person1# and #Person2# to arrive early for the meeting so they decide to travel by subway instead of driving their cars. They will also go together since it would help each other in finding directions\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">typical_p <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.9</span>:</pre>\n"
      ],
      "text/plain": [
       "typical_p \u001b[1;36m0.9\u001b[0m:"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Mister Ewing asks #Person1# and #Person2# to arrive early for the meeting so they will have time to meet some colleagues in their East York Branch Office. They decide to travel by subway instead of driving due\n"
     ]
    }
   ],
   "source": [
    "# repitition_penalty really blows test_df['text'][1] out of the water...`\n",
    "\n",
    "analyze_generation_params(text, persisted_model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c9c719fe-4b25-484c-89de-a4906e9f0afa",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}