evaluation/original_model_evaluation.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "gpuType": "T4"
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "code",
      "execution_count": 1,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "DX49e1YIvRT3",
        "outputId": "f00382ed-3099-47a8-a35e-992445ae6891"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Cloning into 'lm-evaluation-harness'...\n",
            "remote: Enumerating objects: 40620, done.\u001b[K\n",
            "remote: Counting objects: 100% (1008/1008), done.\u001b[K\n",
            "remote: Compressing objects: 100% (474/474), done.\u001b[K\n",
            "remote: Total 40620 (delta 629), reused 856 (delta 533), pack-reused 39612 (from 1)\u001b[K\n",
            "Receiving objects: 100% (40620/40620), 27.04 MiB | 14.27 MiB/s, done.\n",
            "Resolving deltas: 100% (28494/28494), done.\n",
            "Obtaining file:///content/lm-evaluation-harness\n",
            "  Installing build dependencies ... \u001b[?25l\u001b[?25hdone\n",
            "  Checking if build backend supports build_editable ... \u001b[?25l\u001b[?25hdone\n",
            "  Getting requirements to build editable ... \u001b[?25l\u001b[?25hdone\n",
            "  Preparing editable metadata (pyproject.toml) ... \u001b[?25l\u001b[?25hdone\n",
            "Requirement already satisfied: accelerate>=0.26.0 in /usr/local/lib/python3.10/dist-packages (from lm_eval==0.4.4) (0.34.2)\n",
            "Collecting evaluate (from lm_eval==0.4.4)\n",
            "  Downloading evaluate-0.4.3-py3-none-any.whl.metadata (9.2 kB)\n",
            "Collecting datasets>=2.16.0 (from lm_eval==0.4.4)\n",
            "  Downloading datasets-3.0.0-py3-none-any.whl.metadata (19 kB)\n",
            "Collecting jsonlines (from lm_eval==0.4.4)\n",
            "  Downloading jsonlines-4.0.0-py3-none-any.whl.metadata (1.6 kB)\n",
            "Requirement already satisfied: numexpr in /usr/local/lib/python3.10/dist-packages (from lm_eval==0.4.4) (2.10.1)\n",
            "Collecting peft>=0.2.0 (from lm_eval==0.4.4)\n",
            "  Downloading peft-0.12.0-py3-none-any.whl.metadata (13 kB)\n",
            "Collecting pybind11>=2.6.2 (from lm_eval==0.4.4)\n",
            "  Downloading pybind11-2.13.6-py3-none-any.whl.metadata (9.5 kB)\n",
            "Collecting pytablewriter (from lm_eval==0.4.4)\n",
            "  Downloading pytablewriter-1.2.0-py3-none-any.whl.metadata (37 kB)\n",
            "Collecting rouge-score>=0.0.4 (from lm_eval==0.4.4)\n",
            "  Downloading rouge_score-0.1.2.tar.gz (17 kB)\n",
            "  Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "Collecting sacrebleu>=1.5.0 (from lm_eval==0.4.4)\n",
            "  Downloading sacrebleu-2.4.3-py3-none-any.whl.metadata (51 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m51.8/51.8 kB\u001b[0m \u001b[31m3.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: scikit-learn>=0.24.1 in /usr/local/lib/python3.10/dist-packages (from lm_eval==0.4.4) (1.3.2)\n",
            "Collecting sqlitedict (from lm_eval==0.4.4)\n",
            "  Downloading sqlitedict-2.1.0.tar.gz (21 kB)\n",
            "  Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "Requirement already satisfied: torch>=1.8 in /usr/local/lib/python3.10/dist-packages (from lm_eval==0.4.4) (2.4.0+cu121)\n",
            "Collecting tqdm-multiprocess (from lm_eval==0.4.4)\n",
            "  Downloading tqdm_multiprocess-0.0.11-py3-none-any.whl.metadata (5.7 kB)\n",
            "Requirement already satisfied: transformers>=4.1 in /usr/local/lib/python3.10/dist-packages (from lm_eval==0.4.4) (4.44.2)\n",
            "Collecting zstandard (from lm_eval==0.4.4)\n",
            "  Downloading zstandard-0.23.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (3.0 kB)\n",
            "Collecting dill (from lm_eval==0.4.4)\n",
            "  Downloading dill-0.3.8-py3-none-any.whl.metadata (10 kB)\n",
            "Collecting word2number (from lm_eval==0.4.4)\n",
            "  Downloading word2number-1.1.zip (9.7 kB)\n",
            "  Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "Requirement already satisfied: more-itertools in /usr/local/lib/python3.10/dist-packages (from lm_eval==0.4.4) (10.3.0)\n",
            "Requirement already satisfied: numpy<3.0.0,>=1.17 in /usr/local/lib/python3.10/dist-packages (from accelerate>=0.26.0->lm_eval==0.4.4) (1.26.4)\n",
            "Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.10/dist-packages (from accelerate>=0.26.0->lm_eval==0.4.4) (24.1)\n",
            "Requirement already satisfied: psutil in /usr/local/lib/python3.10/dist-packages (from accelerate>=0.26.0->lm_eval==0.4.4) (5.9.5)\n",
            "Requirement already satisfied: pyyaml in /usr/local/lib/python3.10/dist-packages (from accelerate>=0.26.0->lm_eval==0.4.4) (6.0.2)\n",
            "Requirement already satisfied: huggingface-hub>=0.21.0 in /usr/local/lib/python3.10/dist-packages (from accelerate>=0.26.0->lm_eval==0.4.4) (0.24.6)\n",
            "Requirement already satisfied: safetensors>=0.4.3 in /usr/local/lib/python3.10/dist-packages (from accelerate>=0.26.0->lm_eval==0.4.4) (0.4.5)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from datasets>=2.16.0->lm_eval==0.4.4) (3.16.0)\n",
            "Collecting pyarrow>=15.0.0 (from datasets>=2.16.0->lm_eval==0.4.4)\n",
            "  Downloading pyarrow-17.0.0-cp310-cp310-manylinux_2_28_x86_64.whl.metadata (3.3 kB)\n",
            "Requirement already satisfied: pandas in /usr/local/lib/python3.10/dist-packages (from datasets>=2.16.0->lm_eval==0.4.4) (2.1.4)\n",
            "Requirement already satisfied: requests>=2.32.2 in /usr/local/lib/python3.10/dist-packages (from datasets>=2.16.0->lm_eval==0.4.4) (2.32.3)\n",
            "Requirement already satisfied: tqdm>=4.66.3 in /usr/local/lib/python3.10/dist-packages (from datasets>=2.16.0->lm_eval==0.4.4) (4.66.5)\n",
            "Collecting xxhash (from datasets>=2.16.0->lm_eval==0.4.4)\n",
            "  Downloading xxhash-3.5.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (12 kB)\n",
            "Collecting multiprocess (from datasets>=2.16.0->lm_eval==0.4.4)\n",
            "  Downloading multiprocess-0.70.16-py310-none-any.whl.metadata (7.2 kB)\n",
            "Requirement already satisfied: fsspec<=2024.6.1,>=2023.1.0 in /usr/local/lib/python3.10/dist-packages (from fsspec[http]<=2024.6.1,>=2023.1.0->datasets>=2.16.0->lm_eval==0.4.4) (2024.6.1)\n",
            "Requirement already satisfied: aiohttp in /usr/local/lib/python3.10/dist-packages (from datasets>=2.16.0->lm_eval==0.4.4) (3.10.5)\n",
            "Requirement already satisfied: absl-py in /usr/local/lib/python3.10/dist-packages (from rouge-score>=0.0.4->lm_eval==0.4.4) (1.4.0)\n",
            "Requirement already satisfied: nltk in /usr/local/lib/python3.10/dist-packages (from rouge-score>=0.0.4->lm_eval==0.4.4) (3.8.1)\n",
            "Requirement already satisfied: six>=1.14.0 in /usr/local/lib/python3.10/dist-packages (from rouge-score>=0.0.4->lm_eval==0.4.4) (1.16.0)\n",
            "Collecting portalocker (from sacrebleu>=1.5.0->lm_eval==0.4.4)\n",
            "  Downloading portalocker-2.10.1-py3-none-any.whl.metadata (8.5 kB)\n",
            "Requirement already satisfied: regex in /usr/local/lib/python3.10/dist-packages (from sacrebleu>=1.5.0->lm_eval==0.4.4) (2024.5.15)\n",
            "Requirement already satisfied: tabulate>=0.8.9 in /usr/local/lib/python3.10/dist-packages (from sacrebleu>=1.5.0->lm_eval==0.4.4) (0.9.0)\n",
            "Collecting colorama (from sacrebleu>=1.5.0->lm_eval==0.4.4)\n",
            "  Downloading colorama-0.4.6-py2.py3-none-any.whl.metadata (17 kB)\n",
            "Requirement already satisfied: lxml in /usr/local/lib/python3.10/dist-packages (from sacrebleu>=1.5.0->lm_eval==0.4.4) (4.9.4)\n",
            "Requirement already satisfied: scipy>=1.5.0 in /usr/local/lib/python3.10/dist-packages (from scikit-learn>=0.24.1->lm_eval==0.4.4) (1.13.1)\n",
            "Requirement already satisfied: joblib>=1.1.1 in /usr/local/lib/python3.10/dist-packages (from scikit-learn>=0.24.1->lm_eval==0.4.4) (1.4.2)\n",
            "Requirement already satisfied: threadpoolctl>=2.0.0 in /usr/local/lib/python3.10/dist-packages (from scikit-learn>=0.24.1->lm_eval==0.4.4) (3.5.0)\n",
            "Requirement already satisfied: typing-extensions>=4.8.0 in /usr/local/lib/python3.10/dist-packages (from torch>=1.8->lm_eval==0.4.4) (4.12.2)\n",
            "Requirement already satisfied: sympy in /usr/local/lib/python3.10/dist-packages (from torch>=1.8->lm_eval==0.4.4) (1.13.2)\n",
            "Requirement already satisfied: networkx in /usr/local/lib/python3.10/dist-packages (from torch>=1.8->lm_eval==0.4.4) (3.3)\n",
            "Requirement already satisfied: jinja2 in /usr/local/lib/python3.10/dist-packages (from torch>=1.8->lm_eval==0.4.4) (3.1.4)\n",
            "Requirement already satisfied: tokenizers<0.20,>=0.19 in /usr/local/lib/python3.10/dist-packages (from transformers>=4.1->lm_eval==0.4.4) (0.19.1)\n",
            "Requirement already satisfied: attrs>=19.2.0 in /usr/local/lib/python3.10/dist-packages (from jsonlines->lm_eval==0.4.4) (24.2.0)\n",
            "Requirement already satisfied: setuptools>=38.3.0 in /usr/local/lib/python3.10/dist-packages (from pytablewriter->lm_eval==0.4.4) (71.0.4)\n",
            "Collecting DataProperty<2,>=1.0.1 (from pytablewriter->lm_eval==0.4.4)\n",
            "  Downloading DataProperty-1.0.1-py3-none-any.whl.metadata (11 kB)\n",
            "Collecting mbstrdecoder<2,>=1.0.0 (from pytablewriter->lm_eval==0.4.4)\n",
            "  Downloading mbstrdecoder-1.1.3-py3-none-any.whl.metadata (4.0 kB)\n",
            "Collecting pathvalidate<4,>=2.3.0 (from pytablewriter->lm_eval==0.4.4)\n",
            "  Downloading pathvalidate-3.2.1-py3-none-any.whl.metadata (12 kB)\n",
            "Collecting tabledata<2,>=1.3.1 (from pytablewriter->lm_eval==0.4.4)\n",
            "  Downloading tabledata-1.3.3-py3-none-any.whl.metadata (3.7 kB)\n",
            "Collecting tcolorpy<1,>=0.0.5 (from pytablewriter->lm_eval==0.4.4)\n",
            "  Downloading tcolorpy-0.1.6-py3-none-any.whl.metadata (6.4 kB)\n",
            "Collecting typepy<2,>=1.3.2 (from typepy[datetime]<2,>=1.3.2->pytablewriter->lm_eval==0.4.4)\n",
            "  Downloading typepy-1.3.2-py3-none-any.whl.metadata (9.3 kB)\n",
            "Requirement already satisfied: aiohappyeyeballs>=2.3.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets>=2.16.0->lm_eval==0.4.4) (2.4.0)\n",
            "Requirement already satisfied: aiosignal>=1.1.2 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets>=2.16.0->lm_eval==0.4.4) (1.3.1)\n",
            "Requirement already satisfied: frozenlist>=1.1.1 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets>=2.16.0->lm_eval==0.4.4) (1.4.1)\n",
            "Requirement already satisfied: multidict<7.0,>=4.5 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets>=2.16.0->lm_eval==0.4.4) (6.1.0)\n",
            "Requirement already satisfied: yarl<2.0,>=1.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets>=2.16.0->lm_eval==0.4.4) (1.11.1)\n",
            "Requirement already satisfied: async-timeout<5.0,>=4.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets>=2.16.0->lm_eval==0.4.4) (4.0.3)\n",
            "Requirement already satisfied: chardet<6,>=3.0.4 in /usr/local/lib/python3.10/dist-packages (from mbstrdecoder<2,>=1.0.0->pytablewriter->lm_eval==0.4.4) (5.2.0)\n",
            "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests>=2.32.2->datasets>=2.16.0->lm_eval==0.4.4) (3.3.2)\n",
            "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests>=2.32.2->datasets>=2.16.0->lm_eval==0.4.4) (3.8)\n",
            "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests>=2.32.2->datasets>=2.16.0->lm_eval==0.4.4) (2.0.7)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests>=2.32.2->datasets>=2.16.0->lm_eval==0.4.4) (2024.8.30)\n",
            "Requirement already satisfied: python-dateutil<3.0.0,>=2.8.0 in /usr/local/lib/python3.10/dist-packages (from typepy[datetime]<2,>=1.3.2->pytablewriter->lm_eval==0.4.4) (2.8.2)\n",
            "Requirement already satisfied: pytz>=2018.9 in /usr/local/lib/python3.10/dist-packages (from typepy[datetime]<2,>=1.3.2->pytablewriter->lm_eval==0.4.4) (2024.2)\n",
            "Requirement already satisfied: MarkupSafe>=2.0 in /usr/local/lib/python3.10/dist-packages (from jinja2->torch>=1.8->lm_eval==0.4.4) (2.1.5)\n",
            "Requirement already satisfied: click in /usr/local/lib/python3.10/dist-packages (from nltk->rouge-score>=0.0.4->lm_eval==0.4.4) (8.1.7)\n",
            "Requirement already satisfied: tzdata>=2022.1 in /usr/local/lib/python3.10/dist-packages (from pandas->datasets>=2.16.0->lm_eval==0.4.4) (2024.1)\n",
            "Requirement already satisfied: mpmath<1.4,>=1.1.0 in /usr/local/lib/python3.10/dist-packages (from sympy->torch>=1.8->lm_eval==0.4.4) (1.3.0)\n",
            "Downloading datasets-3.0.0-py3-none-any.whl (474 kB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m474.3/474.3 kB\u001b[0m \u001b[31m16.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading dill-0.3.8-py3-none-any.whl (116 kB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m116.3/116.3 kB\u001b[0m \u001b[31m8.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading evaluate-0.4.3-py3-none-any.whl (84 kB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m84.0/84.0 kB\u001b[0m \u001b[31m2.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading peft-0.12.0-py3-none-any.whl (296 kB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m296.4/296.4 kB\u001b[0m \u001b[31m20.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading pybind11-2.13.6-py3-none-any.whl (243 kB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m243.3/243.3 kB\u001b[0m \u001b[31m19.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading sacrebleu-2.4.3-py3-none-any.whl (103 kB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m104.0/104.0 kB\u001b[0m \u001b[31m6.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading jsonlines-4.0.0-py3-none-any.whl (8.7 kB)\n",
            "Downloading pytablewriter-1.2.0-py3-none-any.whl (111 kB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m111.1/111.1 kB\u001b[0m \u001b[31m8.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading tqdm_multiprocess-0.0.11-py3-none-any.whl (9.8 kB)\n",
            "Downloading zstandard-0.23.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (5.4 MB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m5.4/5.4 MB\u001b[0m \u001b[31m70.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading DataProperty-1.0.1-py3-none-any.whl (27 kB)\n",
            "Downloading mbstrdecoder-1.1.3-py3-none-any.whl (7.8 kB)\n",
            "Downloading pathvalidate-3.2.1-py3-none-any.whl (23 kB)\n",
            "Downloading pyarrow-17.0.0-cp310-cp310-manylinux_2_28_x86_64.whl (39.9 MB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m39.9/39.9 MB\u001b[0m \u001b[31m11.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading tabledata-1.3.3-py3-none-any.whl (11 kB)\n",
            "Downloading tcolorpy-0.1.6-py3-none-any.whl (8.1 kB)\n",
            "Downloading typepy-1.3.2-py3-none-any.whl (31 kB)\n",
            "Downloading colorama-0.4.6-py2.py3-none-any.whl (25 kB)\n",
            "Downloading multiprocess-0.70.16-py310-none-any.whl (134 kB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m134.8/134.8 kB\u001b[0m \u001b[31m10.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading portalocker-2.10.1-py3-none-any.whl (18 kB)\n",
            "Downloading xxhash-3.5.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (194 kB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m194.1/194.1 kB\u001b[0m \u001b[31m8.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hBuilding wheels for collected packages: lm_eval, rouge-score, sqlitedict, word2number\n",
            "  Building editable for lm_eval (pyproject.toml) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for lm_eval: filename=lm_eval-0.4.4-0.editable-py3-none-any.whl size=19185 sha256=0d7e8daa43e9cb97bd87e08e8b651695b9311a4ead926def1c9070d960090756\n",
            "  Stored in directory: /tmp/pip-ephem-wheel-cache-oaj4l3zp/wheels/dc/8d/a0/ce1a137b6a29fcf5007da91566ee423695e01d20703991091d\n",
            "  Building wheel for rouge-score (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for rouge-score: filename=rouge_score-0.1.2-py3-none-any.whl size=24935 sha256=17ac34b8d7f8be771308bdc03069e2844cc41f22cb684fd4afd15d7895686587\n",
            "  Stored in directory: /root/.cache/pip/wheels/5f/dd/89/461065a73be61a532ff8599a28e9beef17985c9e9c31e541b4\n",
            "  Building wheel for sqlitedict (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for sqlitedict: filename=sqlitedict-2.1.0-py3-none-any.whl size=16863 sha256=053a5fe1dbd0aa9c42e615c79d249db372d250c3429d2820eac8432d1211177f\n",
            "  Stored in directory: /root/.cache/pip/wheels/79/d6/e7/304e0e6cb2221022c26d8161f7c23cd4f259a9e41e8bbcfabd\n",
            "  Building wheel for word2number (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for word2number: filename=word2number-1.1-py3-none-any.whl size=5570 sha256=1ecd7829259a13cdc66291c5724aa0eeea65780ef2b883d3448d366650ccd635\n",
            "  Stored in directory: /root/.cache/pip/wheels/84/ff/26/d3cfbd971e96c5aa3737ecfced81628830d7359b55fbb8ca3b\n",
            "Successfully built lm_eval rouge-score sqlitedict word2number\n",
            "Installing collected packages: word2number, sqlitedict, zstandard, xxhash, tcolorpy, pybind11, pyarrow, portalocker, pathvalidate, mbstrdecoder, jsonlines, dill, colorama, typepy, tqdm-multiprocess, sacrebleu, rouge-score, multiprocess, DataProperty, tabledata, peft, datasets, pytablewriter, evaluate, lm_eval\n",
            "  Attempting uninstall: pyarrow\n",
            "    Found existing installation: pyarrow 14.0.2\n",
            "    Uninstalling pyarrow-14.0.2:\n",
            "      Successfully uninstalled pyarrow-14.0.2\n",
            "\u001b[31mERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.\n",
            "cudf-cu12 24.4.1 requires pyarrow<15.0.0a0,>=14.0.1, but you have pyarrow 17.0.0 which is incompatible.\n",
            "ibis-framework 8.0.0 requires pyarrow<16,>=2, but you have pyarrow 17.0.0 which is incompatible.\u001b[0m\u001b[31m\n",
            "\u001b[0mSuccessfully installed DataProperty-1.0.1 colorama-0.4.6 datasets-3.0.0 dill-0.3.8 evaluate-0.4.3 jsonlines-4.0.0 lm_eval-0.4.4 mbstrdecoder-1.1.3 multiprocess-0.70.16 pathvalidate-3.2.1 peft-0.12.0 portalocker-2.10.1 pyarrow-17.0.0 pybind11-2.13.6 pytablewriter-1.2.0 rouge-score-0.1.2 sacrebleu-2.4.3 sqlitedict-2.1.0 tabledata-1.3.3 tcolorpy-0.1.6 tqdm-multiprocess-0.0.11 typepy-1.3.2 word2number-1.1 xxhash-3.5.0 zstandard-0.23.0\n",
            "Collecting bitsandbytes\n",
            "  Downloading bitsandbytes-0.43.3-py3-none-manylinux_2_24_x86_64.whl.metadata (3.5 kB)\n",
            "Requirement already satisfied: torch in /usr/local/lib/python3.10/dist-packages (from bitsandbytes) (2.4.0+cu121)\n",
            "Requirement already satisfied: numpy in /usr/local/lib/python3.10/dist-packages (from bitsandbytes) (1.26.4)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from torch->bitsandbytes) (3.16.0)\n",
            "Requirement already satisfied: typing-extensions>=4.8.0 in /usr/local/lib/python3.10/dist-packages (from torch->bitsandbytes) (4.12.2)\n",
            "Requirement already satisfied: sympy in /usr/local/lib/python3.10/dist-packages (from torch->bitsandbytes) (1.13.2)\n",
            "Requirement already satisfied: networkx in /usr/local/lib/python3.10/dist-packages (from torch->bitsandbytes) (3.3)\n",
            "Requirement already satisfied: jinja2 in /usr/local/lib/python3.10/dist-packages (from torch->bitsandbytes) (3.1.4)\n",
            "Requirement already satisfied: fsspec in /usr/local/lib/python3.10/dist-packages (from torch->bitsandbytes) (2024.6.1)\n",
            "Requirement already satisfied: MarkupSafe>=2.0 in /usr/local/lib/python3.10/dist-packages (from jinja2->torch->bitsandbytes) (2.1.5)\n",
            "Requirement already satisfied: mpmath<1.4,>=1.1.0 in /usr/local/lib/python3.10/dist-packages (from sympy->torch->bitsandbytes) (1.3.0)\n",
            "Downloading bitsandbytes-0.43.3-py3-none-manylinux_2_24_x86_64.whl (137.5 MB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m137.5/137.5 MB\u001b[0m \u001b[31m6.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hInstalling collected packages: bitsandbytes\n",
            "Successfully installed bitsandbytes-0.43.3\n"
          ]
        }
      ],
      "source": [
        "!git clone https://github.com/EleutherAI/lm-evaluation-harness\n",
        "!pip install -e lm-evaluation-harness/\n",
        "!pip install bitsandbytes"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Check if the GPU is recognized\n",
        "import tensorflow as tf\n",
        "device_name = tf.test.gpu_device_name()\n",
        "if device_name != '/device:GPU:0':\n",
        "    raise SystemError('GPU device not found')\n",
        "print('Found GPU at: {}'.format(device_name))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "DWiACrsJxg-X",
        "outputId": "a6fe88cc-283e-4f3a-8802-3ae851f7b3d9"
      },
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Found GPU at: /device:GPU:0\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Evaluating Original LLM"
      ],
      "metadata": {
        "id": "-qf6P7i6v5sX"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "!lm_eval --model_args pretrained=unsloth/mistral-7b-bnb-4bit --tasks arithmetic --limit 10"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "OTM7rqrdvZBz",
        "outputId": "1e794eee-7778-4524-81c4-c77efd2242dd"
      },
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2024-09-14 13:43:53.838422: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:485] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered\n",
            "2024-09-14 13:43:53.857646: E external/local_xla/xla/stream_executor/cuda/cuda_dnn.cc:8454] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered\n",
            "2024-09-14 13:43:53.863382: E external/local_xla/xla/stream_executor/cuda/cuda_blas.cc:1452] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered\n",
            "2024-09-14 13:43:54.992430: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT\n",
            "2024-09-14:13:43:57,826 INFO     [__main__.py:279] Verbosity set to INFO\n",
            "2024-09-14:13:43:57,836 INFO     [__init__.py:491] `group` and `group_alias` keys in TaskConfigs are deprecated and will be removed in v0.4.5 of lm_eval. The new `tag` field will be used to allow for a shortcut to a group of tasks one does not wish to aggregate metrics across. `group`s which aggregate across subtasks must be only defined in a separate group config file, which will be the official way to create groups that support cross-task aggregation as in `mmlu`. Please see the v0.4.4 patch notes and our documentation: https://github.com/EleutherAI/lm-evaluation-harness/blob/main/docs/new_task_guide.md#advanced-group-configs for more information.\n",
            "2024-09-14:13:44:09,612 WARNING  [__main__.py:312]  --limit SHOULD ONLY BE USED FOR TESTING.REAL METRICS SHOULD NOT BE COMPUTED USING LIMIT.\n",
            "2024-09-14:13:44:09,613 INFO     [__main__.py:376] Selected Tasks: ['arithmetic']\n",
            "2024-09-14:13:44:09,620 INFO     [evaluator.py:161] Setting random seed to 0 | Setting numpy seed to 1234 | Setting torch manual seed to 1234\n",
            "2024-09-14:13:44:09,620 INFO     [evaluator.py:198] Initializing hf model, with arguments: {'pretrained': 'unsloth/mistral-7b-bnb-4bit'}\n",
            "2024-09-14:13:44:09,666 INFO     [huggingface.py:130] Using device 'cuda'\n",
            "config.json: 100% 1.19k/1.19k [00:00<00:00, 7.09MB/s]\n",
            "tokenizer_config.json: 100% 1.02k/1.02k [00:00<00:00, 6.79MB/s]\n",
            "tokenizer.model: 100% 493k/493k [00:00<00:00, 14.1MB/s]\n",
            "tokenizer.json: 100% 1.80M/1.80M [00:00<00:00, 5.58MB/s]\n",
            "special_tokens_map.json: 100% 438/438 [00:00<00:00, 2.54MB/s]\n",
            "2024-09-14:13:44:11,365 INFO     [huggingface.py:366] Model parallel was set to False, max memory was not set, and device map was set to {'': 'cuda'}\n",
            "Unused kwargs: ['_load_in_4bit', '_load_in_8bit', 'quant_method']. These kwargs are not used in <class 'transformers.utils.quantization_config.BitsAndBytesConfig'>.\n",
            "model.safetensors: 100% 4.13G/4.13G [00:26<00:00, 155MB/s]\n",
            "generation_config.json: 100% 155/155 [00:00<00:00, 1.18MB/s]\n",
            "2024-09-14:13:44:52,339 WARNING  [big_modeling.py:451] You shouldn't move a model that is dispatched using accelerate hooks.\n",
            "0000.parquet: 100% 21.8k/21.8k [00:00<00:00, 355kB/s]\n",
            "Generating validation split: 100% 2000/2000 [00:00<00:00, 19185.41 examples/s]\n",
            "0000.parquet: 100% 21.2k/21.2k [00:00<00:00, 19.0MB/s]\n",
            "Generating validation split: 100% 2000/2000 [00:00<00:00, 739736.16 examples/s]\n",
            "0000.parquet: 100% 40.2k/40.2k [00:00<00:00, 19.8MB/s]\n",
            "Generating validation split: 100% 2000/2000 [00:00<00:00, 731097.09 examples/s]\n",
            "0000.parquet: 100% 31.5k/31.5k [00:00<00:00, 514kB/s]\n",
            "Generating validation split: 100% 2000/2000 [00:00<00:00, 696786.11 examples/s]\n",
            "0000.parquet: 100% 28.0k/28.0k [00:00<00:00, 13.3MB/s]\n",
            "Generating validation split: 100% 2000/2000 [00:00<00:00, 781790.12 examples/s]\n",
            "0000.parquet: 100% 40.2k/40.2k [00:00<00:00, 614kB/s]\n",
            "Generating validation split: 100% 2000/2000 [00:00<00:00, 694651.21 examples/s]\n",
            "0000.parquet: 100% 21.0k/21.0k [00:00<00:00, 353kB/s]\n",
            "Generating validation split: 100% 2000/2000 [00:00<00:00, 604148.94 examples/s]\n",
            "0000.parquet: 100% 46.5k/46.5k [00:00<00:00, 10.8MB/s]\n",
            "Generating validation split: 100% 2000/2000 [00:00<00:00, 507262.99 examples/s]\n",
            "0000.parquet: 100% 31.5k/31.5k [00:00<00:00, 11.9MB/s]\n",
            "Generating validation split: 100% 2000/2000 [00:00<00:00, 784421.92 examples/s]\n",
            "0000.parquet: 100% 46.5k/46.5k [00:00<00:00, 746kB/s]\n",
            "Generating validation split: 100% 2000/2000 [00:00<00:00, 725846.50 examples/s]\n",
            "2024-09-14:13:45:04,176 INFO     [evaluator.py:279] Setting fewshot random generator seed to 1234\n",
            "2024-09-14:13:45:04,177 INFO     [evaluator.py:279] Setting fewshot random generator seed to 1234\n",
            "2024-09-14:13:45:04,177 INFO     [evaluator.py:279] Setting fewshot random generator seed to 1234\n",
            "2024-09-14:13:45:04,177 INFO     [evaluator.py:279] Setting fewshot random generator seed to 1234\n",
            "2024-09-14:13:45:04,177 INFO     [evaluator.py:279] Setting fewshot random generator seed to 1234\n",
            "2024-09-14:13:45:04,177 INFO     [evaluator.py:279] Setting fewshot random generator seed to 1234\n",
            "2024-09-14:13:45:04,177 INFO     [evaluator.py:279] Setting fewshot random generator seed to 1234\n",
            "2024-09-14:13:45:04,177 INFO     [evaluator.py:279] Setting fewshot random generator seed to 1234\n",
            "2024-09-14:13:45:04,177 INFO     [evaluator.py:279] Setting fewshot random generator seed to 1234\n",
            "2024-09-14:13:45:04,177 INFO     [evaluator.py:279] Setting fewshot random generator seed to 1234\n",
            "2024-09-14:13:45:04,177 WARNING  [model.py:422] model.chat_template was called with the chat_template set to False or None. Therefore no chat template will be applied. Make sure this is an intended behavior.\n",
            "2024-09-14:13:45:04,179 INFO     [task.py:428] Building contexts for arithmetic_5da on rank 0...\n",
            "100% 10/10 [00:00<00:00, 1102.20it/s]\n",
            "2024-09-14:13:45:04,189 INFO     [task.py:428] Building contexts for arithmetic_3da on rank 0...\n",
            "100% 10/10 [00:00<00:00, 1275.72it/s]\n",
            "2024-09-14:13:45:04,197 INFO     [task.py:428] Building contexts for arithmetic_5ds on rank 0...\n",
            "100% 10/10 [00:00<00:00, 1252.89it/s]\n",
            "2024-09-14:13:45:04,206 INFO     [task.py:428] Building contexts for arithmetic_2da on rank 0...\n",
            "100% 10/10 [00:00<00:00, 1223.69it/s]\n",
            "2024-09-14:13:45:04,215 INFO     [task.py:428] Building contexts for arithmetic_4da on rank 0...\n",
            "100% 10/10 [00:00<00:00, 1338.54it/s]\n",
            "2024-09-14:13:45:04,223 INFO     [task.py:428] Building contexts for arithmetic_2dm on rank 0...\n",
            "100% 10/10 [00:00<00:00, 1303.15it/s]\n",
            "2024-09-14:13:45:04,231 INFO     [task.py:428] Building contexts for arithmetic_3ds on rank 0...\n",
            "100% 10/10 [00:00<00:00, 1310.02it/s]\n",
            "2024-09-14:13:45:04,239 INFO     [task.py:428] Building contexts for arithmetic_4ds on rank 0...\n",
            "100% 10/10 [00:00<00:00, 1313.59it/s]\n",
            "2024-09-14:13:45:04,248 INFO     [task.py:428] Building contexts for arithmetic_2ds on rank 0...\n",
            "100% 10/10 [00:00<00:00, 1245.67it/s]\n",
            "2024-09-14:13:45:04,256 INFO     [task.py:428] Building contexts for arithmetic_1dc on rank 0...\n",
            "100% 10/10 [00:00<00:00, 1341.75it/s]\n",
            "2024-09-14:13:45:04,264 INFO     [evaluator.py:485] Running loglikelihood requests\n",
            "Running loglikelihood requests:   0% 0/100 [00:00<?, ?it/s]We detected that you are passing `past_key_values` as a tuple and this is deprecated and will be removed in v4.43. Please use an appropriate `Cache` class (https://huggingface.co/docs/transformers/v4.41.3/en/internal/generation_utils#transformers.Cache)\n",
            "Running loglikelihood requests: 100% 100/100 [00:53<00:00,  1.88it/s]\n",
            "fatal: not a git repository (or any of the parent directories): .git\n",
            "2024-09-14:13:45:59,724 INFO     [evaluation_tracker.py:269] Output path not provided, skipping saving results aggregated\n",
            "hf (pretrained=unsloth/mistral-7b-bnb-4bit), gen_kwargs: (None), limit: 10.0, num_fewshot: None, batch_size: 1\n",
            "|    Tasks     |Version|Filter|n-shot|Metric|   |Value|   |Stderr|\n",
            "|--------------|------:|------|-----:|------|---|----:|---|-----:|\n",
            "|arithmetic_1dc|      1|none  |     0|acc   |↑  |  0.5|±  |0.1667|\n",
            "|arithmetic_2da|      1|none  |     0|acc   |↑  |  1.0|±  |0.0000|\n",
            "|arithmetic_2dm|      1|none  |     0|acc   |↑  |  0.9|±  |0.1000|\n",
            "|arithmetic_2ds|      1|none  |     0|acc   |↑  |  1.0|±  |0.0000|\n",
            "|arithmetic_3da|      1|none  |     0|acc   |↑  |  1.0|±  |0.0000|\n",
            "|arithmetic_3ds|      1|none  |     0|acc   |↑  |  1.0|±  |0.0000|\n",
            "|arithmetic_4da|      1|none  |     0|acc   |↑  |  0.9|±  |0.1000|\n",
            "|arithmetic_4ds|      1|none  |     0|acc   |↑  |  1.0|±  |0.0000|\n",
            "|arithmetic_5da|      1|none  |     0|acc   |↑  |  0.9|±  |0.1000|\n",
            "|arithmetic_5ds|      1|none  |     0|acc   |↑  |  0.9|±  |0.1000|\n",
            "\n"
          ]
        }
      ]
    }
  ]
}