Group_10_Fine_Tuning_Implementation_Random_Baseline.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "r6rq3g8bNeN3"
      },
      "source": [
        "# Processing the data (TensorFlow)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "mMwV-1XSNeN6"
      },
      "source": [
        "Install the Transformers, Datasets, and Evaluate libraries to run this notebook."
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "FineTuning the Model to our Dataset"
      ],
      "metadata": {
        "id": "b_fPzfEcrSbg"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "meanList = []\n",
        "sDeviationList = []\n",
        "varianceList = []"
      ],
      "metadata": {
        "id": "KJawjvAnQvf8"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install transformers datasets evaluate"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "vlMtO2kkBZi1",
        "outputId": "dfda8b1b-2206-4d33-810e-33a1345d7730"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Requirement already satisfied: transformers in /usr/local/lib/python3.10/dist-packages (4.40.0)\n",
            "Requirement already satisfied: datasets in /usr/local/lib/python3.10/dist-packages (2.19.0)\n",
            "Requirement already satisfied: evaluate in /usr/local/lib/python3.10/dist-packages (0.4.1)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from transformers) (3.13.4)\n",
            "Requirement already satisfied: huggingface-hub<1.0,>=0.19.3 in /usr/local/lib/python3.10/dist-packages (from transformers) (0.22.2)\n",
            "Requirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.10/dist-packages (from transformers) (1.25.2)\n",
            "Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.10/dist-packages (from transformers) (24.0)\n",
            "Requirement already satisfied: pyyaml>=5.1 in /usr/local/lib/python3.10/dist-packages (from transformers) (6.0.1)\n",
            "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.10/dist-packages (from transformers) (2023.12.25)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.10/dist-packages (from transformers) (2.31.0)\n",
            "Requirement already satisfied: tokenizers<0.20,>=0.19 in /usr/local/lib/python3.10/dist-packages (from transformers) (0.19.1)\n",
            "Requirement already satisfied: safetensors>=0.4.1 in /usr/local/lib/python3.10/dist-packages (from transformers) (0.4.3)\n",
            "Requirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.10/dist-packages (from transformers) (4.66.2)\n",
            "Requirement already satisfied: pyarrow>=12.0.0 in /usr/local/lib/python3.10/dist-packages (from datasets) (14.0.2)\n",
            "Requirement already satisfied: pyarrow-hotfix in /usr/local/lib/python3.10/dist-packages (from datasets) (0.6)\n",
            "Requirement already satisfied: dill<0.3.9,>=0.3.0 in /usr/local/lib/python3.10/dist-packages (from datasets) (0.3.8)\n",
            "Requirement already satisfied: pandas in /usr/local/lib/python3.10/dist-packages (from datasets) (2.0.3)\n",
            "Requirement already satisfied: xxhash in /usr/local/lib/python3.10/dist-packages (from datasets) (3.4.1)\n",
            "Requirement already satisfied: multiprocess in /usr/local/lib/python3.10/dist-packages (from datasets) (0.70.16)\n",
            "Requirement already satisfied: fsspec[http]<=2024.3.1,>=2023.1.0 in /usr/local/lib/python3.10/dist-packages (from datasets) (2023.6.0)\n",
            "Requirement already satisfied: aiohttp in /usr/local/lib/python3.10/dist-packages (from datasets) (3.9.5)\n",
            "Requirement already satisfied: responses<0.19 in /usr/local/lib/python3.10/dist-packages (from evaluate) (0.18.0)\n",
            "Requirement already satisfied: aiosignal>=1.1.2 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets) (1.3.1)\n",
            "Requirement already satisfied: attrs>=17.3.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets) (23.2.0)\n",
            "Requirement already satisfied: frozenlist>=1.1.1 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets) (1.4.1)\n",
            "Requirement already satisfied: multidict<7.0,>=4.5 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets) (6.0.5)\n",
            "Requirement already satisfied: yarl<2.0,>=1.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets) (1.9.4)\n",
            "Requirement already satisfied: async-timeout<5.0,>=4.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets) (4.0.3)\n",
            "Requirement already satisfied: typing-extensions>=3.7.4.3 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub<1.0,>=0.19.3->transformers) (4.11.0)\n",
            "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests->transformers) (3.3.2)\n",
            "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests->transformers) (3.7)\n",
            "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests->transformers) (2.0.7)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests->transformers) (2024.2.2)\n",
            "Requirement already satisfied: python-dateutil>=2.8.2 in /usr/local/lib/python3.10/dist-packages (from pandas->datasets) (2.8.2)\n",
            "Requirement already satisfied: pytz>=2020.1 in /usr/local/lib/python3.10/dist-packages (from pandas->datasets) (2023.4)\n",
            "Requirement already satisfied: tzdata>=2022.1 in /usr/local/lib/python3.10/dist-packages (from pandas->datasets) (2024.1)\n",
            "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.10/dist-packages (from python-dateutil>=2.8.2->pandas->datasets) (1.16.0)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import pandas as pd\n",
        "from datasets import Dataset\n",
        "import random\n",
        "import numpy as np\n",
        "\n",
        "# getting the same seed and seeing if they are the same\n",
        "\n",
        "# Load the dataset from the CSV file containing the seed information\n",
        "seed_number = 1  # Change this to the seed number you want to use\n",
        "df = pd.read_csv(f\"datasets/fine_tuning_dataset_seed_{seed_number}.csv\")\n",
        "\n",
        "# Convert float columns to string\n",
        "df['question'] = df['question'].astype(str)\n",
        "df['context'] = df['context'].astype(str)\n",
        "df['answer'] = df['answer'].astype(str)\n",
        "\n",
        "# Convert the DataFrame into a Hugging Face dataset\n",
        "dataset_dict = {\n",
        "    \"question\": df[\"question\"],\n",
        "    \"context\": df[\"context\"],\n",
        "    \"answer\": df[\"answer\"]\n",
        "}\n",
        "\n",
        "# Create a Hugging Face dataset\n",
        "custom_dataset = Dataset.from_dict(dataset_dict)\n",
        "\n",
        "# Now, you can split the dataset using the seed value\n",
        "# You would use the same seed number used when saving the dataset\n",
        "seed_used = df['seed'].iloc[0]  # Extract the seed value from the DataFrame\n",
        "splits = custom_dataset.train_test_split(test_size=0.33, seed=seed_used)\n",
        "\n",
        "# Assuming 'test_data' is the test set obtained after splitting the dataset\n",
        "test_data = splits['test']\n",
        "scoreList = []\n",
        "\n",
        "for i in range(len(test_data)):\n",
        "    row = test_data[i]\n",
        "    words_in_context = row['context'].split(\" \")\n",
        "    # Filter out 'nan' if present and extract words without leading/trailing whitespace or newline characters\n",
        "    words_in_context = [word.strip(\"\\r\\n\") for word in words_in_context if word.strip(\"\\r\\n\") != 'nan']\n",
        "    # Check if there are at least 10 non-empty words after filtering 'nan' and empty strings\n",
        "    if len(words_in_context) >= 10:\n",
        "        selected_words = random.sample(words_in_context, 10)\n",
        "    else:\n",
        "        selected_words = words_in_context\n",
        "    # Remove any empty lists from selected_words\n",
        "    selected_words = [word for word in selected_words if word]\n",
        "    Answerlist = list(row['answer'].split(\",\"))\n",
        "    Intersection = set(selected_words).intersection(Answerlist)\n",
        "    HitRate = round(len(Intersection)/len(Answerlist) * 100, 2)\n",
        "    scoreList.append(HitRate)\n",
        "\n",
        "print(scoreList)\n",
        "meanList.append(np.mean(scoreList))\n",
        "sDeviationList.append(np.std(scoreList))\n",
        "varianceList.append(np.var(scoreList))\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "M48h8Crcu1pC",
        "outputId": "2e7a3de0-06c7-4079-e72a-9e98bdcc4448"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import pandas as pd\n",
        "from datasets import Dataset\n",
        "import random\n",
        "import numpy as np\n",
        "\n",
        "# getting the same seed and seeing if they are the same\n",
        "\n",
        "# Load the dataset from the CSV file containing the seed information\n",
        "seed_number = 2  # Change this to the seed number you want to use\n",
        "df = pd.read_csv(f\"datasets/fine_tuning_dataset_seed_{seed_number}.csv\")\n",
        "\n",
        "# Convert float columns to string\n",
        "df['question'] = df['question'].astype(str)\n",
        "df['context'] = df['context'].astype(str)\n",
        "df['answer'] = df['answer'].astype(str)\n",
        "\n",
        "# Convert the DataFrame into a Hugging Face dataset\n",
        "dataset_dict = {\n",
        "    \"question\": df[\"question\"],\n",
        "    \"context\": df[\"context\"],\n",
        "    \"answer\": df[\"answer\"]\n",
        "}\n",
        "\n",
        "# Create a Hugging Face dataset\n",
        "custom_dataset = Dataset.from_dict(dataset_dict)\n",
        "\n",
        "# Now, you can split the dataset using the seed value\n",
        "# You would use the same seed number used when saving the dataset\n",
        "seed_used = df['seed'].iloc[0]  # Extract the seed value from the DataFrame\n",
        "splits = custom_dataset.train_test_split(test_size=0.33, seed=seed_used)\n",
        "\n",
        "# Assuming 'test_data' is the test set obtained after splitting the dataset\n",
        "test_data = splits['test']\n",
        "scoreList = []\n",
        "\n",
        "for i in range(len(test_data)):\n",
        "    row = test_data[i]\n",
        "    words_in_context = row['context'].split(\" \")\n",
        "    # Filter out 'nan' if present and extract words without leading/trailing whitespace or newline characters\n",
        "    words_in_context = [word.strip(\"\\r\\n\") for word in words_in_context if word.strip(\"\\r\\n\") != 'nan']\n",
        "    # Check if there are at least 10 non-empty words after filtering 'nan' and empty strings\n",
        "    if len(words_in_context) >= 10:\n",
        "        selected_words = random.sample(words_in_context, 10)\n",
        "    else:\n",
        "        selected_words = words_in_context\n",
        "    # Remove any empty lists from selected_words\n",
        "    selected_words = [word for word in selected_words if word]\n",
        "    Answerlist = list(row['answer'].split(\",\"))\n",
        "    Intersection = set(selected_words).intersection(Answerlist)\n",
        "    HitRate = round(len(Intersection)/len(Answerlist) * 100, 2)\n",
        "    scoreList.append(HitRate)\n",
        "\n",
        "print(scoreList)\n",
        "meanList.append(np.mean(scoreList))\n",
        "sDeviationList.append(np.std(scoreList))\n",
        "varianceList.append(np.var(scoreList))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "theLscCASAT3",
        "outputId": "6b90160a-acac-4963-9e6c-2812db6e7ed8"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import pandas as pd\n",
        "from datasets import Dataset\n",
        "import random\n",
        "import numpy as np\n",
        "\n",
        "# getting the same seed and seeing if they are the same\n",
        "\n",
        "# Load the dataset from the CSV file containing the seed information\n",
        "seed_number = 3  # Change this to the seed number you want to use\n",
        "df = pd.read_csv(f\"datasets/fine_tuning_dataset_seed_{seed_number}.csv\")\n",
        "\n",
        "# Convert float columns to string\n",
        "df['question'] = df['question'].astype(str)\n",
        "df['context'] = df['context'].astype(str)\n",
        "df['answer'] = df['answer'].astype(str)\n",
        "\n",
        "# Convert the DataFrame into a Hugging Face dataset\n",
        "dataset_dict = {\n",
        "    \"question\": df[\"question\"],\n",
        "    \"context\": df[\"context\"],\n",
        "    \"answer\": df[\"answer\"]\n",
        "}\n",
        "\n",
        "# Create a Hugging Face dataset\n",
        "custom_dataset = Dataset.from_dict(dataset_dict)\n",
        "\n",
        "# Now, you can split the dataset using the seed value\n",
        "# You would use the same seed number used when saving the dataset\n",
        "seed_used = df['seed'].iloc[0]  # Extract the seed value from the DataFrame\n",
        "splits = custom_dataset.train_test_split(test_size=0.33, seed=seed_used)\n",
        "\n",
        "# Assuming 'test_data' is the test set obtained after splitting the dataset\n",
        "test_data = splits['test']\n",
        "scoreList = []\n",
        "\n",
        "for i in range(len(test_data)):\n",
        "    row = test_data[i]\n",
        "    words_in_context = row['context'].split(\" \")\n",
        "    # Filter out 'nan' if present and extract words without leading/trailing whitespace or newline characters\n",
        "    words_in_context = [word.strip(\"\\r\\n\") for word in words_in_context if word.strip(\"\\r\\n\") != 'nan']\n",
        "    # Check if there are at least 10 non-empty words after filtering 'nan' and empty strings\n",
        "    if len(words_in_context) >= 10:\n",
        "        selected_words = random.sample(words_in_context, 10)\n",
        "    else:\n",
        "        selected_words = words_in_context\n",
        "    # Remove any empty lists from selected_words\n",
        "    selected_words = [word for word in selected_words if word]\n",
        "    Answerlist = list(row['answer'].split(\",\"))\n",
        "    Intersection = set(selected_words).intersection(Answerlist)\n",
        "    HitRate = round(len(Intersection)/len(Answerlist) * 100, 2)\n",
        "    scoreList.append(HitRate)\n",
        "\n",
        "print(scoreList)\n",
        "meanList.append(np.mean(scoreList))\n",
        "sDeviationList.append(np.std(scoreList))\n",
        "varianceList.append(np.var(scoreList))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "jQvoABQjSAdL",
        "outputId": "60ccc446-ad25-4b64-89c2-404d46833b4f"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import pandas as pd\n",
        "from datasets import Dataset\n",
        "import random\n",
        "import numpy as np\n",
        "\n",
        "# getting the same seed and seeing if they are the same\n",
        "\n",
        "# Load the dataset from the CSV file containing the seed information\n",
        "seed_number = 4  # Change this to the seed number you want to use\n",
        "df = pd.read_csv(f\"datasets/fine_tuning_dataset_seed_{seed_number}.csv\")\n",
        "\n",
        "# Convert float columns to string\n",
        "df['question'] = df['question'].astype(str)\n",
        "df['context'] = df['context'].astype(str)\n",
        "df['answer'] = df['answer'].astype(str)\n",
        "\n",
        "# Convert the DataFrame into a Hugging Face dataset\n",
        "dataset_dict = {\n",
        "    \"question\": df[\"question\"],\n",
        "    \"context\": df[\"context\"],\n",
        "    \"answer\": df[\"answer\"]\n",
        "}\n",
        "\n",
        "# Create a Hugging Face dataset\n",
        "custom_dataset = Dataset.from_dict(dataset_dict)\n",
        "\n",
        "# Now, you can split the dataset using the seed value\n",
        "# You would use the same seed number used when saving the dataset\n",
        "seed_used = df['seed'].iloc[0]  # Extract the seed value from the DataFrame\n",
        "splits = custom_dataset.train_test_split(test_size=0.33, seed=seed_used)\n",
        "\n",
        "# Assuming 'test_data' is the test set obtained after splitting the dataset\n",
        "test_data = splits['test']\n",
        "scoreList = []\n",
        "\n",
        "for i in range(len(test_data)):\n",
        "    row = test_data[i]\n",
        "    words_in_context = row['context'].split(\" \")\n",
        "    # Filter out 'nan' if present and extract words without leading/trailing whitespace or newline characters\n",
        "    words_in_context = [word.strip(\"\\r\\n\") for word in words_in_context if word.strip(\"\\r\\n\") != 'nan']\n",
        "    # Check if there are at least 10 non-empty words after filtering 'nan' and empty strings\n",
        "    if len(words_in_context) >= 10:\n",
        "        selected_words = random.sample(words_in_context, 10)\n",
        "    else:\n",
        "        selected_words = words_in_context\n",
        "    # Remove any empty lists from selected_words\n",
        "    selected_words = [word for word in selected_words if word]\n",
        "    Answerlist = list(row['answer'].split(\",\"))\n",
        "    Intersection = set(selected_words).intersection(Answerlist)\n",
        "    HitRate = round(len(Intersection)/len(Answerlist) * 100, 2)\n",
        "    scoreList.append(HitRate)\n",
        "\n",
        "print(scoreList)\n",
        "meanList.append(np.mean(scoreList))\n",
        "sDeviationList.append(np.std(scoreList))\n",
        "varianceList.append(np.var(scoreList))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "f_KJR4M1SAmM",
        "outputId": "cb55e09e-360b-43d3-e69b-821effa2a623"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 12.5, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import pandas as pd\n",
        "from datasets import Dataset\n",
        "import random\n",
        "import numpy as np\n",
        "\n",
        "# getting the same seed and seeing if they are the same\n",
        "\n",
        "# Load the dataset from the CSV file containing the seed information\n",
        "seed_number = 5  # Change this to the seed number you want to use\n",
        "df = pd.read_csv(f\"datasets/fine_tuning_dataset_seed_{seed_number}.csv\")\n",
        "\n",
        "# Convert float columns to string\n",
        "df['question'] = df['question'].astype(str)\n",
        "df['context'] = df['context'].astype(str)\n",
        "df['answer'] = df['answer'].astype(str)\n",
        "\n",
        "# Convert the DataFrame into a Hugging Face dataset\n",
        "dataset_dict = {\n",
        "    \"question\": df[\"question\"],\n",
        "    \"context\": df[\"context\"],\n",
        "    \"answer\": df[\"answer\"]\n",
        "}\n",
        "\n",
        "# Create a Hugging Face dataset\n",
        "custom_dataset = Dataset.from_dict(dataset_dict)\n",
        "\n",
        "# Now, you can split the dataset using the seed value\n",
        "# You would use the same seed number used when saving the dataset\n",
        "seed_used = df['seed'].iloc[0]  # Extract the seed value from the DataFrame\n",
        "splits = custom_dataset.train_test_split(test_size=0.33, seed=seed_used)\n",
        "\n",
        "# Assuming 'test_data' is the test set obtained after splitting the dataset\n",
        "test_data = splits['test']\n",
        "scoreList = []\n",
        "\n",
        "for i in range(len(test_data)):\n",
        "    row = test_data[i]\n",
        "    words_in_context = row['context'].split(\" \")\n",
        "    # Filter out 'nan' if present and extract words without leading/trailing whitespace or newline characters\n",
        "    words_in_context = [word.strip(\"\\r\\n\") for word in words_in_context if word.strip(\"\\r\\n\") != 'nan']\n",
        "    # Check if there are at least 10 non-empty words after filtering 'nan' and empty strings\n",
        "    if len(words_in_context) >= 10:\n",
        "        selected_words = random.sample(words_in_context, 10)\n",
        "    else:\n",
        "        selected_words = words_in_context\n",
        "    # Remove any empty lists from selected_words\n",
        "    selected_words = [word for word in selected_words if word]\n",
        "    Answerlist = list(row['answer'].split(\",\"))\n",
        "    Intersection = set(selected_words).intersection(Answerlist)\n",
        "    HitRate = round(len(Intersection)/len(Answerlist) * 100, 2)\n",
        "    scoreList.append(HitRate)\n",
        "\n",
        "print(scoreList)\n",
        "meanList.append(np.mean(scoreList))\n",
        "sDeviationList.append(np.std(scoreList))\n",
        "varianceList.append(np.var(scoreList))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "FoybdPfwSA1M",
        "outputId": "d94fc7e5-0b85-48e9-9554-f53e5d714cec"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "print('mean: ', meanList)\n",
        "print('standard deviation: ', sDeviationList)\n",
        "print('variance: ', varianceList)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "rEZb1yHoSKo8",
        "outputId": "ab50e3cf-0c12-400c-a36b-f27deb3194c2"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "mean:  [0.0, 0.0, 0.0, 0.06313131313131314, 0.0]\n",
            "standard deviation:  [0.0, 0.0, 0.0, 0.8860902050264016, 0.0]\n",
            "variance:  [0.0, 0.0, 0.0, 0.7851558514437305, 0.0]\n"
          ]
        }
      ]
    }
  ],
  "metadata": {
    "colab": {
      "provenance": [],
      "gpuType": "T4",
      "machine_shape": "hm"
    },
    "language_info": {
      "name": "python"
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "nbformat": 4,
  "nbformat_minor": 0
}