Shayan_IRI_LAW_RAG.ipynb

{
  "cells": [
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "b22ah9hn-XYW",
        "outputId": "6eff2dd6-d3ec-4c0c-c4bd-d894f0afd45a"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Mounted at /content/drive\n"
          ]
        }
      ],
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/drive')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "rD6UKIWPYjYW",
        "outputId": "c68be404-40f2-40a1-bd48-1e988462930c"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m280.0/280.0 kB\u001b[0m \u001b[31m4.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m806.2/806.2 kB\u001b[0m \u001b[31m25.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.8/1.8 MB\u001b[0m \u001b[31m34.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.8/1.8 MB\u001b[0m \u001b[31m44.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m156.5/156.5 kB\u001b[0m \u001b[31m16.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m525.5/525.5 kB\u001b[0m \u001b[31m33.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m16.9/16.9 MB\u001b[0m \u001b[31m45.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m227.4/227.4 kB\u001b[0m \u001b[31m22.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m69.7/69.7 kB\u001b[0m \u001b[31m7.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m252.4/252.4 kB\u001b[0m \u001b[31m28.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m64.2/64.2 kB\u001b[0m \u001b[31m5.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m421.5/421.5 kB\u001b[0m \u001b[31m29.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m274.7/274.7 kB\u001b[0m \u001b[31m24.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m981.5/981.5 kB\u001b[0m \u001b[31m45.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25h  Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m3.4/3.4 MB\u001b[0m \u001b[31m53.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m2.4/2.4 MB\u001b[0m \u001b[31m51.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m92.1/92.1 kB\u001b[0m \u001b[31m7.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m60.8/60.8 kB\u001b[0m \u001b[31m4.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m41.2/41.2 kB\u001b[0m \u001b[31m4.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m5.4/5.4 MB\u001b[0m \u001b[31m73.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m6.8/6.8 MB\u001b[0m \u001b[31m77.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m58.4/58.4 kB\u001b[0m \u001b[31m7.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m105.7/105.7 kB\u001b[0m \u001b[31m9.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m67.3/67.3 kB\u001b[0m \u001b[31m7.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25h  Installing build dependencies ... \u001b[?25l\u001b[?25hdone\n",
            "  Getting requirements to build wheel ... \u001b[?25l\u001b[?25hdone\n",
            "  Preparing metadata (pyproject.toml) ... \u001b[?25l\u001b[?25hdone\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m698.9/698.9 kB\u001b[0m \u001b[31m26.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.6/1.6 MB\u001b[0m \u001b[31m64.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m67.6/67.6 kB\u001b[0m \u001b[31m7.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m138.5/138.5 kB\u001b[0m \u001b[31m14.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25h  Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m307.9/307.9 kB\u001b[0m \u001b[31m22.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m75.6/75.6 kB\u001b[0m \u001b[31m8.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m7.8/7.8 MB\u001b[0m \u001b[31m51.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m129.9/129.9 kB\u001b[0m \u001b[31m11.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.8/1.8 MB\u001b[0m \u001b[31m43.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m510.5/510.5 kB\u001b[0m \u001b[31m27.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m71.1/71.1 kB\u001b[0m \u001b[31m6.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m49.4/49.4 kB\u001b[0m \u001b[31m4.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m71.5/71.5 kB\u001b[0m \u001b[31m7.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m77.8/77.8 kB\u001b[0m \u001b[31m10.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m58.3/58.3 kB\u001b[0m \u001b[31m7.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m46.0/46.0 kB\u001b[0m \u001b[31m5.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m50.8/50.8 kB\u001b[0m \u001b[31m5.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m341.4/341.4 kB\u001b[0m \u001b[31m32.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m3.4/3.4 MB\u001b[0m \u001b[31m53.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.3/1.3 MB\u001b[0m \u001b[31m62.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m116.3/116.3 kB\u001b[0m \u001b[31m12.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m134.8/134.8 kB\u001b[0m \u001b[31m13.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m86.8/86.8 kB\u001b[0m \u001b[31m9.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25h  Building wheel for pypika (pyproject.toml) ... \u001b[?25l\u001b[?25hdone\n",
            "  Building wheel for ffmpy (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Building wheel for langdetect (setup.py) ... \u001b[?25l\u001b[?25hdone\n"
          ]
        }
      ],
      "source": [
        "!pip install -q accelerate langchain langchain_community unstructured sentence-transformers chromadb gradio openai langchain-openai tqdm ragas"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "M-9mYdfBwmPx"
      },
      "outputs": [],
      "source": [
        "from langchain.document_loaders import DirectoryLoader\n",
        "from langchain.text_splitter import RecursiveCharacterTextSplitter\n",
        "from langchain.schema import Document\n",
        "from langchain_community.embeddings import HuggingFaceEmbeddings\n",
        "from langchain.vectorstores.chroma import Chroma\n",
        "import os\n",
        "import shutil\n",
        "import torch\n",
        "import openai\n",
        "from getpass import getpass\n",
        "import re"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "oX4puMM5-r8N",
        "outputId": "d758f283-9c69-47e6-f4fe-8955d4436cf4"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Please provide your OpenAI Key: ··········\n"
          ]
        }
      ],
      "source": [
        "openai.api_key = getpass(\"Please provide your OpenAI Key: \")\n",
        "os.environ[\"OPENAI_API_KEY\"] = openai.api_key"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "NZ4kWNHuY0me"
      },
      "outputs": [],
      "source": [
        "device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "rs1gEJcnXVFm"
      },
      "source": [
        "# Build the Vector DB"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "Y2rc8KMUZGPE"
      },
      "outputs": [],
      "source": [
        "%%capture\n",
        "CHROMA_PATH = \"./chroma\"\n",
        "DATA_PATH = \"./datasets\"\n",
        "\n",
        "embeding_model_name = \"sentence-transformers/LaBSE\"\n",
        "embeding_model_kwargs = {'device': device}\n",
        "embeding_encode_kwargs = {'normalize_embeddings': False}\n",
        "\n",
        "embedding_function = HuggingFaceEmbeddings(model_name=embeding_model_name,model_kwargs=embeding_model_kwargs,encode_kwargs=embeding_encode_kwargs)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "jHrmtUyPrjCr"
      },
      "source": [
        "## Naive Document Splitting"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "G7cBYCFE-TAC"
      },
      "outputs": [],
      "source": [
        "with open('/content/business_law.txt', 'r') as file:\n",
        "  text = file.read()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "M3hITnhIZGwU"
      },
      "outputs": [],
      "source": [
        "def generate_data_store():\n",
        "    documents = load_documents()\n",
        "    chunks = split_text(documents)\n",
        "    save_to_chroma(chunks)\n",
        "\n",
        "\n",
        "def load_documents():\n",
        "    loader = DirectoryLoader(DATA_PATH, glob=\"*.txt\")\n",
        "    documents = loader.load()\n",
        "    return documents\n",
        "\n",
        "\n",
        "def split_text(documents: list[Document]):\n",
        "    text_splitter = RecursiveCharacterTextSplitter(\n",
        "        chunk_size=300,\n",
        "        chunk_overlap=100,\n",
        "        length_function=len,\n",
        "        add_start_index=True,\n",
        "    )\n",
        "    chunks = text_splitter.split_documents(documents)\n",
        "    print(f\"Split {len(documents)} documents into {len(chunks)} chunks.\")\n",
        "\n",
        "    return chunks\n",
        "\n",
        "\n",
        "def save_to_chroma(chunks: list[Document]):\n",
        "    if os.path.exists(CHROMA_PATH):\n",
        "        shutil.rmtree(CHROMA_PATH)\n",
        "\n",
        "    db = Chroma.from_documents(\n",
        "        chunks,\n",
        "        embedding_function,\n",
        "        persist_directory=CHROMA_PATH\n",
        "    )\n",
        "\n",
        "    db.persist()\n",
        "    print(f\"Saved {len(chunks)} chunks to {CHROMA_PATH}.\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "xm8nco-PZVhC"
      },
      "outputs": [],
      "source": [
        "generate_data_store()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "L0U_1NBagUQq"
      },
      "outputs": [],
      "source": [
        "# backup the sqlite database"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "r3AUMgyjrrE_"
      },
      "source": [
        "## Chunking based on Law Articles"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "p_DUbZ6gejN9"
      },
      "outputs": [],
      "source": [
        "BUSINESS_LAW_TXT_PATH = '/content/drive/MyDrive/IRI_LAW/datasets/business_law.txt'\n",
        "\n",
        "with open(BUSINESS_LAW_TXT_PATH, 'r') as file:\n",
        "  text = file.read()\n",
        "\n",
        "# preprocess\n",
        "temp = ''\n",
        "for c in text:\n",
        "  if c != '\\u200c':\n",
        "    temp += c\n",
        "\n",
        "text = temp"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "0bQ9nDNee2By"
      },
      "outputs": [],
      "source": [
        "pattern = r'(^ماده) (\\d+)'\n",
        "\n",
        "matches = re.finditer(pattern, text, re.MULTILINE)\n",
        "indexes = [m.start() for m in matches]\n",
        "\n",
        "splitted = []\n",
        "for i, j in zip([0] + indexes, indexes + [len(text)]):\n",
        "  splitted.append(text[i:j])"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "VTYxtdu2eD3W"
      },
      "outputs": [],
      "source": [
        "from langchain.docstore.document import Document\n",
        "\n",
        "chunks = []\n",
        "for d in splitted:\n",
        "  chunks.append(Document(page_content=d))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "_pT6PsDOZL7x"
      },
      "outputs": [],
      "source": [
        "from langchain_openai import OpenAIEmbeddings\n",
        "embedding_function = OpenAIEmbeddings()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "g6379DiEeFi4"
      },
      "outputs": [],
      "source": [
        "db = Chroma.from_documents(\n",
        "    chunks,\n",
        "    embedding_function,\n",
        "    # persist_directory='/content/'\n",
        ")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "PGEk-Qi_eYhd",
        "outputId": "42df7d22-54ae-4732-94e1-46cab4040d67"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "[(Document(page_content='ماده 415 - ورشكستگي تاجر بحكم محكمه بدايت در موارد ذيل اعلام مي شود:\\nالف) بر حسب اظهار خود تاجر.\\nب) بموجب تقاضاي يك يا چند نفر از طلبكارها.\\nج) بر حسب تقاضاي مدعيالعموم بدايت.\\n\\n'),\n",
              "  0.8090258016076803),\n",
              " (Document(page_content='ماده 350 - هر گاه معامله مشروط بشرط تعليقي باشد دلال پس از حصول شرط مستحق اجرت خواهد بود.\\n\\n'),\n",
              "  0.8085643989836073),\n",
              " (Document(page_content='ماده 475 - حكم فوق درباره دعوي خياراتي نيز مجري خواهد بود كه تاجر ورشكسته نسبت باموال متصرفي خود يا ديگران دارد مشروط براينكه بر ضرر طلبكارها نباشد.\\n\\nفصل هفتم - در قرارداد ارفاقي و تصفيه حساب تاجر ورشكسته\\n\\nمبحث اول - در دعوت طلبكارها و مجمع عمومي آنها\\n\\n'),\n",
              "  0.8084869752228929),\n",
              " (Document(page_content='ماده 1 - تاجر كسي است كه شغل معمولي خود را معاملات تجارتي قرار بدهد.\\n\\n'),\n",
              "  0.8081855198889187),\n",
              " (Document(page_content='ماده 533 - هر گاه كسي مال التجارة بتاجر ورشكسته فروخته وليكن هنوز آن جنس نه بخود تاجر ورشكسته تسليم شده و نه بكس ديگر كه بحساب او بياورد آن كس ميتواند باندازة كه وجه آنرا نگرفته از تسليم مال التجاره امتناع كند.\\n\\n'),\n",
              "  0.794085776408357)]"
            ]
          },
          "execution_count": 11,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "question = 'شرایط تاجر بودن چیست؟'\n",
        "r = db.similarity_search_with_relevance_scores(question, k=5)\n",
        "r"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "oS3mSsVsZ0KJ"
      },
      "outputs": [],
      "source": [
        "def find_relevant_results(query, k=3):\n",
        "  results = db.similarity_search_with_relevance_scores(query, k=k)\n",
        "\n",
        "  return results"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "4xOHcgBtsspM"
      },
      "outputs": [],
      "source": [
        "# db.persist()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "CePNk2lds2bu"
      },
      "outputs": [],
      "source": [
        "# backup the sqlite database"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "po6_FyCKZbyy"
      },
      "source": [
        "# Load the Vector DB"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "DI2BzS6LgWXw"
      },
      "outputs": [],
      "source": [
        "!cp -r /content/drive/MyDrive/IRI_LAW/chroma/Main/ ./chroma"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "1mfMlJXYC4Da"
      },
      "outputs": [],
      "source": [
        "from langchain_openai import OpenAIEmbeddings\n",
        "embedding_function = OpenAIEmbeddings()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "kKj8NsY9ZhJ_"
      },
      "outputs": [],
      "source": [
        "CHROMA_PATH = \"./chroma\"\n",
        "db = Chroma(persist_directory=CHROMA_PATH, embedding_function=embedding_function)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "c_psFXw3kAad"
      },
      "source": [
        "# Configure RAG\n",
        "\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "e1120PcctrMO"
      },
      "source": [
        "## Init Language Model"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "nbOCzKUsQ3Qn"
      },
      "source": [
        "## MaralGPT-7B"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "CzwdojH-kJui"
      },
      "outputs": [],
      "source": [
        "from transformers import AutoModelForCausalLM, AutoTokenizer, GenerationConfig\n",
        "\n",
        "model_name_or_id = \"MaralGPT/Maral-7B-alpha-1\"\n",
        "tokenizer = AutoTokenizer.from_pretrained(model_name_or_id)\n",
        "llm = AutoModelForCausalLM.from_pretrained(model_name_or_id, torch_dtype=torch.bfloat16, device_map=\"auto\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "QvY_s4FeT_SC"
      },
      "outputs": [],
      "source": [
        "from typing import Any, List, Mapping, Optional\n",
        "\n",
        "from langchain_core.callbacks.manager import CallbackManagerForLLMRun\n",
        "from langchain_core.language_models.llms import LLM\n",
        "\n",
        "\n",
        "class CustomLLM(LLM):\n",
        "\n",
        "    @property\n",
        "    def _llm_type(self) -> str:\n",
        "        return \"MistralForCausalLM\"\n",
        "\n",
        "    def _call(\n",
        "        self,\n",
        "        prompt: str,\n",
        "        stop: Optional[List[str]] = None,\n",
        "        run_manager: Optional[CallbackManagerForLLMRun] = None,\n",
        "        **kwargs: Any,\n",
        "    ) -> str:\n",
        "      inputs = tokenizer(prompt, return_tensors=\"pt\").to(\"cuda\")\n",
        "      outputs = llm.generate(**inputs, generation_config=generation_config)\n",
        "      return tokenizer.decode(outputs[0], skip_special_tokens=True)\n",
        "\n",
        "llm = CustomLLM()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "Sr0vSOOflHME"
      },
      "outputs": [],
      "source": [
        "generation_config = GenerationConfig(\n",
        "    do_sample=True,\n",
        "    top_k=1,\n",
        "    temperature=0.5,\n",
        "    max_new_tokens=300,\n",
        "    pad_token_id=tokenizer.eos_token_id\n",
        ")\n",
        "def process_query(query):\n",
        "    results = find_relevant_results(query)\n",
        "    if len(results) == 0 or results[0][1] < 0.25:\n",
        "        return (\"اطلاعاتی که مرتبط با سوال شما باشد را در پایگاه دانش خود پیدا نکردم!\")\n",
        "    context = \"\\n\\n---\\n\\n\".join([doc.page_content for doc, _score in results])\n",
        "    prompt = prompt.format(context=context, question=query)\n",
        "    inputs = tokenizer(prompt, return_tensors=\"pt\").to(device)\n",
        "    outputs = llm.generate(**inputs, generation_config=generation_config)\n",
        "    return tokenizer.decode(outputs[0], skip_special_tokens=True)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "MP0XHU-DQ_36"
      },
      "source": [
        "## GPT 3.5-turbu"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "1g0kqyIB4ZLi"
      },
      "outputs": [],
      "source": [
        "from langchain_openai import ChatOpenAI\n",
        "\n",
        "llm = ChatOpenAI(model_name=\"gpt-3.5-turbo\", temperature=0)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "91XN8rlAtvey"
      },
      "source": [
        "## Create Prompt Template"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "ihA0PdmTlbC1"
      },
      "outputs": [],
      "source": [
        "from langchain.prompts import ChatPromptTemplate\n",
        "\n",
        "PROMPT_TEMPLATE = \"\"\"\n",
        "فقط بر اساس متن زیر به سوال پاسخ دهید:\n",
        "\n",
        "{context}\n",
        "\n",
        "---\n",
        "\n",
        "با توجه به متن بالا به سوال پاسخ دهید:\n",
        "{question}\n",
        "\n",
        "پاسخ:\n",
        "\"\"\"\n",
        "\n",
        "prompt = ChatPromptTemplate.from_template(PROMPT_TEMPLATE)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "fBcB0gzgNTvV"
      },
      "source": [
        "## Chain All Elements"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "xwqdDR5DOrjW"
      },
      "outputs": [],
      "source": [
        "retriever = db.as_retriever(search_type=\"similarity_score_threshold\", search_kwargs={\"score_threshold\": .25, \"k\": 5})"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "PiRwJ1dqNAtj"
      },
      "outputs": [],
      "source": [
        "from operator import itemgetter\n",
        "from langchain_core.runnables import RunnablePassthrough\n",
        "\n",
        "qa_chain = (\n",
        "    {\"context\": itemgetter(\"question\") | retriever, \"question\": itemgetter(\"question\")}\n",
        "    | RunnablePassthrough.assign(context=itemgetter(\"context\"))\n",
        "    | {\"response\": prompt | llm, \"context\": itemgetter(\"context\")}\n",
        ")"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "s_mp4GXmNL73"
      },
      "source": [
        "# Process a Single Question"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "NxTdDblKJsvk"
      },
      "source": [
        "## Serve in Gradio"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 643
        },
        "id": "MKG25KjoVONc",
        "outputId": "acfd0e9d-bfa6-452f-e5af-969e8039726b"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Setting queue=True in a Colab notebook requires sharing enabled. Setting `share=True` (you can turn this off by setting `share=False` in `launch()` explicitly).\n",
            "\n",
            "Colab notebook detected. To show errors in colab notebook, set debug=True in launch()\n",
            "Running on public URL: https://a35a7594ce11e9a47b.gradio.live\n",
            "\n",
            "This share link expires in 72 hours. For free permanent hosting and GPU upgrades, run `gradio deploy` from Terminal to deploy to Spaces (https://huggingface.co/spaces)\n"
          ]
        },
        {
          "data": {
            "text/html": [
              "<div><iframe src=\"https://a35a7594ce11e9a47b.gradio.live\" width=\"100%\" height=\"500\" allow=\"autoplay; camera; microphone; clipboard-read; clipboard-write;\" frameborder=\"0\" allowfullscreen></iframe></div>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        }
      ],
      "source": [
        "import gradio as gr\n",
        "\n",
        "def run_query(query):\n",
        "  resp = qa_chain.invoke({\"question\" : query})\n",
        "  answer = resp[\"response\"].content\n",
        "  context = [context.page_content for context in resp[\"context\"]]\n",
        "  if len(context) == 0:\n",
        "    context = ['ماده مرتبط با سوال شما یافت نشد.']\n",
        "  return 'زمینه: \\n' + '\\n'.join(context) + '\\nپاسخ:\\n' + answer\n",
        "\n",
        "with gr.Blocks() as demo:\n",
        "    query = gr.Textbox(label=\"سوال\", rtl=True, lines=5)\n",
        "    output = gr.Textbox(label=\"پاسخ\", rtl=True, lines=10)\n",
        "    greet_btn = gr.Button(\"Query\")\n",
        "    greet_btn.click(fn=run_query, inputs=query, outputs=output, api_name=\"run_query\")\n",
        "\n",
        "demo.launch();"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "YhFXOKbgSqp0",
        "outputId": "5f3245eb-b728-4956-e7e8-27890afa969e"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Closing server running on port: 7860\n"
          ]
        }
      ],
      "source": [
        "demo.close()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Rqegq27Yweq0"
      },
      "source": [
        "# Evaluation"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "n5XXjsg3BFu7"
      },
      "outputs": [],
      "source": [
        "# import locale\n",
        "# locale.getpreferredencoding = lambda: \"UTF-8\""
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "v2Osn1gLwq9n"
      },
      "outputs": [],
      "source": [
        "!cp -r /content/drive/MyDrive/IRI_LAW/evaluation ./evaluation"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 255
        },
        "id": "1Ifz6cbmw3Ic",
        "outputId": "9467125f-8923-441b-88d5-690283456615"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "hard_new.csv: 28\n",
            "easy_new.csv: 29\n",
            "medium_new.csv: 28\n"
          ]
        },
        {
          "data": {
            "application/vnd.google.colaboratory.intrinsic+json": {
              "summary": "{\n  \"name\": \"df\",\n  \"rows\": 85,\n  \"fields\": [\n    {\n      \"column\": \"Question\",\n      \"properties\": {\n        \"dtype\": \"string\",\n        \"num_unique_values\": 85,\n        \"samples\": [\n          \"\\u062d\\u062f\\u0648\\u062f \\u0645\\u0633\\u0626\\u0648\\u0644\\u06cc\\u062a \\u0645\\u062f\\u06cc\\u0631 \\u06cc\\u0627 \\u0645\\u062f\\u06cc\\u0631\\u0627\\u0646 \\u0634\\u0631\\u06a9\\u062a \\u062a\\u0636\\u0627\\u0645\\u0646\\u06cc \\u0686\\u0647 \\u0645\\u06cc\\u0632\\u0627\\u0646 \\u0627\\u0633\\u062a\\u061f\",\n          \"\\u0686\\u0647 \\u0634\\u0631\\u0627\\u06cc\\u0637\\u06cc \\u0628\\u0631\\u0627\\u06cc \\u062a\\u0634\\u06a9\\u06cc\\u0644 \\u0648 \\u062b\\u0628\\u062a \\u06cc\\u06a9 \\u0634\\u0631\\u06a9\\u062a \\u0628\\u0627 \\u0645\\u0633\\u0626\\u0648\\u0644\\u06cc\\u062a \\u0645\\u062d\\u062f\\u0648\\u062f \\u0648\\u062c\\u0648\\u062f \\u062f\\u0627\\u0631\\u062f \\u0648 \\u0686\\u06af\\u0648\\u0646\\u0647 \\u0645\\u06cc\\u200c\\u062a\\u0648\\u0627\\u0646 \\u0627\\u0633\\u0627\\u0633\\u0646\\u0627\\u0645\\u0647 \\u0622\\u0646 \\u0631\\u0627 \\u062a\\u063a\\u06cc\\u06cc\\u0631 \\u062f\\u0627\\u062f\\u061f\",\n          \"\\u0686\\u0647 \\u0645\\u062f\\u0627\\u0631\\u06a9\\u06cc \\u0628\\u0631\\u0627\\u06cc \\u062a\\u0634\\u06a9\\u06cc\\u0644 \\u0634\\u0631\\u06a9\\u062a \\u0628\\u0627 \\u0645\\u0633\\u0626\\u0648\\u0644\\u06cc\\u062a \\u0645\\u062d\\u062f\\u0648\\u062f \\u0646\\u06cc\\u0627\\u0632 \\u0627\\u0633\\u062a\\u061f\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"Answer\",\n      \"properties\": {\n        \"dtype\": \"string\",\n        \"num_unique_values\": 85,\n        \"samples\": [\n          \"\\u0645\\u0633\\u0626\\u0648\\u0644\\u06cc\\u062a \\u0645\\u062f\\u06cc\\u0631 \\u0634\\u0631\\u06a9\\u062a \\u062f\\u0631 \\u0645\\u0642\\u0627\\u0628\\u0644 \\u0634\\u0631\\u06a9\\u0627\\u0621 \\u0647\\u0645\\u0627\\u0646 \\u0645\\u0633\\u0648\\u0648\\u0644\\u06cc\\u062a\\u06cc \\u0627\\u0633\\u062a \\u06a9\\u0647 \\u0648\\u06a9\\u06cc\\u0644 \\u062f\\u0631 \\u0645\\u0642\\u0627\\u0628\\u0644 \\u0645\\u0648\\u06a9\\u0644 \\u062f\\u0627\\u0631\\u062f. (\\u0645\\u0648\\u0627\\u062f 121 \\u0648 51)\",\n          \"\\u0628\\u0631\\u0627\\u06cc \\u062a\\u0634\\u06a9\\u06cc\\u0644 \\u06cc\\u06a9 \\u0634\\u0631\\u06a9\\u062a \\u0628\\u0627 \\u0645\\u0633\\u0626\\u0648\\u0644\\u06cc\\u062a \\u0645\\u062d\\u062f\\u0648\\u062f \\u0646\\u06cc\\u0627\\u0632 \\u0628\\u0647 \\u067e\\u0631\\u062f\\u0627\\u062e\\u062a \\u06a9\\u0627\\u0645\\u0644 \\u0633\\u0631\\u0645\\u0627\\u06cc\\u0647 \\u0646\\u0642\\u062f\\u06cc \\u0648 \\u062a\\u0633\\u0644\\u06cc\\u0645 \\u0633\\u0647\\u0645\\u200c\\u0627\\u0644\\u0634\\u0631\\u06a9\\u0647\\u200c\\u0647\\u0627\\u06cc \\u063a\\u06cc\\u0631 \\u0646\\u0642\\u062f\\u06cc \\u0627\\u0633\\u062a. \\u0647\\u0631\\u06af\\u0648\\u0646\\u0647 \\u062a\\u063a\\u06cc\\u06cc\\u0631 \\u062f\\u0631 \\u0627\\u0633\\u0627\\u0633\\u0646\\u0627\\u0645\\u0647 \\u0646\\u06cc\\u0627\\u0632 \\u0628\\u0647 \\u0627\\u06a9\\u062b\\u0631\\u06cc\\u062a \\u0639\\u062f\\u062f\\u06cc \\u0634\\u0631\\u06a9\\u0627\\u0621 \\u062f\\u0627\\u0631\\u062f. (\\u0645\\u0648\\u0627\\u062f 96\\u060c 97 \\u0648 111)\",\n          \"\\u0628\\u0631\\u0627\\u06cc \\u062a\\u0634\\u06a9\\u06cc\\u0644 \\u0634\\u0631\\u06a9\\u062a \\u0628\\u0627 \\u0645\\u0633\\u0626\\u0648\\u0644\\u06cc\\u062a \\u0645\\u062d\\u062f\\u0648\\u062f \\u0646\\u06cc\\u0627\\u0632 \\u0627\\u0633\\u062a \\u06a9\\u0647 \\u062a\\u0645\\u0627\\u0645 \\u0633\\u0631\\u0645\\u0627\\u06cc\\u0647 \\u0646\\u0642\\u062f\\u06cc \\u067e\\u0631\\u062f\\u0627\\u062e\\u062a \\u0634\\u062f\\u0647 \\u0648 \\u0633\\u0647\\u0645\\u200c\\u0627\\u0644\\u0634\\u0631\\u06a9\\u0647\\u200c\\u0647\\u0627\\u06cc \\u063a\\u06cc\\u0631 \\u0646\\u0642\\u062f\\u06cc \\u062a\\u0642\\u0648\\u06cc\\u0645 \\u0648 \\u062a\\u0633\\u0644\\u06cc\\u0645 \\u0634\\u0648\\u062f. (\\u0645\\u0648\\u0627\\u062f 96 \\u0648 97)\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    }\n  ]\n}",
              "type": "dataframe",
              "variable_name": "df"
            },
            "text/html": [
              "\n",
              "  <div id=\"df-ed5e08c4-dcda-48e1-a479-85c19a3858a9\" class=\"colab-df-container\">\n",
              "    <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Question</th>\n",
              "      <th>Answer</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>چه شرایطی برای تشکیل و ثبت یک شرکت با مسئولیت ...</td>\n",
              "      <td>برای تشکیل یک شرکت با مسئولیت محدود نیاز به پر...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>در یک شرکت با مسئولیت محدود، چگونه مدیریت انجا...</td>\n",
              "      <td>مدیریت شرکت با مسئولیت محدود توسط یک یا چند مد...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>چه فرآیندی باید برای انحلال یک شرکت با مسئولیت...</td>\n",
              "      <td>انحلال شرکت ممکن است در صورت تصمیم شرکاء یا بر...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>تفاوت‌های اصلی بین شرکت‌های با مسئولیت محدود و...</td>\n",
              "      <td>در شرکت با مسئولیت محدود، مدیریت می‌تواند توسط...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>چگونه می‌توان یک شرکت تضامنی را به شرکت سهامی ...</td>\n",
              "      <td>تبدیل یک شرکت تضامنی به شرکت سهامی نیاز به تصو...</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "    <div class=\"colab-df-buttons\">\n",
              "\n",
              "  <div class=\"colab-df-container\">\n",
              "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-ed5e08c4-dcda-48e1-a479-85c19a3858a9')\"\n",
              "            title=\"Convert this dataframe to an interactive table.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
              "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "\n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    .colab-df-buttons div {\n",
              "      margin-bottom: 4px;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "    <script>\n",
              "      const buttonEl =\n",
              "        document.querySelector('#df-ed5e08c4-dcda-48e1-a479-85c19a3858a9 button.colab-df-convert');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      async function convertToInteractive(key) {\n",
              "        const element = document.querySelector('#df-ed5e08c4-dcda-48e1-a479-85c19a3858a9');\n",
              "        const dataTable =\n",
              "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                    [key], {});\n",
              "        if (!dataTable) return;\n",
              "\n",
              "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "          + ' to learn more about interactive tables.';\n",
              "        element.innerHTML = '';\n",
              "        dataTable['output_type'] = 'display_data';\n",
              "        await google.colab.output.renderOutput(dataTable, element);\n",
              "        const docLink = document.createElement('div');\n",
              "        docLink.innerHTML = docLinkHtml;\n",
              "        element.appendChild(docLink);\n",
              "      }\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "\n",
              "<div id=\"df-f8988cc9-646b-4367-b12a-41b23541d4a0\">\n",
              "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-f8988cc9-646b-4367-b12a-41b23541d4a0')\"\n",
              "            title=\"Suggest charts\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\">\n",
              "    <g>\n",
              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
              "    </g>\n",
              "</svg>\n",
              "  </button>\n",
              "\n",
              "<style>\n",
              "  .colab-df-quickchart {\n",
              "      --bg-color: #E8F0FE;\n",
              "      --fill-color: #1967D2;\n",
              "      --hover-bg-color: #E2EBFA;\n",
              "      --hover-fill-color: #174EA6;\n",
              "      --disabled-fill-color: #AAA;\n",
              "      --disabled-bg-color: #DDD;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "      --bg-color: #3B4455;\n",
              "      --fill-color: #D2E3FC;\n",
              "      --hover-bg-color: #434B5C;\n",
              "      --hover-fill-color: #FFFFFF;\n",
              "      --disabled-bg-color: #3B4455;\n",
              "      --disabled-fill-color: #666;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart {\n",
              "    background-color: var(--bg-color);\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: var(--fill-color);\n",
              "    height: 32px;\n",
              "    padding: 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: var(--hover-bg-color);\n",
              "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: var(--button-hover-fill-color);\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart-complete:disabled,\n",
              "  .colab-df-quickchart-complete:disabled:hover {\n",
              "    background-color: var(--disabled-bg-color);\n",
              "    fill: var(--disabled-fill-color);\n",
              "    box-shadow: none;\n",
              "  }\n",
              "\n",
              "  .colab-df-spinner {\n",
              "    border: 2px solid var(--fill-color);\n",
              "    border-color: transparent;\n",
              "    border-bottom-color: var(--fill-color);\n",
              "    animation:\n",
              "      spin 1s steps(1) infinite;\n",
              "  }\n",
              "\n",
              "  @keyframes spin {\n",
              "    0% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "      border-left-color: var(--fill-color);\n",
              "    }\n",
              "    20% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    30% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    40% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    60% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    80% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "    90% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "  }\n",
              "</style>\n",
              "\n",
              "  <script>\n",
              "    async function quickchart(key) {\n",
              "      const quickchartButtonEl =\n",
              "        document.querySelector('#' + key + ' button');\n",
              "      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
              "      quickchartButtonEl.classList.add('colab-df-spinner');\n",
              "      try {\n",
              "        const charts = await google.colab.kernel.invokeFunction(\n",
              "            'suggestCharts', [key], {});\n",
              "      } catch (error) {\n",
              "        console.error('Error during call to suggestCharts:', error);\n",
              "      }\n",
              "      quickchartButtonEl.classList.remove('colab-df-spinner');\n",
              "      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
              "    }\n",
              "    (() => {\n",
              "      let quickchartButtonEl =\n",
              "        document.querySelector('#df-f8988cc9-646b-4367-b12a-41b23541d4a0 button');\n",
              "      quickchartButtonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "    })();\n",
              "  </script>\n",
              "</div>\n",
              "    </div>\n",
              "  </div>\n"
            ],
            "text/plain": [
              "                                            Question  \\\n",
              "0  چه شرایطی برای تشکیل و ثبت یک شرکت با مسئولیت ...   \n",
              "1  در یک شرکت با مسئولیت محدود، چگونه مدیریت انجا...   \n",
              "2  چه فرآیندی باید برای انحلال یک شرکت با مسئولیت...   \n",
              "3  تفاوت‌های اصلی بین شرکت‌های با مسئولیت محدود و...   \n",
              "4  چگونه می‌توان یک شرکت تضامنی را به شرکت سهامی ...   \n",
              "\n",
              "                                              Answer  \n",
              "0  برای تشکیل یک شرکت با مسئولیت محدود نیاز به پر...  \n",
              "1  مدیریت شرکت با مسئولیت محدود توسط یک یا چند مد...  \n",
              "2  انحلال شرکت ممکن است در صورت تصمیم شرکاء یا بر...  \n",
              "3  در شرکت با مسئولیت محدود، مدیریت می‌تواند توسط...  \n",
              "4  تبدیل یک شرکت تضامنی به شرکت سهامی نیاز به تصو...  "
            ]
          },
          "execution_count": 56,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "import pandas as pd\n",
        "\n",
        "BUSSINESS_LAW_PATH = './evaluation/business_law/'\n",
        "\n",
        "dfs = []\n",
        "\n",
        "for file_name in os.listdir(BUSSINESS_LAW_PATH):\n",
        "  if '_' in file_name:\n",
        "    file_path = os.path.join(BUSSINESS_LAW_PATH, file_name)\n",
        "    print(f'{file_name}: {len(pd.read_csv(file_path))}')\n",
        "    dfs.append(pd.read_csv(file_path))\n",
        "\n",
        "df = pd.concat(dfs)\n",
        "\n",
        "df.head()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "-lrn9luNEbjB",
        "outputId": "524cd7fe-c266-41fb-d81f-296df5beb1f0"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "85"
            ]
          },
          "execution_count": 57,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "len(df)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 51,
      "metadata": {
        "id": "iwTuBgtv1iLo"
      },
      "outputs": [],
      "source": [
        "test_questions = df[\"Question\"].values.tolist()\n",
        "test_groundtruths = df[\"Answer\"].values.tolist()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 52,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "a8mcK1NV1pFi",
        "outputId": "b7bae4b9-014b-4894-b73f-0c141ee36851"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Processing: 100%|██████████| 28/28 [01:50<00:00,  3.95s/it]\n"
          ]
        }
      ],
      "source": [
        "from tqdm import tqdm\n",
        "\n",
        "answers = []\n",
        "contexts = []\n",
        "\n",
        "for question in tqdm(test_questions, desc='Processing'):\n",
        "  response = qa_chain.invoke({\"question\" : question})\n",
        "  answers.append(response[\"response\"].content)\n",
        "  contexts.append([context.page_content for context in response[\"context\"]])"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 54,
      "metadata": {
        "id": "BCsf0T4w2nJ8"
      },
      "outputs": [],
      "source": [
        "from ragas import evaluate\n",
        "from ragas.metrics import (\n",
        "    faithfulness,\n",
        "    answer_relevancy,\n",
        "    answer_correctness,\n",
        "    context_recall,\n",
        "    context_precision,\n",
        ")\n",
        "\n",
        "metrics = [\n",
        "    faithfulness,\n",
        "    answer_relevancy,\n",
        "    context_recall,\n",
        "    context_precision,\n",
        "    answer_correctness,\n",
        "]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 55,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 49,
          "referenced_widgets": [
            "e1163e1065d142cf8619c431acea4978",
            "8d0a2c12bc004a2cabe66de87bd07fe5",
            "1f5d86741cc84d489ff76be8c99ae947",
            "5a7e8a8be8dc42ceac0617e62ad7af2d",
            "033a9ee137fa4d6b92bb1068fb704a0e",
            "fcf8eea6ba8e46dc995331c33606650a",
            "340586c4ea9d46d68242eda4a1e51c40",
            "f6d8b4c43f684d8d82bd6cbac613d03a",
            "d806b0c6718f4f2d95dcb8f29e853347",
            "af60982013464e0dacea6d79d04beb54",
            "545e442e6a434ec9b3a009ca2a93f649"
          ]
        },
        "id": "8dcpAhGm2tFU",
        "outputId": "76249196-28fb-44e7-f4ab-bdae0a97baba"
      },
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Evaluating:   0%|          | 0/28 [00:00<?, ?it/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "e1163e1065d142cf8619c431acea4978"
            }
          },
          "metadata": {}
        }
      ],
      "source": [
        "from datasets import Dataset\n",
        "\n",
        "response_dataset = Dataset.from_dict({\n",
        "    \"question\" : test_questions,\n",
        "    \"answer\" : answers,\n",
        "    \"contexts\" : contexts,\n",
        "    \"ground_truth\" : test_groundtruths\n",
        "})\n",
        "results = evaluate(response_dataset, metrics)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 49,
      "metadata": {
        "id": "a8GGibnQRzjb"
      },
      "outputs": [],
      "source": [
        "res_df = results.to_pandas()\n",
        "res_df.to_csv('results.csv', index=True)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "Tu6L1EDHP5a9"
      },
      "outputs": [],
      "source": [
        "import csv\n",
        "\n",
        "data = []\n",
        "for q, ans, c, gt in zip(test_questions, answers, contexts, test_groundtruths):\n",
        "  data.append({\n",
        "      'Question': q,\n",
        "      'Answer': ans,\n",
        "      'Contexts': c,\n",
        "      'Ground Truth': gt\n",
        "  })\n",
        "with open(\"output2.csv\", mode='w', newline='') as file:\n",
        "    writer = csv.DictWriter(file, fieldnames=list(data[0].keys()))\n",
        "    writer.writeheader()\n",
        "    for row in data:\n",
        "        writer.writerow(row)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "7C1-rNhhQRxz"
      },
      "outputs": [],
      "source": []
    }
  ],
  "metadata": {
    "colab": {
      "collapsed_sections": [
        "jHrmtUyPrjCr",
        "po6_FyCKZbyy"
      ],
      "provenance": []
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    },
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "e1163e1065d142cf8619c431acea4978": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_8d0a2c12bc004a2cabe66de87bd07fe5",
              "IPY_MODEL_1f5d86741cc84d489ff76be8c99ae947",
              "IPY_MODEL_5a7e8a8be8dc42ceac0617e62ad7af2d"
            ],
            "layout": "IPY_MODEL_033a9ee137fa4d6b92bb1068fb704a0e"
          }
        },
        "8d0a2c12bc004a2cabe66de87bd07fe5": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_fcf8eea6ba8e46dc995331c33606650a",
            "placeholder": "​",
            "style": "IPY_MODEL_340586c4ea9d46d68242eda4a1e51c40",
            "value": "Evaluating: 100%"
          }
        },
        "1f5d86741cc84d489ff76be8c99ae947": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_f6d8b4c43f684d8d82bd6cbac613d03a",
            "max": 28,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_d806b0c6718f4f2d95dcb8f29e853347",
            "value": 28
          }
        },
        "5a7e8a8be8dc42ceac0617e62ad7af2d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_af60982013464e0dacea6d79d04beb54",
            "placeholder": "​",
            "style": "IPY_MODEL_545e442e6a434ec9b3a009ca2a93f649",
            "value": " 28/28 [00:39&lt;00:00,  1.78s/it]"
          }
        },
        "033a9ee137fa4d6b92bb1068fb704a0e": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "fcf8eea6ba8e46dc995331c33606650a": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "340586c4ea9d46d68242eda4a1e51c40": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "f6d8b4c43f684d8d82bd6cbac613d03a": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "d806b0c6718f4f2d95dcb8f29e853347": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "af60982013464e0dacea6d79d04beb54": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "545e442e6a434ec9b3a009ca2a93f649": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        }
      }
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}