Anupam_IISc_Workshop.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "collapsed_sections": [
        "OUlHzvBl0yHB"
      ],
      "authorship_tag": "ABX9TyNDRgBc07/SonzwlO6qvaym",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/amaze18/freeGPT/blob/main/Anupam_IISc_Workshop.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## **Install required dependencies to run this notebook**"
      ],
      "metadata": {
        "id": "HhZn_biZ10dq"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "#Make following parameter 1 if you have restarted using this colab notebook\n",
        "#after a restart of kernel or re-logging into gmail account\n",
        "running_after_kernel_restart=1\n",
        "if(running_after_kernel_restart==1):\n",
        "  ! pip install gradio\n",
        "  ! pip install langchain\n",
        "  ! pip install dotenv\n",
        "  ! pip install\n",
        "  ! pip install sentence_transformers\n",
        "  ! pip install unstructured\n",
        "  ! pip install youtube-transcript-api\n",
        "  ! pip install faiss\n",
        "  ! pip install faiss-cpu\n",
        "  ! pip install python-dotenv\n",
        "  ! pip install unstructured\n",
        "  ! pip install pdfminer.six\n",
        "  ! pip install openai\n",
        "  ! pip install tiktoken\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ATbgx5HUfq12",
        "outputId": "7609052f-2291-418b-ae49-6b883eb8074c"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Collecting gradio\n",
            "  Downloading gradio-4.13.0-py3-none-any.whl (16.6 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m16.6/16.6 MB\u001b[0m \u001b[31m52.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting aiofiles<24.0,>=22.0 (from gradio)\n",
            "  Downloading aiofiles-23.2.1-py3-none-any.whl (15 kB)\n",
            "Requirement already satisfied: altair<6.0,>=4.2.0 in /usr/local/lib/python3.10/dist-packages (from gradio) (4.2.2)\n",
            "Collecting fastapi (from gradio)\n",
            "  Downloading fastapi-0.108.0-py3-none-any.whl (92 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m92.0/92.0 kB\u001b[0m \u001b[31m8.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting ffmpy (from gradio)\n",
            "  Downloading ffmpy-0.3.1.tar.gz (5.5 kB)\n",
            "  Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "Collecting gradio-client==0.8.0 (from gradio)\n",
            "  Downloading gradio_client-0.8.0-py3-none-any.whl (305 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m305.1/305.1 kB\u001b[0m \u001b[31m39.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting httpx (from gradio)\n",
            "  Downloading httpx-0.26.0-py3-none-any.whl (75 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m75.9/75.9 kB\u001b[0m \u001b[31m12.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: huggingface-hub>=0.19.3 in /usr/local/lib/python3.10/dist-packages (from gradio) (0.20.1)\n",
            "Requirement already satisfied: importlib-resources<7.0,>=1.3 in /usr/local/lib/python3.10/dist-packages (from gradio) (6.1.1)\n",
            "Requirement already satisfied: jinja2<4.0 in /usr/local/lib/python3.10/dist-packages (from gradio) (3.1.2)\n",
            "Requirement already satisfied: markupsafe~=2.0 in /usr/local/lib/python3.10/dist-packages (from gradio) (2.1.3)\n",
            "Requirement already satisfied: matplotlib~=3.0 in /usr/local/lib/python3.10/dist-packages (from gradio) (3.7.1)\n",
            "Requirement already satisfied: numpy~=1.0 in /usr/local/lib/python3.10/dist-packages (from gradio) (1.23.5)\n",
            "Collecting orjson~=3.0 (from gradio)\n",
            "  Downloading orjson-3.9.10-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (138 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m138.7/138.7 kB\u001b[0m \u001b[31m19.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: packaging in /usr/local/lib/python3.10/dist-packages (from gradio) (23.2)\n",
            "Requirement already satisfied: pandas<3.0,>=1.0 in /usr/local/lib/python3.10/dist-packages (from gradio) (1.5.3)\n",
            "Requirement already satisfied: pillow<11.0,>=8.0 in /usr/local/lib/python3.10/dist-packages (from gradio) (9.4.0)\n",
            "Collecting pydantic>=2.0 (from gradio)\n",
            "  Downloading pydantic-2.5.3-py3-none-any.whl (381 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m381.9/381.9 kB\u001b[0m \u001b[31m17.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting pydub (from gradio)\n",
            "  Downloading pydub-0.25.1-py2.py3-none-any.whl (32 kB)\n",
            "Collecting python-multipart (from gradio)\n",
            "  Downloading python_multipart-0.0.6-py3-none-any.whl (45 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m45.7/45.7 kB\u001b[0m \u001b[31m6.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: pyyaml<7.0,>=5.0 in /usr/local/lib/python3.10/dist-packages (from gradio) (6.0.1)\n",
            "Collecting semantic-version~=2.0 (from gradio)\n",
            "  Downloading semantic_version-2.10.0-py2.py3-none-any.whl (15 kB)\n",
            "Collecting tomlkit==0.12.0 (from gradio)\n",
            "  Downloading tomlkit-0.12.0-py3-none-any.whl (37 kB)\n",
            "Requirement already satisfied: typer[all]<1.0,>=0.9 in /usr/local/lib/python3.10/dist-packages (from gradio) (0.9.0)\n",
            "Requirement already satisfied: typing-extensions~=4.0 in /usr/local/lib/python3.10/dist-packages (from gradio) (4.5.0)\n",
            "Collecting uvicorn>=0.14.0 (from gradio)\n",
            "  Downloading uvicorn-0.25.0-py3-none-any.whl (60 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m60.3/60.3 kB\u001b[0m \u001b[31m9.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: fsspec in /usr/local/lib/python3.10/dist-packages (from gradio-client==0.8.0->gradio) (2023.6.0)\n",
            "Collecting websockets<12.0,>=10.0 (from gradio-client==0.8.0->gradio)\n",
            "  Downloading websockets-11.0.3-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl (129 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m129.9/129.9 kB\u001b[0m \u001b[31m17.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: entrypoints in /usr/local/lib/python3.10/dist-packages (from altair<6.0,>=4.2.0->gradio) (0.4)\n",
            "Requirement already satisfied: jsonschema>=3.0 in /usr/local/lib/python3.10/dist-packages (from altair<6.0,>=4.2.0->gradio) (4.19.2)\n",
            "Requirement already satisfied: toolz in /usr/local/lib/python3.10/dist-packages (from altair<6.0,>=4.2.0->gradio) (0.12.0)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from huggingface-hub>=0.19.3->gradio) (3.13.1)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.10/dist-packages (from huggingface-hub>=0.19.3->gradio) (2.31.0)\n",
            "Requirement already satisfied: tqdm>=4.42.1 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub>=0.19.3->gradio) (4.66.1)\n",
            "Requirement already satisfied: contourpy>=1.0.1 in /usr/local/lib/python3.10/dist-packages (from matplotlib~=3.0->gradio) (1.2.0)\n",
            "Requirement already satisfied: cycler>=0.10 in /usr/local/lib/python3.10/dist-packages (from matplotlib~=3.0->gradio) (0.12.1)\n",
            "Requirement already satisfied: fonttools>=4.22.0 in /usr/local/lib/python3.10/dist-packages (from matplotlib~=3.0->gradio) (4.47.0)\n",
            "Requirement already satisfied: kiwisolver>=1.0.1 in /usr/local/lib/python3.10/dist-packages (from matplotlib~=3.0->gradio) (1.4.5)\n",
            "Requirement already satisfied: pyparsing>=2.3.1 in /usr/local/lib/python3.10/dist-packages (from matplotlib~=3.0->gradio) (3.1.1)\n",
            "Requirement already satisfied: python-dateutil>=2.7 in /usr/local/lib/python3.10/dist-packages (from matplotlib~=3.0->gradio) (2.8.2)\n",
            "Requirement already satisfied: pytz>=2020.1 in /usr/local/lib/python3.10/dist-packages (from pandas<3.0,>=1.0->gradio) (2023.3.post1)\n",
            "Collecting annotated-types>=0.4.0 (from pydantic>=2.0->gradio)\n",
            "  Downloading annotated_types-0.6.0-py3-none-any.whl (12 kB)\n",
            "Collecting pydantic-core==2.14.6 (from pydantic>=2.0->gradio)\n",
            "  Downloading pydantic_core-2.14.6-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (2.1 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m2.1/2.1 MB\u001b[0m \u001b[31m37.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting typing-extensions~=4.0 (from gradio)\n",
            "  Downloading typing_extensions-4.9.0-py3-none-any.whl (32 kB)\n",
            "Requirement already satisfied: click<9.0.0,>=7.1.1 in /usr/local/lib/python3.10/dist-packages (from typer[all]<1.0,>=0.9->gradio) (8.1.7)\n",
            "Collecting colorama<0.5.0,>=0.4.3 (from typer[all]<1.0,>=0.9->gradio)\n",
            "  Downloading colorama-0.4.6-py2.py3-none-any.whl (25 kB)\n",
            "Collecting shellingham<2.0.0,>=1.3.0 (from typer[all]<1.0,>=0.9->gradio)\n",
            "  Downloading shellingham-1.5.4-py2.py3-none-any.whl (9.8 kB)\n",
            "Requirement already satisfied: rich<14.0.0,>=10.11.0 in /usr/local/lib/python3.10/dist-packages (from typer[all]<1.0,>=0.9->gradio) (13.7.0)\n",
            "Collecting h11>=0.8 (from uvicorn>=0.14.0->gradio)\n",
            "  Downloading h11-0.14.0-py3-none-any.whl (58 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m58.3/58.3 kB\u001b[0m \u001b[31m7.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting starlette<0.33.0,>=0.29.0 (from fastapi->gradio)\n",
            "  Downloading starlette-0.32.0.post1-py3-none-any.whl (70 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m70.0/70.0 kB\u001b[0m \u001b[31m9.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: anyio in /usr/local/lib/python3.10/dist-packages (from httpx->gradio) (3.7.1)\n",
            "Requirement already satisfied: certifi in /usr/local/lib/python3.10/dist-packages (from httpx->gradio) (2023.11.17)\n",
            "Collecting httpcore==1.* (from httpx->gradio)\n",
            "  Downloading httpcore-1.0.2-py3-none-any.whl (76 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m76.9/76.9 kB\u001b[0m \u001b[31m11.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: idna in /usr/local/lib/python3.10/dist-packages (from httpx->gradio) (3.6)\n",
            "Requirement already satisfied: sniffio in /usr/local/lib/python3.10/dist-packages (from httpx->gradio) (1.3.0)\n",
            "Requirement already satisfied: attrs>=22.2.0 in /usr/local/lib/python3.10/dist-packages (from jsonschema>=3.0->altair<6.0,>=4.2.0->gradio) (23.1.0)\n",
            "Requirement already satisfied: jsonschema-specifications>=2023.03.6 in /usr/local/lib/python3.10/dist-packages (from jsonschema>=3.0->altair<6.0,>=4.2.0->gradio) (2023.11.2)\n",
            "Requirement already satisfied: referencing>=0.28.4 in /usr/local/lib/python3.10/dist-packages (from jsonschema>=3.0->altair<6.0,>=4.2.0->gradio) (0.32.0)\n",
            "Requirement already satisfied: rpds-py>=0.7.1 in /usr/local/lib/python3.10/dist-packages (from jsonschema>=3.0->altair<6.0,>=4.2.0->gradio) (0.15.2)\n",
            "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.10/dist-packages (from python-dateutil>=2.7->matplotlib~=3.0->gradio) (1.16.0)\n",
            "Requirement already satisfied: markdown-it-py>=2.2.0 in /usr/local/lib/python3.10/dist-packages (from rich<14.0.0,>=10.11.0->typer[all]<1.0,>=0.9->gradio) (3.0.0)\n",
            "Requirement already satisfied: pygments<3.0.0,>=2.13.0 in /usr/local/lib/python3.10/dist-packages (from rich<14.0.0,>=10.11.0->typer[all]<1.0,>=0.9->gradio) (2.16.1)\n",
            "Requirement already satisfied: exceptiongroup in /usr/local/lib/python3.10/dist-packages (from anyio->httpx->gradio) (1.2.0)\n",
            "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests->huggingface-hub>=0.19.3->gradio) (3.3.2)\n",
            "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests->huggingface-hub>=0.19.3->gradio) (2.0.7)\n",
            "Requirement already satisfied: mdurl~=0.1 in /usr/local/lib/python3.10/dist-packages (from markdown-it-py>=2.2.0->rich<14.0.0,>=10.11.0->typer[all]<1.0,>=0.9->gradio) (0.1.2)\n",
            "Building wheels for collected packages: ffmpy\n",
            "  Building wheel for ffmpy (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for ffmpy: filename=ffmpy-0.3.1-py3-none-any.whl size=5579 sha256=129d0ea79445239986e2077630e6cf582c7acfd2d39b76b05fed7b1f7a14feee\n",
            "  Stored in directory: /root/.cache/pip/wheels/01/a6/d1/1c0828c304a4283b2c1639a09ad86f83d7c487ef34c6b4a1bf\n",
            "Successfully built ffmpy\n",
            "Installing collected packages: pydub, ffmpy, websockets, typing-extensions, tomlkit, shellingham, semantic-version, python-multipart, orjson, h11, colorama, annotated-types, aiofiles, uvicorn, starlette, pydantic-core, httpcore, pydantic, httpx, gradio-client, fastapi, gradio\n",
            "  Attempting uninstall: typing-extensions\n",
            "    Found existing installation: typing_extensions 4.5.0\n",
            "    Uninstalling typing_extensions-4.5.0:\n",
            "      Successfully uninstalled typing_extensions-4.5.0\n",
            "  Attempting uninstall: pydantic\n",
            "    Found existing installation: pydantic 1.10.13\n",
            "    Uninstalling pydantic-1.10.13:\n",
            "      Successfully uninstalled pydantic-1.10.13\n",
            "\u001b[31mERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.\n",
            "lida 0.0.10 requires kaleido, which is not installed.\n",
            "llmx 0.0.15a0 requires cohere, which is not installed.\n",
            "llmx 0.0.15a0 requires openai, which is not installed.\n",
            "llmx 0.0.15a0 requires tiktoken, which is not installed.\n",
            "tensorflow-probability 0.22.0 requires typing-extensions<4.6.0, but you have typing-extensions 4.9.0 which is incompatible.\u001b[0m\u001b[31m\n",
            "\u001b[0mSuccessfully installed aiofiles-23.2.1 annotated-types-0.6.0 colorama-0.4.6 fastapi-0.108.0 ffmpy-0.3.1 gradio-4.13.0 gradio-client-0.8.0 h11-0.14.0 httpcore-1.0.2 httpx-0.26.0 orjson-3.9.10 pydantic-2.5.3 pydantic-core-2.14.6 pydub-0.25.1 python-multipart-0.0.6 semantic-version-2.10.0 shellingham-1.5.4 starlette-0.32.0.post1 tomlkit-0.12.0 typing-extensions-4.9.0 uvicorn-0.25.0 websockets-11.0.3\n",
            "Collecting langchain\n",
            "  Downloading langchain-0.1.0-py3-none-any.whl (797 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m798.0/798.0 kB\u001b[0m \u001b[31m5.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: PyYAML>=5.3 in /usr/local/lib/python3.10/dist-packages (from langchain) (6.0.1)\n",
            "Requirement already satisfied: SQLAlchemy<3,>=1.4 in /usr/local/lib/python3.10/dist-packages (from langchain) (2.0.23)\n",
            "Requirement already satisfied: aiohttp<4.0.0,>=3.8.3 in /usr/local/lib/python3.10/dist-packages (from langchain) (3.9.1)\n",
            "Requirement already satisfied: async-timeout<5.0.0,>=4.0.0 in /usr/local/lib/python3.10/dist-packages (from langchain) (4.0.3)\n",
            "Collecting dataclasses-json<0.7,>=0.5.7 (from langchain)\n",
            "  Downloading dataclasses_json-0.6.3-py3-none-any.whl (28 kB)\n",
            "Collecting jsonpatch<2.0,>=1.33 (from langchain)\n",
            "  Downloading jsonpatch-1.33-py2.py3-none-any.whl (12 kB)\n",
            "Collecting langchain-community<0.1,>=0.0.9 (from langchain)\n",
            "  Downloading langchain_community-0.0.9-py3-none-any.whl (1.5 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.5/1.5 MB\u001b[0m \u001b[31m28.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting langchain-core<0.2,>=0.1.7 (from langchain)\n",
            "  Downloading langchain_core-0.1.7-py3-none-any.whl (214 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m214.3/214.3 kB\u001b[0m \u001b[31m29.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting langsmith<0.1.0,>=0.0.77 (from langchain)\n",
            "  Downloading langsmith-0.0.77-py3-none-any.whl (48 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m48.2/48.2 kB\u001b[0m \u001b[31m7.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: numpy<2,>=1 in /usr/local/lib/python3.10/dist-packages (from langchain) (1.23.5)\n",
            "Requirement already satisfied: pydantic<3,>=1 in /usr/local/lib/python3.10/dist-packages (from langchain) (2.5.3)\n",
            "Requirement already satisfied: requests<3,>=2 in /usr/local/lib/python3.10/dist-packages (from langchain) (2.31.0)\n",
            "Requirement already satisfied: tenacity<9.0.0,>=8.1.0 in /usr/local/lib/python3.10/dist-packages (from langchain) (8.2.3)\n",
            "Requirement already satisfied: attrs>=17.3.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp<4.0.0,>=3.8.3->langchain) (23.1.0)\n",
            "Requirement already satisfied: multidict<7.0,>=4.5 in /usr/local/lib/python3.10/dist-packages (from aiohttp<4.0.0,>=3.8.3->langchain) (6.0.4)\n",
            "Requirement already satisfied: yarl<2.0,>=1.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp<4.0.0,>=3.8.3->langchain) (1.9.4)\n",
            "Requirement already satisfied: frozenlist>=1.1.1 in /usr/local/lib/python3.10/dist-packages (from aiohttp<4.0.0,>=3.8.3->langchain) (1.4.1)\n",
            "Requirement already satisfied: aiosignal>=1.1.2 in /usr/local/lib/python3.10/dist-packages (from aiohttp<4.0.0,>=3.8.3->langchain) (1.3.1)\n",
            "Collecting marshmallow<4.0.0,>=3.18.0 (from dataclasses-json<0.7,>=0.5.7->langchain)\n",
            "  Downloading marshmallow-3.20.1-py3-none-any.whl (49 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m49.4/49.4 kB\u001b[0m \u001b[31m6.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting typing-inspect<1,>=0.4.0 (from dataclasses-json<0.7,>=0.5.7->langchain)\n",
            "  Downloading typing_inspect-0.9.0-py3-none-any.whl (8.8 kB)\n",
            "Collecting jsonpointer>=1.9 (from jsonpatch<2.0,>=1.33->langchain)\n",
            "  Downloading jsonpointer-2.4-py2.py3-none-any.whl (7.8 kB)\n",
            "Requirement already satisfied: anyio<5,>=3 in /usr/local/lib/python3.10/dist-packages (from langchain-core<0.2,>=0.1.7->langchain) (3.7.1)\n",
            "Requirement already satisfied: packaging<24.0,>=23.2 in /usr/local/lib/python3.10/dist-packages (from langchain-core<0.2,>=0.1.7->langchain) (23.2)\n",
            "Requirement already satisfied: annotated-types>=0.4.0 in /usr/local/lib/python3.10/dist-packages (from pydantic<3,>=1->langchain) (0.6.0)\n",
            "Requirement already satisfied: pydantic-core==2.14.6 in /usr/local/lib/python3.10/dist-packages (from pydantic<3,>=1->langchain) (2.14.6)\n",
            "Requirement already satisfied: typing-extensions>=4.6.1 in /usr/local/lib/python3.10/dist-packages (from pydantic<3,>=1->langchain) (4.9.0)\n",
            "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests<3,>=2->langchain) (3.3.2)\n",
            "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests<3,>=2->langchain) (3.6)\n",
            "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests<3,>=2->langchain) (2.0.7)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests<3,>=2->langchain) (2023.11.17)\n",
            "Requirement already satisfied: greenlet!=0.4.17 in /usr/local/lib/python3.10/dist-packages (from SQLAlchemy<3,>=1.4->langchain) (3.0.2)\n",
            "Requirement already satisfied: sniffio>=1.1 in /usr/local/lib/python3.10/dist-packages (from anyio<5,>=3->langchain-core<0.2,>=0.1.7->langchain) (1.3.0)\n",
            "Requirement already satisfied: exceptiongroup in /usr/local/lib/python3.10/dist-packages (from anyio<5,>=3->langchain-core<0.2,>=0.1.7->langchain) (1.2.0)\n",
            "Collecting mypy-extensions>=0.3.0 (from typing-inspect<1,>=0.4.0->dataclasses-json<0.7,>=0.5.7->langchain)\n",
            "  Downloading mypy_extensions-1.0.0-py3-none-any.whl (4.7 kB)\n",
            "Installing collected packages: mypy-extensions, marshmallow, jsonpointer, typing-inspect, jsonpatch, langsmith, dataclasses-json, langchain-core, langchain-community, langchain\n",
            "Successfully installed dataclasses-json-0.6.3 jsonpatch-1.33 jsonpointer-2.4 langchain-0.1.0 langchain-community-0.0.9 langchain-core-0.1.7 langsmith-0.0.77 marshmallow-3.20.1 mypy-extensions-1.0.0 typing-inspect-0.9.0\n",
            "Collecting dotenv\n",
            "  Downloading dotenv-0.0.5.tar.gz (2.4 kB)\n",
            "  \u001b[1;31merror\u001b[0m: \u001b[1msubprocess-exited-with-error\u001b[0m\n",
            "  \n",
            "  \u001b[31m×\u001b[0m \u001b[32mpython setup.py egg_info\u001b[0m did not run successfully.\n",
            "  \u001b[31m│\u001b[0m exit code: \u001b[1;36m1\u001b[0m\n",
            "  \u001b[31m╰─>\u001b[0m See above for output.\n",
            "  \n",
            "  \u001b[1;35mnote\u001b[0m: This error originates from a subprocess, and is likely not a problem with pip.\n",
            "  Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25herror\n",
            "\u001b[1;31merror\u001b[0m: \u001b[1mmetadata-generation-failed\u001b[0m\n",
            "\n",
            "\u001b[31m×\u001b[0m Encountered error while generating package metadata.\n",
            "\u001b[31m╰─>\u001b[0m See above for output.\n",
            "\n",
            "\u001b[1;35mnote\u001b[0m: This is an issue with the package mentioned above, not pip.\n",
            "\u001b[1;36mhint\u001b[0m: See above for details.\n",
            "\u001b[31mERROR: You must give at least one requirement to install (see \"pip help install\")\u001b[0m\u001b[31m\n",
            "\u001b[0mCollecting sentence_transformers\n",
            "  Downloading sentence-transformers-2.2.2.tar.gz (85 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m86.0/86.0 kB\u001b[0m \u001b[31m1.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25h  Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "Requirement already satisfied: transformers<5.0.0,>=4.6.0 in /usr/local/lib/python3.10/dist-packages (from sentence_transformers) (4.35.2)\n",
            "Requirement already satisfied: tqdm in /usr/local/lib/python3.10/dist-packages (from sentence_transformers) (4.66.1)\n",
            "Requirement already satisfied: torch>=1.6.0 in /usr/local/lib/python3.10/dist-packages (from sentence_transformers) (2.1.0+cu121)\n",
            "Requirement already satisfied: torchvision in /usr/local/lib/python3.10/dist-packages (from sentence_transformers) (0.16.0+cu121)\n",
            "Requirement already satisfied: numpy in /usr/local/lib/python3.10/dist-packages (from sentence_transformers) (1.23.5)\n",
            "Requirement already satisfied: scikit-learn in /usr/local/lib/python3.10/dist-packages (from sentence_transformers) (1.2.2)\n",
            "Requirement already satisfied: scipy in /usr/local/lib/python3.10/dist-packages (from sentence_transformers) (1.11.4)\n",
            "Requirement already satisfied: nltk in /usr/local/lib/python3.10/dist-packages (from sentence_transformers) (3.8.1)\n",
            "Collecting sentencepiece (from sentence_transformers)\n",
            "  Downloading sentencepiece-0.1.99-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (1.3 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.3/1.3 MB\u001b[0m \u001b[31m11.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: huggingface-hub>=0.4.0 in /usr/local/lib/python3.10/dist-packages (from sentence_transformers) (0.20.1)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from huggingface-hub>=0.4.0->sentence_transformers) (3.13.1)\n",
            "Requirement already satisfied: fsspec>=2023.5.0 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub>=0.4.0->sentence_transformers) (2023.6.0)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.10/dist-packages (from huggingface-hub>=0.4.0->sentence_transformers) (2.31.0)\n",
            "Requirement already satisfied: pyyaml>=5.1 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub>=0.4.0->sentence_transformers) (6.0.1)\n",
            "Requirement already satisfied: typing-extensions>=3.7.4.3 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub>=0.4.0->sentence_transformers) (4.9.0)\n",
            "Requirement already satisfied: packaging>=20.9 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub>=0.4.0->sentence_transformers) (23.2)\n",
            "Requirement already satisfied: sympy in /usr/local/lib/python3.10/dist-packages (from torch>=1.6.0->sentence_transformers) (1.12)\n",
            "Requirement already satisfied: networkx in /usr/local/lib/python3.10/dist-packages (from torch>=1.6.0->sentence_transformers) (3.2.1)\n",
            "Requirement already satisfied: jinja2 in /usr/local/lib/python3.10/dist-packages (from torch>=1.6.0->sentence_transformers) (3.1.2)\n",
            "Requirement already satisfied: triton==2.1.0 in /usr/local/lib/python3.10/dist-packages (from torch>=1.6.0->sentence_transformers) (2.1.0)\n",
            "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.10/dist-packages (from transformers<5.0.0,>=4.6.0->sentence_transformers) (2023.6.3)\n",
            "Requirement already satisfied: tokenizers<0.19,>=0.14 in /usr/local/lib/python3.10/dist-packages (from transformers<5.0.0,>=4.6.0->sentence_transformers) (0.15.0)\n",
            "Requirement already satisfied: safetensors>=0.3.1 in /usr/local/lib/python3.10/dist-packages (from transformers<5.0.0,>=4.6.0->sentence_transformers) (0.4.1)\n",
            "Requirement already satisfied: click in /usr/local/lib/python3.10/dist-packages (from nltk->sentence_transformers) (8.1.7)\n",
            "Requirement already satisfied: joblib in /usr/local/lib/python3.10/dist-packages (from nltk->sentence_transformers) (1.3.2)\n",
            "Requirement already satisfied: threadpoolctl>=2.0.0 in /usr/local/lib/python3.10/dist-packages (from scikit-learn->sentence_transformers) (3.2.0)\n",
            "Requirement already satisfied: pillow!=8.3.*,>=5.3.0 in /usr/local/lib/python3.10/dist-packages (from torchvision->sentence_transformers) (9.4.0)\n",
            "Requirement already satisfied: MarkupSafe>=2.0 in /usr/local/lib/python3.10/dist-packages (from jinja2->torch>=1.6.0->sentence_transformers) (2.1.3)\n",
            "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests->huggingface-hub>=0.4.0->sentence_transformers) (3.3.2)\n",
            "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests->huggingface-hub>=0.4.0->sentence_transformers) (3.6)\n",
            "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests->huggingface-hub>=0.4.0->sentence_transformers) (2.0.7)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests->huggingface-hub>=0.4.0->sentence_transformers) (2023.11.17)\n",
            "Requirement already satisfied: mpmath>=0.19 in /usr/local/lib/python3.10/dist-packages (from sympy->torch>=1.6.0->sentence_transformers) (1.3.0)\n",
            "Building wheels for collected packages: sentence_transformers\n",
            "  Building wheel for sentence_transformers (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for sentence_transformers: filename=sentence_transformers-2.2.2-py3-none-any.whl size=125923 sha256=3e0adefd98c5a3ef5840d40b829a76ed46efbd8cc533688b1dbf54a63063e82f\n",
            "  Stored in directory: /root/.cache/pip/wheels/62/f2/10/1e606fd5f02395388f74e7462910fe851042f97238cbbd902f\n",
            "Successfully built sentence_transformers\n",
            "Installing collected packages: sentencepiece, sentence_transformers\n",
            "Successfully installed sentence_transformers-2.2.2 sentencepiece-0.1.99\n",
            "Collecting unstructured\n",
            "  Downloading unstructured-0.11.8-py3-none-any.whl (1.8 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.8/1.8 MB\u001b[0m \u001b[31m8.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: chardet in /usr/local/lib/python3.10/dist-packages (from unstructured) (5.2.0)\n",
            "Collecting filetype (from unstructured)\n",
            "  Downloading filetype-1.2.0-py2.py3-none-any.whl (19 kB)\n",
            "Collecting python-magic (from unstructured)\n",
            "  Downloading python_magic-0.4.27-py2.py3-none-any.whl (13 kB)\n",
            "Requirement already satisfied: lxml in /usr/local/lib/python3.10/dist-packages (from unstructured) (4.9.4)\n",
            "Requirement already satisfied: nltk in /usr/local/lib/python3.10/dist-packages (from unstructured) (3.8.1)\n",
            "Requirement already satisfied: tabulate in /usr/local/lib/python3.10/dist-packages (from unstructured) (0.9.0)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.10/dist-packages (from unstructured) (2.31.0)\n",
            "Requirement already satisfied: beautifulsoup4 in /usr/local/lib/python3.10/dist-packages (from unstructured) (4.11.2)\n",
            "Collecting emoji (from unstructured)\n",
            "  Downloading emoji-2.9.0-py2.py3-none-any.whl (397 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m397.5/397.5 kB\u001b[0m \u001b[31m44.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: dataclasses-json in /usr/local/lib/python3.10/dist-packages (from unstructured) (0.6.3)\n",
            "Collecting python-iso639 (from unstructured)\n",
            "  Downloading python_iso639-2024.1.2-py3-none-any.whl (274 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m274.7/274.7 kB\u001b[0m \u001b[31m35.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting langdetect (from unstructured)\n",
            "  Downloading langdetect-1.0.9.tar.gz (981 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m981.5/981.5 kB\u001b[0m \u001b[31m57.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25h  Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "Requirement already satisfied: numpy in /usr/local/lib/python3.10/dist-packages (from unstructured) (1.23.5)\n",
            "Collecting rapidfuzz (from unstructured)\n",
            "  Downloading rapidfuzz-3.6.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (3.4 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m3.4/3.4 MB\u001b[0m \u001b[31m55.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting backoff (from unstructured)\n",
            "  Downloading backoff-2.2.1-py3-none-any.whl (15 kB)\n",
            "Requirement already satisfied: typing-extensions in /usr/local/lib/python3.10/dist-packages (from unstructured) (4.9.0)\n",
            "Collecting unstructured-client (from unstructured)\n",
            "  Downloading unstructured_client-0.15.1-py3-none-any.whl (20 kB)\n",
            "Requirement already satisfied: wrapt in /usr/local/lib/python3.10/dist-packages (from unstructured) (1.14.1)\n",
            "Requirement already satisfied: soupsieve>1.2 in /usr/local/lib/python3.10/dist-packages (from beautifulsoup4->unstructured) (2.5)\n",
            "Requirement already satisfied: marshmallow<4.0.0,>=3.18.0 in /usr/local/lib/python3.10/dist-packages (from dataclasses-json->unstructured) (3.20.1)\n",
            "Requirement already satisfied: typing-inspect<1,>=0.4.0 in /usr/local/lib/python3.10/dist-packages (from dataclasses-json->unstructured) (0.9.0)\n",
            "Requirement already satisfied: six in /usr/local/lib/python3.10/dist-packages (from langdetect->unstructured) (1.16.0)\n",
            "Requirement already satisfied: click in /usr/local/lib/python3.10/dist-packages (from nltk->unstructured) (8.1.7)\n",
            "Requirement already satisfied: joblib in /usr/local/lib/python3.10/dist-packages (from nltk->unstructured) (1.3.2)\n",
            "Requirement already satisfied: regex>=2021.8.3 in /usr/local/lib/python3.10/dist-packages (from nltk->unstructured) (2023.6.3)\n",
            "Requirement already satisfied: tqdm in /usr/local/lib/python3.10/dist-packages (from nltk->unstructured) (4.66.1)\n",
            "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests->unstructured) (3.3.2)\n",
            "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests->unstructured) (3.6)\n",
            "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests->unstructured) (2.0.7)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests->unstructured) (2023.11.17)\n",
            "Collecting jsonpath-python>=1.0.6 (from unstructured-client->unstructured)\n",
            "  Downloading jsonpath_python-1.0.6-py3-none-any.whl (7.6 kB)\n",
            "Requirement already satisfied: mypy-extensions>=1.0.0 in /usr/local/lib/python3.10/dist-packages (from unstructured-client->unstructured) (1.0.0)\n",
            "Requirement already satisfied: packaging>=23.1 in /usr/local/lib/python3.10/dist-packages (from unstructured-client->unstructured) (23.2)\n",
            "Requirement already satisfied: python-dateutil>=2.8.2 in /usr/local/lib/python3.10/dist-packages (from unstructured-client->unstructured) (2.8.2)\n",
            "Building wheels for collected packages: langdetect\n",
            "  Building wheel for langdetect (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for langdetect: filename=langdetect-1.0.9-py3-none-any.whl size=993225 sha256=13dc4791be5f96d8212c0feefe90c09a424764db93da7cefb04d50c5f66b0f40\n",
            "  Stored in directory: /root/.cache/pip/wheels/95/03/7d/59ea870c70ce4e5a370638b5462a7711ab78fba2f655d05106\n",
            "Successfully built langdetect\n",
            "Installing collected packages: filetype, rapidfuzz, python-magic, python-iso639, langdetect, jsonpath-python, emoji, backoff, unstructured-client, unstructured\n",
            "Successfully installed backoff-2.2.1 emoji-2.9.0 filetype-1.2.0 jsonpath-python-1.0.6 langdetect-1.0.9 python-iso639-2024.1.2 python-magic-0.4.27 rapidfuzz-3.6.1 unstructured-0.11.8 unstructured-client-0.15.1\n",
            "Collecting youtube-transcript-api\n",
            "  Downloading youtube_transcript_api-0.6.2-py3-none-any.whl (24 kB)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.10/dist-packages (from youtube-transcript-api) (2.31.0)\n",
            "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests->youtube-transcript-api) (3.3.2)\n",
            "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests->youtube-transcript-api) (3.6)\n",
            "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests->youtube-transcript-api) (2.0.7)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests->youtube-transcript-api) (2023.11.17)\n",
            "Installing collected packages: youtube-transcript-api\n",
            "Successfully installed youtube-transcript-api-0.6.2\n",
            "\u001b[31mERROR: Could not find a version that satisfies the requirement faiss (from versions: none)\u001b[0m\u001b[31m\n",
            "\u001b[0m\u001b[31mERROR: No matching distribution found for faiss\u001b[0m\u001b[31m\n",
            "\u001b[0mCollecting faiss-cpu\n",
            "  Downloading faiss_cpu-1.7.4-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (17.6 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m17.6/17.6 MB\u001b[0m \u001b[31m46.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hInstalling collected packages: faiss-cpu\n",
            "Successfully installed faiss-cpu-1.7.4\n",
            "Collecting python-dotenv\n",
            "  Downloading python_dotenv-1.0.0-py3-none-any.whl (19 kB)\n",
            "Installing collected packages: python-dotenv\n",
            "Successfully installed python-dotenv-1.0.0\n",
            "Requirement already satisfied: unstructured in /usr/local/lib/python3.10/dist-packages (0.11.8)\n",
            "Requirement already satisfied: chardet in /usr/local/lib/python3.10/dist-packages (from unstructured) (5.2.0)\n",
            "Requirement already satisfied: filetype in /usr/local/lib/python3.10/dist-packages (from unstructured) (1.2.0)\n",
            "Requirement already satisfied: python-magic in /usr/local/lib/python3.10/dist-packages (from unstructured) (0.4.27)\n",
            "Requirement already satisfied: lxml in /usr/local/lib/python3.10/dist-packages (from unstructured) (4.9.4)\n",
            "Requirement already satisfied: nltk in /usr/local/lib/python3.10/dist-packages (from unstructured) (3.8.1)\n",
            "Requirement already satisfied: tabulate in /usr/local/lib/python3.10/dist-packages (from unstructured) (0.9.0)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.10/dist-packages (from unstructured) (2.31.0)\n",
            "Requirement already satisfied: beautifulsoup4 in /usr/local/lib/python3.10/dist-packages (from unstructured) (4.11.2)\n",
            "Requirement already satisfied: emoji in /usr/local/lib/python3.10/dist-packages (from unstructured) (2.9.0)\n",
            "Requirement already satisfied: dataclasses-json in /usr/local/lib/python3.10/dist-packages (from unstructured) (0.6.3)\n",
            "Requirement already satisfied: python-iso639 in /usr/local/lib/python3.10/dist-packages (from unstructured) (2024.1.2)\n",
            "Requirement already satisfied: langdetect in /usr/local/lib/python3.10/dist-packages (from unstructured) (1.0.9)\n",
            "Requirement already satisfied: numpy in /usr/local/lib/python3.10/dist-packages (from unstructured) (1.23.5)\n",
            "Requirement already satisfied: rapidfuzz in /usr/local/lib/python3.10/dist-packages (from unstructured) (3.6.1)\n",
            "Requirement already satisfied: backoff in /usr/local/lib/python3.10/dist-packages (from unstructured) (2.2.1)\n",
            "Requirement already satisfied: typing-extensions in /usr/local/lib/python3.10/dist-packages (from unstructured) (4.9.0)\n",
            "Requirement already satisfied: unstructured-client in /usr/local/lib/python3.10/dist-packages (from unstructured) (0.15.1)\n",
            "Requirement already satisfied: wrapt in /usr/local/lib/python3.10/dist-packages (from unstructured) (1.14.1)\n",
            "Requirement already satisfied: soupsieve>1.2 in /usr/local/lib/python3.10/dist-packages (from beautifulsoup4->unstructured) (2.5)\n",
            "Requirement already satisfied: marshmallow<4.0.0,>=3.18.0 in /usr/local/lib/python3.10/dist-packages (from dataclasses-json->unstructured) (3.20.1)\n",
            "Requirement already satisfied: typing-inspect<1,>=0.4.0 in /usr/local/lib/python3.10/dist-packages (from dataclasses-json->unstructured) (0.9.0)\n",
            "Requirement already satisfied: six in /usr/local/lib/python3.10/dist-packages (from langdetect->unstructured) (1.16.0)\n",
            "Requirement already satisfied: click in /usr/local/lib/python3.10/dist-packages (from nltk->unstructured) (8.1.7)\n",
            "Requirement already satisfied: joblib in /usr/local/lib/python3.10/dist-packages (from nltk->unstructured) (1.3.2)\n",
            "Requirement already satisfied: regex>=2021.8.3 in /usr/local/lib/python3.10/dist-packages (from nltk->unstructured) (2023.6.3)\n",
            "Requirement already satisfied: tqdm in /usr/local/lib/python3.10/dist-packages (from nltk->unstructured) (4.66.1)\n",
            "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests->unstructured) (3.3.2)\n",
            "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests->unstructured) (3.6)\n",
            "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests->unstructured) (2.0.7)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests->unstructured) (2023.11.17)\n",
            "Requirement already satisfied: jsonpath-python>=1.0.6 in /usr/local/lib/python3.10/dist-packages (from unstructured-client->unstructured) (1.0.6)\n",
            "Requirement already satisfied: mypy-extensions>=1.0.0 in /usr/local/lib/python3.10/dist-packages (from unstructured-client->unstructured) (1.0.0)\n",
            "Requirement already satisfied: packaging>=23.1 in /usr/local/lib/python3.10/dist-packages (from unstructured-client->unstructured) (23.2)\n",
            "Requirement already satisfied: python-dateutil>=2.8.2 in /usr/local/lib/python3.10/dist-packages (from unstructured-client->unstructured) (2.8.2)\n",
            "Collecting pdfminer.six\n",
            "  Downloading pdfminer.six-20231228-py3-none-any.whl (5.6 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m5.6/5.6 MB\u001b[0m \u001b[31m22.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: charset-normalizer>=2.0.0 in /usr/local/lib/python3.10/dist-packages (from pdfminer.six) (3.3.2)\n",
            "Requirement already satisfied: cryptography>=36.0.0 in /usr/local/lib/python3.10/dist-packages (from pdfminer.six) (41.0.7)\n",
            "Requirement already satisfied: cffi>=1.12 in /usr/local/lib/python3.10/dist-packages (from cryptography>=36.0.0->pdfminer.six) (1.16.0)\n",
            "Requirement already satisfied: pycparser in /usr/local/lib/python3.10/dist-packages (from cffi>=1.12->cryptography>=36.0.0->pdfminer.six) (2.21)\n",
            "Installing collected packages: pdfminer.six\n",
            "Successfully installed pdfminer.six-20231228\n",
            "Collecting openai\n",
            "  Downloading openai-1.6.1-py3-none-any.whl (225 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m225.4/225.4 kB\u001b[0m \u001b[31m2.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: anyio<5,>=3.5.0 in /usr/local/lib/python3.10/dist-packages (from openai) (3.7.1)\n",
            "Requirement already satisfied: distro<2,>=1.7.0 in /usr/lib/python3/dist-packages (from openai) (1.7.0)\n",
            "Requirement already satisfied: httpx<1,>=0.23.0 in /usr/local/lib/python3.10/dist-packages (from openai) (0.26.0)\n",
            "Requirement already satisfied: pydantic<3,>=1.9.0 in /usr/local/lib/python3.10/dist-packages (from openai) (2.5.3)\n",
            "Requirement already satisfied: sniffio in /usr/local/lib/python3.10/dist-packages (from openai) (1.3.0)\n",
            "Requirement already satisfied: tqdm>4 in /usr/local/lib/python3.10/dist-packages (from openai) (4.66.1)\n",
            "Requirement already satisfied: typing-extensions<5,>=4.7 in /usr/local/lib/python3.10/dist-packages (from openai) (4.9.0)\n",
            "Requirement already satisfied: idna>=2.8 in /usr/local/lib/python3.10/dist-packages (from anyio<5,>=3.5.0->openai) (3.6)\n",
            "Requirement already satisfied: exceptiongroup in /usr/local/lib/python3.10/dist-packages (from anyio<5,>=3.5.0->openai) (1.2.0)\n",
            "Requirement already satisfied: certifi in /usr/local/lib/python3.10/dist-packages (from httpx<1,>=0.23.0->openai) (2023.11.17)\n",
            "Requirement already satisfied: httpcore==1.* in /usr/local/lib/python3.10/dist-packages (from httpx<1,>=0.23.0->openai) (1.0.2)\n",
            "Requirement already satisfied: h11<0.15,>=0.13 in /usr/local/lib/python3.10/dist-packages (from httpcore==1.*->httpx<1,>=0.23.0->openai) (0.14.0)\n",
            "Requirement already satisfied: annotated-types>=0.4.0 in /usr/local/lib/python3.10/dist-packages (from pydantic<3,>=1.9.0->openai) (0.6.0)\n",
            "Requirement already satisfied: pydantic-core==2.14.6 in /usr/local/lib/python3.10/dist-packages (from pydantic<3,>=1.9.0->openai) (2.14.6)\n",
            "Installing collected packages: openai\n",
            "\u001b[31mERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.\n",
            "llmx 0.0.15a0 requires cohere, which is not installed.\n",
            "llmx 0.0.15a0 requires tiktoken, which is not installed.\u001b[0m\u001b[31m\n",
            "\u001b[0mSuccessfully installed openai-1.6.1\n",
            "Collecting tiktoken\n",
            "  Downloading tiktoken-0.5.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (2.0 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m2.0/2.0 MB\u001b[0m \u001b[31m10.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: regex>=2022.1.18 in /usr/local/lib/python3.10/dist-packages (from tiktoken) (2023.6.3)\n",
            "Requirement already satisfied: requests>=2.26.0 in /usr/local/lib/python3.10/dist-packages (from tiktoken) (2.31.0)\n",
            "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests>=2.26.0->tiktoken) (3.3.2)\n",
            "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests>=2.26.0->tiktoken) (3.6)\n",
            "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests>=2.26.0->tiktoken) (2.0.7)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests>=2.26.0->tiktoken) (2023.11.17)\n",
            "Installing collected packages: tiktoken\n",
            "\u001b[31mERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.\n",
            "llmx 0.0.15a0 requires cohere, which is not installed.\u001b[0m\u001b[31m\n",
            "\u001b[0mSuccessfully installed tiktoken-0.5.2\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# **Model Development**"
      ],
      "metadata": {
        "id": "9NZUMKUl1FLN"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "CjeExmsNdc2r"
      },
      "outputs": [],
      "source": [
        "import openai\n",
        "from langchain.chat_models import ChatOpenAI\n",
        "import os\n",
        "from langchain.llms import OpenAI\n",
        "from langchain import HuggingFacePipeline\n",
        "from langchain.chains import RetrievalQA\n",
        "from langchain.embeddings import HuggingFaceEmbeddings\n",
        "from langchain.document_loaders import YoutubeLoader\n",
        "from langchain.text_splitter import RecursiveCharacterTextSplitter\n",
        "from langchain.vectorstores import FAISS\n",
        "from langchain.chains import LLMChain\n",
        "from dotenv import find_dotenv, load_dotenv\n",
        "from langchain.prompts.chat import (\n",
        "    ChatPromptTemplate,\n",
        "    SystemMessagePromptTemplate,\n",
        "    HumanMessagePromptTemplate,\n",
        ")\n",
        "\n",
        "from langchain.chains import RetrievalQA\n",
        "from langchain.embeddings import HuggingFaceEmbeddings\n",
        "from langchain.embeddings.openai import OpenAIEmbeddings\n",
        "import textwrap\n",
        "chunk_size = 100\n",
        "openai.api_key =''\n",
        "\n",
        "\n",
        "# Different Large Language Models\n",
        "model_name = \"sentence-transformers/all-mpnet-base-v2\"\n",
        "model_name1 = \"sentence-transformers/LaBSE\"\n",
        "model_name= 'intfloat/e5-large-v2'\n",
        "model_name = 'all-MiniLM-L6-v2'\n",
        "model_name=\"google/flan-t5-base\"\n",
        "model_name=\"google/flan-t5-xl\"\n",
        "model_name=\"google/flan-t5-large\"\n",
        "model_kwargs = {'device': 'cpu'}\n",
        "encode_kwargs = {'normalize_embeddings': False}\n",
        "\n",
        "\n",
        "#creating a database\n",
        "def creating_db(video_url, chunk, overlap):\n",
        "\n",
        "    loader= YoutubeLoader.from_youtube_url(video_url)\n",
        "    transcript= loader.load()\n",
        "\n",
        "\n",
        "    #to breakdown the enormous amount of tokens we will get from the transcript as we have a limited set we can input\n",
        "    text_splitter= RecursiveCharacterTextSplitter(chunk_size=chunk, chunk_overlap=overlap)\n",
        "\n",
        "    #this is just a list with the bunch of splits from the above\n",
        "    docs= text_splitter.split_documents(transcript)\n",
        "\n",
        "\n",
        "    #the final database\n",
        "    '''\n",
        "    when a user asks a question, this database will be used to perform the similarity search and\n",
        "    generate output based on that\n",
        "    '''\n",
        "    if(len(docs)>0):\n",
        "\n",
        "      openai.api_key =\"sk-\"\n",
        "      hf= OpenAIEmbeddings(model=\"text-embedding-ada-002\", openai_api_key=openai.api_key)\n",
        "\n",
        "      db = FAISS.from_documents(docs, hf)\n",
        "      db.save_local(\"faiss_index_youtube\")\n",
        "\n",
        "    else:\n",
        "      docs = \"NA\"\n",
        "\n",
        "    return db\n",
        "\n",
        "\n",
        "\n",
        "#creating another function to get response from querying the above database\n",
        "def get_response(db, query,opt, chunk, overlap, k=5):\n",
        "\n",
        "    '''\n",
        "    gpt can handle up to 4097 tokens. Setting the chunksize to 1000 and k to 4 maximizes\n",
        "    the number of tokens to analyze.\n",
        "    '''\n",
        "\n",
        "    docs= db.similarity_search(query, k=k)\n",
        "\n",
        "\n",
        "    #joining them into one single string\n",
        "    docs_page_content = \" \".join([d.page_content for d in docs])\n",
        "\n",
        "    #chat= ChatOpenAI(temperature=0.4)\n",
        "    llm =  HuggingFacePipeline.from_model_id(model_id=model_name, task=\"text2text-generation\", model_kwargs={\"temperature\":1e-1, \"max_length\" : chunk_size})\n",
        "\n",
        "\n",
        "    #template for the system message prompt\n",
        "\n",
        "    template= '''\n",
        "              You are a helpful assistant who can answer question from Youtube videos based on the video's transcript: {docs}\n",
        "              Only use the factual information from transcript to answer the question.\n",
        "              If you feel like you don't have enough information to answer the question, say: \"Sorry, I cannot answer that\".\n",
        "              Your answer should be verbose and detailed.\n",
        "              '''\n",
        "\n",
        "    system_message_prompt= SystemMessagePromptTemplate.from_template(template)\n",
        "\n",
        "    #Human question prompt\n",
        "\n",
        "    human_template= 'Answer the following question: {question}'\n",
        "\n",
        "    human_message_prompt= HumanMessagePromptTemplate.from_template(human_template)\n",
        "\n",
        "    chat_prompt= ChatPromptTemplate.from_messages(\n",
        "        [system_message_prompt, human_message_prompt]\n",
        "\n",
        "    )\n",
        "\n",
        "\n",
        "    #chaining\n",
        "    ctype=['stuff', 'map_reduce', 'refine', 'map_rerank']\n",
        "\n",
        "    #chain= LLMChain(llm=llm, prompt=chat_prompt)\n",
        "    retriever = db.as_retriever(search_type='similarity', search_kwargs={\"k\": 5} )\n",
        "    llm = OpenAI(model='text-davinci-003',temperature=0, openai_api_key=openai.api_key) #, do_sample=True)\n",
        "\n",
        "    try:\n",
        "     qa = RetrievalQA.from_chain_type(llm=llm, chain_type=ctype[0], retriever=retriever, return_source_documents=True)\n",
        "     res = qa(query)\n",
        "    except Exception as e:\n",
        "      res=str(e)\n",
        "\n",
        "\n",
        "\n",
        "    return res, docs\n"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "**Initiating Chat with Youtube over prompt**"
      ],
      "metadata": {
        "id": "LgbXig1A2lBo"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import pandas as pd\n",
        "import pprint\n",
        "#User prompt URL\n",
        "video_url= 'https://www.youtube.com/watch?v=mHWYqJpRoHI'\n",
        "\n",
        "\n",
        "openai.api_key =\"sk-\"\n",
        "#Hardcoded Question\n",
        "query= [\"Whose work is being celebrated in this video\",\"WHo all are talking in this video\",\n",
        "    'Which topics are they talking in this youtube video']\n",
        "import warnings\n",
        "warnings.filterwarnings('ignore')\n",
        "#User prompt question\n",
        "#query= input('Please enter your question :: ')\n",
        "\n",
        "\n",
        "opts= [1]\n",
        "chunks=[1000]\n",
        "overlaps=[60]\n",
        "response_hyperparam=[]\n",
        "for opt in opts:\n",
        "  for chunk in chunks:\n",
        "    for overlap in overlaps:\n",
        "      for q in query:\n",
        "        print(\"Question ::\", q)\n",
        "        hf= OpenAIEmbeddings(model=\"text-embedding-ada-002\", openai_api_key=openai.api_key)\n",
        "        try:\n",
        "          db=FAISS.load_local(\"faiss_index_youtube\", hf)\n",
        "        except:\n",
        "          db= creating_db(video_url, chunk, overlap)\n",
        "\n",
        "        response, docs = get_response(db, q, opt, chunk, overlap, k=5)\n",
        "\n",
        "        # print(\"Answer from the GPT 3.5 model :: \", opt, chunk, overlap) # inventory management\n",
        "        #pprint.pprint(response['result'])\n",
        "        # print(response)\n",
        "        # print(response['result'])\n",
        "        try:\n",
        "          pprint.pprint(response['result'])\n",
        "        except:\n",
        "          response_hyperparam.append([opt, chunk, overlap, response])\n",
        "\n",
        "pd.DataFrame(response_hyperparam).to_csv(\"response_hyperparam_2.csv\") #, columns=['Opt', 'Chunk size', 'Overlap size', 'Answer'])"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "dSs7n9nxIsgE",
        "outputId": "a73e86c5-f243-4f9d-bdf8-9fd6bb26569e"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Question :: Whose work is being celebrated in this video\n",
            "(' The work of Sumit and his co-author Victor Martin is being celebrated in '\n",
            " 'this video.')\n",
            "Question :: WHo all are talking in this video\n",
            "' Bhagwan and Sumit are talking in this video.'\n",
            "Question :: Which topics are they talking in this youtube video\n",
            "(' They are talking about a paper titled \"A Model for Integrated Assortment '\n",
            " 'and Inventory Planning\" and how it solves a problem of understanding what '\n",
            " 'products to stock and how much of each product to stock in retail stores.')\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "HNmsfRVKjY75"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# **Gradio based User Interface for Search Q&A**"
      ],
      "metadata": {
        "id": "OUlHzvBl0yHB"
      }
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "dBaDxSjZ3wpa"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "import os\n",
        "import glob\n",
        "import argparse\n",
        "import gradio as gr\n",
        "\n",
        "def parse_arguments():\n",
        "    parser = argparse.ArgumentParser(description='privateGPT: Ask questions to your documents without an internet connection, '\n",
        "                                                 'using the power of LLMs.')\n",
        "    parser.add_argument(\"--hide-source\", \"-S\", action='store_true',\n",
        "                        help='Use this flag to disable printing of source documents used for answers.')\n",
        "\n",
        "    parser.add_argument(\"--mute-stream\", \"-M\",\n",
        "                        action='store_true',\n",
        "                        help='Use this flag to disable the streaming StdOut callback for LLMs.')\n",
        "\n",
        "    return parser.parse_args()\n",
        "\n",
        "\n",
        "#creating another function to get response from querying the above database\n",
        "def get_response(db, query,opt, chunk, overlap, k=5):\n",
        "    print(\"query ::\", query)\n",
        "\n",
        "    '''\n",
        "    gpt can handle up to 4097 tokens. Setting the chunksize to 1000 and k to 4 maximizes\n",
        "    the number of tokens to analyze.\n",
        "    '''\n",
        "\n",
        "    docs= db.similarity_search(query, k=k)\n",
        "\n",
        "\n",
        "    #joining them into one single string\n",
        "    docs_page_content = \" \".join([d.page_content for d in docs])\n",
        "\n",
        "    #chat= ChatOpenAI(temperature=0.4)\n",
        "    llm =  HuggingFacePipeline.from_model_id(model_id=model_name, task=\"text2text-generation\", model_kwargs={\"temperature\":1e-1, \"max_length\" : chunk_size})\n",
        "\n",
        "\n",
        "    #template for the system message prompt\n",
        "\n",
        "    template= '''\n",
        "              You are a helpful assistant who can answer question from Youtube videos based on the video's transcript: {docs}\n",
        "              Only use the factual information from transcript to answer the question.\n",
        "              If you feel like you don't have enough information to answer the question, say: \"Sorry, I cannot answer that\".\n",
        "              Your answer should be verbose and detailed.\n",
        "              '''\n",
        "\n",
        "    system_message_prompt= SystemMessagePromptTemplate.from_template(template)\n",
        "\n",
        "    #Human question prompt\n",
        "\n",
        "    human_template= 'Answer the following question: {question}'\n",
        "\n",
        "    human_message_prompt= HumanMessagePromptTemplate.from_template(human_template)\n",
        "\n",
        "    chat_prompt= ChatPromptTemplate.from_messages(\n",
        "        [system_message_prompt, human_message_prompt]\n",
        "\n",
        "    )\n",
        "\n",
        "\n",
        "    #chaining\n",
        "    ctype=['stuff', 'map_reduce', 'refine', 'map_rerank']\n",
        "\n",
        "    #chain= LLMChain(llm=llm, prompt=chat_prompt)\n",
        "    retriever = db.as_retriever(search_type='similarity', search_kwargs={\"k\": 4} )\n",
        "    llm = OpenAI(model='text-davinci-003',temperature=0, openai_api_key=openai.api_key) #, do_sample=True)\n",
        "\n",
        "    try:\n",
        "     qa = RetrievalQA.from_chain_type(llm=llm, chain_type=ctype[0], retriever=retriever, return_source_documents=True)\n",
        "     res = qa(query)\n",
        "    except Exception as e:\n",
        "      res=str(e)\n",
        "\n",
        "    return res, docs\n",
        "\n",
        "\n",
        "def creating_db(video_url, chunk, overlap):\n",
        "\n",
        "    loader= YoutubeLoader.from_youtube_url(video_url)\n",
        "    transcript= loader.load()\n",
        "\n",
        "    #to breakdown the enormous amount of tokens we will get from the transcript as we have a limited set we can input\n",
        "    text_splitter= RecursiveCharacterTextSplitter(chunk_size=chunk, chunk_overlap=overlap)\n",
        "\n",
        "    #this is just a list with the bunch of splits from the above\n",
        "    docs= text_splitter.split_documents(transcript)\n",
        "\n",
        "\n",
        "    #the final database\n",
        "    '''\n",
        "    when a user asks a question, this database will be used to perform the similarity search and\n",
        "    generate output based on that\n",
        "    '''\n",
        "    if(len(docs)>0):\n",
        "\n",
        "      openai.api_key =\"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\"\n",
        "      hf= OpenAIEmbeddings(model=\"text-embedding-ada-002\", openai_api_key=openai.api_key)\n",
        "\n",
        "      #db = FAISS.from_documents(docs, hf)\n",
        "\n",
        "      try:\n",
        "          db=FAISS.load_local(\"faiss_index_youtube\", hf)\n",
        "\n",
        "      except:\n",
        "          db= creating_db(video_url, chunk, overlap)\n",
        "          db.save_local(\"faiss_index_youtube\")\n",
        "\n",
        "    else:\n",
        "      docs = \"NA\"\n",
        "\n",
        "    return db\n",
        "\n",
        "\n",
        "\n",
        "with gr.Blocks() as mychatbot:  # Blocks is a low-level API that allows\n",
        "                                # you to create custom web applications\n",
        "    chatbot = gr.Chatbot(label=\"Chat with Anupam: Ask questions to your documents without an internet connection\")      # displays a chatbot\n",
        "    question = gr.Textbox(label=\"Question\")     # for user to ask a question\n",
        "    clear = gr.Button(\"Clear Conversation History\")  # Clear button\n",
        "    kill = gr.Button(\"Stop Current Search\")  # Clear button\n",
        "\n",
        "    # function to clear the conversation\n",
        "    def clear_messages():\n",
        "        global messages\n",
        "        messages = []    # reset the messages list\n",
        "\n",
        "\n",
        "    def kill_search():\n",
        "        print(\"killing\")\n",
        "        kill =1\n",
        "        exit()    # reset the messages list\n",
        "\n",
        "    def chat(message, chat_history, kill):\n",
        "        global messages\n",
        "        clear_messages()\n",
        "        print(\"chat function launched...\")\n",
        "        print(message)\n",
        "        messages.append({\"role\": \"user\", \"content\": message})\n",
        "        opt=3\n",
        "        chunk= 900\n",
        "        overlap = 50\n",
        "        video_url= message\n",
        "        video_url='https://www.youtube.com/watch?v=mHWYqJpRoHI'\n",
        "        db=creating_db(video_url, chunk, overlap)\n",
        "        print(\"query ::\", query[0])\n",
        "        response, doc = get_response(db, message,opt, chunk, overlap, k=10)\n",
        "\n",
        "\n",
        "        print(\"private gpt response recieved...\")\n",
        "        #print(response)\n",
        "        pprint.pprint(response['result'])\n",
        "        print(\"content....\")\n",
        "        content = response['result'] # + \"\\n\" + \"Sources:\"+ \"\\n >\" + \":\" +\"\\n\"  +\"Content: \"+\"\\n\"  +doc\n",
        "        #print(content)\n",
        "\n",
        "\n",
        "\n",
        "        chat_history.append((message, content))\n",
        "        return \"\", chat_history\n",
        "\n",
        "    # wire up the event handler for Submit button (when user press Enter)\n",
        "    question.submit(fn = chat,\n",
        "                    inputs = [question, chatbot],\n",
        "                    outputs = [question, chatbot])\n",
        "\n",
        "    # wire up the event handler for the Clear Conversation button\n",
        "\n",
        "    clear.click(fn = clear_messages,\n",
        "                inputs = None,\n",
        "                outputs = chatbot,\n",
        "                queue = False)\n",
        "    kill.click(fn = kill_search,\n",
        "                inputs = None,\n",
        "                outputs = chatbot,\n",
        "                queue = False)\n",
        "\n",
        "mychatbot.launch(share=True, debug=True)"
      ],
      "metadata": {
        "id": "ImM95JHgGinz",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 758
        },
        "outputId": "2770e010-265e-4a24-a376-0bee394de08a"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Colab notebook detected. This cell will run indefinitely so that you can see errors and logs. To turn off, set debug=False in launch().\n",
            "Running on public URL: https://737086805475e0babf.gradio.live\n",
            "\n",
            "This share link expires in 72 hours. For free permanent hosting and GPU upgrades, run `gradio deploy` from Terminal to deploy to Spaces (https://huggingface.co/spaces)\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ],
            "text/html": [
              "<div><iframe src=\"https://737086805475e0babf.gradio.live\" width=\"100%\" height=\"500\" allow=\"autoplay; camera; microphone; clipboard-read; clipboard-write;\" frameborder=\"0\" allowfullscreen></iframe></div>"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "chat function launched...\n",
            "Whose work is being talked in this video\n",
            "query :: Whose work is being celebrated in this video\n",
            "query :: Whose work is being talked in this video\n",
            "private gpt response recieved...\n",
            "\" Sumit's work is being talked about in this video.\"\n",
            "content....\n",
            "Keyboard interruption in main thread... closing server.\n",
            "Killing tunnel 127.0.0.1:7860 <> https://737086805475e0babf.gradio.live\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": []
          },
          "metadata": {},
          "execution_count": 13
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "szQzJF8j1Nfx"
      },
      "execution_count": null,
      "outputs": []
    }
  ]
}