Chat_youtube.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "source": [
        "## **Install required dependencies to run this notebook**"
      ],
      "metadata": {
        "id": "HhZn_biZ10dq"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "#Make following parameter 1 if you have restarted using this colab notebook\n",
        "#after a restart of kernel or re-logging into gmail account\n",
        "running_after_kernel_restart=1\n",
        "if(running_after_kernel_restart==1):\n",
        "  ! pip install gradio\n",
        "  ! pip install langchain\n",
        "  ! pip install dotenv\n",
        "  ! pip install\n",
        "  ! pip install sentence_transformers\n",
        "  ! pip install chromadb\n",
        "  !pip install unstructured\n",
        "  ! pip install youtube-transcript-api\n",
        "  ! pip install faiss\n",
        "  ! pip install faiss-cpu\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ATbgx5HUfq12",
        "outputId": "b661a15b-7428-4ed1-e024-4abad7a6fe04"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Collecting gradio\n",
            "  Downloading gradio-3.34.0-py3-none-any.whl (20.0 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m20.0/20.0 MB\u001b[0m \u001b[31m61.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting aiofiles (from gradio)\n",
            "  Downloading aiofiles-23.1.0-py3-none-any.whl (14 kB)\n",
            "Collecting aiohttp (from gradio)\n",
            "  Downloading aiohttp-3.8.4-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (1.0 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.0/1.0 MB\u001b[0m \u001b[31m63.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: altair>=4.2.0 in /usr/local/lib/python3.10/dist-packages (from gradio) (4.2.2)\n",
            "Collecting fastapi (from gradio)\n",
            "  Downloading fastapi-0.96.1-py3-none-any.whl (57 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m57.1/57.1 kB\u001b[0m \u001b[31m7.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting ffmpy (from gradio)\n",
            "  Downloading ffmpy-0.3.0.tar.gz (4.8 kB)\n",
            "  Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "Collecting gradio-client>=0.2.6 (from gradio)\n",
            "  Downloading gradio_client-0.2.6-py3-none-any.whl (288 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m288.3/288.3 kB\u001b[0m \u001b[31m29.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting httpx (from gradio)\n",
            "  Downloading httpx-0.24.1-py3-none-any.whl (75 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m75.4/75.4 kB\u001b[0m \u001b[31m9.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting huggingface-hub>=0.14.0 (from gradio)\n",
            "  Downloading huggingface_hub-0.15.1-py3-none-any.whl (236 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m236.8/236.8 kB\u001b[0m \u001b[31m24.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: jinja2 in /usr/local/lib/python3.10/dist-packages (from gradio) (3.1.2)\n",
            "Requirement already satisfied: markdown-it-py[linkify]>=2.0.0 in /usr/local/lib/python3.10/dist-packages (from gradio) (2.2.0)\n",
            "Requirement already satisfied: markupsafe in /usr/local/lib/python3.10/dist-packages (from gradio) (2.1.2)\n",
            "Requirement already satisfied: matplotlib in /usr/local/lib/python3.10/dist-packages (from gradio) (3.7.1)\n",
            "Collecting mdit-py-plugins<=0.3.3 (from gradio)\n",
            "  Downloading mdit_py_plugins-0.3.3-py3-none-any.whl (50 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m50.5/50.5 kB\u001b[0m \u001b[31m6.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: numpy in /usr/local/lib/python3.10/dist-packages (from gradio) (1.22.4)\n",
            "Collecting orjson (from gradio)\n",
            "  Downloading orjson-3.9.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (136 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m137.0/137.0 kB\u001b[0m \u001b[31m16.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: pandas in /usr/local/lib/python3.10/dist-packages (from gradio) (1.5.3)\n",
            "Requirement already satisfied: pillow in /usr/local/lib/python3.10/dist-packages (from gradio) (8.4.0)\n",
            "Requirement already satisfied: pydantic in /usr/local/lib/python3.10/dist-packages (from gradio) (1.10.7)\n",
            "Collecting pydub (from gradio)\n",
            "  Downloading pydub-0.25.1-py2.py3-none-any.whl (32 kB)\n",
            "Requirement already satisfied: pygments>=2.12.0 in /usr/local/lib/python3.10/dist-packages (from gradio) (2.14.0)\n",
            "Collecting python-multipart (from gradio)\n",
            "  Downloading python_multipart-0.0.6-py3-none-any.whl (45 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m45.7/45.7 kB\u001b[0m \u001b[31m5.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: pyyaml in /usr/local/lib/python3.10/dist-packages (from gradio) (6.0)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.10/dist-packages (from gradio) (2.27.1)\n",
            "Collecting semantic-version (from gradio)\n",
            "  Downloading semantic_version-2.10.0-py2.py3-none-any.whl (15 kB)\n",
            "Requirement already satisfied: typing-extensions in /usr/local/lib/python3.10/dist-packages (from gradio) (4.5.0)\n",
            "Collecting uvicorn>=0.14.0 (from gradio)\n",
            "  Downloading uvicorn-0.22.0-py3-none-any.whl (58 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m58.3/58.3 kB\u001b[0m \u001b[31m7.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting websockets>=10.0 (from gradio)\n",
            "  Downloading websockets-11.0.3-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl (129 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m129.9/129.9 kB\u001b[0m \u001b[31m11.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: entrypoints in /usr/local/lib/python3.10/dist-packages (from altair>=4.2.0->gradio) (0.4)\n",
            "Requirement already satisfied: jsonschema>=3.0 in /usr/local/lib/python3.10/dist-packages (from altair>=4.2.0->gradio) (4.3.3)\n",
            "Requirement already satisfied: toolz in /usr/local/lib/python3.10/dist-packages (from altair>=4.2.0->gradio) (0.12.0)\n",
            "Requirement already satisfied: fsspec in /usr/local/lib/python3.10/dist-packages (from gradio-client>=0.2.6->gradio) (2023.4.0)\n",
            "Requirement already satisfied: packaging in /usr/local/lib/python3.10/dist-packages (from gradio-client>=0.2.6->gradio) (23.1)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from huggingface-hub>=0.14.0->gradio) (3.12.0)\n",
            "Requirement already satisfied: tqdm>=4.42.1 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub>=0.14.0->gradio) (4.65.0)\n",
            "Requirement already satisfied: mdurl~=0.1 in /usr/local/lib/python3.10/dist-packages (from markdown-it-py[linkify]>=2.0.0->gradio) (0.1.2)\n",
            "Collecting linkify-it-py<3,>=1 (from markdown-it-py[linkify]>=2.0.0->gradio)\n",
            "  Downloading linkify_it_py-2.0.2-py3-none-any.whl (19 kB)\n",
            "Requirement already satisfied: python-dateutil>=2.8.1 in /usr/local/lib/python3.10/dist-packages (from pandas->gradio) (2.8.2)\n",
            "Requirement already satisfied: pytz>=2020.1 in /usr/local/lib/python3.10/dist-packages (from pandas->gradio) (2022.7.1)\n",
            "Requirement already satisfied: click>=7.0 in /usr/local/lib/python3.10/dist-packages (from uvicorn>=0.14.0->gradio) (8.1.3)\n",
            "Collecting h11>=0.8 (from uvicorn>=0.14.0->gradio)\n",
            "  Downloading h11-0.14.0-py3-none-any.whl (58 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m58.3/58.3 kB\u001b[0m \u001b[31m4.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: attrs>=17.3.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->gradio) (23.1.0)\n",
            "Requirement already satisfied: charset-normalizer<4.0,>=2.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->gradio) (2.0.12)\n",
            "Collecting multidict<7.0,>=4.5 (from aiohttp->gradio)\n",
            "  Downloading multidict-6.0.4-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (114 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m114.5/114.5 kB\u001b[0m \u001b[31m12.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting async-timeout<5.0,>=4.0.0a3 (from aiohttp->gradio)\n",
            "  Downloading async_timeout-4.0.2-py3-none-any.whl (5.8 kB)\n",
            "Collecting yarl<2.0,>=1.0 (from aiohttp->gradio)\n",
            "  Downloading yarl-1.9.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (268 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m268.8/268.8 kB\u001b[0m \u001b[31m29.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting frozenlist>=1.1.1 (from aiohttp->gradio)\n",
            "  Downloading frozenlist-1.3.3-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl (149 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m149.6/149.6 kB\u001b[0m \u001b[31m17.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting aiosignal>=1.1.2 (from aiohttp->gradio)\n",
            "  Downloading aiosignal-1.3.1-py3-none-any.whl (7.6 kB)\n",
            "Collecting starlette<0.28.0,>=0.27.0 (from fastapi->gradio)\n",
            "  Downloading starlette-0.27.0-py3-none-any.whl (66 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m67.0/67.0 kB\u001b[0m \u001b[31m8.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: certifi in /usr/local/lib/python3.10/dist-packages (from httpx->gradio) (2022.12.7)\n",
            "Collecting httpcore<0.18.0,>=0.15.0 (from httpx->gradio)\n",
            "  Downloading httpcore-0.17.2-py3-none-any.whl (72 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m72.5/72.5 kB\u001b[0m \u001b[31m9.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: idna in /usr/local/lib/python3.10/dist-packages (from httpx->gradio) (3.4)\n",
            "Requirement already satisfied: sniffio in /usr/local/lib/python3.10/dist-packages (from httpx->gradio) (1.3.0)\n",
            "Requirement already satisfied: contourpy>=1.0.1 in /usr/local/lib/python3.10/dist-packages (from matplotlib->gradio) (1.0.7)\n",
            "Requirement already satisfied: cycler>=0.10 in /usr/local/lib/python3.10/dist-packages (from matplotlib->gradio) (0.11.0)\n",
            "Requirement already satisfied: fonttools>=4.22.0 in /usr/local/lib/python3.10/dist-packages (from matplotlib->gradio) (4.39.3)\n",
            "Requirement already satisfied: kiwisolver>=1.0.1 in /usr/local/lib/python3.10/dist-packages (from matplotlib->gradio) (1.4.4)\n",
            "Requirement already satisfied: pyparsing>=2.3.1 in /usr/local/lib/python3.10/dist-packages (from matplotlib->gradio) (3.0.9)\n",
            "Requirement already satisfied: urllib3<1.27,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests->gradio) (1.26.15)\n",
            "Requirement already satisfied: anyio<5.0,>=3.0 in /usr/local/lib/python3.10/dist-packages (from httpcore<0.18.0,>=0.15.0->httpx->gradio) (3.6.2)\n",
            "Requirement already satisfied: pyrsistent!=0.17.0,!=0.17.1,!=0.17.2,>=0.14.0 in /usr/local/lib/python3.10/dist-packages (from jsonschema>=3.0->altair>=4.2.0->gradio) (0.19.3)\n",
            "Collecting uc-micro-py (from linkify-it-py<3,>=1->markdown-it-py[linkify]>=2.0.0->gradio)\n",
            "  Downloading uc_micro_py-1.0.2-py3-none-any.whl (6.2 kB)\n",
            "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.10/dist-packages (from python-dateutil>=2.8.1->pandas->gradio) (1.16.0)\n",
            "Building wheels for collected packages: ffmpy\n",
            "  Building wheel for ffmpy (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for ffmpy: filename=ffmpy-0.3.0-py3-none-any.whl size=4694 sha256=68ef192ec9e9304fd93a02663cf6816da52b5c6e06b9c23fcec9f6f1bbd62d15\n",
            "  Stored in directory: /root/.cache/pip/wheels/0c/c2/0e/3b9c6845c6a4e35beb90910cc70d9ac9ab5d47402bd62af0df\n",
            "Successfully built ffmpy\n",
            "Installing collected packages: pydub, ffmpy, websockets, uc-micro-py, semantic-version, python-multipart, orjson, multidict, h11, frozenlist, async-timeout, aiofiles, yarl, uvicorn, starlette, mdit-py-plugins, linkify-it-py, huggingface-hub, httpcore, aiosignal, httpx, fastapi, aiohttp, gradio-client, gradio\n",
            "Successfully installed aiofiles-23.1.0 aiohttp-3.8.4 aiosignal-1.3.1 async-timeout-4.0.2 fastapi-0.96.1 ffmpy-0.3.0 frozenlist-1.3.3 gradio-3.34.0 gradio-client-0.2.6 h11-0.14.0 httpcore-0.17.2 httpx-0.24.1 huggingface-hub-0.15.1 linkify-it-py-2.0.2 mdit-py-plugins-0.3.3 multidict-6.0.4 orjson-3.9.1 pydub-0.25.1 python-multipart-0.0.6 semantic-version-2.10.0 starlette-0.27.0 uc-micro-py-1.0.2 uvicorn-0.22.0 websockets-11.0.3 yarl-1.9.2\n",
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Collecting langchain\n",
            "  Downloading langchain-0.0.196-py3-none-any.whl (1.0 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.0/1.0 MB\u001b[0m \u001b[31m34.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: PyYAML>=5.4.1 in /usr/local/lib/python3.10/dist-packages (from langchain) (6.0)\n",
            "Requirement already satisfied: SQLAlchemy<3,>=1.4 in /usr/local/lib/python3.10/dist-packages (from langchain) (2.0.10)\n",
            "Requirement already satisfied: aiohttp<4.0.0,>=3.8.3 in /usr/local/lib/python3.10/dist-packages (from langchain) (3.8.4)\n",
            "Requirement already satisfied: async-timeout<5.0.0,>=4.0.0 in /usr/local/lib/python3.10/dist-packages (from langchain) (4.0.2)\n",
            "Collecting dataclasses-json<0.6.0,>=0.5.7 (from langchain)\n",
            "  Downloading dataclasses_json-0.5.7-py3-none-any.whl (25 kB)\n",
            "Collecting langchainplus-sdk>=0.0.7 (from langchain)\n",
            "  Downloading langchainplus_sdk-0.0.8-py3-none-any.whl (22 kB)\n",
            "Requirement already satisfied: numexpr<3.0.0,>=2.8.4 in /usr/local/lib/python3.10/dist-packages (from langchain) (2.8.4)\n",
            "Requirement already satisfied: numpy<2,>=1 in /usr/local/lib/python3.10/dist-packages (from langchain) (1.22.4)\n",
            "Collecting openapi-schema-pydantic<2.0,>=1.2 (from langchain)\n",
            "  Downloading openapi_schema_pydantic-1.2.4-py3-none-any.whl (90 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m90.0/90.0 kB\u001b[0m \u001b[31m462.4 kB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: pydantic<2,>=1 in /usr/local/lib/python3.10/dist-packages (from langchain) (1.10.7)\n",
            "Requirement already satisfied: requests<3,>=2 in /usr/local/lib/python3.10/dist-packages (from langchain) (2.27.1)\n",
            "Requirement already satisfied: tenacity<9.0.0,>=8.1.0 in /usr/local/lib/python3.10/dist-packages (from langchain) (8.2.2)\n",
            "Requirement already satisfied: attrs>=17.3.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp<4.0.0,>=3.8.3->langchain) (23.1.0)\n",
            "Requirement already satisfied: charset-normalizer<4.0,>=2.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp<4.0.0,>=3.8.3->langchain) (2.0.12)\n",
            "Requirement already satisfied: multidict<7.0,>=4.5 in /usr/local/lib/python3.10/dist-packages (from aiohttp<4.0.0,>=3.8.3->langchain) (6.0.4)\n",
            "Requirement already satisfied: yarl<2.0,>=1.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp<4.0.0,>=3.8.3->langchain) (1.9.2)\n",
            "Requirement already satisfied: frozenlist>=1.1.1 in /usr/local/lib/python3.10/dist-packages (from aiohttp<4.0.0,>=3.8.3->langchain) (1.3.3)\n",
            "Requirement already satisfied: aiosignal>=1.1.2 in /usr/local/lib/python3.10/dist-packages (from aiohttp<4.0.0,>=3.8.3->langchain) (1.3.1)\n",
            "Collecting marshmallow<4.0.0,>=3.3.0 (from dataclasses-json<0.6.0,>=0.5.7->langchain)\n",
            "  Downloading marshmallow-3.19.0-py3-none-any.whl (49 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m49.1/49.1 kB\u001b[0m \u001b[31m4.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting marshmallow-enum<2.0.0,>=1.5.1 (from dataclasses-json<0.6.0,>=0.5.7->langchain)\n",
            "  Downloading marshmallow_enum-1.5.1-py2.py3-none-any.whl (4.2 kB)\n",
            "Collecting typing-inspect>=0.4.0 (from dataclasses-json<0.6.0,>=0.5.7->langchain)\n",
            "  Downloading typing_inspect-0.9.0-py3-none-any.whl (8.8 kB)\n",
            "Requirement already satisfied: typing-extensions>=4.2.0 in /usr/local/lib/python3.10/dist-packages (from pydantic<2,>=1->langchain) (4.5.0)\n",
            "Requirement already satisfied: urllib3<1.27,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests<3,>=2->langchain) (1.26.15)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests<3,>=2->langchain) (2022.12.7)\n",
            "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests<3,>=2->langchain) (3.4)\n",
            "Requirement already satisfied: greenlet!=0.4.17 in /usr/local/lib/python3.10/dist-packages (from SQLAlchemy<3,>=1.4->langchain) (2.0.2)\n",
            "Requirement already satisfied: packaging>=17.0 in /usr/local/lib/python3.10/dist-packages (from marshmallow<4.0.0,>=3.3.0->dataclasses-json<0.6.0,>=0.5.7->langchain) (23.1)\n",
            "Collecting mypy-extensions>=0.3.0 (from typing-inspect>=0.4.0->dataclasses-json<0.6.0,>=0.5.7->langchain)\n",
            "  Downloading mypy_extensions-1.0.0-py3-none-any.whl (4.7 kB)\n",
            "Installing collected packages: mypy-extensions, marshmallow, typing-inspect, openapi-schema-pydantic, marshmallow-enum, langchainplus-sdk, dataclasses-json, langchain\n",
            "Successfully installed dataclasses-json-0.5.7 langchain-0.0.196 langchainplus-sdk-0.0.8 marshmallow-3.19.0 marshmallow-enum-1.5.1 mypy-extensions-1.0.0 openapi-schema-pydantic-1.2.4 typing-inspect-0.9.0\n",
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Collecting dotenv\n",
            "  Downloading dotenv-0.0.5.tar.gz (2.4 kB)\n",
            "  \u001b[1;31merror\u001b[0m: \u001b[1msubprocess-exited-with-error\u001b[0m\n",
            "  \n",
            "  \u001b[31m×\u001b[0m \u001b[32mpython setup.py egg_info\u001b[0m did not run successfully.\n",
            "  \u001b[31m│\u001b[0m exit code: \u001b[1;36m1\u001b[0m\n",
            "  \u001b[31m╰─>\u001b[0m See above for output.\n",
            "  \n",
            "  \u001b[1;35mnote\u001b[0m: This error originates from a subprocess, and is likely not a problem with pip.\n",
            "  Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25herror\n",
            "\u001b[1;31merror\u001b[0m: \u001b[1mmetadata-generation-failed\u001b[0m\n",
            "\n",
            "\u001b[31m×\u001b[0m Encountered error while generating package metadata.\n",
            "\u001b[31m╰─>\u001b[0m See above for output.\n",
            "\n",
            "\u001b[1;35mnote\u001b[0m: This is an issue with the package mentioned above, not pip.\n",
            "\u001b[1;36mhint\u001b[0m: See above for details.\n",
            "\u001b[31mERROR: You must give at least one requirement to install (see \"pip help install\")\u001b[0m\u001b[31m\n",
            "\u001b[0mLooking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Collecting sentence_transformers\n",
            "  Downloading sentence-transformers-2.2.2.tar.gz (85 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m86.0/86.0 kB\u001b[0m \u001b[31m8.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25h  Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "Collecting transformers<5.0.0,>=4.6.0 (from sentence_transformers)\n",
            "  Downloading transformers-4.30.1-py3-none-any.whl (7.2 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m7.2/7.2 MB\u001b[0m \u001b[31m63.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: tqdm in /usr/local/lib/python3.10/dist-packages (from sentence_transformers) (4.65.0)\n",
            "Requirement already satisfied: torch>=1.6.0 in /usr/local/lib/python3.10/dist-packages (from sentence_transformers) (2.0.1+cu118)\n",
            "Requirement already satisfied: torchvision in /usr/local/lib/python3.10/dist-packages (from sentence_transformers) (0.15.2+cu118)\n",
            "Requirement already satisfied: numpy in /usr/local/lib/python3.10/dist-packages (from sentence_transformers) (1.22.4)\n",
            "Requirement already satisfied: scikit-learn in /usr/local/lib/python3.10/dist-packages (from sentence_transformers) (1.2.2)\n",
            "Requirement already satisfied: scipy in /usr/local/lib/python3.10/dist-packages (from sentence_transformers) (1.10.1)\n",
            "Requirement already satisfied: nltk in /usr/local/lib/python3.10/dist-packages (from sentence_transformers) (3.8.1)\n",
            "Collecting sentencepiece (from sentence_transformers)\n",
            "  Downloading sentencepiece-0.1.99-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (1.3 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.3/1.3 MB\u001b[0m \u001b[31m49.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: huggingface-hub>=0.4.0 in /usr/local/lib/python3.10/dist-packages (from sentence_transformers) (0.15.1)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from huggingface-hub>=0.4.0->sentence_transformers) (3.12.0)\n",
            "Requirement already satisfied: fsspec in /usr/local/lib/python3.10/dist-packages (from huggingface-hub>=0.4.0->sentence_transformers) (2023.4.0)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.10/dist-packages (from huggingface-hub>=0.4.0->sentence_transformers) (2.27.1)\n",
            "Requirement already satisfied: pyyaml>=5.1 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub>=0.4.0->sentence_transformers) (6.0)\n",
            "Requirement already satisfied: typing-extensions>=3.7.4.3 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub>=0.4.0->sentence_transformers) (4.5.0)\n",
            "Requirement already satisfied: packaging>=20.9 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub>=0.4.0->sentence_transformers) (23.1)\n",
            "Requirement already satisfied: sympy in /usr/local/lib/python3.10/dist-packages (from torch>=1.6.0->sentence_transformers) (1.11.1)\n",
            "Requirement already satisfied: networkx in /usr/local/lib/python3.10/dist-packages (from torch>=1.6.0->sentence_transformers) (3.1)\n",
            "Requirement already satisfied: jinja2 in /usr/local/lib/python3.10/dist-packages (from torch>=1.6.0->sentence_transformers) (3.1.2)\n",
            "Requirement already satisfied: triton==2.0.0 in /usr/local/lib/python3.10/dist-packages (from torch>=1.6.0->sentence_transformers) (2.0.0)\n",
            "Requirement already satisfied: cmake in /usr/local/lib/python3.10/dist-packages (from triton==2.0.0->torch>=1.6.0->sentence_transformers) (3.25.2)\n",
            "Requirement already satisfied: lit in /usr/local/lib/python3.10/dist-packages (from triton==2.0.0->torch>=1.6.0->sentence_transformers) (16.0.5)\n",
            "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.10/dist-packages (from transformers<5.0.0,>=4.6.0->sentence_transformers) (2022.10.31)\n",
            "Collecting tokenizers!=0.11.3,<0.14,>=0.11.1 (from transformers<5.0.0,>=4.6.0->sentence_transformers)\n",
            "  Downloading tokenizers-0.13.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (7.8 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m7.8/7.8 MB\u001b[0m \u001b[31m60.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting safetensors>=0.3.1 (from transformers<5.0.0,>=4.6.0->sentence_transformers)\n",
            "  Downloading safetensors-0.3.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (1.3 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.3/1.3 MB\u001b[0m \u001b[31m67.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: click in /usr/local/lib/python3.10/dist-packages (from nltk->sentence_transformers) (8.1.3)\n",
            "Requirement already satisfied: joblib in /usr/local/lib/python3.10/dist-packages (from nltk->sentence_transformers) (1.2.0)\n",
            "Requirement already satisfied: threadpoolctl>=2.0.0 in /usr/local/lib/python3.10/dist-packages (from scikit-learn->sentence_transformers) (3.1.0)\n",
            "Requirement already satisfied: pillow!=8.3.*,>=5.3.0 in /usr/local/lib/python3.10/dist-packages (from torchvision->sentence_transformers) (8.4.0)\n",
            "Requirement already satisfied: MarkupSafe>=2.0 in /usr/local/lib/python3.10/dist-packages (from jinja2->torch>=1.6.0->sentence_transformers) (2.1.2)\n",
            "Requirement already satisfied: urllib3<1.27,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests->huggingface-hub>=0.4.0->sentence_transformers) (1.26.15)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests->huggingface-hub>=0.4.0->sentence_transformers) (2022.12.7)\n",
            "Requirement already satisfied: charset-normalizer~=2.0.0 in /usr/local/lib/python3.10/dist-packages (from requests->huggingface-hub>=0.4.0->sentence_transformers) (2.0.12)\n",
            "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests->huggingface-hub>=0.4.0->sentence_transformers) (3.4)\n",
            "Requirement already satisfied: mpmath>=0.19 in /usr/local/lib/python3.10/dist-packages (from sympy->torch>=1.6.0->sentence_transformers) (1.3.0)\n",
            "Building wheels for collected packages: sentence_transformers\n",
            "  Building wheel for sentence_transformers (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for sentence_transformers: filename=sentence_transformers-2.2.2-py3-none-any.whl size=125926 sha256=f373785f6b34ce7f147d2bbda9168a081b1ad2684803b1e5d3eb2acdb5ebcde7\n",
            "  Stored in directory: /root/.cache/pip/wheels/62/f2/10/1e606fd5f02395388f74e7462910fe851042f97238cbbd902f\n",
            "Successfully built sentence_transformers\n",
            "Installing collected packages: tokenizers, sentencepiece, safetensors, transformers, sentence_transformers\n",
            "Successfully installed safetensors-0.3.1 sentence_transformers-2.2.2 sentencepiece-0.1.99 tokenizers-0.13.3 transformers-4.30.1\n",
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Collecting chromadb\n",
            "  Downloading chromadb-0.3.26-py3-none-any.whl (123 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m123.6/123.6 kB\u001b[0m \u001b[31m9.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: pandas>=1.3 in /usr/local/lib/python3.10/dist-packages (from chromadb) (1.5.3)\n",
            "Collecting requests>=2.28 (from chromadb)\n",
            "  Downloading requests-2.31.0-py3-none-any.whl (62 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m62.6/62.6 kB\u001b[0m \u001b[31m7.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: pydantic>=1.9 in /usr/local/lib/python3.10/dist-packages (from chromadb) (1.10.7)\n",
            "Collecting hnswlib>=0.7 (from chromadb)\n",
            "  Downloading hnswlib-0.7.0.tar.gz (33 kB)\n",
            "  Installing build dependencies ... \u001b[?25l\u001b[?25hdone\n",
            "  Getting requirements to build wheel ... \u001b[?25l\u001b[?25hdone\n",
            "  Preparing metadata (pyproject.toml) ... \u001b[?25l\u001b[?25hdone\n",
            "Collecting clickhouse-connect>=0.5.7 (from chromadb)\n",
            "  Downloading clickhouse_connect-0.6.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (965 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m965.1/965.1 kB\u001b[0m \u001b[31m5.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: duckdb>=0.7.1 in /usr/local/lib/python3.10/dist-packages (from chromadb) (0.7.1)\n",
            "Requirement already satisfied: fastapi>=0.85.1 in /usr/local/lib/python3.10/dist-packages (from chromadb) (0.96.1)\n",
            "Requirement already satisfied: uvicorn[standard]>=0.18.3 in /usr/local/lib/python3.10/dist-packages (from chromadb) (0.22.0)\n",
            "Requirement already satisfied: numpy>=1.21.6 in /usr/local/lib/python3.10/dist-packages (from chromadb) (1.22.4)\n",
            "Collecting posthog>=2.4.0 (from chromadb)\n",
            "  Downloading posthog-3.0.1-py2.py3-none-any.whl (37 kB)\n",
            "Requirement already satisfied: typing-extensions>=4.5.0 in /usr/local/lib/python3.10/dist-packages (from chromadb) (4.5.0)\n",
            "Collecting pulsar-client>=3.1.0 (from chromadb)\n",
            "  Downloading pulsar_client-3.2.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (5.3 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m5.3/5.3 MB\u001b[0m \u001b[31m67.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting onnxruntime>=1.14.1 (from chromadb)\n",
            "  Downloading onnxruntime-1.15.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (5.9 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m5.9/5.9 MB\u001b[0m \u001b[31m71.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: tokenizers>=0.13.2 in /usr/local/lib/python3.10/dist-packages (from chromadb) (0.13.3)\n",
            "Requirement already satisfied: tqdm>=4.65.0 in /usr/local/lib/python3.10/dist-packages (from chromadb) (4.65.0)\n",
            "Collecting overrides>=7.3.1 (from chromadb)\n",
            "  Downloading overrides-7.3.1-py3-none-any.whl (17 kB)\n",
            "Requirement already satisfied: certifi in /usr/local/lib/python3.10/dist-packages (from clickhouse-connect>=0.5.7->chromadb) (2022.12.7)\n",
            "Requirement already satisfied: urllib3>=1.26 in /usr/local/lib/python3.10/dist-packages (from clickhouse-connect>=0.5.7->chromadb) (1.26.15)\n",
            "Requirement already satisfied: pytz in /usr/local/lib/python3.10/dist-packages (from clickhouse-connect>=0.5.7->chromadb) (2022.7.1)\n",
            "Collecting zstandard (from clickhouse-connect>=0.5.7->chromadb)\n",
            "  Downloading zstandard-0.21.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (2.7 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m2.7/2.7 MB\u001b[0m \u001b[31m60.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting lz4 (from clickhouse-connect>=0.5.7->chromadb)\n",
            "  Downloading lz4-4.3.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (1.3 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.3/1.3 MB\u001b[0m \u001b[31m52.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: starlette<0.28.0,>=0.27.0 in /usr/local/lib/python3.10/dist-packages (from fastapi>=0.85.1->chromadb) (0.27.0)\n",
            "Collecting coloredlogs (from onnxruntime>=1.14.1->chromadb)\n",
            "  Downloading coloredlogs-15.0.1-py2.py3-none-any.whl (46 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m46.0/46.0 kB\u001b[0m \u001b[31m4.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: flatbuffers in /usr/local/lib/python3.10/dist-packages (from onnxruntime>=1.14.1->chromadb) (23.3.3)\n",
            "Requirement already satisfied: packaging in /usr/local/lib/python3.10/dist-packages (from onnxruntime>=1.14.1->chromadb) (23.1)\n",
            "Requirement already satisfied: protobuf in /usr/local/lib/python3.10/dist-packages (from onnxruntime>=1.14.1->chromadb) (3.20.3)\n",
            "Requirement already satisfied: sympy in /usr/local/lib/python3.10/dist-packages (from onnxruntime>=1.14.1->chromadb) (1.11.1)\n",
            "Requirement already satisfied: python-dateutil>=2.8.1 in /usr/local/lib/python3.10/dist-packages (from pandas>=1.3->chromadb) (2.8.2)\n",
            "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.10/dist-packages (from posthog>=2.4.0->chromadb) (1.16.0)\n",
            "Collecting monotonic>=1.5 (from posthog>=2.4.0->chromadb)\n",
            "  Downloading monotonic-1.6-py2.py3-none-any.whl (8.2 kB)\n",
            "Collecting backoff>=1.10.0 (from posthog>=2.4.0->chromadb)\n",
            "  Downloading backoff-2.2.1-py3-none-any.whl (15 kB)\n",
            "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests>=2.28->chromadb) (2.0.12)\n",
            "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests>=2.28->chromadb) (3.4)\n",
            "Requirement already satisfied: click>=7.0 in /usr/local/lib/python3.10/dist-packages (from uvicorn[standard]>=0.18.3->chromadb) (8.1.3)\n",
            "Requirement already satisfied: h11>=0.8 in /usr/local/lib/python3.10/dist-packages (from uvicorn[standard]>=0.18.3->chromadb) (0.14.0)\n",
            "Collecting httptools>=0.5.0 (from uvicorn[standard]>=0.18.3->chromadb)\n",
            "  Downloading httptools-0.5.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl (414 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m414.1/414.1 kB\u001b[0m \u001b[31m31.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting python-dotenv>=0.13 (from uvicorn[standard]>=0.18.3->chromadb)\n",
            "  Downloading python_dotenv-1.0.0-py3-none-any.whl (19 kB)\n",
            "Requirement already satisfied: pyyaml>=5.1 in /usr/local/lib/python3.10/dist-packages (from uvicorn[standard]>=0.18.3->chromadb) (6.0)\n",
            "Collecting uvloop!=0.15.0,!=0.15.1,>=0.14.0 (from uvicorn[standard]>=0.18.3->chromadb)\n",
            "  Downloading uvloop-0.17.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (4.1 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m4.1/4.1 MB\u001b[0m \u001b[31m59.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting watchfiles>=0.13 (from uvicorn[standard]>=0.18.3->chromadb)\n",
            "  Downloading watchfiles-0.19.0-cp37-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (1.3 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.3/1.3 MB\u001b[0m \u001b[31m11.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: websockets>=10.4 in /usr/local/lib/python3.10/dist-packages (from uvicorn[standard]>=0.18.3->chromadb) (11.0.3)\n",
            "Requirement already satisfied: anyio<5,>=3.4.0 in /usr/local/lib/python3.10/dist-packages (from starlette<0.28.0,>=0.27.0->fastapi>=0.85.1->chromadb) (3.6.2)\n",
            "Collecting humanfriendly>=9.1 (from coloredlogs->onnxruntime>=1.14.1->chromadb)\n",
            "  Downloading humanfriendly-10.0-py2.py3-none-any.whl (86 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m86.8/86.8 kB\u001b[0m \u001b[31m8.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: mpmath>=0.19 in /usr/local/lib/python3.10/dist-packages (from sympy->onnxruntime>=1.14.1->chromadb) (1.3.0)\n",
            "Requirement already satisfied: sniffio>=1.1 in /usr/local/lib/python3.10/dist-packages (from anyio<5,>=3.4.0->starlette<0.28.0,>=0.27.0->fastapi>=0.85.1->chromadb) (1.3.0)\n",
            "Building wheels for collected packages: hnswlib\n",
            "  Building wheel for hnswlib (pyproject.toml) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for hnswlib: filename=hnswlib-0.7.0-cp310-cp310-linux_x86_64.whl size=2119881 sha256=1541c80cce5b9d8f7df6244621097244aa0c193f4799fc380b7cd70fb649e390\n",
            "  Stored in directory: /root/.cache/pip/wheels/8a/ae/ec/235a682e0041fbaeee389843670581ec6c66872db856dfa9a4\n",
            "Successfully built hnswlib\n",
            "Installing collected packages: monotonic, zstandard, uvloop, requests, python-dotenv, pulsar-client, overrides, lz4, humanfriendly, httptools, hnswlib, backoff, watchfiles, posthog, coloredlogs, clickhouse-connect, onnxruntime, chromadb\n",
            "  Attempting uninstall: requests\n",
            "    Found existing installation: requests 2.27.1\n",
            "    Uninstalling requests-2.27.1:\n",
            "      Successfully uninstalled requests-2.27.1\n",
            "\u001b[31mERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.\n",
            "google-colab 1.0.0 requires requests==2.27.1, but you have requests 2.31.0 which is incompatible.\u001b[0m\u001b[31m\n",
            "\u001b[0mSuccessfully installed backoff-2.2.1 chromadb-0.3.26 clickhouse-connect-0.6.2 coloredlogs-15.0.1 hnswlib-0.7.0 httptools-0.5.0 humanfriendly-10.0 lz4-4.3.2 monotonic-1.6 onnxruntime-1.15.0 overrides-7.3.1 posthog-3.0.1 pulsar-client-3.2.0 python-dotenv-1.0.0 requests-2.31.0 uvloop-0.17.0 watchfiles-0.19.0 zstandard-0.21.0\n",
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Collecting unstructured\n",
            "  Downloading unstructured-0.7.3-py3-none-any.whl (1.3 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.3/1.3 MB\u001b[0m \u001b[31m51.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting argilla (from unstructured)\n",
            "  Downloading argilla-1.9.0-py3-none-any.whl (2.5 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m2.5/2.5 MB\u001b[0m \u001b[31m82.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: chardet in /usr/local/lib/python3.10/dist-packages (from unstructured) (4.0.0)\n",
            "Requirement already satisfied: lxml in /usr/local/lib/python3.10/dist-packages (from unstructured) (4.9.2)\n",
            "Collecting msg-parser (from unstructured)\n",
            "  Downloading msg_parser-1.2.0-py2.py3-none-any.whl (101 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m101.8/101.8 kB\u001b[0m \u001b[31m13.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: nltk in /usr/local/lib/python3.10/dist-packages (from unstructured) (3.8.1)\n",
            "Requirement already satisfied: openpyxl in /usr/local/lib/python3.10/dist-packages (from unstructured) (3.0.10)\n",
            "Requirement already satisfied: pandas in /usr/local/lib/python3.10/dist-packages (from unstructured) (1.5.3)\n",
            "Collecting pdfminer.six (from unstructured)\n",
            "  Downloading pdfminer.six-20221105-py3-none-any.whl (5.6 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m5.6/5.6 MB\u001b[0m \u001b[31m55.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: pillow in /usr/local/lib/python3.10/dist-packages (from unstructured) (8.4.0)\n",
            "Collecting pypandoc (from unstructured)\n",
            "  Downloading pypandoc-1.11-py3-none-any.whl (20 kB)\n",
            "Collecting python-docx (from unstructured)\n",
            "  Downloading python-docx-0.8.11.tar.gz (5.6 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m5.6/5.6 MB\u001b[0m \u001b[31m72.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25h  Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "Collecting python-pptx (from unstructured)\n",
            "  Downloading python-pptx-0.6.21.tar.gz (10.1 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m10.1/10.1 MB\u001b[0m \u001b[31m96.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25h  Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "Collecting python-magic (from unstructured)\n",
            "  Downloading python_magic-0.4.27-py2.py3-none-any.whl (13 kB)\n",
            "Requirement already satisfied: markdown in /usr/local/lib/python3.10/dist-packages (from unstructured) (3.4.3)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.10/dist-packages (from unstructured) (2.31.0)\n",
            "Requirement already satisfied: tabulate in /usr/local/lib/python3.10/dist-packages (from unstructured) (0.8.10)\n",
            "Requirement already satisfied: xlrd in /usr/local/lib/python3.10/dist-packages (from unstructured) (2.0.1)\n",
            "Collecting httpx<0.24,>=0.15 (from argilla->unstructured)\n",
            "  Downloading httpx-0.23.3-py3-none-any.whl (71 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m71.5/71.5 kB\u001b[0m \u001b[31m9.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting deprecated~=1.2.0 (from argilla->unstructured)\n",
            "  Downloading Deprecated-1.2.14-py2.py3-none-any.whl (9.6 kB)\n",
            "Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.10/dist-packages (from argilla->unstructured) (23.1)\n",
            "Requirement already satisfied: pydantic>=1.10.7 in /usr/local/lib/python3.10/dist-packages (from argilla->unstructured) (1.10.7)\n",
            "Requirement already satisfied: wrapt<1.15,>=1.13 in /usr/local/lib/python3.10/dist-packages (from argilla->unstructured) (1.14.1)\n",
            "Requirement already satisfied: numpy<1.24.0 in /usr/local/lib/python3.10/dist-packages (from argilla->unstructured) (1.22.4)\n",
            "Requirement already satisfied: tqdm>=4.27.0 in /usr/local/lib/python3.10/dist-packages (from argilla->unstructured) (4.65.0)\n",
            "Requirement already satisfied: backoff in /usr/local/lib/python3.10/dist-packages (from argilla->unstructured) (2.2.1)\n",
            "Requirement already satisfied: monotonic in /usr/local/lib/python3.10/dist-packages (from argilla->unstructured) (1.6)\n",
            "Collecting rich<=13.0.1 (from argilla->unstructured)\n",
            "  Downloading rich-13.0.1-py3-none-any.whl (238 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m238.1/238.1 kB\u001b[0m \u001b[31m27.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: typer<1.0.0,>=0.6.0 in /usr/local/lib/python3.10/dist-packages (from argilla->unstructured) (0.7.0)\n",
            "Requirement already satisfied: python-dateutil>=2.8.1 in /usr/local/lib/python3.10/dist-packages (from pandas->unstructured) (2.8.2)\n",
            "Requirement already satisfied: pytz>=2020.1 in /usr/local/lib/python3.10/dist-packages (from pandas->unstructured) (2022.7.1)\n",
            "Collecting olefile>=0.46 (from msg-parser->unstructured)\n",
            "  Downloading olefile-0.46.zip (112 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m112.2/112.2 kB\u001b[0m \u001b[31m11.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25h  Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "Requirement already satisfied: click in /usr/local/lib/python3.10/dist-packages (from nltk->unstructured) (8.1.3)\n",
            "Requirement already satisfied: joblib in /usr/local/lib/python3.10/dist-packages (from nltk->unstructured) (1.2.0)\n",
            "Requirement already satisfied: regex>=2021.8.3 in /usr/local/lib/python3.10/dist-packages (from nltk->unstructured) (2022.10.31)\n",
            "Requirement already satisfied: et-xmlfile in /usr/local/lib/python3.10/dist-packages (from openpyxl->unstructured) (1.1.0)\n",
            "Requirement already satisfied: charset-normalizer>=2.0.0 in /usr/local/lib/python3.10/dist-packages (from pdfminer.six->unstructured) (2.0.12)\n",
            "Requirement already satisfied: cryptography>=36.0.0 in /usr/local/lib/python3.10/dist-packages (from pdfminer.six->unstructured) (40.0.2)\n",
            "Collecting XlsxWriter>=0.5.7 (from python-pptx->unstructured)\n",
            "  Downloading XlsxWriter-3.1.2-py3-none-any.whl (153 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m153.0/153.0 kB\u001b[0m \u001b[31m16.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests->unstructured) (3.4)\n",
            "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests->unstructured) (1.26.15)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests->unstructured) (2022.12.7)\n",
            "Requirement already satisfied: cffi>=1.12 in /usr/local/lib/python3.10/dist-packages (from cryptography>=36.0.0->pdfminer.six->unstructured) (1.15.1)\n",
            "Collecting httpcore<0.17.0,>=0.15.0 (from httpx<0.24,>=0.15->argilla->unstructured)\n",
            "  Downloading httpcore-0.16.3-py3-none-any.whl (69 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m69.6/69.6 kB\u001b[0m \u001b[31m8.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting rfc3986[idna2008]<2,>=1.3 (from httpx<0.24,>=0.15->argilla->unstructured)\n",
            "  Downloading rfc3986-1.5.0-py2.py3-none-any.whl (31 kB)\n",
            "Requirement already satisfied: sniffio in /usr/local/lib/python3.10/dist-packages (from httpx<0.24,>=0.15->argilla->unstructured) (1.3.0)\n",
            "Requirement already satisfied: typing-extensions>=4.2.0 in /usr/local/lib/python3.10/dist-packages (from pydantic>=1.10.7->argilla->unstructured) (4.5.0)\n",
            "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.10/dist-packages (from python-dateutil>=2.8.1->pandas->unstructured) (1.16.0)\n",
            "Collecting commonmark<0.10.0,>=0.9.0 (from rich<=13.0.1->argilla->unstructured)\n",
            "  Downloading commonmark-0.9.1-py2.py3-none-any.whl (51 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m51.1/51.1 kB\u001b[0m \u001b[31m6.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: pygments<3.0.0,>=2.6.0 in /usr/local/lib/python3.10/dist-packages (from rich<=13.0.1->argilla->unstructured) (2.14.0)\n",
            "Requirement already satisfied: pycparser in /usr/local/lib/python3.10/dist-packages (from cffi>=1.12->cryptography>=36.0.0->pdfminer.six->unstructured) (2.21)\n",
            "Requirement already satisfied: h11<0.15,>=0.13 in /usr/local/lib/python3.10/dist-packages (from httpcore<0.17.0,>=0.15.0->httpx<0.24,>=0.15->argilla->unstructured) (0.14.0)\n",
            "Requirement already satisfied: anyio<5.0,>=3.0 in /usr/local/lib/python3.10/dist-packages (from httpcore<0.17.0,>=0.15.0->httpx<0.24,>=0.15->argilla->unstructured) (3.6.2)\n",
            "Building wheels for collected packages: python-docx, python-pptx, olefile\n",
            "  Building wheel for python-docx (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for python-docx: filename=python_docx-0.8.11-py3-none-any.whl size=184491 sha256=8df565477a642ac7afc4b4f3812000763442fb383c677fc57e9ae9d92821f708\n",
            "  Stored in directory: /root/.cache/pip/wheels/80/27/06/837436d4c3bd989b957a91679966f207bfd71d358d63a8194d\n",
            "  Building wheel for python-pptx (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for python-pptx: filename=python_pptx-0.6.21-py3-none-any.whl size=470935 sha256=9fb17a319779404c7371ec166cd0a6967842c5fbe13992ee364fd5e468b53ef1\n",
            "  Stored in directory: /root/.cache/pip/wheels/ea/dd/74/01b3ec7256a0800b99384e9a0f7620e358afc3a51a59bf9b49\n",
            "  Building wheel for olefile (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for olefile: filename=olefile-0.46-py2.py3-none-any.whl size=35417 sha256=dfc7b62739983a341a1b4c1d311d51c14e2597c6c98d7f41d892c4b63d1d564e\n",
            "  Stored in directory: /root/.cache/pip/wheels/02/39/c0/9eb1f7a42b4b38f6f333b6314d4ed11c46f12a0f7b78194f0d\n",
            "Successfully built python-docx python-pptx olefile\n",
            "Installing collected packages: rfc3986, commonmark, XlsxWriter, rich, python-magic, python-docx, pypandoc, olefile, deprecated, python-pptx, msg-parser, httpcore, pdfminer.six, httpx, argilla, unstructured\n",
            "  Attempting uninstall: rich\n",
            "    Found existing installation: rich 13.3.4\n",
            "    Uninstalling rich-13.3.4:\n",
            "      Successfully uninstalled rich-13.3.4\n",
            "  Attempting uninstall: httpcore\n",
            "    Found existing installation: httpcore 0.17.2\n",
            "    Uninstalling httpcore-0.17.2:\n",
            "      Successfully uninstalled httpcore-0.17.2\n",
            "  Attempting uninstall: httpx\n",
            "    Found existing installation: httpx 0.24.1\n",
            "    Uninstalling httpx-0.24.1:\n",
            "      Successfully uninstalled httpx-0.24.1\n",
            "Successfully installed XlsxWriter-3.1.2 argilla-1.9.0 commonmark-0.9.1 deprecated-1.2.14 httpcore-0.16.3 httpx-0.23.3 msg-parser-1.2.0 olefile-0.46 pdfminer.six-20221105 pypandoc-1.11 python-docx-0.8.11 python-magic-0.4.27 python-pptx-0.6.21 rfc3986-1.5.0 rich-13.0.1 unstructured-0.7.3\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# **Model Development**"
      ],
      "metadata": {
        "id": "9NZUMKUl1FLN"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "CjeExmsNdc2r"
      },
      "outputs": [],
      "source": [
        "#import openai\n",
        "#from langchain.chat_models import ChatOpenAI\n",
        "import os\n",
        "from langchain import HuggingFacePipeline\n",
        "from langchain.chains import RetrievalQA\n",
        "from langchain.embeddings import HuggingFaceEmbeddings\n",
        "from langchain.document_loaders import YoutubeLoader\n",
        "from langchain.text_splitter import RecursiveCharacterTextSplitter\n",
        "from langchain.embeddings.openai import OpenAIEmbeddings\n",
        "from langchain.vectorstores import FAISS\n",
        "from langchain.chains import LLMChain\n",
        "from dotenv import find_dotenv, load_dotenv\n",
        "from langchain.prompts.chat import (\n",
        "    ChatPromptTemplate,\n",
        "    SystemMessagePromptTemplate,\n",
        "    HumanMessagePromptTemplate,\n",
        ")\n",
        "import textwrap\n",
        "chunk_size = 1000\n",
        "#importing the .env file containing the api key\n",
        "load_dotenv(find_dotenv())\n",
        "\n",
        "# Different Large Language Models\n",
        "model_name = \"sentence-transformers/all-mpnet-base-v2\"\n",
        "model_name1 = \"sentence-transformers/LaBSE\"\n",
        "model_name= 'intfloat/e5-large-v2'\n",
        "model_name = 'all-MiniLM-L6-v2'\n",
        "model_name=\"google/flan-t5-base\"\n",
        "model_name=\"google/flan-t5-xl\"\n",
        "model_name=\"google/flan-t5-large\"\n",
        "model_kwargs = {'device': 'cpu'}\n",
        "encode_kwargs = {'normalize_embeddings': False}\n",
        "\n",
        "embeddings = HuggingFaceEmbeddings(\n",
        "            model_name=model_name1,\n",
        "            model_kwargs=model_kwargs,\n",
        "            encode_kwargs=encode_kwargs\n",
        "        )\n",
        "embeddings = HuggingFaceEmbeddings(model_kwargs=model_kwargs)\n",
        "\n",
        "#establishing the key\n",
        "#api_key = os.environ['OPENAI_API_KEY']\n",
        "\n",
        "\n",
        "#creating a database\n",
        "def creating_db(video_url):\n",
        "\n",
        "    loader= YoutubeLoader.from_youtube_url(video_url)\n",
        "    transcript= loader.load()\n",
        "\n",
        "    #to breakdown the enormous amount of tokens we will get from the transcript as we have a limited set we can input\n",
        "    text_splitter= RecursiveCharacterTextSplitter(chunk_size=1000, chunk_overlap=100)\n",
        "\n",
        "    #this is just a list with the bunch of splits from the above\n",
        "    docs= text_splitter.split_documents(transcript)\n",
        "    #print(len(docs))\n",
        "\n",
        "    #the final database\n",
        "    '''\n",
        "    when a user asks a question, this database will be used to perform the similarity search and\n",
        "    generate output based on that\n",
        "    '''\n",
        "    if(len(docs)>0):\n",
        "      db= FAISS.from_documents(docs, embeddings) #embeddings are the vectors we convert the text over into\n",
        "    else:\n",
        "      docs = \"NA\"\n",
        "      db= FAISS.from_documents(docs, embeddings)\n",
        "    return db\n",
        "\n",
        "\n",
        "\n",
        "#creating another function to get response from querying the above database\n",
        "def get_response(db, query, k=10):\n",
        "\n",
        "    '''\n",
        "    gpt can handle up to 4097 tokens. Setting the chunksize to 1000 and k to 4 maximizes\n",
        "    the number of tokens to analyze.\n",
        "    '''\n",
        "\n",
        "    docs= db.similarity_search(query, k=k)\n",
        "\n",
        "    #joining them into one single string\n",
        "    docs_page_content = \" \".join([d.page_content for d in docs])\n",
        "\n",
        "    #chat= ChatOpenAI(temperature=0.4)\n",
        "    llm =  HuggingFacePipeline.from_model_id(model_id=model_name, task=\"text2text-generation\", model_kwargs={\"temperature\":3e-1, \"max_length\" : chunk_size})\n",
        "\n",
        "\n",
        "    #template for the system message prompt\n",
        "\n",
        "    template= '''\n",
        "              You are a helpful assistant who can answer question from Youtube videos based on the video's transcript: {docs}\n",
        "              Only use the factual information from transcript to answer the question.\n",
        "              If you feel like you don't have enough information to answer the question, say: \"Sorry, I cannot answer that\".\n",
        "              Your answer should be verbose and detailed.\n",
        "              '''\n",
        "\n",
        "    system_message_prompt= SystemMessagePromptTemplate.from_template(template)\n",
        "\n",
        "    #Human question prompt\n",
        "\n",
        "    human_template= 'Answer the following question: {question}'\n",
        "\n",
        "    human_message_prompt= HumanMessagePromptTemplate.from_template(human_template)\n",
        "\n",
        "    chat_prompt= ChatPromptTemplate.from_messages(\n",
        "        [system_message_prompt, human_message_prompt]\n",
        "\n",
        "    )\n",
        "\n",
        "\n",
        "    #chaining\n",
        "\n",
        "    #chain= LLMChain(llm=llm, prompt=chat_prompt)\n",
        "    retriever = db.as_retriever(search_type='similarity', search_kwargs={\"k\": 10} )\n",
        "    qa = RetrievalQA.from_chain_type(llm=llm, chain_type=\"stuff\", retriever=retriever, return_source_documents=True)\n",
        "\n",
        "    #response= chain.run(question=query, docs= docs_page_content)\n",
        "    #response = response.replace(\"\\n\", \"\")\n",
        "    res = qa(query)\n",
        "    return res, docs\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "\n"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "**Initiating Chat with Youtube over prompt**"
      ],
      "metadata": {
        "id": "LgbXig1A2lBo"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "#calling the functions:\n",
        "\n",
        "#User prompt URL\n",
        "video_url= input('Please enter the url: ')\n",
        "\n",
        "#Hardcoded URL\n",
        "video_url ='https://www.youtube.com/watch?v=mHWYqJpRoHI'\n",
        "\n",
        "db= creating_db(video_url)\n",
        "\n",
        "#Hardcoded Question\n",
        "query= 'Which topics are they talking in this youtube video' #'https://www.youtube.com/watch?v=V0dahVRNFDY'\n",
        "\n",
        "#User prompt question\n",
        "query= input('Please enter your question :: ')\n",
        "response, docs = get_response(db, query, k=10)\n",
        "print(\"Answer from the Flan-T5-Large model :: \")\n",
        "print(response['result'])"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "rsreKhu72i8X",
        "outputId": "b2f2dad5-f9dc-4848-c9ef-d8de7c97af8c"
      },
      "execution_count": null,
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Please enter the url: https://www.youtube.com/watch?v=-7HKIrGWPxo\n",
            "Please enter your question :: Which topics are they talking in this youtube video\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Token indices sequence length is longer than the specified maximum sequence length for this model (2118 > 512). Running this sequence through the model will result in indexing errors\n",
            "/usr/local/lib/python3.10/dist-packages/transformers/generation/utils.py:1259: UserWarning: You have modified the pretrained model configuration to control generation. This is a deprecated strategy to control generation and will be removed soon, in a future version. Please use a generation configuration file (see https://huggingface.co/docs/transformers/main_classes/text_generation)\n",
            "  warnings.warn(\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Answer from the Flan-T5-Large model :: \n",
            "inventory management\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "**Check the length of docs created post splitting**"
      ],
      "metadata": {
        "id": "fV347vwp2DXX"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "    # Check the length of docs created post splitting\n",
        "\n",
        "    from langchain.document_loaders import YoutubeLoader\n",
        "    video_url='https://www.youtube.com/watch?v=mHWYqJpRoHI'\n",
        "    video_url='https://www.youtube.com/watch?v=kLzU6wa0YYU'\n",
        "    loader= YoutubeLoader.from_youtube_url(video_url)\n",
        "    transcript= loader.load()\n",
        "\n",
        "    #to breakdown the enormous amount of tokens we will get from the transcript as we have a limited set we can input\n",
        "    text_splitter= RecursiveCharacterTextSplitter(chunk_size=1000, chunk_overlap=100)\n",
        "\n",
        "    #this is just a list with the bunch of splits from the above\n",
        "    docs= text_splitter.split_documents(transcript)\n",
        "    print(\"Number of documents created post splitting::\")\n",
        "    print(docs)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "HNmsfRVKjY75",
        "outputId": "2f2cebea-3ae3-4c3d-8ea1-04c5aca92eb4"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[Document(page_content=\"[Music] foreign [Music] management science and I would like everyone to know that management science is the most prestigious journal in operations management so congratulations oh thank you very much bhagwan what is the title of your paper uh it's titled a model for integrated assortment and inventory planning and it's with a co-author Victor Martin is the albinist at the EAC Business School so Sumit tell me a little bit about how you came to this problem why did you decide to work on this so it's very interesting we were looking at a data for different retail stores and what we see is that the assortments that are made available in these stores are not the same right we have large retail stores where we see where they get the full assortment a smaller retail stores they do not get the full assortment they just get an assortment of say the more popular product right and so then we were wondering you know why is this what is driving this yeah and the obvious answer was inventor the\", metadata={'source': 'mHWYqJpRoHI'}), Document(page_content=\"wondering you know why is this what is driving this yeah and the obvious answer was inventor the space consideration yeah obviously right because you have larger stores more space you can offer the full assortment smaller stores you can't right so then uh it's clearly not a one-size-fits-all solution in that one has to make these decisions in a tailored Manner and when we looked at the academic literature we found that there was very little on this topic you know how do you make these decisions together assortment and inventory okay so you looked at the real world you found that this is an important problem and then you went back to the literature that you have studied and you found that nobody had actually solved this problem before so there was a gap so this is a great example of research going in to solve a problem that's important which nobody has looked at before so that's that's a fantastic cue for many of our younger colleagues who are trying to do research in this field okay\", metadata={'source': 'mHWYqJpRoHI'}), Document(page_content=\"a fantastic cue for many of our younger colleagues who are trying to do research in this field okay so it Inventory management is very important for false right and the idea here is that if I have a a store which is in a city let's say it costs money to rent the space so you have to stock things the right way and the trade-off is that if it's not in stock and the customer shows up then you lost that's it right absolutely but at the same time to stock it so much that you're never stalked out is expensive because it requires space and so on and so forth so this problem has been studied for many years but there is something additional in your paper that you do what is that so uh one is thing yes as you say rightly the inventory management problem has been studied quite extensively in operations management most of the work tends to look at a single product so single product okay whereas here we are pushing the frontier a little bit and look at the case where you have an assortment of\", metadata={'source': 'mHWYqJpRoHI'}), Document(page_content=\"here we are pushing the frontier a little bit and look at the case where you have an assortment of products right so so which is actually pretty important because usually if I'm buying let's say yogurt I'm not just interested in one I could buy Denon or I could buy something else so you're saying you could offer me different types of products which are substitutes of each other absolutely so that's the innovation in your paper now you're studying not just once product but many such products right so how do we first of all understand what products to stock about products to stock so that is the assortment question and then on top of that how much of each of these products to stock right which is inventory question right so now it's a complicated problem because uh on top of having different types of different assortment of products which are sort of substitutes the margins you get on each one of these products as a seller might be different absolutely right so that makes it even more\", metadata={'source': 'mHWYqJpRoHI'}), Document(page_content=\"one of these products as a seller might be different absolutely right so that makes it even more complicated you want to have it stock the one that gives you highest margin but at the same time uh you ought to offer a wide variety of products and that's expensive so that is the problem we are trying to solve exactly so as you rightly put it them not all products are equally profitable to the retailer so as a result if you were to offer the entire assortment the full assortment it may be possible that customers go and purchase say the lower margin products right which doesn't uh which may hurt the Retailer's profitability right right and as you also point out stores have limited capacity limited space so you can't stock everything and so one has to really think carefully as to you know how to do this right also then this has been studied in the literature just the inventory problem they sort of look at it as a One-Shot thing that you know how much to stock or not stock but I think\", metadata={'source': 'mHWYqJpRoHI'}), Document(page_content=\"sort of look at it as a One-Shot thing that you know how much to stock or not stock but I think you're looking at it kind of a dynamic way how to replenish it right exactly when it runs out how frequently do you want to replenish it so there's an added Dynamic element which makes the problem even more complex to solve is that right yeah so it's challenging enough as it is because if you think about the inventory problem there is uncertainty in demand there is uncertainty in when you get the stock so that makes it a challenging problem now when you think about the assortment problem that is also difficult it is complicated because the number of possible assortments uh grows exponentially it just blows up with a number of products so let me explain that so if I go from let's say two or three or four assortments then there are many combinations one two and four two and four two and three and four and that's what do you mean that it really quickly jumps up exactly so simple strategy which\", metadata={'source': 'mHWYqJpRoHI'}), Document(page_content=\"four and that's what do you mean that it really quickly jumps up exactly so simple strategy which says I'm going to try out all possible things right just okay so that's where your paper comes in that's where the paper comes in and because now we are trying to make these decisions together together simultaneously so a common approach I would say is to break this into silos where you uh first make the assortment decisions figure out what to stock and then try to figure out how much of which to stock right now that can be sub-optimal because this is really something that you have to consider uh together simultaneously and that is where we are coming in okay so now what I want to do is I want to go and look at the approach you use to solve uh this model that you call it and I think that's very clever so you started by saying okay I'm going to solve the problem exactly what we call you know in close form for a very simple version of this one product that's right right and then you're able\", metadata={'source': 'mHWYqJpRoHI'}), Document(page_content=\"in close form for a very simple version of this one product that's right right and then you're able to solve it in close form what does that mean close form means that we are able to get expression for you know what is the right inventory level for this single product there's an equation there is an equation which and you can plug in and then you can solve this and that gives you a lot of confidence because you can do different kinds of experiments to see yeah exactly okay what is your line of attack then because that's the one we want to solve right that's a realistic situation yeah right so we end up making some approximations uh we uh to make the problem easier to analyze mathematically for example we assume that one can stock say fractional quantities now that really doesn't happen in practice but to make this problem tractable that is something that we make and given those assumptions we are able to go and solve an approximate version of this problem an approximate which may be\", metadata={'source': 'mHWYqJpRoHI'}), Document(page_content=\"we are able to go and solve an approximate version of this problem an approximate which may be good enough because it's better than crude force and it's better than just intuition right absolutely so you're saying let me make some approximations and then how do you know that the approximations that you're making are given giving you good results excellent question so uh there are two ways we approach this one is that we come up with some theoretical guarantees which say that the solution that we get cannot be too far from the uh the correct optimal situation put some bounds we can obtain some bounds and the second is we verify this numerically empirically so we generate test problems these are small enough that we can actually compute the optimal solution and then we see how far we are off from that optimal solution okay so that's a good trick you are saying I'll take a problem to which I know the solution because I can solve it and I'm going to use my method to see how close it got\", metadata={'source': 'mHWYqJpRoHI'}), Document(page_content=\"I know the solution because I can solve it and I'm going to use my method to see how close it got yes and our hope is that for a more complicated problem that it will also Deliver us a good results exactly that's what we're hoping yeah and what do you find how how good are your approximations so we were very very pleasantly surprised because ours is an approximation method but on the test problems that we are and they were very close to Optimal so very close what is very close so in 80 percent of instances we got the optimal solution we were able to recovery percent you were able to get the optimal solution you would have gotten by sort of by brutal force yeah exactly fantastic and for the remaining problems the optimality Gap was a fraction of a person so 0.1 percent for instance so you're saying Your solution is within 0.1 percent of optimal exactly that's pretty impressive now in terms of the solution method this is what you're calling the fractional integer problem can you say\", metadata={'source': 'mHWYqJpRoHI'}), Document(page_content=\"terms of the solution method this is what you're calling the fractional integer problem can you say something about that sure uh so what we mean by a fractional integer problem is that the objective function is a fraction it's a ratio of two terms two terms which is different from an easier optimization problem where the objective is a linear a linear function a straight line I see so because you're taking fraction it becomes a non-linear thing it becomes a non-linear a more complicated problem to solve okay so that's why this is a non-trivial modeling solution that is right okay fantastic so you're telling me that these methods are not just for publication and management science they can actually be used in many situations where this is an important thing that there's an assortment and inventory is a big issue in terms of costs so if you were to get a call from I don't know big basket or one of these companies and say can you solve this for us you can do it for them right I would be\", metadata={'source': 'mHWYqJpRoHI'}), Document(page_content=\"or one of these companies and say can you solve this for us you can do it for them right I would be optimistic yes you'd be optimistic so let's take a look at you know ballpark what we are talking about so let's say I have a company that's like a 100 crores in Revenue um I'm just going to make some numbers up approximately um inventory management the cost would be close to maybe four or five percent that sounds right and if you're able to squeeze half a percent in that optimization we are talking about close to half of one percent of Revenue exactly a significant amount which is huge and for big companies it's even bigger that's right okay exactly okay so Sumit you're going to get lots of calls from people be ready for it but at the same time I hope that researchers also got a glimpse of how you solve these problems there was a method to it right you sort of say let me solve a simple problem get an exact solution equation and then you're going to say okay more complicated I'll get it\", metadata={'source': 'mHWYqJpRoHI'}), Document(page_content=\"get an exact solution equation and then you're going to say okay more complicated I'll get it it's some approximation and then you go to the most realistic case and then you check how good it is okay so both rigor is there but also at the same time you're looking at a problem that is realistic and interesting yes so congratulations thank you for sharing your insights and thank you for telling us how you did it thank you bhagwan it was wonderful yeah thank you [Music] six\", metadata={'source': 'mHWYqJpRoHI'})]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# **Questions asked to the model**"
      ],
      "metadata": {
        "id": "OUlHzvBl0yHB"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "query= 'Which topic are they talking in video'\n",
        "response, docs = get_response(db, query, k=10)\n",
        "print(\"Content of the Youtube transcript::\")\n",
        "print(response['source_documents'][0].page_content)\n",
        "print(response['result'])"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "tNAOEVdnmeRx",
        "outputId": "25a21ce5-1fec-4516-c38e-7d201a118e66"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Token indices sequence length is longer than the specified maximum sequence length for this model (2115 > 512). Running this sequence through the model will result in indexing errors\n",
            "/usr/local/lib/python3.10/dist-packages/transformers/generation/utils.py:1259: UserWarning: You have modified the pretrained model configuration to control generation. This is a deprecated strategy to control generation and will be removed soon, in a future version. Please use a generation configuration file (see https://huggingface.co/docs/transformers/main_classes/text_generation)\n",
            "  warnings.warn(\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Content of the Youtube transcript::\n",
            "here we are pushing the frontier a little bit and look at the case where you have an assortment of products right so so which is actually pretty important because usually if I'm buying let's say yogurt I'm not just interested in one I could buy Denon or I could buy something else so you're saying you could offer me different types of products which are substitutes of each other absolutely so that's the innovation in your paper now you're studying not just once product but many such products right so how do we first of all understand what products to stock about products to stock so that is the assortment question and then on top of that how much of each of these products to stock right which is inventory question right so now it's a complicated problem because uh on top of having different types of different assortment of products which are sort of substitutes the margins you get on each one of these products as a seller might be different absolutely right so that makes it even more\n",
            "Inventory management\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "query= 'Who all are discussing in this youtube video'\n",
        "response, docs = get_response(db, query, k=10)\n",
        "print(response['result'])\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "_CdaAyajsjMQ",
        "outputId": "f13eda14-ab0e-4cb6-c0a0-94f9e6a6462e"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Token indices sequence length is longer than the specified maximum sequence length for this model (2117 > 512). Running this sequence through the model will result in indexing errors\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Sumit\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "query= 'Which Journal are they talking about in this youtube video' #'https://www.youtube.com/watch?v=V0dahVRNFDY' #input('Please enter your question: ')\n",
        "response, docs = get_response(db, query, k=10)\n",
        "print(response['result'])"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "yu4T_hJEtHxA",
        "outputId": "66530047-513f-4fb5-9822-7c1cd8900608"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Token indices sequence length is longer than the specified maximum sequence length for this model (2121 > 512). Running this sequence through the model will result in indexing errors\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Management Science\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "query= 'Summarize things being discussed in this youtube video' #'https://www.youtube.com/watch?v=V0dahVRNFDY' #input('Please enter your question: ')\n",
        "response, docs = get_response(db, query, k=10)\n",
        "print(response['result'])\n",
        "summary= str(response['result'])"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "JvPUHGiRwtsm",
        "outputId": "33ce570b-600b-497a-bf46-6ca73dfcf269"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Token indices sequence length is longer than the specified maximum sequence length for this model (2125 > 512). Running this sequence through the model will result in indexing errors\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Sumit's paper titled a model for integrated assortment and inventory planning explains how to solve the inventory and assortment problems.\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "query= 'What is the novelty of things being discussed in this youtube video' #'https://www.youtube.com/watch?v=V0dahVRNFDY' #input('Please enter your question: ')\n",
        "response, docs = get_response(db, query, k=10)\n",
        "print(response['result'])"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "n2eWOz9HxLCn",
        "outputId": "9ca20f71-c05f-4a12-9f6b-aa95b070ff11"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Token indices sequence length is longer than the specified maximum sequence length for this model (2110 > 512). Running this sequence through the model will result in indexing errors\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "you're studying not just once product but many such products\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "query= 'What problems are  being discussed in this youtube video' #'https://www.youtube.com/watch?v=V0dahVRNFDY' #input('Please enter your question: ')\n",
        "response, docs = get_response(db, query, k=10)\n",
        "problem= response['result']\n",
        "print(response['result'])"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "b1XVddJrzjTh",
        "outputId": "991e6dcf-9bdd-4a62-c0c9-82cb003f2aff"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Token indices sequence length is longer than the specified maximum sequence length for this model (2118 > 512). Running this sequence through the model will result in indexing errors\n",
            "/usr/local/lib/python3.10/dist-packages/transformers/generation/utils.py:1259: UserWarning: You have modified the pretrained model configuration to control generation. This is a deprecated strategy to control generation and will be removed soon, in a future version. Please use a generation configuration file (see https://huggingface.co/docs/transformers/main_classes/text_generation)\n",
            "  warnings.warn(\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "inventory management\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import docx\n",
        "\n",
        "from google.colab import drive\n",
        "drive.mount('/content/drive', force_remount=True)\n",
        "%cd 'MyDrive/source_documents'\n",
        "from docx.shared import Inches\n",
        "%pwd\n",
        "%ls\n",
        "# Create an instance of a word document\n",
        "doc = docx.Document()\n",
        "\n",
        "# Add a Title to the document\n",
        "doc.add_heading('Transcript with Summary', 0)\n",
        "\n",
        "# Adding paragraph with spacing\n",
        "doc.add_heading('Transcript:', 3)\n",
        "transcript= str(response['source_documents'][0].page_content)\n",
        "para = doc.add_paragraph(transcript)\n",
        "print(para)\n",
        "# Adding linspace of 0.5 inches in the paragraph\n",
        "para.paragraph_format.line_spacing = Inches(0.5)\n",
        "\n",
        "# Adding paragraph without spacing\n",
        "doc.add_heading('Summary:', 2.5)\n",
        "doc.add_paragraph(summary)\n",
        "\n",
        "doc.add_heading('Problem discussed:', 2)\n",
        "doc.add_paragraph(problem)\n",
        "%pwd\n",
        "# Now save the document to a location\n",
        "doc.save('transcript.docx')\n",
        "%ls"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "dBaDxSjZ3wpa",
        "outputId": "d8d2da03-677d-454e-c641-6029c3b98baf"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Mounted at /content/drive\n",
            "[Errno 107] Transport endpoint is not connected: 'MyDrive/source_documents'\n",
            "/content/drive/My Drive\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "UsageError: CWD no longer exists - please use %cd to change directory.\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "ImM95JHgGinz"
      },
      "execution_count": null,
      "outputs": []
    }
  ]
}