falcon_7b_instruct_GPTQ.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "gpuType": "T4",
      "authorship_tag": "ABX9TyPplsVYOLFtSedVPbljt2rp",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "GPU",
    "gpuClass": "standard"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/Troyanovsky/Local-LLM-comparison/blob/main/falcon_7b_instruct_GPTQ.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Use Alpaca instruction template for instruct mode"
      ],
      "metadata": {
        "id": "Jwq-CEw288fW"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "y2gvBgVU9x3G",
        "outputId": "3b3dbbcf-0125-435b-9941-305da06fc67c"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "/content\n",
            "fatal: destination path 'text-generation-webui' already exists and is not an empty directory.\n",
            "/content/text-generation-webui\n",
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Collecting git+https://github.com/huggingface/peft@3714aa2fff158fdfa637b2b65952580801d890b2 (from -r requirements.txt (line 17))\n",
            "  Cloning https://github.com/huggingface/peft (to revision 3714aa2fff158fdfa637b2b65952580801d890b2) to /tmp/pip-req-build-dul0fz8b\n",
            "  Running command git clone --filter=blob:none --quiet https://github.com/huggingface/peft /tmp/pip-req-build-dul0fz8b\n",
            "  Running command git rev-parse -q --verify 'sha^3714aa2fff158fdfa637b2b65952580801d890b2'\n",
            "  Running command git fetch -q https://github.com/huggingface/peft 3714aa2fff158fdfa637b2b65952580801d890b2\n",
            "  Running command git checkout -q 3714aa2fff158fdfa637b2b65952580801d890b2\n",
            "  Resolved https://github.com/huggingface/peft to commit 3714aa2fff158fdfa637b2b65952580801d890b2\n",
            "  Installing build dependencies ... \u001b[?25l\u001b[?25hdone\n",
            "  Getting requirements to build wheel ... \u001b[?25l\u001b[?25hdone\n",
            "  Preparing metadata (pyproject.toml) ... \u001b[?25l\u001b[?25hdone\n",
            "Collecting git+https://github.com/huggingface/transformers@e45e756d22206ca8fa9fb057c8c3d8fa79bf81c6 (from -r requirements.txt (line 18))\n",
            "  Cloning https://github.com/huggingface/transformers (to revision e45e756d22206ca8fa9fb057c8c3d8fa79bf81c6) to /tmp/pip-req-build-4nlnx2fe\n",
            "  Running command git clone --filter=blob:none --quiet https://github.com/huggingface/transformers /tmp/pip-req-build-4nlnx2fe\n",
            "  Running command git rev-parse -q --verify 'sha^e45e756d22206ca8fa9fb057c8c3d8fa79bf81c6'\n",
            "  Running command git fetch -q https://github.com/huggingface/transformers e45e756d22206ca8fa9fb057c8c3d8fa79bf81c6\n",
            "  Running command git checkout -q e45e756d22206ca8fa9fb057c8c3d8fa79bf81c6\n",
            "  Resolved https://github.com/huggingface/transformers to commit e45e756d22206ca8fa9fb057c8c3d8fa79bf81c6\n",
            "  Installing build dependencies ... \u001b[?25l\u001b[?25hdone\n",
            "  Getting requirements to build wheel ... \u001b[?25l\u001b[?25hdone\n",
            "  Preparing metadata (pyproject.toml) ... \u001b[?25l\u001b[?25hdone\n",
            "Collecting git+https://github.com/huggingface/accelerate@0226f750257b3bf2cadc4f189f9eef0c764a0467 (from -r requirements.txt (line 19))\n",
            "  Cloning https://github.com/huggingface/accelerate (to revision 0226f750257b3bf2cadc4f189f9eef0c764a0467) to /tmp/pip-req-build-6qu2ri13\n",
            "  Running command git clone --filter=blob:none --quiet https://github.com/huggingface/accelerate /tmp/pip-req-build-6qu2ri13\n",
            "  Running command git rev-parse -q --verify 'sha^0226f750257b3bf2cadc4f189f9eef0c764a0467'\n",
            "  Running command git fetch -q https://github.com/huggingface/accelerate 0226f750257b3bf2cadc4f189f9eef0c764a0467\n",
            "  Running command git checkout -q 0226f750257b3bf2cadc4f189f9eef0c764a0467\n",
            "  Resolved https://github.com/huggingface/accelerate to commit 0226f750257b3bf2cadc4f189f9eef0c764a0467\n",
            "  Installing build dependencies ... \u001b[?25l\u001b[?25hdone\n",
            "  Getting requirements to build wheel ... \u001b[?25l\u001b[?25hdone\n",
            "  Preparing metadata (pyproject.toml) ... \u001b[?25l\u001b[?25hdone\n",
            "Ignoring bitsandbytes: markers 'platform_system == \"Windows\"' don't match your environment\n",
            "Ignoring llama-cpp-python: markers 'platform_system == \"Windows\"' don't match your environment\n",
            "Requirement already satisfied: colorama in /usr/local/lib/python3.10/dist-packages (from -r requirements.txt (line 1)) (0.4.6)\n",
            "Requirement already satisfied: datasets in /usr/local/lib/python3.10/dist-packages (from -r requirements.txt (line 2)) (2.12.0)\n",
            "Requirement already satisfied: einops in /usr/local/lib/python3.10/dist-packages (from -r requirements.txt (line 3)) (0.6.1)\n",
            "Requirement already satisfied: flexgen==0.1.7 in /usr/local/lib/python3.10/dist-packages (from -r requirements.txt (line 4)) (0.1.7)\n",
            "Requirement already satisfied: gradio_client==0.2.5 in /usr/local/lib/python3.10/dist-packages (from -r requirements.txt (line 5)) (0.2.5)\n",
            "Collecting gradio==3.31.0 (from -r requirements.txt (line 6))\n",
            "  Using cached gradio-3.31.0-py3-none-any.whl (17.4 MB)\n",
            "Requirement already satisfied: markdown in /usr/local/lib/python3.10/dist-packages (from -r requirements.txt (line 7)) (3.4.3)\n",
            "Requirement already satisfied: numpy in /usr/local/lib/python3.10/dist-packages (from -r requirements.txt (line 8)) (1.22.4)\n",
            "Requirement already satisfied: pandas in /usr/local/lib/python3.10/dist-packages (from -r requirements.txt (line 9)) (1.5.3)\n",
            "Requirement already satisfied: Pillow>=9.5.0 in /usr/local/lib/python3.10/dist-packages (from -r requirements.txt (line 10)) (9.5.0)\n",
            "Requirement already satisfied: pyyaml in /usr/local/lib/python3.10/dist-packages (from -r requirements.txt (line 11)) (6.0)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.10/dist-packages (from -r requirements.txt (line 12)) (2.27.1)\n",
            "Requirement already satisfied: safetensors==0.3.1 in /usr/local/lib/python3.10/dist-packages (from -r requirements.txt (line 13)) (0.3.1)\n",
            "Requirement already satisfied: sentencepiece in /usr/local/lib/python3.10/dist-packages (from -r requirements.txt (line 14)) (0.1.99)\n",
            "Requirement already satisfied: tqdm in /usr/local/lib/python3.10/dist-packages (from -r requirements.txt (line 15)) (4.65.0)\n",
            "Requirement already satisfied: scipy in /usr/local/lib/python3.10/dist-packages (from -r requirements.txt (line 16)) (1.10.1)\n",
            "Requirement already satisfied: bitsandbytes==0.39.0 in /usr/local/lib/python3.10/dist-packages (from -r requirements.txt (line 20)) (0.39.0)\n",
            "Requirement already satisfied: llama-cpp-python==0.1.56 in /usr/local/lib/python3.10/dist-packages (from -r requirements.txt (line 22)) (0.1.56)\n",
            "Requirement already satisfied: torch>=1.12 in /usr/local/lib/python3.10/dist-packages (from flexgen==0.1.7->-r requirements.txt (line 4)) (2.0.1+cu118)\n",
            "Requirement already satisfied: pulp in /usr/local/lib/python3.10/dist-packages (from flexgen==0.1.7->-r requirements.txt (line 4)) (2.7.0)\n",
            "Requirement already satisfied: attrs in /usr/local/lib/python3.10/dist-packages (from flexgen==0.1.7->-r requirements.txt (line 4)) (23.1.0)\n",
            "Requirement already satisfied: fsspec in /usr/local/lib/python3.10/dist-packages (from gradio_client==0.2.5->-r requirements.txt (line 5)) (2023.4.0)\n",
            "Requirement already satisfied: httpx in /usr/local/lib/python3.10/dist-packages (from gradio_client==0.2.5->-r requirements.txt (line 5)) (0.24.1)\n",
            "Requirement already satisfied: huggingface-hub>=0.13.0 in /usr/local/lib/python3.10/dist-packages (from gradio_client==0.2.5->-r requirements.txt (line 5)) (0.14.1)\n",
            "Requirement already satisfied: packaging in /usr/local/lib/python3.10/dist-packages (from gradio_client==0.2.5->-r requirements.txt (line 5)) (23.1)\n",
            "Requirement already satisfied: typing-extensions in /usr/local/lib/python3.10/dist-packages (from gradio_client==0.2.5->-r requirements.txt (line 5)) (4.5.0)\n",
            "Requirement already satisfied: websockets in /usr/local/lib/python3.10/dist-packages (from gradio_client==0.2.5->-r requirements.txt (line 5)) (11.0.3)\n",
            "Requirement already satisfied: aiofiles in /usr/local/lib/python3.10/dist-packages (from gradio==3.31.0->-r requirements.txt (line 6)) (23.1.0)\n",
            "Requirement already satisfied: aiohttp in /usr/local/lib/python3.10/dist-packages (from gradio==3.31.0->-r requirements.txt (line 6)) (3.8.4)\n",
            "Requirement already satisfied: altair>=4.2.0 in /usr/local/lib/python3.10/dist-packages (from gradio==3.31.0->-r requirements.txt (line 6)) (4.2.2)\n",
            "Requirement already satisfied: fastapi in /usr/local/lib/python3.10/dist-packages (from gradio==3.31.0->-r requirements.txt (line 6)) (0.95.2)\n",
            "Requirement already satisfied: ffmpy in /usr/local/lib/python3.10/dist-packages (from gradio==3.31.0->-r requirements.txt (line 6)) (0.3.0)\n",
            "Requirement already satisfied: jinja2 in /usr/local/lib/python3.10/dist-packages (from gradio==3.31.0->-r requirements.txt (line 6)) (3.1.2)\n",
            "Requirement already satisfied: markdown-it-py[linkify]>=2.0.0 in /usr/local/lib/python3.10/dist-packages (from gradio==3.31.0->-r requirements.txt (line 6)) (2.2.0)\n",
            "Requirement already satisfied: markupsafe in /usr/local/lib/python3.10/dist-packages (from gradio==3.31.0->-r requirements.txt (line 6)) (2.1.2)\n",
            "Requirement already satisfied: matplotlib in /usr/local/lib/python3.10/dist-packages (from gradio==3.31.0->-r requirements.txt (line 6)) (3.7.1)\n",
            "Requirement already satisfied: mdit-py-plugins<=0.3.3 in /usr/local/lib/python3.10/dist-packages (from gradio==3.31.0->-r requirements.txt (line 6)) (0.3.3)\n",
            "Requirement already satisfied: orjson in /usr/local/lib/python3.10/dist-packages (from gradio==3.31.0->-r requirements.txt (line 6)) (3.8.14)\n",
            "Requirement already satisfied: pydantic in /usr/local/lib/python3.10/dist-packages (from gradio==3.31.0->-r requirements.txt (line 6)) (1.10.7)\n",
            "Requirement already satisfied: pydub in /usr/local/lib/python3.10/dist-packages (from gradio==3.31.0->-r requirements.txt (line 6)) (0.25.1)\n",
            "Requirement already satisfied: pygments>=2.12.0 in /usr/local/lib/python3.10/dist-packages (from gradio==3.31.0->-r requirements.txt (line 6)) (2.14.0)\n",
            "Requirement already satisfied: python-multipart in /usr/local/lib/python3.10/dist-packages (from gradio==3.31.0->-r requirements.txt (line 6)) (0.0.6)\n",
            "Requirement already satisfied: semantic-version in /usr/local/lib/python3.10/dist-packages (from gradio==3.31.0->-r requirements.txt (line 6)) (2.10.0)\n",
            "Requirement already satisfied: uvicorn>=0.14.0 in /usr/local/lib/python3.10/dist-packages (from gradio==3.31.0->-r requirements.txt (line 6)) (0.22.0)\n",
            "Requirement already satisfied: pyarrow>=8.0.0 in /usr/local/lib/python3.10/dist-packages (from datasets->-r requirements.txt (line 2)) (9.0.0)\n",
            "Requirement already satisfied: dill<0.3.7,>=0.3.0 in /usr/local/lib/python3.10/dist-packages (from datasets->-r requirements.txt (line 2)) (0.3.6)\n",
            "Requirement already satisfied: xxhash in /usr/local/lib/python3.10/dist-packages (from datasets->-r requirements.txt (line 2)) (3.2.0)\n",
            "Requirement already satisfied: multiprocess in /usr/local/lib/python3.10/dist-packages (from datasets->-r requirements.txt (line 2)) (0.70.14)\n",
            "Requirement already satisfied: responses<0.19 in /usr/local/lib/python3.10/dist-packages (from datasets->-r requirements.txt (line 2)) (0.18.0)\n",
            "Requirement already satisfied: python-dateutil>=2.8.1 in /usr/local/lib/python3.10/dist-packages (from pandas->-r requirements.txt (line 9)) (2.8.2)\n",
            "Requirement already satisfied: pytz>=2020.1 in /usr/local/lib/python3.10/dist-packages (from pandas->-r requirements.txt (line 9)) (2022.7.1)\n",
            "Requirement already satisfied: urllib3<1.27,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests->-r requirements.txt (line 12)) (1.26.15)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests->-r requirements.txt (line 12)) (2022.12.7)\n",
            "Requirement already satisfied: charset-normalizer~=2.0.0 in /usr/local/lib/python3.10/dist-packages (from requests->-r requirements.txt (line 12)) (2.0.12)\n",
            "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests->-r requirements.txt (line 12)) (3.4)\n",
            "Requirement already satisfied: psutil in /usr/local/lib/python3.10/dist-packages (from peft==0.4.0.dev0->-r requirements.txt (line 17)) (5.9.5)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from transformers==4.30.0.dev0->-r requirements.txt (line 18)) (3.12.0)\n",
            "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.10/dist-packages (from transformers==4.30.0.dev0->-r requirements.txt (line 18)) (2022.10.31)\n",
            "Requirement already satisfied: tokenizers!=0.11.3,<0.14,>=0.11.1 in /usr/local/lib/python3.10/dist-packages (from transformers==4.30.0.dev0->-r requirements.txt (line 18)) (0.13.3)\n",
            "Requirement already satisfied: entrypoints in /usr/local/lib/python3.10/dist-packages (from altair>=4.2.0->gradio==3.31.0->-r requirements.txt (line 6)) (0.4)\n",
            "Requirement already satisfied: jsonschema>=3.0 in /usr/local/lib/python3.10/dist-packages (from altair>=4.2.0->gradio==3.31.0->-r requirements.txt (line 6)) (4.3.3)\n",
            "Requirement already satisfied: toolz in /usr/local/lib/python3.10/dist-packages (from altair>=4.2.0->gradio==3.31.0->-r requirements.txt (line 6)) (0.12.0)\n",
            "Requirement already satisfied: multidict<7.0,>=4.5 in /usr/local/lib/python3.10/dist-packages (from aiohttp->gradio==3.31.0->-r requirements.txt (line 6)) (6.0.4)\n",
            "Requirement already satisfied: async-timeout<5.0,>=4.0.0a3 in /usr/local/lib/python3.10/dist-packages (from aiohttp->gradio==3.31.0->-r requirements.txt (line 6)) (4.0.2)\n",
            "Requirement already satisfied: yarl<2.0,>=1.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->gradio==3.31.0->-r requirements.txt (line 6)) (1.9.2)\n",
            "Requirement already satisfied: frozenlist>=1.1.1 in /usr/local/lib/python3.10/dist-packages (from aiohttp->gradio==3.31.0->-r requirements.txt (line 6)) (1.3.3)\n",
            "Requirement already satisfied: aiosignal>=1.1.2 in /usr/local/lib/python3.10/dist-packages (from aiohttp->gradio==3.31.0->-r requirements.txt (line 6)) (1.3.1)\n",
            "Requirement already satisfied: mdurl~=0.1 in /usr/local/lib/python3.10/dist-packages (from markdown-it-py[linkify]>=2.0.0->gradio==3.31.0->-r requirements.txt (line 6)) (0.1.2)\n",
            "Requirement already satisfied: linkify-it-py<3,>=1 in /usr/local/lib/python3.10/dist-packages (from markdown-it-py[linkify]>=2.0.0->gradio==3.31.0->-r requirements.txt (line 6)) (2.0.2)\n",
            "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.10/dist-packages (from python-dateutil>=2.8.1->pandas->-r requirements.txt (line 9)) (1.16.0)\n",
            "Requirement already satisfied: sympy in /usr/local/lib/python3.10/dist-packages (from torch>=1.12->flexgen==0.1.7->-r requirements.txt (line 4)) (1.11.1)\n",
            "Requirement already satisfied: networkx in /usr/local/lib/python3.10/dist-packages (from torch>=1.12->flexgen==0.1.7->-r requirements.txt (line 4)) (3.1)\n",
            "Requirement already satisfied: triton==2.0.0 in /usr/local/lib/python3.10/dist-packages (from torch>=1.12->flexgen==0.1.7->-r requirements.txt (line 4)) (2.0.0)\n",
            "Requirement already satisfied: cmake in /usr/local/lib/python3.10/dist-packages (from triton==2.0.0->torch>=1.12->flexgen==0.1.7->-r requirements.txt (line 4)) (3.25.2)\n",
            "Requirement already satisfied: lit in /usr/local/lib/python3.10/dist-packages (from triton==2.0.0->torch>=1.12->flexgen==0.1.7->-r requirements.txt (line 4)) (16.0.5)\n",
            "Requirement already satisfied: click>=7.0 in /usr/local/lib/python3.10/dist-packages (from uvicorn>=0.14.0->gradio==3.31.0->-r requirements.txt (line 6)) (8.1.3)\n",
            "Requirement already satisfied: h11>=0.8 in /usr/local/lib/python3.10/dist-packages (from uvicorn>=0.14.0->gradio==3.31.0->-r requirements.txt (line 6)) (0.14.0)\n",
            "Requirement already satisfied: starlette<0.28.0,>=0.27.0 in /usr/local/lib/python3.10/dist-packages (from fastapi->gradio==3.31.0->-r requirements.txt (line 6)) (0.27.0)\n",
            "Requirement already satisfied: httpcore<0.18.0,>=0.15.0 in /usr/local/lib/python3.10/dist-packages (from httpx->gradio_client==0.2.5->-r requirements.txt (line 5)) (0.17.2)\n",
            "Requirement already satisfied: sniffio in /usr/local/lib/python3.10/dist-packages (from httpx->gradio_client==0.2.5->-r requirements.txt (line 5)) (1.3.0)\n",
            "Requirement already satisfied: contourpy>=1.0.1 in /usr/local/lib/python3.10/dist-packages (from matplotlib->gradio==3.31.0->-r requirements.txt (line 6)) (1.0.7)\n",
            "Requirement already satisfied: cycler>=0.10 in /usr/local/lib/python3.10/dist-packages (from matplotlib->gradio==3.31.0->-r requirements.txt (line 6)) (0.11.0)\n",
            "Requirement already satisfied: fonttools>=4.22.0 in /usr/local/lib/python3.10/dist-packages (from matplotlib->gradio==3.31.0->-r requirements.txt (line 6)) (4.39.3)\n",
            "Requirement already satisfied: kiwisolver>=1.0.1 in /usr/local/lib/python3.10/dist-packages (from matplotlib->gradio==3.31.0->-r requirements.txt (line 6)) (1.4.4)\n",
            "Requirement already satisfied: pyparsing>=2.3.1 in /usr/local/lib/python3.10/dist-packages (from matplotlib->gradio==3.31.0->-r requirements.txt (line 6)) (3.0.9)\n",
            "Requirement already satisfied: anyio<5.0,>=3.0 in /usr/local/lib/python3.10/dist-packages (from httpcore<0.18.0,>=0.15.0->httpx->gradio_client==0.2.5->-r requirements.txt (line 5)) (3.6.2)\n",
            "Requirement already satisfied: pyrsistent!=0.17.0,!=0.17.1,!=0.17.2,>=0.14.0 in /usr/local/lib/python3.10/dist-packages (from jsonschema>=3.0->altair>=4.2.0->gradio==3.31.0->-r requirements.txt (line 6)) (0.19.3)\n",
            "Requirement already satisfied: uc-micro-py in /usr/local/lib/python3.10/dist-packages (from linkify-it-py<3,>=1->markdown-it-py[linkify]>=2.0.0->gradio==3.31.0->-r requirements.txt (line 6)) (1.0.2)\n",
            "Requirement already satisfied: mpmath>=0.19 in /usr/local/lib/python3.10/dist-packages (from sympy->torch>=1.12->flexgen==0.1.7->-r requirements.txt (line 4)) (1.3.0)\n",
            "Installing collected packages: gradio\n",
            "  Attempting uninstall: gradio\n",
            "    Found existing installation: gradio 3.32.0\n",
            "    Uninstalling gradio-3.32.0:\n",
            "      Successfully uninstalled gradio-3.32.0\n",
            "Successfully installed gradio-3.31.0\n",
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Collecting gradio==3.32.0\n",
            "  Using cached gradio-3.32.0-py3-none-any.whl (19.9 MB)\n",
            "Requirement already satisfied: aiofiles in /usr/local/lib/python3.10/dist-packages (from gradio==3.32.0) (23.1.0)\n",
            "Requirement already satisfied: aiohttp in /usr/local/lib/python3.10/dist-packages (from gradio==3.32.0) (3.8.4)\n",
            "Requirement already satisfied: altair>=4.2.0 in /usr/local/lib/python3.10/dist-packages (from gradio==3.32.0) (4.2.2)\n",
            "Requirement already satisfied: fastapi in /usr/local/lib/python3.10/dist-packages (from gradio==3.32.0) (0.95.2)\n",
            "Requirement already satisfied: ffmpy in /usr/local/lib/python3.10/dist-packages (from gradio==3.32.0) (0.3.0)\n",
            "Requirement already satisfied: gradio-client>=0.2.4 in /usr/local/lib/python3.10/dist-packages (from gradio==3.32.0) (0.2.5)\n",
            "Requirement already satisfied: httpx in /usr/local/lib/python3.10/dist-packages (from gradio==3.32.0) (0.24.1)\n",
            "Requirement already satisfied: huggingface-hub>=0.13.0 in /usr/local/lib/python3.10/dist-packages (from gradio==3.32.0) (0.14.1)\n",
            "Requirement already satisfied: jinja2 in /usr/local/lib/python3.10/dist-packages (from gradio==3.32.0) (3.1.2)\n",
            "Requirement already satisfied: markdown-it-py[linkify]>=2.0.0 in /usr/local/lib/python3.10/dist-packages (from gradio==3.32.0) (2.2.0)\n",
            "Requirement already satisfied: markupsafe in /usr/local/lib/python3.10/dist-packages (from gradio==3.32.0) (2.1.2)\n",
            "Requirement already satisfied: matplotlib in /usr/local/lib/python3.10/dist-packages (from gradio==3.32.0) (3.7.1)\n",
            "Requirement already satisfied: mdit-py-plugins<=0.3.3 in /usr/local/lib/python3.10/dist-packages (from gradio==3.32.0) (0.3.3)\n",
            "Requirement already satisfied: numpy in /usr/local/lib/python3.10/dist-packages (from gradio==3.32.0) (1.22.4)\n",
            "Requirement already satisfied: orjson in /usr/local/lib/python3.10/dist-packages (from gradio==3.32.0) (3.8.14)\n",
            "Requirement already satisfied: pandas in /usr/local/lib/python3.10/dist-packages (from gradio==3.32.0) (1.5.3)\n",
            "Requirement already satisfied: pillow in /usr/local/lib/python3.10/dist-packages (from gradio==3.32.0) (9.5.0)\n",
            "Requirement already satisfied: pydantic in /usr/local/lib/python3.10/dist-packages (from gradio==3.32.0) (1.10.7)\n",
            "Requirement already satisfied: pydub in /usr/local/lib/python3.10/dist-packages (from gradio==3.32.0) (0.25.1)\n",
            "Requirement already satisfied: pygments>=2.12.0 in /usr/local/lib/python3.10/dist-packages (from gradio==3.32.0) (2.14.0)\n",
            "Requirement already satisfied: python-multipart in /usr/local/lib/python3.10/dist-packages (from gradio==3.32.0) (0.0.6)\n",
            "Requirement already satisfied: pyyaml in /usr/local/lib/python3.10/dist-packages (from gradio==3.32.0) (6.0)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.10/dist-packages (from gradio==3.32.0) (2.27.1)\n",
            "Requirement already satisfied: semantic-version in /usr/local/lib/python3.10/dist-packages (from gradio==3.32.0) (2.10.0)\n",
            "Requirement already satisfied: typing-extensions in /usr/local/lib/python3.10/dist-packages (from gradio==3.32.0) (4.5.0)\n",
            "Requirement already satisfied: uvicorn>=0.14.0 in /usr/local/lib/python3.10/dist-packages (from gradio==3.32.0) (0.22.0)\n",
            "Requirement already satisfied: websockets>=10.0 in /usr/local/lib/python3.10/dist-packages (from gradio==3.32.0) (11.0.3)\n",
            "Requirement already satisfied: entrypoints in /usr/local/lib/python3.10/dist-packages (from altair>=4.2.0->gradio==3.32.0) (0.4)\n",
            "Requirement already satisfied: jsonschema>=3.0 in /usr/local/lib/python3.10/dist-packages (from altair>=4.2.0->gradio==3.32.0) (4.3.3)\n",
            "Requirement already satisfied: toolz in /usr/local/lib/python3.10/dist-packages (from altair>=4.2.0->gradio==3.32.0) (0.12.0)\n",
            "Requirement already satisfied: fsspec in /usr/local/lib/python3.10/dist-packages (from gradio-client>=0.2.4->gradio==3.32.0) (2023.4.0)\n",
            "Requirement already satisfied: packaging in /usr/local/lib/python3.10/dist-packages (from gradio-client>=0.2.4->gradio==3.32.0) (23.1)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from huggingface-hub>=0.13.0->gradio==3.32.0) (3.12.0)\n",
            "Requirement already satisfied: tqdm>=4.42.1 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub>=0.13.0->gradio==3.32.0) (4.65.0)\n",
            "Requirement already satisfied: mdurl~=0.1 in /usr/local/lib/python3.10/dist-packages (from markdown-it-py[linkify]>=2.0.0->gradio==3.32.0) (0.1.2)\n",
            "Requirement already satisfied: linkify-it-py<3,>=1 in /usr/local/lib/python3.10/dist-packages (from markdown-it-py[linkify]>=2.0.0->gradio==3.32.0) (2.0.2)\n",
            "Requirement already satisfied: python-dateutil>=2.8.1 in /usr/local/lib/python3.10/dist-packages (from pandas->gradio==3.32.0) (2.8.2)\n",
            "Requirement already satisfied: pytz>=2020.1 in /usr/local/lib/python3.10/dist-packages (from pandas->gradio==3.32.0) (2022.7.1)\n",
            "Requirement already satisfied: click>=7.0 in /usr/local/lib/python3.10/dist-packages (from uvicorn>=0.14.0->gradio==3.32.0) (8.1.3)\n",
            "Requirement already satisfied: h11>=0.8 in /usr/local/lib/python3.10/dist-packages (from uvicorn>=0.14.0->gradio==3.32.0) (0.14.0)\n",
            "Requirement already satisfied: attrs>=17.3.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->gradio==3.32.0) (23.1.0)\n",
            "Requirement already satisfied: charset-normalizer<4.0,>=2.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->gradio==3.32.0) (2.0.12)\n",
            "Requirement already satisfied: multidict<7.0,>=4.5 in /usr/local/lib/python3.10/dist-packages (from aiohttp->gradio==3.32.0) (6.0.4)\n",
            "Requirement already satisfied: async-timeout<5.0,>=4.0.0a3 in /usr/local/lib/python3.10/dist-packages (from aiohttp->gradio==3.32.0) (4.0.2)\n",
            "Requirement already satisfied: yarl<2.0,>=1.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->gradio==3.32.0) (1.9.2)\n",
            "Requirement already satisfied: frozenlist>=1.1.1 in /usr/local/lib/python3.10/dist-packages (from aiohttp->gradio==3.32.0) (1.3.3)\n",
            "Requirement already satisfied: aiosignal>=1.1.2 in /usr/local/lib/python3.10/dist-packages (from aiohttp->gradio==3.32.0) (1.3.1)\n",
            "Requirement already satisfied: starlette<0.28.0,>=0.27.0 in /usr/local/lib/python3.10/dist-packages (from fastapi->gradio==3.32.0) (0.27.0)\n",
            "Requirement already satisfied: certifi in /usr/local/lib/python3.10/dist-packages (from httpx->gradio==3.32.0) (2022.12.7)\n",
            "Requirement already satisfied: httpcore<0.18.0,>=0.15.0 in /usr/local/lib/python3.10/dist-packages (from httpx->gradio==3.32.0) (0.17.2)\n",
            "Requirement already satisfied: idna in /usr/local/lib/python3.10/dist-packages (from httpx->gradio==3.32.0) (3.4)\n",
            "Requirement already satisfied: sniffio in /usr/local/lib/python3.10/dist-packages (from httpx->gradio==3.32.0) (1.3.0)\n",
            "Requirement already satisfied: contourpy>=1.0.1 in /usr/local/lib/python3.10/dist-packages (from matplotlib->gradio==3.32.0) (1.0.7)\n",
            "Requirement already satisfied: cycler>=0.10 in /usr/local/lib/python3.10/dist-packages (from matplotlib->gradio==3.32.0) (0.11.0)\n",
            "Requirement already satisfied: fonttools>=4.22.0 in /usr/local/lib/python3.10/dist-packages (from matplotlib->gradio==3.32.0) (4.39.3)\n",
            "Requirement already satisfied: kiwisolver>=1.0.1 in /usr/local/lib/python3.10/dist-packages (from matplotlib->gradio==3.32.0) (1.4.4)\n",
            "Requirement already satisfied: pyparsing>=2.3.1 in /usr/local/lib/python3.10/dist-packages (from matplotlib->gradio==3.32.0) (3.0.9)\n",
            "Requirement already satisfied: urllib3<1.27,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests->gradio==3.32.0) (1.26.15)\n",
            "Requirement already satisfied: anyio<5.0,>=3.0 in /usr/local/lib/python3.10/dist-packages (from httpcore<0.18.0,>=0.15.0->httpx->gradio==3.32.0) (3.6.2)\n",
            "Requirement already satisfied: pyrsistent!=0.17.0,!=0.17.1,!=0.17.2,>=0.14.0 in /usr/local/lib/python3.10/dist-packages (from jsonschema>=3.0->altair>=4.2.0->gradio==3.32.0) (0.19.3)\n",
            "Requirement already satisfied: uc-micro-py in /usr/local/lib/python3.10/dist-packages (from linkify-it-py<3,>=1->markdown-it-py[linkify]>=2.0.0->gradio==3.32.0) (1.0.2)\n",
            "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.10/dist-packages (from python-dateutil>=2.8.1->pandas->gradio==3.32.0) (1.16.0)\n",
            "Installing collected packages: gradio\n",
            "  Attempting uninstall: gradio\n",
            "    Found existing installation: gradio 3.31.0\n",
            "    Uninstalling gradio-3.31.0:\n",
            "      Successfully uninstalled gradio-3.31.0\n",
            "Successfully installed gradio-3.32.0\n",
            "mkdir: cannot create directory ‘/content/text-generation-webui/repositories’: File exists\n",
            "/content/text-generation-webui/repositories\n",
            "Cloning into 'AutoGPTQ'...\n",
            "remote: Enumerating objects: 2029, done.\u001b[K\n",
            "remote: Counting objects: 100% (407/407), done.\u001b[K\n",
            "remote: Compressing objects: 100% (211/211), done.\u001b[K\n",
            "remote: Total 2029 (delta 278), reused 215 (delta 190), pack-reused 1622\u001b[K\n",
            "Receiving objects: 100% (2029/2029), 7.37 MiB | 11.53 MiB/s, done.\n",
            "Resolving deltas: 100% (1365/1365), done.\n",
            "/content/text-generation-webui/repositories/AutoGPTQ\n",
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Processing /content/text-generation-webui/repositories/AutoGPTQ\n",
            "  Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "Requirement already satisfied: accelerate>=0.19.0 in /usr/local/lib/python3.10/dist-packages (from auto-gptq==0.2.0+cu1180) (0.20.0.dev0)\n",
            "Requirement already satisfied: datasets in /usr/local/lib/python3.10/dist-packages (from auto-gptq==0.2.0+cu1180) (2.12.0)\n",
            "Requirement already satisfied: numpy in /usr/local/lib/python3.10/dist-packages (from auto-gptq==0.2.0+cu1180) (1.22.4)\n",
            "Collecting rouge (from auto-gptq==0.2.0+cu1180)\n",
            "  Downloading rouge-1.0.1-py3-none-any.whl (13 kB)\n",
            "Requirement already satisfied: torch>=1.13.0 in /usr/local/lib/python3.10/dist-packages (from auto-gptq==0.2.0+cu1180) (2.0.1+cu118)\n",
            "Requirement already satisfied: safetensors in /usr/local/lib/python3.10/dist-packages (from auto-gptq==0.2.0+cu1180) (0.3.1)\n",
            "Requirement already satisfied: transformers>=4.26.1 in /usr/local/lib/python3.10/dist-packages (from auto-gptq==0.2.0+cu1180) (4.30.0.dev0)\n",
            "Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.10/dist-packages (from accelerate>=0.19.0->auto-gptq==0.2.0+cu1180) (23.1)\n",
            "Requirement already satisfied: psutil in /usr/local/lib/python3.10/dist-packages (from accelerate>=0.19.0->auto-gptq==0.2.0+cu1180) (5.9.5)\n",
            "Requirement already satisfied: pyyaml in /usr/local/lib/python3.10/dist-packages (from accelerate>=0.19.0->auto-gptq==0.2.0+cu1180) (6.0)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from torch>=1.13.0->auto-gptq==0.2.0+cu1180) (3.12.0)\n",
            "Requirement already satisfied: typing-extensions in /usr/local/lib/python3.10/dist-packages (from torch>=1.13.0->auto-gptq==0.2.0+cu1180) (4.5.0)\n",
            "Requirement already satisfied: sympy in /usr/local/lib/python3.10/dist-packages (from torch>=1.13.0->auto-gptq==0.2.0+cu1180) (1.11.1)\n",
            "Requirement already satisfied: networkx in /usr/local/lib/python3.10/dist-packages (from torch>=1.13.0->auto-gptq==0.2.0+cu1180) (3.1)\n",
            "Requirement already satisfied: jinja2 in /usr/local/lib/python3.10/dist-packages (from torch>=1.13.0->auto-gptq==0.2.0+cu1180) (3.1.2)\n",
            "Requirement already satisfied: triton==2.0.0 in /usr/local/lib/python3.10/dist-packages (from torch>=1.13.0->auto-gptq==0.2.0+cu1180) (2.0.0)\n",
            "Requirement already satisfied: cmake in /usr/local/lib/python3.10/dist-packages (from triton==2.0.0->torch>=1.13.0->auto-gptq==0.2.0+cu1180) (3.25.2)\n",
            "Requirement already satisfied: lit in /usr/local/lib/python3.10/dist-packages (from triton==2.0.0->torch>=1.13.0->auto-gptq==0.2.0+cu1180) (16.0.5)\n",
            "Requirement already satisfied: huggingface-hub<1.0,>=0.14.1 in /usr/local/lib/python3.10/dist-packages (from transformers>=4.26.1->auto-gptq==0.2.0+cu1180) (0.14.1)\n",
            "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.10/dist-packages (from transformers>=4.26.1->auto-gptq==0.2.0+cu1180) (2022.10.31)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.10/dist-packages (from transformers>=4.26.1->auto-gptq==0.2.0+cu1180) (2.27.1)\n",
            "Requirement already satisfied: tokenizers!=0.11.3,<0.14,>=0.11.1 in /usr/local/lib/python3.10/dist-packages (from transformers>=4.26.1->auto-gptq==0.2.0+cu1180) (0.13.3)\n",
            "Requirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.10/dist-packages (from transformers>=4.26.1->auto-gptq==0.2.0+cu1180) (4.65.0)\n",
            "Requirement already satisfied: pyarrow>=8.0.0 in /usr/local/lib/python3.10/dist-packages (from datasets->auto-gptq==0.2.0+cu1180) (9.0.0)\n",
            "Requirement already satisfied: dill<0.3.7,>=0.3.0 in /usr/local/lib/python3.10/dist-packages (from datasets->auto-gptq==0.2.0+cu1180) (0.3.6)\n",
            "Requirement already satisfied: pandas in /usr/local/lib/python3.10/dist-packages (from datasets->auto-gptq==0.2.0+cu1180) (1.5.3)\n",
            "Requirement already satisfied: xxhash in /usr/local/lib/python3.10/dist-packages (from datasets->auto-gptq==0.2.0+cu1180) (3.2.0)\n",
            "Requirement already satisfied: multiprocess in /usr/local/lib/python3.10/dist-packages (from datasets->auto-gptq==0.2.0+cu1180) (0.70.14)\n",
            "Requirement already satisfied: fsspec[http]>=2021.11.1 in /usr/local/lib/python3.10/dist-packages (from datasets->auto-gptq==0.2.0+cu1180) (2023.4.0)\n",
            "Requirement already satisfied: aiohttp in /usr/local/lib/python3.10/dist-packages (from datasets->auto-gptq==0.2.0+cu1180) (3.8.4)\n",
            "Requirement already satisfied: responses<0.19 in /usr/local/lib/python3.10/dist-packages (from datasets->auto-gptq==0.2.0+cu1180) (0.18.0)\n",
            "Requirement already satisfied: six in /usr/local/lib/python3.10/dist-packages (from rouge->auto-gptq==0.2.0+cu1180) (1.16.0)\n",
            "Requirement already satisfied: attrs>=17.3.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets->auto-gptq==0.2.0+cu1180) (23.1.0)\n",
            "Requirement already satisfied: charset-normalizer<4.0,>=2.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets->auto-gptq==0.2.0+cu1180) (2.0.12)\n",
            "Requirement already satisfied: multidict<7.0,>=4.5 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets->auto-gptq==0.2.0+cu1180) (6.0.4)\n",
            "Requirement already satisfied: async-timeout<5.0,>=4.0.0a3 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets->auto-gptq==0.2.0+cu1180) (4.0.2)\n",
            "Requirement already satisfied: yarl<2.0,>=1.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets->auto-gptq==0.2.0+cu1180) (1.9.2)\n",
            "Requirement already satisfied: frozenlist>=1.1.1 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets->auto-gptq==0.2.0+cu1180) (1.3.3)\n",
            "Requirement already satisfied: aiosignal>=1.1.2 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets->auto-gptq==0.2.0+cu1180) (1.3.1)\n",
            "Requirement already satisfied: urllib3<1.27,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests->transformers>=4.26.1->auto-gptq==0.2.0+cu1180) (1.26.15)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests->transformers>=4.26.1->auto-gptq==0.2.0+cu1180) (2022.12.7)\n",
            "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests->transformers>=4.26.1->auto-gptq==0.2.0+cu1180) (3.4)\n",
            "Requirement already satisfied: MarkupSafe>=2.0 in /usr/local/lib/python3.10/dist-packages (from jinja2->torch>=1.13.0->auto-gptq==0.2.0+cu1180) (2.1.2)\n",
            "Requirement already satisfied: python-dateutil>=2.8.1 in /usr/local/lib/python3.10/dist-packages (from pandas->datasets->auto-gptq==0.2.0+cu1180) (2.8.2)\n",
            "Requirement already satisfied: pytz>=2020.1 in /usr/local/lib/python3.10/dist-packages (from pandas->datasets->auto-gptq==0.2.0+cu1180) (2022.7.1)\n",
            "Requirement already satisfied: mpmath>=0.19 in /usr/local/lib/python3.10/dist-packages (from sympy->torch>=1.13.0->auto-gptq==0.2.0+cu1180) (1.3.0)\n",
            "Building wheels for collected packages: auto-gptq\n",
            "  Building wheel for auto-gptq (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for auto-gptq: filename=auto_gptq-0.2.0+cu1180-cp310-cp310-linux_x86_64.whl size=3646764 sha256=30caab40d85598890f90ca09e604132e337b02f90fd3a14f4aa399571e776a97\n",
            "  Stored in directory: /tmp/pip-ephem-wheel-cache-5ejveyil/wheels/bb/49/89/0e08b5201f0934afb5265613a7f34338ee566328f9c8366e64\n",
            "Successfully built auto-gptq\n",
            "Installing collected packages: rouge, auto-gptq\n",
            "Successfully installed auto-gptq-0.2.0+cu1180 rouge-1.0.1\n",
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Requirement already satisfied: einops in /usr/local/lib/python3.10/dist-packages (0.6.1)\n",
            "\n",
            "Download Results:\n",
            "gid   |stat|avg speed  |path/URI\n",
            "======+====+===========+=======================================================\n",
            "908712|\u001b[1;32mOK\u001b[0m  |       0B/s|/content/text-generation-webui/models/falcon-7b-instruct-GPTQ/config.json\n",
            "\n",
            "Status Legend:\n",
            "(OK):download completed.\n",
            "\n",
            "Download Results:\n",
            "gid   |stat|avg speed  |path/URI\n",
            "======+====+===========+=======================================================\n",
            "15981b|\u001b[1;32mOK\u001b[0m  |       0B/s|/content/text-generation-webui/models/falcon-7b-instruct-GPTQ/generation_config.json\n",
            "\n",
            "Status Legend:\n",
            "(OK):download completed.\n",
            "\n",
            "Download Results:\n",
            "gid   |stat|avg speed  |path/URI\n",
            "======+====+===========+=======================================================\n",
            "f383f5|\u001b[1;32mOK\u001b[0m  |       0B/s|/content/text-generation-webui/models/falcon-7b-instruct-GPTQ/special_tokens_map.json\n",
            "\n",
            "Status Legend:\n",
            "(OK):download completed.\n",
            "\n",
            "Download Results:\n",
            "gid   |stat|avg speed  |path/URI\n",
            "======+====+===========+=======================================================\n",
            "1364fb|\u001b[1;32mOK\u001b[0m  |       0B/s|/content/text-generation-webui/models/falcon-7b-instruct-GPTQ/configuration_RW.py\n",
            "\n",
            "Status Legend:\n",
            "(OK):download completed.\n",
            "\n",
            "Download Results:\n",
            "gid   |stat|avg speed  |path/URI\n",
            "======+====+===========+=======================================================\n",
            "892d03|\u001b[1;32mOK\u001b[0m  |       0B/s|/content/text-generation-webui/models/falcon-7b-instruct-GPTQ/gptq_model-4bit-64g.safetensors\n",
            "\n",
            "Status Legend:\n",
            "(OK):download completed.\n",
            "\n",
            "Download Results:\n",
            "gid   |stat|avg speed  |path/URI\n",
            "======+====+===========+=======================================================\n",
            "a0b43c|\u001b[1;32mOK\u001b[0m  |       0B/s|/content/text-generation-webui/models/falcon-7b-instruct-GPTQ/modelling_RW.py\n",
            "\n",
            "Status Legend:\n",
            "(OK):download completed.\n",
            "\n",
            "Download Results:\n",
            "gid   |stat|avg speed  |path/URI\n",
            "======+====+===========+=======================================================\n",
            "1fd26c|\u001b[1;32mOK\u001b[0m  |       0B/s|/content/text-generation-webui/models/falcon-7b-instruct-GPTQ/quantize_config.json\n",
            "\n",
            "Status Legend:\n",
            "(OK):download completed.\n",
            "\n",
            "Download Results:\n",
            "gid   |stat|avg speed  |path/URI\n",
            "======+====+===========+=======================================================\n",
            "6cf18b|\u001b[1;32mOK\u001b[0m  |       0B/s|/content/text-generation-webui/models/falcon-7b-instruct-GPTQ/tokenizer.json\n",
            "\n",
            "Status Legend:\n",
            "(OK):download completed.\n",
            "\n",
            "Download Results:\n",
            "gid   |stat|avg speed  |path/URI\n",
            "======+====+===========+=======================================================\n",
            "d6d3c9|\u001b[1;32mOK\u001b[0m  |       0B/s|/content/text-generation-webui/models/falcon-7b-instruct-GPTQ/tokenizer_config.json\n",
            "\n",
            "Status Legend:\n",
            "(OK):download completed.\n",
            "/content/text-generation-webui\n",
            "WARNING:\u001b[33mtrust_remote_code is enabled. This is dangerous.\u001b[0m\n",
            "WARNING:\u001b[33mThe gradio \"share link\" feature uses a proprietary executable to create a reverse tunnel. Use it with care.\u001b[0m\n",
            "2023-06-01 07:50:05.558088: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT\n",
            "bin /usr/local/lib/python3.10/dist-packages/bitsandbytes/libbitsandbytes_cuda118.so\n",
            "INFO:\u001b[32mLoading falcon-7b-instruct-GPTQ...\u001b[0m\n",
            "WARNING:\u001b[33mThe AutoGPTQ params are: {'model_basename': 'gptq_model-4bit-64g', 'device': 'cuda:0', 'use_triton': False, 'use_safetensors': True, 'trust_remote_code': True, 'max_memory': None}\u001b[0m\n",
            "WARNING:\u001b[33mThe safetensors archive passed at models/falcon-7b-instruct-GPTQ/gptq_model-4bit-64g.safetensors does not contain metadata. Make sure to save your model with the `save_pretrained` method. Defaulting to 'pt' metadata.\u001b[0m\n",
            "WARNING:\u001b[33mcan't get model's sequence length from model config, will set to 4096.\u001b[0m\n",
            "WARNING:\u001b[33mRWGPTQForCausalLM hasn't fused attention module yet, will skip inject fused attention.\u001b[0m\n",
            "WARNING:\u001b[33mRWGPTQForCausalLM hasn't fused mlp module yet, will skip inject fused mlp.\u001b[0m\n",
            "INFO:\u001b[32mLoaded the model in 33.50 seconds.\n",
            "\u001b[0m\n",
            "INFO:\u001b[32mLoading the extension \"gallery\"...\u001b[0m\n",
            "Running on local URL:  http://127.0.0.1:7860\n",
            "Running on public URL: https://68abb438fa285ea62b.gradio.live\n",
            "\n",
            "This share link expires in 72 hours. For free permanent hosting and GPU upgrades (NEW!), check out Spaces: https://huggingface.co/spaces\n",
            "Output generated in 53.14 seconds (1.13 tokens/s, 60 tokens, context 63, seed 2058865511)\n",
            "Output generated in 74.26 seconds (1.23 tokens/s, 91 tokens, context 31, seed 1134088929)\n"
          ]
        }
      ],
      "source": [
        "%cd /content\n",
        "!apt-get -y install -qq aria2\n",
        "\n",
        "!git clone -b V1.1 https://github.com/Troyanovsky/text-generation-webui\n",
        "%cd /content/text-generation-webui\n",
        "!pip install -r requirements.txt\n",
        "!pip install -U gradio==3.32.0\n",
        "\n",
        "!mkdir /content/text-generation-webui/repositories\n",
        "%cd /content/text-generation-webui/repositories\n",
        "!git clone https://github.com/PanQiWei/AutoGPTQ\n",
        "%cd AutoGPTQ\n",
        "!pip install .\n",
        "!pip install einops\n",
        "\n",
        "!aria2c --console-log-level=error -c -x 16 -s 16 -k 1M https://huggingface.co/TheBloke/falcon-7b-instruct-GPTQ/resolve/main/config.json -d /content/text-generation-webui/models/falcon-7b-instruct-GPTQ -o config.json\n",
        "!aria2c --console-log-level=error -c -x 16 -s 16 -k 1M https://huggingface.co/TheBloke/falcon-7b-instruct-GPTQ/resolve/main/generation_config.json -d /content/text-generation-webui/models/falcon-7b-instruct-GPTQ -o generation_config.json\n",
        "!aria2c --console-log-level=error -c -x 16 -s 16 -k 1M https://huggingface.co/TheBloke/falcon-7b-instruct-GPTQ/resolve/main/special_tokens_map.json -d /content/text-generation-webui/models/falcon-7b-instruct-GPTQ -o special_tokens_map.json\n",
        "!aria2c --console-log-level=error -c -x 16 -s 16 -k 1M https://huggingface.co/TheBloke/falcon-7b-instruct-GPTQ/resolve/main/configuration_RW.py -d /content/text-generation-webui/models/falcon-7b-instruct-GPTQ -o configuration_RW.py\n",
        "!aria2c --console-log-level=error -c -x 16 -s 16 -k 1M https://huggingface.co/TheBloke/falcon-7b-instruct-GPTQ/resolve/main/gptq_model-4bit-64g.safetensors -d /content/text-generation-webui/models/falcon-7b-instruct-GPTQ -o gptq_model-4bit-64g.safetensors\n",
        "!aria2c --console-log-level=error -c -x 16 -s 16 -k 1M https://huggingface.co/TheBloke/falcon-7b-instruct-GPTQ/resolve/main/modelling_RW.py -d /content/text-generation-webui/models/falcon-7b-instruct-GPTQ -o modelling_RW.py\n",
        "!aria2c --console-log-level=error -c -x 16 -s 16 -k 1M https://huggingface.co/TheBloke/falcon-7b-instruct-GPTQ/resolve/main/quantize_config.json -d /content/text-generation-webui/models/falcon-7b-instruct-GPTQ -o quantize_config.json\n",
        "!aria2c --console-log-level=error -c -x 16 -s 16 -k 1M https://huggingface.co/TheBloke/falcon-7b-instruct-GPTQ/resolve/main/tokenizer.json -d /content/text-generation-webui/models/falcon-7b-instruct-GPTQ -o tokenizer.json\n",
        "!aria2c --console-log-level=error -c -x 16 -s 16 -k 1M https://huggingface.co/TheBloke/falcon-7b-instruct-GPTQ/resolve/main/tokenizer_config.json -d /content/text-generation-webui/models/falcon-7b-instruct-GPTQ -o tokenizer_config.json\n",
        "\n",
        "%cd /content/text-generation-webui\n",
        "!python server.py --share --chat --autogptq --trust-remote-code"
      ]
    }
  ]
}