legal-nlp/17.0.Vector_Store_Integration.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "0i_BfRnB-b8C"
      },
      "source": [
        "![JohnSnowLabs](https://nlp.johnsnowlabs.com/assets/images/logo.png)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "qv2B6430-vY6"
      },
      "source": [
        " # Colab Setup\n",
        "\n",
        "[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.?com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/legal-nlp/17.0.Vector_Store_Integration.ipynb)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "m_W96p3NH2th"
      },
      "outputs": [],
      "source": [
        "!pip install johnsnowlabs\n",
        "\n",
        "from johnsnowlabs import nlp\n",
        "\n",
        "nlp.install(force_browser=True)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "MAFs7z69liAi",
        "outputId": "9d2e5890-81e5-4724-a017-17ef6ebbf6da"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "👌 Detected license file /content/spark_nlp_for_healthcare_spark_ocr_7163 (22).json\n",
            "👷 Trying to install compatible secrets. Use nlp.settings.enforce_versions=False if you want to install outdated secrets.\n",
            "👌 Launched \u001b[92mcpu optimized\u001b[39m session with with: 🚀Spark-NLP==5.0.1, 💊Spark-Healthcare==5.0.1, running on ⚡ PySpark==3.1.2\n"
          ]
        }
      ],
      "source": [
        "import json\n",
        "import pandas as pd\n",
        "from johnsnowlabs import nlp\n",
        "\n",
        "params = {\"spark.driver.memory\":\"32G\",\n",
        "\"spark.kryoserializer.buffer.max\":\"2000M\",\n",
        "\"spark.driver.maxResultSize\":\"20000M\"}\n",
        "\n",
        "\n",
        "spark = nlp.start(spark_conf=params)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "X6jnjUHnL0xd"
      },
      "outputs": [],
      "source": []
    },
    {
      "cell_type": "code",
      "execution_count": 4,
      "metadata": {
        "id": "zu6tCQOYT-20"
      },
      "outputs": [],
      "source": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "a_cA8zpeAU8C"
      },
      "source": [
        "## Generate Embeddings"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "IPA0awue3iTN"
      },
      "source": [
        "### Loading the data"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "RnB5bzptT7Q_"
      },
      "outputs": [],
      "source": [
        "!wget https://raw.githubusercontent.com/JohnSnowLabs/spark-nlp-workshop/master/legal-nlp/data/mnda_sample.csv"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 5,
      "metadata": {
        "id": "pbXWtuL9UIkO"
      },
      "outputs": [],
      "source": [
        "df = pd.read_csv('mnda_sample.csv')\n",
        "\n",
        "documents = df['text'].tolist()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 6,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "q7suC2EtUwbM",
        "outputId": "3c0699f1-c540-4045-c702-3a6f98345b3d"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "(263, 2)"
            ]
          },
          "execution_count": 6,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "df.shape"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "BDsMFSx53GL8",
        "outputId": "0654e042-ff72-45d1-a93a-881b0940b7f5"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Total documents: 263\n"
          ]
        }
      ],
      "source": [
        "print(f'Total documents: {len(documents)}')"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ObVdJlG14jIL"
      },
      "source": [
        "### Creating the pipeline"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "hK076VGYkn-0",
        "outputId": "1de65a48-7a00-446c-fc29-1ee89bd59452"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "sent_bert_base_uncased_legal download started this may take some time.\n",
            "Approximate size to download 390.8 MB\n",
            "[OK!]\n"
          ]
        }
      ],
      "source": [
        "document_assembler = nlp.DocumentAssembler()\\\n",
        "    .setInputCol(\"text\")\\\n",
        "    .setOutputCol(\"document\")\n",
        "\n",
        "sentence_embeddings = nlp.BertSentenceEmbeddings.pretrained(\"sent_bert_base_uncased_legal\", \"en\")\\\n",
        "  .setInputCols([\"document\"])\\\n",
        "  .setOutputCol(\"sbert_embeddings\")\n",
        "\n",
        "embeddingsFinisher = nlp.EmbeddingsFinisher() \\\n",
        "    .setInputCols(\"sbert_embeddings\") \\\n",
        "    .setOutputCols(\"finished_sentence_embeddings\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "NjcZdzNSl4aJ",
        "outputId": "f141c7ec-c807-4c9a-f21f-0509c6f49f31"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "263"
            ]
          },
          "execution_count": 53,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "data = [[i] for i in documents]\n",
        "len(data)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "XoqKowNFlsfQ"
      },
      "outputs": [],
      "source": [
        "data = spark.createDataFrame(data) \\\n",
        "    .toDF(\"text\")\n",
        "\n",
        "pipeline = nlp.Pipeline().setStages([\n",
        "    document_assembler,\n",
        "    sentence_embeddings,\n",
        "    embeddingsFinisher\n",
        "]).fit(data)\n",
        "\n",
        "result = pipeline.transform(data)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "JrhOh6rVk-D1"
      },
      "outputs": [],
      "source": [
        "resultWithSize = result.selectExpr(\"document.result\",\"explode(finished_sentence_embeddings) as embeddings\").toPandas()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 424
        },
        "id": "V-G4Y5qdmLs3",
        "outputId": "89318624-1192-4df8-8282-3357888c23e9"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "\n",
              "  <div id=\"df-010cbee7-c7b3-4f5f-9471-416a2b8f5fdc\" class=\"colab-df-container\">\n",
              "    <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>result</th>\n",
              "      <th>embeddings</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>[The recipient shall not, during the term of t...</td>\n",
              "      <td>[0.26616281270980835, 0.341601699590683, 0.934...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>[The recipient shall not, during the term of t...</td>\n",
              "      <td>[-0.11213510483503342, 0.10545776784420013, -0...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>[This Agreement shall inure to the benefit of ...</td>\n",
              "      <td>[-0.46788308024406433, 0.47307464480400085, 0....</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>[Unless otherwise provided herein, your obliga...</td>\n",
              "      <td>[-0.49271631240844727, 0.27636203169822693, 0....</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>[The recipient shall not, during the term of t...</td>\n",
              "      <td>[-0.7534305453300476, 0.14203622937202454, 1.0...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>258</th>\n",
              "      <td>[Evaluation Material.  The term\\n    “Evaluati...</td>\n",
              "      <td>[-0.2982212007045746, 0.493274062871933, -0.95...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>259</th>\n",
              "      <td>[Notwithstanding any other provision hereof, t...</td>\n",
              "      <td>[-0.898460865020752, 0.7281344532966614, 0.999...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>260</th>\n",
              "      <td>[In addition, each Party agrees that, without ...</td>\n",
              "      <td>[-0.7481690645217896, 0.9281681180000305, 0.99...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>261</th>\n",
              "      <td>[Return and Destruction of Evaluation\\n    Mat...</td>\n",
              "      <td>[-0.7783533334732056, 0.9506694078445435, 0.99...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>262</th>\n",
              "      <td>[Governing Law; Waiver of Jury\\n    Trial.  Th...</td>\n",
              "      <td>[-0.6941927075386047, 0.8366994261741638, 0.99...</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>263 rows × 2 columns</p>\n",
              "</div>\n",
              "    <div class=\"colab-df-buttons\">\n",
              "\n",
              "  <div class=\"colab-df-container\">\n",
              "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-010cbee7-c7b3-4f5f-9471-416a2b8f5fdc')\"\n",
              "            title=\"Convert this dataframe to an interactive table.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
              "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "\n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    .colab-df-buttons div {\n",
              "      margin-bottom: 4px;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "    <script>\n",
              "      const buttonEl =\n",
              "        document.querySelector('#df-010cbee7-c7b3-4f5f-9471-416a2b8f5fdc button.colab-df-convert');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      async function convertToInteractive(key) {\n",
              "        const element = document.querySelector('#df-010cbee7-c7b3-4f5f-9471-416a2b8f5fdc');\n",
              "        const dataTable =\n",
              "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                    [key], {});\n",
              "        if (!dataTable) return;\n",
              "\n",
              "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "          + ' to learn more about interactive tables.';\n",
              "        element.innerHTML = '';\n",
              "        dataTable['output_type'] = 'display_data';\n",
              "        await google.colab.output.renderOutput(dataTable, element);\n",
              "        const docLink = document.createElement('div');\n",
              "        docLink.innerHTML = docLinkHtml;\n",
              "        element.appendChild(docLink);\n",
              "      }\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "\n",
              "<div id=\"df-e979a685-f711-4542-b67e-326014153ef2\">\n",
              "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-e979a685-f711-4542-b67e-326014153ef2')\"\n",
              "            title=\"Suggest charts.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\">\n",
              "    <g>\n",
              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
              "    </g>\n",
              "</svg>\n",
              "  </button>\n",
              "\n",
              "<style>\n",
              "  .colab-df-quickchart {\n",
              "    background-color: #E8F0FE;\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: #1967D2;\n",
              "    height: 32px;\n",
              "    padding: 0 0 0 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: #E2EBFA;\n",
              "    box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: #174EA6;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "    background-color: #3B4455;\n",
              "    fill: #D2E3FC;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart:hover {\n",
              "    background-color: #434B5C;\n",
              "    box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "    filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "    fill: #FFFFFF;\n",
              "  }\n",
              "</style>\n",
              "\n",
              "  <script>\n",
              "    async function quickchart(key) {\n",
              "      const charts = await google.colab.kernel.invokeFunction(\n",
              "          'suggestCharts', [key], {});\n",
              "    }\n",
              "    (() => {\n",
              "      let quickchartButtonEl =\n",
              "        document.querySelector('#df-e979a685-f711-4542-b67e-326014153ef2 button');\n",
              "      quickchartButtonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "    })();\n",
              "  </script>\n",
              "</div>\n",
              "    </div>\n",
              "  </div>\n"
            ],
            "text/plain": [
              "                                                result  \\\n",
              "0    [The recipient shall not, during the term of t...   \n",
              "1    [The recipient shall not, during the term of t...   \n",
              "2    [This Agreement shall inure to the benefit of ...   \n",
              "3    [Unless otherwise provided herein, your obliga...   \n",
              "4    [The recipient shall not, during the term of t...   \n",
              "..                                                 ...   \n",
              "258  [Evaluation Material.  The term\\n    “Evaluati...   \n",
              "259  [Notwithstanding any other provision hereof, t...   \n",
              "260  [In addition, each Party agrees that, without ...   \n",
              "261  [Return and Destruction of Evaluation\\n    Mat...   \n",
              "262  [Governing Law; Waiver of Jury\\n    Trial.  Th...   \n",
              "\n",
              "                                            embeddings  \n",
              "0    [0.26616281270980835, 0.341601699590683, 0.934...  \n",
              "1    [-0.11213510483503342, 0.10545776784420013, -0...  \n",
              "2    [-0.46788308024406433, 0.47307464480400085, 0....  \n",
              "3    [-0.49271631240844727, 0.27636203169822693, 0....  \n",
              "4    [-0.7534305453300476, 0.14203622937202454, 1.0...  \n",
              "..                                                 ...  \n",
              "258  [-0.2982212007045746, 0.493274062871933, -0.95...  \n",
              "259  [-0.898460865020752, 0.7281344532966614, 0.999...  \n",
              "260  [-0.7481690645217896, 0.9281681180000305, 0.99...  \n",
              "261  [-0.7783533334732056, 0.9506694078445435, 0.99...  \n",
              "262  [-0.6941927075386047, 0.8366994261741638, 0.99...  \n",
              "\n",
              "[263 rows x 2 columns]"
            ]
          },
          "execution_count": 56,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "resultWithSize"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "83hvvJT5TWmU"
      },
      "outputs": [],
      "source": [
        "df_processed = resultWithSize.copy()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 424
        },
        "id": "ukugRqdUKbSf",
        "outputId": "5e35d8eb-7a78-46df-8c19-293588421509"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "\n",
              "  <div id=\"df-3ddf3e88-f886-4553-bec4-10b9c6821011\" class=\"colab-df-container\">\n",
              "    <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>result</th>\n",
              "      <th>embeddings</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>[The recipient shall not, during the term of t...</td>\n",
              "      <td>[0.26616281270980835, 0.341601699590683, 0.934...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>[The recipient shall not, during the term of t...</td>\n",
              "      <td>[-0.11213510483503342, 0.10545776784420013, -0...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>[This Agreement shall inure to the benefit of ...</td>\n",
              "      <td>[-0.46788308024406433, 0.47307464480400085, 0....</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>[Unless otherwise provided herein, your obliga...</td>\n",
              "      <td>[-0.49271631240844727, 0.27636203169822693, 0....</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>[The recipient shall not, during the term of t...</td>\n",
              "      <td>[-0.7534305453300476, 0.14203622937202454, 1.0...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>258</th>\n",
              "      <td>[Evaluation Material.  The term\\n    “Evaluati...</td>\n",
              "      <td>[-0.2982212007045746, 0.493274062871933, -0.95...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>259</th>\n",
              "      <td>[Notwithstanding any other provision hereof, t...</td>\n",
              "      <td>[-0.898460865020752, 0.7281344532966614, 0.999...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>260</th>\n",
              "      <td>[In addition, each Party agrees that, without ...</td>\n",
              "      <td>[-0.7481690645217896, 0.9281681180000305, 0.99...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>261</th>\n",
              "      <td>[Return and Destruction of Evaluation\\n    Mat...</td>\n",
              "      <td>[-0.7783533334732056, 0.9506694078445435, 0.99...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>262</th>\n",
              "      <td>[Governing Law; Waiver of Jury\\n    Trial.  Th...</td>\n",
              "      <td>[-0.6941927075386047, 0.8366994261741638, 0.99...</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>263 rows × 2 columns</p>\n",
              "</div>\n",
              "    <div class=\"colab-df-buttons\">\n",
              "\n",
              "  <div class=\"colab-df-container\">\n",
              "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-3ddf3e88-f886-4553-bec4-10b9c6821011')\"\n",
              "            title=\"Convert this dataframe to an interactive table.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
              "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "\n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    .colab-df-buttons div {\n",
              "      margin-bottom: 4px;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "    <script>\n",
              "      const buttonEl =\n",
              "        document.querySelector('#df-3ddf3e88-f886-4553-bec4-10b9c6821011 button.colab-df-convert');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      async function convertToInteractive(key) {\n",
              "        const element = document.querySelector('#df-3ddf3e88-f886-4553-bec4-10b9c6821011');\n",
              "        const dataTable =\n",
              "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                    [key], {});\n",
              "        if (!dataTable) return;\n",
              "\n",
              "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "          + ' to learn more about interactive tables.';\n",
              "        element.innerHTML = '';\n",
              "        dataTable['output_type'] = 'display_data';\n",
              "        await google.colab.output.renderOutput(dataTable, element);\n",
              "        const docLink = document.createElement('div');\n",
              "        docLink.innerHTML = docLinkHtml;\n",
              "        element.appendChild(docLink);\n",
              "      }\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "\n",
              "<div id=\"df-5b3d568f-17a5-4749-936d-9aae6c095805\">\n",
              "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-5b3d568f-17a5-4749-936d-9aae6c095805')\"\n",
              "            title=\"Suggest charts.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\">\n",
              "    <g>\n",
              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
              "    </g>\n",
              "</svg>\n",
              "  </button>\n",
              "\n",
              "<style>\n",
              "  .colab-df-quickchart {\n",
              "    background-color: #E8F0FE;\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: #1967D2;\n",
              "    height: 32px;\n",
              "    padding: 0 0 0 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: #E2EBFA;\n",
              "    box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: #174EA6;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "    background-color: #3B4455;\n",
              "    fill: #D2E3FC;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart:hover {\n",
              "    background-color: #434B5C;\n",
              "    box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "    filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "    fill: #FFFFFF;\n",
              "  }\n",
              "</style>\n",
              "\n",
              "  <script>\n",
              "    async function quickchart(key) {\n",
              "      const charts = await google.colab.kernel.invokeFunction(\n",
              "          'suggestCharts', [key], {});\n",
              "    }\n",
              "    (() => {\n",
              "      let quickchartButtonEl =\n",
              "        document.querySelector('#df-5b3d568f-17a5-4749-936d-9aae6c095805 button');\n",
              "      quickchartButtonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "    })();\n",
              "  </script>\n",
              "</div>\n",
              "    </div>\n",
              "  </div>\n"
            ],
            "text/plain": [
              "                                                result  \\\n",
              "0    [The recipient shall not, during the term of t...   \n",
              "1    [The recipient shall not, during the term of t...   \n",
              "2    [This Agreement shall inure to the benefit of ...   \n",
              "3    [Unless otherwise provided herein, your obliga...   \n",
              "4    [The recipient shall not, during the term of t...   \n",
              "..                                                 ...   \n",
              "258  [Evaluation Material.  The term\\n    “Evaluati...   \n",
              "259  [Notwithstanding any other provision hereof, t...   \n",
              "260  [In addition, each Party agrees that, without ...   \n",
              "261  [Return and Destruction of Evaluation\\n    Mat...   \n",
              "262  [Governing Law; Waiver of Jury\\n    Trial.  Th...   \n",
              "\n",
              "                                            embeddings  \n",
              "0    [0.26616281270980835, 0.341601699590683, 0.934...  \n",
              "1    [-0.11213510483503342, 0.10545776784420013, -0...  \n",
              "2    [-0.46788308024406433, 0.47307464480400085, 0....  \n",
              "3    [-0.49271631240844727, 0.27636203169822693, 0....  \n",
              "4    [-0.7534305453300476, 0.14203622937202454, 1.0...  \n",
              "..                                                 ...  \n",
              "258  [-0.2982212007045746, 0.493274062871933, -0.95...  \n",
              "259  [-0.898460865020752, 0.7281344532966614, 0.999...  \n",
              "260  [-0.7481690645217896, 0.9281681180000305, 0.99...  \n",
              "261  [-0.7783533334732056, 0.9506694078445435, 0.99...  \n",
              "262  [-0.6941927075386047, 0.8366994261741638, 0.99...  \n",
              "\n",
              "[263 rows x 2 columns]"
            ]
          },
          "execution_count": 58,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "df_processed"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ZV71Q6iAKjHU"
      },
      "source": [
        "## Let's use Weaviate to perform Vector search!"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "2fWZZXRH_i7C"
      },
      "source": [
        "**What is vector search?**\n",
        "\n",
        "Vector search refers to a search method that utilizes vector representations (vector embeddings) of data items to perform similarity-based searches. In vector search, data items such as documents, images, or other objects are transformed into high-dimensional vectors, where each dimension represents a specific feature or attribute of the item.\n",
        "\n",
        "The core idea behind vector search is that similar items will have similar vector representations, making it possible to measure the similarity between items by calculating the distance between their corresponding vectors. The closer the vectors are to each other, the more similar the items are considered to be.\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "GrvV0i2AKkT3",
        "outputId": "7a7959f5-4b0a-42ba-c22d-9e6935c837f1"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Requirement already satisfied: weaviate-client in /usr/local/lib/python3.10/dist-packages (3.22.1)\n",
            "Requirement already satisfied: requests<=2.31.0,>=2.28.0 in /usr/local/lib/python3.10/dist-packages (from weaviate-client) (2.31.0)\n",
            "Requirement already satisfied: validators<=0.21.0,>=0.18.2 in /usr/local/lib/python3.10/dist-packages (from weaviate-client) (0.21.0)\n",
            "Requirement already satisfied: tqdm<5.0.0,>=4.59.0 in /usr/local/lib/python3.10/dist-packages (from weaviate-client) (4.66.1)\n",
            "Requirement already satisfied: authlib>=1.1.0 in /usr/local/lib/python3.10/dist-packages (from weaviate-client) (1.2.1)\n",
            "Requirement already satisfied: cryptography>=3.2 in /usr/local/lib/python3.10/dist-packages (from authlib>=1.1.0->weaviate-client) (41.0.3)\n",
            "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests<=2.31.0,>=2.28.0->weaviate-client) (3.2.0)\n",
            "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests<=2.31.0,>=2.28.0->weaviate-client) (3.4)\n",
            "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests<=2.31.0,>=2.28.0->weaviate-client) (1.26.16)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests<=2.31.0,>=2.28.0->weaviate-client) (2023.7.22)\n",
            "Requirement already satisfied: cffi>=1.12 in /usr/local/lib/python3.10/dist-packages (from cryptography>=3.2->authlib>=1.1.0->weaviate-client) (1.15.1)\n",
            "Requirement already satisfied: pycparser in /usr/local/lib/python3.10/dist-packages (from cffi>=1.12->cryptography>=3.2->authlib>=1.1.0->weaviate-client) (2.21)\n"
          ]
        }
      ],
      "source": [
        "import sys\n",
        "!pip install weaviate-client\n",
        "import weaviate"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "kHwC6le0U2-3"
      },
      "source": [
        "### Register from the Weaviate website and create a cluster."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "UD4ez36pNyZW"
      },
      "outputs": [],
      "source": [
        "auth_config = weaviate.AuthApiKey(api_key=\"GCtvDb8Jsd01xmOdwWSpBUeOg36P4CyW5VlB\")  # Replace w/ your Weaviate instance API key\n",
        "\n",
        "# Instantiate the client\n",
        "client = weaviate.Client(\n",
        "    url=\"https://jsl-ilzkw6m2.weaviate.network\", # Replace w/ your Weaviate cluster URL\n",
        "    auth_client_secret=auth_config\n",
        ")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "z3VNXXgoTBMR",
        "outputId": "db150fa0-f695-42a3-d131-a9989701b9e8"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "True"
            ]
          },
          "execution_count": 63,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "client.is_ready()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "euVkDJ7N5D5E"
      },
      "outputs": [],
      "source": [
        "# uncomment and delete if any schemas already exist\n",
        "# client.schema.delete_all()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "JDt9BuQqVGJL"
      },
      "source": [
        "### Import data with vectors"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "xh-IVxXrJ97h"
      },
      "outputs": [],
      "source": [
        "jsl_data = []\n",
        "\n",
        "for i,j in df_processed.iterrows():\n",
        "  dicti = {}\n",
        "  dicti['Text'] = j['result']\n",
        "  dicti['Vector'] = j['embeddings']\n",
        "  jsl_data.append(dicti)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "hXoQ1KIGKsRj",
        "outputId": "15750c16-96e6-4030-f1c2-58da441330c2"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "dict_keys(['Text', 'Vector'])"
            ]
          },
          "execution_count": 66,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "jsl_data[0].keys()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "GUQyWWX7MjC0"
      },
      "outputs": [],
      "source": [
        "# Class definition object. Weaviate's autoschema feature will infer properties when importing.\n",
        "class_obj = {\n",
        "    \"class\": \"JSL_Document\",\n",
        "    \"vectorizer\": \"none\"\n",
        "}\n",
        "\n",
        "# Add the class to the schema\n",
        "client.schema.create_class(class_obj)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "uD9gxzhm7BtB",
        "outputId": "9964c5da-a0fa-4298-af51-42c0e2f1f62d"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "importing question: 1\n",
            "importing question: 2\n",
            "importing question: 3\n",
            "importing question: 4\n",
            "importing question: 5\n",
            "importing question: 6\n",
            "importing question: 7\n",
            "importing question: 8\n",
            "importing question: 9\n",
            "importing question: 10\n",
            "importing question: 11\n",
            "importing question: 12\n",
            "importing question: 13\n",
            "importing question: 14\n",
            "importing question: 15\n",
            "importing question: 16\n",
            "importing question: 17\n",
            "importing question: 18\n",
            "importing question: 19\n",
            "importing question: 20\n",
            "importing question: 21\n",
            "importing question: 22\n",
            "importing question: 23\n",
            "importing question: 24\n",
            "importing question: 25\n",
            "importing question: 26\n",
            "importing question: 27\n",
            "importing question: 28\n",
            "importing question: 29\n",
            "importing question: 30\n",
            "importing question: 31\n",
            "importing question: 32\n",
            "importing question: 33\n",
            "importing question: 34\n",
            "importing question: 35\n",
            "importing question: 36\n",
            "importing question: 37\n",
            "importing question: 38\n",
            "importing question: 39\n",
            "importing question: 40\n",
            "importing question: 41\n",
            "importing question: 42\n",
            "importing question: 43\n",
            "importing question: 44\n",
            "importing question: 45\n",
            "importing question: 46\n",
            "importing question: 47\n",
            "importing question: 48\n",
            "importing question: 49\n",
            "importing question: 50\n",
            "importing question: 51\n",
            "importing question: 52\n",
            "importing question: 53\n",
            "importing question: 54\n",
            "importing question: 55\n",
            "importing question: 56\n",
            "importing question: 57\n",
            "importing question: 58\n",
            "importing question: 59\n",
            "importing question: 60\n",
            "importing question: 61\n",
            "importing question: 62\n",
            "importing question: 63\n",
            "importing question: 64\n",
            "importing question: 65\n",
            "importing question: 66\n",
            "importing question: 67\n",
            "importing question: 68\n",
            "importing question: 69\n",
            "importing question: 70\n",
            "importing question: 71\n",
            "importing question: 72\n",
            "importing question: 73\n",
            "importing question: 74\n",
            "importing question: 75\n",
            "importing question: 76\n",
            "importing question: 77\n",
            "importing question: 78\n",
            "importing question: 79\n",
            "importing question: 80\n",
            "importing question: 81\n",
            "importing question: 82\n",
            "importing question: 83\n",
            "importing question: 84\n",
            "importing question: 85\n",
            "importing question: 86\n",
            "importing question: 87\n",
            "importing question: 88\n",
            "importing question: 89\n",
            "importing question: 90\n",
            "importing question: 91\n",
            "importing question: 92\n",
            "importing question: 93\n",
            "importing question: 94\n",
            "importing question: 95\n",
            "importing question: 96\n",
            "importing question: 97\n",
            "importing question: 98\n",
            "importing question: 99\n",
            "importing question: 100\n",
            "importing question: 101\n",
            "importing question: 102\n",
            "importing question: 103\n",
            "importing question: 104\n",
            "importing question: 105\n",
            "importing question: 106\n",
            "importing question: 107\n",
            "importing question: 108\n",
            "importing question: 109\n",
            "importing question: 110\n",
            "importing question: 111\n",
            "importing question: 112\n",
            "importing question: 113\n",
            "importing question: 114\n",
            "importing question: 115\n",
            "importing question: 116\n",
            "importing question: 117\n",
            "importing question: 118\n",
            "importing question: 119\n",
            "importing question: 120\n",
            "importing question: 121\n",
            "importing question: 122\n",
            "importing question: 123\n",
            "importing question: 124\n",
            "importing question: 125\n",
            "importing question: 126\n",
            "importing question: 127\n",
            "importing question: 128\n",
            "importing question: 129\n",
            "importing question: 130\n",
            "importing question: 131\n",
            "importing question: 132\n",
            "importing question: 133\n",
            "importing question: 134\n",
            "importing question: 135\n",
            "importing question: 136\n",
            "importing question: 137\n",
            "importing question: 138\n",
            "importing question: 139\n",
            "importing question: 140\n",
            "importing question: 141\n",
            "importing question: 142\n",
            "importing question: 143\n",
            "importing question: 144\n",
            "importing question: 145\n",
            "importing question: 146\n",
            "importing question: 147\n",
            "importing question: 148\n",
            "importing question: 149\n",
            "importing question: 150\n",
            "importing question: 151\n",
            "importing question: 152\n",
            "importing question: 153\n",
            "importing question: 154\n",
            "importing question: 155\n",
            "importing question: 156\n",
            "importing question: 157\n",
            "importing question: 158\n",
            "importing question: 159\n",
            "importing question: 160\n",
            "importing question: 161\n",
            "importing question: 162\n",
            "importing question: 163\n",
            "importing question: 164\n",
            "importing question: 165\n",
            "importing question: 166\n",
            "importing question: 167\n",
            "importing question: 168\n",
            "importing question: 169\n",
            "importing question: 170\n",
            "importing question: 171\n",
            "importing question: 172\n",
            "importing question: 173\n",
            "importing question: 174\n",
            "importing question: 175\n",
            "importing question: 176\n",
            "importing question: 177\n",
            "importing question: 178\n",
            "importing question: 179\n",
            "importing question: 180\n",
            "importing question: 181\n",
            "importing question: 182\n",
            "importing question: 183\n",
            "importing question: 184\n",
            "importing question: 185\n",
            "importing question: 186\n",
            "importing question: 187\n",
            "importing question: 188\n",
            "importing question: 189\n",
            "importing question: 190\n",
            "importing question: 191\n",
            "importing question: 192\n",
            "importing question: 193\n",
            "importing question: 194\n",
            "importing question: 195\n",
            "importing question: 196\n",
            "importing question: 197\n",
            "importing question: 198\n",
            "importing question: 199\n",
            "importing question: 200\n",
            "importing question: 201\n",
            "importing question: 202\n",
            "importing question: 203\n",
            "importing question: 204\n",
            "importing question: 205\n",
            "importing question: 206\n",
            "importing question: 207\n",
            "importing question: 208\n",
            "importing question: 209\n",
            "importing question: 210\n",
            "importing question: 211\n",
            "importing question: 212\n",
            "importing question: 213\n",
            "importing question: 214\n",
            "importing question: 215\n",
            "importing question: 216\n",
            "importing question: 217\n",
            "importing question: 218\n",
            "importing question: 219\n",
            "importing question: 220\n",
            "importing question: 221\n",
            "importing question: 222\n",
            "importing question: 223\n",
            "importing question: 224\n",
            "importing question: 225\n",
            "importing question: 226\n",
            "importing question: 227\n",
            "importing question: 228\n",
            "importing question: 229\n",
            "importing question: 230\n",
            "importing question: 231\n",
            "importing question: 232\n",
            "importing question: 233\n",
            "importing question: 234\n",
            "importing question: 235\n",
            "importing question: 236\n",
            "importing question: 237\n",
            "importing question: 238\n",
            "importing question: 239\n",
            "importing question: 240\n",
            "importing question: 241\n",
            "importing question: 242\n",
            "importing question: 243\n",
            "importing question: 244\n",
            "importing question: 245\n",
            "importing question: 246\n",
            "importing question: 247\n",
            "importing question: 248\n",
            "importing question: 249\n",
            "importing question: 250\n",
            "importing question: 251\n",
            "importing question: 252\n",
            "importing question: 253\n",
            "importing question: 254\n",
            "importing question: 255\n",
            "importing question: 256\n",
            "importing question: 257\n",
            "importing question: 258\n",
            "importing question: 259\n",
            "importing question: 260\n",
            "importing question: 261\n",
            "importing question: 262\n",
            "importing question: 263\n"
          ]
        }
      ],
      "source": [
        "# Configure a batch process\n",
        "with client.batch as batch:\n",
        "    batch.batch_size=256\n",
        "    # Batch import all Questions\n",
        "    for i, d in enumerate(jsl_data):\n",
        "        print(f\"importing question: {i+1}\")\n",
        "\n",
        "        properties = {\n",
        "            \"answer\": d[\"Text\"]\n",
        "        }\n",
        "\n",
        "        client.batch.add_data_object(properties, \"JSL_Document\", vector=d[\"Vector\"])"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "O4QW-dpmVLwX"
      },
      "source": [
        "Let's say you want to find questions related to `Confidential Information` . We can do that by obtaining a vector embedding for `Confidential Information`, and finding objects nearest to it. In this example, we've used spark NLP for creating the legal embeddings. Then, in the following query, we pass that vector to the nearVector operator:"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "H2CKrsbJTWsA"
      },
      "outputs": [],
      "source": [
        "# text for checking\n",
        "\n",
        "text = 'Confidential information'"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "6YcTwE1qTr4J"
      },
      "outputs": [],
      "source": [
        "data = spark.createDataFrame([[text]]) \\\n",
        "    .toDF(\"text\")\n",
        "\n",
        "pipeline = nlp.Pipeline().setStages([\n",
        "    document_assembler,\n",
        "    sentence_embeddings,\n",
        "    embeddingsFinisher\n",
        "]).fit(data)\n",
        "\n",
        "result = pipeline.transform(data)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "gC6_GidtTr4J"
      },
      "outputs": [],
      "source": [
        "resultWithSize = result.selectExpr(\"explode(finished_sentence_embeddings) as embeddings\").toPandas()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "Xqy83IlvULCY"
      },
      "outputs": [],
      "source": [
        "df_test = resultWithSize.copy()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ibtX4qJBTr4K",
        "outputId": "bd363a10-4e83-445d-a01a-25d7f9ecb093"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "0    [-0.8694421052932739, 0.5934111475944519, 1.0,...\n",
              "Name: embeddings, dtype: object"
            ]
          },
          "execution_count": 73,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "# Convert the embeddings from Dense Vector to a list using apply and tolist()\n",
        "# df['embeddings'] = df['embeddings'].apply(lambda x: x.values.tolist())\n",
        "\n",
        "emb = df_test['embeddings']\n",
        "emb"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "TvgWHXLlUZAI"
      },
      "outputs": [],
      "source": [
        "dicti = {'vector':emb[0]}\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "jzDqlZESLzZa",
        "outputId": "23b08b03-6703-4060-e0de-791cd990dcb8"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "{\n",
            "  \"data\": {\n",
            "    \"Get\": {\n",
            "      \"JSL_Document\": [\n",
            "        {\n",
            "          \"answer\": [\n",
            "            \"In addition, this\\n     Agreement and its terms, and the fact and substance of discussions between\\n     the parties concerning the Subject Matter, shall be deemed to be\\n     Confidential Information\"\n",
            "          ]\n",
            "        },\n",
            "        {\n",
            "          \"answer\": [\n",
            "            \"Company may\\nnot sell, transfer, assign, sublicense, or subcontract any right or obligation\\nhereunder without the prior written consent of ASG.\"\n",
            "          ]\n",
            "        },\n",
            "        {\n",
            "          \"answer\": [\n",
            "            \"Assignment. WWC shall not assign this Agreement or its rights or\\n    obligations herein without the prior written consent of WWI.\"\n",
            "          ]\n",
            "        },\n",
            "        {\n",
            "          \"answer\": [\n",
            "            \"This Agreement shall be governed by and construed in accordance with the\\n     laws of the State of Oklahoma, without regard to their conflict of laws\\n     provisions\"\n",
            "          ]\n",
            "        },\n",
            "        {\n",
            "          \"answer\": [\n",
            "            \"In order to facilitate the consideration and negotiation of a possible transaction involving the Xxxxxxxx and Peet\\u2019s, each Party has requested access to certain non-public information regarding the other Party and the other Party\\u2019s subsidiaries\"\n",
            "          ]\n",
            "        },\n",
            "        {\n",
            "          \"answer\": [\n",
            "            \"The recipient shall not hire or employ any of the disclosing party's employees in the European Union for a period of 4 years following the termination of this agreement.\"\n",
            "          ]\n",
            "        },\n",
            "        {\n",
            "          \"answer\": [\n",
            "            \"The recipient shall not, during the term of this agreement and for a period of 3 years following its termination, provide services to any person or entity that competes with the disclosing party in France.\"\n",
            "          ]\n",
            "        },\n",
            "        {\n",
            "          \"answer\": [\n",
            "            \"For the purpose of this Agreement, the term \\\"Evaluation\\nMaterial\\\" shall mean any information, data and knowledge concerning the\\nDisclosing Party, regardless of form, which is delivered or disclosed by or on\\nbehalf of the Disclosing Party to the Recipient in writing, orally or through\\nvisual means, or which the Recipient learns or obtains orally, through\\nobservation, or through analysis of such information, data or knowledge in\\nconnection with the Recipient's consideration of the Proposed Transaction\"\n",
            "          ]\n",
            "        },\n",
            "        {\n",
            "          \"answer\": [\n",
            "            \"For the purpose of this Agreement, the term \\\"Evaluation\\nMaterial\\\" shall mean any information, data and knowledge concerning the\\nDisclosing Party, regardless of form, which is delivered or disclosed by or on\\nbehalf of the Disclosing Party to the Recipient in writing, orally or through\\nvisual means, or which the Recipient learns or obtains orally, through\\nobservation, or through analysis of such information, data or knowledge in\\nconnection with the Recipient's consideration of the Proposed Transaction\"\n",
            "          ]\n",
            "        },\n",
            "        {\n",
            "          \"answer\": [\n",
            "            \"Remedies. Recipient acknowledges and agrees that money damages will not be a\\nsufficient remedy for any breach of this Agreement by Recipient or the Recipient\\u2019s Representatives and that the Disclosing Party shall be entitled to seek specific performance as a remedy for any such breach. Such remedy shall not be deemed to\\nbe the exclusive remedy for Recipient\\u2019s breach of this Agreement, but shall be in addition to all other remedies available at law or equity to the Disclosing Party\"\n",
            "          ]\n",
            "        }\n",
            "      ]\n",
            "    }\n",
            "  }\n",
            "}\n"
          ]
        }
      ],
      "source": [
        "response = (\n",
        "    client.query\n",
        "    .get(\"JSL_Document\", [\"answer\"])\n",
        "    .with_near_vector(\n",
        "            dicti)\n",
        "    .with_limit(10)\n",
        "    .do()\n",
        ")\n",
        "\n",
        "print(json.dumps(response, indent=2))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "HUdm4xbQ8ZVq"
      },
      "outputs": [],
      "source": [
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "MTqcYba-Oeu5"
      },
      "outputs": [],
      "source": []
    }
  ],
  "metadata": {
    "colab": {
      "machine_shape": "hm",
      "provenance": []
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}