evaluate.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/meron/Documents/work/tenacademy/week11/RAG_implementation_LizziAI/venv/lib/python3.10/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
      "  from .autonotebook import tqdm as notebook_tqdm\n"
     ]
    }
   ],
   "source": [
    "\n",
    "\n",
    "import os\n",
    "os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'lizzy_AI.settings')\n",
    "import django\n",
    "django.setup()\n",
    "\n",
    "from RAG_evaluation.create_and_evaluate_ragas_dataset import create_ragas_dataset, evaluate_ragas_dataset\n",
    "\n",
    "from RAG_evaluation.read_qa import read_docx\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 10/10 [00:23<00:00,  2.38s/it]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "Dataset({\n",
       "    features: ['question', 'answer', 'contexts', 'ground_truths'],\n",
       "    num_rows: 10\n",
       "})"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\n",
    "# file_path = \"/home/meron/Documents/work/tenacademy/week11/RAG_implementation_LizziAI/data/Raptor_Q&A2.docx\"\n",
    "file_path = \"/home/meron/Documents/work/tenacademy/week11/RAG_implementation_LizziAI/data/Robinson_Q&A.docx\"\n",
    "\n",
    "# document_name = 'docx/Raptor_Contract_MH5vEw0.docx'\n",
    "document_name = 'docx/Robinson_Advisory_opgMThn.docx'\n",
    "\n",
    "content = read_docx(file_path)\n",
    "# content\n",
    "basic_qa_ragas_dataset = await create_ragas_dataset(content, document_name=document_name)\n",
    "basic_qa_ragas_dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'question': 'Q1: Who are the parties to the Agreement and what are their defined names?',\n",
       " 'answer': 'The parties to the Agreement are Cloud Investments Ltd and the Advisor, Jack Robinson. The defined names are Silvan Joseph as the CEO of Cloud Investments Ltd and Jack Robinson as the Advisor.',\n",
       " 'contexts': ['Personnel: The Advisor may provide the Services only directly or through employees, contractors or agents (“Personnel”), which were specifically approved by the Company, and have committed in writing to confidentiality and ownership rights towards the Advisor and/or the Company, in a form restrictive enough to allow Advisor to comply and enforce the commitments under this undertaking (“Authorized Personnel”). Without limiting the foregoing, the Advisor is liable hereunder for any breach of this undertaking by any Advisor Personnel.\\nThird Parties: The Advisor represents that the performance of the Services and the terms of this Undertaking, do not and will not breach any right, contractual or other, of any employer, customer or other third party. Advisor acknowledges that the Company is relying upon the truthfulness and accuracy of such representations in engaging with the Advisor. Advisor specifically undertakes not to use any proprietary and/or confidential information belonging to the Advisor and/or any third party while performing the Services and in case any Advisor proprietary or confidential information is used, such information will be automatically assigned to the Company and become its full and exclusive property.',\n",
       "  'to (i) comply with all the Company\\'s policies, procedures and objectives, as in effect from time to time; (ii) adhere to any applicable law or provision, pertaining to his services; (iii) not take any action with the intention that the same will cause material damage to the good name of the Company; (iv) keep the contents of this Agreement confidential and not to disclose the existence or contents of this Agreement to any third party without the prior written consent of the Company.Time Tracking: Advisor shall provide the Company with a written report, in a format acceptable by the Company, setting forth the number of hours in which he provided the Services, on a daily basis, as well as an aggregated monthly report at the last day of each calendar month.No Conflicts: Advisor represents that there is no contractual or other legal restriction preventing or limiting him/her from entering into this Agreement or performing any of his/her obligations hereunder. Advisor shall immediately and without delay inform the Company of any affairs and/or matters that might constitute a conflict of interest with his/her services to Company and/or the interests of the Company. Advisor agrees to indemnify, defend and hold harmless the Company and its directors, officers and employees against all claims, liabilities, and costs, including reasonable attorneys\\' fees in connection with any breach by Advisor, of any obligations pursuant to a prior or existing engagement with any other third party, including without limitation other employers or clients.Term: The term of this Agreement shall commence on the Effective Date and shall continue until terminated in accordance with the provisions herein (the \"Term\").  Termination: Either party, at any given time, may terminate this Agreement, for any reason whatsoever, with or without cause, upon fourteen (14) days’ prior written notice. Notwithstanding the above, the Company may terminate this Agreement immediately and without prior notice if',\n",
       "  'Joseph\\t\\t\\t\\tName:\\tJack Robinson\\t\\tTitle: CEO\\t\\t\\t\\t\\tConfidentiality, None Compete and IP Ownership UndertakingAppendix A to Advisory Service Agreement as of June 15th, 2023THIS CONFIDENTIALITY UNDERTAKING (“Undertaking”) is entered into as of June 15th, 2023 (“Effective Date”), by Mr. Jack Robinson, Passport Number 780055578, residing at 1 Rabin st, Tel Aviv, Israel, Email: jackrobinson@gmail.com, (“Advisor”), towards Cloud Investments Ltd (“Company”), as follows:Definitions: (a) Company’s Business: development of an AI-based contract assistant. (b) Services: Software development services and/or any other services provided by the Advisor to the Company. (c) Work Product: Any product of the Services, tangible and/or intangible, including any inventions, discoveries and/or other intellectual property. (d) Confidential Information: any information disclosed by the Company to the Advisor and/or created as a result of the Services (including Work Product), of any type, form (including orally) or media, including (without limitation), code, specifications, architecture, design, data, algorithms, business plans, budget, customers / suppliers lists etc.), but excluding information which prior to the disclosure was publicly available or known to the Advisor (both without breach of any confidentiality obligation towards the Company).',\n",
       "  'Jurisdiction: This Undertaking shall be governed by and construed in accordance with the laws of the State of Israel, without giving effect to its laws pertaining to conflict of laws.\\xa0 The Advisor agrees that any and all disputes in connection with this Undertaking shall be submitted to the exclusive jurisdiction of the competent courts in the city of Tel Aviv-Yafo, Israel.Pre-Undertaking: In case the Advisor has been providing Services to the Company before the Effective Date (“Pre-Undertaking Period”), this Undertaking shall also apply to the Pre-Undertaking Period, including, if relevant, before Company’s incorporation. The Company may assign this undertaking to third parties.IN WITNESS WHEREOF, the undersigned has executed this Undertaking as of the Effective Date.',\n",
       "  'the Term. Advisor further consents that the Company may offset any amounts due to him under this Section from any amounts payable to Advisor under this Agreement. Advisor shall indemnify the Company for any loss or expenses incurred by the Company if it were determined that an alleged employer/employee relationship existed between the Advisor and the Company. Entire Agreement; No Waiver or Assignment: This Agreement together with the Exhibits, which are attached hereto and incorporated herein, set forth the entire Agreement between the parties and shall supersede all previous communications and agreements between the parties, either oral or written. This Agreement may be modified only by a written amendment executed by both parties. This Agreement may not be assigned, sold, delegated or transferred in any manner by Advisor for any reason whatsoever. The Company may assign the Agreement to a successor of all or substantially all of its assets or business, provided the assignee has assumed the Company’s obligations under this Agreement.  Governing Law and Jurisdiction:  This Agreement shall be governed by the laws of the State of Israel, without giving effect to the rules respecting conflicts of laws. The parties consent to the exclusive jurisdiction and venue of Tel Aviv courts for any lawsuit filed arising from or relating to this Agreement.  \\tNotices: Notices under this Agreement shall be delivered to the party’s email address as follows: Company: info@cloudcorp.com, Advisor: jackrobinson@gmail.com, or in any the other means with a proof of acceptance by the other party.IN WITNESS WHEREOF the parties have executed this Agreement as of the date first above written.Cloud Investments Ltd.\\t\\t\\t\\tAdvisor By: ________________________\\t\\tBy:________________________Name:\\tSilvan Joseph\\t\\t\\t\\tName:\\tJack Robinson\\t\\tTitle: CEO\\t\\t\\t\\t\\tConfidentiality, None Compete and IP Ownership UndertakingAppendix A to Advisory Service Agreement as of June 15th, 2023THIS CONFIDENTIALITY UNDERTAKING'],\n",
       " 'ground_truths': ['A1:  Cloud Investments Ltd. (“Company”) and Jack Robinson (“Advisor”)']}"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "basic_qa_ragas_dataset[0]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# # Assuming basic_qa_ragas_dataset is a list of dictionaries\n",
    "# for item in basic_qa_ragas_dataset:\n",
    "#     item[\"answer\"] = str(item[\"answer\"])\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "passing column names as 'ground_truths' is deprecated and will be removed in the next version, please use 'ground_truth' instead. Note that `ground_truth` should be of type string and not Sequence[string] like `ground_truths`\n",
      "Evaluating:   0%|          | 0/70 [00:00<?, ?it/s]Task exception was never retrieved\n",
      "future: <Task finished name='Task-204' coro=<AsyncClient.aclose() done, defined at /home/meron/Documents/work/tenacademy/week11/RAG_implementation_LizziAI/venv/lib/python3.10/site-packages/httpx/_client.py:1996> exception=RuntimeError('Event loop is closed')>\n",
      "Traceback (most recent call last):\n",
      "  File \"/home/meron/Documents/work/tenacademy/week11/RAG_implementation_LizziAI/venv/lib/python3.10/site-packages/httpx/_client.py\", line 2003, in aclose\n",
      "    await self._transport.aclose()\n",
      "  File \"/home/meron/Documents/work/tenacademy/week11/RAG_implementation_LizziAI/venv/lib/python3.10/site-packages/httpx/_transports/default.py\", line 383, in aclose\n",
      "    await self._pool.aclose()\n",
      "  File \"/home/meron/Documents/work/tenacademy/week11/RAG_implementation_LizziAI/venv/lib/python3.10/site-packages/httpcore/_async/connection_pool.py\", line 313, in aclose\n",
      "    await self._close_connections(closing_connections)\n",
      "  File \"/home/meron/Documents/work/tenacademy/week11/RAG_implementation_LizziAI/venv/lib/python3.10/site-packages/httpcore/_async/connection_pool.py\", line 305, in _close_connections\n",
      "    await connection.aclose()\n",
      "  File \"/home/meron/Documents/work/tenacademy/week11/RAG_implementation_LizziAI/venv/lib/python3.10/site-packages/httpcore/_async/connection.py\", line 171, in aclose\n",
      "    await self._connection.aclose()\n",
      "  File \"/home/meron/Documents/work/tenacademy/week11/RAG_implementation_LizziAI/venv/lib/python3.10/site-packages/httpcore/_async/http11.py\", line 252, in aclose\n",
      "    await self._network_stream.aclose()\n",
      "  File \"/home/meron/Documents/work/tenacademy/week11/RAG_implementation_LizziAI/venv/lib/python3.10/site-packages/httpcore/_backends/anyio.py\", line 54, in aclose\n",
      "    await self._stream.aclose()\n",
      "  File \"/home/meron/Documents/work/tenacademy/week11/RAG_implementation_LizziAI/venv/lib/python3.10/site-packages/anyio/streams/tls.py\", line 202, in aclose\n",
      "    await self.transport_stream.aclose()\n",
      "  File \"/home/meron/Documents/work/tenacademy/week11/RAG_implementation_LizziAI/venv/lib/python3.10/site-packages/anyio/_backends/_asyncio.py\", line 1191, in aclose\n",
      "    self._transport.close()\n",
      "  File \"/usr/lib/python3.10/asyncio/selector_events.py\", line 706, in close\n",
      "    self._loop.call_soon(self._call_connection_lost, None)\n",
      "  File \"/usr/lib/python3.10/asyncio/base_events.py\", line 753, in call_soon\n",
      "    self._check_closed()\n",
      "  File \"/usr/lib/python3.10/asyncio/base_events.py\", line 515, in _check_closed\n",
      "    raise RuntimeError('Event loop is closed')\n",
      "RuntimeError: Event loop is closed\n",
      "Task exception was never retrieved\n",
      "future: <Task finished name='Task-205' coro=<AsyncClient.aclose() done, defined at /home/meron/Documents/work/tenacademy/week11/RAG_implementation_LizziAI/venv/lib/python3.10/site-packages/httpx/_client.py:1996> exception=RuntimeError('Event loop is closed')>\n",
      "Traceback (most recent call last):\n",
      "  File \"/home/meron/Documents/work/tenacademy/week11/RAG_implementation_LizziAI/venv/lib/python3.10/site-packages/httpx/_client.py\", line 2003, in aclose\n",
      "    await self._transport.aclose()\n",
      "  File \"/home/meron/Documents/work/tenacademy/week11/RAG_implementation_LizziAI/venv/lib/python3.10/site-packages/httpx/_transports/default.py\", line 383, in aclose\n",
      "    await self._pool.aclose()\n",
      "  File \"/home/meron/Documents/work/tenacademy/week11/RAG_implementation_LizziAI/venv/lib/python3.10/site-packages/httpcore/_async/connection_pool.py\", line 313, in aclose\n",
      "    await self._close_connections(closing_connections)\n",
      "  File \"/home/meron/Documents/work/tenacademy/week11/RAG_implementation_LizziAI/venv/lib/python3.10/site-packages/httpcore/_async/connection_pool.py\", line 305, in _close_connections\n",
      "    await connection.aclose()\n",
      "  File \"/home/meron/Documents/work/tenacademy/week11/RAG_implementation_LizziAI/venv/lib/python3.10/site-packages/httpcore/_async/connection.py\", line 171, in aclose\n",
      "    await self._connection.aclose()\n",
      "  File \"/home/meron/Documents/work/tenacademy/week11/RAG_implementation_LizziAI/venv/lib/python3.10/site-packages/httpcore/_async/http11.py\", line 252, in aclose\n",
      "    await self._network_stream.aclose()\n",
      "  File \"/home/meron/Documents/work/tenacademy/week11/RAG_implementation_LizziAI/venv/lib/python3.10/site-packages/httpcore/_backends/anyio.py\", line 54, in aclose\n",
      "    await self._stream.aclose()\n",
      "  File \"/home/meron/Documents/work/tenacademy/week11/RAG_implementation_LizziAI/venv/lib/python3.10/site-packages/anyio/streams/tls.py\", line 202, in aclose\n",
      "    await self.transport_stream.aclose()\n",
      "  File \"/home/meron/Documents/work/tenacademy/week11/RAG_implementation_LizziAI/venv/lib/python3.10/site-packages/anyio/_backends/_asyncio.py\", line 1191, in aclose\n",
      "    self._transport.close()\n",
      "  File \"/usr/lib/python3.10/asyncio/selector_events.py\", line 706, in close\n",
      "    self._loop.call_soon(self._call_connection_lost, None)\n",
      "  File \"/usr/lib/python3.10/asyncio/base_events.py\", line 753, in call_soon\n",
      "    self._check_closed()\n",
      "  File \"/usr/lib/python3.10/asyncio/base_events.py\", line 515, in _check_closed\n",
      "    raise RuntimeError('Event loop is closed')\n",
      "RuntimeError: Event loop is closed\n",
      "Evaluating:  41%|████▏     | 29/70 [00:14<00:20,  2.02it/s]Runner in Executor raised an exception\n",
      "Traceback (most recent call last):\n",
      "  File \"/home/meron/Documents/work/tenacademy/week11/RAG_implementation_LizziAI/venv/lib/python3.10/site-packages/ragas/executor.py\", line 58, in _aresults\n",
      "    r = await future\n",
      "  File \"/usr/lib/python3.10/asyncio/tasks.py\", line 571, in _wait_for_one\n",
      "    return f.result()  # May raise f.exception().\n",
      "  File \"/home/meron/Documents/work/tenacademy/week11/RAG_implementation_LizziAI/venv/lib/python3.10/site-packages/ragas/executor.py\", line 91, in wrapped_callable_async\n",
      "    return counter, await callable(*args, **kwargs)\n",
      "  File \"/home/meron/Documents/work/tenacademy/week11/RAG_implementation_LizziAI/venv/lib/python3.10/site-packages/ragas/metrics/base.py\", line 91, in ascore\n",
      "    raise e\n",
      "  File \"/home/meron/Documents/work/tenacademy/week11/RAG_implementation_LizziAI/venv/lib/python3.10/site-packages/ragas/metrics/base.py\", line 87, in ascore\n",
      "    score = await self._ascore(row=row, callbacks=group_cm, is_async=is_async)\n",
      "  File \"/home/meron/Documents/work/tenacademy/week11/RAG_implementation_LizziAI/venv/lib/python3.10/site-packages/ragas/metrics/_faithfulness.py\", line 190, in _ascore\n",
      "    assert isinstance(statements, dict), \"Invalid JSON response\"\n",
      "AssertionError: Invalid JSON response\n",
      "Evaluating: 100%|██████████| 70/70 [00:16<00:00,  4.23it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'context_precision': 0.8375, 'faithfulness': 0.8935, 'answer_relevancy': 0.9169, 'context_recall': 0.9718, 'context_relevancy': 0.2266, 'answer_correctness': 0.4516, 'answer_similarity': 0.8444}\n"
     ]
    }
   ],
   "source": [
    "\n",
    "eval_result = evaluate_ragas_dataset(basic_qa_ragas_dataset)\n",
    "print(eval_result)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Robinson_eval_metrics</th>\n",
       "      <th>average_Value</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>context_precision</td>\n",
       "      <td>0.837521</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>faithfulness</td>\n",
       "      <td>0.893472</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>answer_relevancy</td>\n",
       "      <td>0.916916</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>context_recall</td>\n",
       "      <td>0.971791</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>context_relevancy</td>\n",
       "      <td>0.226575</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>answer_correctness</td>\n",
       "      <td>0.451650</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>answer_similarity</td>\n",
       "      <td>0.844361</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  Robinson_eval_metrics  average_Value\n",
       "0     context_precision       0.837521\n",
       "1          faithfulness       0.893472\n",
       "2      answer_relevancy       0.916916\n",
       "3        context_recall       0.971791\n",
       "4     context_relevancy       0.226575\n",
       "5    answer_correctness       0.451650\n",
       "6     answer_similarity       0.844361"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "df_eval = pd.DataFrame(list(eval_result.items()), columns=['Robinson_eval_metrics', 'average_Value'])\n",
    "# df_eval = pd.DataFrame(list(eval_result.items()), columns=['Raptor_eval_metrics', 'average_Value'])\n",
    "\n",
    "df_eval"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "venv",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}