Notebooks/dspy_starter.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\aniru\\AppData\\Local\\Packages\\PythonSoftwareFoundation.Python.3.11_qbz5n2kfra8p0\\LocalCache\\local-packages\\Python311\\site-packages\\tqdm\\auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
      "  from .autonotebook import tqdm as notebook_tqdm\n"
     ]
    }
   ],
   "source": [
    "# Following the tutorial at: https://github.com/stanfordnlp/dspy/blob/main/intro.ipynb\n",
    "# In this tutorial we implement: Vanilla QA, Chain of Thought Reasoning, RAG, MuliHop, MultiHop with RAG\n",
    "\n",
    "import dspy\n",
    "from dspy.datasets import HotPotQA\n",
    "from dspy.teleprompt import BootstrapFewShot\n",
    "from dspy.evaluate.evaluate import Evaluate\n",
    "from dsp.utils import deduplicate\n",
    "\n",
    "\n",
    "# Reading API Key, ideally we would set it to an env variable\n",
    "api_key = open(\"../api_key\").read()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Using OpenAI GPT-3.5 as the langugage model and colbertv2 as the retreival model\n",
    "turbo = dspy.OpenAI(model='gpt-3.5-turbo', api_key=api_key)\n",
    "colbertv2_wiki17_abstracts = dspy.ColBERTv2(url='http://20.102.90.50:2017/wiki17_abstracts')\n",
    "\n",
    "dspy.settings.configure(lm=turbo, rm=colbertv2_wiki17_abstracts)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\aniru\\AppData\\Local\\Packages\\PythonSoftwareFoundation.Python.3.11_qbz5n2kfra8p0\\LocalCache\\local-packages\\Python311\\site-packages\\datasets\\table.py:1421: FutureWarning: promote has been superseded by promote_options='default'.\n",
      "  table = cls._concat_blocks(blocks, axis=0)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "20\n",
      "50\n",
      "For this dataset, training examples have input keys ['question'] and label keys ['answer']\n",
      "For this dataset, dev examples have input keys ['question'] and label keys ['answer', 'gold_titles']\n"
     ]
    }
   ],
   "source": [
    "# Load the QA dataset\n",
    "dataset = HotPotQA(train_seed=1, train_size=20, eval_seed=2023, dev_size=50, test_size=0)\n",
    "\n",
    "# Tell DSPy that the 'question' field is the input. Any other fields are labels and/or metadata.\n",
    "trainset = [x.with_inputs('question') for x in dataset.train]\n",
    "devset = [x.with_inputs('question') for x in dataset.dev]\n",
    "\n",
    "print(len(trainset))\n",
    "print(len(devset))\n",
    "\n",
    "train_example = trainset[0]\n",
    "dev_example = devset[18]\n",
    "print(f\"For this dataset, training examples have input keys {train_example.inputs().keys()} and label keys {train_example.labels().keys()}\")\n",
    "print(f\"For this dataset, dev examples have input keys {dev_example.inputs().keys()} and label keys {dev_example.labels().keys()}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Question: What is the nationality of the chef and restaurateur featured in Restaurant: Impossible?\n",
      "Predicted Answer: American\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "Answer questions with short factoid answers.\n",
      "\n",
      "---\n",
      "\n",
      "Follow the following format.\n",
      "\n",
      "Question: ${question}\n",
      "Answer: often between 1 and 5 words\n",
      "\n",
      "---\n",
      "\n",
      "Question: What is the nationality of the chef and restaurateur featured in Restaurant: Impossible?\n",
      "Answer:\u001b[32m American\u001b[0m\n",
      "\n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# Setting up and testing basic QA\n",
    "class BasicQA(dspy.Signature):\n",
    "    \"\"\"Answer questions with short factoid answers.\"\"\"\n",
    "\n",
    "    question = dspy.InputField()\n",
    "    answer = dspy.OutputField(desc=\"often between 1 and 5 words\")\n",
    "\n",
    "# Simple Text Genreation\n",
    "generate_answer = dspy.Predict(BasicQA)\n",
    "\n",
    "# Call the predictor on a particular input.\n",
    "pred = generate_answer(question=dev_example.question)\n",
    "\n",
    "# Print the input and the prediction.\n",
    "print(f\"Question: {dev_example.question}\")\n",
    "print(f\"Predicted Answer: {pred.answer}\")\n",
    "\n",
    "# Inspecting history\n",
    "turbo.inspect_history(n=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Question: What is the nationality of the chef and restaurateur featured in Restaurant: Impossible?\n",
      "Thought: We know that the chef and restaurateur featured in Restaurant: Impossible is Robert Irvine.\n",
      "Predicted Answer: British\n"
     ]
    }
   ],
   "source": [
    "# Chain of Thought rationale for QA\n",
    "cot = dspy.ChainOfThought(BasicQA)\n",
    "\n",
    "# Call the predictor on the same input.\n",
    "pred = cot(question=dev_example.question)\n",
    "\n",
    "# Print the input, the chain of thought, and the prediction.\n",
    "print(f\"Question: {dev_example.question}\")\n",
    "print(f\"Thought: {pred.rationale.split('.', 1)[1].strip()}\")\n",
    "print(f\"Predicted Answer: {pred.answer}\")\n",
    "\n",
    "# Check history\n",
    "# turbo.inspect_history(n=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Top 3 passages for question: What is the nationality of the chef and restaurateur featured in Restaurant: Impossible? \n",
      " ------------------------------ \n",
      "\n",
      "1] Restaurant: Impossible | Restaurant: Impossible is an American reality television series, featuring chef and restaurateur Robert Irvine, that aired on Food Network from 2011 to 2016. \n",
      "\n",
      "2] Jean Joho | Jean Joho is a French-American chef and restaurateur. He is chef/proprietor of Everest in Chicago (founded in 1986), Paris Club Bistro & Bar and Studio Paris in Chicago, The Eiffel Tower Restaurant in Las Vegas, and Brasserie JO in Boston. \n",
      "\n",
      "3] List of Restaurant: Impossible episodes | This is the list of the episodes for the American cooking and reality television series \"Restaurant Impossible\", produced by Food Network. The premise of the series is that within two days and on a budget of $10,000, celebrity chef Robert Irvine renovates a failing American restaurant with the goal of helping to restore it to profitability and prominence. Irvine is assisted by a designer (usually Taniya Nayak, Cheryl Torrenueva, or Lynn Keagan, but sometimes Vanessa De Leon, Krista Watterworth, Yvette Irene, or Nicole Faccuito), along with general contractor Tom Bury, who sometimes does double duty as both general contractor and designer. After assessing the problems with the restaurant, Robert Irvine typically creates a plan for the new decor, oversees the cleaning of the restaurant, reduces the size of the menu and improves the food, develops a promotional activity, educates the restaurant's owners, or trains the staff, as needed by each restaurant. \n",
      "\n"
     ]
    }
   ],
   "source": [
    "# Test Retreival Model\n",
    "retrieve = dspy.Retrieve(k=3)\n",
    "topK_passages = retrieve(dev_example.question).passages\n",
    "\n",
    "print(f\"Top {retrieve.k} passages for question: {dev_example.question} \\n\", '-' * 30, '\\n')\n",
    "\n",
    "for idx, passage in enumerate(topK_passages):\n",
    "    print(f'{idx+1}]', passage, '\\n')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 55%|█████▌    | 11/20 [00:00<00:00, 733.21it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Bootstrapped 4 full traces after 12 examples in round 0.\n",
      "Question: What castle did David Gregory inherit?\n",
      "Predicted Answer: Kinnairdy Castle\n",
      "Retrieved Contexts (truncated): ['David Gregory (physician) | David Gregory (20 December 1625 – 1720) was a Scottish physician and inventor. His surname is sometimes spelt as Gregorie, the original Scottish spelling. He inherited Kinn...', 'Gregory Tarchaneiotes | Gregory Tarchaneiotes (Greek: Γρηγόριος Ταρχανειώτης , Italian: \"Gregorio Tracanioto\" or \"Tracamoto\" ) was a \"protospatharius\" and the long-reigning catepan of Italy from 998 t...', 'David Gregory (mathematician) | David Gregory (originally spelt Gregorie) FRS (? 1659 – 10 October 1708) was a Scottish mathematician and astronomer. He was professor of mathematics at the University ...']\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "# Testing RAG\n",
    "class GenerateAnswer(dspy.Signature):\n",
    "    \"\"\"Answer questions with short factoid answers.\"\"\"\n",
    "\n",
    "    context = dspy.InputField(desc=\"may contain relevant facts\")\n",
    "    question = dspy.InputField()\n",
    "    answer = dspy.OutputField(desc=\"often between 1 and 5 words\")\n",
    "\n",
    "class RAG(dspy.Module):\n",
    "    def __init__(self, num_passages=3):\n",
    "        super().__init__()\n",
    "\n",
    "        self.retrieve = dspy.Retrieve(k=num_passages)\n",
    "        self.generate_answer = dspy.ChainOfThought(GenerateAnswer)\n",
    "\n",
    "    def forward(self, question):\n",
    "        context = self.retrieve(question).passages\n",
    "        prediction = self.generate_answer(context=context, question=question)\n",
    "        return dspy.Prediction(context=context, answer=prediction.answer)\n",
    "\n",
    "\n",
    "\n",
    "# Validation logic: check that the predicted answer is correct.\n",
    "# Also check that the retrieved context does actually contain that answer.\n",
    "def validate_context_and_answer(example, pred, trace=None):\n",
    "    answer_EM = dspy.evaluate.answer_exact_match(example, pred)\n",
    "    answer_PM = dspy.evaluate.answer_passage_match(example, pred)\n",
    "    return answer_EM and answer_PM\n",
    "\n",
    "# Set up a basic teleprompter, which will compile our RAG program.\n",
    "teleprompter = BootstrapFewShot(metric=validate_context_and_answer)\n",
    "\n",
    "# Compile!\n",
    "compiled_rag = teleprompter.compile(RAG(), trainset=trainset)\n",
    "\n",
    "# Testing RAG with a prompt\n",
    "my_question = \"What castle did David Gregory inherit?\"\n",
    "\n",
    "# Get the prediction. This contains `pred.context` and `pred.answer`.\n",
    "pred = compiled_rag(my_question)\n",
    "\n",
    "# Print the contexts and the answer.\n",
    "print(f\"Question: {my_question}\")\n",
    "print(f\"Predicted Answer: {pred.answer}\")\n",
    "print(f\"Retrieved Contexts (truncated): {[c[:200] + '...' for c in pred.context]}\")\n",
    "\n",
    "# inspect history\n",
    "# turbo.inspect_history(n=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "generate_answer\n",
      "Example({'augmented': True, 'context': ['Tae Kwon Do Times | Tae Kwon Do Times is a magazine devoted to the martial art of taekwondo, and is published in the United States of America. While the title suggests that it focuses on taekwondo exclusively, the magazine also covers other Korean martial arts. \"Tae Kwon Do Times\" has published articles by a wide range of authors, including He-Young Kimm, Thomas Kurz, Scott Shaw, and Mark Van Schuyver.', \"Kwon Tae-man | Kwon Tae-man (born 1941) was an early Korean hapkido practitioner and a pioneer of the art, first in Korea and then in the United States. He formed one of the earliest dojang's for hapkido in the United States in Torrance, California, and has been featured in many magazine articles promoting the art.\", 'Hee Il Cho | Cho Hee Il (born October 13, 1940) is a prominent Korean-American master of taekwondo, holding the rank of 9th \"dan\" in the martial art. He has written 11 martial art books, produced 70 martial art training videos, and has appeared on more than 70 martial arts magazine covers. Cho won several national and international competitions as a taekwondo competitor, and has appeared in several films, including \"Fight to Win\", \"Best of the Best\", \"Bloodsport II\", and \"Bloodsport III\". He founded the Action International Martial Arts Association (AIMAA) in 1980, and is its President. Cho is a member of both \"Black Belt\" magazine\\'s Hall of Fame and \"Tae Kwon Do Times\" magazine\\'s Hall of Fame.'], 'question': 'Which magazine has published articles by Scott Shaw, Tae Kwon Do Times or Southwest Art?', 'rationale': 'produce the answer. We know from the context that \"Tae Kwon Do Times\" is a magazine that covers taekwondo and other Korean martial arts. It has published articles by authors like Scott Shaw. On the other hand, there is no information about Southwest Art magazine in the context.', 'answer': 'Tae Kwon Do Times'}) (input_keys=None)\n",
      "\n"
     ]
    }
   ],
   "source": [
    "for name, parameter in compiled_rag.named_predictors():\n",
    "    print(name)\n",
    "    print(parameter.demos[0])\n",
    "    print()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\aniru\\AppData\\Local\\Packages\\PythonSoftwareFoundation.Python.3.11_qbz5n2kfra8p0\\LocalCache\\local-packages\\Python311\\site-packages\\dspy\\evaluate\\evaluate.py:137: FutureWarning: DataFrame.applymap has been deprecated. Use DataFrame.map instead.\n",
      "  df = df.applymap(truncate_cell)\n"
     ]
    }
   ],
   "source": [
    "# Testing results on the entire QA dataset\n",
    "\n",
    "# Set up the `evaluate_on_hotpotqa` function. We'll use this many times below.\n",
    "evaluate_on_hotpotqa = Evaluate(devset=devset, num_threads=1, display_progress=True, display_table=5)\n",
    "\n",
    "# Evaluate the `compiled_rag` program with the `answer_exact_match` metric.\n",
    "metric = dspy.evaluate.answer_exact_match\n",
    "evaluate_on_hotpotqa(compiled_rag, metric=metric, display=False)\n",
    "\n",
    "# Check if fold titles are retreived\n",
    "def gold_passages_retrieved(example, pred, trace=None):\n",
    "    gold_titles = set(map(dspy.evaluate.normalize_text, example['gold_titles']))\n",
    "    found_titles = set(map(dspy.evaluate.normalize_text, [c.split(' | ')[0] for c in pred.context]))\n",
    "\n",
    "    return gold_titles.issubset(found_titles)\n",
    "\n",
    "compiled_rag_retrieval_score = evaluate_on_hotpotqa(compiled_rag, metric=gold_passages_retrieved, display=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Question: How many storeys are in the castle that David Gregory inherited?\n",
      "Predicted Answer: Three.\n",
      "Retrieved Contexts (truncated): ['David Gregory (physician) | David Gregory (20 December 1625 – 1720) was a Scottish physician and inventor. His surname is sometimes spelt as Gregorie, the original Scottish spelling. He inherited Kinn...', 'David S. Castle | David S. Castle (13 February 1884 – 28 October 1956) was an architect in Texas....', 'Roughan Castle | Roughan Castle is a castle a mile outside Newmills, County Tyrone, Northern Ireland, on the Dungannon to Stewartstown road. It was built about 1618 by Sir Andrew Stewart (d.1639), 2nd...', 'Gregory Tarchaneiotes | Gregory Tarchaneiotes (Greek: Γρηγόριος Ταρχανειώτης , Italian: \"Gregorio Tracanioto\" or \"Tracamoto\" ) was a \"protospatharius\" and the long-reigning catepan of Italy from 998 t...', 'Gregory Parsloe-Parsloe | Sir Gregory Parsloe-Parsloe, 7th Baronet is a fictional character from the Blandings stories of P. G. Wodehouse. The seventh Baronet, who resides at Matchingham Hall, he is t...']\n"
     ]
    }
   ],
   "source": [
    "# Test Multihop search\n",
    "class GenerateSearchQuery(dspy.Signature):\n",
    "    \"\"\"Write a simple search query that will help answer a complex question.\"\"\"\n",
    "\n",
    "    context = dspy.InputField(desc=\"may contain relevant facts\")\n",
    "    question = dspy.InputField()\n",
    "    query = dspy.OutputField()\n",
    "\n",
    "class SimplifiedBaleen(dspy.Module):\n",
    "    def __init__(self, passages_per_hop=3, max_hops=2):\n",
    "        super().__init__()\n",
    "\n",
    "        self.generate_query = [dspy.ChainOfThought(GenerateSearchQuery) for _ in range(max_hops)]\n",
    "        self.retrieve = dspy.Retrieve(k=passages_per_hop)\n",
    "        self.generate_answer = dspy.ChainOfThought(GenerateAnswer)\n",
    "        self.max_hops = max_hops\n",
    "\n",
    "    def forward(self, question):\n",
    "        context = []\n",
    "\n",
    "        for hop in range(self.max_hops):\n",
    "            query = self.generate_query[hop](context=context, question=question).query\n",
    "            passages = self.retrieve(query).passages\n",
    "            context = deduplicate(context + passages)\n",
    "\n",
    "        pred = self.generate_answer(context=context, question=question)\n",
    "        return dspy.Prediction(context=context, answer=pred.answer)\n",
    "\n",
    "\n",
    "# Testing uncompiled/ zero shot method for multihop\n",
    "my_question = \"How many storeys are in the castle that David Gregory inherited?\"\n",
    "\n",
    "# Get the prediction. This contains `pred.context` and `pred.answer`.\n",
    "uncompiled_baleen = SimplifiedBaleen()  # uncompiled (i.e., zero-shot) program\n",
    "pred = uncompiled_baleen(my_question)\n",
    "\n",
    "# Print the contexts and the answer.\n",
    "print(f\"Question: {my_question}\")\n",
    "print(f\"Predicted Answer: {pred.answer}\")\n",
    "print(f\"Retrieved Contexts (truncated): {[c[:200] + '...' for c in pred.context]}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 90%|█████████ | 18/20 [00:00<00:00, 371.13it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Bootstrapped 4 full traces after 19 examples in round 0.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "def validate_context_and_answer_and_hops(example, pred, trace=None):\n",
    "    if not dspy.evaluate.answer_exact_match(example, pred): return False\n",
    "    if not dspy.evaluate.answer_passage_match(example, pred): return False\n",
    "\n",
    "    hops = [example.question] + [outputs.query for *_, outputs in trace if 'query' in outputs]\n",
    "\n",
    "    if max([len(h) for h in hops]) > 100: return False\n",
    "    if any(dspy.evaluate.answer_exact_match_str(hops[idx], hops[:idx], frac=0.8) for idx in range(2, len(hops))): return False\n",
    "\n",
    "    return True\n",
    "teleprompter = BootstrapFewShot(metric=validate_context_and_answer_and_hops)\n",
    "compiled_baleen = teleprompter.compile(SimplifiedBaleen(), teacher=SimplifiedBaleen(passages_per_hop=2), trainset=trainset)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "uncompiled_baleen_retrieval_score = evaluate_on_hotpotqa(uncompiled_baleen, metric=gold_passages_retrieved, display=False)\n",
    "compiled_baleen_retrieval_score = evaluate_on_hotpotqa(compiled_baleen, metric=gold_passages_retrieved, display=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "## Retrieval Score for RAG: 26.0\n",
      "## Retrieval Score for uncompiled Baleen: 40.0\n",
      "## Retrieval Score for compiled Baleen: 60.0\n"
     ]
    }
   ],
   "source": [
    "print(f\"## Retrieval Score for RAG: {compiled_rag_retrieval_score}\")  # note that for RAG, compilation has no effect on the retrieval step\n",
    "print(f\"## Retrieval Score for uncompiled Baleen: {uncompiled_baleen_retrieval_score}\")\n",
    "print(f\"## Retrieval Score for compiled Baleen: {compiled_baleen_retrieval_score}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Inspect history for a compiled multihop\n",
    "# compiled_baleen(\"How many storeys are in the castle that David Gregory inherited?\")\n",
    "# turbo.inspect_history(n=3)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": ".env",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}