dspy_multiagent_fine_tune.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# for vast ai - enter in terminal\n",
    "!python3 -m pip install ipykernel -U --user --force-reinstall && apt update && apt install -y python3-pip"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "!pip3 install flash-attn --no-build-isolation\n",
    "!pip3 install llama-index llama-parse llama-index-embeddings-huggingface llama-index-llms-huggingface llama-agents dspy-ai openpyxl langchain chromadb\n",
    "!pip3 install sentencepiece protobuf evaluate rouge_score absl-py tensorboardX bitsandbytes peft accelerate\n",
    "!cp /workspace/repos/agentic-ai/MASTER\\ -\\ PYTHON\\ -\\ SCORING\\ MODEL\\ -\\ MCG\\ MADISON\\ RIDGE\\ DST\\ -\\ v2.0.xlsx /workspace/data\n",
    "!cp /workspace/repos/agentic-ai/PPM\\ -\\ MCG\\ MADISON\\ RIDGE\\ DST.pdf /workspace/data\n",
    "!pip3 uninstall -y torch\n",
    "!pip3 install torch torchvision torchaudio --index-url https://download.pytorch.org/whl/cu118"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import gc\n",
    "import os\n",
    "\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "from transformers import BitsAndBytesConfig\n",
    "\n",
    "import dspy\n",
    "from dspy.evaluate import Evaluate\n",
    "from dspy.datasets.hotpotqa import HotPotQA\n",
    "from dspy.teleprompt import BootstrapFewShotWithRandomSearch\n",
    "\n",
    "from llama_index.core import SimpleDirectoryReader, VectorStoreIndex\n",
    "from llama_index.embeddings.huggingface import HuggingFaceEmbedding\n",
    "from llama_index.core import VectorStoreIndex, SimpleDirectoryReader, Settings, StorageContext\n",
    "from llama_index.core.embeddings import resolve_embed_model\n",
    "from llama_parse import LlamaParse\n",
    "from llama_index.llms.huggingface import HuggingFaceLLM\n",
    "\n",
    "import chromadb\n",
    "from chromadb.utils import embedding_functions\n",
    "from langchain.text_splitter import SentenceTransformersTokenTextSplitter\n",
    "from llama_index.readers.file import PandasExcelReader\n",
    "# CHROMA_COLLECTION_NAME = \"blockchain_and_ai\"\n",
    "# CHROMADB_DIR = \"/workspace/data/db/\"\n",
    "# from dspy.retrieve.chromadb_rm import ChromadbRM\n",
    "\n",
    "from typing import List, Any, Callable, Optional\n",
    "from pydantic import BaseModel\n",
    "\n",
    "import torch\n",
    "from transformers import AutoModelForCausalLM\n",
    "\n",
    "from train_utils import get_csv_string, randomize_row_values, operators_dict, range_description_json, split_df_by_empty_columns, split_df_by_empty_rows, print_trainable_parameters\n",
    "from models import SpreadSheetAnalyzer\n",
    "\n",
    "from dotenv import load_dotenv\n",
    "load_dotenv('/workspace/repos/agentic-ai/.env')\n",
    "\n",
    "import nest_asyncio\n",
    "nest_asyncio.apply()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "filepath = \"/workspace/data/MASTER - PYTHON - SCORING MODEL - MCG MADISON RIDGE DST - v2.0.xlsx\"\n",
    "disposition_inputs = [\n",
    "  \"Selling Costs\",\n",
    "  \"Disposition Fee\",\n",
    "  \"Net Operating Income\",\n",
    "  \"Loan Assumption/Payoff\",\n",
    "  \"Return of Forecasted Reserves\",\n",
    "  \"CF Y 11\",\n",
    "  \"Return of Maximum Offering Amount\",\n",
    "  \"Projected Terminal Cap Rate\",\n",
    "  \"Cash Flows\"\n",
    "]\n",
    "dfs = pd.read_excel(filepath, sheet_name=\"5 - Disposition Analysis\", header=None)\n",
    "# Splitting the DataFrame by empty columns\n",
    "sub_dfs_by_columns = split_df_by_empty_columns(dfs)\n",
    "\n",
    "# Splitting each sub-DataFrame by empty rows\n",
    "final_split_dfs = []\n",
    "for sub_df in sub_dfs_by_columns:\n",
    "    split_sub_dfs = split_df_by_empty_rows(sub_df)\n",
    "    final_split_dfs.extend([get_csv_string(x) for x in split_sub_dfs if not x.empty])\n",
    "\n",
    "dfs.dropna(axis=0, how='all', inplace=True)\n",
    "dfs.dropna(axis=1, how='all', inplace=True)\n",
    "fee_columns = ['Disposition Fee', 'Selling Costs']\n",
    "cashflow_columns = [1,2,3,4,5,6,7,8,9]\n",
    "ground_truth = dfs[dfs[1].isin(disposition_inputs+cashflow_columns)].iloc[:, :2] # Get only the necessary columns\n",
    "ground_truth.drop(labels=[16, 17], axis=0, inplace=True) # drop the duplicate Selling and Disposition Costs\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "access_token = os.getenv('HF_TOKEN')\n",
    "llama_api_key = os.getenv('LLAMA_API_KEY')\n",
    "\n",
    "print('first model load...')\n",
    "# model_name = \"EleutherAI/gpt-neo-125m\"\n",
    "# model_name = \"microsoft/Phi-3-mini-128k-instruct\" # 128K context window\n",
    "# model_name = \"meta-llama/Meta-Llama-3-8B-Instruct\" # 8K context window\n",
    "# model_name = \"clibrain/mamba-2.8b-instruct-openhermes\" # 8K context window\n",
    "# model_name = \"Qwen/Qwen2-1.5B-Instruct\"\n",
    "model_name = \"mistralai/Mistral-7B-Instruct-v0.3\" # 32K context window\n",
    "llm = dspy.HFModel(model=model_name, hf_device_map='auto', token=access_token)\n",
    "llm.kwargs['max_new_tokens']=100\n",
    "# llm.kwargs['repetition_penalty']=1.1\n",
    "llm.kwargs['temperature']=None\n",
    "llm.kwargs['do_sample']=False\n",
    "llm.kwargs['top_k']=None\n",
    "# llm.kwargs['typical_p']=0.9\n",
    "\n",
    "print('deleting model...')\n",
    "llm.model=None\n",
    "gc.collect()\n",
    "print('reloading model...')\n",
    "\n",
    "quantization_config = BitsAndBytesConfig(\n",
    "        load_in_4bit=True,\n",
    "        bnb_4bit_quant_type=\"nf4\",\n",
    "        bnb_4bit_compute_dtype=torch.float16,\n",
    ")\n",
    "\n",
    "llm.model=AutoModelForCausalLM.from_pretrained(model_name, quantization_config=None, \n",
    "                                               trust_remote_code=True, device_map=\"auto\", \n",
    "                                               attn_implementation=\"flash_attention_2\",  \n",
    "                                               torch_dtype=torch.bfloat16)\n",
    "\n",
    "\n",
    "# from peft import LoraConfig, get_peft_model, prepare_model_for_kbit_training\n",
    "\n",
    "# config = LoraConfig(\n",
    "#     r=16,\n",
    "#     lora_alpha=32,\n",
    "#     target_modules=[\"k_proj\", \"v_proj\", \"q_proj\", \"o_proj\"], # Mistral param names\n",
    "#     lora_dropout=0.05,\n",
    "#     bias=\"none\", #\"none\", \"all\", \"lora_only\"\n",
    "#     task_type=\"CAUSAL_LM\", \n",
    "    \n",
    "# )\n",
    "\n",
    "# llm.model = prepare_model_for_kbit_training(llm.model)\n",
    "# llm.model = get_peft_model(llm.model, config)\n",
    "# print_trainable_parameters(llm.model)\n",
    "\n",
    "if model_name == 'mistralai/Mistral-7B-Instruct-v0.3':\n",
    "    llm.model.generation_config.pad_token_id = llm.tokenizer.eos_token_id\n",
    "    llm.tokenizer.pad_token_id = llm.tokenizer.eos_token_id\n",
    "\n",
    "dspy.settings.configure(lm=llm)\n",
    "\n",
    "######## RAG model\n",
    "# chroma_client = chromadb.PersistentClient(path=CHROMADB_DIR)\n",
    "# collection = chroma_client.get_or_create_collection(name=CHROMA_COLLECTION_NAME)\n",
    "# # text_splitter = SentenceTransformersTokenTextSplitter(tokens_per_chunk=100)\n",
    "\n",
    "# ids = []\n",
    "# documents = []\n",
    "# metadatas = []\n",
    "# # dfs_str = get_csv_string(dfs)\n",
    "# # chunks = text_splitter.create_documents([dfs_str], )\n",
    "# for chunk_no, chunk in enumerate(final_split_dfs):\n",
    "#     ids.append(f\"{chunk_no}\")\n",
    "#     documents.append(chunk)\n",
    "#     # metadatas.append({\"title\":})\n",
    "# if ids:\n",
    "#     collection.upsert(ids=ids, documents=documents)#, metadatas=metadatas)\n",
    "\n",
    "# retriever = dspy.ColBERTv2(url='http://20.102.90.50:2017/wiki17_abstracts')\n",
    "# default_ef = embedding_functions.HuggingFaceEmbeddingFunction(model_name='colbert-ir/colbertv2.0', api_key=access_token)\n",
    "# default_ef = embedding_functions.DefaultEmbeddingFunction()\n",
    "# retriever = ChromadbRM(CHROMA_COLLECTION_NAME, CHROMADB_DIR, default_ef, k=3)\n",
    "\n",
    "# dspy.settings.configure(lm=llm, rm=retriever)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "access_token = os.getenv('HF_TOKEN')\n",
    "llama_api_key = os.getenv('LLAMA_API_KEY')\n",
    "parser = LlamaParse(\n",
    "    api_key=llama_api_key,\n",
    "        result_type=\"text\",\n",
    "        language=\"en\",\n",
    "        varbose=True\n",
    "    )"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "!rm -rf /root/.cache/huggingface/hub/"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "!pip3 install llama-index-embeddings-text-embeddings-inference"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from llama_index.core import VectorStoreIndex, get_response_synthesizer\n",
    "from llama_index.core.retrievers import VectorIndexRetriever\n",
    "from llama_index.core.query_engine import RetrieverQueryEngine\n",
    "\n",
    "\n",
    "# model_name = \"mistralai/Mistral-7B-Instruct-v0.3\" # 32K context window\n",
    "model_name = \"Qwen/Qwen2-1.5B-Instruct\"\n",
    "tokenizer_name = model_name\n",
    "# rm_llm = HuggingFaceLLM(model=dspy_llm.llm, tokenizer_name=tokenizer_name, is_chat_model=True, device_map='auto', max_new_tokens=50, context_window=8000)\n",
    "rm_llm = HuggingFaceLLM(model_name=model_name, tokenizer_name=tokenizer_name, model=llm.model, is_chat_model=True, device_map='auto', max_new_tokens=50, context_window=8000)\n",
    "filepath = \"/workspace/data/MASTER - PYTHON - SCORING MODEL - MCG MADISON RIDGE DST - v2.0.xlsx\"\n",
    "documents = PandasExcelReader(sheet_name=\"5 - Disposition Analysis\").load_data(filepath)\n",
    "# documents = parser.load_data(\"/workspace/data/PPM - MCG MADISON RIDGE DST.pdf\")\n",
    "print(\"Documents created\")\n",
    "\n",
    "Settings.llm = rm_llm\n",
    "Settings.chunk_size = 100\n",
    "Settings.chunk_overlap = 25\n",
    "\n",
    "embed_model_name = \"BAAI/bge-small-en-v1.5\"\n",
    "# embed_model_name = \"Alibaba-NLP/gte-Qwen1.5-7B-instruct\"\n",
    "embed_model = HuggingFaceEmbedding(model_name=embed_model_name)\n",
    "\n",
    "# # for p in embed_model._model.named_parameters():\n",
    "# #     p[1].requires_grad = False\n",
    "\n",
    "# # KeywordTableSimpleRetriever\n",
    "# index = VectorStoreIndex.from_documents(documents, embed_model=embed_model)\n",
    "# # configure retriever\n",
    "# retriever = VectorIndexRetriever(\n",
    "#     index=index,\n",
    "#     similarity_top_k=2,\n",
    "# )\n",
    "# # configure response synthesizer\n",
    "# response_synthesizer = get_response_synthesizer(\n",
    "#     response_mode=\"tree_summarize\",\n",
    "# )\n",
    "# # assemble query engine\n",
    "# query_engine = RetrieverQueryEngine(\n",
    "#     retriever=retriever,\n",
    "#     response_synthesizer=response_synthesizer,\n",
    "# )\n",
    "\n",
    "index = VectorStoreIndex.from_documents(documents, embed_model=embed_model)\n",
    "# index = VectorStoreIndex.from_documents(final_split_dfs, embed_model=embed_model)\n",
    "# index.storage_context.persist(persist_dir=\"/workspace/data/storage/alpha\")\n",
    "# query_engine = index.as_query_engine(llm=rm_llm)\n",
    "query_engine = index.as_retriever(similarity_top_k=2)\n",
    "\n",
    "# Settings.embed_model = embed_model\n",
    "\n",
    "# index.set_index_id(\"vector_index\")\n",
    "# index.storage_context.persist(\"/workspace/data/storage\")\n",
    "# storage_context = StorageContext.from_defaults(persist_dir=\"/workspace/data/storage\")\n",
    "# index = load_index_from_storage(storage_context, index_id=\"vector_index\")\n",
    "# query_engine = index.as_query_engine(response_mode=\"tree_summarize\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "dspy_llm = DspyLlamaIndexWrapper(rm_llm, model_type='chat', max_new_tokens=30)\n",
    "dspy.settings.configure(lm=dspy_llm)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Question: Get the value for Return of Maximum Offering Amount.\n",
    "# Extracted values: Return of Maximum Offering Amount: 44386706.96773932\n",
    "# Question: What is the return on maximum offering amount? Please provide a floating point number less than zero.\n",
    "# Extracted values: Return of Maximum Offering Amount: -77670566.54709445"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Fine Tuning"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "import sys\n",
    "sys.setrecursionlimit(10000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "gt_collect = {}\n",
    "for row,col in ground_truth.iterrows():\n",
    "    if isinstance(col.values[0], int):\n",
    "        name = f\"Cashflows {col.values[0]}\"\n",
    "    else:\n",
    "        name = col.values[0]\n",
    "    value = col.values[1]\n",
    "    gt_collect[name] = str(value)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import random\n",
    "# dfs_str = get_csv_string(dfs)\n",
    "num_rounds = 2\n",
    "train_data = []\n",
    "for _ in range(num_rounds):\n",
    "    # TODO: gradually increase n_samples, random fill in of values in range\n",
    "    # dfs_aug = randomize_row_values(dfs, ground_truth=ground_truth, n_samples=15)\n",
    "    # dfs_str = get_csv_string(dfs_aug)\n",
    "    # dfs_str = get_csv_string(dfs)\n",
    "    \n",
    "    for value_to_extract in gt_collect:\n",
    "\n",
    "        question = f\"Extract the value for the variable name '{value_to_extract}'?\"\n",
    "        answer = f\"{value_to_extract}: {gt_collect[value_to_extract]}\"\n",
    "        train_data.append(dspy.Example(question=question, answer=answer).with_inputs('question'))\n",
    "    \n",
    "random.shuffle(train_data)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2\n",
    "\n",
    "from train_utils import operators_dict, range_description_json\n",
    "from models_testing import SpreadSheetAnalyzer\n",
    "spreadsheeet_ananlyst = SpreadSheetAnalyzer(range_description_json, operators_dict, query_engine=query_engine, num_passages=3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# in dspy.primitives.module.py\n",
    "# def reset_copy(self):\n",
    "#     import llama_index\n",
    "#     obj = copy.deepcopy(self)\n",
    "#     ######################################################\n",
    "#     for attribute_name in dir(obj):\n",
    "#         if not attribute_name.startswith('_'):\n",
    "#             attribute_value = getattr(obj, attribute_name)\n",
    "#             if isinstance(attribute_value, llama_index.core.indices.vector_store.retrievers.retriever.VectorIndexRetriever):\n",
    "#                 setattr(obj, attribute_name, getattr(self, attribute_name))\n",
    "#     ######################################################\n",
    "#     for param in obj.parameters():\n",
    "#         param.reset()\n",
    "\n",
    "#     return obj\n",
    "\n",
    "############################or############################# \n",
    "\n",
    "# in llama_index/core/schema.py comment out line 84\n",
    "# 84  state[\"__private_attribute_values__\"] = {}\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from dspy.teleprompt import BootstrapFewShotWithRandomSearch, BootstrapFinetune, BootstrapFewShot, MIPRO, MIPROv2\n",
    "\n",
    "perc_train = 0.7\n",
    "num_train = int(len(train_data) * perc_train)\n",
    "def validate_answer(pred, example, trace=None):\n",
    "    return example.answer.lower() == pred.answer.lower()\n",
    "metric = dspy.evaluate.metrics.answer_exact_match\n",
    "# metric = dspy.evaluate.metrics.answer_passage_match\n",
    "# metric = validate_answer\n",
    "NUM_THREADS=1\n",
    "TRAIN_NUM=17\n",
    "\n",
    "#Configure model to finetune\n",
    "# config = dict(bf16=True, bsize=1, accumsteps=3, lr=8e-5) #path_prefix=None\n",
    "\n",
    "#Compile program on BootstrapFinetune\n",
    "\n",
    "# finetune_optimizer = MIPROv2(prompt_model=llm, task_model=llm, metric=metric, num_candidates=10, init_temperature=1.2, minibatch_size=1)\n",
    "# kwargs = dict(num_threads=NUM_THREADS, display_progress=True, display_table=0)\n",
    "# compiled_prompt_opt = finetune_optimizer.compile(spreadsheeet_ananlyst, trainset=train_data[:TRAIN_NUM], \n",
    "#                                                  num_batches=200, max_bootstrapped_demos=3, max_labeled_demos=5, \n",
    "#                                                  eval_kwargs=kwargs, requires_permission_to_run=False)\n",
    "\n",
    "# finetune_optimizer = MIPRO(prompt_model=llm, task_model=llm, metric=metric, num_candidates=10, init_temperature=1.2)\n",
    "# kwargs = dict(num_threads=NUM_THREADS, display_progress=True, display_table=0)\n",
    "# compiled_prompt_opt = finetune_optimizer.compile(spreadsheeet_ananlyst, trainset=train_data[:TRAIN_NUM], num_trials=100, \n",
    "#                                                  max_bootstrapped_demos=3, max_labeled_demos=5, eval_kwargs=kwargs,\n",
    "#                                                  requires_permission_to_run=False)\n",
    "\n",
    "config = dict(target=model_name, epochs=3, bf16=True, bsize=1, accumsteps=3, lr=7e-5) #path_prefix=None\n",
    "finetune_optimizer = BootstrapFinetune(metric=metric)\n",
    "finetune_program = finetune_optimizer.compile(spreadsheeet_ananlyst, trainset=train_data, **config)\n",
    "\n",
    "# finetune_optimizer = BootstrapFewShot(metric=metric, max_bootstrapped_demos=8, max_labeled_demos=8)\n",
    "# finetune_program = finetune_optimizer.compile(spreadsheeet_ananlyst, trainset=train_data)\n",
    "\n",
    "# #Load program and activate model's parameters in program before evaluation\n",
    "# ckpt_path = \"saved_checkpoint_path_from_finetuning\"\n",
    "# LM = dspy.HFModel(checkpoint=ckpt_path, model=model_name)\n",
    "\n",
    "# for p in finetune_program.predictors():\n",
    "#     p.lm = LM\n",
    "#     p.activated = False"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Load program and activate model's parameters in program before evaluation\n",
    "ckpt_path = \"/workspace/repos/finetuning_ckpts/NPBHZ93JIRMS2.all/checkpoint-120\"\n",
    "LM = dspy.HFModel(checkpoint=ckpt_path, model=model_name, hf_device_map='cuda:0')\n",
    "\n",
    "for p in spreadsheeet_ananlyst.predictors():\n",
    "    p.lm = LM\n",
    "    p.activated = False"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# perc_train = 0.7\n",
    "# num_train = int(len(train_data) * perc_train)\n",
    "# metric = dspy.evaluate.metrics.answer_exact_match\n",
    "\n",
    "scores = []\n",
    "for x in train_data[num_train:num_train+34]:\n",
    "    pred = finetune_program(**x.inputs())\n",
    "    score = metric(x, pred)\n",
    "    scores.append(score)\n",
    "np.mean(scores)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "saved_checkpoint_path_from_finetuning = '/workspace/repos/finetuning_ckpts/NFAI903XCHAMQ.all/checkpoint-53'\n",
    "llm.model=None\n",
    "llm.model=AutoModelForCausalLM.from_pretrained(saved_checkpoint_path_from_finetuning, quantization_config=None, \n",
    "                                               trust_remote_code=True, device_map=\"auto\", \n",
    "                                               attn_implementation=\"flash_attention_2\",  \n",
    "                                               torch_dtype=torch.bfloat16)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2\n",
    "\n",
    "from train_utils import operators_dict, range_description_json\n",
    "from models_testing import SpreadSheetAnalyzer\n",
    "spreadsheeet_ananlyst = SpreadSheetAnalyzer(range_description_json, operators_dict, query_engine=query_engine, num_passages=3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# dfs_str = get_csv_string(dfs)\n",
    "collection = []\n",
    "for value_to_extract in gt_collect:\n",
    "    # if value_to_extract==\"Selling Costs\":\n",
    "        # continue\n",
    "    question = f\"Extract the value for the variable name '{value_to_extract}'?\"\n",
    "    print(question)\n",
    "    pred = spreadsheeet_ananlyst(question, verbose=True)\n",
    "    print(pred.answer)\n",
    "    collection.append((pred, f\"{value_to_extract}: {gt_collect[value_to_extract]}\"))\n",
    "    break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(pred.answer)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for i in collection:\n",
    "    print(i[0].answer,\"---\", i[1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "np.mean([i[0].answer == i[1] for i in collection])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# dfs_str = get_csv_string(dfs)\n",
    "collection = []\n",
    "for value_to_extract in gt_collect:\n",
    "    question = f\"Extract the value for the variable name '{value_to_extract}'?\"\n",
    "    print(question)\n",
    "    pred = finetune_program(question, verbose=True)\n",
    "    print(pred.answer)\n",
    "    collection.append((pred, f\"{value_to_extract}: {gt_collect[value_to_extract]}\"))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for i in collection:\n",
    "    print(i[0].answer,\"---\", i[1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "np.mean([i[0].answer == i[1] for i in collection])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "llm.model()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from dspy.teleprompt.signature_opt_typed import optimize_signature\n",
    "from dspy.evaluate.metrics import answer_exact_match\n",
    "from dspy.functional import TypedChainOfThought\n",
    "\n",
    "compiled_program = optimize_signature(\n",
    "    student=TypedChainOfThought(\"question -> answer\"),\n",
    "    evaluator=Evaluate(devset=devset, metric=answer_exact_match, num_threads=10, display_progress=True),\n",
    "    n_iterations=50,\n",
    ").program"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}