gpt_corporate_lobbying.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import pandas as pd\n",
    "import re\n",
    "import backoff\n",
    "import time\n",
    "import json\n",
    "from openai.error import RateLimitError\n",
    "from importlib.metadata import version\n",
    "from typing import List, Callable, Dict\n",
    "from langchain.llms import OpenAI\n",
    "from langchain.chains import LLMChain\n",
    "from langchain.prompts import PromptTemplate\n",
    "from langchain.chains.summarize import load_summarize_chain\n",
    "from langchain.chains.question_answering import load_qa_chain\n",
    "from transformers import GPT2TokenizerFast\n",
    "from langchain.text_splitter import CharacterTextSplitter\n",
    "from langchain.docstore.document import Document\n",
    "from langchain.agents import ZeroShotAgent, Tool, AgentExecutor\n",
    "from langchain.serpapi import SerpAPIWrapper\n",
    "from sklearn.metrics import precision_score\n",
    "from sklearn.metrics import recall_score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'0.0.55'"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "os.environ[\"OPENAI_API_KEY\"] = OAI_KEY\n",
    "os.environ[\"SERPAPI_API_KEY\"] = SERP_KEY\n",
    "version('langchain')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "d = pd.read_csv(FP + \"legislation_relevance_dataset_for_llm_evaluation_unbalanced.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "500\n",
      "485\n"
     ]
    }
   ],
   "source": [
    "print(d.shape[0])\n",
    "d = d.drop_duplicates(subset=['company_name', 'summary_text'], keep='first')\n",
    "print(d.shape[0])\n",
    "d = d.reset_index(drop=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    344\n",
       "1    141\n",
       "Name: relevance, dtype: int64"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "d.relevance.value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Lobbyist:\n",
    "    def __init__(self, data: pd.DataFrame):\n",
    "        self.d = data\n",
    "        self.model_name = \"text-davinci-003\"\n",
    "        self.model_name_older = \"text-davinci-002\"\n",
    "        self.temp = 0\n",
    "        \n",
    "        self.tokenizer = GPT2TokenizerFast.from_pretrained(\"gpt2\")\n",
    "        self.text_splitter = CharacterTextSplitter.from_tiktoken_encoder(chunk_size=4000, \n",
    "                                                                        chunk_overlap=0, \n",
    "                                                                        separator = \" \")\n",
    "        self.llm_summarize = OpenAI(temperature=self.temp, model_name=self.model_name)\n",
    "        self.llm = OpenAI(temperature=self.temp, model_name=self.model_name, max_tokens=-1)\n",
    "        self.llm_older = OpenAI(temperature=self.temp, model_name=self.model_name_older, max_tokens=-1)\n",
    "        \n",
    "        ###### SUMMARY CHAIN ######\n",
    "        self.sum_template = \"\"\"Slightly summarize the following Congressional bill summary:\n",
    "                                {context}\n",
    "                                SUMMARY:\"\"\"\n",
    "        self.sum_prompt = PromptTemplate(template=self.sum_template, input_variables=[\"context\"])\n",
    "        self.sum_combine_template = \"\"\"Given the following summaries of parts of this bill, create a final summary of the bill. \n",
    "                                        =========\n",
    "                                        SUMMARIES FROM PARTS OF THE OVERALL RESPONSE: {summaries}\n",
    "                                        =========\n",
    "                                        FINAL SUMMARY:\"\"\"\n",
    "        self.sum_combine_prompt = PromptTemplate(template=self.sum_combine_template, \n",
    "                                                 input_variables=[\"summaries\"])\n",
    "        self.sum_chain = load_qa_chain(llm=self.llm_summarize, \n",
    "                                        chain_type=\"map_reduce\", \n",
    "                                        question_prompt = self.sum_prompt,\n",
    "                                        combine_prompt = self.sum_combine_prompt,\n",
    "                                        collapse_prompt = self.sum_combine_prompt)\n",
    "        \n",
    "        ###### ZERO SHOT QA CHAIN ######\n",
    "        self.zero_shot_template = \"\"\"You are a lobbyist analyzing Congressional bills for their potential impacts on companies. \n",
    "                            Given the title and summary of the bill, plus information on the company from its 10K SEC filing, it is your job to determine if a bill is at least somewhat relevant to a company (in terms of whether it could impact the company if it was later enacted). \n",
    "                            Official title of bill: {official_title}\n",
    "                            Official summary of bill: {summary_text}\n",
    "                            Official subjects of bill: {subjects}\n",
    "                            Company name: {company_name}\n",
    "                            Company business description: {business_description}\n",
    "                            Is this bill potentially relevant to this company? \n",
    "                            Answer in this format:\n",
    "                            ANSWER: 'YES' or 'NO' (use all caps). EXPLANATION: the step-by-step reasoning you undertook to formulate a response. CONFIDENCE: integer between 0 and 100 for your estimate of confidence in your answer (1 is low confidence and 99 is high)\n",
    "                            \"\"\"\n",
    "        self.zero_shot_temp  = PromptTemplate(template=self.zero_shot_template, input_variables = [\"official_title\", \"summary_text\", \"subjects\",\n",
    "                                                                     \"company_name\", \"business_description\"])\n",
    "        self.zero_shot_chain = LLMChain(llm=self.llm, prompt=self.zero_shot_temp)\n",
    "        \n",
    "        self.zero_shot_chain_older = LLMChain(llm=self.llm_older, prompt=self.zero_shot_temp)\n",
    "        \n",
    "        ###### AGENT QA CHAIN ######\n",
    "        self.self_ask_agent_search = SerpAPIWrapper()\n",
    "        \n",
    "        self.self_ask_agent_tools = [\n",
    "            Tool(\n",
    "                name=\"Search\",\n",
    "                func=self.self_ask_agent_search.run,\n",
    "                description=\"useful for learning more about a company or bill\"\n",
    "            )\n",
    "        ]\n",
    "        \n",
    "        self.self_ask_agent_prefix = \"\"\"You are a lobbyist analyzing Congressional bills for their impacts on companies. \n",
    "            Given the title and summary of the bill, plus information on the company from its 10K SEC filing, it is your job to determine if a bill is at least somewhat relevant to a company in terms of whether it could impact the company if it was enacted. \n",
    "            You have access to the following tools:\"\"\"\n",
    "        self.self_ask_agent_suffix = \"\"\"\n",
    "            Official title of bill: {official_title}\n",
    "            Official summary of bill: {summary_text}\n",
    "            Official subjects of bill: {subjects}\n",
    "            Company name: {company_name}\n",
    "            Company business description: {business_description}\n",
    "            Question: {input} \n",
    "            {agent_scratchpad}\"\"\"\n",
    "        self.self_ask_prompt = ZeroShotAgent.create_prompt(\n",
    "                    tools=self.self_ask_agent_tools,\n",
    "                    prefix=self.self_ask_agent_prefix,\n",
    "                    suffix=self.self_ask_agent_suffix,\n",
    "                    input_variables=[\"official_title\", \"summary_text\", \"subjects\",\n",
    "                                     \"company_name\", \"business_description\" ,\n",
    "                                     \"input\", \"agent_scratchpad\"]\n",
    "                )\n",
    "        self.self_ask_llm_chain = LLMChain(llm=self.llm, prompt=self.self_ask_prompt)\n",
    "        self.self_ask_agent = ZeroShotAgent(llm_chain=self.self_ask_llm_chain, \n",
    "                                            tools=self.self_ask_agent_tools)\n",
    "        self.self_ask_agent_executor = AgentExecutor.from_agent_and_tools(agent=self.self_ask_agent, \n",
    "                                                                          tools=self.self_ask_agent_tools, \n",
    "                                                                          max_iterations=3, \n",
    "                                                                          early_stopping_method=\"generate\",\n",
    "                                                                          verbose=True)\n",
    "        \n",
    "        ###### ZERO SHOT LETTER GENERATION ######\n",
    "        self.zero_shot_letter_template = \"\"\"You are a lobbyist analyzing Congressional bills for their impacts on companies. \n",
    "                            You have identified the following bill to be relevant to the company you work for.\n",
    "                            Given the title and summary of the bill, plus information on your company from its 10K SEC filing, it is your job to now write a persuasive letter to sponsor of the bill to convince them to add provisions to the bill that would make it better for the bottom-line of your company.\n",
    "                            Sign the letter as the general counsel of your company (put the actual name of your company).\n",
    "                            Official title of bill: {official_title}\n",
    "                            Official summary of bill: {summary_text}\n",
    "                            Official subjects of bill: {subjects}\n",
    "                            Company name: {company_name}\n",
    "                            Company business description: {business_description}\n",
    "                            YOUR LETTER: \n",
    "                            \"\"\"\n",
    "        self.zero_shot_letter_temp  = PromptTemplate(template=self.zero_shot_letter_template, \n",
    "                                                     input_variables = [\"official_title\", \"summary_text\", \"subjects\",\n",
    "                                                                        \"company_name\", \"business_description\"])\n",
    "        self.zero_shot_letter = LLMChain(llm=self.llm, prompt=self.zero_shot_letter_temp)\n",
    "    \n",
    "    def compute_length(self, text: str) -> int:\n",
    "        tokenized_text = self.tokenizer.tokenize(text)\n",
    "        nt = len(tokenized_text)\n",
    "        return nt\n",
    "\n",
    "    def compute_length_shorten(self, text: str) -> str:\n",
    "        nt = self.compute_length(text)\n",
    "        while nt > 4000:\n",
    "            texts = self.text_splitter.split_text(text)\n",
    "            docs = [Document(page_content=t) for t in texts]\n",
    "            text = self.sum_chain.run(docs)\n",
    "            nt = self.compute_length(text)\n",
    "        return text\n",
    "\n",
    "    def add_summary_col(self):\n",
    "        results = list()\n",
    "        for i in range(self.d.shape[0]):\n",
    "            s = self.compute_length_shorten(self.d.summary_text[i])\n",
    "            results.append({\"summary_text_summarized\": s})\n",
    "        self.d = pd.concat([self.d, pd.DataFrame(results)], axis=1)\n",
    "    \n",
    "    def convert_output(self, text: str) -> Dict:\n",
    "        _out = dict()\n",
    "        if \"YES\" in text:\n",
    "            _out['prediction'] = 1\n",
    "        elif \"NO\" in text:\n",
    "            _out['prediction'] = 0\n",
    "        elif \"Yes\" in text:\n",
    "            _out['prediction'] = 1\n",
    "        elif \"No\" in text:\n",
    "            _out['prediction'] = 0\n",
    "        elif \"yes\" in text:\n",
    "            _out['prediction'] = 1\n",
    "        elif \"no\" in text:\n",
    "            _out['prediction'] = 0\n",
    "        else:\n",
    "            _out['prediction'] = -999\n",
    "        \n",
    "        match = re.search('CONFIDENCE: (\\d+)', text)\n",
    "        if match:\n",
    "            _out['prob'] = int(match.group(1))\n",
    "        else:\n",
    "            _out['prob'] = -999\n",
    "        \n",
    "        return _out\n",
    "    \n",
    "    def dummy_predictor(self, i: int) -> str:\n",
    "        return \"NO and 100\"\n",
    "    \n",
    "    @backoff.on_exception(backoff.expo, (RateLimitError))\n",
    "    def zero_shot_predictor(self, i: int) -> str:\n",
    "        return self.zero_shot_chain.predict(official_title = self.d.official_title[i], \n",
    "                                            summary_text = self.d.summary_text_summarized[i], \n",
    "                                            subjects = self.d.subjects[i], \n",
    "                                            company_name = self.d.company_name[i], \n",
    "                                            business_description = self.d.business_description[i])\n",
    "    \n",
    "    @backoff.on_exception(backoff.expo, (RateLimitError))\n",
    "    def zero_shot_predictor_older(self, i: int) -> str:\n",
    "        return self.zero_shot_chain_older.predict(official_title = self.d.official_title[i], \n",
    "                                            summary_text = self.d.summary_text_summarized[i], \n",
    "                                            subjects = self.d.subjects[i], \n",
    "                                            company_name = self.d.company_name[i], \n",
    "                                            business_description = self.d.business_description[i])\n",
    "    \n",
    "    @backoff.on_exception(backoff.expo, (RateLimitError))\n",
    "    def agent_predictor(self, i: int) -> str:\n",
    "        return self.self_ask_agent_executor.run(official_title= self.d.official_title[i], \n",
    "                                                summary_text=self.d.summary_text_summarized[i], \n",
    "                                                subjects=self.d.subjects[i], \n",
    "                                                company_name=self.d.company_name[i],\n",
    "                                                business_description=self.d.business_description[i],\n",
    "                                               input=\"Is this bill potentially relevant to this company? ('YES' or 'NO'; use all caps)\")\n",
    "    \n",
    "    @backoff.on_exception(backoff.expo, (RateLimitError))\n",
    "    def zero_shot_letter_generator(self, i: int) -> str:\n",
    "        return self.zero_shot_letter.predict(official_title = self.d.official_title[i], \n",
    "                                            summary_text = self.d.summary_text_summarized[i], \n",
    "                                            subjects = self.d.subjects[i], \n",
    "                                            company_name = self.d.company_name[i], \n",
    "                                            business_description = self.d.business_description[i])\n",
    "    \n",
    "    def evaluate(self, func: Callable, \n",
    "                 n = int, m = int) -> List:\n",
    "        results = list()\n",
    "        for i in range(n, m):\n",
    "            try:\n",
    "                out = func(i)\n",
    "                pred = self.convert_output(out)\n",
    "                pp = pred['prediction']\n",
    "                gt = self.d.relevance[i]\n",
    "\n",
    "                if pp == -999:\n",
    "                    correct = False\n",
    "                    print(\"-999\")\n",
    "                elif pp == 2:\n",
    "                    pp = 1\n",
    "                    correct = gt == pp\n",
    "                else:\n",
    "                    correct = gt == pp\n",
    "\n",
    "                results.append({\"text_response\": out,\n",
    "                                \"numeric_prediction\": pp,\n",
    "                                \"probability\": pred['prob'],\n",
    "                                \"ground_truth\": int(gt),\n",
    "                                \"correct\": bool(correct)})\n",
    "            except Exception as e:\n",
    "                print(f\"At data {i}, error: {e}\")\n",
    "                time.sleep(10)\n",
    "                try:\n",
    "                    out = func(i)\n",
    "                    pred = self.convert_output(out)\n",
    "                    pp = pred['prediction']\n",
    "                    gt = self.d.relevance[i]\n",
    "\n",
    "                    if pp == -999:\n",
    "                        correct = False\n",
    "                        print(\"-999\")\n",
    "                    elif pp == 2:\n",
    "                        pp = 1\n",
    "                        correct = gt == pp\n",
    "                    else:\n",
    "                        correct = gt == pp\n",
    "\n",
    "                    results.append({\"text_response\": out,\n",
    "                                    \"numeric_prediction\": pp,\n",
    "                                    \"probability\": pred['prob'],\n",
    "                                    \"ground_truth\": int(gt),\n",
    "                                    \"correct\": bool(correct)})\n",
    "                except Exception as e2:\n",
    "                    print(f\"At data {i}, error 2: {e2}\")\n",
    "                    results.append({\"text_response\": \"NA\",\n",
    "                                    \"numeric_prediction\": \"NA\",\n",
    "                                    \"probability\": \"NA\",\n",
    "                                    \"ground_truth\": \"NA\",\n",
    "                                    \"correct\": \"NA\"})\n",
    "            finally:\n",
    "                with open(FP + \"gpt_data.json\", \"wt\") as output_file:\n",
    "                    json.dump(results, output_file)\n",
    "        \n",
    "        return results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Token indices sequence length is longer than the specified maximum sequence length for this model (7266 > 1024). Running this sequence through the model will result in indexing errors\n"
     ]
    }
   ],
   "source": [
    "l = Lobbyist(d)\n",
    "l.add_summary_col()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "dum_result = l.evaluate(l.dummy_predictor, 0, l.d.shape[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.709278350515464\n"
     ]
    }
   ],
   "source": [
    "dum = pd.DataFrame(dum_result)\n",
    "print(dum.correct.mean())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "result_chain_older = l.evaluate(l.zero_shot_predictor_older, 0, l.d.shape[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1    351\n",
       "0    134\n",
       "Name: numeric_prediction, dtype: int64"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "result_chain_older = pd.DataFrame(result_chain_older)\n",
    "result_chain_older.numeric_prediction.value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "probability  numeric_prediction\n",
       " 80          1                     316\n",
       " 1           0                      56\n",
       "-999         0                      38\n",
       " 99          0                      22\n",
       "-999         1                      21\n",
       " 95          1                      11\n",
       " 80          0                       6\n",
       " 100         0                       5\n",
       " 95          0                       3\n",
       " 60          0                       2\n",
       "             1                       2\n",
       " 90          0                       2\n",
       " 70          1                       1\n",
       "dtype: int64"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "result_chain_older[['probability', 'numeric_prediction']].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "ANSWER: YES\n",
      "EXPLANATION: The bill is relevant to Alkermes Plc because it could impact the company's products ARISTADA and VIVITROL.\n",
      "CONFIDENCE: 80\n"
     ]
    }
   ],
   "source": [
    "print(result_chain_older.text_response[6])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.5216494845360825\n",
      "0.9219858156028369\n",
      "0.35755813953488375\n",
      "0.37037037037037035\n",
      "0.9219858156028369\n"
     ]
    }
   ],
   "source": [
    "print(result_chain_older.correct.mean())\n",
    "print(result_chain_older[result_chain_older.ground_truth == 1].correct.mean())\n",
    "print(result_chain_older[result_chain_older.ground_truth == 0].correct.mean())\n",
    "result_chain_older_metadata = pd.concat([l.d, result_chain_older], axis=1)\n",
    "print(precision_score(result_chain_older_metadata.ground_truth, result_chain_older_metadata.numeric_prediction))\n",
    "print(recall_score(result_chain_older_metadata.ground_truth, result_chain_older_metadata.numeric_prediction))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "485\n",
      "0.8292682926829268\n",
      "41\n",
      "0.49324324324324326\n"
     ]
    }
   ],
   "source": [
    "print(result_chain_older.shape[0])\n",
    "print(result_chain_older[result_chain_older.probability > 90].correct.mean())\n",
    "print(result_chain_older[result_chain_older.probability > 90].shape[0])\n",
    "print(result_chain_older[result_chain_older.probability <= 90].correct.mean())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "At data 54, error: The server is overloaded or not ready yet.\n",
      "At data 350, error: The server is overloaded or not ready yet.\n"
     ]
    }
   ],
   "source": [
    "result_chain = l.evaluate(l.zero_shot_predictor, 0, l.d.shape[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    363\n",
       "1    122\n",
       "Name: numeric_prediction, dtype: int64"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "result_chain = pd.DataFrame(result_chain)\n",
    "result_chain.numeric_prediction.value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "probability  numeric_prediction\n",
       "99           0                     347\n",
       "95           1                      68\n",
       "90           1                      51\n",
       "95           0                      12\n",
       "100          0                       4\n",
       "99           1                       2\n",
       "80           1                       1\n",
       "dtype: int64"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "result_chain[['probability', 'numeric_prediction']].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "ANSWER: YES. EXPLANATION: Alkermes Plc is a biopharmaceutical company that develops and commercializes products designed to address unmet medical needs of patients in major therapeutic areas, including addiction and schizophrenia. This bill requires the Centers for Medicare & Medicaid Services (CMS) to negotiate with pharmaceutical companies regarding prices for drugs covered under the Medicare prescription drug benefit, which could potentially impact Alkermes Plc's products. CONFIDENCE: 95\n"
     ]
    }
   ],
   "source": [
    "print(result_chain.text_response[6])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.7505154639175258\n",
      "0.5035460992907801\n",
      "0.8517441860465116\n",
      "0.5819672131147541\n",
      "0.5035460992907801\n"
     ]
    }
   ],
   "source": [
    "print(result_chain.correct.mean())\n",
    "print(result_chain[result_chain.ground_truth == 1].correct.mean())\n",
    "print(result_chain[result_chain.ground_truth == 0].correct.mean())\n",
    "result_chain_metadata = pd.concat([l.d, result_chain], axis=1)\n",
    "print(precision_score(result_chain_metadata.ground_truth, result_chain_metadata.numeric_prediction))\n",
    "print(recall_score(result_chain_metadata.ground_truth, result_chain_metadata.numeric_prediction))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "letters = list()\n",
    "for i in result_chain[result_chain.numeric_prediction == 1].index[:5]:\n",
    "    letters.append(l.zero_shot_letter_generator(i))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Dear [Sponsor of the Bill],\n",
      "\n",
      "I am writing on behalf of Alkermes Plc, a fully integrated, global biopharmaceutical company that applies its scientific expertise and proprietary technologies to research, develop and commercialize pharmaceutical products that are designed to address unmet medical needs of patients in major therapeutic areas.\n",
      "\n",
      "We are writing to express our support for the Medicare Negotiation and Competitive Licensing Act of 2019. We believe that this bill is an important step in ensuring that Medicare beneficiaries have access to the medications they need at a price they can afford.\n",
      "\n",
      "We are particularly supportive of the provisions in the bill that would require the Centers for Medicare & Medicaid Services (CMS) to negotiate with pharmaceutical companies regarding prices for drugs covered under the Medicare prescription drug benefit. We believe that this will help to ensure that the prices of these drugs are fair and reasonable.\n",
      "\n",
      "We are also supportive of the provisions in the bill that would allow for competitive licensing of drugs if the CMS is unable to negotiate the price of a drug. This will help to ensure that the prices of these drugs are kept in check and that Medicare beneficiaries have access to the medications they need.\n",
      "\n",
      "At Alkermes, we develop and commercialize products designed to address the unmet needs of patients suffering from addiction and schizophrenia. We have two key marketed products, ARISTADA and VIVITROL, which are used to treat these conditions. We believe that the provisions in the bill will help to ensure that our products are available to Medicare beneficiaries at a price they can afford.\n",
      "\n",
      "We would like to suggest that the bill be amended to include provisions that would provide additional incentives for pharmaceutical companies to negotiate with the CMS. We believe that this would help to ensure that the prices of drugs are kept in check and that Medicare beneficiaries have access to the medications they need.\n",
      "\n",
      "We thank you for your consideration and look forward to working with you to ensure that the Medicare Negotiation and Competitive Licensing Act of 2019 is passed in its amended form.\n",
      "\n",
      "Sincerely,\n",
      "\n",
      "[Name],\n",
      "General Counsel\n",
      "Alkermes Plc\n"
     ]
    }
   ],
   "source": [
    "print(letters[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "485\n",
      "0.789838337182448\n",
      "433\n",
      "0.4230769230769231\n"
     ]
    }
   ],
   "source": [
    "print(result_chain.shape[0])\n",
    "print(result_chain[result_chain.probability > 90].correct.mean())\n",
    "print(result_chain[result_chain.probability > 90].shape[0])\n",
    "print(result_chain[result_chain.probability <= 90].correct.mean())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "result_agent = l.evaluate(l.agent_predictor, 0, l.d.shape[0])\n",
    "result_agent = pd.DataFrame(result_agent)\n",
    "result_agent.numeric_prediction.value_counts()\n",
    "print(result_agent[result_agent.numeric_prediction != \"NA\"].correct.mean())"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}