copilots-reloaded-rag-dataset.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "82d3d136-694c-459e-bbfd-9193c0e51139",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "\n",
    "df = pd.read_csv(\"FNBr-cxns-ces.csv\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ee70d378-57e1-4199-942d-36ab702f7555",
   "metadata": {},
   "source": [
    "Filter out portuguese names and definitions for English constructions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "b57415f9-dd35-455a-93a5-cfc73b8ed9f5",
   "metadata": {},
   "outputs": [],
   "source": [
    "df = df[(df[\"cxnLanguage\"] == 1) | (df[\"txtLanguage\"] == 2)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "id": "e0d15f54-6c19-4f57-867f-314905072090",
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_with_translation(row, col):\n",
    "    col_pt = col + \"_pt\"\n",
    "    \n",
    "    if isinstance(row[col_pt], str):\n",
    "        return f\"{row[col]} (in portuguese: {row[col_pt]})\"\n",
    "    else:\n",
    "        return row[col]\n",
    "\n",
    "prompts = []\n",
    "\n",
    "for idx, group in df.groupby([\"cxnLanguage\", \"idConstruction\"]):\n",
    "    language = \"Brazilian Portuguese\" if idx[0] == 1 else \"English\"\n",
    "\n",
    "    pt_group = group[group[\"txtLanguage\"] == 1]\n",
    "    en_group = group[group[\"txtLanguage\"] == 2]\n",
    "    flat_group = en_group.merge(pt_group, on=[\"idConstruction\", \"idConstructionElement\"],\n",
    "                                how=\"left\", suffixes=(\"\", \"_pt\"))\n",
    "\n",
    "    cxnName = get_with_translation(flat_group.iloc[0], \"cxnName\")\n",
    "    cxnDef = get_with_translation(flat_group.iloc[0], \"cxnDefinition\")\n",
    "\n",
    "    text = f\"\"\"In the context of the theory of Construction Grammar and of the {language} language, the {cxnName} construction can be defined as such: \"{cxnDef}\".\n",
    "The {len(flat_group)} construction elements (CEs) of this construction are:\\n\"\"\"\n",
    "\n",
    "    for _, row in flat_group.iterrows():\n",
    "        ceName = row[\"ceName\"]\n",
    "        ceDef = row[\"ceDefinition\"]\n",
    "        text += f\"- {ceName}: {ceDef}\"\n",
    "\n",
    "        if isinstance(row[\"ceName_pt\"], str):\n",
    "            ceNamePt = row[\"ceName_pt\"]\n",
    "            ceDefPt = row[\"ceDefinition_pt\"]\n",
    "            text += f\" ({ceNamePt}: {ceDefPt})\\n\"\n",
    "        else:\n",
    "            text += \"\\n\"\n",
    "\n",
    "    prompts.append(text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "id": "683b8d84-53db-452e-b171-132d882a21ed",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of prompts: 230\n",
      "e.g.:\n",
      " In the context of the theory of Construction Grammar and of the Brazilian Portuguese language, the verb_modification (in portuguese: Auxiliação) construction can be defined as such: \"verb_modification (in portuguese: Construção genérica que congrega as construções de tempo, modo, modalidade e aspecto.)\".\n",
      "The 2 construction elements (CEs) of this construction are:\n",
      "- Verbo: verb_modification_verb (Verbo: Verbo principal.)\n",
      "- Auxiliar: verb_modification_modifier (Auxiliar: Auxiliar indicativo de tempo, modo, modalidade ou aspecto.)\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(\"Number of prompts:\" , len(prompts))\n",
    "print(\"e.g.:\\n\", prompts[187])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "id": "23a9d2e6-ca2a-488b-a8ec-c03d6f8d3327",
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "\n",
    "with open(\"constructicon-prompts.json\", \"w\") as fp:\n",
    "    json.dump(prompts, fp)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "copilots-kernel",
   "language": "python",
   "name": "copilots-kernel"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}