2-Template-SetFit.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### NLBSE'24 SetFit Baseline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "BASE_MODEL = \"sentence-transformers/all-mpnet-base-v2\"\n",
    "RANDOM_SEED = 42\n",
    "OUTPUT_PATH = 'output/setfit'\n",
    "\n",
    "!mkdir -p $OUTPUT_PATH"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/python/3.10.13/lib/python3.10/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
      "  from .autonotebook import tqdm as notebook_tqdm\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "DatasetDict({\n",
       "    train: Dataset({\n",
       "        features: ['repo', 'created_at', 'label', 'title', 'body'],\n",
       "        num_rows: 1500\n",
       "    })\n",
       "    test: Dataset({\n",
       "        features: ['repo', 'created_at', 'label', 'title', 'body'],\n",
       "        num_rows: 1500\n",
       "    })\n",
       "})"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from datasets import Dataset\n",
    "\n",
    "ds = Dataset.from_csv({ \"train\": \"data/issues_train.csv\", \"test\": \"data/issues_test.csv\" })\n",
    "ds"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['facebook/react', 'tensorflow/tensorflow', 'microsoft/vscode', 'bitcoin/bitcoin', 'opencv/opencv']\n"
     ]
    }
   ],
   "source": [
    "repos = ds[\"train\"].unique(\"repo\")\n",
    "print(repos)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th>label</th>\n",
       "      <th>bug</th>\n",
       "      <th>feature</th>\n",
       "      <th>question</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>repo</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>bitcoin/bitcoin</th>\n",
       "      <td>100</td>\n",
       "      <td>100</td>\n",
       "      <td>100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>facebook/react</th>\n",
       "      <td>100</td>\n",
       "      <td>100</td>\n",
       "      <td>100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>microsoft/vscode</th>\n",
       "      <td>100</td>\n",
       "      <td>100</td>\n",
       "      <td>100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>opencv/opencv</th>\n",
       "      <td>100</td>\n",
       "      <td>100</td>\n",
       "      <td>100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>tensorflow/tensorflow</th>\n",
       "      <td>100</td>\n",
       "      <td>100</td>\n",
       "      <td>100</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "label                  bug  feature  question\n",
       "repo                                         \n",
       "bitcoin/bitcoin        100      100       100\n",
       "facebook/react         100      100       100\n",
       "microsoft/vscode       100      100       100\n",
       "opencv/opencv          100      100       100\n",
       "tensorflow/tensorflow  100      100       100"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ds[\"train\"].to_pandas().groupby([\"repo\", \"label\"]).size().unstack(fill_value=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "import re\n",
    "\n",
    "def process_dataset(example):\n",
    "\n",
    "    # concatenate title and body\n",
    "    text = (example['title'] or \"\") + \" \" + (example['body'] or \"\")\n",
    "    \n",
    "    example['text'] = text\n",
    "    return example"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "DatasetDict({\n",
       "    train: Dataset({\n",
       "        features: ['repo', 'label', 'text'],\n",
       "        num_rows: 1500\n",
       "    })\n",
       "    test: Dataset({\n",
       "        features: ['repo', 'label', 'text'],\n",
       "        num_rows: 1500\n",
       "    })\n",
       "})"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ds = ds.shuffle(seed=RANDOM_SEED)\n",
    "ds = ds.map(process_dataset)\n",
    "ds = ds.select_columns(['repo', 'label', 'text'])\n",
    "ds"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Failed to detect the name of this notebook, you can set it manually with the WANDB_NOTEBOOK_NAME environment variable to enable code saving.\n",
      "\u001b[34m\u001b[1mwandb\u001b[0m: Currently logged in as: \u001b[33mrafaelkallis\u001b[0m. Use \u001b[1m`wandb login --relogin`\u001b[0m to force relogin\n",
      "/usr/local/python/3.10.13/lib/python3.10/site-packages/wandb/sdk/lib/ipython.py:77: DeprecationWarning: Importing display from IPython.core.display is deprecated since IPython 7.14, please import from IPython display\n",
      "  from IPython.core.display import HTML, display  # type: ignore\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "Tracking run with wandb version 0.16.2"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "Run data is saved locally in <code>/workspaces/issue-report-classification/wandb/run-20240116_135510-h8ao1gpt</code>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "Syncing run <strong><a href='https://wandb.ai/rafaelkallis/NLBSE%2724%20Issue%20Report%20Classification%20-%20SetFit/runs/h8ao1gpt' target=\"_blank\">facebook/react</a></strong> to <a href='https://wandb.ai/rafaelkallis/NLBSE%2724%20Issue%20Report%20Classification%20-%20SetFit' target=\"_blank\">Weights & Biases</a> (<a href='https://wandb.me/run' target=\"_blank\">docs</a>)<br/>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       " View project at <a href='https://wandb.ai/rafaelkallis/NLBSE%2724%20Issue%20Report%20Classification%20-%20SetFit' target=\"_blank\">https://wandb.ai/rafaelkallis/NLBSE%2724%20Issue%20Report%20Classification%20-%20SetFit</a>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       " View run at <a href='https://wandb.ai/rafaelkallis/NLBSE%2724%20Issue%20Report%20Classification%20-%20SetFit/runs/h8ao1gpt' target=\"_blank\">https://wandb.ai/rafaelkallis/NLBSE%2724%20Issue%20Report%20Classification%20-%20SetFit/runs/h8ao1gpt</a>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "model_head.pkl not found on HuggingFace Hub, initialising classification head with random weights. You should TRAIN this model on a downstream task to use it for predictions and inference.\n",
      "***** Running training *****\n",
      "  Num examples = 750\n",
      "  Num epochs = 1\n",
      "  Total optimization steps = 750\n",
      "  Total train batch size = 16\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='750' max='750' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [750/750 06:42, Epoch 1/0]\n",
       "    </div>\n",
       "    <table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       " <tr style=\"text-align: left;\">\n",
       "      <th>Step</th>\n",
       "      <th>Training Loss</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table><p>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Batches: 100%|██████████| 38/38 [00:01<00:00, 22.72it/s]\n",
      "wandb: WARNING Source type is set to 'repo' but some required information is missing from the environment. A job will not be created from this run. See https://docs.wandb.ai/guides/launch/create-job\n",
      "/usr/local/python/3.10.13/lib/python3.10/site-packages/wandb/sdk/lib/ipython.py:77: DeprecationWarning: Importing display from IPython.core.display is deprecated since IPython 7.14, please import from IPython display\n",
      "  from IPython.core.display import HTML, display  # type: ignore\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<style>\n",
       "    table.wandb td:nth-child(1) { padding: 0 10px; text-align: left ; width: auto;} td:nth-child(2) {text-align: left ; width: 100%}\n",
       "    .wandb-row { display: flex; flex-direction: row; flex-wrap: wrap; justify-content: flex-start; width: 100% }\n",
       "    .wandb-col { display: flex; flex-direction: column; flex-basis: 100%; flex: 1; padding: 10px; }\n",
       "    </style>\n",
       "<div class=\"wandb-row\"><div class=\"wandb-col\"><h3>Run history:</h3><br/><table class=\"wandb\"><tr><td>train/embedding_loss</td><td>█▆▆▆▅▂▂▂▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁</td></tr><tr><td>train/epoch</td><td>▁▁▁▂▂▂▂▂▂▃▃▃▃▃▄▄▄▄▄▄▅▅▅▅▅▅▆▆▆▆▆▇▇▇▇▇▇███</td></tr><tr><td>train/global_step</td><td>▁▁▁▂▂▂▂▂▂▃▃▃▃▃▄▄▄▄▄▄▅▅▅▅▅▅▆▆▆▆▆▇▇▇▇▇▇███</td></tr><tr><td>train/learning_rate</td><td>▂▃▅▇███▇▇▇▇▇▆▆▆▆▆▅▅▅▅▅▄▄▄▄▄▃▃▃▃▃▂▂▂▂▂▁▁▁</td></tr><tr><td>train/train_runtime</td><td>▁</td></tr><tr><td>train/train_samples_per_second</td><td>▁</td></tr><tr><td>train/train_steps_per_second</td><td>▁</td></tr></table><br/></div><div class=\"wandb-col\"><h3>Run summary:</h3><br/><table class=\"wandb\"><tr><td>train/embedding_loss</td><td>0.0007</td></tr><tr><td>train/epoch</td><td>1.0</td></tr><tr><td>train/global_step</td><td>750</td></tr><tr><td>train/learning_rate</td><td>0.0</td></tr><tr><td>train/train_runtime</td><td>403.2109</td></tr><tr><td>train/train_samples_per_second</td><td>29.761</td></tr><tr><td>train/train_steps_per_second</td><td>1.86</td></tr></table><br/></div></div>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       " View run <strong style=\"color:#cdcd00\">facebook/react</strong> at: <a href='https://wandb.ai/rafaelkallis/NLBSE%2724%20Issue%20Report%20Classification%20-%20SetFit/runs/h8ao1gpt' target=\"_blank\">https://wandb.ai/rafaelkallis/NLBSE%2724%20Issue%20Report%20Classification%20-%20SetFit/runs/h8ao1gpt</a><br/>Synced 5 W&B file(s), 0 media file(s), 0 artifact file(s) and 0 other file(s)"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "Find logs at: <code>./wandb/run-20240116_135510-h8ao1gpt/logs</code>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/python/3.10.13/lib/python3.10/site-packages/wandb/sdk/lib/ipython.py:77: DeprecationWarning: Importing display from IPython.core.display is deprecated since IPython 7.14, please import from IPython display\n",
      "  from IPython.core.display import HTML, display  # type: ignore\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "Changes to your `wandb` environment variables will be ignored because your `wandb` session has already started. For more information on how to modify your settings with `wandb.init()` arguments, please refer to <a href='https://wandb.me/wandb-init' target=\"_blank\">the W&B docs</a>."
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/python/3.10.13/lib/python3.10/site-packages/wandb/sdk/lib/ipython.py:77: DeprecationWarning: Importing display from IPython.core.display is deprecated since IPython 7.14, please import from IPython display\n",
      "  from IPython.core.display import HTML, display  # type: ignore\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "Tracking run with wandb version 0.16.2"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "Run data is saved locally in <code>/workspaces/issue-report-classification/wandb/run-20240116_140203-4net7f7w</code>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "Syncing run <strong><a href='https://wandb.ai/rafaelkallis/NLBSE%2724%20Issue%20Report%20Classification%20-%20SetFit/runs/4net7f7w' target=\"_blank\">tensorflow/tensorflow</a></strong> to <a href='https://wandb.ai/rafaelkallis/NLBSE%2724%20Issue%20Report%20Classification%20-%20SetFit' target=\"_blank\">Weights & Biases</a> (<a href='https://wandb.me/run' target=\"_blank\">docs</a>)<br/>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       " View project at <a href='https://wandb.ai/rafaelkallis/NLBSE%2724%20Issue%20Report%20Classification%20-%20SetFit' target=\"_blank\">https://wandb.ai/rafaelkallis/NLBSE%2724%20Issue%20Report%20Classification%20-%20SetFit</a>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       " View run at <a href='https://wandb.ai/rafaelkallis/NLBSE%2724%20Issue%20Report%20Classification%20-%20SetFit/runs/4net7f7w' target=\"_blank\">https://wandb.ai/rafaelkallis/NLBSE%2724%20Issue%20Report%20Classification%20-%20SetFit/runs/4net7f7w</a>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Filter: 100%|██████████| 1500/1500 [00:00<00:00, 174699.58 examples/s]\n",
      "Filter: 100%|██████████| 1500/1500 [00:00<00:00, 195314.04 examples/s]\n",
      "model_head.pkl not found on HuggingFace Hub, initialising classification head with random weights. You should TRAIN this model on a downstream task to use it for predictions and inference.\n",
      "Map: 100%|██████████| 300/300 [00:00<00:00, 5668.39 examples/s]\n",
      "***** Running training *****\n",
      "  Num examples = 750\n",
      "  Num epochs = 1\n",
      "  Total optimization steps = 750\n",
      "  Total train batch size = 16\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='750' max='750' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [750/750 06:57, Epoch 1/0]\n",
       "    </div>\n",
       "    <table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       " <tr style=\"text-align: left;\">\n",
       "      <th>Step</th>\n",
       "      <th>Training Loss</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table><p>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Batches: 100%|██████████| 38/38 [00:01<00:00, 19.99it/s]\n",
      "wandb: WARNING Source type is set to 'repo' but some required information is missing from the environment. A job will not be created from this run. See https://docs.wandb.ai/guides/launch/create-job\n",
      "/usr/local/python/3.10.13/lib/python3.10/site-packages/wandb/sdk/lib/ipython.py:77: DeprecationWarning: Importing display from IPython.core.display is deprecated since IPython 7.14, please import from IPython display\n",
      "  from IPython.core.display import HTML, display  # type: ignore\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<style>\n",
       "    table.wandb td:nth-child(1) { padding: 0 10px; text-align: left ; width: auto;} td:nth-child(2) {text-align: left ; width: 100%}\n",
       "    .wandb-row { display: flex; flex-direction: row; flex-wrap: wrap; justify-content: flex-start; width: 100% }\n",
       "    .wandb-col { display: flex; flex-direction: column; flex-basis: 100%; flex: 1; padding: 10px; }\n",
       "    </style>\n",
       "<div class=\"wandb-row\"><div class=\"wandb-col\"><h3>Run history:</h3><br/><table class=\"wandb\"><tr><td>train/embedding_loss</td><td>██▇█▃▃▃▂▃▁▁▃▁▁▁▁▁▁▁▁▁▁▁▁▁▁▂▁▁▁▁▂▂▁▁▁▁▁▁▁</td></tr><tr><td>train/epoch</td><td>▁▁▁▂▂▂▂▂▂▃▃▃▃▃▄▄▄▄▄▄▅▅▅▅▅▅▆▆▆▆▆▇▇▇▇▇▇███</td></tr><tr><td>train/global_step</td><td>▁▁▁▂▂▂▂▂▂▃▃▃▃▃▄▄▄▄▄▄▅▅▅▅▅▅▆▆▆▆▆▇▇▇▇▇▇███</td></tr><tr><td>train/learning_rate</td><td>▂▃▅▇███▇▇▇▇▇▆▆▆▆▆▅▅▅▅▅▄▄▄▄▄▃▃▃▃▃▂▂▂▂▂▁▁▁</td></tr><tr><td>train/train_runtime</td><td>▁</td></tr><tr><td>train/train_samples_per_second</td><td>▁</td></tr><tr><td>train/train_steps_per_second</td><td>▁</td></tr></table><br/></div><div class=\"wandb-col\"><h3>Run summary:</h3><br/><table class=\"wandb\"><tr><td>train/embedding_loss</td><td>0.0001</td></tr><tr><td>train/epoch</td><td>1.0</td></tr><tr><td>train/global_step</td><td>750</td></tr><tr><td>train/learning_rate</td><td>0.0</td></tr><tr><td>train/train_runtime</td><td>417.4734</td></tr><tr><td>train/train_samples_per_second</td><td>28.744</td></tr><tr><td>train/train_steps_per_second</td><td>1.797</td></tr></table><br/></div></div>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       " View run <strong style=\"color:#cdcd00\">tensorflow/tensorflow</strong> at: <a href='https://wandb.ai/rafaelkallis/NLBSE%2724%20Issue%20Report%20Classification%20-%20SetFit/runs/4net7f7w' target=\"_blank\">https://wandb.ai/rafaelkallis/NLBSE%2724%20Issue%20Report%20Classification%20-%20SetFit/runs/4net7f7w</a><br/>Synced 5 W&B file(s), 0 media file(s), 0 artifact file(s) and 0 other file(s)"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "Find logs at: <code>./wandb/run-20240116_140203-4net7f7w/logs</code>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/python/3.10.13/lib/python3.10/site-packages/wandb/sdk/lib/ipython.py:77: DeprecationWarning: Importing display from IPython.core.display is deprecated since IPython 7.14, please import from IPython display\n",
      "  from IPython.core.display import HTML, display  # type: ignore\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "Changes to your `wandb` environment variables will be ignored because your `wandb` session has already started. For more information on how to modify your settings with `wandb.init()` arguments, please refer to <a href='https://wandb.me/wandb-init' target=\"_blank\">the W&B docs</a>."
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/python/3.10.13/lib/python3.10/site-packages/wandb/sdk/lib/ipython.py:77: DeprecationWarning: Importing display from IPython.core.display is deprecated since IPython 7.14, please import from IPython display\n",
      "  from IPython.core.display import HTML, display  # type: ignore\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "Tracking run with wandb version 0.16.2"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "Run data is saved locally in <code>/workspaces/issue-report-classification/wandb/run-20240116_140912-he3uy24e</code>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "Syncing run <strong><a href='https://wandb.ai/rafaelkallis/NLBSE%2724%20Issue%20Report%20Classification%20-%20SetFit/runs/he3uy24e' target=\"_blank\">microsoft/vscode</a></strong> to <a href='https://wandb.ai/rafaelkallis/NLBSE%2724%20Issue%20Report%20Classification%20-%20SetFit' target=\"_blank\">Weights & Biases</a> (<a href='https://wandb.me/run' target=\"_blank\">docs</a>)<br/>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       " View project at <a href='https://wandb.ai/rafaelkallis/NLBSE%2724%20Issue%20Report%20Classification%20-%20SetFit' target=\"_blank\">https://wandb.ai/rafaelkallis/NLBSE%2724%20Issue%20Report%20Classification%20-%20SetFit</a>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       " View run at <a href='https://wandb.ai/rafaelkallis/NLBSE%2724%20Issue%20Report%20Classification%20-%20SetFit/runs/he3uy24e' target=\"_blank\">https://wandb.ai/rafaelkallis/NLBSE%2724%20Issue%20Report%20Classification%20-%20SetFit/runs/he3uy24e</a>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Filter: 100%|██████████| 1500/1500 [00:00<00:00, 168975.26 examples/s]\n",
      "Filter: 100%|██████████| 1500/1500 [00:00<00:00, 172927.71 examples/s]\n",
      "model_head.pkl not found on HuggingFace Hub, initialising classification head with random weights. You should TRAIN this model on a downstream task to use it for predictions and inference.\n",
      "Map: 100%|██████████| 300/300 [00:00<00:00, 7208.28 examples/s]\n",
      "***** Running training *****\n",
      "  Num examples = 750\n",
      "  Num epochs = 1\n",
      "  Total optimization steps = 750\n",
      "  Total train batch size = 16\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='750' max='750' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [750/750 06:49, Epoch 1/0]\n",
       "    </div>\n",
       "    <table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       " <tr style=\"text-align: left;\">\n",
       "      <th>Step</th>\n",
       "      <th>Training Loss</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table><p>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Batches: 100%|██████████| 38/38 [00:01<00:00, 26.28it/s]\n",
      "wandb: WARNING Source type is set to 'repo' but some required information is missing from the environment. A job will not be created from this run. See https://docs.wandb.ai/guides/launch/create-job\n",
      "/usr/local/python/3.10.13/lib/python3.10/site-packages/wandb/sdk/lib/ipython.py:77: DeprecationWarning: Importing display from IPython.core.display is deprecated since IPython 7.14, please import from IPython display\n",
      "  from IPython.core.display import HTML, display  # type: ignore\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<style>\n",
       "    table.wandb td:nth-child(1) { padding: 0 10px; text-align: left ; width: auto;} td:nth-child(2) {text-align: left ; width: 100%}\n",
       "    .wandb-row { display: flex; flex-direction: row; flex-wrap: wrap; justify-content: flex-start; width: 100% }\n",
       "    .wandb-col { display: flex; flex-direction: column; flex-basis: 100%; flex: 1; padding: 10px; }\n",
       "    </style>\n",
       "<div class=\"wandb-row\"><div class=\"wandb-col\"><h3>Run history:</h3><br/><table class=\"wandb\"><tr><td>train/embedding_loss</td><td>█▆▆▃▃▄▃▃▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁</td></tr><tr><td>train/epoch</td><td>▁▁▁▂▂▂▂▂▂▃▃▃▃▃▄▄▄▄▄▄▅▅▅▅▅▅▆▆▆▆▆▇▇▇▇▇▇███</td></tr><tr><td>train/global_step</td><td>▁▁▁▂▂▂▂▂▂▃▃▃▃▃▄▄▄▄▄▄▅▅▅▅▅▅▆▆▆▆▆▇▇▇▇▇▇███</td></tr><tr><td>train/learning_rate</td><td>▂▃▅▇███▇▇▇▇▇▆▆▆▆▆▅▅▅▅▅▄▄▄▄▄▃▃▃▃▃▂▂▂▂▂▁▁▁</td></tr><tr><td>train/train_runtime</td><td>▁</td></tr><tr><td>train/train_samples_per_second</td><td>▁</td></tr><tr><td>train/train_steps_per_second</td><td>▁</td></tr></table><br/></div><div class=\"wandb-col\"><h3>Run summary:</h3><br/><table class=\"wandb\"><tr><td>train/embedding_loss</td><td>0.0001</td></tr><tr><td>train/epoch</td><td>1.0</td></tr><tr><td>train/global_step</td><td>750</td></tr><tr><td>train/learning_rate</td><td>0.0</td></tr><tr><td>train/train_runtime</td><td>409.5611</td></tr><tr><td>train/train_samples_per_second</td><td>29.3</td></tr><tr><td>train/train_steps_per_second</td><td>1.831</td></tr></table><br/></div></div>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       " View run <strong style=\"color:#cdcd00\">microsoft/vscode</strong> at: <a href='https://wandb.ai/rafaelkallis/NLBSE%2724%20Issue%20Report%20Classification%20-%20SetFit/runs/he3uy24e' target=\"_blank\">https://wandb.ai/rafaelkallis/NLBSE%2724%20Issue%20Report%20Classification%20-%20SetFit/runs/he3uy24e</a><br/>Synced 5 W&B file(s), 0 media file(s), 0 artifact file(s) and 0 other file(s)"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "Find logs at: <code>./wandb/run-20240116_140912-he3uy24e/logs</code>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/python/3.10.13/lib/python3.10/site-packages/wandb/sdk/lib/ipython.py:77: DeprecationWarning: Importing display from IPython.core.display is deprecated since IPython 7.14, please import from IPython display\n",
      "  from IPython.core.display import HTML, display  # type: ignore\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "Changes to your `wandb` environment variables will be ignored because your `wandb` session has already started. For more information on how to modify your settings with `wandb.init()` arguments, please refer to <a href='https://wandb.me/wandb-init' target=\"_blank\">the W&B docs</a>."
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/python/3.10.13/lib/python3.10/site-packages/wandb/sdk/lib/ipython.py:77: DeprecationWarning: Importing display from IPython.core.display is deprecated since IPython 7.14, please import from IPython display\n",
      "  from IPython.core.display import HTML, display  # type: ignore\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "Tracking run with wandb version 0.16.2"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "Run data is saved locally in <code>/workspaces/issue-report-classification/wandb/run-20240116_141613-5v9qs1fj</code>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "Syncing run <strong><a href='https://wandb.ai/rafaelkallis/NLBSE%2724%20Issue%20Report%20Classification%20-%20SetFit/runs/5v9qs1fj' target=\"_blank\">bitcoin/bitcoin</a></strong> to <a href='https://wandb.ai/rafaelkallis/NLBSE%2724%20Issue%20Report%20Classification%20-%20SetFit' target=\"_blank\">Weights & Biases</a> (<a href='https://wandb.me/run' target=\"_blank\">docs</a>)<br/>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       " View project at <a href='https://wandb.ai/rafaelkallis/NLBSE%2724%20Issue%20Report%20Classification%20-%20SetFit' target=\"_blank\">https://wandb.ai/rafaelkallis/NLBSE%2724%20Issue%20Report%20Classification%20-%20SetFit</a>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       " View run at <a href='https://wandb.ai/rafaelkallis/NLBSE%2724%20Issue%20Report%20Classification%20-%20SetFit/runs/5v9qs1fj' target=\"_blank\">https://wandb.ai/rafaelkallis/NLBSE%2724%20Issue%20Report%20Classification%20-%20SetFit/runs/5v9qs1fj</a>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Filter: 100%|██████████| 1500/1500 [00:00<00:00, 163869.87 examples/s]\n",
      "Filter: 100%|██████████| 1500/1500 [00:00<00:00, 168009.61 examples/s]\n",
      "model_head.pkl not found on HuggingFace Hub, initialising classification head with random weights. You should TRAIN this model on a downstream task to use it for predictions and inference.\n",
      "Map: 100%|██████████| 300/300 [00:00<00:00, 6401.43 examples/s]\n",
      "***** Running training *****\n",
      "  Num examples = 750\n",
      "  Num epochs = 1\n",
      "  Total optimization steps = 750\n",
      "  Total train batch size = 16\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='750' max='750' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [750/750 07:00, Epoch 1/0]\n",
       "    </div>\n",
       "    <table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       " <tr style=\"text-align: left;\">\n",
       "      <th>Step</th>\n",
       "      <th>Training Loss</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table><p>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Batches: 100%|██████████| 38/38 [00:01<00:00, 23.26it/s]\n",
      "wandb: WARNING Source type is set to 'repo' but some required information is missing from the environment. A job will not be created from this run. See https://docs.wandb.ai/guides/launch/create-job\n",
      "/usr/local/python/3.10.13/lib/python3.10/site-packages/wandb/sdk/lib/ipython.py:77: DeprecationWarning: Importing display from IPython.core.display is deprecated since IPython 7.14, please import from IPython display\n",
      "  from IPython.core.display import HTML, display  # type: ignore\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<style>\n",
       "    table.wandb td:nth-child(1) { padding: 0 10px; text-align: left ; width: auto;} td:nth-child(2) {text-align: left ; width: 100%}\n",
       "    .wandb-row { display: flex; flex-direction: row; flex-wrap: wrap; justify-content: flex-start; width: 100% }\n",
       "    .wandb-col { display: flex; flex-direction: column; flex-basis: 100%; flex: 1; padding: 10px; }\n",
       "    </style>\n",
       "<div class=\"wandb-row\"><div class=\"wandb-col\"><h3>Run history:</h3><br/><table class=\"wandb\"><tr><td>train/embedding_loss</td><td>█▇▇▇▃▅▆▄▃▅▂▂▂▃▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁</td></tr><tr><td>train/epoch</td><td>▁▁▁▂▂▂▂▂▂▃▃▃▃▃▄▄▄▄▄▄▅▅▅▅▅▅▆▆▆▆▆▇▇▇▇▇▇███</td></tr><tr><td>train/global_step</td><td>▁▁▁▂▂▂▂▂▂▃▃▃▃▃▄▄▄▄▄▄▅▅▅▅▅▅▆▆▆▆▆▇▇▇▇▇▇███</td></tr><tr><td>train/learning_rate</td><td>▂▃▅▇███▇▇▇▇▇▆▆▆▆▆▅▅▅▅▅▄▄▄▄▄▃▃▃▃▃▂▂▂▂▂▁▁▁</td></tr><tr><td>train/train_runtime</td><td>▁</td></tr><tr><td>train/train_samples_per_second</td><td>▁</td></tr><tr><td>train/train_steps_per_second</td><td>▁</td></tr></table><br/></div><div class=\"wandb-col\"><h3>Run summary:</h3><br/><table class=\"wandb\"><tr><td>train/embedding_loss</td><td>0.0003</td></tr><tr><td>train/epoch</td><td>1.0</td></tr><tr><td>train/global_step</td><td>750</td></tr><tr><td>train/learning_rate</td><td>0.0</td></tr><tr><td>train/train_runtime</td><td>421.5564</td></tr><tr><td>train/train_samples_per_second</td><td>28.466</td></tr><tr><td>train/train_steps_per_second</td><td>1.779</td></tr></table><br/></div></div>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       " View run <strong style=\"color:#cdcd00\">bitcoin/bitcoin</strong> at: <a href='https://wandb.ai/rafaelkallis/NLBSE%2724%20Issue%20Report%20Classification%20-%20SetFit/runs/5v9qs1fj' target=\"_blank\">https://wandb.ai/rafaelkallis/NLBSE%2724%20Issue%20Report%20Classification%20-%20SetFit/runs/5v9qs1fj</a><br/>Synced 5 W&B file(s), 0 media file(s), 0 artifact file(s) and 0 other file(s)"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "Find logs at: <code>./wandb/run-20240116_141613-5v9qs1fj/logs</code>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/python/3.10.13/lib/python3.10/site-packages/wandb/sdk/lib/ipython.py:77: DeprecationWarning: Importing display from IPython.core.display is deprecated since IPython 7.14, please import from IPython display\n",
      "  from IPython.core.display import HTML, display  # type: ignore\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "Changes to your `wandb` environment variables will be ignored because your `wandb` session has already started. For more information on how to modify your settings with `wandb.init()` arguments, please refer to <a href='https://wandb.me/wandb-init' target=\"_blank\">the W&B docs</a>."
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/python/3.10.13/lib/python3.10/site-packages/wandb/sdk/lib/ipython.py:77: DeprecationWarning: Importing display from IPython.core.display is deprecated since IPython 7.14, please import from IPython display\n",
      "  from IPython.core.display import HTML, display  # type: ignore\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "Tracking run with wandb version 0.16.2"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "Run data is saved locally in <code>/workspaces/issue-report-classification/wandb/run-20240116_142331-6raktlfg</code>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "Syncing run <strong><a href='https://wandb.ai/rafaelkallis/NLBSE%2724%20Issue%20Report%20Classification%20-%20SetFit/runs/6raktlfg' target=\"_blank\">opencv/opencv</a></strong> to <a href='https://wandb.ai/rafaelkallis/NLBSE%2724%20Issue%20Report%20Classification%20-%20SetFit' target=\"_blank\">Weights & Biases</a> (<a href='https://wandb.me/run' target=\"_blank\">docs</a>)<br/>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       " View project at <a href='https://wandb.ai/rafaelkallis/NLBSE%2724%20Issue%20Report%20Classification%20-%20SetFit' target=\"_blank\">https://wandb.ai/rafaelkallis/NLBSE%2724%20Issue%20Report%20Classification%20-%20SetFit</a>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       " View run at <a href='https://wandb.ai/rafaelkallis/NLBSE%2724%20Issue%20Report%20Classification%20-%20SetFit/runs/6raktlfg' target=\"_blank\">https://wandb.ai/rafaelkallis/NLBSE%2724%20Issue%20Report%20Classification%20-%20SetFit/runs/6raktlfg</a>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Filter: 100%|██████████| 1500/1500 [00:00<00:00, 125135.87 examples/s]\n",
      "Filter: 100%|██████████| 1500/1500 [00:00<00:00, 132714.34 examples/s]\n",
      "model_head.pkl not found on HuggingFace Hub, initialising classification head with random weights. You should TRAIN this model on a downstream task to use it for predictions and inference.\n",
      "Map: 100%|██████████| 300/300 [00:00<00:00, 4178.23 examples/s]\n",
      "***** Running training *****\n",
      "  Num examples = 750\n",
      "  Num epochs = 1\n",
      "  Total optimization steps = 750\n",
      "  Total train batch size = 16\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='750' max='750' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [750/750 07:06, Epoch 1/0]\n",
       "    </div>\n",
       "    <table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       " <tr style=\"text-align: left;\">\n",
       "      <th>Step</th>\n",
       "      <th>Training Loss</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table><p>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Batches: 100%|██████████| 38/38 [00:01<00:00, 22.01it/s]\n",
      "wandb: WARNING Source type is set to 'repo' but some required information is missing from the environment. A job will not be created from this run. See https://docs.wandb.ai/guides/launch/create-job\n",
      "/usr/local/python/3.10.13/lib/python3.10/site-packages/wandb/sdk/lib/ipython.py:77: DeprecationWarning: Importing display from IPython.core.display is deprecated since IPython 7.14, please import from IPython display\n",
      "  from IPython.core.display import HTML, display  # type: ignore\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<style>\n",
       "    table.wandb td:nth-child(1) { padding: 0 10px; text-align: left ; width: auto;} td:nth-child(2) {text-align: left ; width: 100%}\n",
       "    .wandb-row { display: flex; flex-direction: row; flex-wrap: wrap; justify-content: flex-start; width: 100% }\n",
       "    .wandb-col { display: flex; flex-direction: column; flex-basis: 100%; flex: 1; padding: 10px; }\n",
       "    </style>\n",
       "<div class=\"wandb-row\"><div class=\"wandb-col\"><h3>Run history:</h3><br/><table class=\"wandb\"><tr><td>train/embedding_loss</td><td>▇█▅▆▅▃▅▄▂▁▂▁▁▁▁▁▁▁▁▁▁▁▁▁▁▂▁▁▁▁▁▁▁▁▁▁▁▁▁▁</td></tr><tr><td>train/epoch</td><td>▁▁▁▂▂▂▂▂▂▃▃▃▃▃▄▄▄▄▄▄▅▅▅▅▅▅▆▆▆▆▆▇▇▇▇▇▇███</td></tr><tr><td>train/global_step</td><td>▁▁▁▂▂▂▂▂▂▃▃▃▃▃▄▄▄▄▄▄▅▅▅▅▅▅▆▆▆▆▆▇▇▇▇▇▇███</td></tr><tr><td>train/learning_rate</td><td>▂▃▅▇███▇▇▇▇▇▆▆▆▆▆▅▅▅▅▅▄▄▄▄▄▃▃▃▃▃▂▂▂▂▂▁▁▁</td></tr><tr><td>train/train_runtime</td><td>▁</td></tr><tr><td>train/train_samples_per_second</td><td>▁</td></tr><tr><td>train/train_steps_per_second</td><td>▁</td></tr></table><br/></div><div class=\"wandb-col\"><h3>Run summary:</h3><br/><table class=\"wandb\"><tr><td>train/embedding_loss</td><td>0.0003</td></tr><tr><td>train/epoch</td><td>1.0</td></tr><tr><td>train/global_step</td><td>750</td></tr><tr><td>train/learning_rate</td><td>0.0</td></tr><tr><td>train/train_runtime</td><td>427.1565</td></tr><tr><td>train/train_samples_per_second</td><td>28.093</td></tr><tr><td>train/train_steps_per_second</td><td>1.756</td></tr></table><br/></div></div>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       " View run <strong style=\"color:#cdcd00\">opencv/opencv</strong> at: <a href='https://wandb.ai/rafaelkallis/NLBSE%2724%20Issue%20Report%20Classification%20-%20SetFit/runs/6raktlfg' target=\"_blank\">https://wandb.ai/rafaelkallis/NLBSE%2724%20Issue%20Report%20Classification%20-%20SetFit/runs/6raktlfg</a><br/>Synced 5 W&B file(s), 0 media file(s), 0 artifact file(s) and 0 other file(s)"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "Find logs at: <code>./wandb/run-20240116_142331-6raktlfg/logs</code>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import wandb\n",
    "from datetime import datetime\n",
    "from setfit import SetFitModel, Trainer, TrainingArguments\n",
    "\n",
    "group = datetime.utcnow().replace(microsecond=0).isoformat()\n",
    "\n",
    "references = {}\n",
    "predictions = {}\n",
    "\n",
    "def repo_eq(repo: str):\n",
    "    return lambda example: example['repo'] == repo\n",
    "\n",
    "for repo in repos:\n",
    "    wandb.init(\n",
    "        project=\"NLBSE'24 Issue Report Classification - SetFit\", \n",
    "        group=group,\n",
    "        name=repo, \n",
    "    )\n",
    "\n",
    "    train_set = ds.filter(repo_eq(repo))['train']\n",
    "\n",
    "    model = SetFitModel.from_pretrained(BASE_MODEL)\n",
    "\n",
    "    args = TrainingArguments(\n",
    "        output_dir=f'{OUTPUT_PATH}/{repo.replace(\"/\", \"-\")}',\n",
    "        save_strategy=\"no\",\n",
    "        report_to=\"wandb\",\n",
    "        run_name=repo,\n",
    "        logging_steps=1,\n",
    "        seed=RANDOM_SEED,\n",
    "        batch_size=(16, 2),\n",
    "        num_epochs=1,\n",
    "        num_iterations=20,\n",
    "    )\n",
    "\n",
    "    trainer = Trainer(\n",
    "        model=model,\n",
    "        args=args,\n",
    "        train_dataset=train_set,\n",
    "    )\n",
    "\n",
    "    trainer.train()\n",
    "    \n",
    "    test_set = ds.filter(repo_eq(repo))['test']\n",
    "\n",
    "    references[repo] = list(test_set['label'])\n",
    "\n",
    "    predictions[repo] = list(model.predict(test_set['text'], batch_size=8, show_progress_bar=True))\n",
    "\n",
    "    wandb.finish()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import classification_report\n",
    "from numpy import mean\n",
    "\n",
    "results = {}\n",
    "metrics = ['precision', 'recall', 'f1-score']\n",
    "labels = ['bug', 'feature', 'question']\n",
    "\n",
    "for repo in repos:\n",
    "  results[repo] = classification_report(references[repo], predictions[repo], digits=4, output_dict=True)\n",
    "  results[repo]['average'] = results[repo]['weighted avg']\n",
    "  results[repo] = {label: {metric: results[repo][label][metric] for metric in metrics} for label in labels + ['average']}\n",
    "\n",
    "results['overall'] = {label: {metric: mean([results[repo][label][metric] for repo in repos]) for metric in metrics} for label in labels + ['average']}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Repository               Label     Precision  Recall     F1\n",
      "---------------------------------------------------------------\n",
      "facebook/react           bug       0.9065     0.9700     0.9372     \n",
      "facebook/react           feature   0.8627     0.8800     0.8713     \n",
      "facebook/react           question  0.8571     0.7800     0.8168     \n",
      "facebook/react           average   0.8755     0.8767     0.8751     \n",
      "---------------------------------------------------------------\n",
      "tensorflow/tensorflow    bug       0.9375     0.9000     0.9184     \n",
      "tensorflow/tensorflow    feature   0.8958     0.8600     0.8776     \n",
      "tensorflow/tensorflow    question  0.7778     0.8400     0.8077     \n",
      "tensorflow/tensorflow    average   0.8704     0.8667     0.8679     \n",
      "---------------------------------------------------------------\n",
      "microsoft/vscode         bug       0.8061     0.7900     0.7980     \n",
      "microsoft/vscode         feature   0.7565     0.8700     0.8093     \n",
      "microsoft/vscode         question  0.8621     0.7500     0.8021     \n",
      "microsoft/vscode         average   0.8082     0.8033     0.8031     \n",
      "---------------------------------------------------------------\n",
      "bitcoin/bitcoin          bug       0.7353     0.7500     0.7426     \n",
      "bitcoin/bitcoin          feature   0.8660     0.8400     0.8528     \n",
      "bitcoin/bitcoin          question  0.6535     0.6600     0.6567     \n",
      "bitcoin/bitcoin          average   0.7516     0.7500     0.7507     \n",
      "---------------------------------------------------------------\n",
      "opencv/opencv            bug       0.7812     0.7500     0.7653     \n",
      "opencv/opencv            feature   0.8889     0.8800     0.8844     \n",
      "opencv/opencv            question  0.8000     0.8400     0.8195     \n",
      "opencv/opencv            average   0.8234     0.8233     0.8231     \n",
      "---------------------------------------------------------------\n",
      "overall                  bug       0.8333     0.8320     0.8323     \n",
      "overall                  feature   0.8540     0.8660     0.8591     \n",
      "overall                  question  0.7901     0.7740     0.7806     \n",
      "overall                  average   0.8258     0.8240     0.8240     \n"
     ]
    }
   ],
   "source": [
    "import json\n",
    "import os\n",
    "\n",
    "output_file_name = 'results.json'\n",
    "\n",
    "with open(os.path.join(OUTPUT_PATH, output_file_name), 'w') as fp:\n",
    "    json.dump(results, fp, indent=2)\n",
    "\n",
    "print(f\"Repository{' '*15}Label     Precision  Recall     F1\")\n",
    "for repo in repos + ['overall']:\n",
    "  print(\"-\"*63)\n",
    "  for label in labels + ['average']:\n",
    "    out = f\"{repo:<25}{label:<10}\"\n",
    "    for metric in metrics:\n",
    "      out += f\"{results[repo][label][metric]:<10.4f} \"\n",
    "    print(out)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}