3-Template-RoBERTa.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### NLBSE'24 RoBERTa Baseline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "BASE_MODEL = \"roberta-base\"\n",
    "RANDOM_SEED = 42\n",
    "OUTPUT_PATH = 'output/roberta'\n",
    "\n",
    "!mkdir -p $OUTPUT_PATH"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/python/3.10.13/lib/python3.10/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
      "  from .autonotebook import tqdm as notebook_tqdm\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "DatasetDict({\n",
       "    train: Dataset({\n",
       "        features: ['repo', 'created_at', 'label', 'title', 'body'],\n",
       "        num_rows: 1500\n",
       "    })\n",
       "    test: Dataset({\n",
       "        features: ['repo', 'created_at', 'label', 'title', 'body'],\n",
       "        num_rows: 1500\n",
       "    })\n",
       "})"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from datasets import Dataset\n",
    "\n",
    "ds = Dataset.from_csv({ \"train\": \"data/issues_train.csv\", \"test\": \"data/issues_test.csv\" })\n",
    "ds"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['facebook/react', 'tensorflow/tensorflow', 'microsoft/vscode', 'bitcoin/bitcoin', 'opencv/opencv']\n"
     ]
    }
   ],
   "source": [
    "repos = ds[\"train\"].unique(\"repo\")\n",
    "print(repos)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th>label</th>\n",
       "      <th>bug</th>\n",
       "      <th>feature</th>\n",
       "      <th>question</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>repo</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>bitcoin/bitcoin</th>\n",
       "      <td>100</td>\n",
       "      <td>100</td>\n",
       "      <td>100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>facebook/react</th>\n",
       "      <td>100</td>\n",
       "      <td>100</td>\n",
       "      <td>100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>microsoft/vscode</th>\n",
       "      <td>100</td>\n",
       "      <td>100</td>\n",
       "      <td>100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>opencv/opencv</th>\n",
       "      <td>100</td>\n",
       "      <td>100</td>\n",
       "      <td>100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>tensorflow/tensorflow</th>\n",
       "      <td>100</td>\n",
       "      <td>100</td>\n",
       "      <td>100</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "label                  bug  feature  question\n",
       "repo                                         \n",
       "bitcoin/bitcoin        100      100       100\n",
       "facebook/react         100      100       100\n",
       "microsoft/vscode       100      100       100\n",
       "opencv/opencv          100      100       100\n",
       "tensorflow/tensorflow  100      100       100"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ds[\"train\"].to_pandas().groupby([\"repo\", \"label\"]).size().unstack(fill_value=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "import re\n",
    "\n",
    "label2id = { \"bug\": 0, \"feature\": 1, \"question\": 2 }\n",
    "id2label = { 0: \"bug\", 1: \"feature\", 2: \"question\" }\n",
    "\n",
    "def process_dataset(example):\n",
    "\n",
    "    example['label'] = label2id[example['label']]\n",
    "\n",
    "    # concatenate title and body\n",
    "    text = (example['title'] or \"\") + \" \" + (example['body'] or \"\")\n",
    "\n",
    "    # Remove strings between triple quotes\n",
    "    text = re.sub(r'```.*?```', ' ', text, flags=re.DOTALL)\n",
    "\n",
    "    # Remove new lines\n",
    "    text = re.sub(r'\\n', ' ', text)\n",
    "\n",
    "    # Remove links\n",
    "    text = re.sub(r'http[s]?://(?:[a-zA-Z]|[0-9]|[$-_@.&+]|[!*\\\\(\\\\),]|(?:%[0-9a-fA-F][0-9a-fA-F]))+', ' ', text)\n",
    "\n",
    "    # Remove digits\n",
    "    text = re.sub(r'\\d+', ' ', text)\n",
    "\n",
    "    # Remove special characters except the question marks\n",
    "    text = re.sub(r'[^a-zA-Z0-9?\\s]', ' ', text)\n",
    "    text = re.sub(r'\\s+', ' ', text)\n",
    "    \n",
    "    example['text'] = text\n",
    "\n",
    "    return example\n",
    "\n",
    "ds = ds.shuffle(seed=RANDOM_SEED)\n",
    "ds = ds.map(process_dataset)\n",
    "ds = ds.select_columns(['repo', 'label', 'text'])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Failed to detect the name of this notebook, you can set it manually with the WANDB_NOTEBOOK_NAME environment variable to enable code saving.\n",
      "\u001b[34m\u001b[1mwandb\u001b[0m: Currently logged in as: \u001b[33mrafaelkallis\u001b[0m. Use \u001b[1m`wandb login --relogin`\u001b[0m to force relogin\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "Tracking run with wandb version 0.16.2"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "Run data is saved locally in <code>/workspaces/issue-report-classification/wandb/run-20240116_210749-2pg2vivk</code>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "Syncing run <strong><a href='https://wandb.ai/rafaelkallis/NLBSE%2724%20Issue%20Report%20Classification%20-%20RoBERTa/runs/2pg2vivk' target=\"_blank\">facebook/react</a></strong> to <a href='https://wandb.ai/rafaelkallis/NLBSE%2724%20Issue%20Report%20Classification%20-%20RoBERTa' target=\"_blank\">Weights & Biases</a> (<a href='https://wandb.me/run' target=\"_blank\">docs</a>)<br/>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       " View project at <a href='https://wandb.ai/rafaelkallis/NLBSE%2724%20Issue%20Report%20Classification%20-%20RoBERTa' target=\"_blank\">https://wandb.ai/rafaelkallis/NLBSE%2724%20Issue%20Report%20Classification%20-%20RoBERTa</a>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       " View run at <a href='https://wandb.ai/rafaelkallis/NLBSE%2724%20Issue%20Report%20Classification%20-%20RoBERTa/runs/2pg2vivk' target=\"_blank\">https://wandb.ai/rafaelkallis/NLBSE%2724%20Issue%20Report%20Classification%20-%20RoBERTa/runs/2pg2vivk</a>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Some weights of RobertaForSequenceClassification were not initialized from the model checkpoint at roberta-base and are newly initialized: ['classifier.out_proj.bias', 'classifier.dense.bias', 'classifier.dense.weight', 'classifier.out_proj.weight']\n",
      "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n",
      "You're using a RobertaTokenizerFast tokenizer. Please note that with a fast tokenizer, using the `__call__` method is faster than using a method to encode the text followed by a call to the `pad` method to get a padded encoding.\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='190' max='190' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [190/190 01:29, Epoch 10/10]\n",
       "    </div>\n",
       "    <table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       " <tr style=\"text-align: left;\">\n",
       "      <th>Epoch</th>\n",
       "      <th>Training Loss</th>\n",
       "      <th>Validation Loss</th>\n",
       "      <th>F1</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>0.889000</td>\n",
       "      <td>0.683371</td>\n",
       "      <td>0.505019</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>0.682300</td>\n",
       "      <td>0.613906</td>\n",
       "      <td>0.639636</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>0.546800</td>\n",
       "      <td>0.547347</td>\n",
       "      <td>0.789345</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>0.432200</td>\n",
       "      <td>0.551864</td>\n",
       "      <td>0.760293</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>5</td>\n",
       "      <td>0.311100</td>\n",
       "      <td>0.650390</td>\n",
       "      <td>0.750040</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>6</td>\n",
       "      <td>0.113300</td>\n",
       "      <td>0.607732</td>\n",
       "      <td>0.839547</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>7</td>\n",
       "      <td>0.076900</td>\n",
       "      <td>0.873719</td>\n",
       "      <td>0.827400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>8</td>\n",
       "      <td>0.021200</td>\n",
       "      <td>0.854718</td>\n",
       "      <td>0.841026</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>9</td>\n",
       "      <td>0.003900</td>\n",
       "      <td>0.868349</td>\n",
       "      <td>0.843438</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>10</td>\n",
       "      <td>0.003000</td>\n",
       "      <td>0.881902</td>\n",
       "      <td>0.844094</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table><p>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "wandb: WARNING Source type is set to 'repo' but some required information is missing from the environment. A job will not be created from this run. See https://docs.wandb.ai/guides/launch/create-job\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<style>\n",
       "    table.wandb td:nth-child(1) { padding: 0 10px; text-align: left ; width: auto;} td:nth-child(2) {text-align: left ; width: 100%}\n",
       "    .wandb-row { display: flex; flex-direction: row; flex-wrap: wrap; justify-content: flex-start; width: 100% }\n",
       "    .wandb-col { display: flex; flex-direction: column; flex-basis: 100%; flex: 1; padding: 10px; }\n",
       "    </style>\n",
       "<div class=\"wandb-row\"><div class=\"wandb-col\"><h3>Run history:</h3><br/><table class=\"wandb\"><tr><td>eval/f1</td><td>▁▄▇▆▆█████</td></tr><tr><td>eval/loss</td><td>▄▂▁▁▃▂█▇██</td></tr><tr><td>eval/runtime</td><td>▁▂▄▅▆▆▇▇▇█</td></tr><tr><td>eval/samples_per_second</td><td>█▆▅▄▃▃▂▂▂▁</td></tr><tr><td>eval/steps_per_second</td><td>█▇▅▄▃▃▂▂▂▁</td></tr><tr><td>train/epoch</td><td>▁▁▂▂▃▃▃▃▄▄▅▅▆▆▆▆▇▇███</td></tr><tr><td>train/global_step</td><td>▁▁▂▂▃▃▃▃▄▄▅▅▆▆▆▆▇▇███</td></tr><tr><td>train/learning_rate</td><td>█▇▆▆▅▄▃▃▂▁</td></tr><tr><td>train/loss</td><td>█▆▅▄▃▂▂▁▁▁</td></tr><tr><td>train/total_flos</td><td>▁</td></tr><tr><td>train/train_loss</td><td>▁</td></tr><tr><td>train/train_runtime</td><td>▁</td></tr><tr><td>train/train_samples_per_second</td><td>▁</td></tr><tr><td>train/train_steps_per_second</td><td>▁</td></tr></table><br/></div><div class=\"wandb-col\"><h3>Run summary:</h3><br/><table class=\"wandb\"><tr><td>eval/f1</td><td>0.84409</td></tr><tr><td>eval/loss</td><td>0.8819</td></tr><tr><td>eval/runtime</td><td>2.3665</td></tr><tr><td>eval/samples_per_second</td><td>126.768</td></tr><tr><td>eval/steps_per_second</td><td>16.057</td></tr><tr><td>train/epoch</td><td>10.0</td></tr><tr><td>train/global_step</td><td>190</td></tr><tr><td>train/learning_rate</td><td>0.0</td></tr><tr><td>train/loss</td><td>0.003</td></tr><tr><td>train/total_flos</td><td>789340253184000.0</td></tr><tr><td>train/train_loss</td><td>0.30796</td></tr><tr><td>train/train_runtime</td><td>89.8833</td></tr><tr><td>train/train_samples_per_second</td><td>33.377</td></tr><tr><td>train/train_steps_per_second</td><td>2.114</td></tr></table><br/></div></div>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       " View run <strong style=\"color:#cdcd00\">facebook/react</strong> at: <a href='https://wandb.ai/rafaelkallis/NLBSE%2724%20Issue%20Report%20Classification%20-%20RoBERTa/runs/2pg2vivk' target=\"_blank\">https://wandb.ai/rafaelkallis/NLBSE%2724%20Issue%20Report%20Classification%20-%20RoBERTa/runs/2pg2vivk</a><br/>Synced 5 W&B file(s), 0 media file(s), 0 artifact file(s) and 0 other file(s)"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "Find logs at: <code>./wandb/run-20240116_210749-2pg2vivk/logs</code>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "Tracking run with wandb version 0.16.2"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "Run data is saved locally in <code>/workspaces/issue-report-classification/wandb/run-20240116_210928-vt3diyqt</code>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "Syncing run <strong><a href='https://wandb.ai/rafaelkallis/NLBSE%2724%20Issue%20Report%20Classification%20-%20RoBERTa/runs/vt3diyqt' target=\"_blank\">tensorflow/tensorflow</a></strong> to <a href='https://wandb.ai/rafaelkallis/NLBSE%2724%20Issue%20Report%20Classification%20-%20RoBERTa' target=\"_blank\">Weights & Biases</a> (<a href='https://wandb.me/run' target=\"_blank\">docs</a>)<br/>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       " View project at <a href='https://wandb.ai/rafaelkallis/NLBSE%2724%20Issue%20Report%20Classification%20-%20RoBERTa' target=\"_blank\">https://wandb.ai/rafaelkallis/NLBSE%2724%20Issue%20Report%20Classification%20-%20RoBERTa</a>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       " View run at <a href='https://wandb.ai/rafaelkallis/NLBSE%2724%20Issue%20Report%20Classification%20-%20RoBERTa/runs/vt3diyqt' target=\"_blank\">https://wandb.ai/rafaelkallis/NLBSE%2724%20Issue%20Report%20Classification%20-%20RoBERTa/runs/vt3diyqt</a>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Some weights of RobertaForSequenceClassification were not initialized from the model checkpoint at roberta-base and are newly initialized: ['classifier.out_proj.bias', 'classifier.dense.bias', 'classifier.dense.weight', 'classifier.out_proj.weight']\n",
      "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n",
      "Map: 100%|██████████| 300/300 [00:00<00:00, 2123.36 examples/s]\n",
      "You're using a RobertaTokenizerFast tokenizer. Please note that with a fast tokenizer, using the `__call__` method is faster than using a method to encode the text followed by a call to the `pad` method to get a padded encoding.\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='190' max='190' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [190/190 01:29, Epoch 10/10]\n",
       "    </div>\n",
       "    <table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       " <tr style=\"text-align: left;\">\n",
       "      <th>Epoch</th>\n",
       "      <th>Training Loss</th>\n",
       "      <th>Validation Loss</th>\n",
       "      <th>F1</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>0.989800</td>\n",
       "      <td>0.835612</td>\n",
       "      <td>0.594050</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>0.720400</td>\n",
       "      <td>0.732002</td>\n",
       "      <td>0.674492</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>0.592000</td>\n",
       "      <td>0.624695</td>\n",
       "      <td>0.732933</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>0.480800</td>\n",
       "      <td>0.479299</td>\n",
       "      <td>0.822421</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>5</td>\n",
       "      <td>0.285500</td>\n",
       "      <td>0.473744</td>\n",
       "      <td>0.869514</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>6</td>\n",
       "      <td>0.193900</td>\n",
       "      <td>0.498568</td>\n",
       "      <td>0.865892</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>7</td>\n",
       "      <td>0.126900</td>\n",
       "      <td>0.722005</td>\n",
       "      <td>0.831306</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>8</td>\n",
       "      <td>0.086500</td>\n",
       "      <td>0.708205</td>\n",
       "      <td>0.844257</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>9</td>\n",
       "      <td>0.069600</td>\n",
       "      <td>0.739826</td>\n",
       "      <td>0.858154</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>10</td>\n",
       "      <td>0.040800</td>\n",
       "      <td>0.724963</td>\n",
       "      <td>0.858454</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table><p>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "wandb: WARNING Source type is set to 'repo' but some required information is missing from the environment. A job will not be created from this run. See https://docs.wandb.ai/guides/launch/create-job\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<style>\n",
       "    table.wandb td:nth-child(1) { padding: 0 10px; text-align: left ; width: auto;} td:nth-child(2) {text-align: left ; width: 100%}\n",
       "    .wandb-row { display: flex; flex-direction: row; flex-wrap: wrap; justify-content: flex-start; width: 100% }\n",
       "    .wandb-col { display: flex; flex-direction: column; flex-basis: 100%; flex: 1; padding: 10px; }\n",
       "    </style>\n",
       "<div class=\"wandb-row\"><div class=\"wandb-col\"><h3>Run history:</h3><br/><table class=\"wandb\"><tr><td>eval/f1</td><td>▁▃▅▇██▇▇██</td></tr><tr><td>eval/loss</td><td>█▆▄▁▁▁▆▆▆▆</td></tr><tr><td>eval/runtime</td><td>▁▃▄▂▃▄▅▆▆█</td></tr><tr><td>eval/samples_per_second</td><td>█▆▅▇▆▅▄▃▃▁</td></tr><tr><td>eval/steps_per_second</td><td>█▆▅▇▆▅▄▃▃▁</td></tr><tr><td>train/epoch</td><td>▁▁▂▂▃▃▃▃▄▄▅▅▆▆▆▆▇▇███</td></tr><tr><td>train/global_step</td><td>▁▁▂▂▃▃▃▃▄▄▅▅▆▆▆▆▇▇███</td></tr><tr><td>train/learning_rate</td><td>█▇▆▆▅▄▃▃▂▁</td></tr><tr><td>train/loss</td><td>█▆▅▄▃▂▂▁▁▁</td></tr><tr><td>train/total_flos</td><td>▁</td></tr><tr><td>train/train_loss</td><td>▁</td></tr><tr><td>train/train_runtime</td><td>▁</td></tr><tr><td>train/train_samples_per_second</td><td>▁</td></tr><tr><td>train/train_steps_per_second</td><td>▁</td></tr></table><br/></div><div class=\"wandb-col\"><h3>Run summary:</h3><br/><table class=\"wandb\"><tr><td>eval/f1</td><td>0.85845</td></tr><tr><td>eval/loss</td><td>0.72496</td></tr><tr><td>eval/runtime</td><td>2.3871</td></tr><tr><td>eval/samples_per_second</td><td>125.675</td></tr><tr><td>eval/steps_per_second</td><td>15.919</td></tr><tr><td>train/epoch</td><td>10.0</td></tr><tr><td>train/global_step</td><td>190</td></tr><tr><td>train/learning_rate</td><td>0.0</td></tr><tr><td>train/loss</td><td>0.0408</td></tr><tr><td>train/total_flos</td><td>789340253184000.0</td></tr><tr><td>train/train_loss</td><td>0.35861</td></tr><tr><td>train/train_runtime</td><td>90.3392</td></tr><tr><td>train/train_samples_per_second</td><td>33.208</td></tr><tr><td>train/train_steps_per_second</td><td>2.103</td></tr></table><br/></div></div>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       " View run <strong style=\"color:#cdcd00\">tensorflow/tensorflow</strong> at: <a href='https://wandb.ai/rafaelkallis/NLBSE%2724%20Issue%20Report%20Classification%20-%20RoBERTa/runs/vt3diyqt' target=\"_blank\">https://wandb.ai/rafaelkallis/NLBSE%2724%20Issue%20Report%20Classification%20-%20RoBERTa/runs/vt3diyqt</a><br/>Synced 5 W&B file(s), 0 media file(s), 0 artifact file(s) and 0 other file(s)"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "Find logs at: <code>./wandb/run-20240116_210928-vt3diyqt/logs</code>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "Tracking run with wandb version 0.16.2"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "Run data is saved locally in <code>/workspaces/issue-report-classification/wandb/run-20240116_211125-id1ttsbl</code>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "Syncing run <strong><a href='https://wandb.ai/rafaelkallis/NLBSE%2724%20Issue%20Report%20Classification%20-%20RoBERTa/runs/id1ttsbl' target=\"_blank\">microsoft/vscode</a></strong> to <a href='https://wandb.ai/rafaelkallis/NLBSE%2724%20Issue%20Report%20Classification%20-%20RoBERTa' target=\"_blank\">Weights & Biases</a> (<a href='https://wandb.me/run' target=\"_blank\">docs</a>)<br/>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       " View project at <a href='https://wandb.ai/rafaelkallis/NLBSE%2724%20Issue%20Report%20Classification%20-%20RoBERTa' target=\"_blank\">https://wandb.ai/rafaelkallis/NLBSE%2724%20Issue%20Report%20Classification%20-%20RoBERTa</a>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       " View run at <a href='https://wandb.ai/rafaelkallis/NLBSE%2724%20Issue%20Report%20Classification%20-%20RoBERTa/runs/id1ttsbl' target=\"_blank\">https://wandb.ai/rafaelkallis/NLBSE%2724%20Issue%20Report%20Classification%20-%20RoBERTa/runs/id1ttsbl</a>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Some weights of RobertaForSequenceClassification were not initialized from the model checkpoint at roberta-base and are newly initialized: ['classifier.out_proj.bias', 'classifier.dense.bias', 'classifier.dense.weight', 'classifier.out_proj.weight']\n",
      "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n",
      "Map: 100%|██████████| 300/300 [00:00<00:00, 2127.63 examples/s]\n",
      "Map: 100%|██████████| 300/300 [00:00<00:00, 2272.63 examples/s]\n",
      "You're using a RobertaTokenizerFast tokenizer. Please note that with a fast tokenizer, using the `__call__` method is faster than using a method to encode the text followed by a call to the `pad` method to get a padded encoding.\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='190' max='190' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [190/190 01:29, Epoch 10/10]\n",
       "    </div>\n",
       "    <table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       " <tr style=\"text-align: left;\">\n",
       "      <th>Epoch</th>\n",
       "      <th>Training Loss</th>\n",
       "      <th>Validation Loss</th>\n",
       "      <th>F1</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>1.029700</td>\n",
       "      <td>0.975167</td>\n",
       "      <td>0.566626</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>0.885500</td>\n",
       "      <td>0.836753</td>\n",
       "      <td>0.666281</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>0.688200</td>\n",
       "      <td>0.802347</td>\n",
       "      <td>0.671059</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>0.595800</td>\n",
       "      <td>0.664130</td>\n",
       "      <td>0.740537</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>5</td>\n",
       "      <td>0.389100</td>\n",
       "      <td>0.606848</td>\n",
       "      <td>0.789923</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>6</td>\n",
       "      <td>0.219200</td>\n",
       "      <td>0.653700</td>\n",
       "      <td>0.777000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>7</td>\n",
       "      <td>0.120800</td>\n",
       "      <td>0.760761</td>\n",
       "      <td>0.790686</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>8</td>\n",
       "      <td>0.075400</td>\n",
       "      <td>0.962399</td>\n",
       "      <td>0.760114</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>9</td>\n",
       "      <td>0.044400</td>\n",
       "      <td>0.996846</td>\n",
       "      <td>0.767218</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>10</td>\n",
       "      <td>0.034700</td>\n",
       "      <td>1.037996</td>\n",
       "      <td>0.764312</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table><p>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "wandb: WARNING Source type is set to 'repo' but some required information is missing from the environment. A job will not be created from this run. See https://docs.wandb.ai/guides/launch/create-job\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<style>\n",
       "    table.wandb td:nth-child(1) { padding: 0 10px; text-align: left ; width: auto;} td:nth-child(2) {text-align: left ; width: 100%}\n",
       "    .wandb-row { display: flex; flex-direction: row; flex-wrap: wrap; justify-content: flex-start; width: 100% }\n",
       "    .wandb-col { display: flex; flex-direction: column; flex-basis: 100%; flex: 1; padding: 10px; }\n",
       "    </style>\n",
       "<div class=\"wandb-row\"><div class=\"wandb-col\"><h3>Run history:</h3><br/><table class=\"wandb\"><tr><td>eval/f1</td><td>▁▄▄▆███▇▇▇</td></tr><tr><td>eval/loss</td><td>▇▅▄▂▁▂▃▇▇█</td></tr><tr><td>eval/runtime</td><td>▁▃▄▆▆▇▇▇██</td></tr><tr><td>eval/samples_per_second</td><td>█▆▅▃▃▂▂▂▁▁</td></tr><tr><td>eval/steps_per_second</td><td>█▆▅▃▃▂▂▂▁▁</td></tr><tr><td>train/epoch</td><td>▁▁▂▂▃▃▃▃▄▄▅▅▆▆▆▆▇▇███</td></tr><tr><td>train/global_step</td><td>▁▁▂▂▃▃▃▃▄▄▅▅▆▆▆▆▇▇███</td></tr><tr><td>train/learning_rate</td><td>█▇▆▆▅▄▃▃▂▁</td></tr><tr><td>train/loss</td><td>█▇▆▅▃▂▂▁▁▁</td></tr><tr><td>train/total_flos</td><td>▁</td></tr><tr><td>train/train_loss</td><td>▁</td></tr><tr><td>train/train_runtime</td><td>▁</td></tr><tr><td>train/train_samples_per_second</td><td>▁</td></tr><tr><td>train/train_steps_per_second</td><td>▁</td></tr></table><br/></div><div class=\"wandb-col\"><h3>Run summary:</h3><br/><table class=\"wandb\"><tr><td>eval/f1</td><td>0.76431</td></tr><tr><td>eval/loss</td><td>1.038</td></tr><tr><td>eval/runtime</td><td>2.3838</td></tr><tr><td>eval/samples_per_second</td><td>125.851</td></tr><tr><td>eval/steps_per_second</td><td>15.941</td></tr><tr><td>train/epoch</td><td>10.0</td></tr><tr><td>train/global_step</td><td>190</td></tr><tr><td>train/learning_rate</td><td>0.0</td></tr><tr><td>train/loss</td><td>0.0347</td></tr><tr><td>train/total_flos</td><td>789340253184000.0</td></tr><tr><td>train/train_loss</td><td>0.40829</td></tr><tr><td>train/train_runtime</td><td>90.1936</td></tr><tr><td>train/train_samples_per_second</td><td>33.262</td></tr><tr><td>train/train_steps_per_second</td><td>2.107</td></tr></table><br/></div></div>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       " View run <strong style=\"color:#cdcd00\">microsoft/vscode</strong> at: <a href='https://wandb.ai/rafaelkallis/NLBSE%2724%20Issue%20Report%20Classification%20-%20RoBERTa/runs/id1ttsbl' target=\"_blank\">https://wandb.ai/rafaelkallis/NLBSE%2724%20Issue%20Report%20Classification%20-%20RoBERTa/runs/id1ttsbl</a><br/>Synced 5 W&B file(s), 0 media file(s), 0 artifact file(s) and 0 other file(s)"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "Find logs at: <code>./wandb/run-20240116_211125-id1ttsbl/logs</code>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "Tracking run with wandb version 0.16.2"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "Run data is saved locally in <code>/workspaces/issue-report-classification/wandb/run-20240116_211316-u5o89x48</code>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "Syncing run <strong><a href='https://wandb.ai/rafaelkallis/NLBSE%2724%20Issue%20Report%20Classification%20-%20RoBERTa/runs/u5o89x48' target=\"_blank\">bitcoin/bitcoin</a></strong> to <a href='https://wandb.ai/rafaelkallis/NLBSE%2724%20Issue%20Report%20Classification%20-%20RoBERTa' target=\"_blank\">Weights & Biases</a> (<a href='https://wandb.me/run' target=\"_blank\">docs</a>)<br/>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       " View project at <a href='https://wandb.ai/rafaelkallis/NLBSE%2724%20Issue%20Report%20Classification%20-%20RoBERTa' target=\"_blank\">https://wandb.ai/rafaelkallis/NLBSE%2724%20Issue%20Report%20Classification%20-%20RoBERTa</a>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       " View run at <a href='https://wandb.ai/rafaelkallis/NLBSE%2724%20Issue%20Report%20Classification%20-%20RoBERTa/runs/u5o89x48' target=\"_blank\">https://wandb.ai/rafaelkallis/NLBSE%2724%20Issue%20Report%20Classification%20-%20RoBERTa/runs/u5o89x48</a>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Some weights of RobertaForSequenceClassification were not initialized from the model checkpoint at roberta-base and are newly initialized: ['classifier.out_proj.bias', 'classifier.dense.bias', 'classifier.dense.weight', 'classifier.out_proj.weight']\n",
      "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n",
      "Map: 100%|██████████| 300/300 [00:00<00:00, 2067.63 examples/s]\n",
      "Map: 100%|██████████| 300/300 [00:00<00:00, 1932.63 examples/s]\n",
      "You're using a RobertaTokenizerFast tokenizer. Please note that with a fast tokenizer, using the `__call__` method is faster than using a method to encode the text followed by a call to the `pad` method to get a padded encoding.\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='190' max='190' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [190/190 01:30, Epoch 10/10]\n",
       "    </div>\n",
       "    <table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       " <tr style=\"text-align: left;\">\n",
       "      <th>Epoch</th>\n",
       "      <th>Training Loss</th>\n",
       "      <th>Validation Loss</th>\n",
       "      <th>F1</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>1.081400</td>\n",
       "      <td>0.970499</td>\n",
       "      <td>0.445159</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>0.771600</td>\n",
       "      <td>0.764188</td>\n",
       "      <td>0.653771</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>0.533500</td>\n",
       "      <td>0.667387</td>\n",
       "      <td>0.710790</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>0.325200</td>\n",
       "      <td>0.815736</td>\n",
       "      <td>0.737051</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>5</td>\n",
       "      <td>0.252200</td>\n",
       "      <td>0.812353</td>\n",
       "      <td>0.726487</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>6</td>\n",
       "      <td>0.119900</td>\n",
       "      <td>1.133980</td>\n",
       "      <td>0.735907</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>7</td>\n",
       "      <td>0.069300</td>\n",
       "      <td>1.336177</td>\n",
       "      <td>0.725464</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>8</td>\n",
       "      <td>0.036700</td>\n",
       "      <td>1.468985</td>\n",
       "      <td>0.707075</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>9</td>\n",
       "      <td>0.025200</td>\n",
       "      <td>1.412592</td>\n",
       "      <td>0.734412</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>10</td>\n",
       "      <td>0.038000</td>\n",
       "      <td>1.354312</td>\n",
       "      <td>0.752635</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table><p>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "wandb: WARNING Source type is set to 'repo' but some required information is missing from the environment. A job will not be created from this run. See https://docs.wandb.ai/guides/launch/create-job\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<style>\n",
       "    table.wandb td:nth-child(1) { padding: 0 10px; text-align: left ; width: auto;} td:nth-child(2) {text-align: left ; width: 100%}\n",
       "    .wandb-row { display: flex; flex-direction: row; flex-wrap: wrap; justify-content: flex-start; width: 100% }\n",
       "    .wandb-col { display: flex; flex-direction: column; flex-basis: 100%; flex: 1; padding: 10px; }\n",
       "    </style>\n",
       "<div class=\"wandb-row\"><div class=\"wandb-col\"><h3>Run history:</h3><br/><table class=\"wandb\"><tr><td>eval/f1</td><td>▁▆▇█▇█▇▇██</td></tr><tr><td>eval/loss</td><td>▄▂▁▂▂▅▇██▇</td></tr><tr><td>eval/runtime</td><td>▁▃▅▆▇▇██▇▇</td></tr><tr><td>eval/samples_per_second</td><td>█▆▄▃▂▂▁▁▂▂</td></tr><tr><td>eval/steps_per_second</td><td>█▆▄▃▂▂▁▁▂▂</td></tr><tr><td>train/epoch</td><td>▁▁▂▂▃▃▃▃▄▄▅▅▆▆▆▆▇▇███</td></tr><tr><td>train/global_step</td><td>▁▁▂▂▃▃▃▃▄▄▅▅▆▆▆▆▇▇███</td></tr><tr><td>train/learning_rate</td><td>█▇▆▆▅▄▃▃▂▁</td></tr><tr><td>train/loss</td><td>█▆▄▃▃▂▁▁▁▁</td></tr><tr><td>train/total_flos</td><td>▁</td></tr><tr><td>train/train_loss</td><td>▁</td></tr><tr><td>train/train_runtime</td><td>▁</td></tr><tr><td>train/train_samples_per_second</td><td>▁</td></tr><tr><td>train/train_steps_per_second</td><td>▁</td></tr></table><br/></div><div class=\"wandb-col\"><h3>Run summary:</h3><br/><table class=\"wandb\"><tr><td>eval/f1</td><td>0.75263</td></tr><tr><td>eval/loss</td><td>1.35431</td></tr><tr><td>eval/runtime</td><td>2.3901</td></tr><tr><td>eval/samples_per_second</td><td>125.515</td></tr><tr><td>eval/steps_per_second</td><td>15.899</td></tr><tr><td>train/epoch</td><td>10.0</td></tr><tr><td>train/global_step</td><td>190</td></tr><tr><td>train/learning_rate</td><td>0.0</td></tr><tr><td>train/loss</td><td>0.038</td></tr><tr><td>train/total_flos</td><td>789340253184000.0</td></tr><tr><td>train/train_loss</td><td>0.32531</td></tr><tr><td>train/train_runtime</td><td>90.6359</td></tr><tr><td>train/train_samples_per_second</td><td>33.099</td></tr><tr><td>train/train_steps_per_second</td><td>2.096</td></tr></table><br/></div></div>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       " View run <strong style=\"color:#cdcd00\">bitcoin/bitcoin</strong> at: <a href='https://wandb.ai/rafaelkallis/NLBSE%2724%20Issue%20Report%20Classification%20-%20RoBERTa/runs/u5o89x48' target=\"_blank\">https://wandb.ai/rafaelkallis/NLBSE%2724%20Issue%20Report%20Classification%20-%20RoBERTa/runs/u5o89x48</a><br/>Synced 5 W&B file(s), 0 media file(s), 0 artifact file(s) and 0 other file(s)"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "Find logs at: <code>./wandb/run-20240116_211316-u5o89x48/logs</code>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "Tracking run with wandb version 0.16.2"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "Run data is saved locally in <code>/workspaces/issue-report-classification/wandb/run-20240116_211459-apzan6ow</code>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "Syncing run <strong><a href='https://wandb.ai/rafaelkallis/NLBSE%2724%20Issue%20Report%20Classification%20-%20RoBERTa/runs/apzan6ow' target=\"_blank\">opencv/opencv</a></strong> to <a href='https://wandb.ai/rafaelkallis/NLBSE%2724%20Issue%20Report%20Classification%20-%20RoBERTa' target=\"_blank\">Weights & Biases</a> (<a href='https://wandb.me/run' target=\"_blank\">docs</a>)<br/>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       " View project at <a href='https://wandb.ai/rafaelkallis/NLBSE%2724%20Issue%20Report%20Classification%20-%20RoBERTa' target=\"_blank\">https://wandb.ai/rafaelkallis/NLBSE%2724%20Issue%20Report%20Classification%20-%20RoBERTa</a>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       " View run at <a href='https://wandb.ai/rafaelkallis/NLBSE%2724%20Issue%20Report%20Classification%20-%20RoBERTa/runs/apzan6ow' target=\"_blank\">https://wandb.ai/rafaelkallis/NLBSE%2724%20Issue%20Report%20Classification%20-%20RoBERTa/runs/apzan6ow</a>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Some weights of RobertaForSequenceClassification were not initialized from the model checkpoint at roberta-base and are newly initialized: ['classifier.out_proj.bias', 'classifier.dense.bias', 'classifier.dense.weight', 'classifier.out_proj.weight']\n",
      "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n",
      "Map: 100%|██████████| 300/300 [00:00<00:00, 1869.55 examples/s]\n",
      "Map: 100%|██████████| 300/300 [00:00<00:00, 2020.36 examples/s]\n",
      "You're using a RobertaTokenizerFast tokenizer. Please note that with a fast tokenizer, using the `__call__` method is faster than using a method to encode the text followed by a call to the `pad` method to get a padded encoding.\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='190' max='190' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [190/190 01:30, Epoch 10/10]\n",
       "    </div>\n",
       "    <table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       " <tr style=\"text-align: left;\">\n",
       "      <th>Epoch</th>\n",
       "      <th>Training Loss</th>\n",
       "      <th>Validation Loss</th>\n",
       "      <th>F1</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>1.075100</td>\n",
       "      <td>0.968205</td>\n",
       "      <td>0.593196</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>0.939800</td>\n",
       "      <td>0.817039</td>\n",
       "      <td>0.561382</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>0.727100</td>\n",
       "      <td>0.832950</td>\n",
       "      <td>0.642537</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>0.632600</td>\n",
       "      <td>0.727519</td>\n",
       "      <td>0.628608</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>5</td>\n",
       "      <td>0.564300</td>\n",
       "      <td>0.866814</td>\n",
       "      <td>0.556447</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>6</td>\n",
       "      <td>0.472500</td>\n",
       "      <td>0.845133</td>\n",
       "      <td>0.640861</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>7</td>\n",
       "      <td>0.358900</td>\n",
       "      <td>0.818358</td>\n",
       "      <td>0.677320</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>8</td>\n",
       "      <td>0.322300</td>\n",
       "      <td>0.881727</td>\n",
       "      <td>0.670452</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>9</td>\n",
       "      <td>0.249400</td>\n",
       "      <td>0.882026</td>\n",
       "      <td>0.722118</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>10</td>\n",
       "      <td>0.222000</td>\n",
       "      <td>0.845077</td>\n",
       "      <td>0.742175</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table><p>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "wandb: WARNING Source type is set to 'repo' but some required information is missing from the environment. A job will not be created from this run. See https://docs.wandb.ai/guides/launch/create-job\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<style>\n",
       "    table.wandb td:nth-child(1) { padding: 0 10px; text-align: left ; width: auto;} td:nth-child(2) {text-align: left ; width: 100%}\n",
       "    .wandb-row { display: flex; flex-direction: row; flex-wrap: wrap; justify-content: flex-start; width: 100% }\n",
       "    .wandb-col { display: flex; flex-direction: column; flex-basis: 100%; flex: 1; padding: 10px; }\n",
       "    </style>\n",
       "<div class=\"wandb-row\"><div class=\"wandb-col\"><h3>Run history:</h3><br/><table class=\"wandb\"><tr><td>eval/f1</td><td>▂▁▄▄▁▄▆▅▇█</td></tr><tr><td>eval/loss</td><td>█▄▄▁▅▄▄▅▅▄</td></tr><tr><td>eval/runtime</td><td>▁▃▄▅▆▇▇█▇█</td></tr><tr><td>eval/samples_per_second</td><td>█▆▅▄▃▂▂▁▂▁</td></tr><tr><td>eval/steps_per_second</td><td>█▆▅▄▃▂▂▁▂▁</td></tr><tr><td>train/epoch</td><td>▁▁▂▂▃▃▃▃▄▄▅▅▆▆▆▆▇▇███</td></tr><tr><td>train/global_step</td><td>▁▁▂▂▃▃▃▃▄▄▅▅▆▆▆▆▇▇███</td></tr><tr><td>train/learning_rate</td><td>█▇▆▆▅▄▃▃▂▁</td></tr><tr><td>train/loss</td><td>█▇▅▄▄▃▂▂▁▁</td></tr><tr><td>train/total_flos</td><td>▁</td></tr><tr><td>train/train_loss</td><td>▁</td></tr><tr><td>train/train_runtime</td><td>▁</td></tr><tr><td>train/train_samples_per_second</td><td>▁</td></tr><tr><td>train/train_steps_per_second</td><td>▁</td></tr></table><br/></div><div class=\"wandb-col\"><h3>Run summary:</h3><br/><table class=\"wandb\"><tr><td>eval/f1</td><td>0.74218</td></tr><tr><td>eval/loss</td><td>0.84508</td></tr><tr><td>eval/runtime</td><td>2.4025</td></tr><tr><td>eval/samples_per_second</td><td>124.872</td></tr><tr><td>eval/steps_per_second</td><td>15.817</td></tr><tr><td>train/epoch</td><td>10.0</td></tr><tr><td>train/global_step</td><td>190</td></tr><tr><td>train/learning_rate</td><td>0.0</td></tr><tr><td>train/loss</td><td>0.222</td></tr><tr><td>train/total_flos</td><td>789340253184000.0</td></tr><tr><td>train/train_loss</td><td>0.55641</td></tr><tr><td>train/train_runtime</td><td>91.1251</td></tr><tr><td>train/train_samples_per_second</td><td>32.922</td></tr><tr><td>train/train_steps_per_second</td><td>2.085</td></tr></table><br/></div></div>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       " View run <strong style=\"color:#cdcd00\">opencv/opencv</strong> at: <a href='https://wandb.ai/rafaelkallis/NLBSE%2724%20Issue%20Report%20Classification%20-%20RoBERTa/runs/apzan6ow' target=\"_blank\">https://wandb.ai/rafaelkallis/NLBSE%2724%20Issue%20Report%20Classification%20-%20RoBERTa/runs/apzan6ow</a><br/>Synced 5 W&B file(s), 0 media file(s), 0 artifact file(s) and 0 other file(s)"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "Find logs at: <code>./wandb/run-20240116_211459-apzan6ow/logs</code>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import wandb\n",
    "from datetime import datetime\n",
    "import evaluate\n",
    "from transformers import TrainingArguments, Trainer, DataCollatorWithPadding\n",
    "from transformers import RobertaTokenizerFast, RobertaConfig, RobertaForSequenceClassification, TextClassificationPipeline\n",
    "import numpy as np\n",
    "\n",
    "group = datetime.utcnow().replace(microsecond=0).isoformat()\n",
    "\n",
    "metric = evaluate.load(\"f1\")\n",
    "\n",
    "def repo_eq(repo: str):\n",
    "    return lambda example: example['repo'] == repo\n",
    "\n",
    "references = {}\n",
    "predictions = {}\n",
    "\n",
    "for repo in repos:\n",
    "    wandb.init(\n",
    "        project=\"NLBSE'24 Issue Report Classification - RoBERTa\", \n",
    "        group=group,\n",
    "        name=repo, \n",
    "    )\n",
    "\n",
    "    device=\"cuda\"\n",
    "    truncation=True\n",
    "    padding=\"max_length\"\n",
    "    max_length=512\n",
    "\n",
    "    tokenizer = RobertaTokenizerFast.from_pretrained(BASE_MODEL)\n",
    "\n",
    "    config = RobertaConfig.from_pretrained(BASE_MODEL, num_labels=3)\n",
    "\n",
    "    model = RobertaForSequenceClassification.from_pretrained(BASE_MODEL, config=config)\n",
    "  \n",
    "    classifier = TextClassificationPipeline(model=model, tokenizer=tokenizer, device=device, truncation=truncation, padding=padding, max_length=max_length)\n",
    "\n",
    "    data_collator = DataCollatorWithPadding(tokenizer=tokenizer)\n",
    "\n",
    "    def preprocess_function(examples):\n",
    "      return tokenizer(examples['text'], truncation=truncation, padding=padding, max_length=max_length)\n",
    "\n",
    "    train_set = ds.filter(repo_eq(repo))[\"train\"]\n",
    "    train_set = train_set.map(preprocess_function, batched=True)\n",
    "    train_set.set_format(type=\"torch\", columns=[\"input_ids\", \"attention_mask\", \"label\"])\n",
    "\n",
    "    test_set = ds.filter(repo_eq(repo))[\"test\"]\n",
    "    references[repo] = [id2label[id] for id in test_set['label']]\n",
    "    test_set = test_set.map(preprocess_function, batched=True)\n",
    "    test_set.set_format(type=\"torch\", columns=[\"input_ids\", \"attention_mask\", \"label\"])\n",
    "\n",
    "    def compute_metrics(eval_pred):\n",
    "        logits, labels = eval_pred\n",
    "        predictions = np.argmax(logits, axis=-1)\n",
    "        return metric.compute(predictions=predictions, references=labels, average=\"weighted\")\n",
    "\n",
    "    training_args = TrainingArguments(\n",
    "        output_dir=f\"{OUTPUT_PATH}/{repo.replace('/', '-')}\",\n",
    "        evaluation_strategy=\"epoch\",\n",
    "        logging_strategy=\"epoch\",\n",
    "        save_strategy=\"no\",\n",
    "        report_to=\"wandb\",\n",
    "        run_name=repo,\n",
    "        seed=RANDOM_SEED,\n",
    "        num_train_epochs=10,\n",
    "        per_device_train_batch_size=16,\n",
    "    )\n",
    "\n",
    "    trainer = Trainer(\n",
    "        model=model,\n",
    "        tokenizer=tokenizer,\n",
    "        args=training_args,\n",
    "        train_dataset=train_set,\n",
    "        eval_dataset=test_set,\n",
    "        compute_metrics=compute_metrics,\n",
    "        data_collator=data_collator,\n",
    "    )\n",
    "\n",
    "    trainer.train()\n",
    "\n",
    "    predictions[repo] = classifier(test_set['text'])\n",
    "    predictions[repo] = [pred['label'] for pred in predictions[repo]]\n",
    "    predictions[repo] = [model.config.label2id[label] for label in predictions[repo]]\n",
    "    predictions[repo] = [id2label[id] for id in predictions[repo]]\n",
    "    \n",
    "    wandb.finish()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import classification_report\n",
    "from numpy import mean\n",
    "\n",
    "results = {}\n",
    "metrics = ['precision', 'recall', 'f1-score']\n",
    "labels = ['bug', 'feature', 'question']\n",
    "\n",
    "for repo in repos:\n",
    "  results[repo] = classification_report(references[repo], predictions[repo], digits=4, output_dict=True)\n",
    "  results[repo]['average'] = results[repo]['weighted avg']\n",
    "  results[repo] = {label: {metric: results[repo][label][metric] for metric in metrics} for label in labels + ['average']}\n",
    "\n",
    "results['overall'] = {label: {metric: mean([results[repo][label][metric] for repo in repos]) for metric in metrics} for label in labels + ['average']}\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Repository               Label     Precision  Recall     F1\n",
      "---------------------------------------------------------------\n",
      "facebook/react           bug       0.8909     0.9800     0.9333     \n",
      "facebook/react           feature   0.8200     0.8200     0.8200     \n",
      "facebook/react           question  0.8222     0.7400     0.7789     \n",
      "facebook/react           average   0.8444     0.8467     0.8441     \n",
      "---------------------------------------------------------------\n",
      "tensorflow/tensorflow    bug       0.9659     0.8500     0.9043     \n",
      "tensorflow/tensorflow    feature   0.8641     0.8900     0.8768     \n",
      "tensorflow/tensorflow    question  0.7615     0.8300     0.7943     \n",
      "tensorflow/tensorflow    average   0.8638     0.8567     0.8585     \n",
      "---------------------------------------------------------------\n",
      "microsoft/vscode         bug       0.7895     0.7500     0.7692     \n",
      "microsoft/vscode         feature   0.6803     0.8300     0.7477     \n",
      "microsoft/vscode         question  0.8554     0.7100     0.7760     \n",
      "microsoft/vscode         average   0.7751     0.7633     0.7643     \n",
      "---------------------------------------------------------------\n",
      "bitcoin/bitcoin          bug       0.7426     0.7500     0.7463     \n",
      "bitcoin/bitcoin          feature   0.8333     0.8500     0.8416     \n",
      "bitcoin/bitcoin          question  0.6804     0.6600     0.6701     \n",
      "bitcoin/bitcoin          average   0.7521     0.7533     0.7526     \n",
      "---------------------------------------------------------------\n",
      "opencv/opencv            bug       0.6542     0.7000     0.6763     \n",
      "opencv/opencv            feature   0.8667     0.6500     0.7429     \n",
      "opencv/opencv            question  0.7458     0.8800     0.8073     \n",
      "opencv/opencv            average   0.7555     0.7433     0.7422     \n",
      "---------------------------------------------------------------\n",
      "overall                  bug       0.8086     0.8060     0.8059     \n",
      "overall                  feature   0.8129     0.8080     0.8058     \n",
      "overall                  question  0.7731     0.7640     0.7653     \n",
      "overall                  average   0.7982     0.7927     0.7923     \n"
     ]
    }
   ],
   "source": [
    "import json\n",
    "import os\n",
    "\n",
    "output_file_name = 'results.json'\n",
    "\n",
    "with open(os.path.join(OUTPUT_PATH, output_file_name), 'w') as fp:\n",
    "    json.dump(results, fp, indent=2)\n",
    "\n",
    "print(f\"Repository{' '*15}Label     Precision  Recall     F1\")\n",
    "for repo in repos + ['overall']:\n",
    "  print(\"-\"*63)\n",
    "  for label in labels + ['average']:\n",
    "    out = f\"{repo:<25}{label:<10}\"\n",
    "    for metric in metrics:\n",
    "      out += f\"{results[repo][label][metric]:<10.4f} \"\n",
    "    print(out)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}