yahoo_bert.ipynb

{
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "name": "python",
   "version": "3.10.13",
   "mimetype": "text/x-python",
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "pygments_lexer": "ipython3",
   "nbconvert_exporter": "python",
   "file_extension": ".py"
  },
  "kaggle": {
   "accelerator": "gpu",
   "dataSources": [
    {
     "sourceId": 9175240,
     "sourceType": "datasetVersion",
     "datasetId": 5545075
    }
   ],
   "dockerImageVersionId": 30747,
   "isInternetEnabled": true,
   "language": "python",
   "sourceType": "notebook",
   "isGpuEnabled": true
  }
 },
 "nbformat_minor": 5,
 "nbformat": 4,
 "cells": [
  {
   "cell_type": "code",
   "source": [
    "!pip install evaluate accelerate\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from sklearn.metrics import accuracy_score, classification_report\n",
    "from transformers import AutoModelForSequenceClassification\n",
    "from transformers import AutoTokenizer\n",
    "from transformers import TrainingArguments, Trainer, EvalPrediction\n",
    "import evaluate\n",
    "\n",
    "%matplotlib inline"
   ],
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-08-13T14:35:34.067595Z",
     "start_time": "2024-08-13T14:35:30.815275Z"
    },
    "execution": {
     "iopub.status.busy": "2024-08-14T22:18:42.785078Z",
     "iopub.execute_input": "2024-08-14T22:18:42.785381Z",
     "iopub.status.idle": "2024-08-14T22:19:14.846872Z",
     "shell.execute_reply.started": "2024-08-14T22:18:42.785347Z",
     "shell.execute_reply": "2024-08-14T22:19:14.846045Z"
    },
    "trusted": true
   },
   "execution_count": 1,
   "outputs": [
    {
     "name": "stdout",
     "text": "Collecting evaluate\n  Downloading evaluate-0.4.2-py3-none-any.whl.metadata (9.3 kB)\nRequirement already satisfied: accelerate in /opt/conda/lib/python3.10/site-packages (0.32.1)\nRequirement already satisfied: datasets>=2.0.0 in /opt/conda/lib/python3.10/site-packages (from evaluate) (2.20.0)\nRequirement already satisfied: numpy>=1.17 in /opt/conda/lib/python3.10/site-packages (from evaluate) (1.26.4)\nRequirement already satisfied: dill in /opt/conda/lib/python3.10/site-packages (from evaluate) (0.3.8)\nRequirement already satisfied: pandas in /opt/conda/lib/python3.10/site-packages (from evaluate) (2.2.2)\nRequirement already satisfied: requests>=2.19.0 in /opt/conda/lib/python3.10/site-packages (from evaluate) (2.32.3)\nRequirement already satisfied: tqdm>=4.62.1 in /opt/conda/lib/python3.10/site-packages (from evaluate) (4.66.4)\nRequirement already satisfied: xxhash in /opt/conda/lib/python3.10/site-packages (from evaluate) (3.4.1)\nRequirement already satisfied: multiprocess in /opt/conda/lib/python3.10/site-packages (from evaluate) (0.70.16)\nRequirement already satisfied: fsspec>=2021.05.0 in /opt/conda/lib/python3.10/site-packages (from fsspec[http]>=2021.05.0->evaluate) (2024.5.0)\nRequirement already satisfied: huggingface-hub>=0.7.0 in /opt/conda/lib/python3.10/site-packages (from evaluate) (0.23.4)\nRequirement already satisfied: packaging in /opt/conda/lib/python3.10/site-packages (from evaluate) (21.3)\nRequirement already satisfied: psutil in /opt/conda/lib/python3.10/site-packages (from accelerate) (5.9.3)\nRequirement already satisfied: pyyaml in /opt/conda/lib/python3.10/site-packages (from accelerate) (6.0.1)\nRequirement already satisfied: torch>=1.10.0 in /opt/conda/lib/python3.10/site-packages (from accelerate) (2.1.2)\nRequirement already satisfied: safetensors>=0.3.1 in /opt/conda/lib/python3.10/site-packages (from accelerate) (0.4.3)\nRequirement already satisfied: filelock in /opt/conda/lib/python3.10/site-packages (from datasets>=2.0.0->evaluate) (3.13.1)\nRequirement already satisfied: pyarrow>=15.0.0 in /opt/conda/lib/python3.10/site-packages (from datasets>=2.0.0->evaluate) (16.1.0)\nRequirement already satisfied: pyarrow-hotfix in /opt/conda/lib/python3.10/site-packages (from datasets>=2.0.0->evaluate) (0.6)\nRequirement already satisfied: aiohttp in /opt/conda/lib/python3.10/site-packages (from datasets>=2.0.0->evaluate) (3.9.1)\nRequirement already satisfied: typing-extensions>=3.7.4.3 in /opt/conda/lib/python3.10/site-packages (from huggingface-hub>=0.7.0->evaluate) (4.9.0)\nRequirement already satisfied: pyparsing!=3.0.5,>=2.0.2 in /opt/conda/lib/python3.10/site-packages (from packaging->evaluate) (3.1.1)\nRequirement already satisfied: charset-normalizer<4,>=2 in /opt/conda/lib/python3.10/site-packages (from requests>=2.19.0->evaluate) (3.3.2)\nRequirement already satisfied: idna<4,>=2.5 in /opt/conda/lib/python3.10/site-packages (from requests>=2.19.0->evaluate) (3.6)\nRequirement already satisfied: urllib3<3,>=1.21.1 in /opt/conda/lib/python3.10/site-packages (from requests>=2.19.0->evaluate) (1.26.18)\nRequirement already satisfied: certifi>=2017.4.17 in /opt/conda/lib/python3.10/site-packages (from requests>=2.19.0->evaluate) (2024.7.4)\nRequirement already satisfied: sympy in /opt/conda/lib/python3.10/site-packages (from torch>=1.10.0->accelerate) (1.13.0)\nRequirement already satisfied: networkx in /opt/conda/lib/python3.10/site-packages (from torch>=1.10.0->accelerate) (3.2.1)\nRequirement already satisfied: jinja2 in /opt/conda/lib/python3.10/site-packages (from torch>=1.10.0->accelerate) (3.1.2)\nRequirement already satisfied: python-dateutil>=2.8.2 in /opt/conda/lib/python3.10/site-packages (from pandas->evaluate) (2.9.0.post0)\nRequirement already satisfied: pytz>=2020.1 in /opt/conda/lib/python3.10/site-packages (from pandas->evaluate) (2023.3.post1)\nRequirement already satisfied: tzdata>=2022.7 in /opt/conda/lib/python3.10/site-packages (from pandas->evaluate) (2023.4)\nRequirement already satisfied: attrs>=17.3.0 in /opt/conda/lib/python3.10/site-packages (from aiohttp->datasets>=2.0.0->evaluate) (23.2.0)\nRequirement already satisfied: multidict<7.0,>=4.5 in /opt/conda/lib/python3.10/site-packages (from aiohttp->datasets>=2.0.0->evaluate) (6.0.4)\nRequirement already satisfied: yarl<2.0,>=1.0 in /opt/conda/lib/python3.10/site-packages (from aiohttp->datasets>=2.0.0->evaluate) (1.9.3)\nRequirement already satisfied: frozenlist>=1.1.1 in /opt/conda/lib/python3.10/site-packages (from aiohttp->datasets>=2.0.0->evaluate) (1.4.1)\nRequirement already satisfied: aiosignal>=1.1.2 in /opt/conda/lib/python3.10/site-packages (from aiohttp->datasets>=2.0.0->evaluate) (1.3.1)\nRequirement already satisfied: async-timeout<5.0,>=4.0 in /opt/conda/lib/python3.10/site-packages (from aiohttp->datasets>=2.0.0->evaluate) (4.0.3)\nRequirement already satisfied: six>=1.5 in /opt/conda/lib/python3.10/site-packages (from python-dateutil>=2.8.2->pandas->evaluate) (1.16.0)\nRequirement already satisfied: MarkupSafe>=2.0 in /opt/conda/lib/python3.10/site-packages (from jinja2->torch>=1.10.0->accelerate) (2.1.3)\nRequirement already satisfied: mpmath<1.4,>=1.1.0 in /opt/conda/lib/python3.10/site-packages (from sympy->torch>=1.10.0->accelerate) (1.3.0)\nDownloading evaluate-0.4.2-py3-none-any.whl (84 kB)\n\u001B[2K   \u001B[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001B[0m \u001B[32m84.1/84.1 kB\u001B[0m \u001B[31m3.6 MB/s\u001B[0m eta \u001B[36m0:00:00\u001B[0m\n\u001B[?25hInstalling collected packages: evaluate\nSuccessfully installed evaluate-0.4.2\n",
     "output_type": "stream"
    },
    {
     "name": "stderr",
     "text": "2024-08-14 22:19:04.126826: E external/local_xla/xla/stream_executor/cuda/cuda_dnn.cc:9261] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered\n2024-08-14 22:19:04.126921: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:607] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered\n2024-08-14 22:19:04.250147: E external/local_xla/xla/stream_executor/cuda/cuda_blas.cc:1515] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered\n",
     "output_type": "stream"
    }
   ],
   "id": "6a72e2abf27b0177"
  },
  {
   "cell_type": "code",
   "source": [
    "# Load split data\n",
    "yahoo_train = pd.read_csv(\"/kaggle/input/ugursayahoo-finance-news-sentences/yahoo_train.csv\")\n",
    "yahoo_test = pd.read_csv(\"/kaggle/input/ugursayahoo-finance-news-sentences/yahoo_test.csv\")"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2024-08-13T14:35:34.131596Z",
     "start_time": "2024-08-13T14:35:34.067595Z"
    },
    "execution": {
     "iopub.status.busy": "2024-08-14T22:19:14.848657Z",
     "iopub.execute_input": "2024-08-14T22:19:14.849696Z",
     "iopub.status.idle": "2024-08-14T22:19:14.973257Z",
     "shell.execute_reply.started": "2024-08-14T22:19:14.849659Z",
     "shell.execute_reply": "2024-08-14T22:19:14.972288Z"
    },
    "trusted": true
   },
   "execution_count": 2,
   "outputs": [],
   "id": "c3714c627e9d3448"
  },
  {
   "cell_type": "code",
   "source": [
    "yahoo_train.head()"
   ],
   "metadata": {
    "execution": {
     "iopub.status.busy": "2024-08-14T22:19:14.974438Z",
     "iopub.execute_input": "2024-08-14T22:19:14.974726Z",
     "iopub.status.idle": "2024-08-14T22:19:14.991537Z",
     "shell.execute_reply.started": "2024-08-14T22:19:14.974701Z",
     "shell.execute_reply": "2024-08-14T22:19:14.990604Z"
    },
    "trusted": true
   },
   "execution_count": 3,
   "outputs": [
    {
     "execution_count": 3,
     "output_type": "execute_result",
     "data": {
      "text/plain": "   label                                               text\n0      1  “A significant portion of the agenda will focu...\n1      0  From a year ago, it advanced 4% for a second m...\n2      1  There are reports that Google, which is also b...\n3      2  In addition, a downgrade of China’s outlook by...\n4      0  But at the COP28 summit in Dubai, several init...",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>label</th>\n      <th>text</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>1</td>\n      <td>“A significant portion of the agenda will focu...</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>0</td>\n      <td>From a year ago, it advanced 4% for a second m...</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>1</td>\n      <td>There are reports that Google, which is also b...</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>2</td>\n      <td>In addition, a downgrade of China’s outlook by...</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>0</td>\n      <td>But at the COP28 summit in Dubai, several init...</td>\n    </tr>\n  </tbody>\n</table>\n</div>"
     },
     "metadata": {}
    }
   ],
   "id": "41915044425cd97b"
  },
  {
   "cell_type": "code",
   "source": [
    "from datasets import Dataset\n",
    "\n",
    "ds_train = Dataset.from_pandas(yahoo_train)\n",
    "ds_test = Dataset.from_pandas(yahoo_test)"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2024-08-13T14:35:36.271517Z",
     "start_time": "2024-08-13T14:35:36.250214Z"
    },
    "execution": {
     "iopub.status.busy": "2024-08-14T22:19:18.570067Z",
     "iopub.execute_input": "2024-08-14T22:19:18.570443Z",
     "iopub.status.idle": "2024-08-14T22:19:18.635368Z",
     "shell.execute_reply.started": "2024-08-14T22:19:18.570408Z",
     "shell.execute_reply": "2024-08-14T22:19:18.634439Z"
    },
    "trusted": true
   },
   "execution_count": 4,
   "outputs": [],
   "id": "195a6d7502d1056c"
  },
  {
   "cell_type": "code",
   "source": [
    "MODEL_NAME = \"google-bert/bert-base-uncased\"\n",
    "tokenizer = AutoTokenizer.from_pretrained(MODEL_NAME)\n",
    "\n",
    "\n",
    "def tokenize(datapoint):\n",
    "    return tokenizer(datapoint['text'], padding='max_length', truncation=True)\n",
    "\n",
    "\n",
    "tokenized_train_df = ds_train.map(tokenize, batched=True)\n",
    "tokenized_test_df = ds_test.map(tokenize, batched=True)"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2024-08-13T14:36:17.231778Z",
     "start_time": "2024-08-13T14:36:14.890352Z"
    },
    "execution": {
     "iopub.status.busy": "2024-08-14T22:19:19.760161Z",
     "iopub.execute_input": "2024-08-14T22:19:19.760794Z",
     "iopub.status.idle": "2024-08-14T22:19:31.049158Z",
     "shell.execute_reply.started": "2024-08-14T22:19:19.760759Z",
     "shell.execute_reply": "2024-08-14T22:19:31.048361Z"
    },
    "trusted": true
   },
   "execution_count": 5,
   "outputs": [
    {
     "output_type": "display_data",
     "data": {
      "text/plain": "tokenizer_config.json:   0%|          | 0.00/48.0 [00:00<?, ?B/s]",
      "application/vnd.jupyter.widget-view+json": {
       "version_major": 2,
       "version_minor": 0,
       "model_id": "db3bc04b2ecd475ba12d352b6746bb41"
      }
     },
     "metadata": {}
    },
    {
     "output_type": "display_data",
     "data": {
      "text/plain": "config.json:   0%|          | 0.00/570 [00:00<?, ?B/s]",
      "application/vnd.jupyter.widget-view+json": {
       "version_major": 2,
       "version_minor": 0,
       "model_id": "18919872c46f4bf089f8a634f30d445a"
      }
     },
     "metadata": {}
    },
    {
     "output_type": "display_data",
     "data": {
      "text/plain": "vocab.txt:   0%|          | 0.00/232k [00:00<?, ?B/s]",
      "application/vnd.jupyter.widget-view+json": {
       "version_major": 2,
       "version_minor": 0,
       "model_id": "3543ac74ffc14742ac69beade90702fa"
      }
     },
     "metadata": {}
    },
    {
     "output_type": "display_data",
     "data": {
      "text/plain": "tokenizer.json:   0%|          | 0.00/466k [00:00<?, ?B/s]",
      "application/vnd.jupyter.widget-view+json": {
       "version_major": 2,
       "version_minor": 0,
       "model_id": "0eb6f6fb276b430a9073510302caceca"
      }
     },
     "metadata": {}
    },
    {
     "output_type": "display_data",
     "data": {
      "text/plain": "Map:   0%|          | 0/17522 [00:00<?, ? examples/s]",
      "application/vnd.jupyter.widget-view+json": {
       "version_major": 2,
       "version_minor": 0,
       "model_id": "5403a11817d44e198dec5cc3fbffa381"
      }
     },
     "metadata": {}
    },
    {
     "output_type": "display_data",
     "data": {
      "text/plain": "Map:   0%|          | 0/7510 [00:00<?, ? examples/s]",
      "application/vnd.jupyter.widget-view+json": {
       "version_major": 2,
       "version_minor": 0,
       "model_id": "1538229297ca4d9aa59dd978e463c650"
      }
     },
     "metadata": {}
    }
   ],
   "id": "dead4bf049d6a72b"
  },
  {
   "cell_type": "code",
   "source": [
    "model = AutoModelForSequenceClassification.from_pretrained(MODEL_NAME, num_labels=3)\n",
    "\n",
    "training_args = TrainingArguments(\n",
    "    output_dir=\"yahoo_bert\",\n",
    "    eval_strategy=\"epoch\",\n",
    "    per_device_train_batch_size=24,\n",
    "    per_device_eval_batch_size=24,\n",
    "    num_train_epochs=5,\n",
    "    report_to=\"none\",\n",
    ")\n",
    "\n",
    "metric = evaluate.load(\"accuracy\")"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2024-08-13T14:37:03.369850Z",
     "start_time": "2024-08-13T14:37:00.761156Z"
    },
    "execution": {
     "iopub.status.busy": "2024-08-14T22:19:31.050724Z",
     "iopub.execute_input": "2024-08-14T22:19:31.051015Z",
     "iopub.status.idle": "2024-08-14T22:19:33.905259Z",
     "shell.execute_reply.started": "2024-08-14T22:19:31.050989Z",
     "shell.execute_reply": "2024-08-14T22:19:33.904399Z"
    },
    "trusted": true
   },
   "execution_count": 6,
   "outputs": [
    {
     "output_type": "display_data",
     "data": {
      "text/plain": "model.safetensors:   0%|          | 0.00/440M [00:00<?, ?B/s]",
      "application/vnd.jupyter.widget-view+json": {
       "version_major": 2,
       "version_minor": 0,
       "model_id": "9a214484e0c340f08765fc861d8729e2"
      }
     },
     "metadata": {}
    },
    {
     "name": "stderr",
     "text": "Some weights of BertForSequenceClassification were not initialized from the model checkpoint at google-bert/bert-base-uncased and are newly initialized: ['classifier.bias', 'classifier.weight']\nYou should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n",
     "output_type": "stream"
    },
    {
     "output_type": "display_data",
     "data": {
      "text/plain": "Downloading builder script:   0%|          | 0.00/4.20k [00:00<?, ?B/s]",
      "application/vnd.jupyter.widget-view+json": {
       "version_major": 2,
       "version_minor": 0,
       "model_id": "b160a1ca741c4b5c9d88ea3362604d2b"
      }
     },
     "metadata": {}
    }
   ],
   "id": "fc3959d9bb20b4a2"
  },
  {
   "cell_type": "code",
   "source": [
    "def compute_metrics(eval_pred):\n",
    "    logits, labels = eval_pred\n",
    "    y_pred = np.argmax(logits, axis=-1)\n",
    "    return metric.compute(predictions=y_pred, references=labels)"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2024-08-13T14:37:03.723397Z",
     "start_time": "2024-08-13T14:37:03.720029Z"
    },
    "execution": {
     "iopub.status.busy": "2024-08-14T22:19:36.410373Z",
     "iopub.execute_input": "2024-08-14T22:19:36.411041Z",
     "iopub.status.idle": "2024-08-14T22:19:36.416204Z",
     "shell.execute_reply.started": "2024-08-14T22:19:36.411004Z",
     "shell.execute_reply": "2024-08-14T22:19:36.415118Z"
    },
    "trusted": true
   },
   "execution_count": 7,
   "outputs": [],
   "id": "de764eb4d2da7c6b"
  },
  {
   "cell_type": "code",
   "source": [
    "trainer = Trainer(\n",
    "    model=model,\n",
    "    args=training_args,\n",
    "    train_dataset=tokenized_train_df,\n",
    "    eval_dataset=tokenized_test_df,\n",
    "    compute_metrics=compute_metrics,\n",
    ")\n",
    "\n",
    "trainer.train()"
   ],
   "metadata": {
    "collapsed": false,
    "is_executing": true,
    "ExecuteTime": {
     "start_time": "2024-08-13T14:37:06.883440Z"
    },
    "execution": {
     "iopub.status.busy": "2024-08-14T22:19:38.646531Z",
     "iopub.execute_input": "2024-08-14T22:19:38.646902Z",
     "iopub.status.idle": "2024-08-14T23:51:03.914226Z",
     "shell.execute_reply.started": "2024-08-14T22:19:38.646873Z",
     "shell.execute_reply": "2024-08-14T23:51:03.913316Z"
    },
    "trusted": true
   },
   "execution_count": 8,
   "outputs": [
    {
     "output_type": "display_data",
     "data": {
      "text/plain": "<IPython.core.display.HTML object>",
      "text/html": "\n    <div>\n      \n      <progress value='3655' max='3655' style='width:300px; height:20px; vertical-align: middle;'></progress>\n      [3655/3655 1:31:22, Epoch 5/5]\n    </div>\n    <table border=\"1\" class=\"dataframe\">\n  <thead>\n <tr style=\"text-align: left;\">\n      <th>Epoch</th>\n      <th>Training Loss</th>\n      <th>Validation Loss</th>\n      <th>Accuracy</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <td>1</td>\n      <td>0.385000</td>\n      <td>0.269857</td>\n      <td>0.903196</td>\n    </tr>\n    <tr>\n      <td>2</td>\n      <td>0.200500</td>\n      <td>0.347259</td>\n      <td>0.910519</td>\n    </tr>\n    <tr>\n      <td>3</td>\n      <td>0.046000</td>\n      <td>0.464147</td>\n      <td>0.917443</td>\n    </tr>\n    <tr>\n      <td>4</td>\n      <td>0.031300</td>\n      <td>0.520912</td>\n      <td>0.918109</td>\n    </tr>\n    <tr>\n      <td>5</td>\n      <td>0.005900</td>\n      <td>0.555070</td>\n      <td>0.920506</td>\n    </tr>\n  </tbody>\n</table><p>"
     },
     "metadata": {}
    },
    {
     "execution_count": 8,
     "output_type": "execute_result",
     "data": {
      "text/plain": "TrainOutput(global_step=3655, training_loss=0.11049416410939313, metrics={'train_runtime': 5484.5434, 'train_samples_per_second': 15.974, 'train_steps_per_second': 0.666, 'total_flos': 2.305136652715008e+16, 'train_loss': 0.11049416410939313, 'epoch': 5.0})"
     },
     "metadata": {}
    }
   ],
   "id": "78964398268c67c2"
  },
  {
   "cell_type": "code",
   "source": [
    "import torch\n",
    "torch.cuda.empty_cache()"
   ],
   "metadata": {
    "execution": {
     "iopub.status.busy": "2024-08-14T22:17:47.096042Z",
     "iopub.execute_input": "2024-08-14T22:17:47.096403Z",
     "iopub.status.idle": "2024-08-14T22:17:47.102796Z",
     "shell.execute_reply.started": "2024-08-14T22:17:47.096369Z",
     "shell.execute_reply": "2024-08-14T22:17:47.101696Z"
    },
    "trusted": true
   },
   "execution_count": 19,
   "outputs": [],
   "id": "a618168f231a4d56"
  },
  {
   "cell_type": "code",
   "source": [
    "from sklearn.metrics import accuracy_score, classification_report\n",
    "from transformers import EvalPrediction\n",
    "\n",
    "# Define the evaluation metric\n",
    "def eval_metrics(p: EvalPrediction):\n",
    "    preds = np.argmax(p.predictions, axis=1)\n",
    "    report = classification_report(p.label_ids, preds)\n",
    "    print(report)\n",
    "    acc = accuracy_score(p.label_ids, preds)\n",
    "    return {\"accuracy\": acc}\n",
    "\n",
    "\n",
    "# Define training arguments\n",
    "eval_args = TrainingArguments(\n",
    "    output_dir=\"./results_bert_yahoo\",\n",
    "    do_train=False,\n",
    "    do_eval=True,\n",
    "    per_device_eval_batch_size=16,\n",
    "    report_to=\"none\",\n",
    "    logging_dir=\"./logs\",\n",
    "    logging_steps=10,\n",
    ")\n",
    "\n",
    "# Initialize the Trainer\n",
    "trainer = Trainer(\n",
    "    model=model,\n",
    "    args=eval_args,\n",
    "    eval_dataset=tokenized_test_df,\n",
    "    compute_metrics=eval_metrics,\n",
    ")\n",
    "\n",
    "# Run evaluation\n",
    "eval_result = trainer.evaluate()\n",
    "\n",
    "# Print results\n",
    "print(eval_result)"
   ],
   "metadata": {
    "execution": {
     "iopub.status.busy": "2024-08-14T23:59:04.449865Z",
     "iopub.execute_input": "2024-08-14T23:59:04.450829Z",
     "iopub.status.idle": "2024-08-15T00:01:15.885488Z",
     "shell.execute_reply.started": "2024-08-14T23:59:04.450782Z",
     "shell.execute_reply": "2024-08-15T00:01:15.884565Z"
    },
    "trusted": true
   },
   "execution_count": 9,
   "outputs": [
    {
     "output_type": "display_data",
     "data": {
      "text/plain": "<IPython.core.display.HTML object>",
      "text/html": "\n    <div>\n      \n      <progress value='470' max='470' style='width:300px; height:20px; vertical-align: middle;'></progress>\n      [470/470 02:11]\n    </div>\n    "
     },
     "metadata": {}
    },
    {
     "name": "stdout",
     "text": "              precision    recall  f1-score   support\n\n           0       0.90      0.91      0.91      2204\n           1       0.94      0.92      0.93      3167\n           2       0.90      0.93      0.92      2139\n\n    accuracy                           0.92      7510\n   macro avg       0.92      0.92      0.92      7510\nweighted avg       0.92      0.92      0.92      7510\n\n{'eval_loss': 0.5550699234008789, 'eval_accuracy': 0.9205059920106524, 'eval_runtime': 131.3924, 'eval_samples_per_second': 57.157, 'eval_steps_per_second': 3.577}\n",
     "output_type": "stream"
    }
   ],
   "id": "a76722178e052b16"
  },
  {
   "cell_type": "code",
   "source": [
    "import shutil\n",
    "shutil.make_archive('yahoo-bert-uncased', 'zip', '/kaggle/working/yahoo_bert/checkpoint-2500')"
   ],
   "metadata": {
    "execution": {
     "iopub.status.busy": "2024-08-15T00:06:54.585272Z",
     "iopub.execute_input": "2024-08-15T00:06:54.585932Z",
     "iopub.status.idle": "2024-08-15T00:08:01.981093Z",
     "shell.execute_reply.started": "2024-08-15T00:06:54.585898Z",
     "shell.execute_reply": "2024-08-15T00:08:01.980146Z"
    },
    "trusted": true
   },
   "execution_count": 10,
   "outputs": [
    {
     "execution_count": 10,
     "output_type": "execute_result",
     "data": {
      "text/plain": "'/kaggle/working/yahoo-bert-uncased.zip'"
     },
     "metadata": {}
    }
   ],
   "id": "cd724b4367b116ea"
  }
 ]
}