fakenews.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "f:\\Users\\mario\\.conda\\envs\\parking\\Lib\\site-packages\\tqdm\\auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
      "  from .autonotebook import tqdm as notebook_tqdm\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "\n",
    "from transformers import AutoTokenizer, DataCollatorForTokenClassification, AutoTokenizer,\\\n",
    "AutoModelForTokenClassification, AutoModelForMaskedLM, TrainingArguments, Trainer,AutoModelForSequenceClassification,DataCollatorWithPadding\n",
    "import evaluate\n",
    "import torch\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "from datasets import load_dataset,Dataset, DatasetDict\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Some weights of BertForSequenceClassification were not initialized from the model checkpoint at mor40/BulBERT-chitanka-model and are newly initialized: ['classifier.bias', 'bert.pooler.dense.weight', 'classifier.weight', 'bert.pooler.dense.bias']\n",
      "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
     ]
    }
   ],
   "source": [
    "# Load model\n",
    "model_checkpoint = \"mor40/BulBERT-chitanka-model\"\n",
    "model_raw = AutoModelForSequenceClassification.from_pretrained(model_checkpoint, num_labels=2)\n",
    "tokenizer = AutoTokenizer.from_pretrained(model_checkpoint)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "DatasetDict({\n",
       "    train: Dataset({\n",
       "        features: ['title', 'content', 'date_published', 'url', 'label'],\n",
       "        num_rows: 1990\n",
       "    })\n",
       "    validation: Dataset({\n",
       "        features: ['title', 'content', 'date_published', 'url', 'label'],\n",
       "        num_rows: 221\n",
       "    })\n",
       "    test: Dataset({\n",
       "        features: ['title', 'content', 'date_published', 'url', 'label'],\n",
       "        num_rows: 701\n",
       "    })\n",
       "})"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#@title Load Dataset\n",
    "hf_dataset = load_dataset(\"bgglue/bgglue\",\"fakenews\")\n",
    "hf_dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'Positive: 661 Negative: 1329'"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\"Positive: \" + str(hf_dataset[\"train\"][\"label\"].count(1)) + \" Negative: \" + str(hf_dataset[\"train\"][\"label\"].count(0))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'Руска медия ни припомни предсказанията на Ванга и Варсофоний за Сирия! Сирия ще падне и...'"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "hf_dataset[\"train\"][1]['title']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'Медията esoreiter ни припомня за предсказанията на баба Ванга и на монаха Варсофоний. Става на въпрос за това, което са предрекли по адрес на Сирия. Първо да разгледаме какво е казала нашата пророчица. Според нея Сирия ще падне в краката на победителя, но той няма да е, който очакваме. Падането на Дамаск ще е знак, че на света идва нов Месия. Ванга е предрекла и, че древно учение ще се върне на почит след падането на Сирия. То ще донесе мир и хората ще живеят добре. Предсказанието на монаха Варсофоний, който след себе си е оставил няколко точни прогнози за бъдещето, една от които за случващото се в Украйна и за Сирия. „На север от Таврида (б.а. Кримския полуостров) и на изток от Средиземно море ще се лее много кръв. Злото ще дойде от Запада, но ще може да се пребори“, прогнозирал монахът.'"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "hf_dataset[\"train\"][1]['content']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'2017-04-12 12:24:00'"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "hf_dataset[\"train\"][1]['date_published']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'http://spodeli.eu/ruska-mediya-ni-pripomni-predskazaniyata-na-vanga-i-varsofoniy-za-siriya-siriya-shte-padne-i-articles-15943.html'"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "hf_dataset[\"train\"][1]['url']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_domain(example):\n",
    "    example[\"domain\"] =example[\"url\"].split(\"/\")[2]\n",
    "    return example\n",
    "\n",
    "hf_dataset = hf_dataset.map(get_domain)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "def concatenate_features(example):\n",
    "    example[\"features\"] = example[\"domain\"] + \" \" + example['title'] + \" \" + example[\"content\"]\n",
    "    return example\n",
    "\n",
    "hf_dataset = hf_dataset.map(concatenate_features)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Train the model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>input_text</th>\n",
       "      <th>labels</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>petel.bg Petel.bg - новини - \"България днес\": ...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>spodeli.eu Руска медия ни припомни предсказани...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>www.dunavmost.bg:443 Направиха Ванга почетен г...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>www.ekipnews.com Коя е мистериозната баба Ванг...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>bulbox.net Рецептите на Баба Ванга срещу безсъ...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2653</th>\n",
       "      <td>offnews.bg Деси Тенекеджиева снима филма за Лю...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2654</th>\n",
       "      <td>news.data.bg Почина дъщерята на Ванга / Новини...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2655</th>\n",
       "      <td>skafeto.com Завеща ни го Ванга. Чудотворната й...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2656</th>\n",
       "      <td>www.blitz.bg Пророчествата се сбъдват! Ванга: ...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2657</th>\n",
       "      <td>www.novini.bg Топясновидец: Ще има атентат в Б...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>2658 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                             input_text  labels\n",
       "0     petel.bg Petel.bg - новини - \"България днес\": ...       1\n",
       "1     spodeli.eu Руска медия ни припомни предсказани...       0\n",
       "2     www.dunavmost.bg:443 Направиха Ванга почетен г...       1\n",
       "3     www.ekipnews.com Коя е мистериозната баба Ванг...       0\n",
       "4     bulbox.net Рецептите на Баба Ванга срещу безсъ...       0\n",
       "...                                                 ...     ...\n",
       "2653  offnews.bg Деси Тенекеджиева снима филма за Лю...       1\n",
       "2654  news.data.bg Почина дъщерята на Ванга / Новини...       1\n",
       "2655  skafeto.com Завеща ни го Ванга. Чудотворната й...       1\n",
       "2656  www.blitz.bg Пророчествата се сбъдват! Ванга: ...       1\n",
       "2657  www.novini.bg Топясновидец: Ще има атентат в Б...       1\n",
       "\n",
       "[2658 rows x 2 columns]"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from imblearn.over_sampling import RandomOverSampler\n",
    "\n",
    "\n",
    "X = hf_dataset[\"train\"]['features']\n",
    "y = hf_dataset[\"train\"]['label']\n",
    "X_arr = np.array(X).reshape(-1, 1)\n",
    "# Initialize the RandomOverSampler\n",
    "oversampler = RandomOverSampler(sampling_strategy='auto', random_state=42)\n",
    "\n",
    "# Apply oversampling to your data\n",
    "X_resampled, y_resampled = oversampler.fit_resample(X_arr, y)\n",
    "flattened_X = [item for sublist in X_resampled for item in sublist]\n",
    "\n",
    "# Now you have X_resampled and y_resampled with oversampled data\n",
    "result_dataset = pd.DataFrame({\"input_text\": flattened_X, \"labels\":y_resampled })\n",
    "result_dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "labels\n",
       "1    1329\n",
       "0    1329\n",
       "Name: count, dtype: int64"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "result_dataset[\"labels\"].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_dataset = Dataset.from_pandas(result_dataset)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Map: 100%|██████████| 2658/2658 [00:05<00:00, 458.84 examples/s]\n"
     ]
    }
   ],
   "source": [
    "def tokenize(batch):\n",
    " return tokenizer(batch[\"input_text\"],  truncation=True)\n",
    "\n",
    "train_tokenzied = train_dataset.map(tokenize, batched=True, batch_size=None).remove_columns([\"input_text\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Dataset({\n",
       "    features: ['labels', 'input_ids', 'token_type_ids', 'attention_mask'],\n",
       "    num_rows: 2658\n",
       "})"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_tokenzied"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Dataset({\n",
       "    features: ['title', 'content', 'date_published', 'url', 'label', 'domain', 'features'],\n",
       "    num_rows: 221\n",
       "})"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "hf_dataset[\"validation\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "def tokenize(batch):\n",
    " return tokenizer(batch[\"features\"],  truncation=True)\n",
    "\n",
    "validation_tokenized = hf_dataset[\"validation\"].map(tokenize, batched=True, batch_size=None).remove_columns(['title', 'content', 'date_published', 'url', 'domain', 'features'])\n",
    "validation_tokenized =validation_tokenized.rename_column(\"label\",\"labels\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Dataset({\n",
       "    features: ['labels', 'input_ids', 'token_type_ids', 'attention_mask'],\n",
       "    num_rows: 221\n",
       "})"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "validation_tokenized"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "#@title Define model training args\n",
    "accuracy = evaluate.load(\"accuracy\")\n",
    "def compute_metrics(eval_pred):\n",
    "    predictions, labels = eval_pred\n",
    "    predictions = np.argmax(predictions, axis=1)\n",
    "    return accuracy.compute(predictions=predictions, references=labels)\n",
    "\n",
    "\n",
    "data_collator = DataCollatorWithPadding(tokenizer=tokenizer)\n",
    "\n",
    "args = TrainingArguments(\n",
    "    \"BulBERT-fakenews-5epochs\",\n",
    "    evaluation_strategy=\"epoch\",\n",
    "    save_strategy=\"epoch\",\n",
    "    learning_rate=2e-5,\n",
    "    num_train_epochs=5,\n",
    "    weight_decay=0.01,\n",
    "    per_device_train_batch_size=32,\n",
    "    per_device_eval_batch_size=32,\n",
    "    push_to_hub=True,\n",
    ")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 0/420 [00:00<?, ?it/s]You're using a BertTokenizerFast tokenizer. Please note that with a fast tokenizer, using the `__call__` method is faster than using a method to encode the text followed by a call to the `pad` method to get a padded encoding.\n",
      "  0%|          | 2/420 [03:36<12:42:35, 109.46s/it]"
     ]
    }
   ],
   "source": [
    "#@title Train\n",
    "trainer = Trainer(\n",
    "    model=model_raw,\n",
    "    args=args,\n",
    "    train_dataset=train_tokenzied,\n",
    "    eval_dataset=validation_tokenized,\n",
    "    data_collator=data_collator,\n",
    "    compute_metrics=compute_metrics,\n",
    "    tokenizer=tokenizer,\n",
    ")\n",
    "trainer.train()\n",
    "\n",
    "trainer.push_to_hub(commit_message=\"Training complete\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Test the model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "from transformers.tools.text_classification import AutoModelForSequenceClassification\n",
    "checkpoint = \"mor40/BulBERT-fakenews-5epochs\"\n",
    "model = AutoModelForSequenceClassification.from_pretrained(checkpoint)\n",
    "tokenizer = AutoTokenizer.from_pretrained(checkpoint)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Map: 100%|██████████| 221/221 [03:28<00:00,  1.06 examples/s]\n",
      "Map: 100%|██████████| 701/701 [16:48<00:00,  1.44s/ examples]\n"
     ]
    }
   ],
   "source": [
    "def get_predictions(example, model):\n",
    "  inputs = tokenizer(example[\"features\"], return_tensors=\"pt\",truncation = True)\n",
    "  # Run through model\n",
    "  with torch.no_grad():\n",
    "        logits = model(**inputs).logits[0]\n",
    "\n",
    "  label_id = np.argmax(logits).item()\n",
    "  example[\"predicted_label\"] = label_id\n",
    "\n",
    "  return example\n",
    "\n",
    "dataset_validation = hf_dataset[\"validation\"].map(get_predictions,fn_kwargs={\"model\": model})\n",
    "dataset_test = hf_dataset[\"test\"].map(get_predictions,fn_kwargs={\"model\": model})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Dataset({\n",
       "    features: ['title', 'content', 'date_published', 'url', 'label', 'domain', 'features', 'predicted_label'],\n",
       "    num_rows: 701\n",
       "})"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataset_test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy:  0.9049773755656109\n"
     ]
    }
   ],
   "source": [
    "correct = 0\n",
    "for i, example in enumerate(dataset_validation):\n",
    "  if example[\"predicted_label\"] == example[\"label\"]: correct+=1\n",
    "print(\"Accuracy: \" ,correct / len(dataset_validation))\n",
    "# Credible - 0\n",
    "# Humorous- 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "predictions = []\n",
    "for i, example in enumerate(dataset_test):\n",
    "  rec = {}\n",
    "  rec['inedex'] = i\n",
    "  rec['label'] = \"credible\" if example[\"predicted_label\"] == 0 else \"fake\" \n",
    "  predictions.append(rec)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[{'inedex': 0, 'label': 'credible'},\n",
       " {'inedex': 1, 'label': 'credible'},\n",
       " {'inedex': 2, 'label': 'credible'},\n",
       " {'inedex': 3, 'label': 'fake'},\n",
       " {'inedex': 4, 'label': 'credible'}]"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "predictions[0:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "import jsonlines\n",
    "\n",
    "with jsonlines.open('predictions-fakenews-5-epochs.jsonl', 'w') as writer:\n",
    "    writer.write_all(predictions)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "huggingface",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}