notebooks/preprocesing.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import re\n",
    "import string\n",
    "\n",
    "import pandas as pd\n",
    "\n",
    "from time import perf_counter \n",
    "\n",
    "from nltk.corpus import stopwords\n",
    "from nltk.stem import PorterStemmer\n",
    "from nltk import word_tokenize\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "from sklearn.naive_bayes import GaussianNB\n",
    "from sklearn.metrics import accuracy_score\n",
    "\n",
    "re_url = re.compile(r'(?:http|ftp|https)://(?:[\\w_-]+(?:(?:\\.[\\w_-]+)+))(?:[\\w.,@?^=%&:/~+#-]*[\\w@?^=%&/~+#-])?')\n",
    "re_email = re.compile('(?:[a-z0-9!#$%&\\'*+/=?^_`{|}~-]+(?:\\.[a-z0-9!#$%&\\'*+/=?^_`{|}~-]+)*|\"(?:[\\x01-\\x08\\x0b\\x0c\\x0e-\\x1f\\x21\\x23-\\x5b\\x5d-\\x7f]|\\\\[\\x01-\\x09\\x0b\\x0c\\x0e-\\x7f])*\")@(?:(?:[a-z0-9](?:[a-z0-9-]*[a-z0-9])?\\.)+[a-z0-9](?:[a-z0-9-]*[a-z0-9])?|\\[(?:(?:(2(5[0-5]|[0-4][0-9])|1[0-9][0-9]|[1-9]?[0-9]))\\.){3}(?:(2(5[0-5]|[0-4][0-9])|1[0-9][0-9]|[1-9]?[0-9])|[a-z0-9-]*[a-z0-9]:(?:[\\x01-\\x08\\x0b\\x0c\\x0e-\\x1f\\x21-\\x5a\\x53-\\x7f]|\\\\[\\x01-\\x09\\x0b\\x0c\\x0e-\\x7f])+)\\])')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Read dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_full = pd.read_csv('../data/processed/20newsgroup_raw.csv', sep=';')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Preprocessing"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Clean Header"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Wall time: 36.4 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "def clean_header(text):\n",
    "    text = re.sub(r'(From:\\s+[^\\n]+\\n)', '', text)\n",
    "    text = re.sub(r'(Subject:[^\\n]+\\n)', '', text)\n",
    "    text = re.sub(r'(([\\sA-Za-z0-9\\-]+)?[A|a]rchive-name:[^\\n]+\\n)', '', text)\n",
    "    text = re.sub(r'(Last-modified:[^\\n]+\\n)', '', text)\n",
    "    text = re.sub(r'(Version:[^\\n]+\\n)', '', text)\n",
    "\n",
    "    return text\n",
    "\n",
    "\n",
    "df_full['text_cleaned'] = df_full['text'].apply(clean_header)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Clean text\n",
    "\n",
    "https://nlp.stanford.edu/IR-book/html/htmledition/normalization-equivalence-classing-of-terms-1.html"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Wall time: 13.6 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "def clean_text(text):        \n",
    "    text = text.lower()\n",
    "    text = text.strip()\n",
    "    text = re.sub(re_url, '', text)\n",
    "    text = re.sub(re_email, '', text)\n",
    "    text = re.sub(f'[{re.escape(string.punctuation)}]', '', text)\n",
    "    text = re.sub(r'(\\d+)', ' ', text)\n",
    "    text = re.sub(r'(\\s+)', ' ', text)\n",
    "    \n",
    "    return text\n",
    "\n",
    "df_full['text_cleaned'] = df_full['text_cleaned'].apply(clean_text)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Remove stop words\n",
    "\n",
    "https://nlp.stanford.edu/IR-book/html/htmledition/dropping-common-terms-stop-words-1.html"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Wall time: 9.87 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "stop_words = stopwords.words('english')\n",
    "\n",
    "df_full['text_cleaned'] = df_full['text_cleaned'].str.split() \\\n",
    "    .apply(lambda x: ' '.join([word for word in x if word not in stop_words]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Save dataset cleaned"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_full.to_csv('../data/processed/20newsgroup_preprocessed.csv', index=False, sep=';')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Stemming\n",
    "\n",
    "https://nlp.stanford.edu/IR-book/html/htmledition/stemming-and-lemmatization-1.html"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Wall time: 1min 14s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "stemmer = PorterStemmer()\n",
    "\n",
    "df_full['text_cleaned'] = df_full['text_cleaned'].str.split() \\\n",
    "    .apply(lambda x: ' '.join([stemmer.stem(word) for word in x]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Divide in train and test"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Balance classes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "rec.sport.hockey            999\n",
       "soc.religion.christian      997\n",
       "rec.motorcycles             994\n",
       "rec.sport.baseball          994\n",
       "sci.crypt                   991\n",
       "sci.med                     990\n",
       "rec.autos                   990\n",
       "sci.space                   987\n",
       "comp.os.ms-windows.misc     985\n",
       "comp.sys.ibm.pc.hardware    982\n",
       "sci.electronics             981\n",
       "comp.windows.x              980\n",
       "comp.graphics               973\n",
       "misc.forsale                972\n",
       "comp.sys.mac.hardware       961\n",
       "talk.politics.mideast       940\n",
       "talk.politics.guns          910\n",
       "alt.atheism                 799\n",
       "talk.politics.misc          775\n",
       "talk.religion.misc          628\n",
       "Name: target, dtype: int64"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_full['target'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "X = df_full['text_cleaned']\n",
    "y = df_full['target']\n",
    "\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=45, stratify=y)\n",
    "# X_train, X_val, y_train, y_val = train_test_split(X_train, y_train, test_size=0.2, random_state=45, stratify=y) # 0.25 x 0.8 = 0.2"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### TF-IDF (Term frequency-inverse document frequency)\n",
    "\n",
    "https://nlp.stanford.edu/IR-book/html/htmledition/tf-idf-weighting-1.html"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "vectorizer = TfidfVectorizer(min_df=5, max_df=0.4)\n",
    "\n",
    "tfidf_matrix = vectorizer.fit(X_train)\n",
    "\n",
    "X_train_tfidf = tfidf_matrix.transform(X_train)\n",
    "X_test_tfidf = tfidf_matrix.transform(X_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Baseline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = GaussianNB()\n",
    "model.fit(X_train_tfidf.toarray(), y_train)\n",
    "\n",
    "y_pred = model.predict(X_test_tfidf.toarray())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.7721720658523632"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "accuracy_score(y_test, y_pred)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Save result preprocessing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_preprocessing = pd.DataFrame(tfidf_matrix.transform(X).toarray())\n",
    "df_preprocessing['target'] = y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Wall time: 5min 59s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "df_preprocessing.to_csv('../data/processed/newsgroup_vectorized.gz', index=False, compression='gzip')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": true,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": true
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}