2_file_read_data_and_collate.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import string\n",
    "import re\n",
    "from nltk.stem import *\n",
    "import nltk\n",
    "import math"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Global variables\n",
    "folder_path = 'articles_all'\n",
    "file_contents_list = []\n",
    "\n",
    "\n",
    "file_name_number_mapping_dict = {}\n",
    "number_file_name_mapping_dict = {}\n",
    "inverted_index = {}\n",
    "num_docs = 1\n",
    "idf_dict = {}\n",
    "tf_dict = {}\n",
    "file_to_vec_dict = {}\n",
    "query_to_vec_dict = {}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "class fileContents:\n",
    "    file_name = \"\"\n",
    "    file_number = 0\n",
    "    contents_raw = \"\"\n",
    "    contents_tokens = []"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "def tokenize_and_remove_punctuations(s):\n",
    "    translator = str.maketrans('','',string.punctuation)\n",
    "    modified_string = s.translate(translator)\n",
    "    modified_string = ''.join([i for i in modified_string if not i.isdigit()])\n",
    "    return nltk.word_tokenize(modified_string)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_stopwords():\n",
    "    stop_words = [word for word in open('stopwords.txt','r').read().split('\\n')]\n",
    "    return stop_words"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "def parse_data(contents):\n",
    "    contents = contents.lower()\n",
    "    title_start = contents.find('<title>')\n",
    "    title_end = contents.find('</title>')\n",
    "    title = contents[title_start+len('<title>'):title_end]\n",
    "    text_start = contents.find('<text>')\n",
    "    text_end = contents.find('</text>')\n",
    "    text = contents[text_start+len('<text>'):text_end]\n",
    "    return title+\" \"+text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "def stem_words(tokens):\n",
    "    stemmer = PorterStemmer()\n",
    "    stemmed_words = [stemmer.stem(token) for token in tokens]\n",
    "    return stemmed_words"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "def remove_stop_words(tokens):\n",
    "    stop_words = get_stopwords()\n",
    "    filtered_words = [token for token in tokens if token not in stop_words and len(token) > 2]\n",
    "    return filtered_words"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_vocabulary(data):\n",
    "    tokens = []\n",
    "    for token_list in data.values():\n",
    "        tokens = tokens + token_list\n",
    "    fdist = nltk.FreqDist(tokens)\n",
    "    return list(fdist.keys())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "def preprocess_data(contents):\n",
    "    dataDict = {}\n",
    "    data_list = []\n",
    "    for content in contents:\n",
    "\n",
    "        tokens = tokenize_and_remove_punctuations(content)\n",
    "        filtered_tokens = remove_stop_words(tokens)\n",
    "        stemmed_tokens = stem_words(filtered_tokens)\n",
    "\n",
    "        if len(stemmed_tokens) !=  0:\n",
    "            data_list.append(stemmed_tokens)\n",
    "\n",
    "    return data_list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": [
    "def read_data(path):\n",
    "    curr_file_num = 1\n",
    "    \n",
    "    for filename in os.listdir(path):\n",
    "        # print (filename)\n",
    "        if curr_file_num%1000 == 0:\n",
    "            print (curr_file_num)\n",
    "\n",
    "        global file_contents_list\n",
    "\n",
    "        file_data = fileContents()\n",
    "        # contents = []\n",
    "        \n",
    "        data = parse_data(open(path + '/' + filename,'r').read())\n",
    "    \n",
    "        data = data.replace('\\n', ' ').replace(\"'''\", '').replace(\"''\", '').replace(\",\", ' ').strip()\n",
    "\n",
    "        # print (data)\n",
    "        file_data.contents_raw = data\n",
    "        file_data.file_num = curr_file_num\n",
    "        file_data.file_name = filename\n",
    "\n",
    "        data_list = data.split()\n",
    "        file_data.contents_tokens = preprocess_data(data_list)\n",
    "\n",
    "        file_contents_list.append(file_data)\n",
    "\n",
    "        curr_file_num += 1\n",
    "    return  inverted_index "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1000\n",
      "2000\n",
      "3000\n",
      "4000\n",
      "5000\n",
      "6000\n",
      "7000\n",
      "8000\n",
      "9000\n",
      "10000\n",
      "11000\n",
      "12000\n",
      "13000\n",
      "14000\n",
      "15000\n",
      "16000\n",
      "17000\n",
      "18000\n",
      "19000\n",
      "20000\n",
      "21000\n",
      "22000\n",
      "23000\n",
      "24000\n",
      "25000\n",
      "26000\n",
      "27000\n",
      "28000\n",
      "29000\n",
      "30000\n",
      "31000\n",
      "32000\n",
      "33000\n",
      "34000\n",
      "35000\n",
      "36000\n",
      "37000\n",
      "38000\n",
      "39000\n",
      "40000\n",
      "41000\n",
      "42000\n",
      "43000\n",
      "44000\n",
      "45000\n",
      "46000\n",
      "47000\n",
      "48000\n",
      "49000\n",
      "50000\n",
      "51000\n",
      "52000\n",
      "53000\n",
      "54000\n",
      "55000\n",
      "56000\n",
      "57000\n",
      "58000\n",
      "59000\n",
      "60000\n",
      "61000\n",
      "62000\n",
      "63000\n",
      "64000\n",
      "65000\n",
      "66000\n",
      "67000\n",
      "68000\n",
      "69000\n",
      "70000\n"
     ]
    }
   ],
   "source": [
    "file_data = read_data(folder_path)\n",
    "\n",
    "# for entry in file_contents_list:\n",
    "#     print (f'{entry.file_name}!@#{entry.file_num}!@#{entry.contents_raw}!@#{entry.contents_tokens}')\n",
    "\n",
    "with open('collated_data.txt', 'w') as file:\n",
    "    for entry in file_contents_list:\n",
    "        line =   entry.file_name + \"!@#\" + str(entry.file_num) + \"!@#\" + entry.contents_raw + \"!@#\"  + str(entry.contents_tokens) + \"\\n\"\n",
    "        file.write(line)\n",
    "    \n"
   ]
  }
 ],
 "metadata": {
  "interpreter": {
   "hash": "916dbcbb3f70747c44a77c7bcd40155683ae19c65e1c03b4aa3499c5328201f1"
  },
  "kernelspec": {
   "display_name": "Python 3.9.7 64-bit",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.7"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}