rednotebook.ipynb

{
    "cells": [
        {
            "cell_type": "markdown",
            "id": "07ab75af",
            "metadata": {},
            "source": [
                "# Import Libraries"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": null,
            "id": "fc4c3dfb",
            "metadata": {},
            "outputs": [],
            "source": [
                "from python import FileManager\n",
                "from python import WordCleaner\n",
                "from python import Indexer\n",
                "from python import Matcher\n",
                "from python import Evaluater\n",
                "from python import Clusterer\n",
                "from sklearn.feature_extraction.text import TfidfVectorizer\n",
                "from sklearn.decomposition import TruncatedSVD\n",
                "from nltk.tokenize import word_tokenize\n",
                "from tqdm import tqdm\n",
                "import torch\n",
                "import numpy as np\n",
                "%load_ext autoreload\n",
                "%autoreload 2"
            ]
        },
        {
            "cell_type": "markdown",
            "id": "f86b2947",
            "metadata": {},
            "source": [
                "# Dataset Manipulation "
            ]
        },
        {
            "cell_type": "markdown",
            "id": "069c2740",
            "metadata": {},
            "source": [
                "## Load Files"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": null,
            "id": "91d69ec5",
            "metadata": {},
            "outputs": [],
            "source": [
                "from sentence_transformers import SentenceTransformer\n",
                "\n",
                "model = SentenceTransformer('sentence-transformers/all-MiniLM-L6-v2')\n",
                "\n",
                "indexer = Indexer.Indexer(TfidfVectorizer(),model)\n",
                "dataset = FileManager.csv_to_dict(\"wikir/csv/R0.csv\")\n",
                "datasets = [dataset]\n",
                "dataset_keys = list(datasets[-1].keys())"
            ]
        },
        {
            "cell_type": "markdown",
            "id": "15979398",
            "metadata": {},
            "source": [
                "### The Ultimate Loader"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": null,
            "id": "b3281041",
            "metadata": {},
            "outputs": [],
            "source": [
                "vectorizer = TfidfVectorizer()\n",
                "\n",
                "dataset = {}\n",
                "for i in range(0,4):\n",
                "    dataset = dataset | FileManager.csv_to_dict(f\"lotte/lemlot{i}.csv\")\n",
                "datasets = [dataset]\n",
                "dataset_keys = list(datasets[-1].keys())"
            ]
        },
        {
            "cell_type": "markdown",
            "id": "ae375500",
            "metadata": {},
            "source": [
                "## Remove stop words"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": null,
            "id": "65723f2b",
            "metadata": {},
            "outputs": [],
            "source": [
                "dataset = datasets[-1]\n",
                "filtered_dataset = {}\n",
                "for key in dataset:\n",
                "    filtered_dataset[key] = WordCleaner.remove_stop_words(dataset[key])\n",
                "datasets.append(filtered_dataset)"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": null,
            "id": "b2f4d135",
            "metadata": {},
            "outputs": [],
            "source": [
                "dataset = datasets[-1]\n",
                "no_singles_dataset = {}\n",
                "for key in tqdm(dataset):\n",
                "    no_singles_dataset[key] = WordCleaner.remove_single_letters(dataset[key])\n",
                "datasets.append(no_singles_dataset)"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": null,
            "id": "45492cbd",
            "metadata": {},
            "outputs": [],
            "source": [
                "dataset = datasets[0]\n",
                "processed_dataset = {}\n",
                "for key in tqdm(dataset):\n",
                "    x = WordCleaner.process_capital_punctuation(dataset[key])\n",
                "    if len(x) < 2:\n",
                "        processed_dataset[key] = \" \".join(x) + ' 0'\n",
                "    else:\n",
                "        processed_dataset[key] = \" \".join(x)\n",
                "datasets.append(processed_dataset)"
            ]
        },
        {
            "cell_type": "markdown",
            "id": "6621f9e3",
            "metadata": {},
            "source": [
                "## Stem"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": null,
            "id": "4377f840",
            "metadata": {},
            "outputs": [],
            "source": [
                "dataset = datasets[-1]\n",
                "stemmed_dataset = {}\n",
                "for row in dataset:\n",
                "    stemmed_dataset[row] = WordCleaner.stem(dataset[row], \"Snowball\")\n",
                "datasets.append(stemmed_dataset)"
            ]
        },
        {
            "cell_type": "markdown",
            "id": "8bd7a58d",
            "metadata": {},
            "source": [
                "## Lemmatize"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": null,
            "id": "43fe0f11",
            "metadata": {},
            "outputs": [],
            "source": [
                "dataset = datasets[-1]\n",
                "lemmad_dataset = {}\n",
                "for row in dataset:\n",
                "    lemmad_dataset[row] = WordCleaner.lemmatize(dataset[row])\n",
                "datasets.append(lemmad_dataset)"
            ]
        },
        {
            "cell_type": "markdown",
            "id": "c9c0bcf1",
            "metadata": {},
            "source": [
                "## Synonym Map"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": null,
            "id": "6a65dbf2",
            "metadata": {},
            "outputs": [],
            "source": [
                "from multiprocessing import Pool\n",
                "from tqdm import tqdm\n",
                "\n",
                "dataset = datasets[-1]\n",
                "mapped_2 = {}\n",
                "\n",
                "# Create a pool of workers\n",
                "with Pool() as p:\n",
                "    # Wrap your iterator (dataset) with tqdm for a progress bar\n",
                "    for row in tqdm(dataset):\n",
                "        # Apply the function to each word in the row in parallel\n",
                "        mapped_2[row] = p.map(WordCleaner.get_unified_synonym_2, dataset[row])\n",
                "datasets.append(mapped_2)"
            ]
        },
        {
            "cell_type": "markdown",
            "id": "75f368df",
            "metadata": {},
            "source": [
                "## Calculating tf-idf for the document"
            ]
        },
        {
            "cell_type": "markdown",
            "id": "48b60ed8",
            "metadata": {},
            "source": [
                "### using Scikit Learn"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": null,
            "id": "98eff815",
            "metadata": {},
            "outputs": [],
            "source": [
                "tfidf_matrix = Indexer.calculate_tf_idf(datasets[-1])"
            ]
        },
        {
            "cell_type": "markdown",
            "id": "5abfc76b",
            "metadata": {},
            "source": [
                "## Clustering"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": null,
            "id": "47e4bde7",
            "metadata": {},
            "outputs": [],
            "source": [
                "c = Clusterer.Clusterer(tfidf_matrix,6)\n",
                "topics=c.getTopics(datasets[-1],keys=dataset_keys)\n",
                "c.plot(size=(21, 15))"
            ]
        },
        {
            "cell_type": "markdown",
            "id": "17659ba6",
            "metadata": {},
            "source": [
                "# Queries"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": null,
            "id": "db528201",
            "metadata": {},
            "outputs": [],
            "source": [
                "queries = FileManager.csv_to_dict(\"lotte/gold/forum_queries.tsv\",delimiter=\"\\t\")"
            ]
        },
        {
            "cell_type": "markdown",
            "id": "63e5cbd5",
            "metadata": {},
            "source": [
                "### Text Processing"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": null,
            "id": "9dcc6d1d",
            "metadata": {},
            "outputs": [],
            "source": [
                "from multiprocessing import Pool\n",
                "\n",
                "for key in queries.keys():\n",
                "    queries[key] = WordCleaner.remove_single_letters(queries[key])\n",
                "    queries[key] = WordCleaner.process_capital_punctuation(queries[key])\n",
                "    queries[key] = WordCleaner.remove_stop_words(queries[key],mode=\"spacy\")\n",
                "    queries[key] = WordCleaner.remove_single_letters(queries[key])\n",
                "with Pool() as p:\n",
                "    for row in tqdm(queries):\n",
                "        queries[row] = p.map(WordCleaner.get_unified_synonym_2, queries[row])\n",
                "for key in queries.keys():\n",
                "    queries[key] = WordCleaner.lemmatize(queries[key])"
            ]
        },
        {
            "cell_type": "markdown",
            "id": "0320dc0f",
            "metadata": {},
            "source": [
                "### Calculate TF-IDF"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": null,
            "id": "260722c3",
            "metadata": {},
            "outputs": [],
            "source": [
                "queries_matrices = {}\n",
                "for key in tqdm(queries.keys()):\n",
                "    queries_matrices[key] = indexer.calculate_doc_tf_idf([\" \".join(queries[key])])"
            ]
        },
        {
            "cell_type": "markdown",
            "id": "f7d46388",
            "metadata": {},
            "source": [
                "### Calculate Cosine Similarity"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": null,
            "id": "562157b2",
            "metadata": {},
            "outputs": [],
            "source": [
                "queriesAnswers = {}\n",
                "for key in tqdm(queries_matrices.keys()):\n",
                "    queriesAnswers[key] = Matcher.get_query_answers(tfidf_matrix,queries_matrices[key],dataset_keys,0.35)"
            ]
        },
        {
            "cell_type": "markdown",
            "id": "1edcaba6",
            "metadata": {},
            "source": [
                "# Evaluation"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": null,
            "id": "b968cff3",
            "metadata": {},
            "outputs": [],
            "source": [
                "Evaluater.evaluate(\"lotte/gold/both_qrels\",\"lotte/gold/both_35T.run\")"
            ]
        },
        {
            "cell_type": "markdown",
            "id": "e3311fe2",
            "metadata": {},
            "source": [
                "# Write To Files"
            ]
        },
        {
            "cell_type": "markdown",
            "id": "731ad4ac",
            "metadata": {},
            "source": [
                "## Dataset"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": null,
            "id": "13c2eea2",
            "metadata": {},
            "outputs": [],
            "source": [
                "FileManager.write_dataset_to_file(\"wikirRML.csv\",datasets[-1])"
            ]
        },
        {
            "cell_type": "markdown",
            "id": "d8a34c7e",
            "metadata": {},
            "source": [
                "## Run File"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": null,
            "id": "d525af42",
            "metadata": {},
            "outputs": [],
            "source": [
                "FileManager.write_runfile_to_file(\"Flot35.run\",queries,queriesAnswers)"
            ]
        },
        {
            "cell_type": "markdown",
            "id": "7bdbe7ab",
            "metadata": {},
            "source": [
                "## Model"
            ]
        },
        {
            "cell_type": "markdown",
            "id": "3094d4e4",
            "metadata": {},
            "source": [
                "### Write"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": null,
            "id": "379b09d8",
            "metadata": {},
            "outputs": [],
            "source": [
                "FileManager.write_model_to_drive(\"wikir_RMLN\",vectorizer, dataset_keys, tfidf_matrix)"
            ]
        },
        {
            "cell_type": "markdown",
            "id": "34d03817",
            "metadata": {},
            "source": [
                "### Read"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": null,
            "id": "b8da4498",
            "metadata": {},
            "outputs": [],
            "source": [
                "vectorizer, dataset_keys, tfidf_matrix, corpus_matrix = FileManager.load_model_from_drive(\"python/model/lotte\")"
            ]
        },
        {
            "cell_type": "markdown",
            "id": "aadbb0b4",
            "metadata": {},
            "source": [
                "# Test Sentence-Transformers"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": null,
            "id": "8b87f3d5",
            "metadata": {},
            "outputs": [],
            "source": [
                "dataset = datasets[-1]\n",
                "embeddings = []\n",
                "keys = list(dataset.keys())\n",
                "values = list(dataset.values())\n",
                "batch_size = 10000  # Adjust this value based on your memory capacity\n",
                "\n",
                "for i in tqdm(range(0, len(keys), batch_size)):\n",
                "    batch_keys = keys[i:i+batch_size]\n",
                "    batch_values = values[i:i+batch_size]\n",
                "    batch_embeddings = model.encode(batch_values)\n",
                "    for embedding in batch_embeddings:\n",
                "        embeddings.append(embedding)"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": null,
            "id": "01007786",
            "metadata": {},
            "outputs": [],
            "source": [
                "import numpy\n",
                "# corpus_matrix = np.array(embeddings)\n",
                "# numpy.save(\"wikir_sent_embed\",corpus_matrix)\n",
                "corpus_matrix = numpy.load(\"lotte_sent_embed2.npy\")"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": null,
            "id": "566a2663",
            "metadata": {},
            "outputs": [],
            "source": [
                "dataset_keys = FileManager.load_keys(\"lotteLSA\")"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": null,
            "id": "1c559fd2",
            "metadata": {},
            "outputs": [],
            "source": [
                "queries = FileManager.csv_to_dict(\"lotte/queries1.tsv\",delimiter=\"\\t\",skip=False)\n",
                "embeddingsQ = {}\n",
                "for key in tqdm(queries.keys()):\n",
                "    embeddingsQ[key] = model.encode(\" \".join(queries[key]))\n",
                "    embeddingsQ[key] = embeddingsQ[key].reshape(1,-1)"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": null,
            "id": "5abae459",
            "metadata": {},
            "outputs": [],
            "source": [
                "queries_answers = {}\n",
                "for key in tqdm(queries.keys()):\n",
                "    queries_answers[key] = Matcher.get_query_answers(corpus_matrix, embeddingsQ[key], dataset_keys, 0.5)"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": null,
            "id": "d86b49ac",
            "metadata": {},
            "outputs": [],
            "source": [
                "# Write to run file\n",
                "FileManager.write_runfile_to_file('2Flottesent50.run', queries, queries_answers)"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": null,
            "id": "c4900369",
            "metadata": {},
            "outputs": [],
            "source": [
                "c = Clusterer.Clusterer(corpus_matrix,6)\n",
                "c.getTopics(datasets[-1],keys=dataset_keys)\n",
                "c.plot(size=(21, 15))"
            ]
        }
    ],
    "metadata": {
        "kernelspec": {
            "display_name": "Python 3 (ipykernel)",
            "language": "python",
            "name": "python3"
        },
        "language_info": {
            "codemirror_mode": {
                "name": "ipython",
                "version": 3
            },
            "file_extension": ".py",
            "mimetype": "text/x-python",
            "name": "python",
            "nbconvert_exporter": "python",
            "pygments_lexer": "ipython3",
            "version": "3.11.7"
        }
    },
    "nbformat": 4,
    "nbformat_minor": 5
}