Mediacloud_PCA.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import ast \n",
    "from collections import Counter\n",
    "import csv\n",
    "from sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer, TfidfTransformer\n",
    "from factor_analyzer.factor_analyzer import calculate_kmo\n",
    "import numpy as np\n",
    "from sklearn.decomposition import PCA\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "import matplotlib.pyplot as plt\n",
    "from advanced_pca import CustomPCA\n",
    "import gensim\n",
    "import scipy\n",
    "import seaborn as sns; sns.set()\n",
    "from sklearn.cluster import MiniBatchKMeans, KMeans\n",
    "from sklearn.manifold import TSNE\n",
    "import matplotlib.cm as cm\n",
    "from sklearn.cluster import DBSCAN"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The data is used from the 'Mediacloud_Analysis.ipynb' (<a href='Mediacloud_Analysis.ipynb'>link</a>). It already contains preprocessed and tokenized text for each article. Also it has a column with corona terms specifically and their frequency. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#reading the dataframe with pre-processed tokens\n",
    "df = pd.read_csv(\"preprocessed_results/mediacloud_parsed_corona_df_feb.csv\")\n",
    "temp = pd.read_csv(\"preprocessed_results/mediacloud_parsed_corona_df_may.csv\")\n",
    "temp_2 = pd.read_csv(\"preprocessed_results/mediacloud_parsed_corona_df_sep.csv\")\n",
    "df = pd.concat([df,temp])\n",
    "df = pd.concat([df,temp_2])\n",
    "df = df[~df.Text.isnull()] #removing rows withh no text\n",
    "\n",
    "df['tokens'] = df['tokens'].apply(ast.literal_eval) #transforming string of tokens to list\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# sample = df.sample(n=1)\n",
    "\n",
    "temp = pd.read_csv('ncov-or-cov-19-or-covid-or-all-story-urls-20201012133126.csv')\n",
    "sample = temp.sample(n=1)\n",
    "print(sample['url'].values[0])\n",
    "print(sample['title'].values[0])\n",
    "print(sample['publish_date'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "For further procedures we use 500 most frequent tokens, that are later manually reviewed. All names, countries, dates as well as words that do not carry any strong meaning are excluded. They are saved to the 'most_frequent_tokens.csv' file"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#finding 500 most frequent tokens\n",
    "flatten_tokens = [token for sublist in df['tokens'].tolist() for token in sublist]\n",
    "counter_tokens = Counter(flatten_tokens)\n",
    "most_frequent = counter_tokens.most_common(500)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#saving them to csv file\n",
    "with open('most_frequent_tokens.csv', \"w\") as the_file:\n",
    "    csv.register_dialect(\"custom\", delimiter=\",\", skipinitialspace=True)\n",
    "    writer = csv.writer(the_file, dialect=\"custom\")\n",
    "    for tup in most_frequent:\n",
    "        writer.writerow(tup)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "#finding 500 most frequent tokens for SEPTEMBER\n",
    "flatten_tokens = [token for sublist in df['bigrams'][85298:].tolist() for token in sublist]\n",
    "counter_tokens = Counter(flatten_tokens)\n",
    "most_frequent = counter_tokens.most_common(500)\n",
    "\n",
    "#saving them to csv file\n",
    "with open('most_frequent_bigrams_SEP.csv', \"w\") as the_file:\n",
    "    csv.register_dialect(\"custom\", delimiter=\",\", skipinitialspace=True)\n",
    "    writer = csv.writer(the_file, dialect=\"custom\")\n",
    "    for tup in most_frequent:\n",
    "        writer.writerow(tup)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Unigrams"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Reading file with reviewed tokens (<a href=\"most_frequent_tokens_cleaned_v2.csv\">file link</a>)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "tokens = pd.read_csv('most_frequent_tokens_cleaned_v2.csv', header=None, names=['token', 'frequency'])\n",
    "#tokens['tfidf'] = 0"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Firstly the original tokenized texts are converted to the tfidf scores. The result is sparse tfidf matrix. After that for each row only tfidf scores of frequent tokens are kept (for each sparse vector we match id of the tfidf value with dictionary token and check if this token is in the clean list). As a result for each row in the dataframe there is a vector of length n (nuber of cleaned frequent tokens) with tfidf values."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def dummy_fun(doc):\n",
    "    return doc\n",
    "\n",
    "cv = CountVectorizer(analyzer='word',\n",
    "    tokenizer=dummy_fun,\n",
    "    preprocessor=dummy_fun,\n",
    "    token_pattern=None)\n",
    "data = cv.fit_transform(df['tokens'])\n",
    "\n",
    "tfidf_transformer = TfidfTransformer()\n",
    "tfidf_matrix = tfidf_transformer.fit_transform(data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "tfidf_dict = cv.get_feature_names()  #all tokens there are in the original texts\n",
    "df['transformed_tokens'] = np.empty((len(df), 0)).tolist()\n",
    "\n",
    "for i in range(tfidf_matrix.shape[0]):\n",
    "    print(i)\n",
    "    df.at[i, 'transformed_tokens'] = [tfidf_matrix[i].toarray()[0][j] for j in range(len(tfidf_dict)) if tfidf_dict[j] in tokens['token'].tolist()]\n",
    "            "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "temp = df['transformed_tokens'].tolist()\n",
    "temp = [np.array(x) for x in temp]\n",
    "\n",
    "tfidf_frequent = np.array(temp)\n",
    "tfidf_frequent.shape #= [np.array(token_list) for token_list in tokens_transformed]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(\"tfidf_transformed_tokens.csv\", \"w\", newline=\"\") as f:\n",
    "    writer = csv.writer(f)\n",
    "    writer.writerows(temp)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "KMO score is calculated (according to the <a href=\"https://www.tandfonline.com/doi/full/10.1080/1369183X.2017.1282813\">paper</a>). KMO is a measure for sampling adequacy applied in factor analysis. It informs about the general strength of the relationship among items and thus indicates whether an item (i.e. a word) should be included in a factor analysis or not. Following Backhaus et al. (2006), terms with a KMO value below .50 were subsequently excluded."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "kmo_all,kmo_model=calculate_kmo(tfidf_frequent)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "kmo_model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "features_pca = np.zeros((tfidf_frequent.shape[0], len(kmo_all)))\n",
    "for i in range(len(kmo_all)):\n",
    "    if kmo_all[i] > 0.5:  #keeping only those that have kmo over 0.5\n",
    "        features_pca[i] = tfidf_frequent[i]\n",
    "    \n",
    "print(len(features_pca), tfidf_frequent.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Running PCA on the filtered tokens. PCA is applied using <a href=\"https://pypi.org/project/advanced-pca/\"> advanced PCA package</a>. For each number of components factor loadings are calculated (for each term) based on the <a href=\"https://www.r-bloggers.com/p-is-for-principal-components-analysis-pca/\">tutorial here</a>. Only significant terms are taken (with a threshold of 0.1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "scaler = StandardScaler()\n",
    "features_pca_scaled = scaler.fit_transform(features_pca)\n",
    "\n",
    "pca_results = {'Num_of_components': [],\n",
    "                'Explained_variance': [],\n",
    "               'Sum_Explained_variance': [],\n",
    "               'Terms':[]\n",
    "                }\n",
    "for n in range (3, 21):\n",
    "    pca_model = (CustomPCA(n_components=n)\n",
    "                    .fit(features_pca_scaled))\n",
    "    pca_results['Num_of_components'].append(n)\n",
    "    pca_results['Explained_variance'].append(pca_model.explained_variance_ratio_)\n",
    "    pca_results['Sum_Explained_variance'].append(sum(pca_model.explained_variance_ratio_))\n",
    "    all_terms = []\n",
    "    for i in range(n):\n",
    "        scores = [score for score in pca_model.components_[i].round(1) if score>0.1 or score<-0.1]\n",
    "#             tokens_sign = (pca_model.components_[i].round(1)>0.1) or (pca_model.components_[i].round(1)<-0.1)\n",
    "        terms = tokens.token[(pca_model.components_[i].round(1)>0.1) | (pca_model.components_[i].round(1)<-0.1)]\n",
    "        all_terms.append(list(zip(terms, scores)))\n",
    "    pca_results['Terms'].append(all_terms)\n",
    "    \n",
    "pca_results_df = pd.DataFrame(pca_results)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Example with a custom PCA with 3 components, printing variance ratio for each component and factor loadings:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "pca_model = (CustomPCA(n_components=5)\n",
    "                    .fit(features_pca_scaled))\n",
    "print(pca_model.explained_variance_ratio_)\n",
    "pca_model.components_[1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "pca_results_df['Terms'][0]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Saving results of the PCA to the csv file 'results/mediacloud_pca_results_shortlist.csv'. Plot the sum of explained variance based on the number of components:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "pca_results_df.to_csv('results/mediacloud_pca_results_shortlist.csv')\n",
    "pca_results_df.plot.line(x='Num_of_components', y='Sum_Explained_variance')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Save the 'significant' terms for all components (each n of components) with corresponding factor loadings to csv file 'results/pca_terms.csv':"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "pca_results_df['Terms'].to_csv('results/pca_terms.csv')\n",
    "print(pca_results_df['Terms'][5])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "A plot that shows cumulative explained variance and explained variance of each component (with max 20):"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "cummulative_pca = PCA(n_components=20).fit(features_pca_scaled)\n",
    "\n",
    "fig, ax = plt.subplots(figsize=(8,6))\n",
    "x_values = range(1, cummulative_pca.n_components_+1)\n",
    "ax.plot(x_values, cummulative_pca.explained_variance_ratio_, lw=2, label='explained variance')\n",
    "ax.plot(x_values, np.cumsum(cummulative_pca.explained_variance_ratio_), lw=2, label='cumulative explained variance')\n",
    "ax.set_title('PCA on filtered tokens : explained variance of components')\n",
    "ax.set_xlabel('principal component')\n",
    "ax.set_ylabel('explained variance')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Bigrams"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Creating bigrams from the original texts. The bigrams are then saved to file 'most_frequent_tokens_bigrams.csv' and reviewed the same way as the unigrams in the file 'most_frequent_tokens_bigrams.csv' (<a href='most_frequent_tokens_bigrams.csv'>link</a>). The final list contains 87 terms"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "bigram = gensim.models.Phrases(df['tokens'], min_count=3, threshold=50) # higher threshold fewer phrases.\n",
    "bigram_mod = gensim.models.phrases.Phraser(bigram)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def make_bigrams(texts):\n",
    "    return [bigram_mod[doc] for doc in texts]\n",
    "\n",
    "df['bigrams'] = make_bigrams(df['tokens'])\n",
    "df['bigrams']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "flatten_bigrams = [token for sublist in df['bigrams'].tolist() for token in sublist]\n",
    "counter_bigrams = Counter(flatten_bigrams)\n",
    "most_frequent = counter_bigrams.most_common(500)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#saving them to csv file\n",
    "with open('most_frequent_tokens_bigrams.csv', \"w\") as the_file:\n",
    "    csv.register_dialect(\"custom\", delimiter=\",\", skipinitialspace=True)\n",
    "    writer = csv.writer(the_file, dialect=\"custom\")\n",
    "    for tup in most_frequent:\n",
    "        writer.writerow(tup)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# tokens_bigrams = pd.read_csv('most_frequent_tokens_bigrams.csv', header=None, names=['token', 'frequency'])\n",
    "cv = CountVectorizer(analyzer='word',\n",
    "    tokenizer=dummy_fun,\n",
    "    preprocessor=dummy_fun,\n",
    "    token_pattern=None)\n",
    "data = cv.fit_transform(df['bigrams'])\n",
    "\n",
    "tfidf_transformer = TfidfTransformer()\n",
    "tfidf_matrix = tfidf_transformer.fit_transform(data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "tfidf_matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "tfidf_dict_bigrams = cv.get_feature_names()  #all tokens there are in the original texts\n",
    "df['transformed_tokens_bigrams'] = np.empty((len(df), 0)).tolist()\n",
    "\n",
    "for i in range(tfidf_matrix.shape[0]):\n",
    "    print(i)\n",
    "    df.at[i, 'transformed_tokens_bigrams'] = [tfidf_matrix[i].toarray()[0][j] for j in range(len(tfidf_dict_bigrams)) if tfidf_dict_bigrams[j] in tokens_bigrams['token'].tolist()]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(\"tfidf_transformed_bigrams.csv\", \"w\", newline=\"\") as f:\n",
    "    writer = csv.writer(f)\n",
    "    writer.writerows(df['transformed_tokens_bigrams'].tolist())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "temp = df['transformed_tokens_bigrams'].tolist()\n",
    "temp = [np.array(x) for x in temp]\n",
    "\n",
    "tfidf_frequent_bigrams = np.array(temp)\n",
    "tfidf_frequent_bigrams.shape #= [np.array(token_list) for token_list in tokens_transformed]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "kmo_all_bi,kmo_model_bi=calculate_kmo(np.array(tfidf_frequent_bigrams))\n",
    "kmo_model_bi"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "features_bigrams = np.zeros((tfidf_frequent_bigrams.shape[0], len(kmo_all_bi)))\n",
    "for i in range(len(kmo_all_bi)):\n",
    "    if kmo_all_bi[i] > 0.5:  #keeping only those that have kmo over 0.5\n",
    "        features_bigrams[i] = tfidf_frequent_bigrams[i]\n",
    "    \n",
    "print(len(features_bigrams), tfidf_frequent_bigrams.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "scaler = StandardScaler()\n",
    "features_bi_scaled = scaler.fit_transform(features_bigrams)\n",
    "\n",
    "pca_results_bi = {'Num_of_components': [],\n",
    "                'Explained_variance': [],\n",
    "               'Terms':[]\n",
    "                }\n",
    "for n in range (3, 21):\n",
    "    pca_model = (CustomPCA(n_components=n)\n",
    "                    .fit(features_bi_scaled))\n",
    "    pca_results_bi['Num_of_components'].append(n)\n",
    "    pca_results_bi['Explained_variance'].append(sum(pca_model.explained_variance_ratio_))\n",
    "    all_terms = []\n",
    "    for i in range(n):\n",
    "        scores = [score for score in pca_model.components_[i].round(1) if score>0.1]\n",
    "#             tokens_sign = (pca_model.components_[i].round(1)>0.1) or (pca_model.components_[i].round(1)<-0.1)\n",
    "        terms = tokens_bigrams.token[pca_model.components_[i].round(1)>0.1]\n",
    "        all_terms.append(list(zip(terms, scores)))\n",
    "    pca_results_bi['Terms'].append(all_terms)\n",
    "    \n",
    "pca_results_bi_df = pd.DataFrame(pca_results_bi)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "pca_model = (CustomPCA(n_components=3)\n",
    "                    .fit(features_bi_scaled))\n",
    "print(pca_model.explained_variance_ratio_)\n",
    "pca_model.components_[1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "pca_results_bi_df['Terms'][0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "temp = tokens_bigrams['token'].tolist()\n",
    "pca_dict = {}\n",
    "for token in temp:\n",
    "    pca_dict[token] = []\n",
    "    for topic in pca_results_bi_df['Terms'][17]:\n",
    "        if token in [term[0] for term in topic]:\n",
    "            pca_dict[token].append([term[1] for term in topic if term[0]==token][0])\n",
    "        else:\n",
    "            pca_dict[token].append(0)\n",
    "            \n",
    "\n",
    "pca_df = pd.DataFrame(pca_dict).transpose()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "pca_df[pca_df[5]!=0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "pca_results_bi_df.to_csv('results/mediacloud_pca_bigrams_results_shortlist.csv')\n",
    "pca_results_bi_df.plot.line(x='Num_of_components', y='Explained_variance')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "cummulative_pca = PCA(n_components=20).fit(features_bi_scaled)\n",
    "\n",
    "fig, ax = plt.subplots(figsize=(8,6))\n",
    "x_values = range(1, cummulative_pca.n_components_+1)\n",
    "ax.plot(x_values, cummulative_pca.explained_variance_ratio_, lw=2, label='explained variance')\n",
    "ax.plot(x_values, np.cumsum(cummulative_pca.explained_variance_ratio_), lw=2, label='cumulative explained variance')\n",
    "ax.set_title('PCA on filtered tokens : explained variance of components')\n",
    "ax.set_xlabel('principal component')\n",
    "ax.set_ylabel('explained variance')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Toy example\n",
    "\n",
    "The perfect curated list is created, that contains 39 words for 4 frames: economic, medical, travel and restrictions/prevention. The list is available <a href=\"most_frequent_tokens_toy.csv\">here</a>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "tokens_toy = pd.read_csv('most_frequent_tokens_toy.csv', header=None, names=['token', 'frequency'])\n",
    "toy = tokens_toy['token'].sort_values().tolist()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "bigrams_sep = pd.read_csv('most_frequent_bigrams_SEP.csv', header=None, names=['token', 'frequency'])\n",
    "tokens_sep = bigrams_sep['token'].sort_values().tolist()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# tokens_bigrams = pd.read_csv('most_frequent_tokens_bigrams.csv', header=None, names=['token', 'frequency'])\n",
    "def dummy_fun(doc):\n",
    "    return doc\n",
    "\n",
    "cv = CountVectorizer(analyzer='word',\n",
    "    tokenizer=dummy_fun,\n",
    "    preprocessor=dummy_fun,\n",
    "    token_pattern=None)\n",
    "data = cv.fit_transform(df['bigrams'][85298:])\n",
    "\n",
    "tfidf_transformer = TfidfTransformer()\n",
    "tfidf_matrix = tfidf_transformer.fit_transform(data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.reset_index(inplace=True)\n",
    "df.drop(['Unnamed: 0','Unnamed: 0.1'],axis=1,inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "tfidf_matrix.shape[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# tfidf_dict = cv.get_feature_names()  #all tokens there are in the original texts\n",
    "tfidf_dict_bigrams = cv.get_feature_names()\n",
    "transformed_tokens_sep = np.empty((tfidf_matrix.shape[0], 86))\n",
    "\n",
    "for i in range(0, tfidf_matrix.shape[0]):\n",
    "    print(i)\n",
    "#     print([tfidf_matrix[i].toarray()[0][j] for j in range(len(tfidf_dict_bigrams)) if tfidf_dict_bigrams[j] in tokens_sep])\n",
    "    transformed_tokens_sep[i] = [tfidf_matrix[i].toarray()[0][j] for j in range(len(tfidf_dict_bigrams)) if tfidf_dict_bigrams[j] in tokens_sep]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(\"tfidf_transformed_toy_sep.csv\", \"w\", newline=\"\") as f:\n",
    "    writer = csv.writer(f)\n",
    "    writer.writerows(df['transformed_tokens_toy2'][1136:].tolist())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(\"tfidf_transformed_toy_sep.csv\", newline='') as csvfile:\n",
    "    data = list(csv.reader(csvfile))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "temp = data\n",
    "temp = [np.array(x) for x in temp]\n",
    "\n",
    "tfidf_frequent_toy = np.array(temp)\n",
    "tfidf_frequent_toy.shape #= [np.array(token_list) for token_list in tokens_transformed]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "tfidf_frequent_sep = transformed_tokens_sep[:1136].astype(np.float)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "kmo_all_toy,kmo_model_toy=calculate_kmo(tfidf_frequent_sep)\n",
    "kmo_model_toy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "features_sep = np.zeros((tfidf_frequent_sep.shape[0], len(kmo_all_toy)))\n",
    "for i in range(len(kmo_all_toy)):\n",
    "    if kmo_all_toy[i] > 0.5:  #keeping only those that have kmo over 0.5\n",
    "        features_sep[i] = tfidf_frequent_sep[i]\n",
    "    \n",
    "print(len(features_sep), tfidf_frequent_sep.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Kmeans clustering. For each number of k model is created and fitted on above features (consisting of 36 manually chosen words). Number of texts assigned to each cluster is printed below. Then top words are presented and a tsne graph of them in 2d"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "random_state = 20\n",
    "k = 3\n",
    "\n",
    "model = KMeans(n_clusters=k, random_state=random_state)\n",
    "clusters = model.fit_predict(features_sep)\n",
    "# tsne = TSNE().fit_transform(features_sep)\n",
    "Counter(clusters)\n",
    "# max_items = np.random.choice(range(features_toy.shape[0]), size=10000, replace=False)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#FEBRUARY\n",
    "def get_top_keywords(data, clusters, labels, n_terms):\n",
    "    df = pd.DataFrame(data).groupby(clusters).mean()\n",
    "    \n",
    "    for i,r in df.iterrows():\n",
    "        print('\\nCluster {}'.format(i))\n",
    "        print(','.join([labels[t] for t in np.argsort(r)[-n_terms:]]))\n",
    "            \n",
    "get_top_keywords(features_toy, clusters, tokens_toy, 5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#SEPTEMBER\n",
    "get_top_keywords(features_sep, clusters, tokens_sep, 5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#kmeans and dbscan, 3 to 5 k\n",
    "\n",
    "def plot_tsne_pca(tsne, labels):\n",
    "    max_label = max(labels)\n",
    "\n",
    "    label_subset = [cm.hsv(i/max_label) for i in labels]\n",
    "    \n",
    "    plt.scatter(tsne[:, 0], tsne[:, 1], c=label_subset)\n",
    "    plt.title('TSNE Cluster Plot')\n",
    "    \n",
    "plot_tsne_pca(tsne[clusters!=0], clusters[clusters!=0])\n",
    "# plot_tsne_pca(tsne, clusters)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "DBSCAN."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "eps = 3\n",
    "min_samples = 3\n",
    "\n",
    "dbscan = {\n",
    "    'eps':[],\n",
    "    'min_samples':[],\n",
    "    'labels':[]\n",
    "}\n",
    "\n",
    "for eps in np.arange(0.01,0.05, 0.01):\n",
    "    for min_samples in range (3, 10, 1):\n",
    "        db1 = DBSCAN(eps=eps, min_samples=min_samples).fit(features_toy)\n",
    "        labels1 = db1.labels_\n",
    "        print(f\"eps: {eps}, min samples: {min_samples}\")\n",
    "        print(Counter(labels1))\n",
    "        dbscan['eps'].append(eps)\n",
    "        dbscan['min_samples'].append(min_samples)\n",
    "        dbscan['labels'].append(labels1)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "PCA. Number of components ranging from 3 to 5, printing explained variance ratio, factor loading matrix and significant terms for each component."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "scaler = StandardScaler()\n",
    "features_toy_scaled = scaler.fit_transform(features_sep)\n",
    "pca_model_toy = (CustomPCA(n_components=4)\n",
    "                    .fit(features_toy_scaled))\n",
    "print(pca_model_toy.explained_variance_ratio_)\n",
    "pca_model_toy.components_[1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "all_terms = []\n",
    "for i in range(4):\n",
    "    scores = [score for score in pca_model_toy.components_[i].round(2) if score>=0.2]\n",
    "    print(scores)\n",
    "    terms = bigrams_sep.token[pca_model_toy.components_[i].round(2)>=0.2]\n",
    "    all_terms.append(list(zip(terms, scores)))\n",
    "    \n",
    "all_terms"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "pca_model_toy.components_[3]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}