bitermplus_notebook-Re-Star.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "de6b21cb",
   "metadata": {},
   "source": [
    "BTM_bitermplus\n",
    "========\n",
    "\n",
    "Model fitting\n",
    "-------------\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "403c1017",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "f6e84915",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(17953, 8)\n",
      "Index(['ids', 'diagnosis', 'section', 'pilot_number', 'inter_time', 'response',\n",
      "       'Nresponse', 'LIWC_response'],\n",
      "      dtype='object')\n"
     ]
    }
   ],
   "source": [
    "from nltk.tokenize import word_tokenize\n",
    "from nltk.stem import PorterStemmer, WordNetLemmatizer\n",
    "\n",
    "# read the data that we prepared for analysis\n",
    "df = pd.read_csv('all_respones_for_all_interviewees_respones_LIWC_words_V2.csv')\n",
    "print(df.shape)\n",
    "print(df.columns)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "c48d810d",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[nltk_data] Downloading package averaged_perceptron_tagger to\n",
      "[nltk_data]     /Users/lilifang/nltk_data...\n",
      "[nltk_data]   Package averaged_perceptron_tagger is already up-to-\n",
      "[nltk_data]       date!\n",
      "[nltk_data] Downloading package wordnet to\n",
      "[nltk_data]     /Users/lilifang/nltk_data...\n",
      "[nltk_data]   Package wordnet is already up-to-date!\n"
     ]
    }
   ],
   "source": [
    "import nltk\n",
    "nltk.download('averaged_perceptron_tagger')\n",
    "nltk.download('wordnet')\n",
    "\n",
    "from nltk.stem import WordNetLemmatizer\n",
    "from nltk import pos_tag\n",
    "from nltk.corpus import wordnet\n",
    "\n",
    "def get_wordnet_pos(tag):\n",
    "    if tag.startswith('J'):\n",
    "        return wordnet.ADJ\n",
    "    elif tag.startswith('V'):\n",
    "        return wordnet.VERB\n",
    "    elif tag.startswith('N'):\n",
    "        return wordnet.NOUN\n",
    "    elif tag.startswith('R'):\n",
    "        return wordnet.ADV\n",
    "    else:\n",
    "        return wordnet.NOUN  # Default POS tag\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "282a8978",
   "metadata": {},
   "outputs": [],
   "source": [
    "import re\n",
    "from nltk.tokenize import word_tokenize\n",
    "# from nltk.stem.porter import PorterStemmer\n",
    "from nltk.corpus import stopwords\n",
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "import string\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "\n",
    "\n",
    "def clean_text2(sentence): \n",
    "    pattern = re.compile(r'[^a-z]+')\n",
    "    sentence = sentence.lower()\n",
    "    sentence = pattern.sub(' ', sentence).strip()\n",
    "    # Tokenize\n",
    "    word_list = word_tokenize(sentence)\n",
    "    # stop words\n",
    "    stopwords_list = stopwords.words('english') \n",
    "    new_stopwords = [\"um\", \"oh\", \"okay\", \"mhm\",\"gon\", \"na\",\"ah\",\"uh\",\"yes\",'yep',\"yeah\",\"no\",\"hm\",\"wow\",'.','?','-','--',':','mm','oop']\n",
    "    #new_stopwords = [\"um\", \"oh\", \"okay\", \"mhm\",\"ah\",\"uh\",\"yes\",\"get\",\"yeah\",\"no\",\"hm\",\"wow\",'.','?','-','--',':','mm']\n",
    "    stopwords_list.extend(new_stopwords)   \n",
    "    # puctuation\n",
    "    punct = set(string.punctuation)\n",
    "    # remove stop words\n",
    "    word_list = [word for word in word_list if word not in stopwords_list]\n",
    "    # remove very small words, length < 3\n",
    "    # they don't contribute any useful information\n",
    "    word_list = [word for word in word_list if len(word) >= 1]\n",
    "    # remove punctuation\n",
    "    word_list = [word for word in word_list if word not in punct] \n",
    "    \n",
    "    # Define the corpus of documents as a list of strings\n",
    "    lemmatizer = WordNetLemmatizer()\n",
    "    \n",
    "    pos_tags = pos_tag(word_list)\n",
    "\n",
    "    lemmas = []\n",
    "    for word, tag in pos_tags:\n",
    "        wordnet_tag = get_wordnet_pos(tag)\n",
    "        lemma = lemmatizer.lemmatize(word, pos=wordnet_tag)\n",
    "        lemmas.append(lemma)\n",
    "\n",
    "    lemmatized_sentence = \" \".join(lemmas)\n",
    "    #print(lemmatized_sentence) \n",
    "    return lemmatized_sentence\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "a4f2d19f",
   "metadata": {},
   "outputs": [],
   "source": [
    "df['response_lemar'] = df['response'].apply(lambda x: clean_text2(str(x)))\n",
    "# df['response_stem'] = df['response_stem'].apply(lambda x: ' '.join(x))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "7e6e266c",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "import pandas as pd\n",
    "\n",
    "# Define the parameters\n",
    "min_df = 0.0025\n",
    "max_df = 0.5\n",
    "# Create a CountVectorizer object\n",
    "vectorizer = CountVectorizer(stop_words='english', min_df=min_df, max_df=max_df)\n",
    "# Fit the vectorizer to the text data\n",
    "X = vectorizer.fit_transform(df['response_lemar'])\n",
    "# Get the feature names (i.e., the words)\n",
    "feature_names = vectorizer.get_feature_names_out()\n",
    "\n",
    "# Convert the sparse matrix to a dense matrix\n",
    "dense_matrix = X.toarray()\n",
    "\n",
    "# Create a new dataframe with the filtered words\n",
    "filtered_words_df = pd.DataFrame(data=dense_matrix, columns=feature_names)\n",
    "\n",
    "# Add the new column to the original dataframe\n",
    "df['filtered_text_0.0025'] = filtered_words_df.apply(lambda x: ' '.join([word for word, count in zip(feature_names, x) if count > 0]), axis=1)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "57a22c5b",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(13957, 11)\n"
     ]
    }
   ],
   "source": [
    "df['words_count'] = df['response'].str.split().str.len()\n",
    "df=df[df['words_count']>2]\n",
    "print(df.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "776c4df7",
   "metadata": {},
   "outputs": [],
   "source": [
    "# !pip install bitermplus\n",
    "import bitermplus as btm\n",
    "import numpy as np\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "60ef293e",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "6e3e2568",
   "metadata": {},
   "outputs": [],
   "source": [
    "old_texts = df['filtered_text_0.0025'].str.strip().tolist()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "b86c850a",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['like say', 'say', 'mum', '', 'cause forget like picture thought want']\n"
     ]
    }
   ],
   "source": [
    "print(old_texts[0:5])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "6812d7ba",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Index(['ids', 'diagnosis', 'section', 'pilot_number', 'inter_time', 'response',\n",
      "       'Nresponse', 'LIWC_response', 'words_count', 'response_lemar',\n",
      "       'filtered_text_0.0025'],\n",
      "      dtype='object')\n"
     ]
    }
   ],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "id": "6092db51",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of distinct words: 306\n"
     ]
    }
   ],
   "source": [
    "# Tokenize the text into words\n",
    "tokens = []\n",
    "\n",
    "for item in old_texts:\n",
    "    try:\n",
    "        new_i = item.split(' ')\n",
    "        # print(new_i)\n",
    "        for i in new_i:\n",
    "            tokens.append(i)\n",
    "    except:\n",
    "        pass\n",
    "\n",
    "# Create a set to store unique words\n",
    "unique_words = set()\n",
    "\n",
    "# Iterate through the tokens and add them to the set\n",
    "for token in tokens:\n",
    "    unique_words.add(token)\n",
    "\n",
    "# Count the unique words\n",
    "num_unique_words = len(unique_words)\n",
    "\n",
    "print(\"Number of distinct words:\", num_unique_words)\n",
    "\n",
    "# before lemmatize Number of distinct words: 8432  \n",
    "# after lemmatize, number of distinct words: 4071\n",
    "# filter lemmatize, number of distanct words: 0.01, 85\n",
    "# filtered_text_0.005: 174\n",
    "# filtered_text_0.0025: 306"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "id": "db7bedcf",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['like', 'say', 'say', 'mum', '']\n"
     ]
    }
   ],
   "source": [
    "print(tokens[0:5])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "id": "653db62c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>response</th>\n",
       "      <th>response_lemar</th>\n",
       "      <th>filtered_text_0.0025</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Just says like</td>\n",
       "      <td>say like</td>\n",
       "      <td>like say</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Okay. There's something you said. Something</td>\n",
       "      <td>something say something</td>\n",
       "      <td>say</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>Just my mum  so --</td>\n",
       "      <td>mum</td>\n",
       "      <td>mum</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>You can't be out here</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>With the picture  cause I had to rush cause I ...</td>\n",
       "      <td>picture cause rush cause forget like like thou...</td>\n",
       "      <td>cause forget like picture thought want</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>I do dance on Saturday  I do singing at home e...</td>\n",
       "      <td>dance saturday sing home every day do draw sin...</td>\n",
       "      <td>day draw home read</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>So I'm reading this book by an author called R...</td>\n",
       "      <td>read book author call rachel renee russell cal...</td>\n",
       "      <td>book like read</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>It's about this girl. It's in the form of a di...</td>\n",
       "      <td>girl form diary speaking trouble school good t...</td>\n",
       "      <td>able book finish good happen school thing trouble</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>What? The book or about me?</td>\n",
       "      <td>book</td>\n",
       "      <td>book</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>So acting childish  often struggling with a lo...</td>\n",
       "      <td>act childish often struggle lot thing think tw...</td>\n",
       "      <td>act lot struggle thing think</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                             response  \\\n",
       "3                                      Just says like   \n",
       "4         Okay. There's something you said. Something   \n",
       "7                                  Just my mum  so --   \n",
       "8                              You can't be out here    \n",
       "9   With the picture  cause I had to rush cause I ...   \n",
       "11  I do dance on Saturday  I do singing at home e...   \n",
       "12  So I'm reading this book by an author called R...   \n",
       "13  It's about this girl. It's in the form of a di...   \n",
       "15                        What? The book or about me?   \n",
       "16  So acting childish  often struggling with a lo...   \n",
       "\n",
       "                                       response_lemar  \\\n",
       "3                                            say like   \n",
       "4                             something say something   \n",
       "7                                                 mum   \n",
       "8                                                       \n",
       "9   picture cause rush cause forget like like thou...   \n",
       "11  dance saturday sing home every day do draw sin...   \n",
       "12  read book author call rachel renee russell cal...   \n",
       "13  girl form diary speaking trouble school good t...   \n",
       "15                                               book   \n",
       "16  act childish often struggle lot thing think tw...   \n",
       "\n",
       "                                 filtered_text_0.0025  \n",
       "3                                            like say  \n",
       "4                                                 say  \n",
       "7                                                 mum  \n",
       "8                                                      \n",
       "9              cause forget like picture thought want  \n",
       "11                                 day draw home read  \n",
       "12                                     book like read  \n",
       "13  able book finish good happen school thing trouble  \n",
       "15                                               book  \n",
       "16                       act lot struggle thing think  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "display(df[['response','response_lemar','filtered_text_0.0025']].head(10))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "id": "449dda5f",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(13957, 12)\n",
      "(13957, 12)\n"
     ]
    }
   ],
   "source": [
    "print(df.shape)\n",
    "df=df.dropna(subset=['filtered_text_0.0025'])\n",
    "print(df.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "id": "93b70958",
   "metadata": {},
   "outputs": [],
   "source": [
    "df['index'] = pd.DataFrame(range(df.shape[0]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "id": "93adb556",
   "metadata": {},
   "outputs": [],
   "source": [
    "texts = df['filtered_text_0.0025'].str.strip().tolist()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "a61c5e79",
   "metadata": {},
   "outputs": [],
   "source": [
    "# import nltk\n",
    "# nltk.download('punkt')\n",
    "# stopwords = nltk.corpus.stopwords.words('english')\n",
    "# new_stopwords = [\"um\", \"oh\", \"okay\", \"mhm\",\"ah\",\"uh\",\"yes\",\"get\",'yep',\"yeah\",\"no\",\"hm\",\"wow\",'.','?','-','--',':','mm','Mm','Oop']\n",
    "# stopwords.extend(new_stopwords)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "id": "5a98770e",
   "metadata": {},
   "outputs": [],
   "source": [
    "X, vocabulary, vocab_dict = btm.get_words_freqs(texts)\n",
    "docs_vec = btm.get_vectorized_docs(texts, vocabulary)\n",
    "biterms = btm.get_biterms(docs_vec)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "id": "c7bab5e4",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "13957"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "13957"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "10897"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "display(len(texts),len(docs_vec),len(biterms))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "id": "f53ba942",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████| 20/20 [00:00<00:00, 84.20it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "3\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████| 20/20 [00:00<00:00, 80.43it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████| 20/20 [00:00<00:00, 72.19it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "5\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████| 20/20 [00:00<00:00, 65.56it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "6\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████| 20/20 [00:00<00:00, 58.86it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "7\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████| 20/20 [00:00<00:00, 59.17it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "8\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████| 20/20 [00:00<00:00, 57.64it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "9\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████| 20/20 [00:00<00:00, 54.58it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "10\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████| 20/20 [00:00<00:00, 51.55it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "11\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████| 20/20 [00:00<00:00, 47.48it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "12\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████| 20/20 [00:00<00:00, 45.24it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "13\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████| 20/20 [00:00<00:00, 43.33it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "14\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████| 20/20 [00:00<00:00, 41.23it/s]\n"
     ]
    }
   ],
   "source": [
    "# finding best topic_num\n",
    "   \n",
    "metrics=[]\n",
    "for topic_num in range(2,15):\n",
    "    print(topic_num)\n",
    "    model = btm.BTM(X, vocabulary, seed=124, T=topic_num, M=20, alpha=50/10, beta=0.01)\n",
    "    model.fit(biterms, iterations=20)\n",
    "    # Get a phi matrix\n",
    "    #phi = tmp.get_phi(model)\n",
    "    #entropy = tmp.entropy(phi)\n",
    "    perplexity = model.perplexity_\n",
    "    coherence = model.coherence_\n",
    "    metrics.append([topic_num,perplexity,coherence])\n",
    "metrics=pd.DataFrame(metrics)\n",
    "metrics.columns=['Topic number', 'perplexity','coherence']\n",
    "metrics['coherence mean']=[np.mean(x) for x in metrics['coherence'].tolist()]\n",
    "  "
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8ff2b98f",
   "metadata": {},
   "source": [
    "##  Choosing the Best Coherence Score\n",
    "There is no one way to determine whether the coherence score is good or bad. The score and its value depend on the data that it’s calculated from. For instance, in one case, the score of 0.5 might be good enough but in another case not acceptable. The only rule is that we want to maximize this score.\n",
    "\n",
    "Usually, the coherence score will increase with the increase in the number of topics. This increase will become smaller as the number of topics gets higher. The trade-off between the number of topics and coherence score can be achieved using the so-called elbow technique. The method implies plotting coherence score as a function of the number of topics. We use the elbow of the curve to select the number of topics.\n",
    "\n",
    "The idea behind this method is that we want to choose a point after which the diminishing increase of coherence score is no longer worth the additional increase of the number of topics. The example of elbow cutoff at n\\_topics = 3 is shown below:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "id": "d2c36d77",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Topic number</th>\n",
       "      <th>perplexity</th>\n",
       "      <th>coherence</th>\n",
       "      <th>coherence mean</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2</td>\n",
       "      <td>inf</td>\n",
       "      <td>[-439.5016097180295, -438.78047975119284]</td>\n",
       "      <td>-439.141045</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>3</td>\n",
       "      <td>3.381857e+260</td>\n",
       "      <td>[-441.3578568481386, -473.63010635683526, -440...</td>\n",
       "      <td>-451.703858</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>4</td>\n",
       "      <td>3.869143e+273</td>\n",
       "      <td>[-453.7403830509875, -460.60543369677407, -443...</td>\n",
       "      <td>-450.630384</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>5</td>\n",
       "      <td>1.379481e+11</td>\n",
       "      <td>[-456.6900096787568, -472.4628861203783, -487....</td>\n",
       "      <td>-462.502802</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>6</td>\n",
       "      <td>1.058289e+63</td>\n",
       "      <td>[-467.4566694227514, -434.24939313945, -479.22...</td>\n",
       "      <td>-457.580639</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>7</td>\n",
       "      <td>5.023411e+100</td>\n",
       "      <td>[-473.12810145276944, -442.8799558432416, -490...</td>\n",
       "      <td>-467.542055</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>8</td>\n",
       "      <td>3.204137e+129</td>\n",
       "      <td>[-468.46545716516664, -462.57909828084223, -44...</td>\n",
       "      <td>-470.191901</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>9</td>\n",
       "      <td>6.125661e+149</td>\n",
       "      <td>[-444.401258455495, -469.1048875485441, -495.2...</td>\n",
       "      <td>-476.566894</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>10</td>\n",
       "      <td>1.000000e+00</td>\n",
       "      <td>[-467.06880940193355, -482.0960872775805, -459...</td>\n",
       "      <td>-477.596970</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>11</td>\n",
       "      <td>2.914873e+01</td>\n",
       "      <td>[-470.62499241095645, -462.9192613408833, -472...</td>\n",
       "      <td>-471.663902</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>12</td>\n",
       "      <td>2.666256e+01</td>\n",
       "      <td>[-456.17761417830053, -464.1825857958031, -451...</td>\n",
       "      <td>-479.642802</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>13</td>\n",
       "      <td>2.460780e+01</td>\n",
       "      <td>[-455.41491269226475, -471.54762845357493, -42...</td>\n",
       "      <td>-478.032793</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>14</td>\n",
       "      <td>2.271767e+01</td>\n",
       "      <td>[-464.1745559305036, -492.4175882400565, -490....</td>\n",
       "      <td>-480.495604</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    Topic number     perplexity  \\\n",
       "0              2            inf   \n",
       "1              3  3.381857e+260   \n",
       "2              4  3.869143e+273   \n",
       "3              5   1.379481e+11   \n",
       "4              6   1.058289e+63   \n",
       "5              7  5.023411e+100   \n",
       "6              8  3.204137e+129   \n",
       "7              9  6.125661e+149   \n",
       "8             10   1.000000e+00   \n",
       "9             11   2.914873e+01   \n",
       "10            12   2.666256e+01   \n",
       "11            13   2.460780e+01   \n",
       "12            14   2.271767e+01   \n",
       "\n",
       "                                            coherence  coherence mean  \n",
       "0           [-439.5016097180295, -438.78047975119284]     -439.141045  \n",
       "1   [-441.3578568481386, -473.63010635683526, -440...     -451.703858  \n",
       "2   [-453.7403830509875, -460.60543369677407, -443...     -450.630384  \n",
       "3   [-456.6900096787568, -472.4628861203783, -487....     -462.502802  \n",
       "4   [-467.4566694227514, -434.24939313945, -479.22...     -457.580639  \n",
       "5   [-473.12810145276944, -442.8799558432416, -490...     -467.542055  \n",
       "6   [-468.46545716516664, -462.57909828084223, -44...     -470.191901  \n",
       "7   [-444.401258455495, -469.1048875485441, -495.2...     -476.566894  \n",
       "8   [-467.06880940193355, -482.0960872775805, -459...     -477.596970  \n",
       "9   [-470.62499241095645, -462.9192613408833, -472...     -471.663902  \n",
       "10  [-456.17761417830053, -464.1825857958031, -451...     -479.642802  \n",
       "11  [-455.41491269226475, -471.54762845357493, -42...     -478.032793  \n",
       "12  [-464.1745559305036, -492.4175882400565, -490....     -480.495604  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "display(pd.DataFrame(metrics)) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "id": "4e581a3c",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████| 500/500 [00:09<00:00, 53.52it/s]\n",
      "/opt/anaconda3/lib/python3.8/site-packages/tmplot/_helpers.py:38: UserWarning: Please install \"tomotopy\" package to analyze its models.\n",
      "Run `pip install tomotopy` in the console.\n",
      "  warn(\n"
     ]
    }
   ],
   "source": [
    "import tmplot as tmp\n",
    "optimal_topic_number=10\n",
    "model = btm.BTM(X, vocabulary, seed=1234, T=optimal_topic_number, M=20)\n",
    "model.fit(biterms, iterations=500)\n",
    "# Get a phi matrix\n",
    "phi = tmp.get_phi(model)\n",
    "# entropy = tmp.entropy(phi)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "id": "c4796177",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████| 13957/13957 [00:00<00:00, 204106.19it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "117.15953029362926\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "p_zd = model.transform(docs_vec)\n",
    "perplexity = btm.perplexity(model.matrix_topics_words_, p_zd, X, 10)\n",
    "coherence = btm.coherence(model.matrix_topics_words_, X, M=20)\n",
    "print(perplexity)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "id": "62730f38",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Word</th>\n",
       "      <th>Index</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>like</td>\n",
       "      <td>146</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>say</td>\n",
       "      <td>229</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>mum</td>\n",
       "      <td>176</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>cause</td>\n",
       "      <td>38</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>forget</td>\n",
       "      <td>95</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>300</th>\n",
       "      <td>night</td>\n",
       "      <td>183</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>301</th>\n",
       "      <td>sense</td>\n",
       "      <td>234</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>302</th>\n",
       "      <td>couple</td>\n",
       "      <td>51</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>303</th>\n",
       "      <td>dont</td>\n",
       "      <td>66</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>304</th>\n",
       "      <td>im</td>\n",
       "      <td>131</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>305 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       Word  Index\n",
       "0      like    146\n",
       "1       say    229\n",
       "2       mum    176\n",
       "3     cause     38\n",
       "4    forget     95\n",
       "..      ...    ...\n",
       "300   night    183\n",
       "301   sense    234\n",
       "302  couple     51\n",
       "303    dont     66\n",
       "304      im    131\n",
       "\n",
       "[305 rows x 2 columns]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "vocab_dict_pd=pd.DataFrame(vocab_dict.items(), columns=['Word', 'Index'])\n",
    "display(vocab_dict_pd)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "id": "f2327a33",
   "metadata": {},
   "outputs": [],
   "source": [
    "import tmplot as tmp"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "id": "c58bf936",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[9 9 9 ... 5 5 4]\n"
     ]
    }
   ],
   "source": [
    "print(model.labels_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "id": "0f74b54a",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "13957\n"
     ]
    }
   ],
   "source": [
    "#Print the top 15 words for each topic\n",
    "print(len(model.labels_))\n",
    "labels = pd.DataFrame(model.labels_)\n",
    "new_id = pd.DataFrame(range(len(model.labels_)))\n",
    "df_with_label=pd.DataFrame()\n",
    "df_with_label['index']=new_id\n",
    "df_with_label['topic_label']=labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "id": "32227d4f",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(13957, 2)\n"
     ]
    }
   ],
   "source": [
    "print(df_with_label.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "id": "b3ae0387",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(13957, 12)\n"
     ]
    }
   ],
   "source": [
    "df['index']=range(0,len(df))\n",
    "print(df.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "id": "f232563d",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(13957, 13)\n"
     ]
    }
   ],
   "source": [
    "new_df_with_label = pd.merge(df,df_with_label,on='index',how='left')\n",
    "print(new_df_with_label.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "id": "6992074e",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0        like\n",
      "1       thing\n",
      "2      really\n",
      "3         lot\n",
      "4      people\n",
      "5        time\n",
      "6       quite\n",
      "7       think\n",
      "8        make\n",
      "9        calm\n",
      "10      place\n",
      "11       feel\n",
      "12    usually\n",
      "13        try\n",
      "14       help\n",
      "15      focus\n",
      "16        bit\n",
      "17      stuff\n",
      "18      noise\n",
      "19       sort\n",
      "Name: Terms, dtype: object\n",
      "0 like thing really lot people time quite think make calm place feel usually try help focus bit stuff noise sort\n",
      "0          like\n",
      "1     attention\n",
      "2           pay\n",
      "3        really\n",
      "4           try\n",
      "5          talk\n",
      "6      distract\n",
      "7         focus\n",
      "8         think\n",
      "9        lesson\n",
      "10          lot\n",
      "11       friend\n",
      "12         work\n",
      "13       people\n",
      "14          say\n",
      "15         know\n",
      "16         kind\n",
      "17      teacher\n",
      "18        thing\n",
      "19        quite\n",
      "Name: Terms, dtype: object\n",
      "1 like attention pay really try talk distract focus think lesson lot friend work people say know kind teacher thing quite\n",
      "0         like\n",
      "1         make\n",
      "2        thing\n",
      "3         feel\n",
      "4        think\n",
      "5         time\n",
      "6         know\n",
      "7          say\n",
      "8         play\n",
      "9         good\n",
      "10        want\n",
      "11        game\n",
      "12      really\n",
      "13        read\n",
      "14        look\n",
      "15    question\n",
      "16         end\n",
      "17         lot\n",
      "18      people\n",
      "19        book\n",
      "Name: Terms, dtype: object\n",
      "2 like make thing feel think time know say play good want game really read look question end lot people book\n",
      "0       like\n",
      "1     really\n",
      "2       play\n",
      "3       feel\n",
      "4      thing\n",
      "5        bit\n",
      "6       make\n",
      "7       time\n",
      "8       room\n",
      "9        mum\n",
      "10      want\n",
      "11    school\n",
      "12      come\n",
      "13      game\n",
      "14     happy\n",
      "15       try\n",
      "16    friend\n",
      "17       say\n",
      "18      home\n",
      "19     watch\n",
      "Name: Terms, dtype: object\n",
      "3 like really play feel thing bit make time room mum want school come game happy try friend say home watch\n",
      "0           like\n",
      "1            say\n",
      "2         really\n",
      "3           know\n",
      "4           feel\n",
      "5        teacher\n",
      "6          think\n",
      "7          thing\n",
      "8           make\n",
      "9           time\n",
      "10        people\n",
      "11          want\n",
      "12        school\n",
      "13    understand\n",
      "14         stuff\n",
      "15          tell\n",
      "16          work\n",
      "17           way\n",
      "18           try\n",
      "19           lot\n",
      "Name: Terms, dtype: object\n",
      "4 like say really know feel teacher think thing make time people want school understand stuff tell work way try lot\n",
      "0        like\n",
      "1        feel\n",
      "2      really\n",
      "3        make\n",
      "4       think\n",
      "5        know\n",
      "6        calm\n",
      "7       happy\n",
      "8       thing\n",
      "9         bit\n",
      "10      angry\n",
      "11       kind\n",
      "12       good\n",
      "13      quite\n",
      "14     people\n",
      "15       help\n",
      "16        try\n",
      "17        bad\n",
      "18      guess\n",
      "19    feeling\n",
      "Name: Terms, dtype: object\n",
      "5 like feel really make think know calm happy thing bit angry kind good quite people help try bad guess feeling\n",
      "0             like\n",
      "1           friend\n",
      "2           really\n",
      "3             know\n",
      "4           people\n",
      "5             talk\n",
      "6              say\n",
      "7             kind\n",
      "8            think\n",
      "9             want\n",
      "10    conversation\n",
      "11            feel\n",
      "12            good\n",
      "13          person\n",
      "14           group\n",
      "15         friends\n",
      "16           thing\n",
      "17             lot\n",
      "18            make\n",
      "19            time\n",
      "Name: Terms, dtype: object\n",
      "6 like friend really know people talk say kind think want conversation feel good person group friends thing lot make time\n",
      "0         like\n",
      "1         time\n",
      "2          day\n",
      "3       really\n",
      "4        think\n",
      "5       school\n",
      "6         know\n",
      "7       happen\n",
      "8       minute\n",
      "9        maybe\n",
      "10       quite\n",
      "11        long\n",
      "12         say\n",
      "13         lot\n",
      "14        kind\n",
      "15         bit\n",
      "16       thing\n",
      "17    probably\n",
      "18        home\n",
      "19         end\n",
      "Name: Terms, dtype: object\n",
      "7 like time day really think school know happen minute maybe quite long say lot kind bit thing probably home end\n",
      "0         like\n",
      "1         know\n",
      "2       really\n",
      "3         feel\n",
      "4        thing\n",
      "5          bit\n",
      "6        think\n",
      "7         want\n",
      "8          say\n",
      "9         make\n",
      "10         mum\n",
      "11         try\n",
      "12      option\n",
      "13      stress\n",
      "14       quite\n",
      "15        pick\n",
      "16        time\n",
      "17        kind\n",
      "18        tell\n",
      "19    probably\n",
      "Name: Terms, dtype: object\n",
      "8 like know really feel thing bit think want say make mum try option stress quite pick time kind tell probably\n",
      "0       like\n",
      "1        say\n",
      "2     really\n",
      "3       know\n",
      "4      think\n",
      "5       feel\n",
      "6      start\n",
      "7     people\n",
      "8       talk\n",
      "9        bit\n",
      "10      tell\n",
      "11       try\n",
      "12     thing\n",
      "13      kind\n",
      "14      want\n",
      "15     angry\n",
      "16      time\n",
      "17      calm\n",
      "18      make\n",
      "19       ask\n",
      "Name: Terms, dtype: object\n",
      "9 like say really know think feel start people talk bit tell try thing kind want angry time calm make ask\n"
     ]
    }
   ],
   "source": [
    "nf =open('topic_words.csv','wt')\n",
    "nf.write('topic,reprensentative_words'+'\\n')\n",
    "for i in range(10):\n",
    "    terms_probs = tmp.calc_terms_probs_ratio(phi, topic=i, lambda_=1)\n",
    "    print( terms_probs['Terms'][0:20])\n",
    "    words = ' '.join(row for row in terms_probs['Terms'][0:20])\n",
    "    print(i,words)\n",
    "    nf.write(str(i)+','+str(words)+'\\n')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "id": "6a00655f",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(10, 2)\n",
      "Index(['topic', 'reprensentative_words'], dtype='object')\n",
      "Index(['topic_label', 'reprensentative_words'], dtype='object')\n"
     ]
    }
   ],
   "source": [
    "topic_words_df = pd.read_csv('topic_words.csv')\n",
    "print(topic_words_df.shape)\n",
    "print(topic_words_df.columns)\n",
    "topic_words_df. rename(columns = {'topic':'topic_label'}, inplace = True)\n",
    "print(topic_words_df.columns)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 164,
   "id": "14da7299",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ids</th>\n",
       "      <th>diagnosis</th>\n",
       "      <th>section</th>\n",
       "      <th>pilot_number</th>\n",
       "      <th>inter_time</th>\n",
       "      <th>response</th>\n",
       "      <th>Nresponse</th>\n",
       "      <th>LIWC_response</th>\n",
       "      <th>response_lemar</th>\n",
       "      <th>filtered_text_0.0025</th>\n",
       "      <th>words_count</th>\n",
       "      <th>index</th>\n",
       "      <th>topic_label</th>\n",
       "      <th>reprensentative_words</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>3</td>\n",
       "      <td>ASD.ADHD</td>\n",
       "      <td>Activity Three</td>\n",
       "      <td>974_16_06_2022</td>\n",
       "      <td>[00:01:08]</td>\n",
       "      <td>Just says like</td>\n",
       "      <td>True</td>\n",
       "      <td>says</td>\n",
       "      <td>say like</td>\n",
       "      <td>like say</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>9</td>\n",
       "      <td>like say really know think feel start people t...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>4</td>\n",
       "      <td>ASD.ADHD</td>\n",
       "      <td>Activity Three</td>\n",
       "      <td>974_16_06_2022</td>\n",
       "      <td>[00:01:18]</td>\n",
       "      <td>Okay. There's something you said. Something</td>\n",
       "      <td>True</td>\n",
       "      <td>something you</td>\n",
       "      <td>something say something</td>\n",
       "      <td>say</td>\n",
       "      <td>6</td>\n",
       "      <td>1</td>\n",
       "      <td>9</td>\n",
       "      <td>like say really know think feel start people t...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>7</td>\n",
       "      <td>ASD.ADHD</td>\n",
       "      <td>Icebreaker</td>\n",
       "      <td>974_16_06_2022</td>\n",
       "      <td>[00:03:16]</td>\n",
       "      <td>Just my mum  so --</td>\n",
       "      <td>True</td>\n",
       "      <td>mum</td>\n",
       "      <td>mum</td>\n",
       "      <td>mum</td>\n",
       "      <td>5</td>\n",
       "      <td>2</td>\n",
       "      <td>9</td>\n",
       "      <td>like say really know think feel start people t...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>8</td>\n",
       "      <td>ASD.ADHD</td>\n",
       "      <td>Icebreaker</td>\n",
       "      <td>974_16_06_2022</td>\n",
       "      <td>[00:03:53]</td>\n",
       "      <td>You can't be out here</td>\n",
       "      <td>True</td>\n",
       "      <td>can't</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>5</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>like thing really lot people time quite think ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>9</td>\n",
       "      <td>ASD.ADHD</td>\n",
       "      <td>Icebreaker</td>\n",
       "      <td>974_16_06_2022</td>\n",
       "      <td>[00:04:16]</td>\n",
       "      <td>With the picture  cause I had to rush cause I ...</td>\n",
       "      <td>True</td>\n",
       "      <td>picture didn't thought but couldn't wanted</td>\n",
       "      <td>picture cause rush cause forget like like thou...</td>\n",
       "      <td>cause forget like picture thought want</td>\n",
       "      <td>38</td>\n",
       "      <td>4</td>\n",
       "      <td>9</td>\n",
       "      <td>like say really know think feel start people t...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   ids diagnosis         section    pilot_number  inter_time  \\\n",
       "0    3  ASD.ADHD  Activity Three  974_16_06_2022  [00:01:08]   \n",
       "1    4  ASD.ADHD  Activity Three  974_16_06_2022  [00:01:18]   \n",
       "2    7  ASD.ADHD      Icebreaker  974_16_06_2022  [00:03:16]   \n",
       "3    8  ASD.ADHD      Icebreaker  974_16_06_2022  [00:03:53]   \n",
       "4    9  ASD.ADHD      Icebreaker  974_16_06_2022  [00:04:16]   \n",
       "\n",
       "                                            response  Nresponse  \\\n",
       "0                                     Just says like       True   \n",
       "1        Okay. There's something you said. Something       True   \n",
       "2                                 Just my mum  so --       True   \n",
       "3                             You can't be out here        True   \n",
       "4  With the picture  cause I had to rush cause I ...       True   \n",
       "\n",
       "                                LIWC_response  \\\n",
       "0                                        says   \n",
       "1                               something you   \n",
       "2                                         mum   \n",
       "3                                       can't   \n",
       "4  picture didn't thought but couldn't wanted   \n",
       "\n",
       "                                      response_lemar  \\\n",
       "0                                           say like   \n",
       "1                            something say something   \n",
       "2                                                mum   \n",
       "3                                                      \n",
       "4  picture cause rush cause forget like like thou...   \n",
       "\n",
       "                     filtered_text_0.0025  words_count  index  topic_label  \\\n",
       "0                                like say            3      0            9   \n",
       "1                                     say            6      1            9   \n",
       "2                                     mum            5      2            9   \n",
       "3                                                    5      3            0   \n",
       "4  cause forget like picture thought want           38      4            9   \n",
       "\n",
       "                               reprensentative_words  \n",
       "0  like say really know think feel start people t...  \n",
       "1  like say really know think feel start people t...  \n",
       "2  like say really know think feel start people t...  \n",
       "3  like thing really lot people time quite think ...  \n",
       "4  like say really know think feel start people t...  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "(13957, 14)"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "df_with_label_topicwords = pd.merge(new_df_with_label,topic_words_df,on='topic_label',how='left')\n",
    "display(df_with_label_topicwords.head())\n",
    "display(df_with_label_topicwords.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "id": "2f1d4351",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 165,
   "id": "032a9451",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_with_label_topicwords['new_words_count'] = df_with_label_topicwords['filtered_text_0.0025'].str.split().str.len()\n",
    "df_with_label_topicwords=df_with_label_topicwords[df_with_label_topicwords['new_words_count']>0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 166,
   "id": "69cb62c6",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(13208, 15)\n"
     ]
    }
   ],
   "source": [
    "print(df_with_label_topicwords.shape)\n",
    "df_with_label_topicwords.to_csv('filtered_text_final_0.0025.csv',index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "id": "eca52347",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>topic_label</th>\n",
       "      <th>ids</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>950</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>260</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>460</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>666</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>2908</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>5</td>\n",
       "      <td>1790</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>6</td>\n",
       "      <td>655</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>7</td>\n",
       "      <td>1201</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>8</td>\n",
       "      <td>487</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>9</td>\n",
       "      <td>4580</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   topic_label   ids\n",
       "0            0   950\n",
       "1            1   260\n",
       "2            2   460\n",
       "3            3   666\n",
       "4            4  2908\n",
       "5            5  1790\n",
       "6            6   655\n",
       "7            7  1201\n",
       "8            8   487\n",
       "9            9  4580"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "new_count_df= df_with_label_topicwords.groupby('topic_label',as_index=False).count()\n",
    "display(new_count_df[['topic_label','ids']])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "id": "8615bf45",
   "metadata": {},
   "outputs": [],
   "source": [
    "from typing import List, Union, Tuple, Dict, Sequence, Any\n",
    "def get_biterms_new(\n",
    "        docs: List[np.ndarray],\n",
    "        win: int = 15) -> List[List[int]]:\n",
    "    biterms = []\n",
    "    doc_index=0\n",
    "    for doc in docs:\n",
    "        doc_biterms = []\n",
    "        doc_len = len(doc)\n",
    "        for i in range(doc_len-1):\n",
    "            for j in range(i+1, min(i + win, doc_len)):\n",
    "                wi = min(doc[i], doc[j])\n",
    "                wj = max(doc[i], doc[j])\n",
    "                doc_biterms.append([wi, wj])\n",
    "        biterms.append([doc_index,doc_biterms])\n",
    "        doc_index=doc_index+1\n",
    "    return biterms\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "id": "19565ea0",
   "metadata": {},
   "outputs": [],
   "source": [
    "biterms_new = get_biterms_new(docs_vec)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 173,
   "id": "d6396f5e",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>index</th>\n",
       "      <th>biterms in doc</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>[[146, 229]]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>[]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>[]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>[]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>[[38, 95], [38, 146], [38, 200], [38, 270], [3...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13952</th>\n",
       "      <td>13952</td>\n",
       "      <td>[]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13953</th>\n",
       "      <td>13953</td>\n",
       "      <td>[]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13954</th>\n",
       "      <td>13954</td>\n",
       "      <td>[]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13955</th>\n",
       "      <td>13955</td>\n",
       "      <td>[[85, 218]]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13956</th>\n",
       "      <td>13956</td>\n",
       "      <td>[[38, 85], [38, 255], [85, 255]]</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>13957 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       index                                     biterms in doc\n",
       "0          0                                       [[146, 229]]\n",
       "1          1                                                 []\n",
       "2          2                                                 []\n",
       "3          3                                                 []\n",
       "4          4  [[38, 95], [38, 146], [38, 200], [38, 270], [3...\n",
       "...      ...                                                ...\n",
       "13952  13952                                                 []\n",
       "13953  13953                                                 []\n",
       "13954  13954                                                 []\n",
       "13955  13955                                        [[85, 218]]\n",
       "13956  13956                   [[38, 85], [38, 255], [85, 255]]\n",
       "\n",
       "[13957 rows x 2 columns]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "biterms_new=pd.DataFrame(biterms_new)\n",
    "biterms_new.columns=['index','biterms in doc']\n",
    "display(biterms_new)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 174,
   "id": "fdea1de4",
   "metadata": {},
   "outputs": [],
   "source": [
    "label_number = 1\n",
    "doc_id_in_topic=df_with_label_topicwords['index'][df_with_label_topicwords['topic_label']==label_number].values.tolist()\n",
    "# doc_id_in_topic"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 175,
   "id": "d580ff99",
   "metadata": {},
   "outputs": [],
   "source": [
    "BitermDataDoc=biterms_new['biterms in doc'][biterms_new['index'].isin(doc_id_in_topic)].values.tolist()\n",
    "# BitermDataDoc[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 176,
   "id": "1ff7f960",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>O</th>\n",
       "      <th>D</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>17</td>\n",
       "      <td>195</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>17</td>\n",
       "      <td>263</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>17</td>\n",
       "      <td>287</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    O    D\n",
       "0  17  195\n",
       "1  17  263\n",
       "2  17  287"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "BitermsSetDoc=[]\n",
    "for row in BitermDataDoc:\n",
    "    BitermsSetDoc=BitermsSetDoc+row\n",
    "BitermsSetDoc=pd.DataFrame(BitermsSetDoc)   \n",
    "BitermsSetDoc.columns=['O','D']\n",
    "BitermsSetDoc=BitermsSetDoc[BitermsSetDoc['O']!=BitermsSetDoc['D']]\n",
    "display(BitermsSetDoc.head(3))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 177,
   "id": "81b0b202",
   "metadata": {},
   "outputs": [],
   "source": [
    "BitermsSetDoc_count= BitermsSetDoc[['O','D']].value_counts().reset_index()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 178,
   "id": "5619f082",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>O</th>\n",
       "      <th>D</th>\n",
       "      <th>0</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>17</td>\n",
       "      <td>195</td>\n",
       "      <td>103</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>146</td>\n",
       "      <td>195</td>\n",
       "      <td>41</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>17</td>\n",
       "      <td>146</td>\n",
       "      <td>41</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     O    D    0\n",
       "0   17  195  103\n",
       "1  146  195   41\n",
       "2   17  146   41"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "display(BitermsSetDoc_count.head(3))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 179,
   "id": "ba0d4788",
   "metadata": {},
   "outputs": [],
   "source": [
    "vocab_dict_df=pd.DataFrame(vocab_dict.items(), columns=['O word', 'O'])\n",
    "vocab_dict_df['D']=vocab_dict_df['O']\n",
    "vocab_dict_df['D word']=vocab_dict_df['O word']\n",
    "BitermsSetDoc_count=pd.merge(BitermsSetDoc_count, vocab_dict_df[['O word', 'O']], how='left', on='O')\n",
    "BitermsSetDoc_count=pd.merge(BitermsSetDoc_count, vocab_dict_df[['D word', 'D']], how='left', on='D')\n",
    "# display(BitermsSetDoc_count)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 180,
   "id": "70369232",
   "metadata": {},
   "outputs": [],
   "source": [
    "BitermsSetDoc_count['Edge weight']=BitermsSetDoc_count[0]\n",
    "top_BitermsSetDoc_count=BitermsSetDoc_count.sort_values(by='Edge weight',ascending=False).head(1000)\n",
    "# top_BitermsSetDoc_count"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 181,
   "id": "049a6f51",
   "metadata": {},
   "outputs": [],
   "source": [
    "import networkx as nx\n",
    "def plotNetwork(df):\n",
    "    G = nx.DiGraph()\n",
    "    for index, row in df.iterrows():\n",
    "        G.add_edge(row['O word'], row['D word'], weight=row['Edge weight'])  \n",
    "    return G"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 182,
   "id": "7184a5bc",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(1000, 3)\n"
     ]
    }
   ],
   "source": [
    "color_lis = ['lightblue','pink','lightskyblue','darkseagreen','gold',\n",
    "             'bisque','tomato','yellowgreen','royalblue','plum']\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "from matplotlib.gridspec import GridSpec\n",
    "path1= './BTM-biterms-filtered-data-summary/filter_text_final_0.0025/'\n",
    "# Create a new figure and subplots\n",
    "Net_data =top_BitermsSetDoc_count[[\"O word\",\"D word\",\"Edge weight\"]]\n",
    "Net_data_new=Net_data\n",
    "print(Net_data_new.shape)\n",
    "G0 = plotNetwork(Net_data_new)\n",
    "#nx.write_gml(G0, 'topic2.gml')\n",
    "weights = dict(G0.degree(weight='weight'))\n",
    "# Sort the dictionary by weights in descending order\n",
    "sorted_weights = {k: v for k, v in sorted(weights.items(), key=lambda item: item[1], reverse=True)}\n",
    "# Keep only the top 20 words (nodes) with largest weights\n",
    "top_words = list(sorted_weights.keys())[0:50]\n",
    "# Create a new network with only the top 20 words and their corresponding edges\n",
    "#plt.savefig(\"Graph_topic_\"+str(i)+\".png\", format=\"PNG\")\n",
    "new_G = G0.subgraph(top_words)\n",
    "nx.write_gml(new_G, path1+str(label_number)+'_for_Re-star.gml')\n",
    "\n",
    "top_BitermsSetDoc_count.to_csv(path1+'filter_text_final_0.0025_biterms_summary'+str(label_number)+'.csv',index=False)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}