bow-with-preprocessing-and-advanced-features.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 53,
   "id": "facc992d",
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "\n",
    "import re\n",
    "from bs4 import BeautifulSoup\n",
    "\n",
    "import warnings\n",
    "warnings.filterwarnings('ignore')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "id": "274f7263",
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.read_csv('train.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "id": "2f4033b3",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(404290, 6)"
      ]
     },
     "execution_count": 55,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "id": "77385d6f",
   "metadata": {},
   "outputs": [],
   "source": [
    "# new_df = df.sample(30000)\n",
    "# new_df = df.sample(300)\n",
    "new_df = df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "id": "146800f5",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>qid1</th>\n",
       "      <th>qid2</th>\n",
       "      <th>question1</th>\n",
       "      <th>question2</th>\n",
       "      <th>is_duplicate</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>356118</th>\n",
       "      <td>356118</td>\n",
       "      <td>485393</td>\n",
       "      <td>485394</td>\n",
       "      <td>What is the problems of trade manufactured goo...</td>\n",
       "      <td>Which voting groups did Hilary do better than ...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>309345</th>\n",
       "      <td>309345</td>\n",
       "      <td>433321</td>\n",
       "      <td>30782</td>\n",
       "      <td>Do spammers make money?</td>\n",
       "      <td>How can you look at someone's private Instagra...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>213099</th>\n",
       "      <td>213099</td>\n",
       "      <td>318345</td>\n",
       "      <td>318346</td>\n",
       "      <td>Why do corrupt and evil people flourish while ...</td>\n",
       "      <td>Is Karma real? If so, then why do we see bad p...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>122902</th>\n",
       "      <td>122902</td>\n",
       "      <td>198822</td>\n",
       "      <td>198823</td>\n",
       "      <td>Where was the wheel invented?</td>\n",
       "      <td>When was the wheel invented?</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>205461</th>\n",
       "      <td>205461</td>\n",
       "      <td>308611</td>\n",
       "      <td>308612</td>\n",
       "      <td>Is it too late to learn playing drums at the a...</td>\n",
       "      <td>Is it too late to learn drums at age 34?</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "            id    qid1    qid2  \\\n",
       "356118  356118  485393  485394   \n",
       "309345  309345  433321   30782   \n",
       "213099  213099  318345  318346   \n",
       "122902  122902  198822  198823   \n",
       "205461  205461  308611  308612   \n",
       "\n",
       "                                                question1  \\\n",
       "356118  What is the problems of trade manufactured goo...   \n",
       "309345                            Do spammers make money?   \n",
       "213099  Why do corrupt and evil people flourish while ...   \n",
       "122902                      Where was the wheel invented?   \n",
       "205461  Is it too late to learn playing drums at the a...   \n",
       "\n",
       "                                                question2  is_duplicate  \n",
       "356118  Which voting groups did Hilary do better than ...             0  \n",
       "309345  How can you look at someone's private Instagra...             0  \n",
       "213099  Is Karma real? If so, then why do we see bad p...             1  \n",
       "122902                       When was the wheel invented?             0  \n",
       "205461           Is it too late to learn drums at age 34?             0  "
      ]
     },
     "execution_count": 57,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "new_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "id": "f02d5248",
   "metadata": {},
   "outputs": [],
   "source": [
    "def preprocess(q):\n",
    "    \n",
    "    q = str(q).lower().strip()\n",
    "    \n",
    "    # Replace certain special characters with their string equivalents\n",
    "    q = q.replace('%', ' percent')\n",
    "    q = q.replace('$', ' dollar ')\n",
    "    q = q.replace('₹', ' rupee ')\n",
    "    q = q.replace('€', ' euro ')\n",
    "    q = q.replace('@', ' at ')\n",
    "    \n",
    "    # The pattern '[math]' appears around 900 times in the whole dataset.\n",
    "    q = q.replace('[math]', '')\n",
    "    \n",
    "    # Replacing some numbers with string equivalents (not perfect, can be done better to account for more cases)\n",
    "    q = q.replace(',000,000,000 ', 'b ')\n",
    "    q = q.replace(',000,000 ', 'm ')\n",
    "    q = q.replace(',000 ', 'k ')\n",
    "    q = re.sub(r'([0-9]+)000000000', r'\\1b', q)\n",
    "    q = re.sub(r'([0-9]+)000000', r'\\1m', q)\n",
    "    q = re.sub(r'([0-9]+)000', r'\\1k', q)\n",
    "    \n",
    "    # Decontracting words\n",
    "    # https://en.wikipedia.org/wiki/Wikipedia%3aList_of_English_contractions\n",
    "    # https://stackoverflow.com/a/19794953\n",
    "    contractions = { \n",
    "    \"ain't\": \"am not\",\n",
    "    \"aren't\": \"are not\",\n",
    "    \"can't\": \"can not\",\n",
    "    \"can't've\": \"can not have\",\n",
    "    \"'cause\": \"because\",\n",
    "    \"could've\": \"could have\",\n",
    "    \"couldn't\": \"could not\",\n",
    "    \"couldn't've\": \"could not have\",\n",
    "    \"didn't\": \"did not\",\n",
    "    \"doesn't\": \"does not\",\n",
    "    \"don't\": \"do not\",\n",
    "    \"hadn't\": \"had not\",\n",
    "    \"hadn't've\": \"had not have\",\n",
    "    \"hasn't\": \"has not\",\n",
    "    \"haven't\": \"have not\",\n",
    "    \"he'd\": \"he would\",\n",
    "    \"he'd've\": \"he would have\",\n",
    "    \"he'll\": \"he will\",\n",
    "    \"he'll've\": \"he will have\",\n",
    "    \"he's\": \"he is\",\n",
    "    \"how'd\": \"how did\",\n",
    "    \"how'd'y\": \"how do you\",\n",
    "    \"how'll\": \"how will\",\n",
    "    \"how's\": \"how is\",\n",
    "    \"i'd\": \"i would\",\n",
    "    \"i'd've\": \"i would have\",\n",
    "    \"i'll\": \"i will\",\n",
    "    \"i'll've\": \"i will have\",\n",
    "    \"i'm\": \"i am\",\n",
    "    \"i've\": \"i have\",\n",
    "    \"isn't\": \"is not\",\n",
    "    \"it'd\": \"it would\",\n",
    "    \"it'd've\": \"it would have\",\n",
    "    \"it'll\": \"it will\",\n",
    "    \"it'll've\": \"it will have\",\n",
    "    \"it's\": \"it is\",\n",
    "    \"let's\": \"let us\",\n",
    "    \"ma'am\": \"madam\",\n",
    "    \"mayn't\": \"may not\",\n",
    "    \"might've\": \"might have\",\n",
    "    \"mightn't\": \"might not\",\n",
    "    \"mightn't've\": \"might not have\",\n",
    "    \"must've\": \"must have\",\n",
    "    \"mustn't\": \"must not\",\n",
    "    \"mustn't've\": \"must not have\",\n",
    "    \"needn't\": \"need not\",\n",
    "    \"needn't've\": \"need not have\",\n",
    "    \"o'clock\": \"of the clock\",\n",
    "    \"oughtn't\": \"ought not\",\n",
    "    \"oughtn't've\": \"ought not have\",\n",
    "    \"shan't\": \"shall not\",\n",
    "    \"sha'n't\": \"shall not\",\n",
    "    \"shan't've\": \"shall not have\",\n",
    "    \"she'd\": \"she would\",\n",
    "    \"she'd've\": \"she would have\",\n",
    "    \"she'll\": \"she will\",\n",
    "    \"she'll've\": \"she will have\",\n",
    "    \"she's\": \"she is\",\n",
    "    \"should've\": \"should have\",\n",
    "    \"shouldn't\": \"should not\",\n",
    "    \"shouldn't've\": \"should not have\",\n",
    "    \"so've\": \"so have\",\n",
    "    \"so's\": \"so as\",\n",
    "    \"that'd\": \"that would\",\n",
    "    \"that'd've\": \"that would have\",\n",
    "    \"that's\": \"that is\",\n",
    "    \"there'd\": \"there would\",\n",
    "    \"there'd've\": \"there would have\",\n",
    "    \"there's\": \"there is\",\n",
    "    \"they'd\": \"they would\",\n",
    "    \"they'd've\": \"they would have\",\n",
    "    \"they'll\": \"they will\",\n",
    "    \"they'll've\": \"they will have\",\n",
    "    \"they're\": \"they are\",\n",
    "    \"they've\": \"they have\",\n",
    "    \"to've\": \"to have\",\n",
    "    \"wasn't\": \"was not\",\n",
    "    \"we'd\": \"we would\",\n",
    "    \"we'd've\": \"we would have\",\n",
    "    \"we'll\": \"we will\",\n",
    "    \"we'll've\": \"we will have\",\n",
    "    \"we're\": \"we are\",\n",
    "    \"we've\": \"we have\",\n",
    "    \"weren't\": \"were not\",\n",
    "    \"what'll\": \"what will\",\n",
    "    \"what'll've\": \"what will have\",\n",
    "    \"what're\": \"what are\",\n",
    "    \"what's\": \"what is\",\n",
    "    \"what've\": \"what have\",\n",
    "    \"when's\": \"when is\",\n",
    "    \"when've\": \"when have\",\n",
    "    \"where'd\": \"where did\",\n",
    "    \"where's\": \"where is\",\n",
    "    \"where've\": \"where have\",\n",
    "    \"who'll\": \"who will\",\n",
    "    \"who'll've\": \"who will have\",\n",
    "    \"who's\": \"who is\",\n",
    "    \"who've\": \"who have\",\n",
    "    \"why's\": \"why is\",\n",
    "    \"why've\": \"why have\",\n",
    "    \"will've\": \"will have\",\n",
    "    \"won't\": \"will not\",\n",
    "    \"won't've\": \"will not have\",\n",
    "    \"would've\": \"would have\",\n",
    "    \"wouldn't\": \"would not\",\n",
    "    \"wouldn't've\": \"would not have\",\n",
    "    \"y'all\": \"you all\",\n",
    "    \"y'all'd\": \"you all would\",\n",
    "    \"y'all'd've\": \"you all would have\",\n",
    "    \"y'all're\": \"you all are\",\n",
    "    \"y'all've\": \"you all have\",\n",
    "    \"you'd\": \"you would\",\n",
    "    \"you'd've\": \"you would have\",\n",
    "    \"you'll\": \"you will\",\n",
    "    \"you'll've\": \"you will have\",\n",
    "    \"you're\": \"you are\",\n",
    "    \"you've\": \"you have\"\n",
    "    }\n",
    "\n",
    "    q_decontracted = []\n",
    "\n",
    "    for word in q.split():\n",
    "        if word in contractions:\n",
    "            word = contractions[word]\n",
    "\n",
    "        q_decontracted.append(word)\n",
    "\n",
    "    q = ' '.join(q_decontracted)\n",
    "    q = q.replace(\"'ve\", \" have\")\n",
    "    q = q.replace(\"n't\", \" not\")\n",
    "    q = q.replace(\"'re\", \" are\")\n",
    "    q = q.replace(\"'ll\", \" will\")\n",
    "    \n",
    "    # Removing HTML tags\n",
    "    q = BeautifulSoup(q)\n",
    "    q = q.get_text()\n",
    "    \n",
    "    # Remove punctuations\n",
    "    pattern = re.compile('\\W')\n",
    "    q = re.sub(pattern, ' ', q).strip()\n",
    "\n",
    "    \n",
    "    return q\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "id": "50e651e1",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'i have already  was not done'"
      ]
     },
     "execution_count": 59,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "preprocess(\"I've already! wasn't <b>done</b>?\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "id": "882f5f41",
   "metadata": {},
   "outputs": [],
   "source": [
    "new_df['question1'] = new_df['question1'].apply(preprocess)\n",
    "new_df['question2'] = new_df['question2'].apply(preprocess)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "id": "18d190d8",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>qid1</th>\n",
       "      <th>qid2</th>\n",
       "      <th>question1</th>\n",
       "      <th>question2</th>\n",
       "      <th>is_duplicate</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>356118</th>\n",
       "      <td>356118</td>\n",
       "      <td>485393</td>\n",
       "      <td>485394</td>\n",
       "      <td>what is the problems of trade manufactured goo...</td>\n",
       "      <td>which voting groups did hilary do better than ...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>309345</th>\n",
       "      <td>309345</td>\n",
       "      <td>433321</td>\n",
       "      <td>30782</td>\n",
       "      <td>do spammers make money</td>\n",
       "      <td>how can you look at someone s private instagra...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>213099</th>\n",
       "      <td>213099</td>\n",
       "      <td>318345</td>\n",
       "      <td>318346</td>\n",
       "      <td>why do corrupt and evil people flourish while ...</td>\n",
       "      <td>is karma real  if so  then why do we see bad p...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>122902</th>\n",
       "      <td>122902</td>\n",
       "      <td>198822</td>\n",
       "      <td>198823</td>\n",
       "      <td>where was the wheel invented</td>\n",
       "      <td>when was the wheel invented</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>205461</th>\n",
       "      <td>205461</td>\n",
       "      <td>308611</td>\n",
       "      <td>308612</td>\n",
       "      <td>is it too late to learn playing drums at the a...</td>\n",
       "      <td>is it too late to learn drums at age 34</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "            id    qid1    qid2  \\\n",
       "356118  356118  485393  485394   \n",
       "309345  309345  433321   30782   \n",
       "213099  213099  318345  318346   \n",
       "122902  122902  198822  198823   \n",
       "205461  205461  308611  308612   \n",
       "\n",
       "                                                question1  \\\n",
       "356118  what is the problems of trade manufactured goo...   \n",
       "309345                             do spammers make money   \n",
       "213099  why do corrupt and evil people flourish while ...   \n",
       "122902                       where was the wheel invented   \n",
       "205461  is it too late to learn playing drums at the a...   \n",
       "\n",
       "                                                question2  is_duplicate  \n",
       "356118  which voting groups did hilary do better than ...             0  \n",
       "309345  how can you look at someone s private instagra...             0  \n",
       "213099  is karma real  if so  then why do we see bad p...             1  \n",
       "122902                        when was the wheel invented             0  \n",
       "205461            is it too late to learn drums at age 34             0  "
      ]
     },
     "execution_count": 61,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "new_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "id": "46941372",
   "metadata": {},
   "outputs": [],
   "source": [
    "new_df['q1_len'] = new_df['question1'].str.len() \n",
    "new_df['q2_len'] = new_df['question2'].str.len()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "id": "4d901ff6",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>qid1</th>\n",
       "      <th>qid2</th>\n",
       "      <th>question1</th>\n",
       "      <th>question2</th>\n",
       "      <th>is_duplicate</th>\n",
       "      <th>q1_len</th>\n",
       "      <th>q2_len</th>\n",
       "      <th>q1_num_words</th>\n",
       "      <th>q2_num_words</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>356118</th>\n",
       "      <td>356118</td>\n",
       "      <td>485393</td>\n",
       "      <td>485394</td>\n",
       "      <td>what is the problems of trade manufactured goo...</td>\n",
       "      <td>which voting groups did hilary do better than ...</td>\n",
       "      <td>0</td>\n",
       "      <td>91</td>\n",
       "      <td>86</td>\n",
       "      <td>13</td>\n",
       "      <td>14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>309345</th>\n",
       "      <td>309345</td>\n",
       "      <td>433321</td>\n",
       "      <td>30782</td>\n",
       "      <td>do spammers make money</td>\n",
       "      <td>how can you look at someone s private instagra...</td>\n",
       "      <td>0</td>\n",
       "      <td>22</td>\n",
       "      <td>78</td>\n",
       "      <td>4</td>\n",
       "      <td>13</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>213099</th>\n",
       "      <td>213099</td>\n",
       "      <td>318345</td>\n",
       "      <td>318346</td>\n",
       "      <td>why do corrupt and evil people flourish while ...</td>\n",
       "      <td>is karma real  if so  then why do we see bad p...</td>\n",
       "      <td>1</td>\n",
       "      <td>69</td>\n",
       "      <td>117</td>\n",
       "      <td>12</td>\n",
       "      <td>27</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>122902</th>\n",
       "      <td>122902</td>\n",
       "      <td>198822</td>\n",
       "      <td>198823</td>\n",
       "      <td>where was the wheel invented</td>\n",
       "      <td>when was the wheel invented</td>\n",
       "      <td>0</td>\n",
       "      <td>28</td>\n",
       "      <td>27</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>205461</th>\n",
       "      <td>205461</td>\n",
       "      <td>308611</td>\n",
       "      <td>308612</td>\n",
       "      <td>is it too late to learn playing drums at the a...</td>\n",
       "      <td>is it too late to learn drums at age 34</td>\n",
       "      <td>0</td>\n",
       "      <td>54</td>\n",
       "      <td>39</td>\n",
       "      <td>13</td>\n",
       "      <td>10</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "            id    qid1    qid2  \\\n",
       "356118  356118  485393  485394   \n",
       "309345  309345  433321   30782   \n",
       "213099  213099  318345  318346   \n",
       "122902  122902  198822  198823   \n",
       "205461  205461  308611  308612   \n",
       "\n",
       "                                                question1  \\\n",
       "356118  what is the problems of trade manufactured goo...   \n",
       "309345                             do spammers make money   \n",
       "213099  why do corrupt and evil people flourish while ...   \n",
       "122902                       where was the wheel invented   \n",
       "205461  is it too late to learn playing drums at the a...   \n",
       "\n",
       "                                                question2  is_duplicate  \\\n",
       "356118  which voting groups did hilary do better than ...             0   \n",
       "309345  how can you look at someone s private instagra...             0   \n",
       "213099  is karma real  if so  then why do we see bad p...             1   \n",
       "122902                        when was the wheel invented             0   \n",
       "205461            is it too late to learn drums at age 34             0   \n",
       "\n",
       "        q1_len  q2_len  q1_num_words  q2_num_words  \n",
       "356118      91      86            13            14  \n",
       "309345      22      78             4            13  \n",
       "213099      69     117            12            27  \n",
       "122902      28      27             5             5  \n",
       "205461      54      39            13            10  "
      ]
     },
     "execution_count": 63,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "new_df['q1_num_words'] = new_df['question1'].apply(lambda row: len(row.split(\" \")))\n",
    "new_df['q2_num_words'] = new_df['question2'].apply(lambda row: len(row.split(\" \")))\n",
    "new_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "id": "3efabfa0",
   "metadata": {},
   "outputs": [],
   "source": [
    "def common_words(row):\n",
    "    w1 = set(map(lambda word: word.lower().strip(), row['question1'].split(\" \")))\n",
    "    w2 = set(map(lambda word: word.lower().strip(), row['question2'].split(\" \")))    \n",
    "    return len(w1 & w2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "id": "59901295",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>qid1</th>\n",
       "      <th>qid2</th>\n",
       "      <th>question1</th>\n",
       "      <th>question2</th>\n",
       "      <th>is_duplicate</th>\n",
       "      <th>q1_len</th>\n",
       "      <th>q2_len</th>\n",
       "      <th>q1_num_words</th>\n",
       "      <th>q2_num_words</th>\n",
       "      <th>word_common</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>356118</th>\n",
       "      <td>356118</td>\n",
       "      <td>485393</td>\n",
       "      <td>485394</td>\n",
       "      <td>what is the problems of trade manufactured goo...</td>\n",
       "      <td>which voting groups did hilary do better than ...</td>\n",
       "      <td>0</td>\n",
       "      <td>91</td>\n",
       "      <td>86</td>\n",
       "      <td>13</td>\n",
       "      <td>14</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>309345</th>\n",
       "      <td>309345</td>\n",
       "      <td>433321</td>\n",
       "      <td>30782</td>\n",
       "      <td>do spammers make money</td>\n",
       "      <td>how can you look at someone s private instagra...</td>\n",
       "      <td>0</td>\n",
       "      <td>22</td>\n",
       "      <td>78</td>\n",
       "      <td>4</td>\n",
       "      <td>13</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>213099</th>\n",
       "      <td>213099</td>\n",
       "      <td>318345</td>\n",
       "      <td>318346</td>\n",
       "      <td>why do corrupt and evil people flourish while ...</td>\n",
       "      <td>is karma real  if so  then why do we see bad p...</td>\n",
       "      <td>1</td>\n",
       "      <td>69</td>\n",
       "      <td>117</td>\n",
       "      <td>12</td>\n",
       "      <td>27</td>\n",
       "      <td>8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>122902</th>\n",
       "      <td>122902</td>\n",
       "      <td>198822</td>\n",
       "      <td>198823</td>\n",
       "      <td>where was the wheel invented</td>\n",
       "      <td>when was the wheel invented</td>\n",
       "      <td>0</td>\n",
       "      <td>28</td>\n",
       "      <td>27</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>205461</th>\n",
       "      <td>205461</td>\n",
       "      <td>308611</td>\n",
       "      <td>308612</td>\n",
       "      <td>is it too late to learn playing drums at the a...</td>\n",
       "      <td>is it too late to learn drums at age 34</td>\n",
       "      <td>0</td>\n",
       "      <td>54</td>\n",
       "      <td>39</td>\n",
       "      <td>13</td>\n",
       "      <td>10</td>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "            id    qid1    qid2  \\\n",
       "356118  356118  485393  485394   \n",
       "309345  309345  433321   30782   \n",
       "213099  213099  318345  318346   \n",
       "122902  122902  198822  198823   \n",
       "205461  205461  308611  308612   \n",
       "\n",
       "                                                question1  \\\n",
       "356118  what is the problems of trade manufactured goo...   \n",
       "309345                             do spammers make money   \n",
       "213099  why do corrupt and evil people flourish while ...   \n",
       "122902                       where was the wheel invented   \n",
       "205461  is it too late to learn playing drums at the a...   \n",
       "\n",
       "                                                question2  is_duplicate  \\\n",
       "356118  which voting groups did hilary do better than ...             0   \n",
       "309345  how can you look at someone s private instagra...             0   \n",
       "213099  is karma real  if so  then why do we see bad p...             1   \n",
       "122902                        when was the wheel invented             0   \n",
       "205461            is it too late to learn drums at age 34             0   \n",
       "\n",
       "        q1_len  q2_len  q1_num_words  q2_num_words  word_common  \n",
       "356118      91      86            13            14            1  \n",
       "309345      22      78             4            13            0  \n",
       "213099      69     117            12            27            8  \n",
       "122902      28      27             5             5            4  \n",
       "205461      54      39            13            10            9  "
      ]
     },
     "execution_count": 65,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "new_df['word_common'] = new_df.apply(common_words, axis=1)\n",
    "new_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "id": "5014884b",
   "metadata": {},
   "outputs": [],
   "source": [
    "def total_words(row):\n",
    "    w1 = set(map(lambda word: word.lower().strip(), row['question1'].split(\" \")))\n",
    "    w2 = set(map(lambda word: word.lower().strip(), row['question2'].split(\" \")))    \n",
    "    return (len(w1) + len(w2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "id": "cef922b4",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>qid1</th>\n",
       "      <th>qid2</th>\n",
       "      <th>question1</th>\n",
       "      <th>question2</th>\n",
       "      <th>is_duplicate</th>\n",
       "      <th>q1_len</th>\n",
       "      <th>q2_len</th>\n",
       "      <th>q1_num_words</th>\n",
       "      <th>q2_num_words</th>\n",
       "      <th>word_common</th>\n",
       "      <th>word_total</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>356118</th>\n",
       "      <td>356118</td>\n",
       "      <td>485393</td>\n",
       "      <td>485394</td>\n",
       "      <td>what is the problems of trade manufactured goo...</td>\n",
       "      <td>which voting groups did hilary do better than ...</td>\n",
       "      <td>0</td>\n",
       "      <td>91</td>\n",
       "      <td>86</td>\n",
       "      <td>13</td>\n",
       "      <td>14</td>\n",
       "      <td>1</td>\n",
       "      <td>27</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>309345</th>\n",
       "      <td>309345</td>\n",
       "      <td>433321</td>\n",
       "      <td>30782</td>\n",
       "      <td>do spammers make money</td>\n",
       "      <td>how can you look at someone s private instagra...</td>\n",
       "      <td>0</td>\n",
       "      <td>22</td>\n",
       "      <td>78</td>\n",
       "      <td>4</td>\n",
       "      <td>13</td>\n",
       "      <td>0</td>\n",
       "      <td>17</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>213099</th>\n",
       "      <td>213099</td>\n",
       "      <td>318345</td>\n",
       "      <td>318346</td>\n",
       "      <td>why do corrupt and evil people flourish while ...</td>\n",
       "      <td>is karma real  if so  then why do we see bad p...</td>\n",
       "      <td>1</td>\n",
       "      <td>69</td>\n",
       "      <td>117</td>\n",
       "      <td>12</td>\n",
       "      <td>27</td>\n",
       "      <td>8</td>\n",
       "      <td>37</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>122902</th>\n",
       "      <td>122902</td>\n",
       "      <td>198822</td>\n",
       "      <td>198823</td>\n",
       "      <td>where was the wheel invented</td>\n",
       "      <td>when was the wheel invented</td>\n",
       "      <td>0</td>\n",
       "      <td>28</td>\n",
       "      <td>27</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>4</td>\n",
       "      <td>10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>205461</th>\n",
       "      <td>205461</td>\n",
       "      <td>308611</td>\n",
       "      <td>308612</td>\n",
       "      <td>is it too late to learn playing drums at the a...</td>\n",
       "      <td>is it too late to learn drums at age 34</td>\n",
       "      <td>0</td>\n",
       "      <td>54</td>\n",
       "      <td>39</td>\n",
       "      <td>13</td>\n",
       "      <td>10</td>\n",
       "      <td>9</td>\n",
       "      <td>23</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "            id    qid1    qid2  \\\n",
       "356118  356118  485393  485394   \n",
       "309345  309345  433321   30782   \n",
       "213099  213099  318345  318346   \n",
       "122902  122902  198822  198823   \n",
       "205461  205461  308611  308612   \n",
       "\n",
       "                                                question1  \\\n",
       "356118  what is the problems of trade manufactured goo...   \n",
       "309345                             do spammers make money   \n",
       "213099  why do corrupt and evil people flourish while ...   \n",
       "122902                       where was the wheel invented   \n",
       "205461  is it too late to learn playing drums at the a...   \n",
       "\n",
       "                                                question2  is_duplicate  \\\n",
       "356118  which voting groups did hilary do better than ...             0   \n",
       "309345  how can you look at someone s private instagra...             0   \n",
       "213099  is karma real  if so  then why do we see bad p...             1   \n",
       "122902                        when was the wheel invented             0   \n",
       "205461            is it too late to learn drums at age 34             0   \n",
       "\n",
       "        q1_len  q2_len  q1_num_words  q2_num_words  word_common  word_total  \n",
       "356118      91      86            13            14            1          27  \n",
       "309345      22      78             4            13            0          17  \n",
       "213099      69     117            12            27            8          37  \n",
       "122902      28      27             5             5            4          10  \n",
       "205461      54      39            13            10            9          23  "
      ]
     },
     "execution_count": 67,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "new_df['word_total'] = new_df.apply(total_words, axis=1)\n",
    "new_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "id": "25ae574e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>qid1</th>\n",
       "      <th>qid2</th>\n",
       "      <th>question1</th>\n",
       "      <th>question2</th>\n",
       "      <th>is_duplicate</th>\n",
       "      <th>q1_len</th>\n",
       "      <th>q2_len</th>\n",
       "      <th>q1_num_words</th>\n",
       "      <th>q2_num_words</th>\n",
       "      <th>word_common</th>\n",
       "      <th>word_total</th>\n",
       "      <th>word_share</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>356118</th>\n",
       "      <td>356118</td>\n",
       "      <td>485393</td>\n",
       "      <td>485394</td>\n",
       "      <td>what is the problems of trade manufactured goo...</td>\n",
       "      <td>which voting groups did hilary do better than ...</td>\n",
       "      <td>0</td>\n",
       "      <td>91</td>\n",
       "      <td>86</td>\n",
       "      <td>13</td>\n",
       "      <td>14</td>\n",
       "      <td>1</td>\n",
       "      <td>27</td>\n",
       "      <td>0.04</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>309345</th>\n",
       "      <td>309345</td>\n",
       "      <td>433321</td>\n",
       "      <td>30782</td>\n",
       "      <td>do spammers make money</td>\n",
       "      <td>how can you look at someone s private instagra...</td>\n",
       "      <td>0</td>\n",
       "      <td>22</td>\n",
       "      <td>78</td>\n",
       "      <td>4</td>\n",
       "      <td>13</td>\n",
       "      <td>0</td>\n",
       "      <td>17</td>\n",
       "      <td>0.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>213099</th>\n",
       "      <td>213099</td>\n",
       "      <td>318345</td>\n",
       "      <td>318346</td>\n",
       "      <td>why do corrupt and evil people flourish while ...</td>\n",
       "      <td>is karma real  if so  then why do we see bad p...</td>\n",
       "      <td>1</td>\n",
       "      <td>69</td>\n",
       "      <td>117</td>\n",
       "      <td>12</td>\n",
       "      <td>27</td>\n",
       "      <td>8</td>\n",
       "      <td>37</td>\n",
       "      <td>0.22</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>122902</th>\n",
       "      <td>122902</td>\n",
       "      <td>198822</td>\n",
       "      <td>198823</td>\n",
       "      <td>where was the wheel invented</td>\n",
       "      <td>when was the wheel invented</td>\n",
       "      <td>0</td>\n",
       "      <td>28</td>\n",
       "      <td>27</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>4</td>\n",
       "      <td>10</td>\n",
       "      <td>0.40</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>205461</th>\n",
       "      <td>205461</td>\n",
       "      <td>308611</td>\n",
       "      <td>308612</td>\n",
       "      <td>is it too late to learn playing drums at the a...</td>\n",
       "      <td>is it too late to learn drums at age 34</td>\n",
       "      <td>0</td>\n",
       "      <td>54</td>\n",
       "      <td>39</td>\n",
       "      <td>13</td>\n",
       "      <td>10</td>\n",
       "      <td>9</td>\n",
       "      <td>23</td>\n",
       "      <td>0.39</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "            id    qid1    qid2  \\\n",
       "356118  356118  485393  485394   \n",
       "309345  309345  433321   30782   \n",
       "213099  213099  318345  318346   \n",
       "122902  122902  198822  198823   \n",
       "205461  205461  308611  308612   \n",
       "\n",
       "                                                question1  \\\n",
       "356118  what is the problems of trade manufactured goo...   \n",
       "309345                             do spammers make money   \n",
       "213099  why do corrupt and evil people flourish while ...   \n",
       "122902                       where was the wheel invented   \n",
       "205461  is it too late to learn playing drums at the a...   \n",
       "\n",
       "                                                question2  is_duplicate  \\\n",
       "356118  which voting groups did hilary do better than ...             0   \n",
       "309345  how can you look at someone s private instagra...             0   \n",
       "213099  is karma real  if so  then why do we see bad p...             1   \n",
       "122902                        when was the wheel invented             0   \n",
       "205461            is it too late to learn drums at age 34             0   \n",
       "\n",
       "        q1_len  q2_len  q1_num_words  q2_num_words  word_common  word_total  \\\n",
       "356118      91      86            13            14            1          27   \n",
       "309345      22      78             4            13            0          17   \n",
       "213099      69     117            12            27            8          37   \n",
       "122902      28      27             5             5            4          10   \n",
       "205461      54      39            13            10            9          23   \n",
       "\n",
       "        word_share  \n",
       "356118        0.04  \n",
       "309345        0.00  \n",
       "213099        0.22  \n",
       "122902        0.40  \n",
       "205461        0.39  "
      ]
     },
     "execution_count": 68,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "new_df['word_share'] = round(new_df['word_common']/new_df['word_total'],2)\n",
    "new_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "id": "a3cf2612",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[nltk_data] Downloading package stopwords to\n",
      "[nltk_data]     C:\\Users\\Mutahhir\\AppData\\Roaming\\nltk_data...\n",
      "[nltk_data]   Package stopwords is already up-to-date!\n"
     ]
    }
   ],
   "source": [
    "# Advanced Features\n",
    "import nltk\n",
    "from nltk.corpus import stopwords\n",
    "nltk.download('stopwords')\n",
    "\n",
    "\n",
    "def fetch_token_features(row):\n",
    "    \n",
    "    q1 = row['question1']\n",
    "    q2 = row['question2']\n",
    "    \n",
    "    SAFE_DIV = 0.0001 \n",
    "\n",
    "    STOP_WORDS = stopwords.words(\"english\")\n",
    "    \n",
    "    token_features = [0.0]*8\n",
    "    \n",
    "    # Converting the Sentence into Tokens: \n",
    "    q1_tokens = q1.split()\n",
    "    q2_tokens = q2.split()\n",
    "    \n",
    "    if len(q1_tokens) == 0 or len(q2_tokens) == 0:\n",
    "        return token_features\n",
    "\n",
    "    # Get the non-stopwords in Questions\n",
    "    q1_words = set([word for word in q1_tokens if word not in STOP_WORDS])\n",
    "    q2_words = set([word for word in q2_tokens if word not in STOP_WORDS])\n",
    "    \n",
    "    #Get the stopwords in Questions\n",
    "    q1_stops = set([word for word in q1_tokens if word in STOP_WORDS])\n",
    "    q2_stops = set([word for word in q2_tokens if word in STOP_WORDS])\n",
    "    \n",
    "    # Get the common non-stopwords from Question pair\n",
    "    common_word_count = len(q1_words.intersection(q2_words))\n",
    "    \n",
    "    # Get the common stopwords from Question pair\n",
    "    common_stop_count = len(q1_stops.intersection(q2_stops))\n",
    "    \n",
    "    # Get the common Tokens from Question pair\n",
    "    common_token_count = len(set(q1_tokens).intersection(set(q2_tokens)))\n",
    "    \n",
    "    \n",
    "    token_features[0] = common_word_count / (min(len(q1_words), len(q2_words)) + SAFE_DIV)\n",
    "    token_features[1] = common_word_count / (max(len(q1_words), len(q2_words)) + SAFE_DIV)\n",
    "    token_features[2] = common_stop_count / (min(len(q1_stops), len(q2_stops)) + SAFE_DIV)\n",
    "    token_features[3] = common_stop_count / (max(len(q1_stops), len(q2_stops)) + SAFE_DIV)\n",
    "    token_features[4] = common_token_count / (min(len(q1_tokens), len(q2_tokens)) + SAFE_DIV)\n",
    "    token_features[5] = common_token_count / (max(len(q1_tokens), len(q2_tokens)) + SAFE_DIV)\n",
    "    \n",
    "    # Last word of both question is same or not\n",
    "    token_features[6] = int(q1_tokens[-1] == q2_tokens[-1])\n",
    "    \n",
    "    # First word of both question is same or not\n",
    "    token_features[7] = int(q1_tokens[0] == q2_tokens[0])\n",
    "    \n",
    "    return token_features\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "id": "1dfca25e",
   "metadata": {},
   "outputs": [],
   "source": [
    "token_features = new_df.apply(fetch_token_features, axis=1)\n",
    "\n",
    "new_df[\"cwc_min\"]       = list(map(lambda x: x[0], token_features))\n",
    "new_df[\"cwc_max\"]       = list(map(lambda x: x[1], token_features))\n",
    "new_df[\"csc_min\"]       = list(map(lambda x: x[2], token_features))\n",
    "new_df[\"csc_max\"]       = list(map(lambda x: x[3], token_features))\n",
    "new_df[\"ctc_min\"]       = list(map(lambda x: x[4], token_features))\n",
    "new_df[\"ctc_max\"]       = list(map(lambda x: x[5], token_features))\n",
    "new_df[\"last_word_eq\"]  = list(map(lambda x: x[6], token_features))\n",
    "new_df[\"first_word_eq\"] = list(map(lambda x: x[7], token_features))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "id": "66b803a7",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>qid1</th>\n",
       "      <th>qid2</th>\n",
       "      <th>question1</th>\n",
       "      <th>question2</th>\n",
       "      <th>is_duplicate</th>\n",
       "      <th>q1_len</th>\n",
       "      <th>q2_len</th>\n",
       "      <th>q1_num_words</th>\n",
       "      <th>q2_num_words</th>\n",
       "      <th>...</th>\n",
       "      <th>word_total</th>\n",
       "      <th>word_share</th>\n",
       "      <th>cwc_min</th>\n",
       "      <th>cwc_max</th>\n",
       "      <th>csc_min</th>\n",
       "      <th>csc_max</th>\n",
       "      <th>ctc_min</th>\n",
       "      <th>ctc_max</th>\n",
       "      <th>last_word_eq</th>\n",
       "      <th>first_word_eq</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>356118</th>\n",
       "      <td>356118</td>\n",
       "      <td>485393</td>\n",
       "      <td>485394</td>\n",
       "      <td>what is the problems of trade manufactured goo...</td>\n",
       "      <td>which voting groups did hilary do better than ...</td>\n",
       "      <td>0</td>\n",
       "      <td>91</td>\n",
       "      <td>86</td>\n",
       "      <td>13</td>\n",
       "      <td>14</td>\n",
       "      <td>...</td>\n",
       "      <td>27</td>\n",
       "      <td>0.04</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.166664</td>\n",
       "      <td>0.166664</td>\n",
       "      <td>0.076922</td>\n",
       "      <td>0.071428</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>309345</th>\n",
       "      <td>309345</td>\n",
       "      <td>433321</td>\n",
       "      <td>30782</td>\n",
       "      <td>do spammers make money</td>\n",
       "      <td>how can you look at someone s private instagra...</td>\n",
       "      <td>0</td>\n",
       "      <td>22</td>\n",
       "      <td>78</td>\n",
       "      <td>4</td>\n",
       "      <td>13</td>\n",
       "      <td>...</td>\n",
       "      <td>17</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>213099</th>\n",
       "      <td>213099</td>\n",
       "      <td>318345</td>\n",
       "      <td>318346</td>\n",
       "      <td>why do corrupt and evil people flourish while ...</td>\n",
       "      <td>is karma real  if so  then why do we see bad p...</td>\n",
       "      <td>1</td>\n",
       "      <td>69</td>\n",
       "      <td>117</td>\n",
       "      <td>12</td>\n",
       "      <td>27</td>\n",
       "      <td>...</td>\n",
       "      <td>37</td>\n",
       "      <td>0.22</td>\n",
       "      <td>0.571420</td>\n",
       "      <td>0.333331</td>\n",
       "      <td>0.999975</td>\n",
       "      <td>0.307690</td>\n",
       "      <td>0.666661</td>\n",
       "      <td>0.319999</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>122902</th>\n",
       "      <td>122902</td>\n",
       "      <td>198822</td>\n",
       "      <td>198823</td>\n",
       "      <td>where was the wheel invented</td>\n",
       "      <td>when was the wheel invented</td>\n",
       "      <td>0</td>\n",
       "      <td>28</td>\n",
       "      <td>27</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>...</td>\n",
       "      <td>10</td>\n",
       "      <td>0.40</td>\n",
       "      <td>0.999950</td>\n",
       "      <td>0.999950</td>\n",
       "      <td>0.666644</td>\n",
       "      <td>0.666644</td>\n",
       "      <td>0.799984</td>\n",
       "      <td>0.799984</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>205461</th>\n",
       "      <td>205461</td>\n",
       "      <td>308611</td>\n",
       "      <td>308612</td>\n",
       "      <td>is it too late to learn playing drums at the a...</td>\n",
       "      <td>is it too late to learn drums at age 34</td>\n",
       "      <td>0</td>\n",
       "      <td>54</td>\n",
       "      <td>39</td>\n",
       "      <td>13</td>\n",
       "      <td>10</td>\n",
       "      <td>...</td>\n",
       "      <td>23</td>\n",
       "      <td>0.39</td>\n",
       "      <td>0.799984</td>\n",
       "      <td>0.666656</td>\n",
       "      <td>0.999980</td>\n",
       "      <td>0.714276</td>\n",
       "      <td>0.899991</td>\n",
       "      <td>0.692302</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 21 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "            id    qid1    qid2  \\\n",
       "356118  356118  485393  485394   \n",
       "309345  309345  433321   30782   \n",
       "213099  213099  318345  318346   \n",
       "122902  122902  198822  198823   \n",
       "205461  205461  308611  308612   \n",
       "\n",
       "                                                question1  \\\n",
       "356118  what is the problems of trade manufactured goo...   \n",
       "309345                             do spammers make money   \n",
       "213099  why do corrupt and evil people flourish while ...   \n",
       "122902                       where was the wheel invented   \n",
       "205461  is it too late to learn playing drums at the a...   \n",
       "\n",
       "                                                question2  is_duplicate  \\\n",
       "356118  which voting groups did hilary do better than ...             0   \n",
       "309345  how can you look at someone s private instagra...             0   \n",
       "213099  is karma real  if so  then why do we see bad p...             1   \n",
       "122902                        when was the wheel invented             0   \n",
       "205461            is it too late to learn drums at age 34             0   \n",
       "\n",
       "        q1_len  q2_len  q1_num_words  q2_num_words  ...  word_total  \\\n",
       "356118      91      86            13            14  ...          27   \n",
       "309345      22      78             4            13  ...          17   \n",
       "213099      69     117            12            27  ...          37   \n",
       "122902      28      27             5             5  ...          10   \n",
       "205461      54      39            13            10  ...          23   \n",
       "\n",
       "        word_share   cwc_min   cwc_max   csc_min   csc_max   ctc_min  \\\n",
       "356118        0.04  0.000000  0.000000  0.166664  0.166664  0.076922   \n",
       "309345        0.00  0.000000  0.000000  0.000000  0.000000  0.000000   \n",
       "213099        0.22  0.571420  0.333331  0.999975  0.307690  0.666661   \n",
       "122902        0.40  0.999950  0.999950  0.666644  0.666644  0.799984   \n",
       "205461        0.39  0.799984  0.666656  0.999980  0.714276  0.899991   \n",
       "\n",
       "         ctc_max  last_word_eq  first_word_eq  \n",
       "356118  0.071428             0              0  \n",
       "309345  0.000000             0              0  \n",
       "213099  0.319999             0              0  \n",
       "122902  0.799984             1              0  \n",
       "205461  0.692302             0              1  \n",
       "\n",
       "[5 rows x 21 columns]"
      ]
     },
     "execution_count": 71,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "new_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "id": "24bc37a6",
   "metadata": {},
   "outputs": [],
   "source": [
    "import distance\n",
    "\n",
    "def fetch_length_features(row):\n",
    "    \n",
    "    q1 = row['question1']\n",
    "    q2 = row['question2']\n",
    "    \n",
    "    length_features = [0.0]*3\n",
    "    \n",
    "    # Converting the Sentence into Tokens: \n",
    "    q1_tokens = q1.split()\n",
    "    q2_tokens = q2.split()\n",
    "    \n",
    "    if len(q1_tokens) == 0 or len(q2_tokens) == 0:\n",
    "        return length_features\n",
    "    \n",
    "    # Absolute length features\n",
    "    length_features[0] = abs(len(q1_tokens) - len(q2_tokens))\n",
    "    \n",
    "    #Average Token Length of both Questions\n",
    "    length_features[1] = (len(q1_tokens) + len(q2_tokens))/2\n",
    "    \n",
    "    strs = list(distance.lcsubstrings(q1, q2))\n",
    "    if len(strs)==0:\n",
    "        length_features[2] = 0\n",
    "    else:\n",
    "        length_features[2] = len(strs[0]) / (min(len(q1), len(q2)) + 1)\n",
    "    \n",
    "    return length_features\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "id": "87817907",
   "metadata": {},
   "outputs": [],
   "source": [
    "length_features = new_df.apply(fetch_length_features, axis=1)\n",
    "\n",
    "new_df['abs_len_diff'] = list(map(lambda x: x[0], length_features))\n",
    "new_df['mean_len'] = list(map(lambda x: x[1], length_features))\n",
    "new_df['longest_substr_ratio'] = list(map(lambda x: x[2], length_features))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "id": "84c29924",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>qid1</th>\n",
       "      <th>qid2</th>\n",
       "      <th>question1</th>\n",
       "      <th>question2</th>\n",
       "      <th>is_duplicate</th>\n",
       "      <th>q1_len</th>\n",
       "      <th>q2_len</th>\n",
       "      <th>q1_num_words</th>\n",
       "      <th>q2_num_words</th>\n",
       "      <th>...</th>\n",
       "      <th>cwc_max</th>\n",
       "      <th>csc_min</th>\n",
       "      <th>csc_max</th>\n",
       "      <th>ctc_min</th>\n",
       "      <th>ctc_max</th>\n",
       "      <th>last_word_eq</th>\n",
       "      <th>first_word_eq</th>\n",
       "      <th>abs_len_diff</th>\n",
       "      <th>mean_len</th>\n",
       "      <th>longest_substr_ratio</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>356118</th>\n",
       "      <td>356118</td>\n",
       "      <td>485393</td>\n",
       "      <td>485394</td>\n",
       "      <td>what is the problems of trade manufactured goo...</td>\n",
       "      <td>which voting groups did hilary do better than ...</td>\n",
       "      <td>0</td>\n",
       "      <td>91</td>\n",
       "      <td>86</td>\n",
       "      <td>13</td>\n",
       "      <td>14</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.166664</td>\n",
       "      <td>0.166664</td>\n",
       "      <td>0.076922</td>\n",
       "      <td>0.071428</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>13.5</td>\n",
       "      <td>0.057471</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>309345</th>\n",
       "      <td>309345</td>\n",
       "      <td>433321</td>\n",
       "      <td>30782</td>\n",
       "      <td>do spammers make money</td>\n",
       "      <td>how can you look at someone s private instagra...</td>\n",
       "      <td>0</td>\n",
       "      <td>22</td>\n",
       "      <td>78</td>\n",
       "      <td>4</td>\n",
       "      <td>13</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>9</td>\n",
       "      <td>8.5</td>\n",
       "      <td>0.130435</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>213099</th>\n",
       "      <td>213099</td>\n",
       "      <td>318345</td>\n",
       "      <td>318346</td>\n",
       "      <td>why do corrupt and evil people flourish while ...</td>\n",
       "      <td>is karma real  if so  then why do we see bad p...</td>\n",
       "      <td>1</td>\n",
       "      <td>69</td>\n",
       "      <td>117</td>\n",
       "      <td>12</td>\n",
       "      <td>27</td>\n",
       "      <td>...</td>\n",
       "      <td>0.333331</td>\n",
       "      <td>0.999975</td>\n",
       "      <td>0.307690</td>\n",
       "      <td>0.666661</td>\n",
       "      <td>0.319999</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>13</td>\n",
       "      <td>18.5</td>\n",
       "      <td>0.128571</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>122902</th>\n",
       "      <td>122902</td>\n",
       "      <td>198822</td>\n",
       "      <td>198823</td>\n",
       "      <td>where was the wheel invented</td>\n",
       "      <td>when was the wheel invented</td>\n",
       "      <td>0</td>\n",
       "      <td>28</td>\n",
       "      <td>27</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>...</td>\n",
       "      <td>0.999950</td>\n",
       "      <td>0.666644</td>\n",
       "      <td>0.666644</td>\n",
       "      <td>0.799984</td>\n",
       "      <td>0.799984</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>0.821429</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>205461</th>\n",
       "      <td>205461</td>\n",
       "      <td>308611</td>\n",
       "      <td>308612</td>\n",
       "      <td>is it too late to learn playing drums at the a...</td>\n",
       "      <td>is it too late to learn drums at age 34</td>\n",
       "      <td>0</td>\n",
       "      <td>54</td>\n",
       "      <td>39</td>\n",
       "      <td>13</td>\n",
       "      <td>10</td>\n",
       "      <td>...</td>\n",
       "      <td>0.666656</td>\n",
       "      <td>0.999980</td>\n",
       "      <td>0.714276</td>\n",
       "      <td>0.899991</td>\n",
       "      <td>0.692302</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>11.5</td>\n",
       "      <td>0.600000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 24 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "            id    qid1    qid2  \\\n",
       "356118  356118  485393  485394   \n",
       "309345  309345  433321   30782   \n",
       "213099  213099  318345  318346   \n",
       "122902  122902  198822  198823   \n",
       "205461  205461  308611  308612   \n",
       "\n",
       "                                                question1  \\\n",
       "356118  what is the problems of trade manufactured goo...   \n",
       "309345                             do spammers make money   \n",
       "213099  why do corrupt and evil people flourish while ...   \n",
       "122902                       where was the wheel invented   \n",
       "205461  is it too late to learn playing drums at the a...   \n",
       "\n",
       "                                                question2  is_duplicate  \\\n",
       "356118  which voting groups did hilary do better than ...             0   \n",
       "309345  how can you look at someone s private instagra...             0   \n",
       "213099  is karma real  if so  then why do we see bad p...             1   \n",
       "122902                        when was the wheel invented             0   \n",
       "205461            is it too late to learn drums at age 34             0   \n",
       "\n",
       "        q1_len  q2_len  q1_num_words  q2_num_words  ...   cwc_max   csc_min  \\\n",
       "356118      91      86            13            14  ...  0.000000  0.166664   \n",
       "309345      22      78             4            13  ...  0.000000  0.000000   \n",
       "213099      69     117            12            27  ...  0.333331  0.999975   \n",
       "122902      28      27             5             5  ...  0.999950  0.666644   \n",
       "205461      54      39            13            10  ...  0.666656  0.999980   \n",
       "\n",
       "         csc_max   ctc_min   ctc_max  last_word_eq  first_word_eq  \\\n",
       "356118  0.166664  0.076922  0.071428             0              0   \n",
       "309345  0.000000  0.000000  0.000000             0              0   \n",
       "213099  0.307690  0.666661  0.319999             0              0   \n",
       "122902  0.666644  0.799984  0.799984             1              0   \n",
       "205461  0.714276  0.899991  0.692302             0              1   \n",
       "\n",
       "        abs_len_diff  mean_len  longest_substr_ratio  \n",
       "356118             1      13.5              0.057471  \n",
       "309345             9       8.5              0.130435  \n",
       "213099            13      18.5              0.128571  \n",
       "122902             0       5.0              0.821429  \n",
       "205461             3      11.5              0.600000  \n",
       "\n",
       "[5 rows x 24 columns]"
      ]
     },
     "execution_count": 74,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "new_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "id": "7308d005",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Fuzzy Features\n",
    "from fuzzywuzzy import fuzz\n",
    "\n",
    "def fetch_fuzzy_features(row):\n",
    "    \n",
    "    q1 = row['question1']\n",
    "    q2 = row['question2']\n",
    "    \n",
    "    fuzzy_features = [0.0]*4\n",
    "    \n",
    "    # fuzz_ratio\n",
    "    fuzzy_features[0] = fuzz.QRatio(q1, q2)\n",
    "\n",
    "    # fuzz_partial_ratio\n",
    "    fuzzy_features[1] = fuzz.partial_ratio(q1, q2)\n",
    "\n",
    "    # token_sort_ratio\n",
    "    fuzzy_features[2] = fuzz.token_sort_ratio(q1, q2)\n",
    "\n",
    "    # token_set_ratio\n",
    "    fuzzy_features[3] = fuzz.token_set_ratio(q1, q2)\n",
    "\n",
    "    return fuzzy_features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "id": "77876d10",
   "metadata": {},
   "outputs": [],
   "source": [
    "fuzzy_features = new_df.apply(fetch_fuzzy_features, axis=1)\n",
    "\n",
    "# Creating new feature columns for fuzzy features\n",
    "new_df['fuzz_ratio'] = list(map(lambda x: x[0], fuzzy_features))\n",
    "new_df['fuzz_partial_ratio'] = list(map(lambda x: x[1], fuzzy_features))\n",
    "new_df['token_sort_ratio'] = list(map(lambda x: x[2], fuzzy_features))\n",
    "new_df['token_set_ratio'] = list(map(lambda x: x[3], fuzzy_features))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "id": "8f7d8ff4",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(300, 28)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>qid1</th>\n",
       "      <th>qid2</th>\n",
       "      <th>question1</th>\n",
       "      <th>question2</th>\n",
       "      <th>is_duplicate</th>\n",
       "      <th>q1_len</th>\n",
       "      <th>q2_len</th>\n",
       "      <th>q1_num_words</th>\n",
       "      <th>q2_num_words</th>\n",
       "      <th>...</th>\n",
       "      <th>ctc_max</th>\n",
       "      <th>last_word_eq</th>\n",
       "      <th>first_word_eq</th>\n",
       "      <th>abs_len_diff</th>\n",
       "      <th>mean_len</th>\n",
       "      <th>longest_substr_ratio</th>\n",
       "      <th>fuzz_ratio</th>\n",
       "      <th>fuzz_partial_ratio</th>\n",
       "      <th>token_sort_ratio</th>\n",
       "      <th>token_set_ratio</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>356118</th>\n",
       "      <td>356118</td>\n",
       "      <td>485393</td>\n",
       "      <td>485394</td>\n",
       "      <td>what is the problems of trade manufactured goo...</td>\n",
       "      <td>which voting groups did hilary do better than ...</td>\n",
       "      <td>0</td>\n",
       "      <td>91</td>\n",
       "      <td>86</td>\n",
       "      <td>13</td>\n",
       "      <td>14</td>\n",
       "      <td>...</td>\n",
       "      <td>0.071428</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>13.5</td>\n",
       "      <td>0.057471</td>\n",
       "      <td>21</td>\n",
       "      <td>29</td>\n",
       "      <td>38</td>\n",
       "      <td>28</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>309345</th>\n",
       "      <td>309345</td>\n",
       "      <td>433321</td>\n",
       "      <td>30782</td>\n",
       "      <td>do spammers make money</td>\n",
       "      <td>how can you look at someone s private instagra...</td>\n",
       "      <td>0</td>\n",
       "      <td>22</td>\n",
       "      <td>78</td>\n",
       "      <td>4</td>\n",
       "      <td>13</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>9</td>\n",
       "      <td>8.5</td>\n",
       "      <td>0.130435</td>\n",
       "      <td>16</td>\n",
       "      <td>36</td>\n",
       "      <td>22</td>\n",
       "      <td>22</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>213099</th>\n",
       "      <td>213099</td>\n",
       "      <td>318345</td>\n",
       "      <td>318346</td>\n",
       "      <td>why do corrupt and evil people flourish while ...</td>\n",
       "      <td>is karma real  if so  then why do we see bad p...</td>\n",
       "      <td>1</td>\n",
       "      <td>69</td>\n",
       "      <td>117</td>\n",
       "      <td>12</td>\n",
       "      <td>27</td>\n",
       "      <td>...</td>\n",
       "      <td>0.319999</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>13</td>\n",
       "      <td>18.5</td>\n",
       "      <td>0.128571</td>\n",
       "      <td>32</td>\n",
       "      <td>45</td>\n",
       "      <td>52</td>\n",
       "      <td>78</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>122902</th>\n",
       "      <td>122902</td>\n",
       "      <td>198822</td>\n",
       "      <td>198823</td>\n",
       "      <td>where was the wheel invented</td>\n",
       "      <td>when was the wheel invented</td>\n",
       "      <td>0</td>\n",
       "      <td>28</td>\n",
       "      <td>27</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>...</td>\n",
       "      <td>0.799984</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>0.821429</td>\n",
       "      <td>95</td>\n",
       "      <td>93</td>\n",
       "      <td>95</td>\n",
       "      <td>95</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>205461</th>\n",
       "      <td>205461</td>\n",
       "      <td>308611</td>\n",
       "      <td>308612</td>\n",
       "      <td>is it too late to learn playing drums at the a...</td>\n",
       "      <td>is it too late to learn drums at age 34</td>\n",
       "      <td>0</td>\n",
       "      <td>54</td>\n",
       "      <td>39</td>\n",
       "      <td>13</td>\n",
       "      <td>10</td>\n",
       "      <td>...</td>\n",
       "      <td>0.692302</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>11.5</td>\n",
       "      <td>0.600000</td>\n",
       "      <td>80</td>\n",
       "      <td>79</td>\n",
       "      <td>80</td>\n",
       "      <td>96</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 28 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "            id    qid1    qid2  \\\n",
       "356118  356118  485393  485394   \n",
       "309345  309345  433321   30782   \n",
       "213099  213099  318345  318346   \n",
       "122902  122902  198822  198823   \n",
       "205461  205461  308611  308612   \n",
       "\n",
       "                                                question1  \\\n",
       "356118  what is the problems of trade manufactured goo...   \n",
       "309345                             do spammers make money   \n",
       "213099  why do corrupt and evil people flourish while ...   \n",
       "122902                       where was the wheel invented   \n",
       "205461  is it too late to learn playing drums at the a...   \n",
       "\n",
       "                                                question2  is_duplicate  \\\n",
       "356118  which voting groups did hilary do better than ...             0   \n",
       "309345  how can you look at someone s private instagra...             0   \n",
       "213099  is karma real  if so  then why do we see bad p...             1   \n",
       "122902                        when was the wheel invented             0   \n",
       "205461            is it too late to learn drums at age 34             0   \n",
       "\n",
       "        q1_len  q2_len  q1_num_words  q2_num_words  ...   ctc_max  \\\n",
       "356118      91      86            13            14  ...  0.071428   \n",
       "309345      22      78             4            13  ...  0.000000   \n",
       "213099      69     117            12            27  ...  0.319999   \n",
       "122902      28      27             5             5  ...  0.799984   \n",
       "205461      54      39            13            10  ...  0.692302   \n",
       "\n",
       "        last_word_eq  first_word_eq  abs_len_diff  mean_len  \\\n",
       "356118             0              0             1      13.5   \n",
       "309345             0              0             9       8.5   \n",
       "213099             0              0            13      18.5   \n",
       "122902             1              0             0       5.0   \n",
       "205461             0              1             3      11.5   \n",
       "\n",
       "        longest_substr_ratio  fuzz_ratio  fuzz_partial_ratio  \\\n",
       "356118              0.057471          21                  29   \n",
       "309345              0.130435          16                  36   \n",
       "213099              0.128571          32                  45   \n",
       "122902              0.821429          95                  93   \n",
       "205461              0.600000          80                  79   \n",
       "\n",
       "        token_sort_ratio  token_set_ratio  \n",
       "356118                38               28  \n",
       "309345                22               22  \n",
       "213099                52               78  \n",
       "122902                95               95  \n",
       "205461                80               96  \n",
       "\n",
       "[5 rows x 28 columns]"
      ]
     },
     "execution_count": 77,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print(new_df.shape)\n",
    "new_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "id": "c1bfbf21",
   "metadata": {},
   "outputs": [],
   "source": [
    "# sns.pairplot(new_df[['ctc_min', 'cwc_min', 'csc_min', 'is_duplicate']],hue='is_duplicate')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "id": "546ae98b",
   "metadata": {},
   "outputs": [],
   "source": [
    "# sns.pairplot(new_df[['ctc_max', 'cwc_max', 'csc_max', 'is_duplicate']],hue='is_duplicate')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "id": "a3a97a57",
   "metadata": {},
   "outputs": [],
   "source": [
    "# sns.pairplot(new_df[['last_word_eq', 'first_word_eq', 'is_duplicate']],hue='is_duplicate')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "id": "db082f02",
   "metadata": {},
   "outputs": [],
   "source": [
    "# sns.pairplot(new_df[['mean_len', 'abs_len_diff','longest_substr_ratio', 'is_duplicate']],hue='is_duplicate')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "id": "f11a410e",
   "metadata": {},
   "outputs": [],
   "source": [
    "# sns.pairplot(new_df[['fuzz_ratio', 'fuzz_partial_ratio','token_sort_ratio','token_set_ratio', 'is_duplicate']],hue='is_duplicate')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "id": "850e5e18",
   "metadata": {},
   "outputs": [],
   "source": [
    "# # Using TSNE for Dimentionality reduction for 15 Features(Generated after cleaning the data) to 3 dimention\n",
    "\n",
    "# from sklearn.preprocessing import MinMaxScaler\n",
    "\n",
    "# X = MinMaxScaler().fit_transform(new_df[['cwc_min', 'cwc_max', 'csc_min', 'csc_max' , 'ctc_min' , 'ctc_max' , 'last_word_eq', 'first_word_eq' , 'abs_len_diff' , 'mean_len' , 'token_set_ratio' , 'token_sort_ratio' ,  'fuzz_ratio' , 'fuzz_partial_ratio' , 'longest_substr_ratio']])\n",
    "# y = new_df['is_duplicate'].values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "id": "2d7f5088",
   "metadata": {},
   "outputs": [],
   "source": [
    "# from sklearn.manifold import TSNE\n",
    "\n",
    "# tsne2d = TSNE(\n",
    "#     n_components=2,\n",
    "#     init='random', # pca\n",
    "#     random_state=101,\n",
    "#     method='barnes_hut',\n",
    "#     n_iter=1000,\n",
    "#     verbose=2,\n",
    "#     angle=0.5\n",
    "# ).fit_transform(X)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "id": "4aa2c677",
   "metadata": {},
   "outputs": [],
   "source": [
    "# x_df = pd.DataFrame({'x':tsne2d[:,0], 'y':tsne2d[:,1] ,'label':y})\n",
    "\n",
    "# # draw the plot in appropriate place in the grid\n",
    "# sns.lmplot(data=x_df, x='x', y='y', hue='label', fit_reg=False, palette=\"Set1\",markers=['s','o'])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "id": "591a9f8e",
   "metadata": {},
   "outputs": [],
   "source": [
    "# tsne3d = TSNE(\n",
    "#     n_components=3,\n",
    "#     init='random', # pca\n",
    "#     random_state=101,\n",
    "#     method='barnes_hut',\n",
    "#     n_iter=1000,\n",
    "#     verbose=2,\n",
    "#     angle=0.5\n",
    "# ).fit_transform(X)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "id": "9fcddfa9",
   "metadata": {},
   "outputs": [],
   "source": [
    "# import plotly.graph_objs as go\n",
    "# import plotly.tools as tls\n",
    "# import plotly.offline as py\n",
    "# py.init_notebook_mode(connected=True)\n",
    "\n",
    "# trace1 = go.Scatter3d(\n",
    "#     x=tsne3d[:,0],\n",
    "#     y=tsne3d[:,1],\n",
    "#     z=tsne3d[:,2],\n",
    "#     mode='markers',\n",
    "#     marker=dict(\n",
    "#         sizemode='diameter',\n",
    "#         color = y,\n",
    "#         colorscale = 'Portland',\n",
    "#         colorbar = dict(title = 'duplicate'),\n",
    "#         line=dict(color='rgb(255, 255, 255)'),\n",
    "#         opacity=0.75\n",
    "#     )\n",
    "# )\n",
    "\n",
    "# data=[trace1]\n",
    "# layout=dict(height=800, width=800, title='3d embedding with engineered features')\n",
    "# fig=dict(data=data, layout=layout)\n",
    "# py.iplot(fig, filename='3DBubble')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "id": "3d7f955c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>question1</th>\n",
       "      <th>question2</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>356118</th>\n",
       "      <td>what is the problems of trade manufactured goo...</td>\n",
       "      <td>which voting groups did hilary do better than ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>309345</th>\n",
       "      <td>do spammers make money</td>\n",
       "      <td>how can you look at someone s private instagra...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>213099</th>\n",
       "      <td>why do corrupt and evil people flourish while ...</td>\n",
       "      <td>is karma real  if so  then why do we see bad p...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>122902</th>\n",
       "      <td>where was the wheel invented</td>\n",
       "      <td>when was the wheel invented</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>205461</th>\n",
       "      <td>is it too late to learn playing drums at the a...</td>\n",
       "      <td>is it too late to learn drums at age 34</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                question1  \\\n",
       "356118  what is the problems of trade manufactured goo...   \n",
       "309345                             do spammers make money   \n",
       "213099  why do corrupt and evil people flourish while ...   \n",
       "122902                       where was the wheel invented   \n",
       "205461  is it too late to learn playing drums at the a...   \n",
       "\n",
       "                                                question2  \n",
       "356118  which voting groups did hilary do better than ...  \n",
       "309345  how can you look at someone s private instagra...  \n",
       "213099  is karma real  if so  then why do we see bad p...  \n",
       "122902                        when was the wheel invented  \n",
       "205461            is it too late to learn drums at age 34  "
      ]
     },
     "execution_count": 88,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ques_df = new_df[['question1','question2']]\n",
    "ques_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "id": "7e0a5953",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(300, 23)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>is_duplicate</th>\n",
       "      <th>q1_len</th>\n",
       "      <th>q2_len</th>\n",
       "      <th>q1_num_words</th>\n",
       "      <th>q2_num_words</th>\n",
       "      <th>word_common</th>\n",
       "      <th>word_total</th>\n",
       "      <th>word_share</th>\n",
       "      <th>cwc_min</th>\n",
       "      <th>cwc_max</th>\n",
       "      <th>...</th>\n",
       "      <th>ctc_max</th>\n",
       "      <th>last_word_eq</th>\n",
       "      <th>first_word_eq</th>\n",
       "      <th>abs_len_diff</th>\n",
       "      <th>mean_len</th>\n",
       "      <th>longest_substr_ratio</th>\n",
       "      <th>fuzz_ratio</th>\n",
       "      <th>fuzz_partial_ratio</th>\n",
       "      <th>token_sort_ratio</th>\n",
       "      <th>token_set_ratio</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>356118</th>\n",
       "      <td>0</td>\n",
       "      <td>91</td>\n",
       "      <td>86</td>\n",
       "      <td>13</td>\n",
       "      <td>14</td>\n",
       "      <td>1</td>\n",
       "      <td>27</td>\n",
       "      <td>0.04</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.071428</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>13.5</td>\n",
       "      <td>0.057471</td>\n",
       "      <td>21</td>\n",
       "      <td>29</td>\n",
       "      <td>38</td>\n",
       "      <td>28</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>309345</th>\n",
       "      <td>0</td>\n",
       "      <td>22</td>\n",
       "      <td>78</td>\n",
       "      <td>4</td>\n",
       "      <td>13</td>\n",
       "      <td>0</td>\n",
       "      <td>17</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>9</td>\n",
       "      <td>8.5</td>\n",
       "      <td>0.130435</td>\n",
       "      <td>16</td>\n",
       "      <td>36</td>\n",
       "      <td>22</td>\n",
       "      <td>22</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>213099</th>\n",
       "      <td>1</td>\n",
       "      <td>69</td>\n",
       "      <td>117</td>\n",
       "      <td>12</td>\n",
       "      <td>27</td>\n",
       "      <td>8</td>\n",
       "      <td>37</td>\n",
       "      <td>0.22</td>\n",
       "      <td>0.571420</td>\n",
       "      <td>0.333331</td>\n",
       "      <td>...</td>\n",
       "      <td>0.319999</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>13</td>\n",
       "      <td>18.5</td>\n",
       "      <td>0.128571</td>\n",
       "      <td>32</td>\n",
       "      <td>45</td>\n",
       "      <td>52</td>\n",
       "      <td>78</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>122902</th>\n",
       "      <td>0</td>\n",
       "      <td>28</td>\n",
       "      <td>27</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>4</td>\n",
       "      <td>10</td>\n",
       "      <td>0.40</td>\n",
       "      <td>0.999950</td>\n",
       "      <td>0.999950</td>\n",
       "      <td>...</td>\n",
       "      <td>0.799984</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>0.821429</td>\n",
       "      <td>95</td>\n",
       "      <td>93</td>\n",
       "      <td>95</td>\n",
       "      <td>95</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>205461</th>\n",
       "      <td>0</td>\n",
       "      <td>54</td>\n",
       "      <td>39</td>\n",
       "      <td>13</td>\n",
       "      <td>10</td>\n",
       "      <td>9</td>\n",
       "      <td>23</td>\n",
       "      <td>0.39</td>\n",
       "      <td>0.799984</td>\n",
       "      <td>0.666656</td>\n",
       "      <td>...</td>\n",
       "      <td>0.692302</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>11.5</td>\n",
       "      <td>0.600000</td>\n",
       "      <td>80</td>\n",
       "      <td>79</td>\n",
       "      <td>80</td>\n",
       "      <td>96</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 23 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        is_duplicate  q1_len  q2_len  q1_num_words  q2_num_words  word_common  \\\n",
       "356118             0      91      86            13            14            1   \n",
       "309345             0      22      78             4            13            0   \n",
       "213099             1      69     117            12            27            8   \n",
       "122902             0      28      27             5             5            4   \n",
       "205461             0      54      39            13            10            9   \n",
       "\n",
       "        word_total  word_share   cwc_min   cwc_max  ...   ctc_max  \\\n",
       "356118          27        0.04  0.000000  0.000000  ...  0.071428   \n",
       "309345          17        0.00  0.000000  0.000000  ...  0.000000   \n",
       "213099          37        0.22  0.571420  0.333331  ...  0.319999   \n",
       "122902          10        0.40  0.999950  0.999950  ...  0.799984   \n",
       "205461          23        0.39  0.799984  0.666656  ...  0.692302   \n",
       "\n",
       "        last_word_eq  first_word_eq  abs_len_diff  mean_len  \\\n",
       "356118             0              0             1      13.5   \n",
       "309345             0              0             9       8.5   \n",
       "213099             0              0            13      18.5   \n",
       "122902             1              0             0       5.0   \n",
       "205461             0              1             3      11.5   \n",
       "\n",
       "        longest_substr_ratio  fuzz_ratio  fuzz_partial_ratio  \\\n",
       "356118              0.057471          21                  29   \n",
       "309345              0.130435          16                  36   \n",
       "213099              0.128571          32                  45   \n",
       "122902              0.821429          95                  93   \n",
       "205461              0.600000          80                  79   \n",
       "\n",
       "        token_sort_ratio  token_set_ratio  \n",
       "356118                38               28  \n",
       "309345                22               22  \n",
       "213099                52               78  \n",
       "122902                95               95  \n",
       "205461                80               96  \n",
       "\n",
       "[5 rows x 23 columns]"
      ]
     },
     "execution_count": 89,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "final_df = new_df.drop(columns=['id','qid1','qid2','question1','question2'])\n",
    "print(final_df.shape)\n",
    "final_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "id": "90a9a865",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "# merge texts\n",
    "questions = list(ques_df['question1']) + list(ques_df['question2'])\n",
    "\n",
    "cv = CountVectorizer(max_features=3000)\n",
    "q1_arr, q2_arr = np.vsplit(cv.fit_transform(questions).toarray(),2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "id": "5ab6270b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(300, 3444)"
      ]
     },
     "execution_count": 91,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "temp_df1 = pd.DataFrame(q1_arr, index= ques_df.index)\n",
    "temp_df2 = pd.DataFrame(q2_arr, index= ques_df.index)\n",
    "temp_df = pd.concat([temp_df1, temp_df2], axis=1)\n",
    "temp_df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "id": "720affe0",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(300, 3467)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>is_duplicate</th>\n",
       "      <th>q1_len</th>\n",
       "      <th>q2_len</th>\n",
       "      <th>q1_num_words</th>\n",
       "      <th>q2_num_words</th>\n",
       "      <th>word_common</th>\n",
       "      <th>word_total</th>\n",
       "      <th>word_share</th>\n",
       "      <th>cwc_min</th>\n",
       "      <th>cwc_max</th>\n",
       "      <th>...</th>\n",
       "      <th>1712</th>\n",
       "      <th>1713</th>\n",
       "      <th>1714</th>\n",
       "      <th>1715</th>\n",
       "      <th>1716</th>\n",
       "      <th>1717</th>\n",
       "      <th>1718</th>\n",
       "      <th>1719</th>\n",
       "      <th>1720</th>\n",
       "      <th>1721</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>356118</th>\n",
       "      <td>0</td>\n",
       "      <td>91</td>\n",
       "      <td>86</td>\n",
       "      <td>13</td>\n",
       "      <td>14</td>\n",
       "      <td>1</td>\n",
       "      <td>27</td>\n",
       "      <td>0.04</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>309345</th>\n",
       "      <td>0</td>\n",
       "      <td>22</td>\n",
       "      <td>78</td>\n",
       "      <td>4</td>\n",
       "      <td>13</td>\n",
       "      <td>0</td>\n",
       "      <td>17</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>213099</th>\n",
       "      <td>1</td>\n",
       "      <td>69</td>\n",
       "      <td>117</td>\n",
       "      <td>12</td>\n",
       "      <td>27</td>\n",
       "      <td>8</td>\n",
       "      <td>37</td>\n",
       "      <td>0.22</td>\n",
       "      <td>0.571420</td>\n",
       "      <td>0.333331</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>122902</th>\n",
       "      <td>0</td>\n",
       "      <td>28</td>\n",
       "      <td>27</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>4</td>\n",
       "      <td>10</td>\n",
       "      <td>0.40</td>\n",
       "      <td>0.999950</td>\n",
       "      <td>0.999950</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>205461</th>\n",
       "      <td>0</td>\n",
       "      <td>54</td>\n",
       "      <td>39</td>\n",
       "      <td>13</td>\n",
       "      <td>10</td>\n",
       "      <td>9</td>\n",
       "      <td>23</td>\n",
       "      <td>0.39</td>\n",
       "      <td>0.799984</td>\n",
       "      <td>0.666656</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 3467 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        is_duplicate  q1_len  q2_len  q1_num_words  q2_num_words  word_common  \\\n",
       "356118             0      91      86            13            14            1   \n",
       "309345             0      22      78             4            13            0   \n",
       "213099             1      69     117            12            27            8   \n",
       "122902             0      28      27             5             5            4   \n",
       "205461             0      54      39            13            10            9   \n",
       "\n",
       "        word_total  word_share   cwc_min   cwc_max  ...  1712  1713  1714  \\\n",
       "356118          27        0.04  0.000000  0.000000  ...     0     0     0   \n",
       "309345          17        0.00  0.000000  0.000000  ...     1     0     0   \n",
       "213099          37        0.22  0.571420  0.333331  ...     0     0     0   \n",
       "122902          10        0.40  0.999950  0.999950  ...     0     0     0   \n",
       "205461          23        0.39  0.799984  0.666656  ...     0     0     0   \n",
       "\n",
       "        1715  1716  1717  1718  1719  1720  1721  \n",
       "356118     0     0     0     0     0     0     0  \n",
       "309345     0     0     0     0     0     0     0  \n",
       "213099     0     0     0     0     0     0     0  \n",
       "122902     0     0     0     0     0     0     0  \n",
       "205461     0     0     0     0     0     0     0  \n",
       "\n",
       "[5 rows x 3467 columns]"
      ]
     },
     "execution_count": 92,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "final_df = pd.concat([final_df, temp_df], axis=1)\n",
    "print(final_df.shape)\n",
    "final_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "id": "eb55a87d",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "X_train,X_test,y_train,y_test = train_test_split(final_df.iloc[:,1:].values,final_df.iloc[:,0].values,test_size=0.2,random_state=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "id": "818b01cf",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.65"
      ]
     },
     "execution_count": 94,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.metrics import accuracy_score\n",
    "rf = RandomForestClassifier()\n",
    "rf.fit(X_train,y_train)\n",
    "y_pred = rf.predict(X_test)\n",
    "accuracy_score(y_test,y_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "id": "e6cd3dee",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.6"
      ]
     },
     "execution_count": 95,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from xgboost import XGBClassifier\n",
    "xgb = XGBClassifier()\n",
    "xgb.fit(X_train,y_train)\n",
    "y_pred1 = xgb.predict(X_test)\n",
    "accuracy_score(y_test,y_pred1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "id": "a5ae43d1",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import confusion_matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "id": "330fdfd3",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[26, 19],\n",
       "       [ 2, 13]], dtype=int64)"
      ]
     },
     "execution_count": 97,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# for random forest model\n",
    "confusion_matrix(y_test,y_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "id": "714ce171",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[28, 17],\n",
       "       [ 7,  8]], dtype=int64)"
      ]
     },
     "execution_count": 98,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# for xgboost model\n",
    "confusion_matrix(y_test,y_pred1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "77f5819c",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "id": "e8d4ec10",
   "metadata": {},
   "outputs": [],
   "source": [
    "def test_common_words(q1,q2):\n",
    "    w1 = set(map(lambda word: word.lower().strip(), q1.split(\" \")))\n",
    "    w2 = set(map(lambda word: word.lower().strip(), q2.split(\" \")))    \n",
    "    return len(w1 & w2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "id": "94e26a13",
   "metadata": {},
   "outputs": [],
   "source": [
    "def test_total_words(q1,q2):\n",
    "    w1 = set(map(lambda word: word.lower().strip(), q1.split(\" \")))\n",
    "    w2 = set(map(lambda word: word.lower().strip(), q2.split(\" \")))    \n",
    "    return (len(w1) + len(w2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "id": "e5638e42",
   "metadata": {},
   "outputs": [],
   "source": [
    "def test_fetch_token_features(q1,q2):\n",
    "    \n",
    "    SAFE_DIV = 0.0001 \n",
    "\n",
    "    STOP_WORDS = stopwords.words(\"english\")\n",
    "    \n",
    "    token_features = [0.0]*8\n",
    "    \n",
    "    # Converting the Sentence into Tokens: \n",
    "    q1_tokens = q1.split()\n",
    "    q2_tokens = q2.split()\n",
    "    \n",
    "    if len(q1_tokens) == 0 or len(q2_tokens) == 0:\n",
    "        return token_features\n",
    "\n",
    "    # Get the non-stopwords in Questions\n",
    "    q1_words = set([word for word in q1_tokens if word not in STOP_WORDS])\n",
    "    q2_words = set([word for word in q2_tokens if word not in STOP_WORDS])\n",
    "    \n",
    "    #Get the stopwords in Questions\n",
    "    q1_stops = set([word for word in q1_tokens if word in STOP_WORDS])\n",
    "    q2_stops = set([word for word in q2_tokens if word in STOP_WORDS])\n",
    "    \n",
    "    # Get the common non-stopwords from Question pair\n",
    "    common_word_count = len(q1_words.intersection(q2_words))\n",
    "    \n",
    "    # Get the common stopwords from Question pair\n",
    "    common_stop_count = len(q1_stops.intersection(q2_stops))\n",
    "    \n",
    "    # Get the common Tokens from Question pair\n",
    "    common_token_count = len(set(q1_tokens).intersection(set(q2_tokens)))\n",
    "    \n",
    "    \n",
    "    token_features[0] = common_word_count / (min(len(q1_words), len(q2_words)) + SAFE_DIV)\n",
    "    token_features[1] = common_word_count / (max(len(q1_words), len(q2_words)) + SAFE_DIV)\n",
    "    token_features[2] = common_stop_count / (min(len(q1_stops), len(q2_stops)) + SAFE_DIV)\n",
    "    token_features[3] = common_stop_count / (max(len(q1_stops), len(q2_stops)) + SAFE_DIV)\n",
    "    token_features[4] = common_token_count / (min(len(q1_tokens), len(q2_tokens)) + SAFE_DIV)\n",
    "    token_features[5] = common_token_count / (max(len(q1_tokens), len(q2_tokens)) + SAFE_DIV)\n",
    "    \n",
    "    # Last word of both question is same or not\n",
    "    token_features[6] = int(q1_tokens[-1] == q2_tokens[-1])\n",
    "    \n",
    "    # First word of both question is same or not\n",
    "    token_features[7] = int(q1_tokens[0] == q2_tokens[0])\n",
    "    \n",
    "    return token_features\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "id": "43bc0a8e",
   "metadata": {},
   "outputs": [],
   "source": [
    "def test_fetch_length_features(q1,q2):\n",
    "    \n",
    "    length_features = [0.0]*3\n",
    "    \n",
    "    # Converting the Sentence into Tokens: \n",
    "    q1_tokens = q1.split()\n",
    "    q2_tokens = q2.split()\n",
    "    \n",
    "    if len(q1_tokens) == 0 or len(q2_tokens) == 0:\n",
    "        return length_features\n",
    "    \n",
    "    # Absolute length features\n",
    "    length_features[0] = abs(len(q1_tokens) - len(q2_tokens))\n",
    "    \n",
    "    #Average Token Length of both Questions\n",
    "    length_features[1] = (len(q1_tokens) + len(q2_tokens))/2\n",
    "    \n",
    "    strs = list(distance.lcsubstrings(q1, q2))\n",
    "    length_features[2] = len(strs[0]) / (min(len(q1), len(q2)) + 1)\n",
    "    \n",
    "    return length_features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "id": "7ebd7a32",
   "metadata": {},
   "outputs": [],
   "source": [
    "def test_fetch_fuzzy_features(q1,q2):\n",
    "    \n",
    "    fuzzy_features = [0.0]*4\n",
    "    \n",
    "    # fuzz_ratio\n",
    "    fuzzy_features[0] = fuzz.QRatio(q1, q2)\n",
    "\n",
    "    # fuzz_partial_ratio\n",
    "    fuzzy_features[1] = fuzz.partial_ratio(q1, q2)\n",
    "\n",
    "    # token_sort_ratio\n",
    "    fuzzy_features[2] = fuzz.token_sort_ratio(q1, q2)\n",
    "\n",
    "    # token_set_ratio\n",
    "    fuzzy_features[3] = fuzz.token_set_ratio(q1, q2)\n",
    "\n",
    "    return fuzzy_features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 104,
   "id": "d4429e08",
   "metadata": {},
   "outputs": [],
   "source": [
    "def query_point_creator(q1,q2):\n",
    "    \n",
    "    input_query = []\n",
    "    \n",
    "    # preprocess\n",
    "    q1 = preprocess(q1)\n",
    "    q2 = preprocess(q2)\n",
    "    \n",
    "    # fetch basic features\n",
    "    input_query.append(len(q1))\n",
    "    input_query.append(len(q2))\n",
    "    \n",
    "    input_query.append(len(q1.split(\" \")))\n",
    "    input_query.append(len(q2.split(\" \")))\n",
    "    \n",
    "    input_query.append(test_common_words(q1,q2))\n",
    "    input_query.append(test_total_words(q1,q2))\n",
    "    input_query.append(round(test_common_words(q1,q2)/test_total_words(q1,q2),2))\n",
    "    \n",
    "    # fetch token features\n",
    "    token_features = test_fetch_token_features(q1,q2)\n",
    "    input_query.extend(token_features)\n",
    "    \n",
    "    # fetch length based features\n",
    "    length_features = test_fetch_length_features(q1,q2)\n",
    "    input_query.extend(length_features)\n",
    "    \n",
    "    # fetch fuzzy features\n",
    "    fuzzy_features = test_fetch_fuzzy_features(q1,q2)\n",
    "    input_query.extend(fuzzy_features)\n",
    "    \n",
    "    # bow feature for q1\n",
    "    q1_bow = cv.transform([q1]).toarray()\n",
    "    \n",
    "    # bow feature for q2\n",
    "    q2_bow = cv.transform([q2]).toarray()\n",
    "    \n",
    "    \n",
    "    \n",
    "    return np.hstack((np.array(input_query).reshape(1,22),q1_bow,q2_bow))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "id": "ade730fe",
   "metadata": {},
   "outputs": [],
   "source": [
    "q1 = 'Where is the capital of India?'\n",
    "q2 = 'What is the current capital of Pakistan?'\n",
    "q3 = 'Which city serves as the capital of India?'\n",
    "q4 = 'What is the business capital of India?'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 106,
   "id": "2f2ef475",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([1], dtype=int64)"
      ]
     },
     "execution_count": 106,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rf.predict(query_point_creator(q1,q4))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 107,
   "id": "58853925",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style>#sk-container-id-1 {\n",
       "  /* Definition of color scheme common for light and dark mode */\n",
       "  --sklearn-color-text: black;\n",
       "  --sklearn-color-line: gray;\n",
       "  /* Definition of color scheme for unfitted estimators */\n",
       "  --sklearn-color-unfitted-level-0: #fff5e6;\n",
       "  --sklearn-color-unfitted-level-1: #f6e4d2;\n",
       "  --sklearn-color-unfitted-level-2: #ffe0b3;\n",
       "  --sklearn-color-unfitted-level-3: chocolate;\n",
       "  /* Definition of color scheme for fitted estimators */\n",
       "  --sklearn-color-fitted-level-0: #f0f8ff;\n",
       "  --sklearn-color-fitted-level-1: #d4ebff;\n",
       "  --sklearn-color-fitted-level-2: #b3dbfd;\n",
       "  --sklearn-color-fitted-level-3: cornflowerblue;\n",
       "\n",
       "  /* Specific color for light theme */\n",
       "  --sklearn-color-text-on-default-background: var(--sg-text-color, var(--theme-code-foreground, var(--jp-content-font-color1, black)));\n",
       "  --sklearn-color-background: var(--sg-background-color, var(--theme-background, var(--jp-layout-color0, white)));\n",
       "  --sklearn-color-border-box: var(--sg-text-color, var(--theme-code-foreground, var(--jp-content-font-color1, black)));\n",
       "  --sklearn-color-icon: #696969;\n",
       "\n",
       "  @media (prefers-color-scheme: dark) {\n",
       "    /* Redefinition of color scheme for dark theme */\n",
       "    --sklearn-color-text-on-default-background: var(--sg-text-color, var(--theme-code-foreground, var(--jp-content-font-color1, white)));\n",
       "    --sklearn-color-background: var(--sg-background-color, var(--theme-background, var(--jp-layout-color0, #111)));\n",
       "    --sklearn-color-border-box: var(--sg-text-color, var(--theme-code-foreground, var(--jp-content-font-color1, white)));\n",
       "    --sklearn-color-icon: #878787;\n",
       "  }\n",
       "}\n",
       "\n",
       "#sk-container-id-1 {\n",
       "  color: var(--sklearn-color-text);\n",
       "}\n",
       "\n",
       "#sk-container-id-1 pre {\n",
       "  padding: 0;\n",
       "}\n",
       "\n",
       "#sk-container-id-1 input.sk-hidden--visually {\n",
       "  border: 0;\n",
       "  clip: rect(1px 1px 1px 1px);\n",
       "  clip: rect(1px, 1px, 1px, 1px);\n",
       "  height: 1px;\n",
       "  margin: -1px;\n",
       "  overflow: hidden;\n",
       "  padding: 0;\n",
       "  position: absolute;\n",
       "  width: 1px;\n",
       "}\n",
       "\n",
       "#sk-container-id-1 div.sk-dashed-wrapped {\n",
       "  border: 1px dashed var(--sklearn-color-line);\n",
       "  margin: 0 0.4em 0.5em 0.4em;\n",
       "  box-sizing: border-box;\n",
       "  padding-bottom: 0.4em;\n",
       "  background-color: var(--sklearn-color-background);\n",
       "}\n",
       "\n",
       "#sk-container-id-1 div.sk-container {\n",
       "  /* jupyter's `normalize.less` sets `[hidden] { display: none; }`\n",
       "     but bootstrap.min.css set `[hidden] { display: none !important; }`\n",
       "     so we also need the `!important` here to be able to override the\n",
       "     default hidden behavior on the sphinx rendered scikit-learn.org.\n",
       "     See: https://github.com/scikit-learn/scikit-learn/issues/21755 */\n",
       "  display: inline-block !important;\n",
       "  position: relative;\n",
       "}\n",
       "\n",
       "#sk-container-id-1 div.sk-text-repr-fallback {\n",
       "  display: none;\n",
       "}\n",
       "\n",
       "div.sk-parallel-item,\n",
       "div.sk-serial,\n",
       "div.sk-item {\n",
       "  /* draw centered vertical line to link estimators */\n",
       "  background-image: linear-gradient(var(--sklearn-color-text-on-default-background), var(--sklearn-color-text-on-default-background));\n",
       "  background-size: 2px 100%;\n",
       "  background-repeat: no-repeat;\n",
       "  background-position: center center;\n",
       "}\n",
       "\n",
       "/* Parallel-specific style estimator block */\n",
       "\n",
       "#sk-container-id-1 div.sk-parallel-item::after {\n",
       "  content: \"\";\n",
       "  width: 100%;\n",
       "  border-bottom: 2px solid var(--sklearn-color-text-on-default-background);\n",
       "  flex-grow: 1;\n",
       "}\n",
       "\n",
       "#sk-container-id-1 div.sk-parallel {\n",
       "  display: flex;\n",
       "  align-items: stretch;\n",
       "  justify-content: center;\n",
       "  background-color: var(--sklearn-color-background);\n",
       "  position: relative;\n",
       "}\n",
       "\n",
       "#sk-container-id-1 div.sk-parallel-item {\n",
       "  display: flex;\n",
       "  flex-direction: column;\n",
       "}\n",
       "\n",
       "#sk-container-id-1 div.sk-parallel-item:first-child::after {\n",
       "  align-self: flex-end;\n",
       "  width: 50%;\n",
       "}\n",
       "\n",
       "#sk-container-id-1 div.sk-parallel-item:last-child::after {\n",
       "  align-self: flex-start;\n",
       "  width: 50%;\n",
       "}\n",
       "\n",
       "#sk-container-id-1 div.sk-parallel-item:only-child::after {\n",
       "  width: 0;\n",
       "}\n",
       "\n",
       "/* Serial-specific style estimator block */\n",
       "\n",
       "#sk-container-id-1 div.sk-serial {\n",
       "  display: flex;\n",
       "  flex-direction: column;\n",
       "  align-items: center;\n",
       "  background-color: var(--sklearn-color-background);\n",
       "  padding-right: 1em;\n",
       "  padding-left: 1em;\n",
       "}\n",
       "\n",
       "\n",
       "/* Toggleable style: style used for estimator/Pipeline/ColumnTransformer box that is\n",
       "clickable and can be expanded/collapsed.\n",
       "- Pipeline and ColumnTransformer use this feature and define the default style\n",
       "- Estimators will overwrite some part of the style using the `sk-estimator` class\n",
       "*/\n",
       "\n",
       "/* Pipeline and ColumnTransformer style (default) */\n",
       "\n",
       "#sk-container-id-1 div.sk-toggleable {\n",
       "  /* Default theme specific background. It is overwritten whether we have a\n",
       "  specific estimator or a Pipeline/ColumnTransformer */\n",
       "  background-color: var(--sklearn-color-background);\n",
       "}\n",
       "\n",
       "/* Toggleable label */\n",
       "#sk-container-id-1 label.sk-toggleable__label {\n",
       "  cursor: pointer;\n",
       "  display: block;\n",
       "  width: 100%;\n",
       "  margin-bottom: 0;\n",
       "  padding: 0.5em;\n",
       "  box-sizing: border-box;\n",
       "  text-align: center;\n",
       "}\n",
       "\n",
       "#sk-container-id-1 label.sk-toggleable__label-arrow:before {\n",
       "  /* Arrow on the left of the label */\n",
       "  content: \"▸\";\n",
       "  float: left;\n",
       "  margin-right: 0.25em;\n",
       "  color: var(--sklearn-color-icon);\n",
       "}\n",
       "\n",
       "#sk-container-id-1 label.sk-toggleable__label-arrow:hover:before {\n",
       "  color: var(--sklearn-color-text);\n",
       "}\n",
       "\n",
       "/* Toggleable content - dropdown */\n",
       "\n",
       "#sk-container-id-1 div.sk-toggleable__content {\n",
       "  max-height: 0;\n",
       "  max-width: 0;\n",
       "  overflow: hidden;\n",
       "  text-align: left;\n",
       "  /* unfitted */\n",
       "  background-color: var(--sklearn-color-unfitted-level-0);\n",
       "}\n",
       "\n",
       "#sk-container-id-1 div.sk-toggleable__content.fitted {\n",
       "  /* fitted */\n",
       "  background-color: var(--sklearn-color-fitted-level-0);\n",
       "}\n",
       "\n",
       "#sk-container-id-1 div.sk-toggleable__content pre {\n",
       "  margin: 0.2em;\n",
       "  border-radius: 0.25em;\n",
       "  color: var(--sklearn-color-text);\n",
       "  /* unfitted */\n",
       "  background-color: var(--sklearn-color-unfitted-level-0);\n",
       "}\n",
       "\n",
       "#sk-container-id-1 div.sk-toggleable__content.fitted pre {\n",
       "  /* unfitted */\n",
       "  background-color: var(--sklearn-color-fitted-level-0);\n",
       "}\n",
       "\n",
       "#sk-container-id-1 input.sk-toggleable__control:checked~div.sk-toggleable__content {\n",
       "  /* Expand drop-down */\n",
       "  max-height: 200px;\n",
       "  max-width: 100%;\n",
       "  overflow: auto;\n",
       "}\n",
       "\n",
       "#sk-container-id-1 input.sk-toggleable__control:checked~label.sk-toggleable__label-arrow:before {\n",
       "  content: \"▾\";\n",
       "}\n",
       "\n",
       "/* Pipeline/ColumnTransformer-specific style */\n",
       "\n",
       "#sk-container-id-1 div.sk-label input.sk-toggleable__control:checked~label.sk-toggleable__label {\n",
       "  color: var(--sklearn-color-text);\n",
       "  background-color: var(--sklearn-color-unfitted-level-2);\n",
       "}\n",
       "\n",
       "#sk-container-id-1 div.sk-label.fitted input.sk-toggleable__control:checked~label.sk-toggleable__label {\n",
       "  background-color: var(--sklearn-color-fitted-level-2);\n",
       "}\n",
       "\n",
       "/* Estimator-specific style */\n",
       "\n",
       "/* Colorize estimator box */\n",
       "#sk-container-id-1 div.sk-estimator input.sk-toggleable__control:checked~label.sk-toggleable__label {\n",
       "  /* unfitted */\n",
       "  background-color: var(--sklearn-color-unfitted-level-2);\n",
       "}\n",
       "\n",
       "#sk-container-id-1 div.sk-estimator.fitted input.sk-toggleable__control:checked~label.sk-toggleable__label {\n",
       "  /* fitted */\n",
       "  background-color: var(--sklearn-color-fitted-level-2);\n",
       "}\n",
       "\n",
       "#sk-container-id-1 div.sk-label label.sk-toggleable__label,\n",
       "#sk-container-id-1 div.sk-label label {\n",
       "  /* The background is the default theme color */\n",
       "  color: var(--sklearn-color-text-on-default-background);\n",
       "}\n",
       "\n",
       "/* On hover, darken the color of the background */\n",
       "#sk-container-id-1 div.sk-label:hover label.sk-toggleable__label {\n",
       "  color: var(--sklearn-color-text);\n",
       "  background-color: var(--sklearn-color-unfitted-level-2);\n",
       "}\n",
       "\n",
       "/* Label box, darken color on hover, fitted */\n",
       "#sk-container-id-1 div.sk-label.fitted:hover label.sk-toggleable__label.fitted {\n",
       "  color: var(--sklearn-color-text);\n",
       "  background-color: var(--sklearn-color-fitted-level-2);\n",
       "}\n",
       "\n",
       "/* Estimator label */\n",
       "\n",
       "#sk-container-id-1 div.sk-label label {\n",
       "  font-family: monospace;\n",
       "  font-weight: bold;\n",
       "  display: inline-block;\n",
       "  line-height: 1.2em;\n",
       "}\n",
       "\n",
       "#sk-container-id-1 div.sk-label-container {\n",
       "  text-align: center;\n",
       "}\n",
       "\n",
       "/* Estimator-specific */\n",
       "#sk-container-id-1 div.sk-estimator {\n",
       "  font-family: monospace;\n",
       "  border: 1px dotted var(--sklearn-color-border-box);\n",
       "  border-radius: 0.25em;\n",
       "  box-sizing: border-box;\n",
       "  margin-bottom: 0.5em;\n",
       "  /* unfitted */\n",
       "  background-color: var(--sklearn-color-unfitted-level-0);\n",
       "}\n",
       "\n",
       "#sk-container-id-1 div.sk-estimator.fitted {\n",
       "  /* fitted */\n",
       "  background-color: var(--sklearn-color-fitted-level-0);\n",
       "}\n",
       "\n",
       "/* on hover */\n",
       "#sk-container-id-1 div.sk-estimator:hover {\n",
       "  /* unfitted */\n",
       "  background-color: var(--sklearn-color-unfitted-level-2);\n",
       "}\n",
       "\n",
       "#sk-container-id-1 div.sk-estimator.fitted:hover {\n",
       "  /* fitted */\n",
       "  background-color: var(--sklearn-color-fitted-level-2);\n",
       "}\n",
       "\n",
       "/* Specification for estimator info (e.g. \"i\" and \"?\") */\n",
       "\n",
       "/* Common style for \"i\" and \"?\" */\n",
       "\n",
       ".sk-estimator-doc-link,\n",
       "a:link.sk-estimator-doc-link,\n",
       "a:visited.sk-estimator-doc-link {\n",
       "  float: right;\n",
       "  font-size: smaller;\n",
       "  line-height: 1em;\n",
       "  font-family: monospace;\n",
       "  background-color: var(--sklearn-color-background);\n",
       "  border-radius: 1em;\n",
       "  height: 1em;\n",
       "  width: 1em;\n",
       "  text-decoration: none !important;\n",
       "  margin-left: 1ex;\n",
       "  /* unfitted */\n",
       "  border: var(--sklearn-color-unfitted-level-1) 1pt solid;\n",
       "  color: var(--sklearn-color-unfitted-level-1);\n",
       "}\n",
       "\n",
       ".sk-estimator-doc-link.fitted,\n",
       "a:link.sk-estimator-doc-link.fitted,\n",
       "a:visited.sk-estimator-doc-link.fitted {\n",
       "  /* fitted */\n",
       "  border: var(--sklearn-color-fitted-level-1) 1pt solid;\n",
       "  color: var(--sklearn-color-fitted-level-1);\n",
       "}\n",
       "\n",
       "/* On hover */\n",
       "div.sk-estimator:hover .sk-estimator-doc-link:hover,\n",
       ".sk-estimator-doc-link:hover,\n",
       "div.sk-label-container:hover .sk-estimator-doc-link:hover,\n",
       ".sk-estimator-doc-link:hover {\n",
       "  /* unfitted */\n",
       "  background-color: var(--sklearn-color-unfitted-level-3);\n",
       "  color: var(--sklearn-color-background);\n",
       "  text-decoration: none;\n",
       "}\n",
       "\n",
       "div.sk-estimator.fitted:hover .sk-estimator-doc-link.fitted:hover,\n",
       ".sk-estimator-doc-link.fitted:hover,\n",
       "div.sk-label-container:hover .sk-estimator-doc-link.fitted:hover,\n",
       ".sk-estimator-doc-link.fitted:hover {\n",
       "  /* fitted */\n",
       "  background-color: var(--sklearn-color-fitted-level-3);\n",
       "  color: var(--sklearn-color-background);\n",
       "  text-decoration: none;\n",
       "}\n",
       "\n",
       "/* Span, style for the box shown on hovering the info icon */\n",
       ".sk-estimator-doc-link span {\n",
       "  display: none;\n",
       "  z-index: 9999;\n",
       "  position: relative;\n",
       "  font-weight: normal;\n",
       "  right: .2ex;\n",
       "  padding: .5ex;\n",
       "  margin: .5ex;\n",
       "  width: min-content;\n",
       "  min-width: 20ex;\n",
       "  max-width: 50ex;\n",
       "  color: var(--sklearn-color-text);\n",
       "  box-shadow: 2pt 2pt 4pt #999;\n",
       "  /* unfitted */\n",
       "  background: var(--sklearn-color-unfitted-level-0);\n",
       "  border: .5pt solid var(--sklearn-color-unfitted-level-3);\n",
       "}\n",
       "\n",
       ".sk-estimator-doc-link.fitted span {\n",
       "  /* fitted */\n",
       "  background: var(--sklearn-color-fitted-level-0);\n",
       "  border: var(--sklearn-color-fitted-level-3);\n",
       "}\n",
       "\n",
       ".sk-estimator-doc-link:hover span {\n",
       "  display: block;\n",
       "}\n",
       "\n",
       "/* \"?\"-specific style due to the `<a>` HTML tag */\n",
       "\n",
       "#sk-container-id-1 a.estimator_doc_link {\n",
       "  float: right;\n",
       "  font-size: 1rem;\n",
       "  line-height: 1em;\n",
       "  font-family: monospace;\n",
       "  background-color: var(--sklearn-color-background);\n",
       "  border-radius: 1rem;\n",
       "  height: 1rem;\n",
       "  width: 1rem;\n",
       "  text-decoration: none;\n",
       "  /* unfitted */\n",
       "  color: var(--sklearn-color-unfitted-level-1);\n",
       "  border: var(--sklearn-color-unfitted-level-1) 1pt solid;\n",
       "}\n",
       "\n",
       "#sk-container-id-1 a.estimator_doc_link.fitted {\n",
       "  /* fitted */\n",
       "  border: var(--sklearn-color-fitted-level-1) 1pt solid;\n",
       "  color: var(--sklearn-color-fitted-level-1);\n",
       "}\n",
       "\n",
       "/* On hover */\n",
       "#sk-container-id-1 a.estimator_doc_link:hover {\n",
       "  /* unfitted */\n",
       "  background-color: var(--sklearn-color-unfitted-level-3);\n",
       "  color: var(--sklearn-color-background);\n",
       "  text-decoration: none;\n",
       "}\n",
       "\n",
       "#sk-container-id-1 a.estimator_doc_link.fitted:hover {\n",
       "  /* fitted */\n",
       "  background-color: var(--sklearn-color-fitted-level-3);\n",
       "}\n",
       "</style><div id=\"sk-container-id-1\" class=\"sk-top-container\"><div class=\"sk-text-repr-fallback\"><pre>CountVectorizer(max_features=3000)</pre><b>In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook. <br />On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.</b></div><div class=\"sk-container\" hidden><div class=\"sk-item\"><div class=\"sk-estimator fitted sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-1\" type=\"checkbox\" checked><label for=\"sk-estimator-id-1\" class=\"sk-toggleable__label fitted sk-toggleable__label-arrow fitted\">&nbsp;&nbsp;CountVectorizer<a class=\"sk-estimator-doc-link fitted\" rel=\"noreferrer\" target=\"_blank\" href=\"https://scikit-learn.org/1.4/modules/generated/sklearn.feature_extraction.text.CountVectorizer.html\">?<span>Documentation for CountVectorizer</span></a><span class=\"sk-estimator-doc-link fitted\">i<span>Fitted</span></span></label><div class=\"sk-toggleable__content fitted\"><pre>CountVectorizer(max_features=3000)</pre></div> </div></div></div></div>"
      ],
      "text/plain": [
       "CountVectorizer(max_features=3000)"
      ]
     },
     "execution_count": 107,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 108,
   "id": "70af166f",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pickle\n",
    "\n",
    "pickle.dump(rf,open('model.pkl','wb'))\n",
    "pickle.dump(cv,open('cv.pkl','wb'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e2612cc9",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}