mark2.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "ae070720",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import warnings\n",
    "warnings.filterwarnings('ignore')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "0b74440e",
   "metadata": {},
   "outputs": [],
   "source": [
    "test_df = pd.read_csv('Doceree-HCP_Test.csv', encoding='latin1')\n",
    "train_df = pd.read_csv('Doceree-HCP_Train.csv', encoding='latin1')\n",
    "#SAMPLE_df = pd.read_csv('Doceree_HCP_Sample_Submission_1.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "6259db1b",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ID</th>\n",
       "      <th>DEVICETYPE</th>\n",
       "      <th>PLATFORM_ID</th>\n",
       "      <th>BIDREQUESTIP</th>\n",
       "      <th>USERPLATFORMUID</th>\n",
       "      <th>USERCITY</th>\n",
       "      <th>USERZIPCODE</th>\n",
       "      <th>USERAGENT</th>\n",
       "      <th>PLATFORMTYPE</th>\n",
       "      <th>CHANNELTYPE</th>\n",
       "      <th>URL</th>\n",
       "      <th>KEYWORDS</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>115501</td>\n",
       "      <td>Desktop</td>\n",
       "      <td>2</td>\n",
       "      <td>75.189.231.103</td>\n",
       "      <td>0d5041ff-f0b6-4d1a-9ad7-0a29f7d485b4</td>\n",
       "      <td>Fayetteville</td>\n",
       "      <td>28305.0</td>\n",
       "      <td>Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_6...</td>\n",
       "      <td>Online Medical Journal</td>\n",
       "      <td>Website</td>\n",
       "      <td>https://www.clinicaladvisor.com/home/features/...</td>\n",
       "      <td>Family Practice|Drainage|Clinical|Dermatology|...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>115502</td>\n",
       "      <td>Mobile</td>\n",
       "      <td>2</td>\n",
       "      <td>24.101.33.158</td>\n",
       "      <td>c8396dd0-969f-4d99-a40b-b7bb1f516154</td>\n",
       "      <td>Conneaut Lake</td>\n",
       "      <td>16316.0</td>\n",
       "      <td>Mozilla/5.0 (iPhone; CPU iPhone OS 15_6_1 like...</td>\n",
       "      <td>Online Medical Journal</td>\n",
       "      <td>Website</td>\n",
       "      <td>https://www.ophthalmologyadvisor.com/topics/ca...</td>\n",
       "      <td>General|Clinical|Operative|Medicine|Cardiology...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>115503</td>\n",
       "      <td>Desktop</td>\n",
       "      <td>2</td>\n",
       "      <td>172.118.216.142</td>\n",
       "      <td>3c97a081-6518-43f8-9f26-369759cfb471</td>\n",
       "      <td>Covina</td>\n",
       "      <td>91724.0</td>\n",
       "      <td>Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7...</td>\n",
       "      <td>Online Medical Journal</td>\n",
       "      <td>Website</td>\n",
       "      <td>https://www.psychiatryadvisor.com/author/tori-...</td>\n",
       "      <td>Abortion|Anxiety Disorders|Apnea|False|Trauma|...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>115504</td>\n",
       "      <td>Desktop</td>\n",
       "      <td>7</td>\n",
       "      <td>71.105.120.171</td>\n",
       "      <td>3e2578c8-f794-41af-a38c-c5cfb3c0f014</td>\n",
       "      <td>Brooklyn</td>\n",
       "      <td>11226.0</td>\n",
       "      <td>Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7...</td>\n",
       "      <td>Online Medical Journal</td>\n",
       "      <td>Website</td>\n",
       "      <td>https://www.cureus.com/articles/105482-diverti...</td>\n",
       "      <td>Health|Male|Neurological Surgery|Otolaryngolog...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>115505</td>\n",
       "      <td>Desktop</td>\n",
       "      <td>2</td>\n",
       "      <td>73.82.211.73</td>\n",
       "      <td>ec2ae7ce-6a8c-4156-98a7-07203e60f483</td>\n",
       "      <td>Marietta</td>\n",
       "      <td>30062.0</td>\n",
       "      <td>Mozilla/5.0 (Windows NT 10.0; Win64; x64) Appl...</td>\n",
       "      <td>Online Medical Journal</td>\n",
       "      <td>Website</td>\n",
       "      <td>https://www.renalandurologynews.com/home/confe...</td>\n",
       "      <td>chronic kidney disease|pain|nephrology|disease...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>115506</td>\n",
       "      <td>Desktop</td>\n",
       "      <td>2</td>\n",
       "      <td>169.147.254.129</td>\n",
       "      <td>0313febc-12e6-4289-8d74-5f0c1aecc385</td>\n",
       "      <td>Kansas City</td>\n",
       "      <td>64113.0</td>\n",
       "      <td>Mozilla/5.0 (Windows NT 10.0; Win64; x64) Appl...</td>\n",
       "      <td>Online Medical Journal</td>\n",
       "      <td>Website</td>\n",
       "      <td>https://www.renalandurologynews.com/home/decis...</td>\n",
       "      <td>Abscess|Liver Transplantation|Transplantation|...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>115507</td>\n",
       "      <td>Desktop</td>\n",
       "      <td>9</td>\n",
       "      <td>66.249.66.2</td>\n",
       "      <td>ce1f2f66-056e-4684-8c24-6d70e632a269</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Bu...</td>\n",
       "      <td>Online Medical Journal</td>\n",
       "      <td>Website</td>\n",
       "      <td>https://www.2minutemedicine.com/studygraphics-...</td>\n",
       "      <td>the|medicine|oncology|</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>115508</td>\n",
       "      <td>Desktop</td>\n",
       "      <td>2</td>\n",
       "      <td>50.236.245.242</td>\n",
       "      <td>8fc0a64c-0c71-4888-aff4-833823802bba</td>\n",
       "      <td>Seattle</td>\n",
       "      <td>98188.0</td>\n",
       "      <td>Mozilla/5.0 (Windows NT 10.0; Win64; x64) Appl...</td>\n",
       "      <td>Online Medical Journal</td>\n",
       "      <td>Website</td>\n",
       "      <td>https://www.cancertherapyadvisor.com/home/canc...</td>\n",
       "      <td>General|Endocrine|False|Medicine|Surgery|Urolo...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>115509</td>\n",
       "      <td>Desktop</td>\n",
       "      <td>2</td>\n",
       "      <td>204.99.118.10</td>\n",
       "      <td>12ec92c0-a55b-45a2-8c4e-4237dcfc8eef</td>\n",
       "      <td>Scottsdale</td>\n",
       "      <td>85260.0</td>\n",
       "      <td>Mozilla/5.0 (Windows NT 10.0; Win64; x64) Appl...</td>\n",
       "      <td>Online Medical Journal</td>\n",
       "      <td>Website</td>\n",
       "      <td>https://www.empr.com/drug/evrysdi/</td>\n",
       "      <td>Medicine|Metabolic|Dermatology|Child|False|Mal...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>115510</td>\n",
       "      <td>Mobile</td>\n",
       "      <td>2</td>\n",
       "      <td>71.228.67.64</td>\n",
       "      <td>d0a2d8ab-6ea1-440e-a2f1-03acd6a1db99</td>\n",
       "      <td>Sarasota</td>\n",
       "      <td>34232.0</td>\n",
       "      <td>Mozilla/5.0 (iPhone; CPU iPhone OS 16_3_1 like...</td>\n",
       "      <td>Online Medical Journal</td>\n",
       "      <td>Website</td>\n",
       "      <td>https://www.clinicaladvisor.com/home/dermatolo...</td>\n",
       "      <td>Hemoptysis|Oral|Rheumatology|Cardiovascular|Sm...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       ID DEVICETYPE  PLATFORM_ID     BIDREQUESTIP  \\\n",
       "0  115501    Desktop            2   75.189.231.103   \n",
       "1  115502     Mobile            2    24.101.33.158   \n",
       "2  115503    Desktop            2  172.118.216.142   \n",
       "3  115504    Desktop            7   71.105.120.171   \n",
       "4  115505    Desktop            2     73.82.211.73   \n",
       "5  115506    Desktop            2  169.147.254.129   \n",
       "6  115507    Desktop            9      66.249.66.2   \n",
       "7  115508    Desktop            2   50.236.245.242   \n",
       "8  115509    Desktop            2    204.99.118.10   \n",
       "9  115510     Mobile            2     71.228.67.64   \n",
       "\n",
       "                        USERPLATFORMUID       USERCITY  USERZIPCODE  \\\n",
       "0  0d5041ff-f0b6-4d1a-9ad7-0a29f7d485b4   Fayetteville      28305.0   \n",
       "1  c8396dd0-969f-4d99-a40b-b7bb1f516154  Conneaut Lake      16316.0   \n",
       "2  3c97a081-6518-43f8-9f26-369759cfb471         Covina      91724.0   \n",
       "3  3e2578c8-f794-41af-a38c-c5cfb3c0f014       Brooklyn      11226.0   \n",
       "4  ec2ae7ce-6a8c-4156-98a7-07203e60f483       Marietta      30062.0   \n",
       "5  0313febc-12e6-4289-8d74-5f0c1aecc385    Kansas City      64113.0   \n",
       "6  ce1f2f66-056e-4684-8c24-6d70e632a269            NaN          NaN   \n",
       "7  8fc0a64c-0c71-4888-aff4-833823802bba        Seattle      98188.0   \n",
       "8  12ec92c0-a55b-45a2-8c4e-4237dcfc8eef     Scottsdale      85260.0   \n",
       "9  d0a2d8ab-6ea1-440e-a2f1-03acd6a1db99       Sarasota      34232.0   \n",
       "\n",
       "                                           USERAGENT            PLATFORMTYPE  \\\n",
       "0  Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_6...  Online Medical Journal   \n",
       "1  Mozilla/5.0 (iPhone; CPU iPhone OS 15_6_1 like...  Online Medical Journal   \n",
       "2  Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7...  Online Medical Journal   \n",
       "3  Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7...  Online Medical Journal   \n",
       "4  Mozilla/5.0 (Windows NT 10.0; Win64; x64) Appl...  Online Medical Journal   \n",
       "5  Mozilla/5.0 (Windows NT 10.0; Win64; x64) Appl...  Online Medical Journal   \n",
       "6  Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Bu...  Online Medical Journal   \n",
       "7  Mozilla/5.0 (Windows NT 10.0; Win64; x64) Appl...  Online Medical Journal   \n",
       "8  Mozilla/5.0 (Windows NT 10.0; Win64; x64) Appl...  Online Medical Journal   \n",
       "9  Mozilla/5.0 (iPhone; CPU iPhone OS 16_3_1 like...  Online Medical Journal   \n",
       "\n",
       "  CHANNELTYPE                                                URL  \\\n",
       "0     Website  https://www.clinicaladvisor.com/home/features/...   \n",
       "1     Website  https://www.ophthalmologyadvisor.com/topics/ca...   \n",
       "2     Website  https://www.psychiatryadvisor.com/author/tori-...   \n",
       "3     Website  https://www.cureus.com/articles/105482-diverti...   \n",
       "4     Website  https://www.renalandurologynews.com/home/confe...   \n",
       "5     Website  https://www.renalandurologynews.com/home/decis...   \n",
       "6     Website  https://www.2minutemedicine.com/studygraphics-...   \n",
       "7     Website  https://www.cancertherapyadvisor.com/home/canc...   \n",
       "8     Website                 https://www.empr.com/drug/evrysdi/   \n",
       "9     Website  https://www.clinicaladvisor.com/home/dermatolo...   \n",
       "\n",
       "                                            KEYWORDS  \n",
       "0  Family Practice|Drainage|Clinical|Dermatology|...  \n",
       "1  General|Clinical|Operative|Medicine|Cardiology...  \n",
       "2  Abortion|Anxiety Disorders|Apnea|False|Trauma|...  \n",
       "3  Health|Male|Neurological Surgery|Otolaryngolog...  \n",
       "4  chronic kidney disease|pain|nephrology|disease...  \n",
       "5  Abscess|Liver Transplantation|Transplantation|...  \n",
       "6                             the|medicine|oncology|  \n",
       "7  General|Endocrine|False|Medicine|Surgery|Urolo...  \n",
       "8  Medicine|Metabolic|Dermatology|Child|False|Mal...  \n",
       "9  Hemoptysis|Oral|Rheumatology|Cardiovascular|Sm...  "
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_df.head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "b6860c67",
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ID</th>\n",
       "      <th>DEVICETYPE</th>\n",
       "      <th>PLATFORM_ID</th>\n",
       "      <th>BIDREQUESTIP</th>\n",
       "      <th>USERPLATFORMUID</th>\n",
       "      <th>USERCITY</th>\n",
       "      <th>USERZIPCODE</th>\n",
       "      <th>USERAGENT</th>\n",
       "      <th>PLATFORMTYPE</th>\n",
       "      <th>CHANNELTYPE</th>\n",
       "      <th>URL</th>\n",
       "      <th>KEYWORDS</th>\n",
       "      <th>TAXONOMY</th>\n",
       "      <th>IS_HCP</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1001</td>\n",
       "      <td>Desktop</td>\n",
       "      <td>2</td>\n",
       "      <td>170.173.0.22</td>\n",
       "      <td>6974dcaa-f932-480e-9fb5-c52e20e1393a</td>\n",
       "      <td>Portland</td>\n",
       "      <td>97206.0</td>\n",
       "      <td>Mozilla/5.0 (Windows NT 10.0; Win64; x64) Appl...</td>\n",
       "      <td>Online Medical Journal</td>\n",
       "      <td>Website</td>\n",
       "      <td>https://www.cancertherapyadvisor.com/home/canc...</td>\n",
       "      <td>General|Endocrine|False|Medicine|Surgery|Urolo...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1002</td>\n",
       "      <td>Desktop</td>\n",
       "      <td>2</td>\n",
       "      <td>65.216.253.25</td>\n",
       "      <td>c12f3f8f-8fcf-484a-90e1-1ac04db8cdcf</td>\n",
       "      <td>Arlington</td>\n",
       "      <td>22202.0</td>\n",
       "      <td>Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7...</td>\n",
       "      <td>Online Medical Journal</td>\n",
       "      <td>Website</td>\n",
       "      <td>https://www.cancertherapyadvisor.com/home/deci...</td>\n",
       "      <td>Bone Marrow|Radiography|Chronic|Oncology|Psych...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1003</td>\n",
       "      <td>Desktop</td>\n",
       "      <td>2</td>\n",
       "      <td>66.232.79.22</td>\n",
       "      <td>a698de4b-e200-46dd-b5fb-40402175ae18</td>\n",
       "      <td>New Meadows</td>\n",
       "      <td>83654.0</td>\n",
       "      <td>Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7...</td>\n",
       "      <td>Online Medical Journal</td>\n",
       "      <td>Website</td>\n",
       "      <td>https://www.cancertherapyadvisor.com/home/canc...</td>\n",
       "      <td>General|Endocrine|False|Medicine|Surgery|Urolo...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1004</td>\n",
       "      <td>Desktop</td>\n",
       "      <td>3</td>\n",
       "      <td>137.54.125.246</td>\n",
       "      <td>45967533-75c8-4fbd-a00c-e6ff20447aaa</td>\n",
       "      <td>NaN</td>\n",
       "      <td>229114624.0</td>\n",
       "      <td>Mozilla/5.0 (Windows NT 10.0; Win64; x64) Appl...</td>\n",
       "      <td>Online Medical Journal</td>\n",
       "      <td>Website</td>\n",
       "      <td>https://globalrph.com/medcalcs/warfarin-mainte...</td>\n",
       "      <td>Dental|Total|Clinical|Pharmacology|Physicians|...</td>\n",
       "      <td>2084P0800X</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1005</td>\n",
       "      <td>Mobile</td>\n",
       "      <td>7</td>\n",
       "      <td>174.202.231.99</td>\n",
       "      <td>a17e25be-532d-4cf5-b916-9308c8c3961f</td>\n",
       "      <td>Houston</td>\n",
       "      <td>77008.0</td>\n",
       "      <td>Mozilla/5.0 (iPhone; CPU iPhone OS 16_1_1 like...</td>\n",
       "      <td>Online Medical Journal</td>\n",
       "      <td>Website</td>\n",
       "      <td>https://www.cureus.com/articles/58184-a-review...</td>\n",
       "      <td>Critical Care|Emergency Medicine|General Pract...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     ID DEVICETYPE  PLATFORM_ID    BIDREQUESTIP  \\\n",
       "0  1001    Desktop            2    170.173.0.22   \n",
       "1  1002    Desktop            2   65.216.253.25   \n",
       "2  1003    Desktop            2    66.232.79.22   \n",
       "3  1004    Desktop            3  137.54.125.246   \n",
       "4  1005     Mobile            7  174.202.231.99   \n",
       "\n",
       "                        USERPLATFORMUID     USERCITY  USERZIPCODE  \\\n",
       "0  6974dcaa-f932-480e-9fb5-c52e20e1393a     Portland      97206.0   \n",
       "1  c12f3f8f-8fcf-484a-90e1-1ac04db8cdcf    Arlington      22202.0   \n",
       "2  a698de4b-e200-46dd-b5fb-40402175ae18  New Meadows      83654.0   \n",
       "3  45967533-75c8-4fbd-a00c-e6ff20447aaa          NaN  229114624.0   \n",
       "4  a17e25be-532d-4cf5-b916-9308c8c3961f      Houston      77008.0   \n",
       "\n",
       "                                           USERAGENT            PLATFORMTYPE  \\\n",
       "0  Mozilla/5.0 (Windows NT 10.0; Win64; x64) Appl...  Online Medical Journal   \n",
       "1  Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7...  Online Medical Journal   \n",
       "2  Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7...  Online Medical Journal   \n",
       "3  Mozilla/5.0 (Windows NT 10.0; Win64; x64) Appl...  Online Medical Journal   \n",
       "4  Mozilla/5.0 (iPhone; CPU iPhone OS 16_1_1 like...  Online Medical Journal   \n",
       "\n",
       "  CHANNELTYPE                                                URL  \\\n",
       "0     Website  https://www.cancertherapyadvisor.com/home/canc...   \n",
       "1     Website  https://www.cancertherapyadvisor.com/home/deci...   \n",
       "2     Website  https://www.cancertherapyadvisor.com/home/canc...   \n",
       "3     Website  https://globalrph.com/medcalcs/warfarin-mainte...   \n",
       "4     Website  https://www.cureus.com/articles/58184-a-review...   \n",
       "\n",
       "                                            KEYWORDS    TAXONOMY  IS_HCP  \n",
       "0  General|Endocrine|False|Medicine|Surgery|Urolo...         NaN     0.0  \n",
       "1  Bone Marrow|Radiography|Chronic|Oncology|Psych...         NaN     0.0  \n",
       "2  General|Endocrine|False|Medicine|Surgery|Urolo...         NaN     0.0  \n",
       "3  Dental|Total|Clinical|Pharmacology|Physicians|...  2084P0800X     1.0  \n",
       "4  Critical Care|Emergency Medicine|General Pract...         NaN     0.0  "
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "7946aa79",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(113937, 14)"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "50494d76",
   "metadata": {},
   "outputs": [],
   "source": [
    "#SAMPLE_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "8d7873c2",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "ID                   int64\n",
       "DEVICETYPE          object\n",
       "PLATFORM_ID          int64\n",
       "BIDREQUESTIP        object\n",
       "USERPLATFORMUID     object\n",
       "USERCITY            object\n",
       "USERZIPCODE        float64\n",
       "USERAGENT           object\n",
       "PLATFORMTYPE        object\n",
       "CHANNELTYPE         object\n",
       "URL                 object\n",
       "KEYWORDS            object\n",
       "TAXONOMY            object\n",
       "IS_HCP             float64\n",
       "dtype: object"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_df.dtypes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "1197013a",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Online Medical Journal      110120\n",
      "Online Learning Portal        2978\n",
      "Medical News Platform          798\n",
      "Electronic Health Record        40\n",
      "Tele-medicine Platform           1\n",
      "Name: PLATFORMTYPE, dtype: int64\n"
     ]
    }
   ],
   "source": [
    "value_counts = train_df['PLATFORMTYPE'].value_counts()\n",
    "print(value_counts)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "a1c73703",
   "metadata": {},
   "outputs": [],
   "source": [
    "columns_to_drop = [\"ID\", \"DEVICETYPE\", \"PLATFORM_ID\", \"CHANNELTYPE\", \"USERAGENT\"]\n",
    "train_df.drop(columns=columns_to_drop, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "7c627484",
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "++- BIDREQUESTIP -++\n",
      "71.105.120.171     1718\n",
      "142.154.205.202    1143\n",
      "68.132.98.211       866\n",
      "108.175.114.100     727\n",
      "108.175.114.101     707\n",
      "                   ... \n",
      "66.66.9.45            1\n",
      "97.80.251.36          1\n",
      "136.32.32.117         1\n",
      "64.127.133.6          1\n",
      "108.41.233.175        1\n",
      "Name: BIDREQUESTIP, Length: 33664, dtype: int64\n",
      "=======================\n",
      "++- USERPLATFORMUID -++\n",
      "3e2578c8-f794-41af-a38c-c5cfb3c0f014    1447\n",
      "d76f7c0b-1a64-4d4b-b36d-86c832be8837    1406\n",
      "d00f28d6-3a50-46cd-92f3-8601bd57ad0e     675\n",
      "fcd3d327-0ad5-425e-8ea8-33ffe6a31543     629\n",
      "fe53f32c-4646-4abe-afea-90b21d6b8531     617\n",
      "                                        ... \n",
      "6759cbf8-cf4f-4d0e-9eb8-00808e69e841       1\n",
      "bd2007ca-d0b5-4620-8078-b7c6db62037c       1\n",
      "0534212a-e0e9-49e7-ace9-608e0d0eae8a       1\n",
      "26a06716-2716-4ac2-9131-fa0f01f84526       1\n",
      "d26887c5-15f9-4b72-80dc-728039529e60       1\n",
      "Name: USERPLATFORMUID, Length: 53780, dtype: int64\n",
      "=======================\n",
      "++- USERCITY -++\n",
      "New York          4943\n",
      "Brooklyn          3237\n",
      "St Louis          2545\n",
      "Los Angeles       2154\n",
      "Houston           1747\n",
      "                  ... \n",
      "Foothill Ranch       1\n",
      "Henrietta            1\n",
      "Wartburg             1\n",
      "Winfield             1\n",
      "Gambier              1\n",
      "Name: USERCITY, Length: 4420, dtype: int64\n",
      "=======================\n",
      "++- USERZIPCODE -++\n",
      "63169.0        2116\n",
      "11226.0        1752\n",
      "22202.0        1259\n",
      "10001.0        1190\n",
      "90060.0        1058\n",
      "               ... \n",
      "56748.0           1\n",
      "77021.0           1\n",
      "74880.0           1\n",
      "93630.0           1\n",
      "760865488.0       1\n",
      "Name: USERZIPCODE, Length: 11278, dtype: int64\n",
      "=======================\n",
      "++- PLATFORMTYPE -++\n",
      "Online Medical Journal      110120\n",
      "Online Learning Portal        2978\n",
      "Medical News Platform          798\n",
      "Electronic Health Record        40\n",
      "Tele-medicine Platform           1\n",
      "Name: PLATFORMTYPE, dtype: int64\n",
      "=======================\n",
      "++- URL -++\n",
      "https://www.clinicaladvisor.com/home/features/clinical-challenge/pain-tingling-and-limited-range-of-motion-after-a-cat-bite-to-the-finger/                                                                                                                                                                                                                                                                                                                                                                                   5310\n",
      "https://www.gastroenterologyadvisor.com/                                                                                                                                                                                                                                                                                                                                                                                                                                                                                     4299\n",
      "https://www.cureus.com/articles/105482-diverticulitis-with-microperforation                                                                                                                                                                                                                                                                                                                                                                                                                                                  2838\n",
      "https://www.clinicaladvisor.com/home/dermatologic-look-alikes/red-rashes-on-the-lower-legs/2/                                                                                                                                                                                                                                                                                                                                                                                                                                2708\n",
      "https://www.rheumatologyadvisor.com/home/general-rheumatology/deep-dive-overview-of-liver-manifestations-in-rheumatic-diseases/                                                                                                                                                                                                                                                                                                                                                                                              2587\n",
      "                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                             ... \n",
      "https://www.infectiousdiseaseadvisor.com/home/topics/covid19/mixed-results-for-paxlovid-in-standard-risk-patients-with-covid-19/?utm_source=newsletter&utm_medium=email&utm_campaign=NWSLT_IDA_UPDT_062222_JM&hmEmail=kTgLcihhhDEW1GwCOOdG13UHIqcIj%2F6O&hmSubId=&NID=1316018294&elqTrackId=e3e7472aa24848f5b776f0dfa4cae5e5&elq=e6d2fec6b6d24ee89c79463b98493739&elqaid=353&elqat=1&elqCampaignId=194                                                                                                                          1\n",
      "https://www.clinicaladvisor.com/home/meeting-coverage/acaai-2022/early-atopic-dermatitis-modifies-the-risk-for-allergic-respiratory-disease/?utm_source=newsletter&utm_medium=email&utm_campaign=NWLTR_CAD_CONF_ACAAI_ASTEPRO_112222_RM&hmEmail=Ysc25hIpkOHdmgLIaUrt9Bhfr1jEQITVgqbP1HaB5lo=&sha256email=c372454efec279dee597fd62192657913f5a5108d9d7a648a2d518ed2f4abcad&hmSubId=&NID=1003876251&elqTrackId=031b0c57d3bc455eb8736ad8693830db&elq=db6bd4eba8354842aff4c502062d50df&elqaid=7476&elqat=1&elqCampaignId=6232       1\n",
      "https://www.rheumatologyadvisor.com/home/news/early-screening-and-management-of-cardiovascular-risk-factors-important-in-psoriatic-arthritis-and-ankylosing-spondylitis/?utm_source=google&utm_medium=ppc&utm_campaign=traffic-drivers-rheum&gclid=EAIaIQobChMIpf6cxoWQ_AIVAqvICh1JwA4XEAEYASAAEgKTN_D_BwE                                                                                                                                                                                                                      1\n",
      "https://allmedx.com/q/Flu vaccine lowers risk of stroke?utm_source=Email8271&utm_medium=email&utm_campaign=Email8271&em_tag=94264\u000b\u000b\u000b                                                                                                                                                                                                                                                                                                                                                                                            1\n",
      "https://radrounds.com/radiology-case-images-teaching-file/bilothorax/?fbclid=IwAR3Yw9WM9wiMWr1a1QPTfo4ktG3z5v7TJqCASTbHHi4jNub55m23gxSvsFg                                                                                                                                                                                                                                                                                                                                                                                      1\n",
      "Name: URL, Length: 5231, dtype: int64\n",
      "=======================\n",
      "++- KEYWORDS -++\n",
      "Family Practice|Drainage|Clinical|Dermatology|Ophthalmology|Medicine|Allergic|General|Diagnosis|Neurology|Hepatology|Health|Cardiovascular|Surgical|Psychiatry|Oral|Urology|Cardiology|Gastroenterology|Orthopedics|Erythema|Arthritis|Asthma|Cellulitis|Pediatrics|Surgery|False|Oncology|Rheumatology                                                                                                                                                                                                                                                                                                                                                                                                                                          5317\n",
      "Transplantation|Technology|Chronic|Neoplasms|Urology|Pancreas|Mental Health|Arthritis|Refractory|Myocardial Infarction|Aged|Cardiology|Oncology|Surgical|Balloon|Anesthesia|Synthetic|Hepatic Transplantation|Diagnosis|Neurology|Ophthalmology|Surgery|Total|Atrial Fibrillation|Overweight|Autoimmune|Atherectomy|Heart Defects|Neonatal|Psychological|Stomach|Metabolic|Hospitals|Graves|Esophagus|Liver|Oral|Rheumatology|Insomnia|Acquired|General|Induced|Gastroenterology|Psychiatry|Angioplasty|Cardiovascular|Child|Arrhythmia|Dermatology|Liver Transplantation|Anxiety Disorders|Delivery|Flu|Small|Resuscitation|Asthma|Pathology|Medicine|Health|Clinical|False|Biliary Tract|Lung|Sleep Disorders|Arrhythmias|Bariatric Surgery    4530\n",
      "Health|Male|Neurological Surgery|Otolaryngology|Preventive Medicine|Surgery|Analgesia|Neutrophils|Resuscitation|Dermatology|Neurology|General|Intravenous|Oral|Surgical|Trauma|Orthopedics|Medicine|Monocytes|Specialties|Preventive|Anastomosis|Physical Medicine & Rehabilitation|Rheumatology|Urology|Critical Care|General Practice|Abscess|Dentistry|Small|Gastroenterology|Cardiac|Technology|Operative|Large|Emergency Medicine|Rehabilitation|Drainage|False|Anesthesiology|Oncology|Ophthalmology|Pediatrics|Transplantation|Clinical|Diagnosis|Metabolic|Pathology|Total|Cardiology|Nuclear Medicine|Psychiatry                                                                                                                        2838\n",
      "Myopathies|Small|Rheumatoid|Psychiatry|Urology|General|False|Chronic|Induced|Total|Neurology|Arthralgia|Joints|Medicine|Health|Connective Tissue Diseases|Intrahepatic|Polyarthritis|Gastroenterology|Autoimmune Diseases|Liver Dysfunction|Liver Diseases|Arthritis|Controlled|Synthetic|Clinical|Metabolic|Cardiology|Rheumatology|Liver|Rheumatic Diseases|Genetic|Autoimmune|Dermatology|Oncology|Ophthalmology                                                                                                                                                                                                                                                                                                                              2741\n",
      "Hemoptysis|Oral|Rheumatology|Cardiovascular|Small|Rheumatoid|Oncology|Ophthalmology|Clinical|Cardiology|Hemorrhage|Health|Biopsy|Hepatology|Hypersensitivity|Refractory|False|Chronic|Psychiatry|Urology|Medicine|Neutrophils|Erythema|Asthma|Diagnosis|Neurology|Orthopedics|Pediatrics|General|Dermatology|Gastroenterology|Surgery|Hematuria|Disseminated Intravascular Coagulation|Arthritis                                                                                                                                                                                                                                                                                                                                                 2708\n",
      "                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                 ... \n",
      "small|hospitals|biliary tract|thrombocytopenia|advanced|rheumatology|hepatology|oncology|angioedema|thyroid carcinoma|physicians|cystic fibrosis|pulmonary fibrosis|induced|alagille syndrome|pulmonary arterial hypertension|neurology|new|pain|myelofibrosis|disease|large|liver|vasculitis|biomarkers|multiple sclerosis|neuromyelitis optica|diagnosis|health|gastroenterology|idiopathic pulmonary fibrosis|cardiology|intrahepatic|the|hematology|clinical|carcinoma|urology|hereditary|hypertension|endocrinology|                                                                                                                                                                                                                           1\n",
      "Medicine|Gastroenterology|Health|Total|Neurology|General|Dermatology|Oral|Psychiatry|Graves|Molecular|Oncology|Graves Disease|Clinical|Ophthalmology|Rheumatology|Urology|False|Cardiology                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                          1\n",
      "biliary tract|rheumatology|stomach|psychiatry|oncology|general|total|pancreas|patient|ophthalmology|neurology|pain|disease|biologics|liver|medicine|gastroenterology|cardiology|the|controlled|hematology|clinical|dermatology|urology|esophagus|endocrinology|                                                                                                                                                                                                                                                                                                                                                                                                                                                                                     1\n",
      "chronic kidney disease|nephrology|rheumatology|psychiatry|lupus|oncology|erectile dysfunction|nephrology|nephritis|general|ophthalmology|chronic|neurology|cardiovascular disease|pain|sleep disorders|disease|large|health|medicine|gastroenterology|cardiovascular|anemia|reproductive|the|clinical|hematology|premature|dermatology|arthritis|carcinoma|transplantation|urology|urinary tract infections|diabetes|hypertension|endocrinology|                                                                                                                                                                                                                                                                                                    1\n",
      "the|lung|educational|pneumonia|radiology|pulmonary fibrosis|dyspnea|technology|pleural effusion|                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                    1\n",
      "Name: KEYWORDS, Length: 2460, dtype: int64\n",
      "=======================\n",
      "++- TAXONOMY -++\n",
      "2084P0800X    7930\n",
      "2084N0400X    6621\n",
      "207Q00000X    3997\n",
      "207R00000X    3190\n",
      "208000000X     837\n",
      "              ... \n",
      "163WC0400X       1\n",
      "246QM0706X       1\n",
      "1835N1003X       1\n",
      "207RI0008X       1\n",
      "207XX0801X       1\n",
      "Name: TAXONOMY, Length: 207, dtype: int64\n",
      "=======================\n",
      "++- IS_HCP -++\n",
      "0.0    79756\n",
      "1.0    34180\n",
      "Name: IS_HCP, dtype: int64\n",
      "=======================\n"
     ]
    }
   ],
   "source": [
    "for column in train_df.columns:\n",
    "    print(\"++- \"+column+\" -++\")\n",
    "    print(train_df[column].value_counts())\n",
    "    print('=======================')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b452094b",
   "metadata": {},
   "source": [
    "Since, multiple URLs are repeating we can use them directly to train the model. Altough breaking them up will be better for accuracy, but nonetheless we can atleast use it meaningfully."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "0fea3545",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Sum of Null Values:\n",
      "BIDREQUESTIP           0\n",
      "USERPLATFORMUID        4\n",
      "USERCITY            6359\n",
      "USERZIPCODE         4592\n",
      "PLATFORMTYPE           0\n",
      "URL                    0\n",
      "KEYWORDS               0\n",
      "TAXONOMY           81624\n",
      "IS_HCP                 1\n",
      "dtype: int64\n",
      "\n"
     ]
    }
   ],
   "source": [
    "null_sum = train_df.isnull().sum()\n",
    "\n",
    "print(\"Sum of Null Values:\")\n",
    "print(null_sum)\n",
    "print()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4a4e13bc",
   "metadata": {},
   "source": [
    "## Dropping the missing values in the USERPLATFORMUID as no valid relationship can be established."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "fe31f6a7",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "59158     152.11.230.114\n",
      "80923      73.59.230.146\n",
      "99416     152.11.230.114\n",
      "102512     73.59.230.146\n",
      "Name: BIDREQUESTIP, dtype: object\n"
     ]
    }
   ],
   "source": [
    "missing_uid_rows = train_df[train_df['USERPLATFORMUID'].isnull()]\n",
    "bidrequestip_values = missing_uid_rows['BIDREQUESTIP']\n",
    "print(bidrequestip_values)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "fd30ae42",
   "metadata": {},
   "outputs": [],
   "source": [
    "train_df.dropna(subset=['USERPLATFORMUID'], inplace=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "33548e20",
   "metadata": {},
   "source": [
    "## for missing value in IS_HCP"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "e27db314",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "73790    47.147.27.131\n",
      "Name: BIDREQUESTIP, dtype: object\n"
     ]
    }
   ],
   "source": [
    "missing_uid_rows = train_df[train_df['IS_HCP'].isnull()]\n",
    "bidrequestip_values = missing_uid_rows['BIDREQUESTIP']\n",
    "print(bidrequestip_values)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "32a4ba2e",
   "metadata": {},
   "source": [
    "the missing value in IS_HCP has this BIDREQIP, so decided to follow the other values at this BIDREQIP"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "031cb125",
   "metadata": {},
   "outputs": [],
   "source": [
    "train_df['IS_HCP'] = train_df.groupby('USERPLATFORMUID')['IS_HCP'].ffill()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "9f20869f",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "       IS_HCP   BIDREQUESTIP\n",
      "412       1.0  47.147.27.131\n",
      "4376      1.0  47.147.27.131\n",
      "17491     1.0  47.147.27.131\n",
      "73790     1.0  47.147.27.131\n"
     ]
    }
   ],
   "source": [
    "filtered_df = train_df[train_df['BIDREQUESTIP'] == '47.147.27.131']\n",
    "print(filtered_df[['IS_HCP', 'BIDREQUESTIP']])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "aed500f9",
   "metadata": {},
   "source": [
    "Since IS_HCP is binary -->!float -->>>int"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "9b106b15",
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "train_df['IS_HCP'] = train_df['IS_HCP'].apply(np.int64)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "196b6be8",
   "metadata": {},
   "source": [
    "We can see the value of 1.0 is inserted into 73790     1.0  47.147.27.131"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "78cf3b5b",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Sum of Null Values:\n",
      "BIDREQUESTIP           0\n",
      "USERPLATFORMUID        0\n",
      "USERCITY            6359\n",
      "USERZIPCODE         4592\n",
      "PLATFORMTYPE           0\n",
      "URL                    0\n",
      "KEYWORDS               0\n",
      "TAXONOMY           81624\n",
      "IS_HCP                 0\n",
      "dtype: int64\n",
      "\n"
     ]
    }
   ],
   "source": [
    "null_sum = train_df.isnull().sum()\n",
    "\n",
    "print(\"Sum of Null Values:\")\n",
    "print(null_sum)\n",
    "print()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "c8ff98cd",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "5009      Plymouth\n",
      "12542      Detroit\n",
      "19351     Plymouth\n",
      "21258      Detroit\n",
      "29326      Detroit\n",
      "42271     Plymouth\n",
      "64092     Plymouth\n",
      "65229     Plymouth\n",
      "66293      Detroit\n",
      "70688      Detroit\n",
      "87318     Plymouth\n",
      "94143      Detroit\n",
      "94208     Plymouth\n",
      "112735         NaN\n",
      "Name: USERCITY, dtype: object\n"
     ]
    }
   ],
   "source": [
    "zipcode = 482012153\n",
    "usercity_values = train_df[train_df['USERZIPCODE'] == zipcode]['USERCITY']\n",
    "print(usercity_values)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "b0173a18",
   "metadata": {},
   "outputs": [],
   "source": [
    "train_df['USERCITY'] = train_df['USERCITY'].fillna(train_df.groupby('USERZIPCODE')['USERCITY'].transform('first'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "f6cdcd8b",
   "metadata": {},
   "outputs": [],
   "source": [
    "# null_sum = train_df.isnull().sum()\n",
    "\n",
    "# print(\"Sum of Null Values:\")\n",
    "# print(null_sum)\n",
    "# print()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "f7d95702",
   "metadata": {},
   "outputs": [],
   "source": [
    "# zipcode = 482012153\n",
    "# usercity_values = train_df[train_df['USERZIPCODE'] == zipcode]['USERCITY']\n",
    "# print(usercity_values)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1e07418c",
   "metadata": {},
   "source": [
    "For Taxanomy"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a902b77f",
   "metadata": {},
   "source": [
    "## handling category"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "4684a84f",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Size of word_list: 1613\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "['gastrointestinal diseases',\n",
       " 'opportunistic infections',\n",
       " 'hamartoma',\n",
       " 'carcinoma',\n",
       " 'large intestine',\n",
       " 'ustekinumab',\n",
       " 'famciclovir',\n",
       " 'cryptococcosis',\n",
       " 'muscle pain',\n",
       " 'FIBRINOGEN',\n",
       " 'nephrology',\n",
       " 'Analgesia',\n",
       " 'glucagon secretion',\n",
       " 'cardiology',\n",
       " 'lipid profile',\n",
       " 'Follicular Lymphoma',\n",
       " 'muscle relaxation',\n",
       " 'lymphatic diseases',\n",
       " 'Minimally Invasive Surgeries',\n",
       " 'lupus anticoagulant',\n",
       " 'hiv infections',\n",
       " 'arteriovenous malformations',\n",
       " 'aphonia',\n",
       " 'Protein S Deficiency',\n",
       " 'transplantation',\n",
       " 'yaws',\n",
       " 'Preschool',\n",
       " 'Cardiomyopathy',\n",
       " 'Allergy Medications',\n",
       " 'Lung',\n",
       " 'Premature',\n",
       " 'af',\n",
       " 'surgical',\n",
       " 'Skeleton',\n",
       " 'new',\n",
       " 'Premedication',\n",
       " 'salivary glands',\n",
       " 'anus',\n",
       " 'epidural abscess',\n",
       " 'Behavior',\n",
       " 'encephalopathy',\n",
       " 'mepolizumab',\n",
       " 'Biliary Fistula',\n",
       " 'catheterization',\n",
       " 'dna sequencing',\n",
       " 'Therapeutic Uses',\n",
       " 'infertility',\n",
       " 'General Hospitals',\n",
       " 'janus kinase 1',\n",
       " 'updated',\n",
       " 'myocardial ischemia',\n",
       " 'insomnia',\n",
       " 'Hematology',\n",
       " 'apixaban',\n",
       " 'rubella',\n",
       " 'staphylococcal scalded skin syndrome',\n",
       " 'Strokes',\n",
       " 'Malnutrition',\n",
       " 'pericardial effusion',\n",
       " 'Hematology & Oncology',\n",
       " 'mucocele',\n",
       " 'thyroid carcinoma',\n",
       " 'general practice',\n",
       " 'otitis media',\n",
       " 'conduct disorders',\n",
       " 'Resuscitation',\n",
       " 'aids dementia complex',\n",
       " 'Clinical Pathology',\n",
       " 'Necrolytic Migratory Erythema',\n",
       " 'radiosurgery',\n",
       " 'eosinophilia',\n",
       " 'diabetic retinopathy',\n",
       " 'vibrio infections',\n",
       " 'dermabrasion',\n",
       " 'myelitis',\n",
       " 'Debridement',\n",
       " 'herpes simplex',\n",
       " 'scabies',\n",
       " 'Metabolic',\n",
       " 'synthetic',\n",
       " 'family medicine',\n",
       " 'postoperative complications',\n",
       " 'croup',\n",
       " 'Myocardial Reperfusion',\n",
       " 'angioedema',\n",
       " 'hepatitis a',\n",
       " 'prostaglandins',\n",
       " 'Polyarthritis',\n",
       " 'Kidney Transplantation',\n",
       " 'Urology',\n",
       " 'Hematuria',\n",
       " 'Neuroendoscopy',\n",
       " 'oral & maxillofacial surgery',\n",
       " 'omalizumab',\n",
       " 'Human Development',\n",
       " 'infant',\n",
       " 'nasal bone',\n",
       " 'Hormone-Dependent',\n",
       " 'Allergic Bronchopulmonary',\n",
       " 'immunocompromised',\n",
       " 'retinal vein occlusion',\n",
       " 'virus',\n",
       " 'respiratory tract infections',\n",
       " 'dyssomnias',\n",
       " 'blood protein disorders',\n",
       " 'heparin',\n",
       " 'reticuloendotheliosis',\n",
       " 'loss of appetite',\n",
       " 'intensive care',\n",
       " 'Radiography',\n",
       " 'cheyne-stokes respiration',\n",
       " 'dyspareunia',\n",
       " 'Pregnancy Complications',\n",
       " 'Lung Transplantation',\n",
       " 'Transluminal Angioplasty',\n",
       " 'intestinal diseases',\n",
       " 'fanconi anemia',\n",
       " 'aneurysm',\n",
       " 'conscious sedation',\n",
       " 'dengue',\n",
       " 'anthropometry',\n",
       " 'cardiac failure',\n",
       " 'obstetric',\n",
       " 'skin rash',\n",
       " 'Dermatology',\n",
       " 'Disseminated Intravascular Coagulation',\n",
       " 'Primary Immunodeficiency Diseases',\n",
       " 'vka',\n",
       " 'Tachycardia',\n",
       " 'blackwater fever',\n",
       " 'taeniasis',\n",
       " 'noac',\n",
       " 'Resistance Training',\n",
       " 'lung injury',\n",
       " 'vascular diseases',\n",
       " 'Hepatology',\n",
       " 'tablet',\n",
       " 'bacterial infections and mycoses',\n",
       " 'Molecular',\n",
       " 'cerebrovascular disorders',\n",
       " 'itching',\n",
       " 'chalazion',\n",
       " 'Infant',\n",
       " 'tachypnea',\n",
       " 'pyuria',\n",
       " 'chromoblastomycosis',\n",
       " 'general practitioners',\n",
       " 'intrahepatic',\n",
       " 'liver abscess',\n",
       " 'Hamartoma',\n",
       " 'Arthritis',\n",
       " 'Genetic',\n",
       " 'marfan syndrome',\n",
       " 'Refractory',\n",
       " 'hemorrhagic fever',\n",
       " 'Epidural Analgesia',\n",
       " 'Neoplastic',\n",
       " 'neurological disorders',\n",
       " 'nidovirales infections',\n",
       " 'immunization',\n",
       " 'Multiple Myeloma',\n",
       " 'drug development',\n",
       " 'fever',\n",
       " 'paraneoplastic syndromes',\n",
       " 'intracranial aneurysm',\n",
       " 'Deep Sedation',\n",
       " 'frontal sinus',\n",
       " 'valacyclovir',\n",
       " 'immunoglobulin a',\n",
       " 'Newborns',\n",
       " 'infliximab',\n",
       " 'Obstetrics & Gynecology',\n",
       " 'enuresis',\n",
       " 'immunoglobulin d',\n",
       " 'sequencing',\n",
       " 'proton therapy',\n",
       " 'hematologic diseases',\n",
       " 'balloon',\n",
       " 'Intracranial Aneurysm',\n",
       " 'renal impairment',\n",
       " 'dwarfism',\n",
       " 'influenza',\n",
       " 'laser',\n",
       " 'Transplantation',\n",
       " 'Anemia',\n",
       " 'false',\n",
       " 'Amyotrophic Lateral Sclerosis',\n",
       " 'Cryptorchidism',\n",
       " 'Diabetes Complications',\n",
       " 'Motor Neuron Disease',\n",
       " 'thoracic surgery',\n",
       " 'pregnancy',\n",
       " 'mandibular advancement',\n",
       " 'myelofibrosis',\n",
       " 'pain relief',\n",
       " 'ascariasis',\n",
       " 'Urogenital',\n",
       " 'erysipeloid',\n",
       " 'Elderly',\n",
       " 'Primary Prevention',\n",
       " 'phlebitis',\n",
       " 'Dietetics',\n",
       " 'varicella-zoster virus',\n",
       " 'rituximab',\n",
       " 'flu',\n",
       " 'Surgical Pathology',\n",
       " 'gingivitis',\n",
       " 'cns infections',\n",
       " 'electrophysiology',\n",
       " 'antithrombin iii deficiency',\n",
       " 'adrenal insufficiency',\n",
       " 'Large Intestine',\n",
       " 'polyneuropathy',\n",
       " 'neuromuscular diseases',\n",
       " 'Impulse Control',\n",
       " 'psychological',\n",
       " 'impulse control',\n",
       " 'metabolic',\n",
       " 'preventive medicine',\n",
       " 'Nephritis',\n",
       " 'CNS',\n",
       " 'latent tuberculosis',\n",
       " 'tonsillitis',\n",
       " 'plague',\n",
       " 'Thyroid Carcinoma',\n",
       " 'pancreas',\n",
       " 'Surgical Intensive',\n",
       " 'acquired immunodeficiency syndrome',\n",
       " 'biologics',\n",
       " 'tinea pedis',\n",
       " 'aortic aneurysm',\n",
       " 'Seasonal',\n",
       " 'hematuria',\n",
       " 'follicular lymphoma',\n",
       " 'clavulanic acid',\n",
       " 'hordeolum',\n",
       " 'patient',\n",
       " 'Granulomatous Disease',\n",
       " 'granuloma inguinale',\n",
       " 'idiopathic pulmonary fibrosis',\n",
       " 'uveitis',\n",
       " 'virus diseases',\n",
       " 'cardiomegaly',\n",
       " 'Ventricular Fibrillation',\n",
       " 'staphylococcus aureus',\n",
       " 'Bile Ducts',\n",
       " 'dvt',\n",
       " 'Otolaryngology',\n",
       " 'rash',\n",
       " 'Neonatal Intensive Care',\n",
       " 'Endodontics',\n",
       " 'pulmonary embolism',\n",
       " 'deep brain stimulation',\n",
       " 'psoriasis',\n",
       " 'anticoagulants',\n",
       " 'pain management',\n",
       " 'rheumatoid arthritis',\n",
       " 'coccidioidomycosis',\n",
       " 'antibody deficiency',\n",
       " 'Afibrinogenemia',\n",
       " 'wound infection',\n",
       " 'pseudoxanthoma elasticum',\n",
       " 'anti-inflammatory agents',\n",
       " 'pharmacology',\n",
       " 'oncology',\n",
       " 'geroscience',\n",
       " 'anastrozole',\n",
       " 'Salivary Gland',\n",
       " 'pneumonia',\n",
       " 'nocardia infections',\n",
       " 'neutrophils',\n",
       " 'clinical pathology',\n",
       " 'Denervation',\n",
       " 'attitude',\n",
       " 'glomerular filtration rate',\n",
       " 'Birth Cohort',\n",
       " 'toothache',\n",
       " 'purpura',\n",
       " 'Health',\n",
       " 'Thymectomy',\n",
       " 'disseminated intravascular coagulation',\n",
       " 'herpes',\n",
       " 'perennial',\n",
       " 'psychiatry',\n",
       " 'Pulmonary Embolism',\n",
       " 'medicine',\n",
       " 'vzv',\n",
       " 'Preventive',\n",
       " 'lung diseases, obstructive',\n",
       " 'Cardiovascular Abnormalities',\n",
       " 'nasal cavity',\n",
       " 'Tooth Extraction',\n",
       " 'ruptured',\n",
       " 'Fatal Familial',\n",
       " 'Cerebrovascular Accidents',\n",
       " 'tetanus',\n",
       " 'trastuzumab',\n",
       " 'non-steroidal anti-inflammatory drug',\n",
       " 'Activities of Daily Living',\n",
       " 'hypertension',\n",
       " 'appendicitis',\n",
       " 'child',\n",
       " 'stomatitis',\n",
       " 'takotsubo cardiomyopathy',\n",
       " 'biopsy',\n",
       " 'herpes zoster oticus',\n",
       " 'doac',\n",
       " 'neurosyphilis',\n",
       " 'liver diseases',\n",
       " 'Neurological Surgery',\n",
       " 'Laminoplasty',\n",
       " 'aged',\n",
       " 'Aortic Aneurysm',\n",
       " 'mental health',\n",
       " 'vte',\n",
       " 'Newborn',\n",
       " 'chagas disease',\n",
       " 'abortion',\n",
       " 'ventricular fibrillation',\n",
       " 'lupus',\n",
       " 'Bone Marrow',\n",
       " 'Gastrointestinal Tract',\n",
       " 'Surgical',\n",
       " 'heart failure',\n",
       " 'difficulty concentrating',\n",
       " 'Minimally Invasive Surgical Procedures',\n",
       " 'Therapeutic Effects',\n",
       " 'Thalassemia',\n",
       " 'Psychiatry',\n",
       " 'rocky mountain spotted fever',\n",
       " 'breathing exercises',\n",
       " 'palatine tonsil',\n",
       " 'erythrasma',\n",
       " 'large',\n",
       " 'varicella',\n",
       " 'tularemia',\n",
       " 'Thoracic Surgical Procedures',\n",
       " 'chronic',\n",
       " 'Orthopedics',\n",
       " 'inborn',\n",
       " 'respiration disorders',\n",
       " 'male',\n",
       " 'bronchiolitis',\n",
       " 'urologic diseases',\n",
       " 'Peripheral Nervous System Diseases',\n",
       " 'Hemothorax',\n",
       " 'skin diseases',\n",
       " 'orthodontics',\n",
       " 'congenital abnormalities',\n",
       " 'postherpetic neuralgia',\n",
       " 'Neuropathology',\n",
       " 'elective surgical procedures',\n",
       " 'Cardiovascular',\n",
       " 'Bevacizumab',\n",
       " 'cefixime',\n",
       " 'acetaminophen',\n",
       " 'Overweight',\n",
       " 'aspirin',\n",
       " 'Health Care Professional',\n",
       " 'leptospirosis',\n",
       " 'Myeloproliferative neoplasms',\n",
       " 'janus kinase 2',\n",
       " 'Sideroblastic',\n",
       " 'Age Groups',\n",
       " 'atrial flutter',\n",
       " 'vomiting',\n",
       " 'cerebrovascular accident',\n",
       " 'Cardiac Rehabilitation',\n",
       " 'chicken pox',\n",
       " 'macrolides',\n",
       " 'Angioplasty',\n",
       " 'Neoplasms by Site',\n",
       " 'shock',\n",
       " 'yellow fever',\n",
       " 'leukopenia',\n",
       " 'lassa fever',\n",
       " 'acquired',\n",
       " 'health care professional',\n",
       " 'Sleep Disorders',\n",
       " 'Anticoagulants',\n",
       " 'tinea',\n",
       " 'cns',\n",
       " 'Interventional Cardiology',\n",
       " 'total',\n",
       " 'Pharmacogenetics',\n",
       " 'angiomatosis',\n",
       " 'deficiency diseases',\n",
       " 'cryoglobulinemia',\n",
       " 'Lipomatosis',\n",
       " 'radiation oncology',\n",
       " 'Mental Processes',\n",
       " 'smallpox',\n",
       " 'laryngeal edema',\n",
       " 'ehlers-danlos syndrome',\n",
       " 'child care',\n",
       " 'Thrombophilia',\n",
       " 'hemorrhagic stroke',\n",
       " 'Rituximab',\n",
       " 'non-high-density lipoprotein cholesterol',\n",
       " 'type 2 diabetes mellitus',\n",
       " 'hemoptysis',\n",
       " 'Neuromuscular Diseases',\n",
       " 'stroke management',\n",
       " 'hepatitis c',\n",
       " 'fabry disease',\n",
       " 'Diabetes Insipidus',\n",
       " 'Hair Diseases',\n",
       " 'pharyngitis',\n",
       " 'Graves',\n",
       " 'Imperforate',\n",
       " 'Congestive Heart Failure',\n",
       " 'Anetoderma',\n",
       " 'cardiovascular infections',\n",
       " 'Abortion',\n",
       " 'exercise therapy',\n",
       " 'empyema',\n",
       " 'vaccine',\n",
       " 'pulmonary edema',\n",
       " 'vitamin k deficiency',\n",
       " 'glucagon',\n",
       " 'Dissecting',\n",
       " 'blood loss',\n",
       " 'pneumonia, viral',\n",
       " 'starvation',\n",
       " 'tabes dorsalis',\n",
       " 'health education',\n",
       " 'newborn',\n",
       " 'Physical Medicine & Rehabilitation',\n",
       " 'hematoma',\n",
       " 'Bursitis',\n",
       " 'retropharyngeal abscess',\n",
       " 'Cellulitis',\n",
       " 'h1n1',\n",
       " 'Surgery',\n",
       " 'premedication',\n",
       " 'nonsteroidal anti-inflammatory drugs',\n",
       " 'molecular pathology',\n",
       " 'psittacosis',\n",
       " 'endurance training',\n",
       " 'Sone',\n",
       " 'endocarditis',\n",
       " 'epididymitis',\n",
       " 'tracheitis',\n",
       " 'anxiety disorders',\n",
       " 'skin disease',\n",
       " 'citalopram',\n",
       " 'metabolic syndrome',\n",
       " 'endocrinology',\n",
       " 'bacteriuria',\n",
       " 'aco',\n",
       " 'nivolumab',\n",
       " 'whipple disease',\n",
       " 'resuscitation',\n",
       " 'pharynx',\n",
       " 'thrombocytosis',\n",
       " 'methylprednisolone',\n",
       " 'bone marrow',\n",
       " 'muscle aches',\n",
       " 'circulating tumour dna',\n",
       " 'cardiac rehabilitation',\n",
       " 'pathology',\n",
       " 'hoarseness',\n",
       " 'liver dysfunction',\n",
       " 'ventricular outflow obstruction',\n",
       " 'heart defects',\n",
       " 'h1n1 vaccine',\n",
       " 'pleural effusion',\n",
       " 'anxiety',\n",
       " 'Heart Diseases',\n",
       " 'brain abscess',\n",
       " 'asymptomatic infections',\n",
       " 'multiple sclerosis',\n",
       " 'ranibizumab',\n",
       " 'Joints',\n",
       " 'hemolytic',\n",
       " 'Sleep Medicine',\n",
       " 'Biopsy',\n",
       " 'Endotracheal',\n",
       " 'bisphosphonate-associated osteonecrosis of the jaw',\n",
       " 'Arthropathy',\n",
       " 'hypotension',\n",
       " 'bronchiolitis obliterans',\n",
       " 'nonvalvular atrial fibrillation',\n",
       " 'abatacept',\n",
       " 'numbness',\n",
       " 'Congenital Abnormalities',\n",
       " 'naproxen',\n",
       " 'Myelodysplastic Syndromes',\n",
       " 'Intensive Care',\n",
       " 'Disruptive',\n",
       " 'Adaptation',\n",
       " 'B-Lymphocytes',\n",
       " 'dietetics',\n",
       " 'Esophagus',\n",
       " 'Neonates',\n",
       " 'ipilimumab',\n",
       " 'acos',\n",
       " 'Decompressive Craniectomy',\n",
       " 'chronic limb-threatening ischemia',\n",
       " 'hemothorax',\n",
       " 'Neuroinflammatory Diseases',\n",
       " 'nervous system',\n",
       " 'Inborn',\n",
       " 'tract neoplasms',\n",
       " 'Angiotensin',\n",
       " 'migraine',\n",
       " \"alzheimer's disease\",\n",
       " 'Blunt Cardiac Injury',\n",
       " 'child well-being',\n",
       " 'copd',\n",
       " 'cough',\n",
       " 'Heart Failure',\n",
       " 'Neurosurgical Procedures',\n",
       " 'Endocrine',\n",
       " 'grandparents',\n",
       " 'metabolic diseases',\n",
       " 'Experimental',\n",
       " 'Cardiac',\n",
       " 'cellulitis',\n",
       " 'Drug Combinations',\n",
       " 'graves disease',\n",
       " 'ischemic stroke',\n",
       " 'relapsing fever',\n",
       " 'new york heart association',\n",
       " 'Neonatology',\n",
       " 'Autoimmune',\n",
       " 'olfactory mucosa',\n",
       " 'febrile neutropenia',\n",
       " 'Hypospadias',\n",
       " 'Aspergillosis',\n",
       " 'Lung Diseases',\n",
       " 'sepsis',\n",
       " 'hospitals',\n",
       " 'pituitary apoplexy',\n",
       " 'oligoclonal bands',\n",
       " 'embolic stroke',\n",
       " 'refractory',\n",
       " 'granuloma annulare',\n",
       " 'keratoconjunctivitis',\n",
       " 'dyspnea',\n",
       " 'nasal mucosa',\n",
       " 'Antibody Deficiency',\n",
       " 'science',\n",
       " 'ethnopharmacology',\n",
       " 'Bone Marrow Failure Disorders',\n",
       " 'Airway Management',\n",
       " 'peripheral nervous system diseases',\n",
       " 'Bronchioles',\n",
       " 'surgical pathology',\n",
       " 'proteomics',\n",
       " 'molluscum contagiosum',\n",
       " 'chemoprevention',\n",
       " 'Breathing Exercises',\n",
       " 'corrective',\n",
       " 'Gastrointestinal Diseases',\n",
       " 'disease',\n",
       " 'next-generation sequencing',\n",
       " 'zoonoses',\n",
       " 'middle ear',\n",
       " 'scarlet fever',\n",
       " 'gastrointestinal tracts',\n",
       " 'cefuroxime',\n",
       " 'thromboembolism',\n",
       " 'Small',\n",
       " 'Apnea',\n",
       " 'radiography',\n",
       " 'covid',\n",
       " 'Papillary',\n",
       " 'central nervous system diseases',\n",
       " 'motor neuron disease',\n",
       " 'diagnosis',\n",
       " 'Bariatric Surgery',\n",
       " 'Small Intestines',\n",
       " 'acute coronary syndrome',\n",
       " 'tracheal stenosis',\n",
       " 'drug therapy',\n",
       " 'Intestine',\n",
       " 'Hemolytic',\n",
       " 'denosumab',\n",
       " 'bacterial infections',\n",
       " 'intestines',\n",
       " 'Malabsorption Syndromes',\n",
       " 'sleep medicine',\n",
       " 'drug combinations',\n",
       " 'educational',\n",
       " 'lung',\n",
       " 'capillary leak syndrome',\n",
       " 'spondylitis',\n",
       " 'primary prevention',\n",
       " 'impetigo',\n",
       " 'hemostatic disorders',\n",
       " 'nasal septum',\n",
       " 'neurological surgery',\n",
       " 'Sports Medicine',\n",
       " 'premature',\n",
       " 'doctor',\n",
       " 'Graves Disease',\n",
       " 'Kidney Diseases',\n",
       " 'Dwarfism',\n",
       " 'filariasis',\n",
       " 'bone diseases',\n",
       " 'pregnancy complications',\n",
       " 'activated protein c resistance',\n",
       " 'granuloma',\n",
       " 'respiratory tract neoplasms',\n",
       " 'depression',\n",
       " 'bevacizumab',\n",
       " 'cardiovascular disease',\n",
       " 'alemtuzumab',\n",
       " 'Lymphedema',\n",
       " 'cysts',\n",
       " 'zostavax',\n",
       " 'gastroenterology',\n",
       " 'Renal Transplantation',\n",
       " 'bartonella infections',\n",
       " 'Shock',\n",
       " 'wgs',\n",
       " 'laryngeal muscles',\n",
       " 'poliomyelitis',\n",
       " 'multiple myeloma',\n",
       " 'social withdrawal',\n",
       " 'renal transplantation',\n",
       " 'Restless Legs Syndrome',\n",
       " 'enhanced',\n",
       " 'lemierre syndrome',\n",
       " 'immune reconstitution inflammatory syndrome',\n",
       " 'dysentery',\n",
       " 'siderosis',\n",
       " 'hypoglycemia',\n",
       " 'thrombocytopenia',\n",
       " 'organ transplant',\n",
       " 'rheumatology',\n",
       " 'Synthetic',\n",
       " 'Pulmonary Medicine',\n",
       " 'hepatitis d',\n",
       " 'Obstetrical',\n",
       " 'critical care medicine',\n",
       " 'patient care',\n",
       " 'spinal cord ischemia',\n",
       " 'critical care',\n",
       " 'Lymphatic System',\n",
       " 'gram-negative bacterial infections',\n",
       " 'Erythema',\n",
       " 'weil disease',\n",
       " 'Myocardial Ischemia',\n",
       " 'Pharmacology',\n",
       " 'Neurodegenerative Diseases',\n",
       " 'pelvic infection',\n",
       " 'surgical oncology',\n",
       " 'mental competency',\n",
       " 'sexually transmitted diseases',\n",
       " 'transfusion-related acute lung injury',\n",
       " 'quadrivalent influenza vaccine',\n",
       " 'irritability',\n",
       " 'diagnostic molecular',\n",
       " 'Liver',\n",
       " 'diphtheria',\n",
       " 'mumps',\n",
       " 'dermatitis',\n",
       " 'dysphonia',\n",
       " 'blepharoplasty',\n",
       " 'neurodegenerative diseases',\n",
       " 'furunculosis',\n",
       " 'Catheterization',\n",
       " 'Activated Protein C Resistance',\n",
       " 'respiratory insufficiency',\n",
       " 'Maze Procedure',\n",
       " 'fibrinogen',\n",
       " 'neurology',\n",
       " 'rheumatoid',\n",
       " 'Surgical Intensive Care',\n",
       " 'Carpal Tunnel Syndrome',\n",
       " 'hepatitis e',\n",
       " 'thoracic outlet syndrome',\n",
       " 'ruxolitinib',\n",
       " 'lymphatic vessels',\n",
       " 'listeriosis',\n",
       " 'forensic',\n",
       " 'polyarthritis',\n",
       " 'brentuximab vedotin',\n",
       " 'congenital',\n",
       " 'Anastomosis',\n",
       " 'Biliary Tract',\n",
       " 'pulmonary disease',\n",
       " 'dpp-4',\n",
       " 'intracranial hemorrhages',\n",
       " 'tick-borne diseases',\n",
       " 'glucose metabolism disorders',\n",
       " 'autoimmune diseases',\n",
       " 'ear diseases',\n",
       " 'congenital heart defect',\n",
       " 'herpes zoster',\n",
       " 'Neonatal',\n",
       " 'blood coagulation disorders',\n",
       " 'acanthamoeba keratitis',\n",
       " 'health personnel',\n",
       " 'Female',\n",
       " 'diabetes',\n",
       " 'Heart Block',\n",
       " 'allergic',\n",
       " 'Chemoprevention',\n",
       " 'hearing disorders',\n",
       " 'Family Practice',\n",
       " 'Medical Oncology',\n",
       " 'Hereditary',\n",
       " 'inhaled corticosteroids',\n",
       " 'endophthalmitis',\n",
       " 'dentition',\n",
       " 'cardiotoxicity',\n",
       " 'body fluids',\n",
       " 'leukoencephalopathy',\n",
       " 'automation',\n",
       " 'Laser Therapy',\n",
       " 'covid-19',\n",
       " 'pain medicine',\n",
       " 'glanders',\n",
       " 'medical oncology',\n",
       " 'dendritic cells',\n",
       " 'aldosterone',\n",
       " 'fatigue',\n",
       " 'tinea cruris',\n",
       " 'operative',\n",
       " 'peritonsillar abscess',\n",
       " 'Anesthesia',\n",
       " 'ankylosis',\n",
       " 'Prostaglandins',\n",
       " 'Neuromuscular Blockade',\n",
       " 'neoplasms',\n",
       " 'common cold',\n",
       " 'weight reduction',\n",
       " 'malnutrition',\n",
       " 'Apixaban',\n",
       " 'hemorrhage',\n",
       " 'advanced',\n",
       " 'pulmonary arterial hypertension',\n",
       " 'adaptation',\n",
       " 'Status Asthmaticus',\n",
       " 'Educational',\n",
       " 'Altitude Sickness',\n",
       " 'carpal tunnel syndrome',\n",
       " 'nasal obstruction',\n",
       " 'blood pressure control',\n",
       " 'carney complex',\n",
       " 'End-Of-Life Care',\n",
       " 'tachycardia',\n",
       " 'autoantibodies',\n",
       " 'Dental',\n",
       " \"parkinson's disease\",\n",
       " 'neonates',\n",
       " 'macrophages',\n",
       " 'Thoracotomy',\n",
       " 'escitalopram',\n",
       " 'hyperemia',\n",
       " 'aspergillosis',\n",
       " 'forensic medicine',\n",
       " 'Liver Dysfunction',\n",
       " 'post-traumatic',\n",
       " 'ophthalmology',\n",
       " 'epidural',\n",
       " 'nausea',\n",
       " 'Phagocytes',\n",
       " 'newborns',\n",
       " 'Neurogenic',\n",
       " 'Exsanguination',\n",
       " 'hepatology',\n",
       " 'cardiac arrest',\n",
       " 'Health Services',\n",
       " 'Allergens',\n",
       " 'antilymphocyte serum',\n",
       " 'cystic fibrosis',\n",
       " 'Diabetes Mellitus',\n",
       " 'neonatal sepsis',\n",
       " 'vocal cord dysfunction',\n",
       " 'intrinsic',\n",
       " 'cidofovir',\n",
       " 'cysticercosis',\n",
       " 'ventricular dysfunction',\n",
       " 'hemopneumothorax',\n",
       " 'beta-Thalassemia',\n",
       " 'fasciitis',\n",
       " 'autoimmune disease',\n",
       " 'Nervous System',\n",
       " 'sglt2',\n",
       " 'neurophysiology',\n",
       " 'Extravascular Lung Water',\n",
       " 'Decompression',\n",
       " 'Vascular Diseases',\n",
       " 'respiratory tract disease',\n",
       " 'pneumoconiosis',\n",
       " 'coronaviridae infections',\n",
       " 'Child Health',\n",
       " 'Leukopenia',\n",
       " 'bradycardia',\n",
       " 'arthropathy',\n",
       " 'reproductive',\n",
       " 'Sleep Deprivation',\n",
       " 'escherichia coli',\n",
       " 'conjunctivitis',\n",
       " 'triple therapy',\n",
       " 'immunity',\n",
       " 'gfr',\n",
       " 'Lymphatic Vessels',\n",
       " 'sleep disorders',\n",
       " 'perfusion',\n",
       " 'tramadol',\n",
       " 'Pathologic Processes',\n",
       " 'chickenpox',\n",
       " 'behavioral sciences',\n",
       " 'molecular diagnostic',\n",
       " 'liver',\n",
       " 'Immune System',\n",
       " 'Rehabilitation',\n",
       " 'Rhinitis',\n",
       " 'neurocysticercosis',\n",
       " 'Cardiovascular Infections',\n",
       " 'cardiac tamponade',\n",
       " 'translocation',\n",
       " 'Jejunum',\n",
       " 'skeleton',\n",
       " 'Hemorrhage',\n",
       " 'eosinophilia-myalgia syndrome',\n",
       " 'erythema infectiosum',\n",
       " 'babesiosis',\n",
       " 'pneumothorax',\n",
       " 'Cardiomyopathies',\n",
       " 'biliary tract',\n",
       " 'Hepatic Transplantation',\n",
       " 'whole genome sequencing',\n",
       " 'baloxavir marboxil',\n",
       " 'paraproteinemias',\n",
       " 'orthopedics',\n",
       " 'intestine',\n",
       " 'mental health recovery',\n",
       " 'neonatal',\n",
       " 'Anxiety Disorders',\n",
       " 'bronchi',\n",
       " 'Secondary Prevention',\n",
       " 'thrombosis',\n",
       " 'waldenstrom macroglobulinemia',\n",
       " 'embolism',\n",
       " 'lung neoplasms',\n",
       " 'eye diseases',\n",
       " 'tract disease',\n",
       " 'cardiovascular risk reduction',\n",
       " 'reperfusion',\n",
       " 'muscular diseases',\n",
       " 'Biliary Atresia',\n",
       " 'malaria',\n",
       " 'orthopedic',\n",
       " 'birth cohort',\n",
       " 'Gastrointestinal Tracts',\n",
       " 'Ophthalmology',\n",
       " 'Long QT Syndrome',\n",
       " 'arthralgia',\n",
       " 'Pain Medicine',\n",
       " 'shingrix',\n",
       " 'cardiovascular',\n",
       " 'oropharynx',\n",
       " 'Large',\n",
       " 'pharmacist',\n",
       " 'Atherectomy',\n",
       " 'keratosis',\n",
       " 'back pain',\n",
       " 'glucocorticoids',\n",
       " 'inherited',\n",
       " 'Intrahepatic',\n",
       " 'neu receptor',\n",
       " 'Psychological',\n",
       " 'epiglottitis',\n",
       " 'Angioedema',\n",
       " 'stomach',\n",
       " 'juvenile dermatomyositis',\n",
       " 'Induced',\n",
       " 'wheezing',\n",
       " 'noonan syndrome',\n",
       " 'Deep Brain Stimulation',\n",
       " 'rheumatic fever',\n",
       " 'Protein C Deficiency',\n",
       " 'rhinitis',\n",
       " 'measles',\n",
       " 'Asthma',\n",
       " 'beta-thalassemia',\n",
       " 'laryngopharyngeal reflux',\n",
       " 'Circadian Rhythm',\n",
       " 'subarachnoid hemorrhage',\n",
       " 'hereditary',\n",
       " 'health services',\n",
       " 'hypoalbuminemia',\n",
       " 'myelodysplastic syndromes',\n",
       " 'candidiasis',\n",
       " 'Oral',\n",
       " 'Infertility',\n",
       " 'Bernard-Soulier Syndrome',\n",
       " 'tracheomalacia',\n",
       " 'nasopharynx',\n",
       " 'encephalitis',\n",
       " 'Dentistry',\n",
       " 'intestinal mucosa',\n",
       " 'controlled',\n",
       " 'Intrinsic',\n",
       " 'Clinical Pharmacology',\n",
       " 'inflammation',\n",
       " 'blunt cardiac injury',\n",
       " 'dental',\n",
       " 'persistent infection',\n",
       " 'Neoplasms',\n",
       " 'vaccines',\n",
       " 'Aneurysm',\n",
       " 'neurologic manifestations',\n",
       " 'hyperventilation',\n",
       " 'alagille syndrome',\n",
       " 'rabies',\n",
       " 'the',\n",
       " 'atrial fibrillation',\n",
       " 'Medical Specialties',\n",
       " 'protein c deficiency',\n",
       " 'delivery',\n",
       " 'cardiomyopathy',\n",
       " 'Hypotension',\n",
       " 'Preventive Medicine',\n",
       " 'Allergic',\n",
       " 'hypospadias',\n",
       " 'drug discovery',\n",
       " 'peripheral arterial disease',\n",
       " 'patient-controlled',\n",
       " 'prothrombin',\n",
       " 'Duodenum',\n",
       " 'General',\n",
       " 'kidney diseases',\n",
       " 'Ischemic Heart Diseases',\n",
       " 'neuromuscular blockade',\n",
       " 'molecular diagnostics',\n",
       " 'pediatrics',\n",
       " 'Dissection',\n",
       " 'cholera',\n",
       " 'conduction',\n",
       " 'bronchopulmonary sequestration',\n",
       " 'Pediatrics',\n",
       " 'Operative',\n",
       " 'vocal cords',\n",
       " 'Pulmonologists',\n",
       " 'retinal artery occlusion',\n",
       " 'Child Care',\n",
       " 'neutropenia',\n",
       " 'Antithrombin III Deficiency',\n",
       " 'Flu',\n",
       " 'dentist',\n",
       " 'tetralogy of fallot',\n",
       " 'General Practitioners',\n",
       " 'hemophilia b',\n",
       " 'botulism',\n",
       " 'neonatal intensive care',\n",
       " 'sitagliptin',\n",
       " 'transplant',\n",
       " 'Paraneoplastic Syndromes',\n",
       " 'abscess',\n",
       " 'Geriatric Medicine',\n",
       " 'acyclovir',\n",
       " 'bile ducts',\n",
       " 'defense mechanisms',\n",
       " 'Eosinophilic Esophagitis',\n",
       " 'respiratory distress syndrome',\n",
       " 'Neurologic Manifestations',\n",
       " 'granulomatosis with polyangiitis',\n",
       " 'health',\n",
       " 'headache',\n",
       " 'amebiasis',\n",
       " 'Dendritic Cells',\n",
       " 'prednisone',\n",
       " 'elimination disorders',\n",
       " 'internal medicine',\n",
       " 'small cell lung carcinoma',\n",
       " 'keratitis',\n",
       " 'herbal medicine',\n",
       " 'neutron capture therapy',\n",
       " 'Neutrophils',\n",
       " 'Hemarthrosis',\n",
       " 'Atrial Flutter',\n",
       " 'thalassemia',\n",
       " 'Heart Defects',\n",
       " 'Antigens',\n",
       " 'Post-Traumatic',\n",
       " 'Rheumatic Diseases',\n",
       " 'Pediatric Emergency Medicine',\n",
       " 'arrhythmias',\n",
       " 'Hemoptysis',\n",
       " 'elephantiasis',\n",
       " 'eosinophilic',\n",
       " 'Keratoacanthoma',\n",
       " 'coronavirus infections',\n",
       " 'Conduct Disorders',\n",
       " 'cervical rib syndrome',\n",
       " 'arrhythmogenic right ventricular dysplasia',\n",
       " 'invasive fungal infections',\n",
       " 'coronary heart disease',\n",
       " 'fetal anomalies',\n",
       " 'Lung Abscess',\n",
       " 'epidural analgesia',\n",
       " 'clinical',\n",
       " 'congestive heart failure',\n",
       " ...]"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#getting all the words to list\n",
    "all_keywords = train_df['KEYWORDS'].str.cat(sep='|')\n",
    "word_list = list(set(all_keywords.split('|')))\n",
    "\n",
    "# Remove empty value from word_list\n",
    "word_list = list(filter(None, word_list))\n",
    "\n",
    "\n",
    "list_size = len(word_list)\n",
    "print(\"Size of word_list:\", list_size)\n",
    "word_list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "0bc6a1e5",
   "metadata": {},
   "outputs": [],
   "source": [
    "# # Iterate over each word in word_list\n",
    "# for word in word_list:\n",
    "#     # Create a new column with the word as the column name and initialize it with 0\n",
    "#     train_df[word] = 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "228632c9",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create a new column for each word in KEYWORDS and initialize it with 0\n",
    "for word in word_list:\n",
    "    train_df[word] = train_df['KEYWORDS'].apply(lambda x: 1 if word in x else 0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "d81cf114",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>BIDREQUESTIP</th>\n",
       "      <th>USERPLATFORMUID</th>\n",
       "      <th>USERCITY</th>\n",
       "      <th>USERZIPCODE</th>\n",
       "      <th>PLATFORMTYPE</th>\n",
       "      <th>URL</th>\n",
       "      <th>KEYWORDS</th>\n",
       "      <th>TAXONOMY</th>\n",
       "      <th>IS_HCP</th>\n",
       "      <th>gastrointestinal diseases</th>\n",
       "      <th>...</th>\n",
       "      <th>pleurisy</th>\n",
       "      <th>pulmonary medicine</th>\n",
       "      <th>Cardiac Output</th>\n",
       "      <th>earache</th>\n",
       "      <th>Cysts</th>\n",
       "      <th>asthma</th>\n",
       "      <th>Embolism</th>\n",
       "      <th>Conscious Sedation</th>\n",
       "      <th>myeloproliferative neoplasms</th>\n",
       "      <th>q fever</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>170.173.0.22</td>\n",
       "      <td>6974dcaa-f932-480e-9fb5-c52e20e1393a</td>\n",
       "      <td>Portland</td>\n",
       "      <td>97206.0</td>\n",
       "      <td>Online Medical Journal</td>\n",
       "      <td>https://www.cancertherapyadvisor.com/home/canc...</td>\n",
       "      <td>General|Endocrine|False|Medicine|Surgery|Urolo...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>65.216.253.25</td>\n",
       "      <td>c12f3f8f-8fcf-484a-90e1-1ac04db8cdcf</td>\n",
       "      <td>Arlington</td>\n",
       "      <td>22202.0</td>\n",
       "      <td>Online Medical Journal</td>\n",
       "      <td>https://www.cancertherapyadvisor.com/home/deci...</td>\n",
       "      <td>Bone Marrow|Radiography|Chronic|Oncology|Psych...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>66.232.79.22</td>\n",
       "      <td>a698de4b-e200-46dd-b5fb-40402175ae18</td>\n",
       "      <td>New Meadows</td>\n",
       "      <td>83654.0</td>\n",
       "      <td>Online Medical Journal</td>\n",
       "      <td>https://www.cancertherapyadvisor.com/home/canc...</td>\n",
       "      <td>General|Endocrine|False|Medicine|Surgery|Urolo...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>137.54.125.246</td>\n",
       "      <td>45967533-75c8-4fbd-a00c-e6ff20447aaa</td>\n",
       "      <td>None</td>\n",
       "      <td>229114624.0</td>\n",
       "      <td>Online Medical Journal</td>\n",
       "      <td>https://globalrph.com/medcalcs/warfarin-mainte...</td>\n",
       "      <td>Dental|Total|Clinical|Pharmacology|Physicians|...</td>\n",
       "      <td>2084P0800X</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>174.202.231.99</td>\n",
       "      <td>a17e25be-532d-4cf5-b916-9308c8c3961f</td>\n",
       "      <td>Houston</td>\n",
       "      <td>77008.0</td>\n",
       "      <td>Online Medical Journal</td>\n",
       "      <td>https://www.cureus.com/articles/58184-a-review...</td>\n",
       "      <td>Critical Care|Emergency Medicine|General Pract...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>24.59.163.238</td>\n",
       "      <td>3b644d8f-b225-4ea3-b370-160ff7858749</td>\n",
       "      <td>Syracuse</td>\n",
       "      <td>13215.0</td>\n",
       "      <td>Online Medical Journal</td>\n",
       "      <td>https://www.cureus.com/articles/131022-the-imp...</td>\n",
       "      <td>small|covid|nephrology|radiology|preventive me...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>67.203.36.94</td>\n",
       "      <td>456631b8-ca59-4e7a-96a0-53bcbf85eaef</td>\n",
       "      <td>Los Angeles</td>\n",
       "      <td>90060.0</td>\n",
       "      <td>Online Medical Journal</td>\n",
       "      <td>https://www.gastroenterologyadvisor.com/</td>\n",
       "      <td>Transplantation|Technology|Chronic|Neoplasms|U...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>24.28.40.219</td>\n",
       "      <td>1e84b963-2a2a-4e1a-a767-33867aa89bd4</td>\n",
       "      <td>Dallas</td>\n",
       "      <td>75204.0</td>\n",
       "      <td>Online Medical Journal</td>\n",
       "      <td>https://www.clinicaladvisor.com/home/features/...</td>\n",
       "      <td>Family Practice|Drainage|Clinical|Dermatology|...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>104.172.116.202</td>\n",
       "      <td>be61a8f7-0f00-487f-8a56-679c819fa01a</td>\n",
       "      <td>Granada Hills</td>\n",
       "      <td>91344.0</td>\n",
       "      <td>Online Medical Journal</td>\n",
       "      <td>https://www.cancertherapyadvisor.com/home/deci...</td>\n",
       "      <td>Intensive Care|Shock|Ischemic Heart Disease|He...</td>\n",
       "      <td>2084P0804X</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>170.178.236.25</td>\n",
       "      <td>ddbea19a-fa49-4d3d-965a-bee29cbe92da</td>\n",
       "      <td>Davenport</td>\n",
       "      <td>52807.0</td>\n",
       "      <td>Online Medical Journal</td>\n",
       "      <td>https://fpnotebook.com/Endo/Pharm/SgltInhbtr.htm</td>\n",
       "      <td>chronic kidney disease|nephrology|radiology|da...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>10 rows × 1622 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      BIDREQUESTIP                       USERPLATFORMUID       USERCITY  \\\n",
       "0     170.173.0.22  6974dcaa-f932-480e-9fb5-c52e20e1393a       Portland   \n",
       "1    65.216.253.25  c12f3f8f-8fcf-484a-90e1-1ac04db8cdcf      Arlington   \n",
       "2     66.232.79.22  a698de4b-e200-46dd-b5fb-40402175ae18    New Meadows   \n",
       "3   137.54.125.246  45967533-75c8-4fbd-a00c-e6ff20447aaa           None   \n",
       "4   174.202.231.99  a17e25be-532d-4cf5-b916-9308c8c3961f        Houston   \n",
       "5    24.59.163.238  3b644d8f-b225-4ea3-b370-160ff7858749       Syracuse   \n",
       "6     67.203.36.94  456631b8-ca59-4e7a-96a0-53bcbf85eaef    Los Angeles   \n",
       "7     24.28.40.219  1e84b963-2a2a-4e1a-a767-33867aa89bd4         Dallas   \n",
       "8  104.172.116.202  be61a8f7-0f00-487f-8a56-679c819fa01a  Granada Hills   \n",
       "9   170.178.236.25  ddbea19a-fa49-4d3d-965a-bee29cbe92da      Davenport   \n",
       "\n",
       "   USERZIPCODE            PLATFORMTYPE  \\\n",
       "0      97206.0  Online Medical Journal   \n",
       "1      22202.0  Online Medical Journal   \n",
       "2      83654.0  Online Medical Journal   \n",
       "3  229114624.0  Online Medical Journal   \n",
       "4      77008.0  Online Medical Journal   \n",
       "5      13215.0  Online Medical Journal   \n",
       "6      90060.0  Online Medical Journal   \n",
       "7      75204.0  Online Medical Journal   \n",
       "8      91344.0  Online Medical Journal   \n",
       "9      52807.0  Online Medical Journal   \n",
       "\n",
       "                                                 URL  \\\n",
       "0  https://www.cancertherapyadvisor.com/home/canc...   \n",
       "1  https://www.cancertherapyadvisor.com/home/deci...   \n",
       "2  https://www.cancertherapyadvisor.com/home/canc...   \n",
       "3  https://globalrph.com/medcalcs/warfarin-mainte...   \n",
       "4  https://www.cureus.com/articles/58184-a-review...   \n",
       "5  https://www.cureus.com/articles/131022-the-imp...   \n",
       "6           https://www.gastroenterologyadvisor.com/   \n",
       "7  https://www.clinicaladvisor.com/home/features/...   \n",
       "8  https://www.cancertherapyadvisor.com/home/deci...   \n",
       "9   https://fpnotebook.com/Endo/Pharm/SgltInhbtr.htm   \n",
       "\n",
       "                                            KEYWORDS    TAXONOMY  IS_HCP  \\\n",
       "0  General|Endocrine|False|Medicine|Surgery|Urolo...         NaN       0   \n",
       "1  Bone Marrow|Radiography|Chronic|Oncology|Psych...         NaN       0   \n",
       "2  General|Endocrine|False|Medicine|Surgery|Urolo...         NaN       0   \n",
       "3  Dental|Total|Clinical|Pharmacology|Physicians|...  2084P0800X       1   \n",
       "4  Critical Care|Emergency Medicine|General Pract...         NaN       0   \n",
       "5  small|covid|nephrology|radiology|preventive me...         NaN       0   \n",
       "6  Transplantation|Technology|Chronic|Neoplasms|U...         NaN       0   \n",
       "7  Family Practice|Drainage|Clinical|Dermatology|...         NaN       0   \n",
       "8  Intensive Care|Shock|Ischemic Heart Disease|He...  2084P0804X       1   \n",
       "9  chronic kidney disease|nephrology|radiology|da...         NaN       0   \n",
       "\n",
       "   gastrointestinal diseases  ...  pleurisy  pulmonary medicine  \\\n",
       "0                          0  ...         0                   0   \n",
       "1                          0  ...         0                   0   \n",
       "2                          0  ...         0                   0   \n",
       "3                          0  ...         0                   0   \n",
       "4                          0  ...         0                   0   \n",
       "5                          0  ...         0                   0   \n",
       "6                          0  ...         0                   0   \n",
       "7                          0  ...         0                   0   \n",
       "8                          0  ...         0                   0   \n",
       "9                          0  ...         0                   0   \n",
       "\n",
       "   Cardiac Output  earache  Cysts  asthma  Embolism  Conscious Sedation  \\\n",
       "0               0        0      0       0         0                   0   \n",
       "1               0        0      0       0         0                   0   \n",
       "2               0        0      0       0         0                   0   \n",
       "3               0        0      0       0         0                   0   \n",
       "4               0        0      0       0         0                   0   \n",
       "5               0        0      0       0         0                   0   \n",
       "6               0        0      0       0         0                   0   \n",
       "7               0        0      0       0         0                   0   \n",
       "8               1        0      0       0         0                   0   \n",
       "9               0        0      0       1         0                   0   \n",
       "\n",
       "   myeloproliferative neoplasms  q fever  \n",
       "0                             0        0  \n",
       "1                             0        0  \n",
       "2                             0        0  \n",
       "3                             0        0  \n",
       "4                             0        0  \n",
       "5                             0        0  \n",
       "6                             0        0  \n",
       "7                             0        0  \n",
       "8                             0        0  \n",
       "9                             0        0  \n",
       "\n",
       "[10 rows x 1622 columns]"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_df.head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "dad8caa9",
   "metadata": {},
   "outputs": [],
   "source": [
    "#test\n",
    "train_df = train_df.drop('TAXONOMY', axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "3c741c48",
   "metadata": {},
   "outputs": [],
   "source": [
    "# train_df.to_csv('train_df_with_KEYWORDS_cols.csv', index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c2c37e4b",
   "metadata": {},
   "source": [
    "# TAXANOMY"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "c381cc48",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>BIDREQUESTIP</th>\n",
       "      <th>USERPLATFORMUID</th>\n",
       "      <th>USERCITY</th>\n",
       "      <th>USERZIPCODE</th>\n",
       "      <th>PLATFORMTYPE</th>\n",
       "      <th>URL</th>\n",
       "      <th>KEYWORDS</th>\n",
       "      <th>IS_HCP</th>\n",
       "      <th>gastrointestinal diseases</th>\n",
       "      <th>opportunistic infections</th>\n",
       "      <th>...</th>\n",
       "      <th>pleurisy</th>\n",
       "      <th>pulmonary medicine</th>\n",
       "      <th>Cardiac Output</th>\n",
       "      <th>earache</th>\n",
       "      <th>Cysts</th>\n",
       "      <th>asthma</th>\n",
       "      <th>Embolism</th>\n",
       "      <th>Conscious Sedation</th>\n",
       "      <th>myeloproliferative neoplasms</th>\n",
       "      <th>q fever</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>170.173.0.22</td>\n",
       "      <td>6974dcaa-f932-480e-9fb5-c52e20e1393a</td>\n",
       "      <td>Portland</td>\n",
       "      <td>97206.0</td>\n",
       "      <td>Online Medical Journal</td>\n",
       "      <td>https://www.cancertherapyadvisor.com/home/canc...</td>\n",
       "      <td>General|Endocrine|False|Medicine|Surgery|Urolo...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>65.216.253.25</td>\n",
       "      <td>c12f3f8f-8fcf-484a-90e1-1ac04db8cdcf</td>\n",
       "      <td>Arlington</td>\n",
       "      <td>22202.0</td>\n",
       "      <td>Online Medical Journal</td>\n",
       "      <td>https://www.cancertherapyadvisor.com/home/deci...</td>\n",
       "      <td>Bone Marrow|Radiography|Chronic|Oncology|Psych...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>66.232.79.22</td>\n",
       "      <td>a698de4b-e200-46dd-b5fb-40402175ae18</td>\n",
       "      <td>New Meadows</td>\n",
       "      <td>83654.0</td>\n",
       "      <td>Online Medical Journal</td>\n",
       "      <td>https://www.cancertherapyadvisor.com/home/canc...</td>\n",
       "      <td>General|Endocrine|False|Medicine|Surgery|Urolo...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>137.54.125.246</td>\n",
       "      <td>45967533-75c8-4fbd-a00c-e6ff20447aaa</td>\n",
       "      <td>None</td>\n",
       "      <td>229114624.0</td>\n",
       "      <td>Online Medical Journal</td>\n",
       "      <td>https://globalrph.com/medcalcs/warfarin-mainte...</td>\n",
       "      <td>Dental|Total|Clinical|Pharmacology|Physicians|...</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>174.202.231.99</td>\n",
       "      <td>a17e25be-532d-4cf5-b916-9308c8c3961f</td>\n",
       "      <td>Houston</td>\n",
       "      <td>77008.0</td>\n",
       "      <td>Online Medical Journal</td>\n",
       "      <td>https://www.cureus.com/articles/58184-a-review...</td>\n",
       "      <td>Critical Care|Emergency Medicine|General Pract...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 1621 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "     BIDREQUESTIP                       USERPLATFORMUID     USERCITY  \\\n",
       "0    170.173.0.22  6974dcaa-f932-480e-9fb5-c52e20e1393a     Portland   \n",
       "1   65.216.253.25  c12f3f8f-8fcf-484a-90e1-1ac04db8cdcf    Arlington   \n",
       "2    66.232.79.22  a698de4b-e200-46dd-b5fb-40402175ae18  New Meadows   \n",
       "3  137.54.125.246  45967533-75c8-4fbd-a00c-e6ff20447aaa         None   \n",
       "4  174.202.231.99  a17e25be-532d-4cf5-b916-9308c8c3961f      Houston   \n",
       "\n",
       "   USERZIPCODE            PLATFORMTYPE  \\\n",
       "0      97206.0  Online Medical Journal   \n",
       "1      22202.0  Online Medical Journal   \n",
       "2      83654.0  Online Medical Journal   \n",
       "3  229114624.0  Online Medical Journal   \n",
       "4      77008.0  Online Medical Journal   \n",
       "\n",
       "                                                 URL  \\\n",
       "0  https://www.cancertherapyadvisor.com/home/canc...   \n",
       "1  https://www.cancertherapyadvisor.com/home/deci...   \n",
       "2  https://www.cancertherapyadvisor.com/home/canc...   \n",
       "3  https://globalrph.com/medcalcs/warfarin-mainte...   \n",
       "4  https://www.cureus.com/articles/58184-a-review...   \n",
       "\n",
       "                                            KEYWORDS  IS_HCP  \\\n",
       "0  General|Endocrine|False|Medicine|Surgery|Urolo...       0   \n",
       "1  Bone Marrow|Radiography|Chronic|Oncology|Psych...       0   \n",
       "2  General|Endocrine|False|Medicine|Surgery|Urolo...       0   \n",
       "3  Dental|Total|Clinical|Pharmacology|Physicians|...       1   \n",
       "4  Critical Care|Emergency Medicine|General Pract...       0   \n",
       "\n",
       "   gastrointestinal diseases  opportunistic infections  ...  pleurisy  \\\n",
       "0                          0                         0  ...         0   \n",
       "1                          0                         0  ...         0   \n",
       "2                          0                         0  ...         0   \n",
       "3                          0                         0  ...         0   \n",
       "4                          0                         0  ...         0   \n",
       "\n",
       "   pulmonary medicine  Cardiac Output  earache  Cysts  asthma  Embolism  \\\n",
       "0                   0               0        0      0       0         0   \n",
       "1                   0               0        0      0       0         0   \n",
       "2                   0               0        0      0       0         0   \n",
       "3                   0               0        0      0       0         0   \n",
       "4                   0               0        0      0       0         0   \n",
       "\n",
       "   Conscious Sedation  myeloproliferative neoplasms  q fever  \n",
       "0                   0                             0        0  \n",
       "1                   0                             0        0  \n",
       "2                   0                             0        0  \n",
       "3                   0                             0        0  \n",
       "4                   0                             0        0  \n",
       "\n",
       "[5 rows x 1621 columns]"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "fd87e890",
   "metadata": {},
   "outputs": [],
   "source": [
    "# from sklearn.preprocessing import LabelEncoder\n",
    "\n",
    "# # Create an instance of LabelEncoder\n",
    "# label_encoder = LabelEncoder()\n",
    "\n",
    "# # Fit and transform the 'KEYWORDS' column\n",
    "# train_df['TAXONOMY_ENCODED'] = label_encoder.fit_transform(train_df['TAXONOMY'])\n",
    "\n",
    "# # Display the encoded values\n",
    "# print(train_df['TAXONOMY_ENCODED'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "0aad1569",
   "metadata": {},
   "outputs": [],
   "source": [
    "# import pandas as pd\n",
    "\n",
    "# # Compute the correlation matrix\n",
    "# correlation_matrix = train_df[['TAXONOMY_ENCODED'] + [\n",
    "#     'diabetic retinopathy',\n",
    "#     'Total',\n",
    "#     'lung diseases',\n",
    "#     'arrhythmias',\n",
    "#     'glanders',\n",
    "#     'fasting plasma glucose',\n",
    "#     'Bradycardia',\n",
    "#     'Tachycardia',\n",
    "#     'herbal medicine',\n",
    "#     'actinomycosis'\n",
    "# ]].corr()\n",
    "\n",
    "# # Display the correlation matrix\n",
    "# print(correlation_matrix)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "e7f47946",
   "metadata": {},
   "outputs": [],
   "source": [
    "# import pandas as pd\n",
    "# import seaborn as sns\n",
    "# import matplotlib.pyplot as plt\n",
    "\n",
    "# # Compute the correlation matrix\n",
    "# correlation_matrix = train_df[['TAXONOMY_ENCODED'] + [\n",
    "#     'diabetic retinopathy',\n",
    "#     'Total',\n",
    "#     'lung diseases',\n",
    "#     'arrhythmias',\n",
    "#     'glanders',\n",
    "#     'fasting plasma glucose',\n",
    "#     'Bradycardia',\n",
    "#     'Tachycardia',\n",
    "#     'herbal medicine',\n",
    "#     'actinomycosis'\n",
    "# ]].corr()\n",
    "\n",
    "# # Plot the correlation matrix as a heatmap\n",
    "# plt.figure(figsize=(10, 8))\n",
    "# sns.heatmap(correlation_matrix, annot=True, cmap='coolwarm', square=True)\n",
    "# plt.title('Correlation Matrix')\n",
    "# plt.show()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "7c21aece",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>BIDREQUESTIP</th>\n",
       "      <th>USERPLATFORMUID</th>\n",
       "      <th>USERCITY</th>\n",
       "      <th>USERZIPCODE</th>\n",
       "      <th>PLATFORMTYPE</th>\n",
       "      <th>URL</th>\n",
       "      <th>KEYWORDS</th>\n",
       "      <th>IS_HCP</th>\n",
       "      <th>gastrointestinal diseases</th>\n",
       "      <th>opportunistic infections</th>\n",
       "      <th>...</th>\n",
       "      <th>pleurisy</th>\n",
       "      <th>pulmonary medicine</th>\n",
       "      <th>Cardiac Output</th>\n",
       "      <th>earache</th>\n",
       "      <th>Cysts</th>\n",
       "      <th>asthma</th>\n",
       "      <th>Embolism</th>\n",
       "      <th>Conscious Sedation</th>\n",
       "      <th>myeloproliferative neoplasms</th>\n",
       "      <th>q fever</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>170.173.0.22</td>\n",
       "      <td>6974dcaa-f932-480e-9fb5-c52e20e1393a</td>\n",
       "      <td>Portland</td>\n",
       "      <td>97206.0</td>\n",
       "      <td>Online Medical Journal</td>\n",
       "      <td>https://www.cancertherapyadvisor.com/home/canc...</td>\n",
       "      <td>General|Endocrine|False|Medicine|Surgery|Urolo...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>65.216.253.25</td>\n",
       "      <td>c12f3f8f-8fcf-484a-90e1-1ac04db8cdcf</td>\n",
       "      <td>Arlington</td>\n",
       "      <td>22202.0</td>\n",
       "      <td>Online Medical Journal</td>\n",
       "      <td>https://www.cancertherapyadvisor.com/home/deci...</td>\n",
       "      <td>Bone Marrow|Radiography|Chronic|Oncology|Psych...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>66.232.79.22</td>\n",
       "      <td>a698de4b-e200-46dd-b5fb-40402175ae18</td>\n",
       "      <td>New Meadows</td>\n",
       "      <td>83654.0</td>\n",
       "      <td>Online Medical Journal</td>\n",
       "      <td>https://www.cancertherapyadvisor.com/home/canc...</td>\n",
       "      <td>General|Endocrine|False|Medicine|Surgery|Urolo...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>137.54.125.246</td>\n",
       "      <td>45967533-75c8-4fbd-a00c-e6ff20447aaa</td>\n",
       "      <td>None</td>\n",
       "      <td>229114624.0</td>\n",
       "      <td>Online Medical Journal</td>\n",
       "      <td>https://globalrph.com/medcalcs/warfarin-mainte...</td>\n",
       "      <td>Dental|Total|Clinical|Pharmacology|Physicians|...</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>174.202.231.99</td>\n",
       "      <td>a17e25be-532d-4cf5-b916-9308c8c3961f</td>\n",
       "      <td>Houston</td>\n",
       "      <td>77008.0</td>\n",
       "      <td>Online Medical Journal</td>\n",
       "      <td>https://www.cureus.com/articles/58184-a-review...</td>\n",
       "      <td>Critical Care|Emergency Medicine|General Pract...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>113932</th>\n",
       "      <td>68.82.97.126</td>\n",
       "      <td>051e4a06-4cb4-4b97-b0b1-d1c6a2b8e2c2</td>\n",
       "      <td>Philadelphia</td>\n",
       "      <td>19120.0</td>\n",
       "      <td>Online Medical Journal</td>\n",
       "      <td>https://www.clinicaladvisor.com/home/consultat...</td>\n",
       "      <td>Cardiovascular|Clinical|False|Male|Oncology|Ps...</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>113933</th>\n",
       "      <td>104.172.11.109</td>\n",
       "      <td>3289bba5-4a50-4b61-a5cc-68596bbdc2bf</td>\n",
       "      <td>Van Nuys</td>\n",
       "      <td>914014213.0</td>\n",
       "      <td>Online Medical Journal</td>\n",
       "      <td>https://www.clinicaladvisor.com/home/features/...</td>\n",
       "      <td>Asthma|Arthritis|General|Clinical|Diagnosis|Rh...</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>113934</th>\n",
       "      <td>174.21.94.113</td>\n",
       "      <td>9c36e90c-caa7-4323-8431-fb84f8293972</td>\n",
       "      <td>Seattle</td>\n",
       "      <td>98105.0</td>\n",
       "      <td>Online Medical Journal</td>\n",
       "      <td>https://www.cureus.com/articles/39455-whats-th...</td>\n",
       "      <td>Rehabilitation|Transplantation|Clinical|Preven...</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>113935</th>\n",
       "      <td>69.253.129.131</td>\n",
       "      <td>afc27760-31f7-4f6e-aa08-90d8a96525d0</td>\n",
       "      <td>Wilmington</td>\n",
       "      <td>19808.0</td>\n",
       "      <td>Online Medical Journal</td>\n",
       "      <td>https://www.dermatologyadvisor.com/home/topics...</td>\n",
       "      <td>pain|disease|autoimmune|multiple sclerosis|gra...</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>113936</th>\n",
       "      <td>108.41.233.175</td>\n",
       "      <td>d26887c5-15f9-4b72-80dc-728039529e60</td>\n",
       "      <td>White Plains</td>\n",
       "      <td>10605.0</td>\n",
       "      <td>Online Medical Journal</td>\n",
       "      <td>https://www.medicalbag.com/</td>\n",
       "      <td>Genetic|Large|Autoimmune|Health|False|Abortion...</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>113933 rows × 1621 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "          BIDREQUESTIP                       USERPLATFORMUID      USERCITY  \\\n",
       "0         170.173.0.22  6974dcaa-f932-480e-9fb5-c52e20e1393a      Portland   \n",
       "1        65.216.253.25  c12f3f8f-8fcf-484a-90e1-1ac04db8cdcf     Arlington   \n",
       "2         66.232.79.22  a698de4b-e200-46dd-b5fb-40402175ae18   New Meadows   \n",
       "3       137.54.125.246  45967533-75c8-4fbd-a00c-e6ff20447aaa          None   \n",
       "4       174.202.231.99  a17e25be-532d-4cf5-b916-9308c8c3961f       Houston   \n",
       "...                ...                                   ...           ...   \n",
       "113932    68.82.97.126  051e4a06-4cb4-4b97-b0b1-d1c6a2b8e2c2  Philadelphia   \n",
       "113933  104.172.11.109  3289bba5-4a50-4b61-a5cc-68596bbdc2bf      Van Nuys   \n",
       "113934   174.21.94.113  9c36e90c-caa7-4323-8431-fb84f8293972       Seattle   \n",
       "113935  69.253.129.131  afc27760-31f7-4f6e-aa08-90d8a96525d0    Wilmington   \n",
       "113936  108.41.233.175  d26887c5-15f9-4b72-80dc-728039529e60  White Plains   \n",
       "\n",
       "        USERZIPCODE            PLATFORMTYPE  \\\n",
       "0           97206.0  Online Medical Journal   \n",
       "1           22202.0  Online Medical Journal   \n",
       "2           83654.0  Online Medical Journal   \n",
       "3       229114624.0  Online Medical Journal   \n",
       "4           77008.0  Online Medical Journal   \n",
       "...             ...                     ...   \n",
       "113932      19120.0  Online Medical Journal   \n",
       "113933  914014213.0  Online Medical Journal   \n",
       "113934      98105.0  Online Medical Journal   \n",
       "113935      19808.0  Online Medical Journal   \n",
       "113936      10605.0  Online Medical Journal   \n",
       "\n",
       "                                                      URL  \\\n",
       "0       https://www.cancertherapyadvisor.com/home/canc...   \n",
       "1       https://www.cancertherapyadvisor.com/home/deci...   \n",
       "2       https://www.cancertherapyadvisor.com/home/canc...   \n",
       "3       https://globalrph.com/medcalcs/warfarin-mainte...   \n",
       "4       https://www.cureus.com/articles/58184-a-review...   \n",
       "...                                                   ...   \n",
       "113932  https://www.clinicaladvisor.com/home/consultat...   \n",
       "113933  https://www.clinicaladvisor.com/home/features/...   \n",
       "113934  https://www.cureus.com/articles/39455-whats-th...   \n",
       "113935  https://www.dermatologyadvisor.com/home/topics...   \n",
       "113936                        https://www.medicalbag.com/   \n",
       "\n",
       "                                                 KEYWORDS  IS_HCP  \\\n",
       "0       General|Endocrine|False|Medicine|Surgery|Urolo...       0   \n",
       "1       Bone Marrow|Radiography|Chronic|Oncology|Psych...       0   \n",
       "2       General|Endocrine|False|Medicine|Surgery|Urolo...       0   \n",
       "3       Dental|Total|Clinical|Pharmacology|Physicians|...       1   \n",
       "4       Critical Care|Emergency Medicine|General Pract...       0   \n",
       "...                                                   ...     ...   \n",
       "113932  Cardiovascular|Clinical|False|Male|Oncology|Ps...       1   \n",
       "113933  Asthma|Arthritis|General|Clinical|Diagnosis|Rh...       1   \n",
       "113934  Rehabilitation|Transplantation|Clinical|Preven...       1   \n",
       "113935  pain|disease|autoimmune|multiple sclerosis|gra...       1   \n",
       "113936  Genetic|Large|Autoimmune|Health|False|Abortion...       1   \n",
       "\n",
       "        gastrointestinal diseases  opportunistic infections  ...  pleurisy  \\\n",
       "0                               0                         0  ...         0   \n",
       "1                               0                         0  ...         0   \n",
       "2                               0                         0  ...         0   \n",
       "3                               0                         0  ...         0   \n",
       "4                               0                         0  ...         0   \n",
       "...                           ...                       ...  ...       ...   \n",
       "113932                          0                         0  ...         0   \n",
       "113933                          0                         0  ...         0   \n",
       "113934                          0                         0  ...         0   \n",
       "113935                          0                         0  ...         0   \n",
       "113936                          0                         0  ...         0   \n",
       "\n",
       "        pulmonary medicine  Cardiac Output  earache  Cysts  asthma  Embolism  \\\n",
       "0                        0               0        0      0       0         0   \n",
       "1                        0               0        0      0       0         0   \n",
       "2                        0               0        0      0       0         0   \n",
       "3                        0               0        0      0       0         0   \n",
       "4                        0               0        0      0       0         0   \n",
       "...                    ...             ...      ...    ...     ...       ...   \n",
       "113932                   0               0        0      0       0         0   \n",
       "113933                   0               0        0      0       0         0   \n",
       "113934                   0               0        0      0       0         0   \n",
       "113935                   0               0        0      0       0         0   \n",
       "113936                   0               0        0      0       0         0   \n",
       "\n",
       "        Conscious Sedation  myeloproliferative neoplasms  q fever  \n",
       "0                        0                             0        0  \n",
       "1                        0                             0        0  \n",
       "2                        0                             0        0  \n",
       "3                        0                             0        0  \n",
       "4                        0                             0        0  \n",
       "...                    ...                           ...      ...  \n",
       "113932                   0                             0        0  \n",
       "113933                   0                             0        0  \n",
       "113934                   0                             0        0  \n",
       "113935                   0                             0        0  \n",
       "113936                   0                             0        0  \n",
       "\n",
       "[113933 rows x 1621 columns]"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "4928976c",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "          BIDREQUESTIP                       USERPLATFORMUID      USERCITY  \\\n",
      "0         170.173.0.22  6974dcaa-f932-480e-9fb5-c52e20e1393a      Portland   \n",
      "1        65.216.253.25  c12f3f8f-8fcf-484a-90e1-1ac04db8cdcf     Arlington   \n",
      "2         66.232.79.22  a698de4b-e200-46dd-b5fb-40402175ae18   New Meadows   \n",
      "3       137.54.125.246  45967533-75c8-4fbd-a00c-e6ff20447aaa          None   \n",
      "4       174.202.231.99  a17e25be-532d-4cf5-b916-9308c8c3961f       Houston   \n",
      "...                ...                                   ...           ...   \n",
      "113936  108.41.233.175  d26887c5-15f9-4b72-80dc-728039529e60  White Plains   \n",
      "59158              NaN                                   NaN           NaN   \n",
      "80923              NaN                                   NaN           NaN   \n",
      "99416              NaN                                   NaN           NaN   \n",
      "102512             NaN                                   NaN           NaN   \n",
      "\n",
      "        USERZIPCODE            PLATFORMTYPE  \\\n",
      "0           97206.0  Online Medical Journal   \n",
      "1           22202.0  Online Medical Journal   \n",
      "2           83654.0  Online Medical Journal   \n",
      "3       229114624.0  Online Medical Journal   \n",
      "4           77008.0  Online Medical Journal   \n",
      "...             ...                     ...   \n",
      "113936      10605.0  Online Medical Journal   \n",
      "59158           NaN                     NaN   \n",
      "80923           NaN                     NaN   \n",
      "99416           NaN                     NaN   \n",
      "102512          NaN                     NaN   \n",
      "\n",
      "                                                      URL  \\\n",
      "0       https://www.cancertherapyadvisor.com/home/canc...   \n",
      "1       https://www.cancertherapyadvisor.com/home/deci...   \n",
      "2       https://www.cancertherapyadvisor.com/home/canc...   \n",
      "3       https://globalrph.com/medcalcs/warfarin-mainte...   \n",
      "4       https://www.cureus.com/articles/58184-a-review...   \n",
      "...                                                   ...   \n",
      "113936                        https://www.medicalbag.com/   \n",
      "59158                                                 NaN   \n",
      "80923                                                 NaN   \n",
      "99416                                                 NaN   \n",
      "102512                                                NaN   \n",
      "\n",
      "                                                 KEYWORDS  IS_HCP  \\\n",
      "0       General|Endocrine|False|Medicine|Surgery|Urolo...     0.0   \n",
      "1       Bone Marrow|Radiography|Chronic|Oncology|Psych...     0.0   \n",
      "2       General|Endocrine|False|Medicine|Surgery|Urolo...     0.0   \n",
      "3       Dental|Total|Clinical|Pharmacology|Physicians|...     1.0   \n",
      "4       Critical Care|Emergency Medicine|General Pract...     0.0   \n",
      "...                                                   ...     ...   \n",
      "113936  Genetic|Large|Autoimmune|Health|False|Abortion...     1.0   \n",
      "59158                                                 NaN     NaN   \n",
      "80923                                                 NaN     NaN   \n",
      "99416                                                 NaN     NaN   \n",
      "102512                                                NaN     NaN   \n",
      "\n",
      "        gastrointestinal diseases  opportunistic infections  ...  asthma  \\\n",
      "0                             0.0                       0.0  ...     0.0   \n",
      "1                             0.0                       0.0  ...     0.0   \n",
      "2                             0.0                       0.0  ...     0.0   \n",
      "3                             0.0                       0.0  ...     0.0   \n",
      "4                             0.0                       0.0  ...     0.0   \n",
      "...                           ...                       ...  ...     ...   \n",
      "113936                        0.0                       0.0  ...     0.0   \n",
      "59158                         NaN                       NaN  ...     NaN   \n",
      "80923                         NaN                       NaN  ...     NaN   \n",
      "99416                         NaN                       NaN  ...     NaN   \n",
      "102512                        NaN                       NaN  ...     NaN   \n",
      "\n",
      "        Embolism  Conscious Sedation  myeloproliferative neoplasms  q fever  \\\n",
      "0            0.0                 0.0                           0.0      0.0   \n",
      "1            0.0                 0.0                           0.0      0.0   \n",
      "2            0.0                 0.0                           0.0      0.0   \n",
      "3            0.0                 0.0                           0.0      0.0   \n",
      "4            0.0                 0.0                           0.0      0.0   \n",
      "...          ...                 ...                           ...      ...   \n",
      "113936       0.0                 0.0                           0.0      0.0   \n",
      "59158        NaN                 NaN                           NaN      NaN   \n",
      "80923        NaN                 NaN                           NaN      NaN   \n",
      "99416        NaN                 NaN                           NaN      NaN   \n",
      "102512       NaN                 NaN                           NaN      NaN   \n",
      "\n",
      "        Electronic Health Record  Medical News Platform  \\\n",
      "0                            0.0                    0.0   \n",
      "1                            0.0                    0.0   \n",
      "2                            0.0                    0.0   \n",
      "3                            0.0                    0.0   \n",
      "4                            0.0                    0.0   \n",
      "...                          ...                    ...   \n",
      "113936                       NaN                    NaN   \n",
      "59158                        0.0                    0.0   \n",
      "80923                        0.0                    0.0   \n",
      "99416                        0.0                    0.0   \n",
      "102512                       0.0                    0.0   \n",
      "\n",
      "        Online Learning Portal  Online Medical Journal  Tele-medicine Platform  \n",
      "0                          0.0                     1.0                     0.0  \n",
      "1                          0.0                     1.0                     0.0  \n",
      "2                          0.0                     1.0                     0.0  \n",
      "3                          0.0                     1.0                     0.0  \n",
      "4                          0.0                     1.0                     0.0  \n",
      "...                        ...                     ...                     ...  \n",
      "113936                     NaN                     NaN                     NaN  \n",
      "59158                      0.0                     1.0                     0.0  \n",
      "80923                      0.0                     1.0                     0.0  \n",
      "99416                      0.0                     1.0                     0.0  \n",
      "102512                     0.0                     1.0                     0.0  \n",
      "\n",
      "[113937 rows x 1626 columns]\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "from sklearn.preprocessing import OneHotEncoder\n",
    "\n",
    "\n",
    "encoder = OneHotEncoder(sparse=False)\n",
    "platform_type = train_df['PLATFORMTYPE'].values.reshape(-1, 1)\n",
    "encoded_features = encoder.fit_transform(platform_type)\n",
    "encoded_df = pd.DataFrame(encoded_features, columns=encoder.categories_[0])\n",
    "\n",
    "\n",
    "df_encoded = pd.concat([train_df, encoded_df], axis=1)\n",
    "\n",
    "\n",
    "print(df_encoded)\n",
    "train_df = df_encoded"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "aa67183a",
   "metadata": {},
   "outputs": [],
   "source": [
    "train_df.drop('PLATFORMTYPE', axis=1, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "id": "0a297533",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(113937, 1625)"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "id": "e215b8f4",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['BIDREQUESTIP', 'USERPLATFORMUID', 'USERCITY', 'USERZIPCODE', 'URL',\n",
       "       'KEYWORDS', 'IS_HCP', 'gastrointestinal diseases',\n",
       "       'opportunistic infections', 'hamartoma',\n",
       "       ...\n",
       "       'asthma', 'Embolism', 'Conscious Sedation',\n",
       "       'myeloproliferative neoplasms', 'q fever', 'Electronic Health Record',\n",
       "       'Medical News Platform', 'Online Learning Portal',\n",
       "       'Online Medical Journal', 'Tele-medicine Platform'],\n",
       "      dtype='object', length=1625)"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_df.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "id": "5e93a55d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "5929"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_df['USERCITY'].isna().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "id": "89d422ca",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "4596"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_df['USERZIPCODE'].isna().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "id": "4cc588e4",
   "metadata": {},
   "outputs": [],
   "source": [
    "train_df.dropna(subset=['USERCITY'], inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "id": "47acacdc",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(108008, 1625)"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "id": "9aadfbe2",
   "metadata": {},
   "outputs": [],
   "source": [
    "train_df.dropna(subset=['USERZIPCODE'], inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "id": "bdfe33fb",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(108006, 1625)"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "id": "b55368cc",
   "metadata": {},
   "outputs": [],
   "source": [
    "columns_to_drop = ['BIDREQUESTIP', 'USERPLATFORMUID','URL','KEYWORDS']\n",
    "train_df.drop(columns_to_drop, axis=1, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "id": "d905a811",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>USERCITY</th>\n",
       "      <th>USERZIPCODE</th>\n",
       "      <th>IS_HCP</th>\n",
       "      <th>gastrointestinal diseases</th>\n",
       "      <th>opportunistic infections</th>\n",
       "      <th>hamartoma</th>\n",
       "      <th>carcinoma</th>\n",
       "      <th>large intestine</th>\n",
       "      <th>ustekinumab</th>\n",
       "      <th>famciclovir</th>\n",
       "      <th>...</th>\n",
       "      <th>asthma</th>\n",
       "      <th>Embolism</th>\n",
       "      <th>Conscious Sedation</th>\n",
       "      <th>myeloproliferative neoplasms</th>\n",
       "      <th>q fever</th>\n",
       "      <th>Electronic Health Record</th>\n",
       "      <th>Medical News Platform</th>\n",
       "      <th>Online Learning Portal</th>\n",
       "      <th>Online Medical Journal</th>\n",
       "      <th>Tele-medicine Platform</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Portland</td>\n",
       "      <td>97206.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Arlington</td>\n",
       "      <td>22202.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>New Meadows</td>\n",
       "      <td>83654.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Houston</td>\n",
       "      <td>77008.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>Syracuse</td>\n",
       "      <td>13215.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 1621 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      USERCITY  USERZIPCODE  IS_HCP  gastrointestinal diseases  \\\n",
       "0     Portland      97206.0     0.0                        0.0   \n",
       "1    Arlington      22202.0     0.0                        0.0   \n",
       "2  New Meadows      83654.0     0.0                        0.0   \n",
       "4      Houston      77008.0     0.0                        0.0   \n",
       "5     Syracuse      13215.0     0.0                        0.0   \n",
       "\n",
       "   opportunistic infections  hamartoma  carcinoma  large intestine  \\\n",
       "0                       0.0        0.0        0.0              0.0   \n",
       "1                       0.0        0.0        0.0              0.0   \n",
       "2                       0.0        0.0        0.0              0.0   \n",
       "4                       0.0        0.0        0.0              0.0   \n",
       "5                       0.0        0.0        0.0              0.0   \n",
       "\n",
       "   ustekinumab  famciclovir  ...  asthma  Embolism  Conscious Sedation  \\\n",
       "0          0.0          0.0  ...     0.0       0.0                 0.0   \n",
       "1          0.0          0.0  ...     0.0       0.0                 0.0   \n",
       "2          0.0          0.0  ...     0.0       0.0                 0.0   \n",
       "4          0.0          0.0  ...     0.0       0.0                 0.0   \n",
       "5          0.0          0.0  ...     0.0       0.0                 0.0   \n",
       "\n",
       "   myeloproliferative neoplasms  q fever  Electronic Health Record  \\\n",
       "0                           0.0      0.0                       0.0   \n",
       "1                           0.0      0.0                       0.0   \n",
       "2                           0.0      0.0                       0.0   \n",
       "4                           0.0      0.0                       0.0   \n",
       "5                           0.0      0.0                       0.0   \n",
       "\n",
       "   Medical News Platform  Online Learning Portal  Online Medical Journal  \\\n",
       "0                    0.0                     0.0                     1.0   \n",
       "1                    0.0                     0.0                     1.0   \n",
       "2                    0.0                     0.0                     1.0   \n",
       "4                    0.0                     0.0                     1.0   \n",
       "5                    0.0                     0.0                     1.0   \n",
       "\n",
       "   Tele-medicine Platform  \n",
       "0                     0.0  \n",
       "1                     0.0  \n",
       "2                     0.0  \n",
       "4                     0.0  \n",
       "5                     0.0  \n",
       "\n",
       "[5 rows x 1621 columns]"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_df.head(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "id": "964d2655",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>USERCITY</th>\n",
       "      <th>USERZIPCODE</th>\n",
       "      <th>IS_HCP</th>\n",
       "      <th>gastrointestinal diseases</th>\n",
       "      <th>opportunistic infections</th>\n",
       "      <th>hamartoma</th>\n",
       "      <th>carcinoma</th>\n",
       "      <th>large intestine</th>\n",
       "      <th>ustekinumab</th>\n",
       "      <th>famciclovir</th>\n",
       "      <th>...</th>\n",
       "      <th>asthma</th>\n",
       "      <th>Embolism</th>\n",
       "      <th>Conscious Sedation</th>\n",
       "      <th>myeloproliferative neoplasms</th>\n",
       "      <th>q fever</th>\n",
       "      <th>Electronic Health Record</th>\n",
       "      <th>Medical News Platform</th>\n",
       "      <th>Online Learning Portal</th>\n",
       "      <th>Online Medical Journal</th>\n",
       "      <th>Tele-medicine Platform</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>3176</td>\n",
       "      <td>8934</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>130</td>\n",
       "      <td>2207</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2700</td>\n",
       "      <td>7585</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1766</td>\n",
       "      <td>7018</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>3877</td>\n",
       "      <td>1351</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 1621 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   USERCITY  USERZIPCODE  IS_HCP  gastrointestinal diseases  \\\n",
       "0      3176         8934     0.0                        0.0   \n",
       "1       130         2207     0.0                        0.0   \n",
       "2      2700         7585     0.0                        0.0   \n",
       "4      1766         7018     0.0                        0.0   \n",
       "5      3877         1351     0.0                        0.0   \n",
       "\n",
       "   opportunistic infections  hamartoma  carcinoma  large intestine  \\\n",
       "0                       0.0        0.0        0.0              0.0   \n",
       "1                       0.0        0.0        0.0              0.0   \n",
       "2                       0.0        0.0        0.0              0.0   \n",
       "4                       0.0        0.0        0.0              0.0   \n",
       "5                       0.0        0.0        0.0              0.0   \n",
       "\n",
       "   ustekinumab  famciclovir  ...  asthma  Embolism  Conscious Sedation  \\\n",
       "0          0.0          0.0  ...     0.0       0.0                 0.0   \n",
       "1          0.0          0.0  ...     0.0       0.0                 0.0   \n",
       "2          0.0          0.0  ...     0.0       0.0                 0.0   \n",
       "4          0.0          0.0  ...     0.0       0.0                 0.0   \n",
       "5          0.0          0.0  ...     0.0       0.0                 0.0   \n",
       "\n",
       "   myeloproliferative neoplasms  q fever  Electronic Health Record  \\\n",
       "0                           0.0      0.0                       0.0   \n",
       "1                           0.0      0.0                       0.0   \n",
       "2                           0.0      0.0                       0.0   \n",
       "4                           0.0      0.0                       0.0   \n",
       "5                           0.0      0.0                       0.0   \n",
       "\n",
       "   Medical News Platform  Online Learning Portal  Online Medical Journal  \\\n",
       "0                    0.0                     0.0                     1.0   \n",
       "1                    0.0                     0.0                     1.0   \n",
       "2                    0.0                     0.0                     1.0   \n",
       "4                    0.0                     0.0                     1.0   \n",
       "5                    0.0                     0.0                     1.0   \n",
       "\n",
       "   Tele-medicine Platform  \n",
       "0                     0.0  \n",
       "1                     0.0  \n",
       "2                     0.0  \n",
       "4                     0.0  \n",
       "5                     0.0  \n",
       "\n",
       "[5 rows x 1621 columns]"
      ]
     },
     "execution_count": 46,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "from sklearn.preprocessing import LabelEncoder\n",
    "\n",
    "city_encoder = LabelEncoder()\n",
    "train_df['USERCITY'] = city_encoder.fit_transform(train_df['USERCITY'])\n",
    "\n",
    "\n",
    "zipcode_encoder = LabelEncoder()\n",
    "train_df['USERZIPCODE'] = zipcode_encoder.fit_transform(train_df['USERZIPCODE'])\n",
    "\n",
    "\n",
    "train_df.head(5)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "id": "e92348e9",
   "metadata": {},
   "outputs": [],
   "source": [
    "# X_train = feature_engineered_train_data.drop('HCP', axis=1)\n",
    "# y_train = feature_engineered_train_data['HCP']\n",
    "\n",
    "# # Split the test dataset into features and target variable\n",
    "# X_test = feature_engineered_test_data.drop('HCP', axis=1)\n",
    "# y_test = feature_engineered_test_data['HCP']\n",
    "\n",
    "# # Define the XGBoost classifier\n",
    "# model = xgb.XGBClassifier()\n",
    "\n",
    "# # Train the model\n",
    "# model.fit(X_train, y_train)\n",
    "\n",
    "# # Make predictions on the test set\n",
    "# y_pred = model.predict(X_test)\n",
    "\n",
    "# # Evaluate the model\n",
    "# accuracy = accuracy_score(y_test, y_pred)\n",
    "# print(\"Accuracy: {:.2f}%\".format(accuracy * 100))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "48a1ba28",
   "metadata": {},
   "source": [
    "Changes made for the test_df file in the below section:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "id": "6de5e8dd",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ID</th>\n",
       "      <th>DEVICETYPE</th>\n",
       "      <th>PLATFORM_ID</th>\n",
       "      <th>BIDREQUESTIP</th>\n",
       "      <th>USERPLATFORMUID</th>\n",
       "      <th>USERCITY</th>\n",
       "      <th>USERZIPCODE</th>\n",
       "      <th>USERAGENT</th>\n",
       "      <th>PLATFORMTYPE</th>\n",
       "      <th>CHANNELTYPE</th>\n",
       "      <th>URL</th>\n",
       "      <th>KEYWORDS</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>115501</td>\n",
       "      <td>Desktop</td>\n",
       "      <td>2</td>\n",
       "      <td>75.189.231.103</td>\n",
       "      <td>0d5041ff-f0b6-4d1a-9ad7-0a29f7d485b4</td>\n",
       "      <td>Fayetteville</td>\n",
       "      <td>28305.0</td>\n",
       "      <td>Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_6...</td>\n",
       "      <td>Online Medical Journal</td>\n",
       "      <td>Website</td>\n",
       "      <td>https://www.clinicaladvisor.com/home/features/...</td>\n",
       "      <td>Family Practice|Drainage|Clinical|Dermatology|...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>115502</td>\n",
       "      <td>Mobile</td>\n",
       "      <td>2</td>\n",
       "      <td>24.101.33.158</td>\n",
       "      <td>c8396dd0-969f-4d99-a40b-b7bb1f516154</td>\n",
       "      <td>Conneaut Lake</td>\n",
       "      <td>16316.0</td>\n",
       "      <td>Mozilla/5.0 (iPhone; CPU iPhone OS 15_6_1 like...</td>\n",
       "      <td>Online Medical Journal</td>\n",
       "      <td>Website</td>\n",
       "      <td>https://www.ophthalmologyadvisor.com/topics/ca...</td>\n",
       "      <td>General|Clinical|Operative|Medicine|Cardiology...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>115503</td>\n",
       "      <td>Desktop</td>\n",
       "      <td>2</td>\n",
       "      <td>172.118.216.142</td>\n",
       "      <td>3c97a081-6518-43f8-9f26-369759cfb471</td>\n",
       "      <td>Covina</td>\n",
       "      <td>91724.0</td>\n",
       "      <td>Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7...</td>\n",
       "      <td>Online Medical Journal</td>\n",
       "      <td>Website</td>\n",
       "      <td>https://www.psychiatryadvisor.com/author/tori-...</td>\n",
       "      <td>Abortion|Anxiety Disorders|Apnea|False|Trauma|...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>115504</td>\n",
       "      <td>Desktop</td>\n",
       "      <td>7</td>\n",
       "      <td>71.105.120.171</td>\n",
       "      <td>3e2578c8-f794-41af-a38c-c5cfb3c0f014</td>\n",
       "      <td>Brooklyn</td>\n",
       "      <td>11226.0</td>\n",
       "      <td>Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7...</td>\n",
       "      <td>Online Medical Journal</td>\n",
       "      <td>Website</td>\n",
       "      <td>https://www.cureus.com/articles/105482-diverti...</td>\n",
       "      <td>Health|Male|Neurological Surgery|Otolaryngolog...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>115505</td>\n",
       "      <td>Desktop</td>\n",
       "      <td>2</td>\n",
       "      <td>73.82.211.73</td>\n",
       "      <td>ec2ae7ce-6a8c-4156-98a7-07203e60f483</td>\n",
       "      <td>Marietta</td>\n",
       "      <td>30062.0</td>\n",
       "      <td>Mozilla/5.0 (Windows NT 10.0; Win64; x64) Appl...</td>\n",
       "      <td>Online Medical Journal</td>\n",
       "      <td>Website</td>\n",
       "      <td>https://www.renalandurologynews.com/home/confe...</td>\n",
       "      <td>chronic kidney disease|pain|nephrology|disease...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       ID DEVICETYPE  PLATFORM_ID     BIDREQUESTIP  \\\n",
       "0  115501    Desktop            2   75.189.231.103   \n",
       "1  115502     Mobile            2    24.101.33.158   \n",
       "2  115503    Desktop            2  172.118.216.142   \n",
       "3  115504    Desktop            7   71.105.120.171   \n",
       "4  115505    Desktop            2     73.82.211.73   \n",
       "\n",
       "                        USERPLATFORMUID       USERCITY  USERZIPCODE  \\\n",
       "0  0d5041ff-f0b6-4d1a-9ad7-0a29f7d485b4   Fayetteville      28305.0   \n",
       "1  c8396dd0-969f-4d99-a40b-b7bb1f516154  Conneaut Lake      16316.0   \n",
       "2  3c97a081-6518-43f8-9f26-369759cfb471         Covina      91724.0   \n",
       "3  3e2578c8-f794-41af-a38c-c5cfb3c0f014       Brooklyn      11226.0   \n",
       "4  ec2ae7ce-6a8c-4156-98a7-07203e60f483       Marietta      30062.0   \n",
       "\n",
       "                                           USERAGENT            PLATFORMTYPE  \\\n",
       "0  Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_6...  Online Medical Journal   \n",
       "1  Mozilla/5.0 (iPhone; CPU iPhone OS 15_6_1 like...  Online Medical Journal   \n",
       "2  Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7...  Online Medical Journal   \n",
       "3  Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7...  Online Medical Journal   \n",
       "4  Mozilla/5.0 (Windows NT 10.0; Win64; x64) Appl...  Online Medical Journal   \n",
       "\n",
       "  CHANNELTYPE                                                URL  \\\n",
       "0     Website  https://www.clinicaladvisor.com/home/features/...   \n",
       "1     Website  https://www.ophthalmologyadvisor.com/topics/ca...   \n",
       "2     Website  https://www.psychiatryadvisor.com/author/tori-...   \n",
       "3     Website  https://www.cureus.com/articles/105482-diverti...   \n",
       "4     Website  https://www.renalandurologynews.com/home/confe...   \n",
       "\n",
       "                                            KEYWORDS  \n",
       "0  Family Practice|Drainage|Clinical|Dermatology|...  \n",
       "1  General|Clinical|Operative|Medicine|Cardiology...  \n",
       "2  Abortion|Anxiety Disorders|Apnea|False|Trauma|...  \n",
       "3  Health|Male|Neurological Surgery|Otolaryngolog...  \n",
       "4  chronic kidney disease|pain|nephrology|disease...  "
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_df.head(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "id": "d3c4f0b1",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "from sklearn.preprocessing import OneHotEncoder, LabelEncoder\n",
    "\n",
    "\n",
    "columns_to_drop = ['ID', 'DEVICETYPE', 'PLATFORM_ID', 'CHANNELTYPE', 'USERAGENT',\n",
    "                   'BIDREQUESTIP', 'USERPLATFORMUID', 'URL']\n",
    "\n",
    "\n",
    "test_df.drop(columns=columns_to_drop, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "id": "f53d414f",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "   USERCITY  USERZIPCODE  gastrointestinal diseases  opportunistic infections  \\\n",
      "0       857         1827                          0                         0   \n",
      "1       559         1091                          0                         0   \n",
      "2       592         5481                          0                         0   \n",
      "3       323          739                          0                         0   \n",
      "4      1609         2008                          0                         0   \n",
      "\n",
      "   hamartoma  carcinoma  large intestine  ustekinumab  famciclovir  \\\n",
      "0          0          0                0            0            0   \n",
      "1          0          0                0            0            0   \n",
      "2          0          0                0            0            0   \n",
      "3          0          0                0            0            0   \n",
      "4          0          1                0            0            0   \n",
      "\n",
      "   cryptococcosis  ...  Cardiac Output  Cysts  asthma  Embolism  \\\n",
      "0               0  ...               0      0       0         0   \n",
      "1               0  ...               0      0       0         0   \n",
      "2               0  ...               0      0       0         0   \n",
      "3               0  ...               0      0       0         0   \n",
      "4               0  ...               0      0       0         0   \n",
      "\n",
      "   Conscious Sedation  myeloproliferative neoplasms  Electronic Health Record  \\\n",
      "0                   0                             0                       0.0   \n",
      "1                   0                             0                       0.0   \n",
      "2                   0                             0                       0.0   \n",
      "3                   0                             0                       0.0   \n",
      "4                   0                             0                       0.0   \n",
      "\n",
      "   Medical News Platform  Online Learning Portal  Online Medical Journal  \n",
      "0                    0.0                     0.0                     1.0  \n",
      "1                    0.0                     0.0                     1.0  \n",
      "2                    0.0                     0.0                     1.0  \n",
      "3                    0.0                     0.0                     1.0  \n",
      "4                    0.0                     0.0                     1.0  \n",
      "\n",
      "[5 rows x 1432 columns]\n"
     ]
    }
   ],
   "source": [
    "test_df['USERCITY'] = test_df['USERCITY'].fillna(test_df.groupby('USERZIPCODE')['USERCITY'].transform('first'))\n",
    "\n",
    "\n",
    "all_keywords = test_df['KEYWORDS'].str.cat(sep='|')\n",
    "word_list = list(set(all_keywords.split('|')))\n",
    "\n",
    "\n",
    "word_list = list(filter(None, word_list))\n",
    "\n",
    "\n",
    "for word in word_list:\n",
    "    test_df[word] = test_df['KEYWORDS'].apply(lambda x: 1 if word in x else 0)\n",
    "\n",
    "\n",
    "columns_to_drop = ['KEYWORDS']\n",
    "test_df.drop(columns_to_drop, axis=1, inplace=True)\n",
    "\n",
    "\n",
    "encoder = OneHotEncoder(sparse=False)\n",
    "platform_type = test_df['PLATFORMTYPE'].values.reshape(-1, 1)\n",
    "encoded_features = encoder.fit_transform(platform_type)\n",
    "encoded_df = pd.DataFrame(encoded_features, columns=encoder.categories_[0])\n",
    "\n",
    "\n",
    "test_df = pd.concat([test_df, encoded_df], axis=1)\n",
    "\n",
    "\n",
    "test_df.drop('PLATFORMTYPE', axis=1, inplace=True)\n",
    "\n",
    "\n",
    "test_df.dropna(subset=['USERCITY', 'USERZIPCODE'], inplace=True)\n",
    "\n",
    "\n",
    "city_encoder = LabelEncoder()\n",
    "test_df['USERCITY'] = city_encoder.fit_transform(test_df['USERCITY'])\n",
    "\n",
    "zipcode_encoder = LabelEncoder()\n",
    "test_df['USERZIPCODE'] = zipcode_encoder.fit_transform(test_df['USERZIPCODE'])\n",
    "\n",
    "# Print the updated DataFrame\n",
    "print(test_df.head(5))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "id": "4b5b93cc",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>USERCITY</th>\n",
       "      <th>USERZIPCODE</th>\n",
       "      <th>gastrointestinal diseases</th>\n",
       "      <th>opportunistic infections</th>\n",
       "      <th>hamartoma</th>\n",
       "      <th>carcinoma</th>\n",
       "      <th>large intestine</th>\n",
       "      <th>ustekinumab</th>\n",
       "      <th>famciclovir</th>\n",
       "      <th>cryptococcosis</th>\n",
       "      <th>...</th>\n",
       "      <th>Cardiac Output</th>\n",
       "      <th>Cysts</th>\n",
       "      <th>asthma</th>\n",
       "      <th>Embolism</th>\n",
       "      <th>Conscious Sedation</th>\n",
       "      <th>myeloproliferative neoplasms</th>\n",
       "      <th>Electronic Health Record</th>\n",
       "      <th>Medical News Platform</th>\n",
       "      <th>Online Learning Portal</th>\n",
       "      <th>Online Medical Journal</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>857</td>\n",
       "      <td>1827</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>559</td>\n",
       "      <td>1091</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>592</td>\n",
       "      <td>5481</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>323</td>\n",
       "      <td>739</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1609</td>\n",
       "      <td>2008</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28487</th>\n",
       "      <td>2436</td>\n",
       "      <td>5898</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28488</th>\n",
       "      <td>323</td>\n",
       "      <td>730</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28489</th>\n",
       "      <td>1693</td>\n",
       "      <td>2340</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28490</th>\n",
       "      <td>903</td>\n",
       "      <td>2355</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28492</th>\n",
       "      <td>2987</td>\n",
       "      <td>1844</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>27008 rows × 1432 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       USERCITY  USERZIPCODE  gastrointestinal diseases  \\\n",
       "0           857         1827                          0   \n",
       "1           559         1091                          0   \n",
       "2           592         5481                          0   \n",
       "3           323          739                          0   \n",
       "4          1609         2008                          0   \n",
       "...         ...          ...                        ...   \n",
       "28487      2436         5898                          0   \n",
       "28488       323          730                          0   \n",
       "28489      1693         2340                          0   \n",
       "28490       903         2355                          0   \n",
       "28492      2987         1844                          0   \n",
       "\n",
       "       opportunistic infections  hamartoma  carcinoma  large intestine  \\\n",
       "0                             0          0          0                0   \n",
       "1                             0          0          0                0   \n",
       "2                             0          0          0                0   \n",
       "3                             0          0          0                0   \n",
       "4                             0          0          1                0   \n",
       "...                         ...        ...        ...              ...   \n",
       "28487                         0          0          0                0   \n",
       "28488                         0          0          0                0   \n",
       "28489                         0          0          0                0   \n",
       "28490                         0          0          0                0   \n",
       "28492                         0          0          0                0   \n",
       "\n",
       "       ustekinumab  famciclovir  cryptococcosis  ...  Cardiac Output  Cysts  \\\n",
       "0                0            0               0  ...               0      0   \n",
       "1                0            0               0  ...               0      0   \n",
       "2                0            0               0  ...               0      0   \n",
       "3                0            0               0  ...               0      0   \n",
       "4                0            0               0  ...               0      0   \n",
       "...            ...          ...             ...  ...             ...    ...   \n",
       "28487            0            0               0  ...               0      0   \n",
       "28488            0            0               0  ...               0      0   \n",
       "28489            0            0               0  ...               0      0   \n",
       "28490            0            0               0  ...               0      0   \n",
       "28492            0            0               0  ...               0      0   \n",
       "\n",
       "       asthma  Embolism  Conscious Sedation  myeloproliferative neoplasms  \\\n",
       "0           0         0                   0                             0   \n",
       "1           0         0                   0                             0   \n",
       "2           0         0                   0                             0   \n",
       "3           0         0                   0                             0   \n",
       "4           0         0                   0                             0   \n",
       "...       ...       ...                 ...                           ...   \n",
       "28487       0         0                   0                             0   \n",
       "28488       0         0                   0                             0   \n",
       "28489       0         0                   0                             0   \n",
       "28490       0         0                   0                             0   \n",
       "28492       0         0                   0                             0   \n",
       "\n",
       "       Electronic Health Record  Medical News Platform  \\\n",
       "0                           0.0                    0.0   \n",
       "1                           0.0                    0.0   \n",
       "2                           0.0                    0.0   \n",
       "3                           0.0                    0.0   \n",
       "4                           0.0                    0.0   \n",
       "...                         ...                    ...   \n",
       "28487                       0.0                    0.0   \n",
       "28488                       0.0                    0.0   \n",
       "28489                       0.0                    0.0   \n",
       "28490                       0.0                    0.0   \n",
       "28492                       0.0                    0.0   \n",
       "\n",
       "       Online Learning Portal  Online Medical Journal  \n",
       "0                         0.0                     1.0  \n",
       "1                         0.0                     1.0  \n",
       "2                         0.0                     1.0  \n",
       "3                         0.0                     1.0  \n",
       "4                         0.0                     1.0  \n",
       "...                       ...                     ...  \n",
       "28487                     0.0                     1.0  \n",
       "28488                     0.0                     1.0  \n",
       "28489                     0.0                     1.0  \n",
       "28490                     0.0                     1.0  \n",
       "28492                     0.0                     1.0  \n",
       "\n",
       "[27008 rows x 1432 columns]"
      ]
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "id": "97ec3918",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>USERCITY</th>\n",
       "      <th>USERZIPCODE</th>\n",
       "      <th>gastrointestinal diseases</th>\n",
       "      <th>opportunistic infections</th>\n",
       "      <th>hamartoma</th>\n",
       "      <th>carcinoma</th>\n",
       "      <th>large intestine</th>\n",
       "      <th>ustekinumab</th>\n",
       "      <th>famciclovir</th>\n",
       "      <th>cryptococcosis</th>\n",
       "      <th>...</th>\n",
       "      <th>Cardiac Output</th>\n",
       "      <th>Cysts</th>\n",
       "      <th>asthma</th>\n",
       "      <th>Embolism</th>\n",
       "      <th>Conscious Sedation</th>\n",
       "      <th>myeloproliferative neoplasms</th>\n",
       "      <th>Electronic Health Record</th>\n",
       "      <th>Medical News Platform</th>\n",
       "      <th>Online Learning Portal</th>\n",
       "      <th>Online Medical Journal</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>857</td>\n",
       "      <td>1827</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>559</td>\n",
       "      <td>1091</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>592</td>\n",
       "      <td>5481</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>323</td>\n",
       "      <td>739</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1609</td>\n",
       "      <td>2008</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28487</th>\n",
       "      <td>2436</td>\n",
       "      <td>5898</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28488</th>\n",
       "      <td>323</td>\n",
       "      <td>730</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28489</th>\n",
       "      <td>1693</td>\n",
       "      <td>2340</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28490</th>\n",
       "      <td>903</td>\n",
       "      <td>2355</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28492</th>\n",
       "      <td>2987</td>\n",
       "      <td>1844</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>27008 rows × 1432 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       USERCITY  USERZIPCODE  gastrointestinal diseases  \\\n",
       "0           857         1827                          0   \n",
       "1           559         1091                          0   \n",
       "2           592         5481                          0   \n",
       "3           323          739                          0   \n",
       "4          1609         2008                          0   \n",
       "...         ...          ...                        ...   \n",
       "28487      2436         5898                          0   \n",
       "28488       323          730                          0   \n",
       "28489      1693         2340                          0   \n",
       "28490       903         2355                          0   \n",
       "28492      2987         1844                          0   \n",
       "\n",
       "       opportunistic infections  hamartoma  carcinoma  large intestine  \\\n",
       "0                             0          0          0                0   \n",
       "1                             0          0          0                0   \n",
       "2                             0          0          0                0   \n",
       "3                             0          0          0                0   \n",
       "4                             0          0          1                0   \n",
       "...                         ...        ...        ...              ...   \n",
       "28487                         0          0          0                0   \n",
       "28488                         0          0          0                0   \n",
       "28489                         0          0          0                0   \n",
       "28490                         0          0          0                0   \n",
       "28492                         0          0          0                0   \n",
       "\n",
       "       ustekinumab  famciclovir  cryptococcosis  ...  Cardiac Output  Cysts  \\\n",
       "0                0            0               0  ...               0      0   \n",
       "1                0            0               0  ...               0      0   \n",
       "2                0            0               0  ...               0      0   \n",
       "3                0            0               0  ...               0      0   \n",
       "4                0            0               0  ...               0      0   \n",
       "...            ...          ...             ...  ...             ...    ...   \n",
       "28487            0            0               0  ...               0      0   \n",
       "28488            0            0               0  ...               0      0   \n",
       "28489            0            0               0  ...               0      0   \n",
       "28490            0            0               0  ...               0      0   \n",
       "28492            0            0               0  ...               0      0   \n",
       "\n",
       "       asthma  Embolism  Conscious Sedation  myeloproliferative neoplasms  \\\n",
       "0           0         0                   0                             0   \n",
       "1           0         0                   0                             0   \n",
       "2           0         0                   0                             0   \n",
       "3           0         0                   0                             0   \n",
       "4           0         0                   0                             0   \n",
       "...       ...       ...                 ...                           ...   \n",
       "28487       0         0                   0                             0   \n",
       "28488       0         0                   0                             0   \n",
       "28489       0         0                   0                             0   \n",
       "28490       0         0                   0                             0   \n",
       "28492       0         0                   0                             0   \n",
       "\n",
       "       Electronic Health Record  Medical News Platform  \\\n",
       "0                           0.0                    0.0   \n",
       "1                           0.0                    0.0   \n",
       "2                           0.0                    0.0   \n",
       "3                           0.0                    0.0   \n",
       "4                           0.0                    0.0   \n",
       "...                         ...                    ...   \n",
       "28487                       0.0                    0.0   \n",
       "28488                       0.0                    0.0   \n",
       "28489                       0.0                    0.0   \n",
       "28490                       0.0                    0.0   \n",
       "28492                       0.0                    0.0   \n",
       "\n",
       "       Online Learning Portal  Online Medical Journal  \n",
       "0                         0.0                     1.0  \n",
       "1                         0.0                     1.0  \n",
       "2                         0.0                     1.0  \n",
       "3                         0.0                     1.0  \n",
       "4                         0.0                     1.0  \n",
       "...                       ...                     ...  \n",
       "28487                     0.0                     1.0  \n",
       "28488                     0.0                     1.0  \n",
       "28489                     0.0                     1.0  \n",
       "28490                     0.0                     1.0  \n",
       "28492                     0.0                     1.0  \n",
       "\n",
       "[27008 rows x 1432 columns]"
      ]
     },
     "execution_count": 52,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "id": "c210ea75",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>USERCITY</th>\n",
       "      <th>USERZIPCODE</th>\n",
       "      <th>IS_HCP</th>\n",
       "      <th>gastrointestinal diseases</th>\n",
       "      <th>opportunistic infections</th>\n",
       "      <th>hamartoma</th>\n",
       "      <th>carcinoma</th>\n",
       "      <th>large intestine</th>\n",
       "      <th>ustekinumab</th>\n",
       "      <th>famciclovir</th>\n",
       "      <th>...</th>\n",
       "      <th>asthma</th>\n",
       "      <th>Embolism</th>\n",
       "      <th>Conscious Sedation</th>\n",
       "      <th>myeloproliferative neoplasms</th>\n",
       "      <th>q fever</th>\n",
       "      <th>Electronic Health Record</th>\n",
       "      <th>Medical News Platform</th>\n",
       "      <th>Online Learning Portal</th>\n",
       "      <th>Online Medical Journal</th>\n",
       "      <th>Tele-medicine Platform</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>3176</td>\n",
       "      <td>8934</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>130</td>\n",
       "      <td>2207</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2700</td>\n",
       "      <td>7585</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1766</td>\n",
       "      <td>7018</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>3877</td>\n",
       "      <td>1351</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>113932</th>\n",
       "      <td>3066</td>\n",
       "      <td>1837</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>113933</th>\n",
       "      <td>4042</td>\n",
       "      <td>10318</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>113934</th>\n",
       "      <td>3569</td>\n",
       "      <td>9052</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>113935</th>\n",
       "      <td>4321</td>\n",
       "      <td>1932</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>113936</th>\n",
       "      <td>4278</td>\n",
       "      <td>994</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>108006 rows × 1621 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        USERCITY  USERZIPCODE  IS_HCP  gastrointestinal diseases  \\\n",
       "0           3176         8934     0.0                        0.0   \n",
       "1            130         2207     0.0                        0.0   \n",
       "2           2700         7585     0.0                        0.0   \n",
       "4           1766         7018     0.0                        0.0   \n",
       "5           3877         1351     0.0                        0.0   \n",
       "...          ...          ...     ...                        ...   \n",
       "113932      3066         1837     1.0                        0.0   \n",
       "113933      4042        10318     1.0                        0.0   \n",
       "113934      3569         9052     1.0                        0.0   \n",
       "113935      4321         1932     1.0                        0.0   \n",
       "113936      4278          994     1.0                        0.0   \n",
       "\n",
       "        opportunistic infections  hamartoma  carcinoma  large intestine  \\\n",
       "0                            0.0        0.0        0.0              0.0   \n",
       "1                            0.0        0.0        0.0              0.0   \n",
       "2                            0.0        0.0        0.0              0.0   \n",
       "4                            0.0        0.0        0.0              0.0   \n",
       "5                            0.0        0.0        0.0              0.0   \n",
       "...                          ...        ...        ...              ...   \n",
       "113932                       0.0        0.0        0.0              0.0   \n",
       "113933                       0.0        0.0        0.0              0.0   \n",
       "113934                       0.0        0.0        0.0              0.0   \n",
       "113935                       0.0        0.0        0.0              0.0   \n",
       "113936                       0.0        0.0        0.0              0.0   \n",
       "\n",
       "        ustekinumab  famciclovir  ...  asthma  Embolism  Conscious Sedation  \\\n",
       "0               0.0          0.0  ...     0.0       0.0                 0.0   \n",
       "1               0.0          0.0  ...     0.0       0.0                 0.0   \n",
       "2               0.0          0.0  ...     0.0       0.0                 0.0   \n",
       "4               0.0          0.0  ...     0.0       0.0                 0.0   \n",
       "5               0.0          0.0  ...     0.0       0.0                 0.0   \n",
       "...             ...          ...  ...     ...       ...                 ...   \n",
       "113932          0.0          0.0  ...     0.0       0.0                 0.0   \n",
       "113933          0.0          0.0  ...     0.0       0.0                 0.0   \n",
       "113934          0.0          0.0  ...     0.0       0.0                 0.0   \n",
       "113935          0.0          0.0  ...     0.0       0.0                 0.0   \n",
       "113936          0.0          0.0  ...     0.0       0.0                 0.0   \n",
       "\n",
       "        myeloproliferative neoplasms  q fever  Electronic Health Record  \\\n",
       "0                                0.0      0.0                       0.0   \n",
       "1                                0.0      0.0                       0.0   \n",
       "2                                0.0      0.0                       0.0   \n",
       "4                                0.0      0.0                       0.0   \n",
       "5                                0.0      0.0                       0.0   \n",
       "...                              ...      ...                       ...   \n",
       "113932                           0.0      0.0                       0.0   \n",
       "113933                           0.0      0.0                       NaN   \n",
       "113934                           0.0      0.0                       NaN   \n",
       "113935                           0.0      0.0                       NaN   \n",
       "113936                           0.0      0.0                       NaN   \n",
       "\n",
       "        Medical News Platform  Online Learning Portal  Online Medical Journal  \\\n",
       "0                         0.0                     0.0                     1.0   \n",
       "1                         0.0                     0.0                     1.0   \n",
       "2                         0.0                     0.0                     1.0   \n",
       "4                         0.0                     0.0                     1.0   \n",
       "5                         0.0                     0.0                     1.0   \n",
       "...                       ...                     ...                     ...   \n",
       "113932                    0.0                     0.0                     1.0   \n",
       "113933                    NaN                     NaN                     NaN   \n",
       "113934                    NaN                     NaN                     NaN   \n",
       "113935                    NaN                     NaN                     NaN   \n",
       "113936                    NaN                     NaN                     NaN   \n",
       "\n",
       "        Tele-medicine Platform  \n",
       "0                          0.0  \n",
       "1                          0.0  \n",
       "2                          0.0  \n",
       "4                          0.0  \n",
       "5                          0.0  \n",
       "...                        ...  \n",
       "113932                     0.0  \n",
       "113933                     NaN  \n",
       "113934                     NaN  \n",
       "113935                     NaN  \n",
       "113936                     NaN  \n",
       "\n",
       "[108006 rows x 1621 columns]"
      ]
     },
     "execution_count": 53,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_df"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "39137e5b",
   "metadata": {},
   "source": [
    "# Inserting the missing columns in the test_df fron the train_df "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "id": "67af21d8",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "train_columns = set(train_df.columns)\n",
    "test_columns = set(test_df.columns)\n",
    "\n",
    "missing_columns = train_columns - test_columns\n",
    "\n",
    "for column in missing_columns:\n",
    "    test_df[column] = 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(108006, 1627)"
      ]
     },
     "execution_count": 68,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(27008, 1627)"
      ]
     },
     "execution_count": 69,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "id": "5a66dead",
   "metadata": {},
   "outputs": [],
   "source": [
    "train_columns = set(train_df.columns)\n",
    "test_columns = set(test_df.columns)\n",
    "\n",
    "missing_columns = test_columns - train_columns\n",
    "\n",
    "for column in missing_columns:\n",
    "    train_df[column] = 0\n",
    "    # print(column)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "neutron capture therapy\n",
      "dpp-4\n",
      "thromboplastin\n",
      "nasal bone\n",
      "retinal vein occlusion\n",
      "Tele-medicine Platform\n",
      "mycoses\n",
      "chagas disease\n",
      "Imperforate\n",
      "Axotomy\n",
      "tracheomalacia\n",
      "cystitis\n",
      "bell palsy\n",
      "blood coagulation disorders\n",
      "impulse control\n",
      "blood protein disorders\n",
      "adrenal hyperplasia\n",
      "elephantiasis\n",
      "empyema\n",
      "Decompressive Craniectomy\n",
      "iliotibial band syndrome\n",
      "hemostatic disorders\n",
      "Chemoprevention\n",
      "intestinal mucosa\n",
      "cervical rib syndrome\n",
      "mediastinitis\n",
      "general hospitals\n",
      "Minimally Invasive Surgical Procedures\n",
      "nasal septum\n",
      "Juvenile Dermatomyositis\n",
      "rocky mountain spotted fever\n",
      "Follicular Lymphoma\n",
      "palatine tonsil\n",
      "muscle relaxation\n",
      "latent tuberculosis\n",
      "blunt cardiac injury\n",
      "tonsillitis\n",
      "persistent infection\n",
      "porphyrias\n",
      "vitamin k deficiency\n",
      "diclofenac\n",
      "erysipelas\n",
      "aphonia\n",
      "Thoracic Surgical Procedures\n",
      "filariasis\n",
      "glanders\n",
      "behcet syndrome\n",
      "biliary atresia\n",
      "parasitic diseases\n",
      "burkitt lymphoma\n",
      "anthropometry\n",
      "tabes dorsalis\n",
      "health education\n",
      "bone marrow diseases\n",
      "hepatic veno-occlusive disease\n",
      "Medical Specialties\n",
      "coinfection\n",
      "varicose veins\n",
      "aortic aneurysm\n",
      "amyotrophic lateral sclerosis\n",
      "rubella syndrome\n",
      "scurvy\n",
      "alemtuzumab\n",
      "babesiosis\n",
      "blackwater fever\n",
      "retropharyngeal abscess\n",
      "taeniasis\n",
      "hordeolum\n",
      "granuloma inguinale\n",
      "inotuzumab ozogamicin\n",
      "rheumatoid factor\n",
      "elective surgical procedures\n",
      "bartonella infections\n",
      "hypospadias\n",
      "drug discovery\n",
      "bacterial infections and mycoses\n",
      "whole genome sequencing\n",
      "paraproteinemias\n",
      "obstetrics & gynecology\n",
      "wgs\n",
      "Congenital Heart Defect\n",
      "chalazion\n",
      "poliomyelitis\n",
      "Lymphopenia\n",
      "relapsing fever\n",
      "bronchi\n",
      "psittacosis\n",
      "leprosy\n",
      "chromoblastomycosis\n",
      "leptospirosis\n",
      "enhanced\n",
      "pharmacogenetics\n",
      "carney complex\n",
      "lemierre syndrome\n",
      "intrahepatic\n",
      "olfactory mucosa\n",
      "Sideroblastic\n",
      "Ischemic Heart Diseases\n",
      "antibiotic prophylaxis\n",
      "janus kinase 1\n",
      "dysentery\n",
      "neuromuscular blockade\n",
      "antibody deficiency\n",
      "siderosis\n",
      "ecchymosis\n",
      "candidemia\n",
      "pseudoxanthoma elasticum\n",
      "panitumumab\n",
      "pituitary apoplexy\n",
      "aco\n",
      "acid-base imbalance\n",
      "thrombocytosis\n",
      "Biliary Atresia\n",
      "fpg\n",
      "mycoplasma infections\n",
      "hyperemia\n",
      "prediabetic state\n",
      "Ablation Techniques\n",
      "Nerve Transfer\n",
      "muscle aches\n",
      "gram-negative bacterial infections\n",
      "bronchopulmonary sequestration\n",
      "weil disease\n",
      "Strokes\n",
      "nocardia infections\n",
      "Heart Rupture\n",
      "prothrombin time\n",
      "ethnopharmacology\n",
      "sterilization\n",
      "intracranial aneurysm\n",
      "frontal sinus\n",
      "vaccination coverage\n",
      "brentuximab vedotin\n",
      "paronychia\n",
      "Birth Cohort\n",
      "quadrivalent influenza vaccine\n",
      "varicocele\n",
      "granulomatous disease\n",
      "diagnostic molecular\n",
      "diabetes insipidus\n",
      "hypopharynx\n",
      "Neutropenia\n",
      "aplastic\n",
      "antilymphocyte serum\n",
      "thymectomy\n",
      "vibrio infections\n",
      "Brugada Syndrome\n",
      "h1n1 vaccine\n",
      "sitagliptin\n",
      "pulmonary aspergillosis\n",
      "furunculosis\n",
      "Cardiovascular Abnormalities\n",
      "temporomandibular joint\n",
      "mitral valve prolapse\n",
      "middle ear\n",
      "tinea capitis\n",
      "chemoprophylaxis\n",
      "high-dose trivalent influenza vaccine\n",
      "cidofovir\n",
      "Mental Processes\n",
      "Fatal Familial\n",
      "Cryptorchidism\n",
      "Diabetes Complications\n",
      "Second Primary\n",
      "ureaplasma infections\n",
      "thoracic surgery\n",
      "hemopneumothorax\n",
      "Cerebrovascular Accidents\n",
      "asymptomatic infections\n",
      "neurophysiology\n",
      "hematology & oncology\n",
      "anaplastic lymphoma kinase\n",
      "non-steroidal anti-inflammatory drug\n",
      "foot deformities\n",
      "Extravascular Lung Water\n",
      "physical therapist\n",
      "onychomycosis\n",
      "sofosbuvir\n",
      "connective tissue diseases\n",
      "tracheal stenosis\n",
      "phlebitis\n",
      "Rituximab\n",
      "preschool\n",
      "IS_HCP\n",
      "bisphosphonate-associated osteonecrosis of the jaw\n",
      "Leukopenia\n",
      "earache\n",
      "lymphatic vessels\n",
      "cns infections\n",
      "hypereosinophilic syndrome\n",
      "stroke management\n",
      "herpangina\n",
      "abatacept\n",
      "q fever\n",
      "herbal medicine\n"
     ]
    }
   ],
   "source": [
    "train_columns = set(train_df.columns)\n",
    "test_columns = set(test_df.columns)\n",
    "\n",
    "missing_columns = train_columns - test_columns\n",
    "\n",
    "for column in missing_columns:\n",
    "    # train_df[column] = 0\n",
    "    print(column)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "27008"
      ]
     },
     "execution_count": 73,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_df['IS_HCP'].eq(0).sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_column_order = train_df.columns\n",
    "\n",
    "test_df = test_df.reindex(columns=train_column_order)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "id": "2b094653",
   "metadata": {},
   "outputs": [
    {
     "ename": "KeyError",
     "evalue": "\"['IS_HCP'] not found in axis\"",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mKeyError\u001b[0m                                  Traceback (most recent call last)",
      "Cell \u001b[1;32mIn[58], line 2\u001b[0m\n\u001b[0;32m      1\u001b[0m \u001b[39m# Drop the \"IS_HCP\" column from test_df\u001b[39;00m\n\u001b[1;32m----> 2\u001b[0m test_df \u001b[39m=\u001b[39m test_df\u001b[39m.\u001b[39;49mdrop(\u001b[39m\"\u001b[39;49m\u001b[39mIS_HCP\u001b[39;49m\u001b[39m\"\u001b[39;49m, axis\u001b[39m=\u001b[39;49m\u001b[39m1\u001b[39;49m)\n",
      "File \u001b[1;32m~\\AppData\\Roaming\\Python\\Python310\\site-packages\\pandas\\util\\_decorators.py:331\u001b[0m, in \u001b[0;36mdeprecate_nonkeyword_arguments.<locals>.decorate.<locals>.wrapper\u001b[1;34m(*args, **kwargs)\u001b[0m\n\u001b[0;32m    325\u001b[0m \u001b[39mif\u001b[39;00m \u001b[39mlen\u001b[39m(args) \u001b[39m>\u001b[39m num_allow_args:\n\u001b[0;32m    326\u001b[0m     warnings\u001b[39m.\u001b[39mwarn(\n\u001b[0;32m    327\u001b[0m         msg\u001b[39m.\u001b[39mformat(arguments\u001b[39m=\u001b[39m_format_argument_list(allow_args)),\n\u001b[0;32m    328\u001b[0m         \u001b[39mFutureWarning\u001b[39;00m,\n\u001b[0;32m    329\u001b[0m         stacklevel\u001b[39m=\u001b[39mfind_stack_level(),\n\u001b[0;32m    330\u001b[0m     )\n\u001b[1;32m--> 331\u001b[0m \u001b[39mreturn\u001b[39;00m func(\u001b[39m*\u001b[39margs, \u001b[39m*\u001b[39m\u001b[39m*\u001b[39mkwargs)\n",
      "File \u001b[1;32m~\\AppData\\Roaming\\Python\\Python310\\site-packages\\pandas\\core\\frame.py:5399\u001b[0m, in \u001b[0;36mDataFrame.drop\u001b[1;34m(self, labels, axis, index, columns, level, inplace, errors)\u001b[0m\n\u001b[0;32m   5251\u001b[0m \u001b[39m@deprecate_nonkeyword_arguments\u001b[39m(version\u001b[39m=\u001b[39m\u001b[39mNone\u001b[39;00m, allowed_args\u001b[39m=\u001b[39m[\u001b[39m\"\u001b[39m\u001b[39mself\u001b[39m\u001b[39m\"\u001b[39m, \u001b[39m\"\u001b[39m\u001b[39mlabels\u001b[39m\u001b[39m\"\u001b[39m])\n\u001b[0;32m   5252\u001b[0m \u001b[39mdef\u001b[39;00m \u001b[39mdrop\u001b[39m(  \u001b[39m# type: ignore[override]\u001b[39;00m\n\u001b[0;32m   5253\u001b[0m     \u001b[39mself\u001b[39m,\n\u001b[1;32m   (...)\u001b[0m\n\u001b[0;32m   5260\u001b[0m     errors: IgnoreRaise \u001b[39m=\u001b[39m \u001b[39m\"\u001b[39m\u001b[39mraise\u001b[39m\u001b[39m\"\u001b[39m,\n\u001b[0;32m   5261\u001b[0m ) \u001b[39m-\u001b[39m\u001b[39m>\u001b[39m DataFrame \u001b[39m|\u001b[39m \u001b[39mNone\u001b[39;00m:\n\u001b[0;32m   5262\u001b[0m \u001b[39m    \u001b[39m\u001b[39m\"\"\"\u001b[39;00m\n\u001b[0;32m   5263\u001b[0m \u001b[39m    Drop specified labels from rows or columns.\u001b[39;00m\n\u001b[0;32m   5264\u001b[0m \n\u001b[1;32m   (...)\u001b[0m\n\u001b[0;32m   5397\u001b[0m \u001b[39m            weight  1.0     0.8\u001b[39;00m\n\u001b[0;32m   5398\u001b[0m \u001b[39m    \"\"\"\u001b[39;00m\n\u001b[1;32m-> 5399\u001b[0m     \u001b[39mreturn\u001b[39;00m \u001b[39msuper\u001b[39;49m()\u001b[39m.\u001b[39;49mdrop(\n\u001b[0;32m   5400\u001b[0m         labels\u001b[39m=\u001b[39;49mlabels,\n\u001b[0;32m   5401\u001b[0m         axis\u001b[39m=\u001b[39;49maxis,\n\u001b[0;32m   5402\u001b[0m         index\u001b[39m=\u001b[39;49mindex,\n\u001b[0;32m   5403\u001b[0m         columns\u001b[39m=\u001b[39;49mcolumns,\n\u001b[0;32m   5404\u001b[0m         level\u001b[39m=\u001b[39;49mlevel,\n\u001b[0;32m   5405\u001b[0m         inplace\u001b[39m=\u001b[39;49minplace,\n\u001b[0;32m   5406\u001b[0m         errors\u001b[39m=\u001b[39;49merrors,\n\u001b[0;32m   5407\u001b[0m     )\n",
      "File \u001b[1;32m~\\AppData\\Roaming\\Python\\Python310\\site-packages\\pandas\\util\\_decorators.py:331\u001b[0m, in \u001b[0;36mdeprecate_nonkeyword_arguments.<locals>.decorate.<locals>.wrapper\u001b[1;34m(*args, **kwargs)\u001b[0m\n\u001b[0;32m    325\u001b[0m \u001b[39mif\u001b[39;00m \u001b[39mlen\u001b[39m(args) \u001b[39m>\u001b[39m num_allow_args:\n\u001b[0;32m    326\u001b[0m     warnings\u001b[39m.\u001b[39mwarn(\n\u001b[0;32m    327\u001b[0m         msg\u001b[39m.\u001b[39mformat(arguments\u001b[39m=\u001b[39m_format_argument_list(allow_args)),\n\u001b[0;32m    328\u001b[0m         \u001b[39mFutureWarning\u001b[39;00m,\n\u001b[0;32m    329\u001b[0m         stacklevel\u001b[39m=\u001b[39mfind_stack_level(),\n\u001b[0;32m    330\u001b[0m     )\n\u001b[1;32m--> 331\u001b[0m \u001b[39mreturn\u001b[39;00m func(\u001b[39m*\u001b[39margs, \u001b[39m*\u001b[39m\u001b[39m*\u001b[39mkwargs)\n",
      "File \u001b[1;32m~\\AppData\\Roaming\\Python\\Python310\\site-packages\\pandas\\core\\generic.py:4505\u001b[0m, in \u001b[0;36mNDFrame.drop\u001b[1;34m(self, labels, axis, index, columns, level, inplace, errors)\u001b[0m\n\u001b[0;32m   4503\u001b[0m \u001b[39mfor\u001b[39;00m axis, labels \u001b[39min\u001b[39;00m axes\u001b[39m.\u001b[39mitems():\n\u001b[0;32m   4504\u001b[0m     \u001b[39mif\u001b[39;00m labels \u001b[39mis\u001b[39;00m \u001b[39mnot\u001b[39;00m \u001b[39mNone\u001b[39;00m:\n\u001b[1;32m-> 4505\u001b[0m         obj \u001b[39m=\u001b[39m obj\u001b[39m.\u001b[39;49m_drop_axis(labels, axis, level\u001b[39m=\u001b[39;49mlevel, errors\u001b[39m=\u001b[39;49merrors)\n\u001b[0;32m   4507\u001b[0m \u001b[39mif\u001b[39;00m inplace:\n\u001b[0;32m   4508\u001b[0m     \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_update_inplace(obj)\n",
      "File \u001b[1;32m~\\AppData\\Roaming\\Python\\Python310\\site-packages\\pandas\\core\\generic.py:4546\u001b[0m, in \u001b[0;36mNDFrame._drop_axis\u001b[1;34m(self, labels, axis, level, errors, only_slice)\u001b[0m\n\u001b[0;32m   4544\u001b[0m         new_axis \u001b[39m=\u001b[39m axis\u001b[39m.\u001b[39mdrop(labels, level\u001b[39m=\u001b[39mlevel, errors\u001b[39m=\u001b[39merrors)\n\u001b[0;32m   4545\u001b[0m     \u001b[39melse\u001b[39;00m:\n\u001b[1;32m-> 4546\u001b[0m         new_axis \u001b[39m=\u001b[39m axis\u001b[39m.\u001b[39;49mdrop(labels, errors\u001b[39m=\u001b[39;49merrors)\n\u001b[0;32m   4547\u001b[0m     indexer \u001b[39m=\u001b[39m axis\u001b[39m.\u001b[39mget_indexer(new_axis)\n\u001b[0;32m   4549\u001b[0m \u001b[39m# Case for non-unique axis\u001b[39;00m\n\u001b[0;32m   4550\u001b[0m \u001b[39melse\u001b[39;00m:\n",
      "File \u001b[1;32m~\\AppData\\Roaming\\Python\\Python310\\site-packages\\pandas\\core\\indexes\\base.py:6934\u001b[0m, in \u001b[0;36mIndex.drop\u001b[1;34m(self, labels, errors)\u001b[0m\n\u001b[0;32m   6932\u001b[0m \u001b[39mif\u001b[39;00m mask\u001b[39m.\u001b[39many():\n\u001b[0;32m   6933\u001b[0m     \u001b[39mif\u001b[39;00m errors \u001b[39m!=\u001b[39m \u001b[39m\"\u001b[39m\u001b[39mignore\u001b[39m\u001b[39m\"\u001b[39m:\n\u001b[1;32m-> 6934\u001b[0m         \u001b[39mraise\u001b[39;00m \u001b[39mKeyError\u001b[39;00m(\u001b[39mf\u001b[39m\u001b[39m\"\u001b[39m\u001b[39m{\u001b[39;00m\u001b[39mlist\u001b[39m(labels[mask])\u001b[39m}\u001b[39;00m\u001b[39m not found in axis\u001b[39m\u001b[39m\"\u001b[39m)\n\u001b[0;32m   6935\u001b[0m     indexer \u001b[39m=\u001b[39m indexer[\u001b[39m~\u001b[39mmask]\n\u001b[0;32m   6936\u001b[0m \u001b[39mreturn\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mdelete(indexer)\n",
      "\u001b[1;31mKeyError\u001b[0m: \"['IS_HCP'] not found in axis\""
     ]
    }
   ],
   "source": [
    "# Drop the \"IS_HCP\" column from test_df\n",
    "# test_df = test_df.drop(\"IS_HCP\", axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "id": "43b2a574",
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "ename": "KeyError",
     "evalue": "'IS_HCP'",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mKeyError\u001b[0m                                  Traceback (most recent call last)",
      "File \u001b[1;32m~\\AppData\\Roaming\\Python\\Python310\\site-packages\\pandas\\core\\indexes\\base.py:3802\u001b[0m, in \u001b[0;36mIndex.get_loc\u001b[1;34m(self, key, method, tolerance)\u001b[0m\n\u001b[0;32m   3801\u001b[0m \u001b[39mtry\u001b[39;00m:\n\u001b[1;32m-> 3802\u001b[0m     \u001b[39mreturn\u001b[39;00m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49m_engine\u001b[39m.\u001b[39;49mget_loc(casted_key)\n\u001b[0;32m   3803\u001b[0m \u001b[39mexcept\u001b[39;00m \u001b[39mKeyError\u001b[39;00m \u001b[39mas\u001b[39;00m err:\n",
      "File \u001b[1;32m~\\AppData\\Roaming\\Python\\Python310\\site-packages\\pandas\\_libs\\index.pyx:138\u001b[0m, in \u001b[0;36mpandas._libs.index.IndexEngine.get_loc\u001b[1;34m()\u001b[0m\n",
      "File \u001b[1;32m~\\AppData\\Roaming\\Python\\Python310\\site-packages\\pandas\\_libs\\index.pyx:165\u001b[0m, in \u001b[0;36mpandas._libs.index.IndexEngine.get_loc\u001b[1;34m()\u001b[0m\n",
      "File \u001b[1;32mpandas\\_libs\\hashtable_class_helper.pxi:5745\u001b[0m, in \u001b[0;36mpandas._libs.hashtable.PyObjectHashTable.get_item\u001b[1;34m()\u001b[0m\n",
      "File \u001b[1;32mpandas\\_libs\\hashtable_class_helper.pxi:5753\u001b[0m, in \u001b[0;36mpandas._libs.hashtable.PyObjectHashTable.get_item\u001b[1;34m()\u001b[0m\n",
      "\u001b[1;31mKeyError\u001b[0m: 'IS_HCP'",
      "\nThe above exception was the direct cause of the following exception:\n",
      "\u001b[1;31mKeyError\u001b[0m                                  Traceback (most recent call last)",
      "Cell \u001b[1;32mIn[60], line 13\u001b[0m\n\u001b[0;32m     10\u001b[0m \u001b[39m# Split the test dataset into features and target variable\u001b[39;00m\n\u001b[0;32m     11\u001b[0m X_test \u001b[39m=\u001b[39m test_df\n\u001b[1;32m---> 13\u001b[0m y_test \u001b[39m=\u001b[39m test_df[\u001b[39m'\u001b[39;49m\u001b[39mIS_HCP\u001b[39;49m\u001b[39m'\u001b[39;49m]\n\u001b[0;32m     15\u001b[0m \u001b[39m# Define the XGBoost regressor\u001b[39;00m\n\u001b[0;32m     16\u001b[0m model \u001b[39m=\u001b[39m xgb\u001b[39m.\u001b[39mXGBRegressor()\n",
      "File \u001b[1;32m~\\AppData\\Roaming\\Python\\Python310\\site-packages\\pandas\\core\\frame.py:3807\u001b[0m, in \u001b[0;36mDataFrame.__getitem__\u001b[1;34m(self, key)\u001b[0m\n\u001b[0;32m   3805\u001b[0m \u001b[39mif\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mcolumns\u001b[39m.\u001b[39mnlevels \u001b[39m>\u001b[39m \u001b[39m1\u001b[39m:\n\u001b[0;32m   3806\u001b[0m     \u001b[39mreturn\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_getitem_multilevel(key)\n\u001b[1;32m-> 3807\u001b[0m indexer \u001b[39m=\u001b[39m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mcolumns\u001b[39m.\u001b[39;49mget_loc(key)\n\u001b[0;32m   3808\u001b[0m \u001b[39mif\u001b[39;00m is_integer(indexer):\n\u001b[0;32m   3809\u001b[0m     indexer \u001b[39m=\u001b[39m [indexer]\n",
      "File \u001b[1;32m~\\AppData\\Roaming\\Python\\Python310\\site-packages\\pandas\\core\\indexes\\base.py:3804\u001b[0m, in \u001b[0;36mIndex.get_loc\u001b[1;34m(self, key, method, tolerance)\u001b[0m\n\u001b[0;32m   3802\u001b[0m     \u001b[39mreturn\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_engine\u001b[39m.\u001b[39mget_loc(casted_key)\n\u001b[0;32m   3803\u001b[0m \u001b[39mexcept\u001b[39;00m \u001b[39mKeyError\u001b[39;00m \u001b[39mas\u001b[39;00m err:\n\u001b[1;32m-> 3804\u001b[0m     \u001b[39mraise\u001b[39;00m \u001b[39mKeyError\u001b[39;00m(key) \u001b[39mfrom\u001b[39;00m \u001b[39merr\u001b[39;00m\n\u001b[0;32m   3805\u001b[0m \u001b[39mexcept\u001b[39;00m \u001b[39mTypeError\u001b[39;00m:\n\u001b[0;32m   3806\u001b[0m     \u001b[39m# If we have a listlike key, _check_indexing_error will raise\u001b[39;00m\n\u001b[0;32m   3807\u001b[0m     \u001b[39m#  InvalidIndexError. Otherwise we fall through and re-raise\u001b[39;00m\n\u001b[0;32m   3808\u001b[0m     \u001b[39m#  the TypeError.\u001b[39;00m\n\u001b[0;32m   3809\u001b[0m     \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_check_indexing_error(key)\n",
      "\u001b[1;31mKeyError\u001b[0m: 'IS_HCP'"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import xgboost as xgb\n",
    "from sklearn.metrics import mean_squared_error\n",
    "from math import sqrt\n",
    "\n",
    "\n",
    "X_train = train_df.drop('IS_HCP', axis=1)\n",
    "y_train = train_df['IS_HCP']\n",
    "\n",
    "\n",
    "X_test = test_df.drop('IS_HCP', axis=1)\n",
    "\n",
    "y_test = test_df['IS_HCP']\n",
    "\n",
    "\n",
    "model = xgb.XGBRegressor()\n",
    "\n",
    "\n",
    "model.fit(X_train, y_train)\n",
    "\n",
    "\n",
    "y_pred = model.predict(X_test)\n",
    "\n",
    "\n",
    "mse = mean_squared_error(y_test, y_pred)\n",
    "print(\"Mean Squared Error (MSE): {:.2f}\".format(mse))\n",
    "\n",
    "\n",
    "rmse = sqrt(mse)\n",
    "print(\"Root Mean Squared Error (RMSE): {:.2f}\".format(rmse))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 97.39%\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import xgboost as xgb\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.metrics import accuracy_score\n",
    "\n",
    "\n",
    "\n",
    "X = train_df.drop('IS_HCP', axis=1)\n",
    "y = train_df['IS_HCP']\n",
    "\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.3, random_state=0)\n",
    "\n",
    "\n",
    "model = xgb.XGBClassifier()\n",
    "\n",
    "\n",
    "model.fit(X_train, y_train)\n",
    "\n",
    "\n",
    "y_pred = model.predict(X_test)\n",
    "\n",
    "\n",
    "accuracy = accuracy_score(y_test, y_pred)\n",
    "print(\"Accuracy: {:.2f}%\".format(accuracy * 100))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Mean Squared Error (MSE): 0.03\n",
      "Root Mean Squared Error (RMSE): 0.16\n"
     ]
    }
   ],
   "source": [
    "mse = mean_squared_error(y_test, y_pred)\n",
    "print(\"Mean Squared Error (MSE): {:.2f}\".format(mse))\n",
    "\n",
    "rmse = sqrt(mse)\n",
    "print(\"Root Mean Squared Error (RMSE): {:.2f}\".format(rmse))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0, 1, 0, ..., 0, 0, 1])"
      ]
     },
     "execution_count": 77,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_pred"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 97.37%\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import xgboost as xgb\n",
    "from sklearn.model_selection import cross_val_score\n",
    "\n",
    "\n",
    "X = train_df.drop('IS_HCP', axis=1)\n",
    "y = train_df['IS_HCP']\n",
    "\n",
    "\n",
    "model = xgb.XGBClassifier()\n",
    "\n",
    "\n",
    "scores = cross_val_score(model, X, y, cv=5)  \n",
    "accuracy = scores.mean()\n",
    "\n",
    "print(\"Accuracy: {:.2f}%\".format(accuracy * 100))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.fit(X, y)\n",
    "X_test = test_df.drop('IS_HCP', axis=1)\n",
    "y_pred = model.predict(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "27008"
      ]
     },
     "execution_count": 84,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(y_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "accuracy = accuracy_score(y_test, y_pred)\n",
    "print(\"Accuracy: {:.2f}%\".format(accuracy * 100))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ff639126",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 97.50%\n"
     ]
    }
   ],
   "source": [
    "\n",
    "print(\"Accuracy: {:.2f}%\".format(accuracy * 100))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}