3.Experiments - XGBoostClassifier (GridSearch Optimization) - AUC 0.88335.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Import Dependencies"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-03-16T08:03:02.869526Z",
     "start_time": "2020-03-16T08:02:59.746171Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "\n",
    "from xgboost import XGBClassifier\n",
    "\n",
    "#from sklearn.pipeline import Pipeline\n",
    "from imblearn.pipeline import Pipeline\n",
    "from sklearn.model_selection import train_test_split, GridSearchCV\n",
    "from sklearn.decomposition import PCA\n",
    "from sklearn.feature_selection import SelectKBest\n",
    "from sklearn.compose import make_column_transformer\n",
    "from sklearn.preprocessing import OneHotEncoder, StandardScaler\n",
    "from sklearn.impute import SimpleImputer\n",
    "from sklearn.metrics import classification_report, roc_auc_score, roc_curve, auc\n",
    "\n",
    "from imblearn.over_sampling import SMOTE\n",
    "\n",
    "# magic word for producing visualizations in notebook\n",
    "%matplotlib inline\n",
    "\n",
    "# and setting the size of all plots.\n",
    "plt.rcParams['figure.figsize'] = [11, 7]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Load Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-03-16T08:03:08.381881Z",
     "start_time": "2020-03-16T08:03:02.902445Z"
    },
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# The C engine doesn't support RegEx so we set it to Python to avoid the warning\n",
    "census = pd.read_csv('data/census-income.data', header=None, sep=',\\s', na_values=['?'], engine='python')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-03-16T08:03:08.567389Z",
     "start_time": "2020-03-16T08:03:08.537465Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>age</th>\n",
       "      <th>class of worker</th>\n",
       "      <th>detailed industry recode</th>\n",
       "      <th>detailed occupation recode</th>\n",
       "      <th>education</th>\n",
       "      <th>wage per hour</th>\n",
       "      <th>enroll in edu inst last wk</th>\n",
       "      <th>marital status</th>\n",
       "      <th>major industry code</th>\n",
       "      <th>major occupation code</th>\n",
       "      <th>...</th>\n",
       "      <th>country of birth father</th>\n",
       "      <th>country of birth mother</th>\n",
       "      <th>country of birth self</th>\n",
       "      <th>citizenship</th>\n",
       "      <th>own business or self employed</th>\n",
       "      <th>fill inc questionnaire for veteran's admin</th>\n",
       "      <th>veterans benefits</th>\n",
       "      <th>weeks worked in year</th>\n",
       "      <th>year</th>\n",
       "      <th>income class</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>73</td>\n",
       "      <td>Not in universe</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>High school graduate</td>\n",
       "      <td>0</td>\n",
       "      <td>Not in universe</td>\n",
       "      <td>Widowed</td>\n",
       "      <td>Not in universe or children</td>\n",
       "      <td>Not in universe</td>\n",
       "      <td>...</td>\n",
       "      <td>United-States</td>\n",
       "      <td>United-States</td>\n",
       "      <td>United-States</td>\n",
       "      <td>Native- Born in the United States</td>\n",
       "      <td>0</td>\n",
       "      <td>Not in universe</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>95</td>\n",
       "      <td>- 50000.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>58</td>\n",
       "      <td>Self-employed-not incorporated</td>\n",
       "      <td>4</td>\n",
       "      <td>34</td>\n",
       "      <td>Some college but no degree</td>\n",
       "      <td>0</td>\n",
       "      <td>Not in universe</td>\n",
       "      <td>Divorced</td>\n",
       "      <td>Construction</td>\n",
       "      <td>Precision production craft &amp; repair</td>\n",
       "      <td>...</td>\n",
       "      <td>United-States</td>\n",
       "      <td>United-States</td>\n",
       "      <td>United-States</td>\n",
       "      <td>Native- Born in the United States</td>\n",
       "      <td>0</td>\n",
       "      <td>Not in universe</td>\n",
       "      <td>2</td>\n",
       "      <td>52</td>\n",
       "      <td>94</td>\n",
       "      <td>- 50000.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>18</td>\n",
       "      <td>Not in universe</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>10th grade</td>\n",
       "      <td>0</td>\n",
       "      <td>High school</td>\n",
       "      <td>Never married</td>\n",
       "      <td>Not in universe or children</td>\n",
       "      <td>Not in universe</td>\n",
       "      <td>...</td>\n",
       "      <td>Vietnam</td>\n",
       "      <td>Vietnam</td>\n",
       "      <td>Vietnam</td>\n",
       "      <td>Foreign born- Not a citizen of U S</td>\n",
       "      <td>0</td>\n",
       "      <td>Not in universe</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>95</td>\n",
       "      <td>- 50000.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>9</td>\n",
       "      <td>Not in universe</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>Children</td>\n",
       "      <td>0</td>\n",
       "      <td>Not in universe</td>\n",
       "      <td>Never married</td>\n",
       "      <td>Not in universe or children</td>\n",
       "      <td>Not in universe</td>\n",
       "      <td>...</td>\n",
       "      <td>United-States</td>\n",
       "      <td>United-States</td>\n",
       "      <td>United-States</td>\n",
       "      <td>Native- Born in the United States</td>\n",
       "      <td>0</td>\n",
       "      <td>Not in universe</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>94</td>\n",
       "      <td>- 50000.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>10</td>\n",
       "      <td>Not in universe</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>Children</td>\n",
       "      <td>0</td>\n",
       "      <td>Not in universe</td>\n",
       "      <td>Never married</td>\n",
       "      <td>Not in universe or children</td>\n",
       "      <td>Not in universe</td>\n",
       "      <td>...</td>\n",
       "      <td>United-States</td>\n",
       "      <td>United-States</td>\n",
       "      <td>United-States</td>\n",
       "      <td>Native- Born in the United States</td>\n",
       "      <td>0</td>\n",
       "      <td>Not in universe</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>94</td>\n",
       "      <td>- 50000.</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 42 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   age                 class of worker  detailed industry recode  \\\n",
       "0   73                 Not in universe                         0   \n",
       "1   58  Self-employed-not incorporated                         4   \n",
       "2   18                 Not in universe                         0   \n",
       "3    9                 Not in universe                         0   \n",
       "4   10                 Not in universe                         0   \n",
       "\n",
       "   detailed occupation recode                   education  wage per hour  \\\n",
       "0                           0        High school graduate              0   \n",
       "1                          34  Some college but no degree              0   \n",
       "2                           0                  10th grade              0   \n",
       "3                           0                    Children              0   \n",
       "4                           0                    Children              0   \n",
       "\n",
       "  enroll in edu inst last wk marital status          major industry code  \\\n",
       "0            Not in universe        Widowed  Not in universe or children   \n",
       "1            Not in universe       Divorced                 Construction   \n",
       "2                High school  Never married  Not in universe or children   \n",
       "3            Not in universe  Never married  Not in universe or children   \n",
       "4            Not in universe  Never married  Not in universe or children   \n",
       "\n",
       "                 major occupation code  ... country of birth father  \\\n",
       "0                      Not in universe  ...           United-States   \n",
       "1  Precision production craft & repair  ...           United-States   \n",
       "2                      Not in universe  ...                 Vietnam   \n",
       "3                      Not in universe  ...           United-States   \n",
       "4                      Not in universe  ...           United-States   \n",
       "\n",
       "  country of birth mother country of birth self  \\\n",
       "0           United-States         United-States   \n",
       "1           United-States         United-States   \n",
       "2                 Vietnam               Vietnam   \n",
       "3           United-States         United-States   \n",
       "4           United-States         United-States   \n",
       "\n",
       "                           citizenship own business or self employed  \\\n",
       "0    Native- Born in the United States                             0   \n",
       "1    Native- Born in the United States                             0   \n",
       "2  Foreign born- Not a citizen of U S                              0   \n",
       "3    Native- Born in the United States                             0   \n",
       "4    Native- Born in the United States                             0   \n",
       "\n",
       "  fill inc questionnaire for veteran's admin  veterans benefits  \\\n",
       "0                            Not in universe                  2   \n",
       "1                            Not in universe                  2   \n",
       "2                            Not in universe                  2   \n",
       "3                            Not in universe                  0   \n",
       "4                            Not in universe                  0   \n",
       "\n",
       "   weeks worked in year  year income class  \n",
       "0                     0    95     - 50000.  \n",
       "1                    52    94     - 50000.  \n",
       "2                     0    95     - 50000.  \n",
       "3                     0    94     - 50000.  \n",
       "4                     0    94     - 50000.  \n",
       "\n",
       "[5 rows x 42 columns]"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "census_columns = [\n",
    "    'age',\n",
    "    'class of worker',\n",
    "    'detailed industry recode',\n",
    "    'detailed occupation recode',\n",
    "    'education',\n",
    "    'wage per hour',\n",
    "    'enroll in edu inst last wk',\n",
    "    'marital status',\n",
    "    'major industry code',\n",
    "    'major occupation code',\n",
    "    'race',\n",
    "    'hispanic origin',\n",
    "    'sex',\n",
    "    'member of a labor union',\n",
    "    'reason for unemployment',\n",
    "    'full or part time employment stat',\n",
    "    'capital gains',\n",
    "    'capital losses',\n",
    "    'dividends from stocks',\n",
    "    'tax filer stat',\n",
    "    'region of previous residence',\n",
    "    'state of previous residence',\n",
    "    'detailed household and family stat',\n",
    "    'detailed household summary in household',\n",
    "    'instance weight',\n",
    "    'migration code-change in msa',\n",
    "    'migration code-change in reg',\n",
    "    'migration code-move within reg',\n",
    "    'live in this house 1 year ago',\n",
    "    'migration prev res in sunbelt',\n",
    "    'num persons worked for employer',\n",
    "    'family members under 18',\n",
    "    'country of birth father',\n",
    "    'country of birth mother',\n",
    "    'country of birth self',\n",
    "    'citizenship',\n",
    "    'own business or self employed',\n",
    "    'fill inc questionnaire for veteran\\'s admin',\n",
    "    'veterans benefits',\n",
    "    'weeks worked in year',\n",
    "    'year',\n",
    "    'income class'\n",
    "]\n",
    "\n",
    "features_to_drop = ['instance weight']\n",
    "\n",
    "categorical_features = ['class of worker',\n",
    "                        'detailed industry recode',\n",
    "                        'detailed occupation recode',\n",
    "                        'education',\n",
    "                        'marital status',\n",
    "                        'major industry code',\n",
    "                        'major occupation code',\n",
    "                        'race',\n",
    "                        'hispanic origin',\n",
    "                        'sex',\n",
    "                        'full or part time employment stat',\n",
    "                        'tax filer stat',\n",
    "                        'detailed household and family stat',\n",
    "                        'detailed household summary in household',\n",
    "                        'migration code-change in msa',\n",
    "                        'migration code-change in reg',\n",
    "                        'migration code-move within reg',\n",
    "                        'live in this house 1 year ago',\n",
    "                        'country of birth father',\n",
    "                        'country of birth mother',\n",
    "                        'country of birth self',\n",
    "                        'citizenship',\n",
    "                        'own business or self employed',\n",
    "                        'veterans benefits',\n",
    "                        'enroll in edu inst last wk',\n",
    "                        'member of a labor union',\n",
    "                        'reason for unemployment',\n",
    "                        'region of previous residence',\n",
    "                        'state of previous residence',\n",
    "                        'migration prev res in sunbelt',\n",
    "                        'family members under 18',\n",
    "                        \"fill inc questionnaire for veteran\\'s admin\",\n",
    "                        'year']\n",
    "\n",
    "numerical_features = ['age',\n",
    "                      'wage per hour',\n",
    "                      'capital gains',\n",
    "                      'capital losses',\n",
    "                      'dividends from stocks',\n",
    "                      'num persons worked for employer',\n",
    "                      'weeks worked in year']\n",
    "\n",
    "census.columns = census_columns\n",
    "census.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Clean Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-03-16T08:03:08.738926Z",
     "start_time": "2020-03-16T08:03:08.723966Z"
    }
   },
   "outputs": [],
   "source": [
    "def clean_dataset(df):\n",
    "    # Drop features\n",
    "    df = df.drop(features_to_drop, axis=1)\n",
    "    \n",
    "    # Fill NaN\n",
    "    df['hispanic origin'] = df['hispanic origin'].fillna('Do not know')\n",
    "    df['migration code-change in msa'] = df['migration code-change in msa'].fillna('Not identifiable')\n",
    "    \n",
    "    df['state of previous residence'] = df['state of previous residence'].fillna('Not in universe')\n",
    "    df['migration code-change in reg'] = df['migration code-change in reg'].fillna('Not in universe')\n",
    "    df['migration code-move within reg'] = df['migration code-move within reg'].fillna('Not in universe')\n",
    "    df['migration prev res in sunbelt'] = df['migration prev res in sunbelt'].fillna('Not in universe')\n",
    "    \n",
    "    df['country of birth father'] = df['country of birth father'].fillna(df['country of birth father'].mode()[0])\n",
    "    df['country of birth mother'] = df['country of birth mother'].fillna(df['country of birth mother'].mode()[0])\n",
    "    df['country of birth self'] = df['country of birth self'].fillna(df['country of birth self'].mode()[0])\n",
    "    \n",
    "    df[categorical_features] = df[categorical_features].astype(str)\n",
    "    \n",
    "    # Encode income class column\n",
    "    df['income class'] = df['income class'].map({'- 50000.': 0, '50000+.': 1})\n",
    "\n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-03-16T08:03:10.127213Z",
     "start_time": "2020-03-16T08:03:08.895507Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>age</th>\n",
       "      <th>class of worker</th>\n",
       "      <th>detailed industry recode</th>\n",
       "      <th>detailed occupation recode</th>\n",
       "      <th>education</th>\n",
       "      <th>wage per hour</th>\n",
       "      <th>enroll in edu inst last wk</th>\n",
       "      <th>marital status</th>\n",
       "      <th>major industry code</th>\n",
       "      <th>major occupation code</th>\n",
       "      <th>...</th>\n",
       "      <th>country of birth father</th>\n",
       "      <th>country of birth mother</th>\n",
       "      <th>country of birth self</th>\n",
       "      <th>citizenship</th>\n",
       "      <th>own business or self employed</th>\n",
       "      <th>fill inc questionnaire for veteran's admin</th>\n",
       "      <th>veterans benefits</th>\n",
       "      <th>weeks worked in year</th>\n",
       "      <th>year</th>\n",
       "      <th>income class</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>73</td>\n",
       "      <td>Not in universe</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>High school graduate</td>\n",
       "      <td>0</td>\n",
       "      <td>Not in universe</td>\n",
       "      <td>Widowed</td>\n",
       "      <td>Not in universe or children</td>\n",
       "      <td>Not in universe</td>\n",
       "      <td>...</td>\n",
       "      <td>United-States</td>\n",
       "      <td>United-States</td>\n",
       "      <td>United-States</td>\n",
       "      <td>Native- Born in the United States</td>\n",
       "      <td>0</td>\n",
       "      <td>Not in universe</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>95</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>58</td>\n",
       "      <td>Self-employed-not incorporated</td>\n",
       "      <td>4</td>\n",
       "      <td>34</td>\n",
       "      <td>Some college but no degree</td>\n",
       "      <td>0</td>\n",
       "      <td>Not in universe</td>\n",
       "      <td>Divorced</td>\n",
       "      <td>Construction</td>\n",
       "      <td>Precision production craft &amp; repair</td>\n",
       "      <td>...</td>\n",
       "      <td>United-States</td>\n",
       "      <td>United-States</td>\n",
       "      <td>United-States</td>\n",
       "      <td>Native- Born in the United States</td>\n",
       "      <td>0</td>\n",
       "      <td>Not in universe</td>\n",
       "      <td>2</td>\n",
       "      <td>52</td>\n",
       "      <td>94</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>18</td>\n",
       "      <td>Not in universe</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>10th grade</td>\n",
       "      <td>0</td>\n",
       "      <td>High school</td>\n",
       "      <td>Never married</td>\n",
       "      <td>Not in universe or children</td>\n",
       "      <td>Not in universe</td>\n",
       "      <td>...</td>\n",
       "      <td>Vietnam</td>\n",
       "      <td>Vietnam</td>\n",
       "      <td>Vietnam</td>\n",
       "      <td>Foreign born- Not a citizen of U S</td>\n",
       "      <td>0</td>\n",
       "      <td>Not in universe</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>95</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>9</td>\n",
       "      <td>Not in universe</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>Children</td>\n",
       "      <td>0</td>\n",
       "      <td>Not in universe</td>\n",
       "      <td>Never married</td>\n",
       "      <td>Not in universe or children</td>\n",
       "      <td>Not in universe</td>\n",
       "      <td>...</td>\n",
       "      <td>United-States</td>\n",
       "      <td>United-States</td>\n",
       "      <td>United-States</td>\n",
       "      <td>Native- Born in the United States</td>\n",
       "      <td>0</td>\n",
       "      <td>Not in universe</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>94</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>10</td>\n",
       "      <td>Not in universe</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>Children</td>\n",
       "      <td>0</td>\n",
       "      <td>Not in universe</td>\n",
       "      <td>Never married</td>\n",
       "      <td>Not in universe or children</td>\n",
       "      <td>Not in universe</td>\n",
       "      <td>...</td>\n",
       "      <td>United-States</td>\n",
       "      <td>United-States</td>\n",
       "      <td>United-States</td>\n",
       "      <td>Native- Born in the United States</td>\n",
       "      <td>0</td>\n",
       "      <td>Not in universe</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>94</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 41 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   age                 class of worker detailed industry recode  \\\n",
       "0   73                 Not in universe                        0   \n",
       "1   58  Self-employed-not incorporated                        4   \n",
       "2   18                 Not in universe                        0   \n",
       "3    9                 Not in universe                        0   \n",
       "4   10                 Not in universe                        0   \n",
       "\n",
       "  detailed occupation recode                   education  wage per hour  \\\n",
       "0                          0        High school graduate              0   \n",
       "1                         34  Some college but no degree              0   \n",
       "2                          0                  10th grade              0   \n",
       "3                          0                    Children              0   \n",
       "4                          0                    Children              0   \n",
       "\n",
       "  enroll in edu inst last wk marital status          major industry code  \\\n",
       "0            Not in universe        Widowed  Not in universe or children   \n",
       "1            Not in universe       Divorced                 Construction   \n",
       "2                High school  Never married  Not in universe or children   \n",
       "3            Not in universe  Never married  Not in universe or children   \n",
       "4            Not in universe  Never married  Not in universe or children   \n",
       "\n",
       "                 major occupation code  ... country of birth father  \\\n",
       "0                      Not in universe  ...           United-States   \n",
       "1  Precision production craft & repair  ...           United-States   \n",
       "2                      Not in universe  ...                 Vietnam   \n",
       "3                      Not in universe  ...           United-States   \n",
       "4                      Not in universe  ...           United-States   \n",
       "\n",
       "  country of birth mother country of birth self  \\\n",
       "0           United-States         United-States   \n",
       "1           United-States         United-States   \n",
       "2                 Vietnam               Vietnam   \n",
       "3           United-States         United-States   \n",
       "4           United-States         United-States   \n",
       "\n",
       "                           citizenship own business or self employed  \\\n",
       "0    Native- Born in the United States                             0   \n",
       "1    Native- Born in the United States                             0   \n",
       "2  Foreign born- Not a citizen of U S                              0   \n",
       "3    Native- Born in the United States                             0   \n",
       "4    Native- Born in the United States                             0   \n",
       "\n",
       "  fill inc questionnaire for veteran's admin  veterans benefits  \\\n",
       "0                            Not in universe                  2   \n",
       "1                            Not in universe                  2   \n",
       "2                            Not in universe                  2   \n",
       "3                            Not in universe                  0   \n",
       "4                            Not in universe                  0   \n",
       "\n",
       "   weeks worked in year  year income class  \n",
       "0                     0    95            0  \n",
       "1                    52    94            0  \n",
       "2                     0    95            0  \n",
       "3                     0    94            0  \n",
       "4                     0    94            0  \n",
       "\n",
       "[5 rows x 41 columns]"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "census = clean_dataset(census)\n",
    "census.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-03-16T08:03:10.745559Z",
     "start_time": "2020-03-16T08:03:10.437384Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 199523 entries, 0 to 199522\n",
      "Data columns (total 41 columns):\n",
      " #   Column                                      Non-Null Count   Dtype \n",
      "---  ------                                      --------------   ----- \n",
      " 0   age                                         199523 non-null  int64 \n",
      " 1   class of worker                             199523 non-null  object\n",
      " 2   detailed industry recode                    199523 non-null  object\n",
      " 3   detailed occupation recode                  199523 non-null  object\n",
      " 4   education                                   199523 non-null  object\n",
      " 5   wage per hour                               199523 non-null  int64 \n",
      " 6   enroll in edu inst last wk                  199523 non-null  object\n",
      " 7   marital status                              199523 non-null  object\n",
      " 8   major industry code                         199523 non-null  object\n",
      " 9   major occupation code                       199523 non-null  object\n",
      " 10  race                                        199523 non-null  object\n",
      " 11  hispanic origin                             199523 non-null  object\n",
      " 12  sex                                         199523 non-null  object\n",
      " 13  member of a labor union                     199523 non-null  object\n",
      " 14  reason for unemployment                     199523 non-null  object\n",
      " 15  full or part time employment stat           199523 non-null  object\n",
      " 16  capital gains                               199523 non-null  int64 \n",
      " 17  capital losses                              199523 non-null  int64 \n",
      " 18  dividends from stocks                       199523 non-null  int64 \n",
      " 19  tax filer stat                              199523 non-null  object\n",
      " 20  region of previous residence                199523 non-null  object\n",
      " 21  state of previous residence                 199523 non-null  object\n",
      " 22  detailed household and family stat          199523 non-null  object\n",
      " 23  detailed household summary in household     199523 non-null  object\n",
      " 24  migration code-change in msa                199523 non-null  object\n",
      " 25  migration code-change in reg                199523 non-null  object\n",
      " 26  migration code-move within reg              199523 non-null  object\n",
      " 27  live in this house 1 year ago               199523 non-null  object\n",
      " 28  migration prev res in sunbelt               199523 non-null  object\n",
      " 29  num persons worked for employer             199523 non-null  int64 \n",
      " 30  family members under 18                     199523 non-null  object\n",
      " 31  country of birth father                     199523 non-null  object\n",
      " 32  country of birth mother                     199523 non-null  object\n",
      " 33  country of birth self                       199523 non-null  object\n",
      " 34  citizenship                                 199523 non-null  object\n",
      " 35  own business or self employed               199523 non-null  object\n",
      " 36  fill inc questionnaire for veteran's admin  199523 non-null  object\n",
      " 37  veterans benefits                           199523 non-null  object\n",
      " 38  weeks worked in year                        199523 non-null  int64 \n",
      " 39  year                                        199523 non-null  object\n",
      " 40  income class                                199523 non-null  int64 \n",
      "dtypes: int64(8), object(33)\n",
      "memory usage: 62.4+ MB\n"
     ]
    }
   ],
   "source": [
    "census.info()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Split train and validation sets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-03-16T08:03:11.020823Z",
     "start_time": "2020-03-16T08:03:10.959985Z"
    }
   },
   "outputs": [],
   "source": [
    "X = census.drop('income class', axis=1)\n",
    "y = census['income class']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-03-16T08:03:11.438704Z",
     "start_time": "2020-03-16T08:03:11.423746Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.06205800834991455"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y.mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-03-16T08:03:11.964299Z",
     "start_time": "2020-03-16T08:03:11.840630Z"
    }
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAWAAAAFPCAYAAABzmxSvAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAgAElEQVR4nO3de5xVdb3/8debm5e8gIqmAkIdTAEJZVKwk5oeETr9jppmkB2w9OD1ZJeTl3qY5uWXpmboMTkmBJiJhqmUpHmQ0k54GQQBUwSRZIIjOgiYIAp+zh/rO7jY7JnZXGbWML6fj8d6zN6f9V3f9V17Nm/WfPfaeysiMDOz5tem6AGYmX1YOYDNzAriADYzK4gD2MysIA5gM7OCOIDNzAriAC6ApCskRVrel/SmpGckXSPpoyVtu6d2n6+w7w6p/36bMZ5Fkm7I3R8nqbryI2qw70GSvlGmvs32sS1J+jdJr0haJ+kP9bQ5I/1Odmnm4bVIm/sctQ+0K3oAH2IrgcHp9u7AYcC5wEhJgyNiRlq3FBgIvFhhvx2Ay4FFwKwKtzkZqK2w7eYaBJwK/KSkfhWwUxPtc4uk//xuA/4T+BXwZj1NHyL7naxupqFZK+UALs66iHgyd/8RSbcBjwP3SPpERKyPiLXAk+W72DqSdoqINRExsyn6b0hEvNzc+6zAPwBtgbERMbu+RhHxOvB6s43KWi1PQbQgEbECuAj4OHA8lP/zTtK/SJoh6e00ffGUpKPT6rfSz5/npjm65/o5XdIESSuA36T+NpqCyO3nJEkvSnpH0p8k9cqtK/tnZ35qQdIVwLeBA3JjGVfaLrdtP0lTJa1Ox3WXpH3K7PM0Sf8laaWkGkk/kNToc1nSBZLmS1oraYGkb+bWXQE8ke4+l/ZzRj39bDQFsTnjktRX0m8krZD0d0lPSzo+t76HpAckrZL0Vmr7DyV9hKRvSrpRUq2kNyT9R1o3QtLC1P9YSTuWbNtN0kRJy9Pj/IikT1Tw2B0g6e60r9WSZkv6cgPth6fnzPL0u5wmqaqkTW9JD6c2b0t6QdL5ufX/KOmJ9FiskjRL0hdL+jhL0vPpd/pXSRdtzj6K5jPglmcasA4YADxculLSx4FJwCjgO8COQH9gj9TkWOAx4GqyP5Uhm8bYN92+Afg18EVgfQPjOAD4MXAZsAb4AdlZes+IeKfCY7kD6JnGdHKqlT1zlNQZ+APwAvBlYBfgWuBRSVUR8W6u+Y+A+8imNo4Dvg88D9xb30Ak/RtwSzqmR4DPAjdK2iEirk1jXQbcCpwOLAQ29yy9wXFJOgj4H2AecA7ZtE8V0DWt3wGYCrwH/BvZ8+AHwB8lHRIRy3P7+jbZ73cY8Hngekl7A58Cvg50A24CXiJ7HJG0B/CntN9zyKZQLgH+W9KBEbGmnsdub2B6av8fwGKgT92469EdmED2GHYg+50+LqlPRCxMbSaTTa19BVgLfALYLe1zN+C3wIPAlYCAQ4COuXF9B/j/ZI/7H8j+HVwlaXVE/Gdj+2gRIsJLMy/AFcAbDaxfCtyWbncHAvh8un8qUNvAtruk9meU1Ov6ub/MNouAG3L3x6W2R+ZqB5AFwjnlxlWybXXu/g3AojL7LG13LbAC2C1XOzztY1jJPieU9DULmNjAY9IG+Bvw85L6T8nm4ndM949J/fdp5Pd3Rmq3y+aMC7gbqAF2qqffc9Jj/LFcrQvwLnBprhbAtJLjW0o2Z51//O4Fnsrdv4osfPfI1Tqlx+D8Bo73h8DbwL71rC/7XCgZXzuyIPx+qu2Vtjmknm2q0vpd61m/G/B34PKS+pXA/5JNJTW4j5aweAqiZVID6+YAu0sar+wKg49sZt8PNd4EgGUR8ee6OxHxV2AGWSg2hcOB30fEqtw+nyb7z+EfS9r+vuT+X8iCqj5dgP3IXljLu4fsH/IhWzDechob17HAPVHPmSbZY/BsfHCGSETUkJ01lz4GU3Nt3gdeAWbkHz9gAbB/7v4/AY8CqyS1k9SObMpqBlng1edY4OGIWNpAm41IOljS/ZJeI/tL6z2ys88DU5PlZGfSoyV9KZ1l571MFrC/lHSipI4l6wcCHwF+VXcs6XgeA/Yhe9wb20fhHMAtTJqz2xN4rdz6iJgHnAh8DJgCvCHpl+lP+EqU7beMZfXU9i1T3xb2pfzYXuOD6ZU6K0ruv0s2FdNQ33V9lfZNmf63VGPj2pPsTLU+W/sYNLb/vYAvkYVhfvksDU8nNDbujUjalew/o67At4DPkE2NPFc3nvSfxiCys9WxwP+m+d5D0/o30/r2ZGfyr0t6SNLHcscC2RRP/limpXrXxvbREngOuOX5LNnvZXp9DSLiIeAhSbsD/0x2idctwNAK+q/080fLnS3sTfaEB6ibB+5Q0mZLw2xpPfvch+wMbWvUhUdp/3Uv8C2nedTS8H9gS4HeZer7sG3GuJxsTvSqMuveKlOr09i4Sw0kOwM9PiI2XD6Znq8bpHWnSGpPFtLXkT2vu0TE+xExHRgsaSeys/cfA78ke32k7vH4POX/05pXyT4245iahM+AW5D0Z9Z1ZH86/ndj7SNiZUT8ErgfqLtCoe7FqobOCCuxt6Qjc2PrRnat8tOptIzsjOPgXJtdyP7x5TV2dlrnKeCEdPZU19+nyOYX/7QF48+rAZaQvfCYdxqwimxapzlMBU4rvTIh5ymgv6QedQVJ+wNHsvWPQd3+ewPPR0R1yTKvke1OUO6KlEbUXd+9tq6QnkvdyzWOiPci4jGygN2X3Attaf2aiPgN2Vls3fN8OtmLw/uVOZbqiHhrc/ZRFJ8BF6edpAHp9q5kr+CeC+wMDI6IslcoSDqbLOQeJguVnmTBMgEgIt6V9ArZP/S5ZGeq9V7T2oA3gDsl1V0FcSVZ6I5L+3lf0oPANyX9lezP32+ntnkvAvsou6RrLtmLj4vK7O/H6fgfkXQdH1wFMYfsyoItlsZ6BfBfkmrJ5kGPTvv7blR+VcfW+gHwDNnVADeSnVkeSvai6liyx/Zi4HeSvk82d3oF2e/iv7bB/n9MdjXAY5JuIXthch+yx+JPEXF3PdvdBAwHnpB0Ddm86sHARyLiR2XaP0k2f/szST8iOxu+Iu0PyC7HI3uB9h6yK046kR37cxGxXNI/A18DHgBeJZvLPptsjpeIWJF+p6MkHUB2/Xwbsjnmz0bEyY3tYzMet6ZT9KuAH8aF7MkYaXmfLLyqgWuAj5a07c7GV0EMJHshbQlZuL5Cdta8Q26bQWSh+07atntpPyX7WMSmV0FUA18gu4xpLdkLQX1KttuH7DKhVcBfgZFsenXDjsDPycI7gHH5fZT0dyjZP7DV6TH5JbBPfY9F6XgreNwvIPvr4l2yf5DfLFl/DFt3FUSj4wL6ks3dv5WWp4Djcus/RhY6b5GF2G+BniV9BHBBSe0PwKQyz7M3Smr7pd/Ha+n3ugj4BdC7kWM+gCzI3ky/n+eAofUdP9m7POeS/Yc8G/hcfoxk00F3pt/DO2TztHcD3dL6T5Bdbrk4jbMGGE3uCo7U7itkU1Rr0tieAr5VyT5awqI0UDMza2aeAzYzK4gD2MysIA5gM7OCOIDNzAriADYzK4ivA0722muv6N69e9HDMLNWZsaMGW9ERNmPCnAAJ927d6e6usV9Q46ZbefSG5XK8hSEmVlBHMBmZgVxAJuZFcQB3EqNGjWKPn360Lt3b37yk+wLiS+77DL69u1Lv379GDRoEEuWLNlku2nTptGvX78Ny4477sgDDzwAwGOPPcZhhx1Gnz59GDFiBOvWrQPgvvvuo3fv3nzmM5+htjb7cuWXX36ZoUMr+XRMsw+xoj+MoqUs/fv3j9Zizpw50bt373j77bfjvffei+OOOy5eeumlWLly5YY2o0aNirPPPrvBfmpra6NTp07x9ttvx/r166NLly4xb968iIi47LLL4o477oiIiIEDB8aqVavi9ttvj5tvvjkiIoYOHRovvfRSEx2h2faDBj4oymfArdALL7zAgAED2HnnnWnXrh1HH300999/P7vt9sF3Eb799ttIDX3zEUyaNIkhQ4aw8847U1tbyw477MCBB2bfKHP88cdz333Zp0S2adOGtWvXsnr1atq3b88TTzzBvvvuS8+ePZvuIM1aAV+G1gr16dOH733ve9TW1rLTTjsxZcoUqqqyr/z63ve+x4QJE9h9992ZNm1ag/1MnDiRb33rWwDstddevPfee1RXV1NVVcWkSZNYvHgxAJdffjknnHAC++23H7/4xS847bTTmDhxYtMepFkr4I+jTKqqqqI1XQc8ZswYbr31VnbZZRd69erFTjvtxE033bRh/Q9/+EPeeecdfvCDH5TdfunSpfTt25clS5bQvn17AKZPn85FF13E2rVrGTRoEA899BAzZ87caLvx48ezYsUKjjjiCG644QY6derEqFGj2HnnnZvuYM1aMEkzIqLsl556CqKVOvPMM3n22Wd5/PHH2WOPPTaZDvjyl7+8YQqhnHvvvZeTTz55Q/gCDBw4kCeeeIKnn36ao446apM+V69ezfjx4znvvPO49NJLGTt2LP379+euu+7atgdn1ko4gFupZcuyLzV+9dVX+fWvf82wYcOYP3/+hvWTJ0/moIMOqnf7u+++m2HDhpXtc+3atVx33XWcc845G63/0Y9+xIUXXkj79u1Zs2YNkmjTpg2rV6/eVodl1qp4DriVOuWUU6itraV9+/bceuutdOrUibPOOot58+bRpk0bDjjgAEaPHg1AdXU1o0eP5o477gBg0aJFLF68mKOPPnqjPq+//np++9vf8v7773Puuedy7LHHbli3ZMkSqqurueKKKwD49re/zYABA+jYseOGy9jMbGOeA062dA64/3cmNMForLnMuH540UOwVs5zwGZmLZAD2MysIA5gM7OCOIDNzAriADYzK4gD2MysIA5gM7OCOIDNzAriADYzK4gD2MysIA5gM7OCOIDNzArSZAEsaaykZZLm5mr3SJqVlkWSZqV6d0lrcutG57bpL2mOpAWSblb6Hh1Je0h6VNL89LNTqiu1WyBptqTDmuoYzcy2RlOeAY8DBucLEfGliOgXEf2A+4Bf51a/XLcuIvIfNHsbMBLomZa6Pi8BpkZET2Bqug8wJNd2ZNrezKzFabIAjojHgeXl1qWz2NOAuxvqQ9K+wG4RMT19u+gE4KS0+kRgfLo9vqQ+IX0h6ZNAx9SPmVmLUtQc8GeA1yJifq7WQ9JMSX+U9JlU2x+oybWpSTWAfSJiKUD6uXdum8X1bGNm1mIU9Y0Yw9j47Hcp0C0iaiX1Bx6Q1Bso973pjX2CfMXbSBpJNk1Bt27dGh20mdm21OxnwJLaAV8A7qmrRcTaiKhNt2cALwMHkp29dslt3gVYkm6/Vje1kH4uS/UaoGs922wkIm6PiKqIqOrcufPWHpqZ2WYpYgrin4AXI2LD1IKkzpLaptsfI3sBbWGaWnhL0oA0bzwceDBtNhkYkW6PKKkPT1dDDABW1k1VmJm1JE15GdrdwHTgE5JqJJ2ZVg1l0xffjgJmS3oOmAScExF1L+CdC9wBLCA7M/5dql8LHC9pPnB8ug8wBViY2v8MOG9bH5uZ2bbQZHPAETGsnvoZZWr3kV2WVq59NdCnTL0WOK5MPYDzN3O4ZmbNzu+EMzMriAPYzKwgDmAzs4I4gM3MCuIANjMriAPYzKwgDmAzs4I4gM3MCuIANjMriAPYzKwgDmAzs4I4gM3MCuIANjMriAPYzKwgDmAzs4I4gM3MCuIANjMriAPYzKwgDmAzs4I4gM3MCuIANjMriAPYzKwgDmAzs4I4gM3MCuIANjMriAPYzKwgTRbAksZKWiZpbq52haS/SZqVls/l1l0qaYGkeZJOyNUHp9oCSZfk6j0kPSVpvqR7JHVI9R3S/QVpffemOkYzs63RlGfA44DBZeo3RUS/tEwBkNQLGAr0Ttv8VFJbSW2BW4EhQC9gWGoLcF3qqyfwJnBmqp8JvBkR/wDclNqZmbU4TRbAEfE4sLzC5icCEyNibUS8AiwADk/LgohYGBHvAhOBEyUJOBaYlLYfD5yU62t8uj0JOC61NzNrUYqYA75A0uw0RdEp1fYHFufa1KRaffU9gRURsa6kvlFfaf3K1H4TkkZKqpZU/frrr2/9kZmZbYbmDuDbgI8D/YClwI2pXu4MNbag3lBfmxYjbo+Iqoio6ty5c0PjNjPb5po1gCPitYhYHxHvAz8jm2KA7Ay2a65pF2BJA/U3gI6S2pXUN+orrd+dyqdCzMyaTbMGsKR9c3dPBuqukJgMDE1XMPQAegJPA88APdMVDx3IXqibHBEBTANOTduPAB7M9TUi3T4VeCy1NzNrUdo13mTLSLobOAbYS1INcDlwjKR+ZFMCi4CzASLieUn3An8B1gHnR8T61M8FwCNAW2BsRDyfdnExMFHS1cBMYEyqjwHulLSA7Mx3aFMdo5nZ1miyAI6IYWXKY8rU6tpfA1xTpj4FmFKmvpAPpjDy9XeAL27WYM3MCuB3wpmZFcQBbGZWEAewmVlBHMBmZgVxAJuZFcQBbGZWEAewmVlBHMBmZgVxAJuZFcQBbGZWEAewmVlBHMBmZgVxAJuZFcQBbGZWEAewmVlBHMBmZgVxAJuZFcQBbGZWEAewmVlBHMBmZgVxAJuZFcQBbGZWEAewmVlBHMBmZgVxAJuZFaTJAljSWEnLJM3N1a6X9KKk2ZLul9Qx1btLWiNpVlpG57bpL2mOpAWSbpakVN9D0qOS5qefnVJdqd2CtJ/DmuoYzcy2RlOeAY8DBpfUHgX6RERf4CXg0ty6lyOiX1rOydVvA0YCPdNS1+clwNSI6AlMTfcBhuTajkzbm5m1OE0WwBHxOLC8pPb7iFiX7j4JdGmoD0n7ArtFxPSICGACcFJafSIwPt0eX1KfEJkngY6pHzOzFqXIOeCvAb/L3e8haaakP0r6TKrtD9Tk2tSkGsA+EbEUIP3cO7fN4nq2MTNrMdoVsVNJ3wPWAXel0lKgW0TUSuoPPCCpN6Aym0dj3Ve6jaSRZNMUdOvWrZKhm5ltM81+BixpBPB54PQ0rUBErI2I2nR7BvAycCDZ2Wt+mqILsCTdfq1uaiH9XJbqNUDXerbZSETcHhFVEVHVuXPnbXF4ZmYVa9YAljQYuBj4l4hYnat3ltQ23f4Y2QtoC9PUwluSBqSrH4YDD6bNJgMj0u0RJfXh6WqIAcDKuqkKM7OWpMmmICTdDRwD7CWpBric7KqHHYBH09VkT6YrHo4CrpS0DlgPnBMRdS/gnUt2RcVOZHPGdfPG1wL3SjoTeBX4YqpPAT4HLABWA19tqmM0M9saTRbAETGsTHlMPW3vA+6rZ1010KdMvRY4rkw9gPM3a7BmZgXwO+HMzAriADYzK4gD2MysIA5gM7OCOIDNzAriADYzK4gD2MysIA5gM7OCOIDNzAriADYzK4gD2MysIA5gM7OCOIDNzAriADYzK4gD2MysIA5gM7OCOIDNzAriADYzK4gD2MysIA5gM7OCVBTAkqZWUjMzs8o1+K3IknYEdib7avlOgNKq3YD9mnhsZmatWmNfS3828A2ysJ3BBwG8Cri1CcdlZtbqNRjAETEKGCXp3yPilmYak5nZh0JjZ8AARMQtko4Euue3iYgJTTQuM7NWr6IAlnQn8HFgFrA+lQNwAJuZbaFKL0OrAj4dEedFxL+n5euNbSRprKRlkubmantIelTS/PSzU6pL0s2SFkiaLemw3DYjUvv5kkbk6v0lzUnb3CxJDe3DzKwlqTSA5wIf3YL+xwGDS2qXAFMjoicwNd0HGAL0TMtI4DbIwhS4HDgCOBy4PBeot6W2ddsNbmQfZmYtRqUBvBfwF0mPSJpctzS2UUQ8DiwvKZ8IjE+3xwMn5eoTIvMk0FHSvsAJwKMRsTwi3gQeBQandbtFxPSIqJsOOamRfZiZtRgVzQEDV2zDfe4TEUsBImKppL1TfX9gca5dTao1VK8pU29oHxuRNJLsDJpu3bptzTGZmW22Sq+C+GNTD4QPrjHeaNdbUK9YRNwO3A5QVVW1WduamW2tSt+K/JakVWl5R9J6Sau2cJ+vpekD0s9lqV4DdM216wIsaaTepUy9oX2YmbUYFQVwROwaEbulZUfgFOA/t3Cfk4G6KxlGAA/m6sPT1RADgJVpGuERYJCkTunFt0HAI2ndW5IGpKsfhpf0VW4fZmYtRqVzwBuJiAckNXplgaS7gWPIPkuihuxqhmuBeyWdCbwKfDE1nwJ8DlgArAa+mva1XNJVwDOp3ZURUffC3rlkV1rsBPwuLTSwDzOzFqPSN2J8IXe3Ddl1wY3OmUbEsHpWHVembQDn19PPWGBsmXo10KdMvbbcPszMWpJKz4D/X+72OmAR2aVeZma2hSq9CuKrTT0QM7MPm0qvgugi6f70tuLXJN0nqUvjW5qZWX0qfSfcz8muLNiP7M0Ov0k1MzPbQpUGcOeI+HlErEvLOKBzE47LzKzVqzSA35D0FUlt0/IVoLYpB2Zm1tpVGsBfA04D/hdYCpxKuk7XzMy2TKWXoV0FjEifRlb3EZE3kAWzmZltgUrPgPvWhS9k704DDm2aIZmZfThUGsBt8t8qkc6At+htzGZmlqk0RG8E/ixpEtlbkE8DrmmyUZmZfQhU+k64CZKqgWPJPof3CxHxlyYdmZlZK1fxNEIKXIeumdk2UukcsJmZbWMOYDOzgjiAzcwK4gA2MyuIA9jMrCAOYDOzgjiAzcwK4gA2MyuIA9jMrCAOYDOzgjiAzcwK4gA2MytIswewpE9ImpVbVkn6hqQrJP0tV/9cbptLJS2QNE/SCbn64FRbIOmSXL2HpKckzZd0j6QOzX2cZmaNafYAjoh5EdEvIvoB/YHVwP1p9U116yJiCoCkXsBQoDcwGPhp3ZeDArcCQ4BewLDUFuC61FdP4E3gzOY6PjOzShU9BXEc8HJE/LWBNicCEyNibUS8AiwADk/LgohYGBHvAhOBEyWJ7HOLJ6XtxwMnNdkRmJltoaIDeChwd+7+BZJmSxqb+wqk/YHFuTY1qVZffU9gRUSsK6mbmbUohQVwmpf9F+BXqXQb8HGgH7CU7GuQIPsGjlKxBfVyYxgpqVpS9euvv74Zozcz23pFngEPAZ6NiNcAIuK1iFgfEe8DPyObYoDsDLZrbrsuwJIG6m8AHSW1K6lvIiJuj4iqiKjq3LnzNjosM7PKFBnAw8hNP0jaN7fuZGBuuj0ZGCppB0k9gJ7A08AzQM90xUMHsumMyRERwDTg1LT9CODBJj0SM7MtUMhXy0vaGTgeODtX/pGkfmTTBYvq1kXE85LuJfs+unXA+RGxPvVzAfAI0BYYGxHPp74uBiZKuhqYCYxp8oMyM9tMhQRwRKwme7EsX/vXBtpfA1xTpj4FmFKmvpAPpjDMzFqkoq+CMDP70HIAm5kVxAFsZlYQB7CZWUEcwGZmBXEAm5kVxAFsZlYQB7CZWUEcwGZmBXEAm5kVxAFsZlYQB7CZWUEcwGZmBXEAm5kVxAFsZlYQB7CZWUEcwGZmBXEAm5kVxAFsZlYQB7CZWUEcwGZmBXEAm5kVxAFsZlYQB7CZWUEcwGZmBXEAm5kVpLAAlrRI0hxJsyRVp9oekh6VND/97JTqknSzpAWSZks6LNfPiNR+vqQRuXr/1P+CtK2a/yjNzOpX9BnwZyOiX0RUpfuXAFMjoicwNd0HGAL0TMtI4DbIAhu4HDgCOBy4vC60U5uRue0GN/3hmJlVrugALnUiMD7dHg+clKtPiMyTQEdJ+wInAI9GxPKIeBN4FBic1u0WEdMjIoAJub7MzFqEIgM4gN9LmiFpZKrtExFLAdLPvVN9f2BxbtuaVGuoXlOmbmbWYrQrcN+fjoglkvYGHpX0YgNty83fxhbUN+40C/6RAN26dWt8xGZm21BhZ8ARsST9XAbcTzaH+1qaPiD9XJaa1wBdc5t3AZY0Uu9Spl46htsjoioiqjp37rwtDsvMrGKFBLCkj0jate42MAiYC0wG6q5kGAE8mG5PBoanqyEGACvTFMUjwCBJndKLb4OAR9K6tyQNSFc/DM/1ZWbWIhQ1BbEPcH+6Mqwd8MuIeFjSM8C9ks4EXgW+mNpPAT4HLABWA18FiIjlkq4CnkntroyI5en2ucA4YCfgd2kxM2sxCgngiFgIfLJMvRY4rkw9gPPr6WssMLZMvRros9WDNTNrIi3tMjQzsw8NB7CZWUEcwGZmBXEAm5kVxAFsZlYQB7CZWUEcwGZmBXEAm5kVxAFsZlYQB7CZWUEcwGZmBXEAm5kVxAFsZlYQB7CZWUEcwGZmBXEAm5kVxAFsZlYQB7CZWUEcwGZmBXEAm5kVxAFsZlYQB7CZWUEcwGZmBXEAm5kVxAFsZlaQZg9gSV0lTZP0gqTnJV2Y6ldI+pukWWn5XG6bSyUtkDRP0gm5+uBUWyDpkly9h6SnJM2XdI+kDs17lGZmjSviDHgd8O2IOBgYAJwvqVdad1NE9EvLFIC0bijQGxgM/FRSW0ltgVuBIUAvYFiun+tSXz2BN4Ezm+vgzMwq1ewBHBFLI+LZdPst4AVg/wY2ORGYGBFrI+IVYAFweFoWRMTCiHgXmAicKEnAscCktP144KSmORozsy1X6BywpO7AocBTqXSBpNmSxkrqlGr7A4tzm9WkWn31PYEVEbGupG5m1qIUFsCSdgHuA74REauA24CPA/2ApcCNdU3LbB5bUC83hpGSqiVVv/7665t5BGZmW6eQAJbUnix874qIXwNExGsRsT4i3gd+RjbFANkZbNfc5l2AJQ3U3wA6SmpXUt9ERNweEVURUdW5c+dtc3BmZhUq4ioIAWOAFyLix7n6vrlmJwNz0+3JwFBJO0jqAfQEngaeAXqmKx46kL1QNzkiApgGnJq2HwE82JTHZGa2Jdo13mSb+zTwr8AcSbNS7btkVzH0I5suWAScDRARz0u6F/gL2RUU50fEegBJFwCPAG2BsRHxfOrvYmCipKuBmWSBb2bWojR7AEfEnyg/TzulgW2uAa4pU59SbruIWMgHUxhmZi2S3wlnZlYQB7CZWUEcwGZmBXEAm5kVxAFsZlYQB7CZbT26JBsAAAdXSURBVGTFihWceuqpHHTQQRx88MFMnz59o/V33XUXffv2pW/fvhx55JE899xzG9Z97WtfY++996ZPnz4bbXPxxRfTt29fhg8fvqF25513MmrUqKY9mBbOAWxmG7nwwgsZPHgwL774Is899xwHH3zwRut79OjBH//4R2bPns1ll13GyJEjN6w744wzePjhhzdqv3LlSv785z8ze/Zs1q9fz5w5c1izZg3jxo3jvPPOa5ZjaqmKeCOGmbVQq1at4vHHH2fcuHEAdOjQgQ4dNv447SOPPHLD7QEDBlBTU7Ph/lFHHcWiRYs2at+mTRveffddIoI1a9bQvn17rr/+er7+9a/Tvn37JjuW7YHPgM1sg4ULF9K5c2e++tWvcuihh3LWWWfx9ttv19t+zJgxDBkypME+d911V0455RQOPfRQevTowe67784zzzzDiSeeuK2Hv91xAJvZBuvWrePZZ5/l3HPPZebMmXzkIx/h2muvLdt22rRpjBkzhuuuu67Rfi+66CJmzZrFjTfeyGWXXcaVV17JHXfcwWmnncbVV1+9rQ9ju+EANrMNunTpQpcuXTjiiCMAOPXUU3n22Wc3aTd79mzOOussHnzwQfbcc8+K+585cyYABx54IBMmTODee+9l7ty5zJ8/f9scwHbGAWxmG3z0ox+la9euzJs3D4CpU6fSq1evjdq8+uqrfOELX+DOO+/kwAMP3Kz+685+33vvPdavXw9kc8SrV6/eNgewnXEAm9lGbrnlFk4//XT69u3LrFmz+O53v8vo0aMZPXo0AFdeeSW1tbWcd9559OvXj6qqqg3bDhs2jIEDBzJv3jy6dOnCmDEffBDhAw88wKc+9Sn2228/OnbsyMCBAznkkEOQxCc/+clmP86WQNnH51pVVVVUV1dv9nb9vzOhCUZjzWXG9cMbb7QNvXrlIc26P9t2un1/zhZtJ2lGRFSVW+czYDOzgjiAzcwK4gA2MyuIA9jMrCAOYDOzgjiAzcwK4gA2MyuIA9jMrCAOYDOzgjiAzcwK4gA2MyuIA9jMrCCtNoAlDZY0T9ICSZcUPR4zs1KtMoAltQVuBYYAvYBhkno1vJWZWfNqlQEMHA4siIiFEfEuMBHwF1CZWYvSWgN4f2Bx7n5NqpmZtRit9WvpVaa2ySfPSxoJjEx3/y5pXpOOavu0F/BG0YNoKrphRNFDaE1a9XOFy8vFSkUOqG9Faw3gGqBr7n4XYElpo4i4Hbi9uQa1PZJUXd+n+Zvl+bmy+VrrFMQzQE9JPSR1AIYCkwsek5nZRlrlGXBErJN0AfAI0BYYGxHPFzwsM7ONtMoABoiIKcCUosfRCniKxirl58pm8rcim5kVpLXOAZuZtXgOYCvLb+W2SkkaK2mZpLlFj2V74wC2Tfit3LaZxgGDix7E9sgBbOX4rdxWsYh4HFhe9Di2Rw5gK8dv5TZrBg5gK6eit3Kb2dZxAFs5Fb2V28y2jgPYyvFbuc2agQPYNhER64C6t3K/ANzrt3JbfSTdDUwHPiGpRtKZRY9pe+F3wpmZFcRnwGZmBXEAm5kVxAFsZlYQB7CZWUEcwGZmBXEA23ZJ0p+LHsPmkrRI0l5Fj8NaDgewbZci4siix2C2tRzAtl2S9Pf08xhJf5A0SdKLku6SpLTuU5L+LOk5SU9L2lXSjpJ+LmmOpJmSPpvaniHpAUm/kfSKpAskfSu1eVLSHqndxyU9LGmGpCckHVRmbLvk9jFb0ill2jyQ+nhe0shUaytpnKS5adtvpvrXJf0l9TWx6R5Va26t9jvh7EPlUKA32edV/A/waUlPA/cAX4qIZyTtBqwBLgSIiENSeP5e0oGpnz6prx2BBcDFEXGopJuA4cBPyL737JyImC/pCOCnwLEl47kMWBkRhwBI6lRmzF+LiOWSdgKekXQf0B3YPyL6pO06praXAD0iYm2uZq2AA9hag6cjogZA0iyyIFsJLI2IZwAiYlVa/4/ALan2oqS/AnUBPC0i3gLekrQS+E2qzwH6StoFOBL4VTrJBtihzHj+iezzM0j7ebNMm69LOjnd7gr0BOYBH5N0C/AQ8Pu0fjZwl6QHgAcqekRsu+ApCGsN1uZuryc7sRDlP0Kz3Edtluvn/dz991OfbYAVEdEvtxxczz7qfY+/pGPIQnpgRHwSmAnsmIL6k8AfgPOBO9Im/0z2DSX9gRmSfOLUSjiArbV6EdhP0qcA0vxvO+Bx4PRUOxDoRnbm2ah0Fv2KpC+m7SXpk2Wa/p7sw4xI7UqnIHYH3oyI1WkaZEBqtxfQJiLuI5vGOExSG6BrREwDLgI6ArtUMl5r+RzA1iqlr1L6EnCLpOeAR8nmdn8KtJU0h2yO+IyIWFt/T5s4HTgz9fk85b+q6WqgU3ox7TngsyXrHwbaSZoNXAU8mer7A39I0yjjgEuBtsAv0nhnAjdFxIrNGK+1YP40NDOzgvgM2MysIA5gM7OCOIDNzAriADYzK4gD2MysIA5gM7OCOIDNzAriADYzK8j/AYBwe8u8YKt7AAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 360x360 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.figure(figsize=(5, 5))\n",
    "total = census.shape[0]\n",
    "ax = sns.countplot(x=\"income class\", data=census)\n",
    "for p in ax.patches:\n",
    "    height = p.get_height()\n",
    "    ax.text(p.get_x() + p.get_width()/2.,\n",
    "            height + 3,\n",
    "            '{0:.2%}'.format(height/total),\n",
    "            ha=\"center\")\n",
    "plt.title('Distribution of income classes', fontsize=15)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-03-16T08:03:12.566688Z",
     "start_time": "2020-03-16T08:03:12.367221Z"
    }
   },
   "outputs": [],
   "source": [
    "X_train, X_val, y_train, y_val = train_test_split(X, y, test_size=0.2, stratify=y, random_state=666)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-03-16T08:03:13.232034Z",
     "start_time": "2020-03-16T08:03:13.217074Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "15.113264688067838"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ratio = float(np.sum(y_train == 0)) / np.sum(y_train == 1)\n",
    "ratio"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Feature Selection"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Tune SelectKBest"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-03-15T10:32:59.594452Z",
     "start_time": "2020-03-15T10:32:59.581488Z"
    }
   },
   "outputs": [],
   "source": [
    "preprocessor = make_column_transformer(\n",
    "    (Pipeline([\n",
    "        ('num', SimpleImputer()),\n",
    "        ('sc', StandardScaler()),\n",
    "    ]), numerical_features),\n",
    "    (Pipeline([\n",
    "        ('num', SimpleImputer(strategy='constant', fill_value='Unknown')),\n",
    "        ( 'enc', OneHotEncoder(categories='auto', sparse = False, handle_unknown='ignore' ))\n",
    "    ]), categorical_features),\n",
    "    remainder='drop'\n",
    ")\n",
    "\n",
    "pipeline = Pipeline([\n",
    "    ('pre', preprocessor),\n",
    "    ('sel', SelectKBest()),\n",
    "    ('clf', XGBClassifier(random_state=666, tree_method='gpu_hist', objective='binary:logistic'))\n",
    "]) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-03-15T10:53:08.699506Z",
     "start_time": "2020-03-15T10:33:12.077638Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 5 folds for each of 20 candidates, totalling 100 fits\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=5)]: Using backend LokyBackend with 5 concurrent workers.\n",
      "[Parallel(n_jobs=5)]: Done  31 tasks      | elapsed:  5.3min\n",
      "[Parallel(n_jobs=5)]: Done 100 out of 100 | elapsed: 19.6min finished\n"
     ]
    }
   ],
   "source": [
    "parameters = {\n",
    "    'sel__k': range(1,101,5),\n",
    "    'clf__learning_rate': [0.1],\n",
    "    'clf__n_estimators': [1000],\n",
    "    'clf__max_depth': [5],\n",
    "    'clf__min_child_weight': [1],\n",
    "    'clf__gamma': [0.0],    \n",
    "    'clf__subsample': [0.8],\n",
    "    'clf__colsample_bytree': [0.8],\n",
    "    'clf__scale_pos_weight': [ratio]\n",
    "}\n",
    "\n",
    "model = GridSearchCV(pipeline, param_grid=parameters, scoring='roc_auc', n_jobs=5, verbose=2)\n",
    "\n",
    "model.fit(X_train, y_train);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-03-15T10:53:10.798776Z",
     "start_time": "2020-03-15T10:53:09.516778Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best score: 0.9460770141835386\n",
      "Best params: {'clf__colsample_bytree': 0.8, 'clf__gamma': 0.0, 'clf__learning_rate': 0.1, 'clf__max_depth': 5, 'clf__min_child_weight': 1, 'clf__n_estimators': 1000, 'clf__scale_pos_weight': 15.113264688067838, 'clf__subsample': 0.8, 'sel__k': 91}\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.99      0.90      0.94     37429\n",
      "           1       0.35      0.84      0.49      2476\n",
      "\n",
      "    accuracy                           0.89     39905\n",
      "   macro avg       0.67      0.87      0.72     39905\n",
      "weighted avg       0.95      0.89      0.91     39905\n",
      "\n",
      "AUC: 0.8665554980110755\n"
     ]
    }
   ],
   "source": [
    "print(\"Best score: %s\" % model.best_score_)\n",
    "print(\"Best params: %s\" % model.best_params_)\n",
    "\n",
    "y_pred = model.predict(X_val)\n",
    "\n",
    "print(classification_report(y_val, y_pred))\n",
    "fpr, tpr, thresholds = roc_curve(y_val, y_pred)\n",
    "\n",
    "print(\"AUC: %s\" % auc(fpr, tpr))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-03-15T11:06:19.803779Z",
     "start_time": "2020-03-15T10:55:33.657198Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 5 folds for each of 10 candidates, totalling 50 fits\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=5)]: Using backend LokyBackend with 5 concurrent workers.\n",
      "[Parallel(n_jobs=5)]: Done  31 tasks      | elapsed:  6.3min\n",
      "[Parallel(n_jobs=5)]: Done  50 out of  50 | elapsed: 10.4min finished\n"
     ]
    }
   ],
   "source": [
    "parameters = {\n",
    "    'sel__k': range(85,95,1),\n",
    "    'clf__learning_rate': [0.1],\n",
    "    'clf__n_estimators': [1000],\n",
    "    'clf__max_depth': [5],\n",
    "    'clf__min_child_weight': [1],\n",
    "    'clf__gamma': [0.0],    \n",
    "    'clf__subsample': [0.8],\n",
    "    'clf__colsample_bytree': [0.8],\n",
    "    'clf__scale_pos_weight': [ratio]\n",
    "}\n",
    "\n",
    "model = GridSearchCV(pipeline, param_grid=parameters, scoring='roc_auc', n_jobs=5, verbose=2)\n",
    "\n",
    "model.fit(X_train, y_train);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-03-15T11:06:24.549088Z",
     "start_time": "2020-03-15T11:06:23.270507Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best score: 0.9463054829332197\n",
      "Best params: {'clf__colsample_bytree': 0.8, 'clf__gamma': 0.0, 'clf__learning_rate': 0.1, 'clf__max_depth': 5, 'clf__min_child_weight': 1, 'clf__n_estimators': 1000, 'clf__scale_pos_weight': 15.113264688067838, 'clf__subsample': 0.8, 'sel__k': 94}\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.99      0.90      0.94     37429\n",
      "           1       0.36      0.84      0.50      2476\n",
      "\n",
      "    accuracy                           0.90     39905\n",
      "   macro avg       0.67      0.87      0.72     39905\n",
      "weighted avg       0.95      0.90      0.91     39905\n",
      "\n",
      "AUC: 0.8694282769345394\n"
     ]
    }
   ],
   "source": [
    "print(\"Best score: %s\" % model.best_score_)\n",
    "print(\"Best params: %s\" % model.best_params_)\n",
    "\n",
    "y_pred = model.predict(X_val)\n",
    "\n",
    "print(classification_report(y_val, y_pred))\n",
    "fpr, tpr, thresholds = roc_curve(y_val, y_pred)\n",
    "\n",
    "print(\"AUC: %s\" % auc(fpr, tpr))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Tune max_depth and min_child_weight"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-03-15T11:32:20.652652Z",
     "start_time": "2020-03-15T11:06:50.799889Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 5 folds for each of 12 candidates, totalling 60 fits\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=5)]: Using backend LokyBackend with 5 concurrent workers.\n",
      "[Parallel(n_jobs=5)]: Done  31 tasks      | elapsed:  7.1min\n",
      "[Parallel(n_jobs=5)]: Done  60 out of  60 | elapsed: 25.3min finished\n"
     ]
    }
   ],
   "source": [
    "parameters = {\n",
    "    'sel__k': [94],\n",
    "    'clf__learning_rate': [0.1],\n",
    "    'clf__n_estimators': [1000],\n",
    "    'clf__max_depth': range(3,10,2),\n",
    "    'clf__min_child_weight': range(1,6,2),\n",
    "    'clf__gamma': [0.0],    \n",
    "    'clf__subsample': [0.8],\n",
    "    'clf__colsample_bytree': [0.8],\n",
    "    'clf__scale_pos_weight': [ratio]\n",
    "}\n",
    "\n",
    "model = GridSearchCV(pipeline, param_grid=parameters, scoring='roc_auc', n_jobs=5, verbose=2)\n",
    "\n",
    "model.fit(X_train, y_train);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-03-15T11:32:22.509683Z",
     "start_time": "2020-03-15T11:32:21.458494Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best score: 0.9507559379499921\n",
      "Best params: {'clf__colsample_bytree': 0.8, 'clf__gamma': 0.0, 'clf__learning_rate': 0.1, 'clf__max_depth': 3, 'clf__min_child_weight': 1, 'clf__n_estimators': 1000, 'clf__scale_pos_weight': 15.113264688067838, 'clf__subsample': 0.8, 'sel__k': 94}\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.99      0.88      0.93     37429\n",
      "           1       0.32      0.88      0.47      2476\n",
      "\n",
      "    accuracy                           0.88     39905\n",
      "   macro avg       0.65      0.88      0.70     39905\n",
      "weighted avg       0.95      0.88      0.90     39905\n",
      "\n",
      "AUC: 0.8775726576513136\n"
     ]
    }
   ],
   "source": [
    "print(\"Best score: %s\" % model.best_score_)\n",
    "print(\"Best params: %s\" % model.best_params_)\n",
    "\n",
    "y_pred = model.predict(X_val)\n",
    "\n",
    "print(classification_report(y_val, y_pred))\n",
    "fpr, tpr, thresholds = roc_curve(y_val, y_pred)\n",
    "\n",
    "print(\"AUC: %s\" % auc(fpr, tpr))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Tune gamma"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-03-15T11:37:48.896647Z",
     "start_time": "2020-03-15T11:34:38.197936Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 5 folds for each of 5 candidates, totalling 25 fits\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=5)]: Using backend LokyBackend with 5 concurrent workers.\n",
      "[Parallel(n_jobs=5)]: Done  25 out of  25 | elapsed:  2.9min finished\n"
     ]
    }
   ],
   "source": [
    "parameters = {\n",
    "    'sel__k': [94],\n",
    "    'clf__learning_rate': [0.1],\n",
    "    'clf__n_estimators': [1000],\n",
    "    'clf__max_depth': [3],\n",
    "    'clf__min_child_weight': [1],\n",
    "    'clf__gamma': [i/10.0 for i in range(0,5)] ,    \n",
    "    'clf__subsample': [0.8],\n",
    "    'clf__colsample_bytree': [0.8],\n",
    "    'clf__scale_pos_weight': [ratio]\n",
    "}\n",
    "\n",
    "model = GridSearchCV(pipeline, param_grid=parameters, scoring='roc_auc', n_jobs=5, verbose=2)\n",
    "\n",
    "model.fit(X_train, y_train);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-03-15T11:37:50.889319Z",
     "start_time": "2020-03-15T11:37:49.780287Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best score: 0.9507559379499921\n",
      "Best params: {'clf__colsample_bytree': 0.8, 'clf__gamma': 0.0, 'clf__learning_rate': 0.1, 'clf__max_depth': 3, 'clf__min_child_weight': 1, 'clf__n_estimators': 1000, 'clf__scale_pos_weight': 15.113264688067838, 'clf__subsample': 0.8, 'sel__k': 94}\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.99      0.88      0.93     37429\n",
      "           1       0.32      0.88      0.47      2476\n",
      "\n",
      "    accuracy                           0.88     39905\n",
      "   macro avg       0.65      0.88      0.70     39905\n",
      "weighted avg       0.95      0.88      0.90     39905\n",
      "\n",
      "AUC: 0.8775726576513136\n"
     ]
    }
   ],
   "source": [
    "print(\"Best score: %s\" % model.best_score_)\n",
    "print(\"Best params: %s\" % model.best_params_)\n",
    "\n",
    "y_pred = model.predict(X_val)\n",
    "\n",
    "print(classification_report(y_val, y_pred))\n",
    "fpr, tpr, thresholds = roc_curve(y_val, y_pred)\n",
    "\n",
    "print(\"AUC: %s\" % auc(fpr, tpr))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Tune subsample and colsample_bytree"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-03-15T11:49:34.535251Z",
     "start_time": "2020-03-15T11:39:14.518191Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 5 folds for each of 16 candidates, totalling 80 fits\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=5)]: Using backend LokyBackend with 5 concurrent workers.\n",
      "[Parallel(n_jobs=5)]: Done  31 tasks      | elapsed:  4.1min\n",
      "[Parallel(n_jobs=5)]: Done  80 out of  80 | elapsed: 10.1min finished\n"
     ]
    }
   ],
   "source": [
    "parameters = {\n",
    "    'sel__k': [94],\n",
    "    'clf__learning_rate': [0.1],\n",
    "    'clf__n_estimators': [1000],\n",
    "    'clf__max_depth': [3],\n",
    "    'clf__min_child_weight': [1],\n",
    "    'clf__gamma': [0.0] ,    \n",
    "    'clf__subsample': [i/10.0 for i in range(6,10)],\n",
    "    'clf__colsample_bytree': [i/10.0 for i in range(6,10)],\n",
    "    'clf__scale_pos_weight': [ratio]\n",
    "}\n",
    "\n",
    "model = GridSearchCV(pipeline, param_grid=parameters, scoring='roc_auc', n_jobs=5, verbose=2)\n",
    "\n",
    "model.fit(X_train, y_train);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-03-15T11:49:36.438143Z",
     "start_time": "2020-03-15T11:49:35.344068Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best score: 0.9510451513577873\n",
      "Best params: {'clf__colsample_bytree': 0.8, 'clf__gamma': 0.0, 'clf__learning_rate': 0.1, 'clf__max_depth': 3, 'clf__min_child_weight': 1, 'clf__n_estimators': 1000, 'clf__scale_pos_weight': 15.113264688067838, 'clf__subsample': 0.9, 'sel__k': 94}\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.99      0.88      0.93     37429\n",
      "           1       0.32      0.88      0.47      2476\n",
      "\n",
      "    accuracy                           0.88     39905\n",
      "   macro avg       0.66      0.88      0.70     39905\n",
      "weighted avg       0.95      0.88      0.90     39905\n",
      "\n",
      "AUC: 0.8787945294895654\n"
     ]
    }
   ],
   "source": [
    "print(\"Best score: %s\" % model.best_score_)\n",
    "print(\"Best params: %s\" % model.best_params_)\n",
    "\n",
    "y_pred = model.predict(X_val)\n",
    "\n",
    "print(classification_report(y_val, y_pred))\n",
    "fpr, tpr, thresholds = roc_curve(y_val, y_pred)\n",
    "\n",
    "print(\"AUC: %s\" % auc(fpr, tpr))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-03-15T12:00:37.427986Z",
     "start_time": "2020-03-15T11:54:45.460657Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 5 folds for each of 9 candidates, totalling 45 fits\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=5)]: Using backend LokyBackend with 5 concurrent workers.\n",
      "[Parallel(n_jobs=5)]: Done  31 tasks      | elapsed:  4.0min\n",
      "[Parallel(n_jobs=5)]: Done  45 out of  45 | elapsed:  5.6min finished\n"
     ]
    }
   ],
   "source": [
    "parameters = {\n",
    "    'sel__k': [94],\n",
    "    'clf__learning_rate': [0.1],\n",
    "    'clf__n_estimators': [1000],\n",
    "    'clf__max_depth': [3],\n",
    "    'clf__min_child_weight': [1],\n",
    "    'clf__gamma': [0.0] ,    \n",
    "    'clf__subsample': [i/100.0 for i in range(85,96,5)],\n",
    "    'clf__colsample_bytree': [i/100.0 for i in range(75,86,5)],\n",
    "    'clf__scale_pos_weight': [ratio]\n",
    "}\n",
    "\n",
    "model = GridSearchCV(pipeline, param_grid=parameters, scoring='roc_auc', n_jobs=5, verbose=2)\n",
    "\n",
    "model.fit(X_train, y_train);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-03-15T12:00:39.375596Z",
     "start_time": "2020-03-15T12:00:38.279840Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best score: 0.9511754546694743\n",
      "Best params: {'clf__colsample_bytree': 0.8, 'clf__gamma': 0.0, 'clf__learning_rate': 0.1, 'clf__max_depth': 3, 'clf__min_child_weight': 1, 'clf__n_estimators': 1000, 'clf__scale_pos_weight': 15.113264688067838, 'clf__subsample': 0.95, 'sel__k': 94}\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.99      0.87      0.93     37429\n",
      "           1       0.32      0.88      0.47      2476\n",
      "\n",
      "    accuracy                           0.88     39905\n",
      "   macro avg       0.65      0.88      0.70     39905\n",
      "weighted avg       0.95      0.88      0.90     39905\n",
      "\n",
      "AUC: 0.877856307241657\n"
     ]
    }
   ],
   "source": [
    "print(\"Best score: %s\" % model.best_score_)\n",
    "print(\"Best params: %s\" % model.best_params_)\n",
    "\n",
    "y_pred = model.predict(X_val)\n",
    "\n",
    "print(classification_report(y_val, y_pred))\n",
    "fpr, tpr, thresholds = roc_curve(y_val, y_pred)\n",
    "\n",
    "print(\"AUC: %s\" % auc(fpr, tpr))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Reducing Learning Rate in order to reduce overfit"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-03-15T12:08:04.283692Z",
     "start_time": "2020-03-15T12:02:39.264735Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 5 folds for each of 8 candidates, totalling 40 fits\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=5)]: Using backend LokyBackend with 5 concurrent workers.\n",
      "[Parallel(n_jobs=5)]: Done  31 tasks      | elapsed:  4.2min\n",
      "[Parallel(n_jobs=5)]: Done  40 out of  40 | elapsed:  5.2min finished\n"
     ]
    }
   ],
   "source": [
    "parameters = {\n",
    "    'sel__k': [94],\n",
    "    'clf__learning_rate': [i/1000.0 for i in range(5,20,2)],\n",
    "    'clf__n_estimators': [1000],\n",
    "    'clf__max_depth': [3],\n",
    "    'clf__min_child_weight': [1],\n",
    "    'clf__gamma': [0.0] ,\n",
    "    'clf__subsample': [0.9],\n",
    "    'clf__colsample_bytree': [0.8],\n",
    "    'clf__scale_pos_weight': [ratio]\n",
    "}\n",
    "\n",
    "model = GridSearchCV(pipeline, param_grid=parameters, scoring='roc_auc', n_jobs=5, verbose=2)\n",
    "\n",
    "model.fit(X_train, y_train);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-03-15T12:08:06.289788Z",
     "start_time": "2020-03-15T12:08:05.180810Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best score: 0.9477107546103735\n",
      "Best params: {'clf__colsample_bytree': 0.8, 'clf__gamma': 0.0, 'clf__learning_rate': 0.019, 'clf__max_depth': 3, 'clf__min_child_weight': 1, 'clf__n_estimators': 1000, 'clf__scale_pos_weight': 15.113264688067838, 'clf__subsample': 0.9, 'sel__k': 94}\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.99      0.85      0.92     37429\n",
      "           1       0.29      0.89      0.43      2476\n",
      "\n",
      "    accuracy                           0.86     39905\n",
      "   macro avg       0.64      0.87      0.68     39905\n",
      "weighted avg       0.95      0.86      0.89     39905\n",
      "\n",
      "AUC: 0.8720041339659091\n"
     ]
    }
   ],
   "source": [
    "print(\"Best score: %s\" % model.best_score_)\n",
    "print(\"Best params: %s\" % model.best_params_)\n",
    "\n",
    "y_pred = model.predict(X_val)\n",
    "\n",
    "print(classification_report(y_val, y_pred))\n",
    "fpr, tpr, thresholds = roc_curve(y_val, y_pred)\n",
    "\n",
    "print(\"AUC: %s\" % auc(fpr, tpr))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Final fit and evaluation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-03-16T08:07:07.081395Z",
     "start_time": "2020-03-16T08:07:07.043496Z"
    }
   },
   "outputs": [],
   "source": [
    "preprocessor = make_column_transformer(\n",
    "    (Pipeline([\n",
    "        ('num', SimpleImputer()),\n",
    "        ('sc', StandardScaler()),\n",
    "    ]), numerical_features),\n",
    "    (Pipeline([\n",
    "        ('num', SimpleImputer(strategy='constant', fill_value='Unknown')),\n",
    "        ( 'enc', OneHotEncoder(categories='auto', sparse = False, handle_unknown='ignore' ))\n",
    "    ]), categorical_features),\n",
    "    remainder='drop'\n",
    ")\n",
    "\n",
    "selectKBest_model = Pipeline([\n",
    "    ('pre', preprocessor),\n",
    "    ('sel', SelectKBest(k=94)),\n",
    "    ('clf', XGBClassifier(\n",
    "        learning_rate=0.1,\n",
    "        n_estimators=1000,\n",
    "        max_depth=3,\n",
    "        min_child_weight=1,\n",
    "        gamma=0.0,    \n",
    "        subsample=0.9,\n",
    "        colsample_bytree=0.8,\n",
    "        scale_pos_weight=ratio,\n",
    "        random_state=666, \n",
    "        tree_method='gpu_hist',\n",
    "        eval_metric='auc',\n",
    "        objective='binary:logistic'))    \n",
    "]) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-03-16T08:07:19.015619Z",
     "start_time": "2020-03-16T08:07:07.787344Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.99      0.88      0.93     37429\n",
      "           1       0.32      0.88      0.47      2476\n",
      "\n",
      "    accuracy                           0.88     39905\n",
      "   macro avg       0.66      0.88      0.70     39905\n",
      "weighted avg       0.95      0.88      0.90     39905\n",
      "\n",
      "AUC: 0.8787945294895654\n"
     ]
    }
   ],
   "source": [
    "selectKBest_model.fit(X_train, y_train);\n",
    "y_pred = selectKBest_model.predict(X_val)\n",
    "\n",
    "print(classification_report(y_val, y_pred))\n",
    "fpr, tpr, thresholds = roc_curve(y_val, y_pred)\n",
    "\n",
    "print(\"AUC: %s\" % auc(fpr, tpr))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Dimensionality Reduction"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true
   },
   "source": [
    "#### Tune PCA"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-03-15T08:55:56.791430Z",
     "start_time": "2020-03-15T08:55:56.780459Z"
    },
    "hidden": true
   },
   "outputs": [],
   "source": [
    "preprocessor = make_column_transformer(\n",
    "    (Pipeline([\n",
    "        ('num', SimpleImputer()),\n",
    "        ('sc', StandardScaler()),\n",
    "    ]), numerical_features),\n",
    "    (Pipeline([\n",
    "        ('num', SimpleImputer(strategy='constant', fill_value='Unknown')),\n",
    "        ( 'enc', OneHotEncoder(categories='auto', sparse = False, handle_unknown='ignore' ))\n",
    "    ]), categorical_features),\n",
    "    remainder='drop'\n",
    ")\n",
    "\n",
    "pipeline = Pipeline([\n",
    "    ('pre', preprocessor),\n",
    "    ('pca', PCA(random_state=666)),\n",
    "    ('clf', XGBClassifier(random_state=666, tree_method='gpu_hist', objective='binary:logistic'))\n",
    "]) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-03-15T09:16:17.889562Z",
     "start_time": "2020-03-15T08:56:44.369996Z"
    },
    "hidden": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 5 folds for each of 20 candidates, totalling 100 fits\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=5)]: Using backend LokyBackend with 5 concurrent workers.\n",
      "[Parallel(n_jobs=5)]: Done  31 tasks      | elapsed:  5.6min\n",
      "[Parallel(n_jobs=5)]: Done 100 out of 100 | elapsed: 19.2min finished\n"
     ]
    }
   ],
   "source": [
    "parameters = {\n",
    "    'pca__n_components': range(1,101,5),\n",
    "    'clf__learning_rate': [0.1],\n",
    "    'clf__n_estimators': [1000],\n",
    "    'clf__max_depth': [5],\n",
    "    'clf__min_child_weight': [1],\n",
    "    'clf__gamma': [0.0],    \n",
    "    'clf__subsample': [0.8],\n",
    "    'clf__colsample_bytree': [0.8],\n",
    "    'clf__scale_pos_weight': [ratio]\n",
    "}\n",
    "\n",
    "model = GridSearchCV(pipeline, param_grid=parameters, scoring='roc_auc', n_jobs=5, verbose=2)\n",
    "\n",
    "model.fit(X_train, y_train);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-03-15T09:18:04.977850Z",
     "start_time": "2020-03-15T09:18:03.973528Z"
    },
    "hidden": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best score: 0.9413946216019532\n",
      "Best params: {'clf__colsample_bytree': 0.8, 'clf__gamma': 0.0, 'clf__learning_rate': 0.1, 'clf__max_depth': 5, 'clf__min_child_weight': 1, 'clf__n_estimators': 1000, 'clf__scale_pos_weight': 15.113264688067838, 'clf__subsample': 0.8, 'pca__n_components': 81}\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.98      0.94      0.96     37429\n",
      "           1       0.44      0.73      0.55      2476\n",
      "\n",
      "    accuracy                           0.92     39905\n",
      "   macro avg       0.71      0.83      0.75     39905\n",
      "weighted avg       0.95      0.92      0.93     39905\n",
      "\n",
      "AUC: 0.8335087399293982\n"
     ]
    }
   ],
   "source": [
    "print(\"Best score: %s\" % model.best_score_)\n",
    "print(\"Best params: %s\" % model.best_params_)\n",
    "\n",
    "y_pred = model.predict(X_val)\n",
    "\n",
    "print(classification_report(y_val, y_pred))\n",
    "fpr, tpr, thresholds = roc_curve(y_val, y_pred)\n",
    "\n",
    "print(\"AUC: %s\" % auc(fpr, tpr))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true
   },
   "source": [
    "#### Tune max_depth and min_child_weight"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-03-15T09:40:41.034358Z",
     "start_time": "2020-03-15T09:18:27.262878Z"
    },
    "hidden": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 5 folds for each of 12 candidates, totalling 60 fits\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=5)]: Using backend LokyBackend with 5 concurrent workers.\n",
      "[Parallel(n_jobs=5)]: Done  31 tasks      | elapsed:  6.5min\n",
      "[Parallel(n_jobs=5)]: Done  60 out of  60 | elapsed: 22.0min finished\n"
     ]
    }
   ],
   "source": [
    "parameters = {\n",
    "    'pca__n_components': [81],\n",
    "    'clf__learning_rate': [0.1],\n",
    "    'clf__n_estimators': [1000],\n",
    "    'clf__max_depth': range(3,10,2),\n",
    "    'clf__min_child_weight': range(1,6,2),\n",
    "    'clf__gamma': [0.0],    \n",
    "    'clf__subsample': [0.8],\n",
    "    'clf__colsample_bytree': [0.8],\n",
    "    'clf__scale_pos_weight': [ratio]\n",
    "}\n",
    "\n",
    "model = GridSearchCV(pipeline, param_grid=parameters, scoring='roc_auc', n_jobs=5, verbose=2)\n",
    "\n",
    "model.fit(X_train, y_train);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-03-15T09:40:51.748974Z",
     "start_time": "2020-03-15T09:40:50.802503Z"
    },
    "hidden": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best score: 0.9454658119081725\n",
      "Best params: {'clf__colsample_bytree': 0.8, 'clf__gamma': 0.0, 'clf__learning_rate': 0.1, 'clf__max_depth': 3, 'clf__min_child_weight': 5, 'clf__n_estimators': 1000, 'clf__scale_pos_weight': 15.113264688067838, 'clf__subsample': 0.8, 'pca__n_components': 81}\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.99      0.89      0.94     37429\n",
      "           1       0.33      0.86      0.48      2476\n",
      "\n",
      "    accuracy                           0.88     39905\n",
      "   macro avg       0.66      0.88      0.71     39905\n",
      "weighted avg       0.95      0.88      0.91     39905\n",
      "\n",
      "AUC: 0.8752542401119517\n"
     ]
    }
   ],
   "source": [
    "print(\"Best score: %s\" % model.best_score_)\n",
    "print(\"Best params: %s\" % model.best_params_)\n",
    "\n",
    "y_pred = model.predict(X_val)\n",
    "\n",
    "print(classification_report(y_val, y_pred))\n",
    "fpr, tpr, thresholds = roc_curve(y_val, y_pred)\n",
    "\n",
    "print(\"AUC: %s\" % auc(fpr, tpr))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true
   },
   "source": [
    "#### Tune gamma"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-03-15T09:45:00.000694Z",
     "start_time": "2020-03-15T09:41:47.412769Z"
    },
    "hidden": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 5 folds for each of 5 candidates, totalling 25 fits\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=5)]: Using backend LokyBackend with 5 concurrent workers.\n",
      "[Parallel(n_jobs=5)]: Done  25 out of  25 | elapsed:  3.0min finished\n"
     ]
    }
   ],
   "source": [
    "parameters = {\n",
    "    'pca__n_components': [81],\n",
    "    'clf__learning_rate': [0.1],\n",
    "    'clf__n_estimators': [1000],\n",
    "    'clf__max_depth': [3],\n",
    "    'clf__min_child_weight': [5],\n",
    "    'clf__gamma': [i/10.0 for i in range(0,5)] ,    \n",
    "    'clf__subsample': [0.8],\n",
    "    'clf__colsample_bytree': [0.8],\n",
    "    'clf__scale_pos_weight': [ratio]\n",
    "}\n",
    "\n",
    "model = GridSearchCV(pipeline, param_grid=parameters, scoring='roc_auc', n_jobs=5, verbose=2)\n",
    "\n",
    "model.fit(X_train, y_train);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-03-15T09:45:14.733203Z",
     "start_time": "2020-03-15T09:45:13.830525Z"
    },
    "hidden": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best score: 0.9454658119081725\n",
      "Best params: {'clf__colsample_bytree': 0.8, 'clf__gamma': 0.0, 'clf__learning_rate': 0.1, 'clf__max_depth': 3, 'clf__min_child_weight': 5, 'clf__n_estimators': 1000, 'clf__scale_pos_weight': 15.113264688067838, 'clf__subsample': 0.8, 'pca__n_components': 81}\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.99      0.89      0.94     37429\n",
      "           1       0.33      0.86      0.48      2476\n",
      "\n",
      "    accuracy                           0.88     39905\n",
      "   macro avg       0.66      0.88      0.71     39905\n",
      "weighted avg       0.95      0.88      0.91     39905\n",
      "\n",
      "AUC: 0.8752542401119517\n"
     ]
    }
   ],
   "source": [
    "print(\"Best score: %s\" % model.best_score_)\n",
    "print(\"Best params: %s\" % model.best_params_)\n",
    "\n",
    "y_pred = model.predict(X_val)\n",
    "\n",
    "print(classification_report(y_val, y_pred))\n",
    "fpr, tpr, thresholds = roc_curve(y_val, y_pred)\n",
    "\n",
    "print(\"AUC: %s\" % auc(fpr, tpr))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true
   },
   "source": [
    "#### Tune subsample and colsample_bytree"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-03-15T09:56:18.843210Z",
     "start_time": "2020-03-15T09:46:12.215826Z"
    },
    "hidden": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 5 folds for each of 16 candidates, totalling 80 fits\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=5)]: Using backend LokyBackend with 5 concurrent workers.\n",
      "[Parallel(n_jobs=5)]: Done  31 tasks      | elapsed:  4.1min\n",
      "[Parallel(n_jobs=5)]: Done  80 out of  80 | elapsed:  9.9min finished\n"
     ]
    }
   ],
   "source": [
    "parameters = {\n",
    "    'pca__n_components': [81],\n",
    "    'clf__learning_rate': [0.1],\n",
    "    'clf__n_estimators': [1000],\n",
    "    'clf__max_depth': [3],\n",
    "    'clf__min_child_weight': [5],\n",
    "    'clf__gamma': [0.0] ,    \n",
    "    'clf__subsample': [i/10.0 for i in range(6,10)],\n",
    "    'clf__colsample_bytree': [i/10.0 for i in range(6,10)],\n",
    "    'clf__scale_pos_weight': [ratio]\n",
    "}\n",
    "\n",
    "model = GridSearchCV(pipeline, param_grid=parameters, scoring='roc_auc', n_jobs=5, verbose=2)\n",
    "\n",
    "model.fit(X_train, y_train);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-03-15T09:58:31.805975Z",
     "start_time": "2020-03-15T09:58:30.923793Z"
    },
    "hidden": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best score: 0.9454658119081725\n",
      "Best params: {'clf__colsample_bytree': 0.8, 'clf__gamma': 0.0, 'clf__learning_rate': 0.1, 'clf__max_depth': 3, 'clf__min_child_weight': 5, 'clf__n_estimators': 1000, 'clf__scale_pos_weight': 15.113264688067838, 'clf__subsample': 0.8, 'pca__n_components': 81}\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.99      0.89      0.94     37429\n",
      "           1       0.33      0.86      0.48      2476\n",
      "\n",
      "    accuracy                           0.88     39905\n",
      "   macro avg       0.66      0.88      0.71     39905\n",
      "weighted avg       0.95      0.88      0.91     39905\n",
      "\n",
      "AUC: 0.8752542401119517\n"
     ]
    }
   ],
   "source": [
    "print(\"Best score: %s\" % model.best_score_)\n",
    "print(\"Best params: %s\" % model.best_params_)\n",
    "\n",
    "y_pred = model.predict(X_val)\n",
    "\n",
    "print(classification_report(y_val, y_pred))\n",
    "fpr, tpr, thresholds = roc_curve(y_val, y_pred)\n",
    "\n",
    "print(\"AUC: %s\" % auc(fpr, tpr))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-03-15T10:06:06.353107Z",
     "start_time": "2020-03-15T10:00:38.409304Z"
    },
    "hidden": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 5 folds for each of 9 candidates, totalling 45 fits\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=5)]: Using backend LokyBackend with 5 concurrent workers.\n",
      "[Parallel(n_jobs=5)]: Done  31 tasks      | elapsed:  3.9min\n",
      "[Parallel(n_jobs=5)]: Done  45 out of  45 | elapsed:  5.2min finished\n"
     ]
    }
   ],
   "source": [
    "parameters = {\n",
    "    'pca__n_components': [81],\n",
    "    'clf__learning_rate': [0.1],\n",
    "    'clf__n_estimators': [1000],\n",
    "    'clf__max_depth': [3],\n",
    "    'clf__min_child_weight': [5],\n",
    "    'clf__gamma': [0.0] ,    \n",
    "    'clf__subsample': [i/100.0 for i in range(75,90,5)],\n",
    "    'clf__colsample_bytree': [i/100.0 for i in range(75,90,5)],\n",
    "    'clf__scale_pos_weight': [ratio]\n",
    "}\n",
    "\n",
    "model = GridSearchCV(pipeline, param_grid=parameters, scoring='roc_auc', n_jobs=5, verbose=2)\n",
    "\n",
    "model.fit(X_train, y_train);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-03-15T10:06:16.423705Z",
     "start_time": "2020-03-15T10:06:15.503164Z"
    },
    "hidden": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best score: 0.9457113990946489\n",
      "Best params: {'clf__colsample_bytree': 0.75, 'clf__gamma': 0.0, 'clf__learning_rate': 0.1, 'clf__max_depth': 3, 'clf__min_child_weight': 5, 'clf__n_estimators': 1000, 'clf__scale_pos_weight': 15.113264688067838, 'clf__subsample': 0.8, 'pca__n_components': 81}\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.99      0.89      0.93     37429\n",
      "           1       0.33      0.86      0.48      2476\n",
      "\n",
      "    accuracy                           0.88     39905\n",
      "   macro avg       0.66      0.87      0.71     39905\n",
      "weighted avg       0.95      0.88      0.91     39905\n",
      "\n",
      "AUC: 0.8712453036014207\n"
     ]
    }
   ],
   "source": [
    "print(\"Best score: %s\" % model.best_score_)\n",
    "print(\"Best params: %s\" % model.best_params_)\n",
    "\n",
    "y_pred = model.predict(X_val)\n",
    "\n",
    "print(classification_report(y_val, y_pred))\n",
    "fpr, tpr, thresholds = roc_curve(y_val, y_pred)\n",
    "\n",
    "print(\"AUC: %s\" % auc(fpr, tpr))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true
   },
   "source": [
    "#### Reducing Learning Rate in order to reduce overfit"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-03-15T10:12:35.447640Z",
     "start_time": "2020-03-15T10:07:15.291820Z"
    },
    "hidden": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 5 folds for each of 8 candidates, totalling 40 fits\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=5)]: Using backend LokyBackend with 5 concurrent workers.\n",
      "[Parallel(n_jobs=5)]: Done  31 tasks      | elapsed:  4.3min\n",
      "[Parallel(n_jobs=5)]: Done  40 out of  40 | elapsed:  5.1min finished\n"
     ]
    }
   ],
   "source": [
    "parameters = {\n",
    "    'pca__n_components': [81],\n",
    "    'clf__learning_rate': [i/1000.0 for i in range(5,20,2)],\n",
    "    'clf__n_estimators': [1000],\n",
    "    'clf__max_depth': [3],\n",
    "    'clf__min_child_weight': [5],\n",
    "    'clf__gamma': [0.0] ,    \n",
    "    'clf__subsample': [0.8],\n",
    "    'clf__colsample_bytree': [0.8],\n",
    "    'clf__scale_pos_weight': [ratio]\n",
    "}\n",
    "\n",
    "model = GridSearchCV(pipeline, param_grid=parameters, scoring='roc_auc', n_jobs=5, verbose=2)\n",
    "\n",
    "model.fit(X_train, y_train);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-03-15T10:12:36.914629Z",
     "start_time": "2020-03-15T10:12:35.945219Z"
    },
    "hidden": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best score: 0.9441843807457004\n",
      "Best params: {'clf__colsample_bytree': 0.8, 'clf__gamma': 0.0, 'clf__learning_rate': 0.019, 'clf__max_depth': 3, 'clf__min_child_weight': 5, 'clf__n_estimators': 1000, 'clf__scale_pos_weight': 15.113264688067838, 'clf__subsample': 0.8, 'pca__n_components': 81}\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.99      0.85      0.92     37429\n",
      "           1       0.28      0.88      0.43      2476\n",
      "\n",
      "    accuracy                           0.85     39905\n",
      "   macro avg       0.64      0.87      0.67     39905\n",
      "weighted avg       0.95      0.85      0.89     39905\n",
      "\n",
      "AUC: 0.868061990583701\n"
     ]
    }
   ],
   "source": [
    "print(\"Best score: %s\" % model.best_score_)\n",
    "print(\"Best params: %s\" % model.best_params_)\n",
    "\n",
    "y_pred = model.predict(X_val)\n",
    "\n",
    "print(classification_report(y_val, y_pred))\n",
    "fpr, tpr, thresholds = roc_curve(y_val, y_pred)\n",
    "\n",
    "print(\"AUC: %s\" % auc(fpr, tpr))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true
   },
   "source": [
    "#### Final fit and evaluation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-03-15T12:19:57.306604Z",
     "start_time": "2020-03-15T12:19:57.262651Z"
    },
    "hidden": true
   },
   "outputs": [],
   "source": [
    "preprocessor = make_column_transformer(\n",
    "    (Pipeline([\n",
    "        ('num', SimpleImputer()),\n",
    "        ('sc', StandardScaler()),\n",
    "    ]), numerical_features),\n",
    "    (Pipeline([\n",
    "        ('num', SimpleImputer(strategy='constant', fill_value='Unknown')),\n",
    "        ( 'enc', OneHotEncoder(categories='auto', sparse = False, handle_unknown='ignore' ))\n",
    "    ]), categorical_features),\n",
    "    remainder='drop'\n",
    ")\n",
    "\n",
    "pca_model = Pipeline([\n",
    "    ('pre', preprocessor),\n",
    "    ('pca', PCA(\n",
    "        n_components=81, \n",
    "        random_state=666)),\n",
    "    ('clf', XGBClassifier(\n",
    "        learning_rate=0.1,\n",
    "        n_estimators=1000,\n",
    "        max_depth=3,\n",
    "        min_child_weight=5,\n",
    "        gamma=0.0,    \n",
    "        subsample=0.8,\n",
    "        colsample_bytree=0.8,\n",
    "        scale_pos_weight=ratio,\n",
    "        random_state=666, \n",
    "        tree_method='gpu_hist',\n",
    "        objective='binary:logistic'))\n",
    "]) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-03-15T12:20:12.275181Z",
     "start_time": "2020-03-15T12:19:57.879019Z"
    },
    "hidden": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.99      0.89      0.94     37429\n",
      "           1       0.33      0.86      0.48      2476\n",
      "\n",
      "    accuracy                           0.88     39905\n",
      "   macro avg       0.66      0.88      0.71     39905\n",
      "weighted avg       0.95      0.88      0.91     39905\n",
      "\n",
      "AUC: 0.8752542401119517\n"
     ]
    }
   ],
   "source": [
    "pca_model.fit(X_train, y_train);\n",
    "y_pred = pca_model.predict(X_val)\n",
    "\n",
    "print(classification_report(y_val, y_pred))\n",
    "fpr, tpr, thresholds = roc_curve(y_val, y_pred)\n",
    "\n",
    "print(\"AUC: %s\" % auc(fpr, tpr))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Test"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Load Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-03-15T10:13:32.099545Z",
     "start_time": "2020-03-15T10:13:29.109275Z"
    }
   },
   "outputs": [],
   "source": [
    "# The C engine doesn't support RegEx so we set it to Python to avoid the warning\n",
    "census_test = pd.read_csv('data/census-income.test', header=None, sep=',\\s', na_values=['?'], engine='python')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-03-15T10:13:32.594668Z",
     "start_time": "2020-03-15T10:13:32.565779Z"
    },
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>age</th>\n",
       "      <th>class of worker</th>\n",
       "      <th>detailed industry recode</th>\n",
       "      <th>detailed occupation recode</th>\n",
       "      <th>education</th>\n",
       "      <th>wage per hour</th>\n",
       "      <th>enroll in edu inst last wk</th>\n",
       "      <th>marital status</th>\n",
       "      <th>major industry code</th>\n",
       "      <th>major occupation code</th>\n",
       "      <th>...</th>\n",
       "      <th>country of birth father</th>\n",
       "      <th>country of birth mother</th>\n",
       "      <th>country of birth self</th>\n",
       "      <th>citizenship</th>\n",
       "      <th>own business or self employed</th>\n",
       "      <th>fill inc questionnaire for veteran's admin</th>\n",
       "      <th>veterans benefits</th>\n",
       "      <th>weeks worked in year</th>\n",
       "      <th>year</th>\n",
       "      <th>income class</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>38</td>\n",
       "      <td>Private</td>\n",
       "      <td>6</td>\n",
       "      <td>36</td>\n",
       "      <td>1st 2nd 3rd or 4th grade</td>\n",
       "      <td>0</td>\n",
       "      <td>Not in universe</td>\n",
       "      <td>Married-civilian spouse present</td>\n",
       "      <td>Manufacturing-durable goods</td>\n",
       "      <td>Machine operators assmblrs &amp; inspctrs</td>\n",
       "      <td>...</td>\n",
       "      <td>Mexico</td>\n",
       "      <td>Mexico</td>\n",
       "      <td>Mexico</td>\n",
       "      <td>Foreign born- Not a citizen of U S</td>\n",
       "      <td>0</td>\n",
       "      <td>Not in universe</td>\n",
       "      <td>2</td>\n",
       "      <td>12</td>\n",
       "      <td>95</td>\n",
       "      <td>- 50000.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>44</td>\n",
       "      <td>Self-employed-not incorporated</td>\n",
       "      <td>37</td>\n",
       "      <td>12</td>\n",
       "      <td>Associates degree-occup /vocational</td>\n",
       "      <td>0</td>\n",
       "      <td>Not in universe</td>\n",
       "      <td>Married-civilian spouse present</td>\n",
       "      <td>Business and repair services</td>\n",
       "      <td>Professional specialty</td>\n",
       "      <td>...</td>\n",
       "      <td>United-States</td>\n",
       "      <td>United-States</td>\n",
       "      <td>United-States</td>\n",
       "      <td>Native- Born in the United States</td>\n",
       "      <td>0</td>\n",
       "      <td>Not in universe</td>\n",
       "      <td>2</td>\n",
       "      <td>26</td>\n",
       "      <td>95</td>\n",
       "      <td>- 50000.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>Not in universe</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>Children</td>\n",
       "      <td>0</td>\n",
       "      <td>Not in universe</td>\n",
       "      <td>Never married</td>\n",
       "      <td>Not in universe or children</td>\n",
       "      <td>Not in universe</td>\n",
       "      <td>...</td>\n",
       "      <td>United-States</td>\n",
       "      <td>United-States</td>\n",
       "      <td>United-States</td>\n",
       "      <td>Native- Born in the United States</td>\n",
       "      <td>0</td>\n",
       "      <td>Not in universe</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>95</td>\n",
       "      <td>- 50000.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>35</td>\n",
       "      <td>Private</td>\n",
       "      <td>29</td>\n",
       "      <td>3</td>\n",
       "      <td>High school graduate</td>\n",
       "      <td>0</td>\n",
       "      <td>Not in universe</td>\n",
       "      <td>Divorced</td>\n",
       "      <td>Transportation</td>\n",
       "      <td>Executive admin and managerial</td>\n",
       "      <td>...</td>\n",
       "      <td>United-States</td>\n",
       "      <td>United-States</td>\n",
       "      <td>United-States</td>\n",
       "      <td>Native- Born in the United States</td>\n",
       "      <td>2</td>\n",
       "      <td>Not in universe</td>\n",
       "      <td>2</td>\n",
       "      <td>52</td>\n",
       "      <td>94</td>\n",
       "      <td>- 50000.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>49</td>\n",
       "      <td>Private</td>\n",
       "      <td>4</td>\n",
       "      <td>34</td>\n",
       "      <td>High school graduate</td>\n",
       "      <td>0</td>\n",
       "      <td>Not in universe</td>\n",
       "      <td>Divorced</td>\n",
       "      <td>Construction</td>\n",
       "      <td>Precision production craft &amp; repair</td>\n",
       "      <td>...</td>\n",
       "      <td>United-States</td>\n",
       "      <td>United-States</td>\n",
       "      <td>United-States</td>\n",
       "      <td>Native- Born in the United States</td>\n",
       "      <td>0</td>\n",
       "      <td>Not in universe</td>\n",
       "      <td>2</td>\n",
       "      <td>50</td>\n",
       "      <td>95</td>\n",
       "      <td>- 50000.</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 42 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   age                 class of worker  detailed industry recode  \\\n",
       "0   38                         Private                         6   \n",
       "1   44  Self-employed-not incorporated                        37   \n",
       "2    2                 Not in universe                         0   \n",
       "3   35                         Private                        29   \n",
       "4   49                         Private                         4   \n",
       "\n",
       "   detailed occupation recode                            education  \\\n",
       "0                          36             1st 2nd 3rd or 4th grade   \n",
       "1                          12  Associates degree-occup /vocational   \n",
       "2                           0                             Children   \n",
       "3                           3                 High school graduate   \n",
       "4                          34                 High school graduate   \n",
       "\n",
       "   wage per hour enroll in edu inst last wk                   marital status  \\\n",
       "0              0            Not in universe  Married-civilian spouse present   \n",
       "1              0            Not in universe  Married-civilian spouse present   \n",
       "2              0            Not in universe                    Never married   \n",
       "3              0            Not in universe                         Divorced   \n",
       "4              0            Not in universe                         Divorced   \n",
       "\n",
       "            major industry code                  major occupation code  ...  \\\n",
       "0   Manufacturing-durable goods  Machine operators assmblrs & inspctrs  ...   \n",
       "1  Business and repair services                 Professional specialty  ...   \n",
       "2   Not in universe or children                        Not in universe  ...   \n",
       "3                Transportation         Executive admin and managerial  ...   \n",
       "4                  Construction    Precision production craft & repair  ...   \n",
       "\n",
       "  country of birth father country of birth mother country of birth self  \\\n",
       "0                  Mexico                  Mexico                Mexico   \n",
       "1           United-States           United-States         United-States   \n",
       "2           United-States           United-States         United-States   \n",
       "3           United-States           United-States         United-States   \n",
       "4           United-States           United-States         United-States   \n",
       "\n",
       "                           citizenship own business or self employed  \\\n",
       "0  Foreign born- Not a citizen of U S                              0   \n",
       "1    Native- Born in the United States                             0   \n",
       "2    Native- Born in the United States                             0   \n",
       "3    Native- Born in the United States                             2   \n",
       "4    Native- Born in the United States                             0   \n",
       "\n",
       "  fill inc questionnaire for veteran's admin  veterans benefits  \\\n",
       "0                            Not in universe                  2   \n",
       "1                            Not in universe                  2   \n",
       "2                            Not in universe                  0   \n",
       "3                            Not in universe                  2   \n",
       "4                            Not in universe                  2   \n",
       "\n",
       "   weeks worked in year  year income class  \n",
       "0                    12    95     - 50000.  \n",
       "1                    26    95     - 50000.  \n",
       "2                     0    95     - 50000.  \n",
       "3                    52    94     - 50000.  \n",
       "4                    50    95     - 50000.  \n",
       "\n",
       "[5 rows x 42 columns]"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "census_test.columns = census_columns\n",
    "census_test.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true
   },
   "source": [
    "### Clean Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-03-15T10:13:33.706377Z",
     "start_time": "2020-03-15T10:13:33.058453Z"
    },
    "hidden": true,
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>age</th>\n",
       "      <th>class of worker</th>\n",
       "      <th>detailed industry recode</th>\n",
       "      <th>detailed occupation recode</th>\n",
       "      <th>education</th>\n",
       "      <th>wage per hour</th>\n",
       "      <th>enroll in edu inst last wk</th>\n",
       "      <th>marital status</th>\n",
       "      <th>major industry code</th>\n",
       "      <th>major occupation code</th>\n",
       "      <th>...</th>\n",
       "      <th>country of birth father</th>\n",
       "      <th>country of birth mother</th>\n",
       "      <th>country of birth self</th>\n",
       "      <th>citizenship</th>\n",
       "      <th>own business or self employed</th>\n",
       "      <th>fill inc questionnaire for veteran's admin</th>\n",
       "      <th>veterans benefits</th>\n",
       "      <th>weeks worked in year</th>\n",
       "      <th>year</th>\n",
       "      <th>income class</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>38</td>\n",
       "      <td>Private</td>\n",
       "      <td>6</td>\n",
       "      <td>36</td>\n",
       "      <td>1st 2nd 3rd or 4th grade</td>\n",
       "      <td>0</td>\n",
       "      <td>Not in universe</td>\n",
       "      <td>Married-civilian spouse present</td>\n",
       "      <td>Manufacturing-durable goods</td>\n",
       "      <td>Machine operators assmblrs &amp; inspctrs</td>\n",
       "      <td>...</td>\n",
       "      <td>Mexico</td>\n",
       "      <td>Mexico</td>\n",
       "      <td>Mexico</td>\n",
       "      <td>Foreign born- Not a citizen of U S</td>\n",
       "      <td>0</td>\n",
       "      <td>Not in universe</td>\n",
       "      <td>2</td>\n",
       "      <td>12</td>\n",
       "      <td>95</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>44</td>\n",
       "      <td>Self-employed-not incorporated</td>\n",
       "      <td>37</td>\n",
       "      <td>12</td>\n",
       "      <td>Associates degree-occup /vocational</td>\n",
       "      <td>0</td>\n",
       "      <td>Not in universe</td>\n",
       "      <td>Married-civilian spouse present</td>\n",
       "      <td>Business and repair services</td>\n",
       "      <td>Professional specialty</td>\n",
       "      <td>...</td>\n",
       "      <td>United-States</td>\n",
       "      <td>United-States</td>\n",
       "      <td>United-States</td>\n",
       "      <td>Native- Born in the United States</td>\n",
       "      <td>0</td>\n",
       "      <td>Not in universe</td>\n",
       "      <td>2</td>\n",
       "      <td>26</td>\n",
       "      <td>95</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>Not in universe</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>Children</td>\n",
       "      <td>0</td>\n",
       "      <td>Not in universe</td>\n",
       "      <td>Never married</td>\n",
       "      <td>Not in universe or children</td>\n",
       "      <td>Not in universe</td>\n",
       "      <td>...</td>\n",
       "      <td>United-States</td>\n",
       "      <td>United-States</td>\n",
       "      <td>United-States</td>\n",
       "      <td>Native- Born in the United States</td>\n",
       "      <td>0</td>\n",
       "      <td>Not in universe</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>95</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>35</td>\n",
       "      <td>Private</td>\n",
       "      <td>29</td>\n",
       "      <td>3</td>\n",
       "      <td>High school graduate</td>\n",
       "      <td>0</td>\n",
       "      <td>Not in universe</td>\n",
       "      <td>Divorced</td>\n",
       "      <td>Transportation</td>\n",
       "      <td>Executive admin and managerial</td>\n",
       "      <td>...</td>\n",
       "      <td>United-States</td>\n",
       "      <td>United-States</td>\n",
       "      <td>United-States</td>\n",
       "      <td>Native- Born in the United States</td>\n",
       "      <td>2</td>\n",
       "      <td>Not in universe</td>\n",
       "      <td>2</td>\n",
       "      <td>52</td>\n",
       "      <td>94</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>49</td>\n",
       "      <td>Private</td>\n",
       "      <td>4</td>\n",
       "      <td>34</td>\n",
       "      <td>High school graduate</td>\n",
       "      <td>0</td>\n",
       "      <td>Not in universe</td>\n",
       "      <td>Divorced</td>\n",
       "      <td>Construction</td>\n",
       "      <td>Precision production craft &amp; repair</td>\n",
       "      <td>...</td>\n",
       "      <td>United-States</td>\n",
       "      <td>United-States</td>\n",
       "      <td>United-States</td>\n",
       "      <td>Native- Born in the United States</td>\n",
       "      <td>0</td>\n",
       "      <td>Not in universe</td>\n",
       "      <td>2</td>\n",
       "      <td>50</td>\n",
       "      <td>95</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 41 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   age                 class of worker detailed industry recode  \\\n",
       "0   38                         Private                        6   \n",
       "1   44  Self-employed-not incorporated                       37   \n",
       "2    2                 Not in universe                        0   \n",
       "3   35                         Private                       29   \n",
       "4   49                         Private                        4   \n",
       "\n",
       "  detailed occupation recode                            education  \\\n",
       "0                         36             1st 2nd 3rd or 4th grade   \n",
       "1                         12  Associates degree-occup /vocational   \n",
       "2                          0                             Children   \n",
       "3                          3                 High school graduate   \n",
       "4                         34                 High school graduate   \n",
       "\n",
       "   wage per hour enroll in edu inst last wk                   marital status  \\\n",
       "0              0            Not in universe  Married-civilian spouse present   \n",
       "1              0            Not in universe  Married-civilian spouse present   \n",
       "2              0            Not in universe                    Never married   \n",
       "3              0            Not in universe                         Divorced   \n",
       "4              0            Not in universe                         Divorced   \n",
       "\n",
       "            major industry code                  major occupation code  ...  \\\n",
       "0   Manufacturing-durable goods  Machine operators assmblrs & inspctrs  ...   \n",
       "1  Business and repair services                 Professional specialty  ...   \n",
       "2   Not in universe or children                        Not in universe  ...   \n",
       "3                Transportation         Executive admin and managerial  ...   \n",
       "4                  Construction    Precision production craft & repair  ...   \n",
       "\n",
       "  country of birth father country of birth mother country of birth self  \\\n",
       "0                  Mexico                  Mexico                Mexico   \n",
       "1           United-States           United-States         United-States   \n",
       "2           United-States           United-States         United-States   \n",
       "3           United-States           United-States         United-States   \n",
       "4           United-States           United-States         United-States   \n",
       "\n",
       "                           citizenship own business or self employed  \\\n",
       "0  Foreign born- Not a citizen of U S                              0   \n",
       "1    Native- Born in the United States                             0   \n",
       "2    Native- Born in the United States                             0   \n",
       "3    Native- Born in the United States                             2   \n",
       "4    Native- Born in the United States                             0   \n",
       "\n",
       "  fill inc questionnaire for veteran's admin  veterans benefits  \\\n",
       "0                            Not in universe                  2   \n",
       "1                            Not in universe                  2   \n",
       "2                            Not in universe                  0   \n",
       "3                            Not in universe                  2   \n",
       "4                            Not in universe                  2   \n",
       "\n",
       "   weeks worked in year  year income class  \n",
       "0                    12    95            0  \n",
       "1                    26    95            0  \n",
       "2                     0    95            0  \n",
       "3                    52    94            0  \n",
       "4                    50    95            0  \n",
       "\n",
       "[5 rows x 41 columns]"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "census_test = clean_dataset(census_test)\n",
    "census_test.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-03-15T10:13:34.402147Z",
     "start_time": "2020-03-15T10:13:34.233145Z"
    },
    "hidden": true,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 99762 entries, 0 to 99761\n",
      "Data columns (total 41 columns):\n",
      " #   Column                                      Non-Null Count  Dtype \n",
      "---  ------                                      --------------  ----- \n",
      " 0   age                                         99762 non-null  int64 \n",
      " 1   class of worker                             99762 non-null  object\n",
      " 2   detailed industry recode                    99762 non-null  object\n",
      " 3   detailed occupation recode                  99762 non-null  object\n",
      " 4   education                                   99762 non-null  object\n",
      " 5   wage per hour                               99762 non-null  int64 \n",
      " 6   enroll in edu inst last wk                  99762 non-null  object\n",
      " 7   marital status                              99762 non-null  object\n",
      " 8   major industry code                         99762 non-null  object\n",
      " 9   major occupation code                       99762 non-null  object\n",
      " 10  race                                        99762 non-null  object\n",
      " 11  hispanic origin                             99762 non-null  object\n",
      " 12  sex                                         99762 non-null  object\n",
      " 13  member of a labor union                     99762 non-null  object\n",
      " 14  reason for unemployment                     99762 non-null  object\n",
      " 15  full or part time employment stat           99762 non-null  object\n",
      " 16  capital gains                               99762 non-null  int64 \n",
      " 17  capital losses                              99762 non-null  int64 \n",
      " 18  dividends from stocks                       99762 non-null  int64 \n",
      " 19  tax filer stat                              99762 non-null  object\n",
      " 20  region of previous residence                99762 non-null  object\n",
      " 21  state of previous residence                 99762 non-null  object\n",
      " 22  detailed household and family stat          99762 non-null  object\n",
      " 23  detailed household summary in household     99762 non-null  object\n",
      " 24  migration code-change in msa                99762 non-null  object\n",
      " 25  migration code-change in reg                99762 non-null  object\n",
      " 26  migration code-move within reg              99762 non-null  object\n",
      " 27  live in this house 1 year ago               99762 non-null  object\n",
      " 28  migration prev res in sunbelt               99762 non-null  object\n",
      " 29  num persons worked for employer             99762 non-null  int64 \n",
      " 30  family members under 18                     99762 non-null  object\n",
      " 31  country of birth father                     99762 non-null  object\n",
      " 32  country of birth mother                     99762 non-null  object\n",
      " 33  country of birth self                       99762 non-null  object\n",
      " 34  citizenship                                 99762 non-null  object\n",
      " 35  own business or self employed               99762 non-null  object\n",
      " 36  fill inc questionnaire for veteran's admin  99762 non-null  object\n",
      " 37  veterans benefits                           99762 non-null  object\n",
      " 38  weeks worked in year                        99762 non-null  int64 \n",
      " 39  year                                        99762 non-null  object\n",
      " 40  income class                                99762 non-null  int64 \n",
      "dtypes: int64(8), object(33)\n",
      "memory usage: 31.2+ MB\n"
     ]
    }
   ],
   "source": [
    "census_test.info()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Predictions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-03-15T10:13:34.927564Z",
     "start_time": "2020-03-15T10:13:34.882863Z"
    }
   },
   "outputs": [],
   "source": [
    "X_test = census_test.drop('income class', axis=1)\n",
    "y_test = census_test['income class']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### SelectKBest"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-03-15T12:20:58.130831Z",
     "start_time": "2020-03-15T12:20:56.341582Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.99      0.88      0.93     93576\n",
      "           1       0.32      0.89      0.47      6186\n",
      "\n",
      "    accuracy                           0.88     99762\n",
      "   macro avg       0.66      0.88      0.70     99762\n",
      "weighted avg       0.95      0.88      0.90     99762\n",
      "\n",
      "AUC: 0.8833547861468454\n"
     ]
    }
   ],
   "source": [
    "y_pred_test = selectKBest_model.predict(X_test)\n",
    "\n",
    "print(classification_report(y_test, y_pred_test))\n",
    "fpr, tpr, thresholds = roc_curve(y_test, y_pred_test)\n",
    "\n",
    "print(\"AUC: %s\" % auc(fpr, tpr))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-03-15T12:20:58.855856Z",
     "start_time": "2020-03-15T12:20:58.688304Z"
    }
   },
   "outputs": [],
   "source": [
    "submission = pd.DataFrame(y_pred_test).astype(float)\n",
    "submission.columns = ['income class']\n",
    "submission.reset_index(inplace=True)\n",
    "submission.to_csv('temp/xgBoost with SelectKBest.csv', index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### PCA"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-03-15T12:21:01.560623Z",
     "start_time": "2020-03-15T12:20:59.473205Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.99      0.89      0.94     93576\n",
      "           1       0.33      0.86      0.48      6186\n",
      "\n",
      "    accuracy                           0.88     99762\n",
      "   macro avg       0.66      0.87      0.71     99762\n",
      "weighted avg       0.95      0.88      0.91     99762\n",
      "\n",
      "AUC: 0.8732813235539102\n"
     ]
    }
   ],
   "source": [
    "y_pred_test = pca_model.predict(X_test)\n",
    "\n",
    "print(classification_report(y_test, y_pred_test))\n",
    "fpr, tpr, thresholds = roc_curve(y_test, y_pred_test)\n",
    "\n",
    "print(\"AUC: %s\" % auc(fpr, tpr))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-03-15T12:21:02.300990Z",
     "start_time": "2020-03-15T12:21:02.133436Z"
    }
   },
   "outputs": [],
   "source": [
    "submission = pd.DataFrame(y_pred_test).astype(float)\n",
    "submission.columns = ['income class']\n",
    "submission.reset_index(inplace=True)\n",
    "submission.to_csv('temp/xgBoost with PCA.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": true
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}