024 lesson Sklearn Part2/23 Lesson Sklearn Details.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Sklearn"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Machine Learning in Python"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "* Simple and efficient tools for predictive data analysis\n",
    "* Accessible to everybody, and reusable in various contexts\n",
    "* Built on NumPy, SciPy, and matplotlib\n",
    "* Open source, commercially usable - BSD license"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# An end-to-end Scikit-Learn worfklow Detailed"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Getting the data ready (Data Preparation)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "* 1 Split the data into features\n",
    "* 2 Filling (imputing the missing value)\n",
    "* 3 Converting non numerical  to numerical"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Standard imports\n",
    "%matplotlib inline\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "import pandas as pd\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>age</th>\n",
       "      <th>sex</th>\n",
       "      <th>cp</th>\n",
       "      <th>trestbps</th>\n",
       "      <th>chol</th>\n",
       "      <th>fbs</th>\n",
       "      <th>restecg</th>\n",
       "      <th>thalach</th>\n",
       "      <th>exang</th>\n",
       "      <th>oldpeak</th>\n",
       "      <th>slope</th>\n",
       "      <th>ca</th>\n",
       "      <th>thal</th>\n",
       "      <th>target</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>63</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>145</td>\n",
       "      <td>233</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>150</td>\n",
       "      <td>0</td>\n",
       "      <td>2.3</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>37</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>130</td>\n",
       "      <td>250</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>187</td>\n",
       "      <td>0</td>\n",
       "      <td>3.5</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>41</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>130</td>\n",
       "      <td>204</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>172</td>\n",
       "      <td>0</td>\n",
       "      <td>1.4</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>56</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>120</td>\n",
       "      <td>236</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>178</td>\n",
       "      <td>0</td>\n",
       "      <td>0.8</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>57</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>120</td>\n",
       "      <td>354</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>163</td>\n",
       "      <td>1</td>\n",
       "      <td>0.6</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   age  sex  cp  trestbps  chol  fbs  restecg  thalach  exang  oldpeak  slope  \\\n",
       "0   63    1   3       145   233    1        0      150      0      2.3      0   \n",
       "1   37    1   2       130   250    0        1      187      0      3.5      0   \n",
       "2   41    0   1       130   204    0        0      172      0      1.4      2   \n",
       "3   56    1   1       120   236    0        1      178      0      0.8      2   \n",
       "4   57    0   0       120   354    0        1      163      1      0.6      2   \n",
       "\n",
       "   ca  thal  target  \n",
       "0   0     1       1  \n",
       "1   0     2       1  \n",
       "2   0     2       1  \n",
       "3   0     2       1  \n",
       "4   0     2       1  "
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# import data \n",
    "heart_disease = pd.read_csv('data/heart-disease.csv')\n",
    "heart_disease.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 1 Splitting the data into features (usually X) and labels (usually y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>age</th>\n",
       "      <th>sex</th>\n",
       "      <th>cp</th>\n",
       "      <th>trestbps</th>\n",
       "      <th>chol</th>\n",
       "      <th>fbs</th>\n",
       "      <th>restecg</th>\n",
       "      <th>thalach</th>\n",
       "      <th>exang</th>\n",
       "      <th>oldpeak</th>\n",
       "      <th>slope</th>\n",
       "      <th>ca</th>\n",
       "      <th>thal</th>\n",
       "      <th>target</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>63</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>145</td>\n",
       "      <td>233</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>150</td>\n",
       "      <td>0</td>\n",
       "      <td>2.3</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>37</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>130</td>\n",
       "      <td>250</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>187</td>\n",
       "      <td>0</td>\n",
       "      <td>3.5</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>41</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>130</td>\n",
       "      <td>204</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>172</td>\n",
       "      <td>0</td>\n",
       "      <td>1.4</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>56</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>120</td>\n",
       "      <td>236</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>178</td>\n",
       "      <td>0</td>\n",
       "      <td>0.8</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>57</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>120</td>\n",
       "      <td>354</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>163</td>\n",
       "      <td>1</td>\n",
       "      <td>0.6</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   age  sex  cp  trestbps  chol  fbs  restecg  thalach  exang  oldpeak  slope  \\\n",
       "0   63    1   3       145   233    1        0      150      0      2.3      0   \n",
       "1   37    1   2       130   250    0        1      187      0      3.5      0   \n",
       "2   41    0   1       130   204    0        0      172      0      1.4      2   \n",
       "3   56    1   1       120   236    0        1      178      0      0.8      2   \n",
       "4   57    0   0       120   354    0        1      163      1      0.6      2   \n",
       "\n",
       "   ca  thal  target  \n",
       "0   0     1       1  \n",
       "1   0     2       1  \n",
       "2   0     2       1  \n",
       "3   0     2       1  \n",
       "4   0     2       1  "
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "heart_disease.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>age</th>\n",
       "      <th>sex</th>\n",
       "      <th>cp</th>\n",
       "      <th>trestbps</th>\n",
       "      <th>chol</th>\n",
       "      <th>fbs</th>\n",
       "      <th>restecg</th>\n",
       "      <th>thalach</th>\n",
       "      <th>exang</th>\n",
       "      <th>oldpeak</th>\n",
       "      <th>slope</th>\n",
       "      <th>ca</th>\n",
       "      <th>thal</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>63</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>145</td>\n",
       "      <td>233</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>150</td>\n",
       "      <td>0</td>\n",
       "      <td>2.3</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>37</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>130</td>\n",
       "      <td>250</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>187</td>\n",
       "      <td>0</td>\n",
       "      <td>3.5</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   age  sex  cp  trestbps  chol  fbs  restecg  thalach  exang  oldpeak  slope  \\\n",
       "0   63    1   3       145   233    1        0      150      0      2.3      0   \n",
       "1   37    1   2       130   250    0        1      187      0      3.5      0   \n",
       "\n",
       "   ca  thal  \n",
       "0   0     1  \n",
       "1   0     2  "
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X = heart_disease.drop('target', axis=1)\n",
    "X.head(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    1\n",
       "1    1\n",
       "Name: target, dtype: int64"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y = heart_disease['target']\n",
    "y.head(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((242, 13), (61, 13), (242,), (61,))"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Splitting the data into training and test sets\n",
    "from sklearn.model_selection import train_test_split\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42) # you can change the test size\n",
    "\n",
    "X_train.shape, X_test.shape, y_train.shape, y_test.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2 Make sure it's all numerical"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Make</th>\n",
       "      <th>Colour</th>\n",
       "      <th>Odometer (KM)</th>\n",
       "      <th>Doors</th>\n",
       "      <th>Price</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Honda</td>\n",
       "      <td>White</td>\n",
       "      <td>35431</td>\n",
       "      <td>4</td>\n",
       "      <td>15323</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>BMW</td>\n",
       "      <td>Blue</td>\n",
       "      <td>192714</td>\n",
       "      <td>5</td>\n",
       "      <td>19943</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Honda</td>\n",
       "      <td>White</td>\n",
       "      <td>84714</td>\n",
       "      <td>4</td>\n",
       "      <td>28343</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Toyota</td>\n",
       "      <td>White</td>\n",
       "      <td>154365</td>\n",
       "      <td>4</td>\n",
       "      <td>13434</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Nissan</td>\n",
       "      <td>Blue</td>\n",
       "      <td>181577</td>\n",
       "      <td>3</td>\n",
       "      <td>14043</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     Make Colour  Odometer (KM)  Doors  Price\n",
       "0   Honda  White          35431      4  15323\n",
       "1     BMW   Blue         192714      5  19943\n",
       "2   Honda  White          84714      4  28343\n",
       "3  Toyota  White         154365      4  13434\n",
       "4  Nissan   Blue         181577      3  14043"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Import car-sales-extended.csv\n",
    "car_sales = pd.read_csv(\"data/car-sales-extended.csv\")\n",
    "car_sales.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 1000 entries, 0 to 999\n",
      "Data columns (total 5 columns):\n",
      " #   Column         Non-Null Count  Dtype \n",
      "---  ------         --------------  ----- \n",
      " 0   Make           1000 non-null   object\n",
      " 1   Colour         1000 non-null   object\n",
      " 2   Odometer (KM)  1000 non-null   int64 \n",
      " 3   Doors          1000 non-null   int64 \n",
      " 4   Price          1000 non-null   int64 \n",
      "dtypes: int64(3), object(2)\n",
      "memory usage: 39.2+ KB\n"
     ]
    }
   ],
   "source": [
    "car_sales.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Split into X & y and train/test\n",
    "X = car_sales.drop(\"Price\", axis=1)\n",
    "y = car_sales[\"Price\"]\n",
    "\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "ename": "ValueError",
     "evalue": "could not convert string to float: 'Honda'",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mValueError\u001b[0m                                Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-16-ecb56ad8f06d>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[0;32m      3\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      4\u001b[0m \u001b[0mmodel\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mRandomForestRegressor\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 5\u001b[1;33m \u001b[0mmodel\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mfit\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mX_train\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0my_train\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m      6\u001b[0m \u001b[0mmodel\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mscore\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mX_test\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0my_test\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\anaconda3\\lib\\site-packages\\sklearn\\ensemble\\_forest.py\u001b[0m in \u001b[0;36mfit\u001b[1;34m(self, X, y, sample_weight)\u001b[0m\n\u001b[0;32m    301\u001b[0m                 \u001b[1;34m\"sparse multilabel-indicator for y is not supported.\"\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    302\u001b[0m             )\n\u001b[1;32m--> 303\u001b[1;33m         X, y = self._validate_data(X, y, multi_output=True,\n\u001b[0m\u001b[0;32m    304\u001b[0m                                    accept_sparse=\"csc\", dtype=DTYPE)\n\u001b[0;32m    305\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[0msample_weight\u001b[0m \u001b[1;32mis\u001b[0m \u001b[1;32mnot\u001b[0m \u001b[1;32mNone\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\anaconda3\\lib\\site-packages\\sklearn\\base.py\u001b[0m in \u001b[0;36m_validate_data\u001b[1;34m(self, X, y, reset, validate_separately, **check_params)\u001b[0m\n\u001b[0;32m    430\u001b[0m                 \u001b[0my\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mcheck_array\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0my\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mcheck_y_params\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    431\u001b[0m             \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 432\u001b[1;33m                 \u001b[0mX\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0my\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mcheck_X_y\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mX\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0my\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mcheck_params\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    433\u001b[0m             \u001b[0mout\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mX\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0my\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    434\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\anaconda3\\lib\\site-packages\\sklearn\\utils\\validation.py\u001b[0m in \u001b[0;36minner_f\u001b[1;34m(*args, **kwargs)\u001b[0m\n\u001b[0;32m     70\u001b[0m                           FutureWarning)\n\u001b[0;32m     71\u001b[0m         \u001b[0mkwargs\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mupdate\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m{\u001b[0m\u001b[0mk\u001b[0m\u001b[1;33m:\u001b[0m \u001b[0marg\u001b[0m \u001b[1;32mfor\u001b[0m \u001b[0mk\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0marg\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mzip\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0msig\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mparameters\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0margs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m}\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 72\u001b[1;33m         \u001b[1;32mreturn\u001b[0m \u001b[0mf\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     73\u001b[0m     \u001b[1;32mreturn\u001b[0m \u001b[0minner_f\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     74\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\anaconda3\\lib\\site-packages\\sklearn\\utils\\validation.py\u001b[0m in \u001b[0;36mcheck_X_y\u001b[1;34m(X, y, accept_sparse, accept_large_sparse, dtype, order, copy, force_all_finite, ensure_2d, allow_nd, multi_output, ensure_min_samples, ensure_min_features, y_numeric, estimator)\u001b[0m\n\u001b[0;32m    793\u001b[0m         \u001b[1;32mraise\u001b[0m \u001b[0mValueError\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m\"y cannot be None\"\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    794\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 795\u001b[1;33m     X = check_array(X, accept_sparse=accept_sparse,\n\u001b[0m\u001b[0;32m    796\u001b[0m                     \u001b[0maccept_large_sparse\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0maccept_large_sparse\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    797\u001b[0m                     \u001b[0mdtype\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mdtype\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0morder\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0morder\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mcopy\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mcopy\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\anaconda3\\lib\\site-packages\\sklearn\\utils\\validation.py\u001b[0m in \u001b[0;36minner_f\u001b[1;34m(*args, **kwargs)\u001b[0m\n\u001b[0;32m     70\u001b[0m                           FutureWarning)\n\u001b[0;32m     71\u001b[0m         \u001b[0mkwargs\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mupdate\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m{\u001b[0m\u001b[0mk\u001b[0m\u001b[1;33m:\u001b[0m \u001b[0marg\u001b[0m \u001b[1;32mfor\u001b[0m \u001b[0mk\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0marg\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mzip\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0msig\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mparameters\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0margs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m}\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 72\u001b[1;33m         \u001b[1;32mreturn\u001b[0m \u001b[0mf\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     73\u001b[0m     \u001b[1;32mreturn\u001b[0m \u001b[0minner_f\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     74\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\anaconda3\\lib\\site-packages\\sklearn\\utils\\validation.py\u001b[0m in \u001b[0;36mcheck_array\u001b[1;34m(array, accept_sparse, accept_large_sparse, dtype, order, copy, force_all_finite, ensure_2d, allow_nd, ensure_min_samples, ensure_min_features, estimator)\u001b[0m\n\u001b[0;32m    596\u001b[0m                     \u001b[0marray\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0marray\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mastype\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mdtype\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mcasting\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;34m\"unsafe\"\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mcopy\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;32mFalse\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    597\u001b[0m                 \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 598\u001b[1;33m                     \u001b[0marray\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mnp\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0masarray\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0marray\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0morder\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0morder\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mdtype\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mdtype\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    599\u001b[0m             \u001b[1;32mexcept\u001b[0m \u001b[0mComplexWarning\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    600\u001b[0m                 raise ValueError(\"Complex data not supported\\n\"\n",
      "\u001b[1;32m~\\anaconda3\\lib\\site-packages\\numpy\\core\\_asarray.py\u001b[0m in \u001b[0;36masarray\u001b[1;34m(a, dtype, order)\u001b[0m\n\u001b[0;32m     81\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     82\u001b[0m     \"\"\"\n\u001b[1;32m---> 83\u001b[1;33m     \u001b[1;32mreturn\u001b[0m \u001b[0marray\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0ma\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mdtype\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mcopy\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;32mFalse\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0morder\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0morder\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     84\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     85\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\anaconda3\\lib\\site-packages\\pandas\\core\\generic.py\u001b[0m in \u001b[0;36m__array__\u001b[1;34m(self, dtype)\u001b[0m\n\u001b[0;32m   1779\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1780\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0m__array__\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mdtype\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;32mNone\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;33m->\u001b[0m \u001b[0mnp\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mndarray\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 1781\u001b[1;33m         \u001b[1;32mreturn\u001b[0m \u001b[0mnp\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0masarray\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_values\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mdtype\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mdtype\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   1782\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1783\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0m__array_wrap__\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mresult\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mcontext\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;32mNone\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\anaconda3\\lib\\site-packages\\numpy\\core\\_asarray.py\u001b[0m in \u001b[0;36masarray\u001b[1;34m(a, dtype, order)\u001b[0m\n\u001b[0;32m     81\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     82\u001b[0m     \"\"\"\n\u001b[1;32m---> 83\u001b[1;33m     \u001b[1;32mreturn\u001b[0m \u001b[0marray\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0ma\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mdtype\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mcopy\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;32mFalse\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0morder\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0morder\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     84\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     85\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mValueError\u001b[0m: could not convert string to float: 'Honda'"
     ]
    }
   ],
   "source": [
    "# Try to predict with random forest on price column (doesn't work)\n",
    "from sklearn.ensemble import RandomForestRegressor\n",
    "\n",
    "model = RandomForestRegressor()\n",
    "model.fit(X_train, y_train)\n",
    "model.score(X_test, y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "4    856\n",
       "5     79\n",
       "3     65\n",
       "Name: Doors, dtype: int64"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "car_sales['Doors'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0.00000e+00, 1.00000e+00, 0.00000e+00, ..., 1.00000e+00,\n",
       "        0.00000e+00, 3.54310e+04],\n",
       "       [1.00000e+00, 0.00000e+00, 0.00000e+00, ..., 0.00000e+00,\n",
       "        1.00000e+00, 1.92714e+05],\n",
       "       [0.00000e+00, 1.00000e+00, 0.00000e+00, ..., 1.00000e+00,\n",
       "        0.00000e+00, 8.47140e+04],\n",
       "       ...,\n",
       "       [0.00000e+00, 0.00000e+00, 1.00000e+00, ..., 1.00000e+00,\n",
       "        0.00000e+00, 6.66040e+04],\n",
       "       [0.00000e+00, 1.00000e+00, 0.00000e+00, ..., 1.00000e+00,\n",
       "        0.00000e+00, 2.15883e+05],\n",
       "       [0.00000e+00, 0.00000e+00, 0.00000e+00, ..., 1.00000e+00,\n",
       "        0.00000e+00, 2.48360e+05]])"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Turn the categories (Make and Colour) into numbers\n",
    "from sklearn.preprocessing import OneHotEncoder\n",
    "from sklearn.compose import ColumnTransformer\n",
    "\n",
    "categorical_features = [\"Make\", \"Colour\", \"Doors\"]\n",
    "one_hot = OneHotEncoder()\n",
    "transformer = ColumnTransformer([(\"one_hot\", \n",
    "                                 one_hot, \n",
    "                                 categorical_features)],\n",
    "                                 remainder=\"passthrough\")\n",
    "transformed_X = transformer.fit_transform(X)\n",
    "transformed_X"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Another way... using pandas and pd.get_dummies()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Make_BMW</th>\n",
       "      <th>Make_Honda</th>\n",
       "      <th>Make_Nissan</th>\n",
       "      <th>Make_Toyota</th>\n",
       "      <th>Colour_Black</th>\n",
       "      <th>Colour_Blue</th>\n",
       "      <th>Colour_Green</th>\n",
       "      <th>Colour_Red</th>\n",
       "      <th>Colour_White</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Make_BMW  Make_Honda  Make_Nissan  Make_Toyota  Colour_Black  Colour_Blue  \\\n",
       "0         0           1            0            0             0            0   \n",
       "1         1           0            0            0             0            1   \n",
       "2         0           1            0            0             0            0   \n",
       "3         0           0            0            1             0            0   \n",
       "4         0           0            1            0             0            1   \n",
       "\n",
       "   Colour_Green  Colour_Red  Colour_White  \n",
       "0             0           0             1  \n",
       "1             0           0             0  \n",
       "2             0           0             1  \n",
       "3             0           0             1  \n",
       "4             0           0             0  "
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dummies = pd.get_dummies(car_sales[[\"Make\", \"Colour\"]])\n",
    "dummies.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "RandomForestRegressor()"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\n",
    "# Let's refit the model\n",
    "np.random.seed(42)\n",
    "X_train, X_test, y_train, y_test = train_test_split(transformed_X,\n",
    "                                                    y,\n",
    "                                                    test_size=0.2)\n",
    "\n",
    "model.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.3235867221569877"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.score(X_test, y_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 3 What if there were missing values?"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "* 1)Fill them  with values \n",
    "* 2)Remove the missing values "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Make</th>\n",
       "      <th>Colour</th>\n",
       "      <th>Odometer (KM)</th>\n",
       "      <th>Doors</th>\n",
       "      <th>Price</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Honda</td>\n",
       "      <td>White</td>\n",
       "      <td>35431.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>15323.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>BMW</td>\n",
       "      <td>Blue</td>\n",
       "      <td>192714.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>19943.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Honda</td>\n",
       "      <td>White</td>\n",
       "      <td>84714.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>28343.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Toyota</td>\n",
       "      <td>White</td>\n",
       "      <td>154365.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>13434.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     Make Colour  Odometer (KM)  Doors    Price\n",
       "0   Honda  White        35431.0    4.0  15323.0\n",
       "1     BMW   Blue       192714.0    5.0  19943.0\n",
       "2   Honda  White        84714.0    4.0  28343.0\n",
       "3  Toyota  White       154365.0    4.0  13434.0"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Import car sales dataframe with missing values\n",
    "car_sales_missing = pd.read_csv(\"data/car-sales-extended-missing-data.csv\")\n",
    "car_sales_missing.head(4)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "car_sales_missing.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Make             49\n",
       "Colour           50\n",
       "Odometer (KM)    50\n",
       "Doors            50\n",
       "Price            50\n",
       "dtype: int64"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "car_sales_missing.isna().sum()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Fill missing data with pandas"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "* What we'll do is fill the rows where categorical values are missing with \"missing\", the numerical features with the mean or 4 for the doors. And drop the rows where the Price is missing.\n",
    "\n",
    "* We could fill Price with the mean, however, since it's the target variable, we don't want to be introducing too many fake labels.\n",
    "\n",
    "* Note: The practice of filling missing data is called imputation. And it's important to remember there's no perfect way to fill missing data. The methods we're using are only one of many. The techniques you use will depend heavily on your dataset. A good place to look would be searching for \"data imputation techniques\"."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Fill the \"Make\" column\n",
    "car_sales_missing[\"Make\"].fillna(\"missing\", inplace=True)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Fill the \"Colour\" column\n",
    "car_sales_missing[\"Colour\"].fillna(\"missing\", inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Fill the \"Odometer (KM)\" column\n",
    "car_sales_missing[\"Odometer (KM)\"].fillna(car_sales_missing[\"Odometer (KM)\"].mean(), inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Fill the \"Doors\" column\n",
    "car_sales_missing[\"Doors\"].fillna(4, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Make              0\n",
       "Colour            0\n",
       "Odometer (KM)     0\n",
       "Doors             0\n",
       "Price            50\n",
       "dtype: int64"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Check our dataframe\n",
    "car_sales_missing.isna().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Remove rows with missing Price labels\n",
    "car_sales_missing.dropna(inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Make             0\n",
       "Colour           0\n",
       "Odometer (KM)    0\n",
       "Doors            0\n",
       "Price            0\n",
       "dtype: int64"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "car_sales_missing.isna().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "950"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(car_sales_missing)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0.00000e+00, 1.00000e+00, 0.00000e+00, ..., 0.00000e+00,\n",
       "        3.54310e+04, 1.53230e+04],\n",
       "       [1.00000e+00, 0.00000e+00, 0.00000e+00, ..., 1.00000e+00,\n",
       "        1.92714e+05, 1.99430e+04],\n",
       "       [0.00000e+00, 1.00000e+00, 0.00000e+00, ..., 0.00000e+00,\n",
       "        8.47140e+04, 2.83430e+04],\n",
       "       ...,\n",
       "       [0.00000e+00, 0.00000e+00, 1.00000e+00, ..., 0.00000e+00,\n",
       "        6.66040e+04, 3.15700e+04],\n",
       "       [0.00000e+00, 1.00000e+00, 0.00000e+00, ..., 0.00000e+00,\n",
       "        2.15883e+05, 4.00100e+03],\n",
       "       [0.00000e+00, 0.00000e+00, 0.00000e+00, ..., 0.00000e+00,\n",
       "        2.48360e+05, 1.27320e+04]])"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Now let's one-hot encode the categorical columns (copied from above)\n",
    "from sklearn.preprocessing import OneHotEncoder\n",
    "from sklearn.compose import ColumnTransformer\n",
    "\n",
    "categorical_features = [\"Make\", \"Colour\", \"Doors\"]\n",
    "one_hot = OneHotEncoder()\n",
    "transformer = ColumnTransformer([(\"one_hot\", \n",
    "                                 one_hot, \n",
    "                                 categorical_features)],\n",
    "                                 remainder=\"passthrough\")\n",
    "transformed_X = transformer.fit_transform(car_sales_missing)\n",
    "transformed_X"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Filling missing data and transforming categorical data with Scikit-Learn"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "* scikit-learn provides another method called SimpleImputer() which allows us to do a similar thing.\n",
    "\n",
    "* SimpleImputer() transforms data by filling missing values with a given strategy."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Make             49\n",
       "Colour           50\n",
       "Odometer (KM)    50\n",
       "Doors            50\n",
       "Price            50\n",
       "dtype: int64"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Reimport the DataFrame\n",
    "car_sales_missing = pd.read_csv(\"data/car-sales-extended-missing-data.csv\")\n",
    "car_sales_missing.isna().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Drop the rows with missing in the \"Price\" column\n",
    "car_sales_missing.dropna(subset=[\"Price\"], inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Make             47\n",
       "Colour           46\n",
       "Odometer (KM)    48\n",
       "Doors            47\n",
       "Price             0\n",
       "dtype: int64"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "car_sales_missing.isna().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Split into X and y\n",
    "X = car_sales_missing.drop(\"Price\", axis=1)\n",
    "y = car_sales_missing[\"Price\"]\n",
    "\n",
    "# Split data into train and test\n",
    "np.random.seed(42)\n",
    "X_train, X_test, y_train, y_test = train_test_split(X,\n",
    "                                                    y,\n",
    "                                                    test_size=0.2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.impute import SimpleImputer\n",
    "from sklearn.compose import ColumnTransformer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Fill categorical values with 'missing' & numerical with mean\n",
    "cat_imputer = SimpleImputer(strategy=\"constant\", fill_value=\"missing\")\n",
    "door_imputer = SimpleImputer(strategy=\"constant\", fill_value=4)\n",
    "num_imputer = SimpleImputer(strategy=\"mean\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Define different column features\n",
    "categorical_features = [\"Make\", \"Colour\"]\n",
    "door_feature = [\"Doors\"]\n",
    "numerical_feature = [\"Odometer (KM)\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([['Honda', 'White', 4.0, 71934.0],\n",
       "       ['Toyota', 'Red', 4.0, 162665.0],\n",
       "       ['Honda', 'White', 4.0, 42844.0],\n",
       "       ...,\n",
       "       ['Toyota', 'White', 4.0, 196225.0],\n",
       "       ['Honda', 'Blue', 4.0, 133117.0],\n",
       "       ['Honda', 'missing', 4.0, 150582.0]], dtype=object)"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "imputer = ColumnTransformer([\n",
    "    (\"cat_imputer\", cat_imputer, categorical_features),\n",
    "    (\"door_imputer\", door_imputer, door_feature),\n",
    "    (\"num_imputer\", num_imputer, numerical_feature)])\n",
    "\n",
    "# Fill train and test values separately\n",
    "filled_X_train = imputer.fit_transform(X_train)\n",
    "filled_X_test = imputer.transform(X_test)\n",
    "\n",
    "# Check filled X_train\n",
    "filled_X_train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Make             0\n",
       "Colour           0\n",
       "Doors            0\n",
       "Odometer (KM)    0\n",
       "dtype: int64"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Get our transformed data array's back into DataFrame's\n",
    "car_sales_filled_train = pd.DataFrame(filled_X_train, \n",
    "                                      columns=[\"Make\", \"Colour\", \"Doors\", \"Odometer (KM)\"])\n",
    "\n",
    "car_sales_filled_test = pd.DataFrame(filled_X_test, \n",
    "                                      columns=[\"Make\", \"Colour\", \"Doors\", \"Odometer (KM)\"])\n",
    "\n",
    "# Check missing data in training set\n",
    "car_sales_filled_train.isna().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Make</th>\n",
       "      <th>Colour</th>\n",
       "      <th>Doors</th>\n",
       "      <th>Odometer (KM)</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Honda</td>\n",
       "      <td>White</td>\n",
       "      <td>4</td>\n",
       "      <td>71934</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Toyota</td>\n",
       "      <td>Red</td>\n",
       "      <td>4</td>\n",
       "      <td>162665</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Honda</td>\n",
       "      <td>White</td>\n",
       "      <td>4</td>\n",
       "      <td>42844</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Honda</td>\n",
       "      <td>White</td>\n",
       "      <td>4</td>\n",
       "      <td>195829</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Honda</td>\n",
       "      <td>Blue</td>\n",
       "      <td>4</td>\n",
       "      <td>219217</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     Make Colour Doors Odometer (KM)\n",
       "0   Honda  White     4         71934\n",
       "1  Toyota    Red     4        162665\n",
       "2   Honda  White     4         42844\n",
       "3   Honda  White     4        195829\n",
       "4   Honda   Blue     4        219217"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "car_sales_filled_train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0.00000e+00, 1.00000e+00, 0.00000e+00, ..., 1.00000e+00,\n",
       "        0.00000e+00, 7.19340e+04],\n",
       "       [0.00000e+00, 0.00000e+00, 0.00000e+00, ..., 1.00000e+00,\n",
       "        0.00000e+00, 1.62665e+05],\n",
       "       [0.00000e+00, 1.00000e+00, 0.00000e+00, ..., 1.00000e+00,\n",
       "        0.00000e+00, 4.28440e+04],\n",
       "       ...,\n",
       "       [0.00000e+00, 0.00000e+00, 0.00000e+00, ..., 1.00000e+00,\n",
       "        0.00000e+00, 1.96225e+05],\n",
       "       [0.00000e+00, 1.00000e+00, 0.00000e+00, ..., 1.00000e+00,\n",
       "        0.00000e+00, 1.33117e+05],\n",
       "       [0.00000e+00, 1.00000e+00, 0.00000e+00, ..., 1.00000e+00,\n",
       "        0.00000e+00, 1.50582e+05]])"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "categorical_features = [\"Make\", \"Colour\", \"Doors\"]\n",
    "one_hot = OneHotEncoder()\n",
    "transformer = ColumnTransformer([(\"one_hot\", \n",
    "                                 one_hot, \n",
    "                                 categorical_features)],\n",
    "                                 remainder=\"passthrough\")\n",
    "\n",
    "# Fill train and test values separately\n",
    "transformed_X_train = transformer.fit_transform(car_sales_filled_train)\n",
    "transformed_X_test = transformer.transform(car_sales_filled_test)\n",
    "\n",
    "# Check transformed and filled X_train\n",
    "transformed_X_train.toarray()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.21229043336119102"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Now we've transformed X, let's see if we can fit a model\n",
    "np.random.seed(42)\n",
    "from sklearn.ensemble import RandomForestRegressor\n",
    "\n",
    "model = RandomForestRegressor()\n",
    "\n",
    "# Make sure to use transformed (filled and one-hot encoded X data)\n",
    "model.fit(transformed_X_train, y_train)\n",
    "model.score(transformed_X_test, y_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Choosing the right estimator/algorithm for your problem"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "https://scikit-learn.org/stable/tutorial/machine_learning_map/index.html"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Picking a machine learning model for a regression problem¶"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "# Import the Boston housing dataset\n",
    "from sklearn.datasets import load_boston\n",
    "boston = load_boston()\n",
    "boston; # imports as dictionary"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>CRIM</th>\n",
       "      <th>ZN</th>\n",
       "      <th>INDUS</th>\n",
       "      <th>CHAS</th>\n",
       "      <th>NOX</th>\n",
       "      <th>RM</th>\n",
       "      <th>AGE</th>\n",
       "      <th>DIS</th>\n",
       "      <th>RAD</th>\n",
       "      <th>TAX</th>\n",
       "      <th>PTRATIO</th>\n",
       "      <th>B</th>\n",
       "      <th>LSTAT</th>\n",
       "      <th>target</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.00632</td>\n",
       "      <td>18.0</td>\n",
       "      <td>2.31</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.538</td>\n",
       "      <td>6.575</td>\n",
       "      <td>65.2</td>\n",
       "      <td>4.0900</td>\n",
       "      <td>1.0</td>\n",
       "      <td>296.0</td>\n",
       "      <td>15.3</td>\n",
       "      <td>396.90</td>\n",
       "      <td>4.98</td>\n",
       "      <td>24.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.02731</td>\n",
       "      <td>0.0</td>\n",
       "      <td>7.07</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.469</td>\n",
       "      <td>6.421</td>\n",
       "      <td>78.9</td>\n",
       "      <td>4.9671</td>\n",
       "      <td>2.0</td>\n",
       "      <td>242.0</td>\n",
       "      <td>17.8</td>\n",
       "      <td>396.90</td>\n",
       "      <td>9.14</td>\n",
       "      <td>21.6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.02729</td>\n",
       "      <td>0.0</td>\n",
       "      <td>7.07</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.469</td>\n",
       "      <td>7.185</td>\n",
       "      <td>61.1</td>\n",
       "      <td>4.9671</td>\n",
       "      <td>2.0</td>\n",
       "      <td>242.0</td>\n",
       "      <td>17.8</td>\n",
       "      <td>392.83</td>\n",
       "      <td>4.03</td>\n",
       "      <td>34.7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.03237</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.18</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.458</td>\n",
       "      <td>6.998</td>\n",
       "      <td>45.8</td>\n",
       "      <td>6.0622</td>\n",
       "      <td>3.0</td>\n",
       "      <td>222.0</td>\n",
       "      <td>18.7</td>\n",
       "      <td>394.63</td>\n",
       "      <td>2.94</td>\n",
       "      <td>33.4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.06905</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.18</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.458</td>\n",
       "      <td>7.147</td>\n",
       "      <td>54.2</td>\n",
       "      <td>6.0622</td>\n",
       "      <td>3.0</td>\n",
       "      <td>222.0</td>\n",
       "      <td>18.7</td>\n",
       "      <td>396.90</td>\n",
       "      <td>5.33</td>\n",
       "      <td>36.2</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      CRIM    ZN  INDUS  CHAS    NOX     RM   AGE     DIS  RAD    TAX  \\\n",
       "0  0.00632  18.0   2.31   0.0  0.538  6.575  65.2  4.0900  1.0  296.0   \n",
       "1  0.02731   0.0   7.07   0.0  0.469  6.421  78.9  4.9671  2.0  242.0   \n",
       "2  0.02729   0.0   7.07   0.0  0.469  7.185  61.1  4.9671  2.0  242.0   \n",
       "3  0.03237   0.0   2.18   0.0  0.458  6.998  45.8  6.0622  3.0  222.0   \n",
       "4  0.06905   0.0   2.18   0.0  0.458  7.147  54.2  6.0622  3.0  222.0   \n",
       "\n",
       "   PTRATIO       B  LSTAT  target  \n",
       "0     15.3  396.90   4.98    24.0  \n",
       "1     17.8  396.90   9.14    21.6  \n",
       "2     17.8  392.83   4.03    34.7  \n",
       "3     18.7  394.63   2.94    33.4  \n",
       "4     18.7  396.90   5.33    36.2  "
      ]
     },
     "execution_count": 46,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "boston_df = pd.DataFrame(boston[\"data\"], columns=boston[\"feature_names\"])\n",
    "boston_df[\"target\"] = pd.Series(boston[\"target\"])\n",
    "boston_df.head()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "506"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\n",
    "# How many samples?\n",
    "len(boston_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.6662221670168519"
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Import the Ridge model class from the linear_model module\n",
    "from sklearn.linear_model import Ridge\n",
    "\n",
    "# Setup random seed\n",
    "np.random.seed(42)\n",
    "\n",
    "# Create the data\n",
    "X = boston_df.drop(\"target\", axis=1)\n",
    "y = boston_df[\"target\"]\n",
    "\n",
    "# Split into train and test sets\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2)\n",
    "\n",
    "# Institate and fit the model (on the training set)\n",
    "model = Ridge()\n",
    "model.fit(X_train, y_train)\n",
    "\n",
    "# Check the score of the model (on the test set)\n",
    "model.score(X_test, y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.8654448653350507"
      ]
     },
     "execution_count": 49,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Import the RandomForestRegressor model class from the ensemble module\n",
    "from sklearn.ensemble import RandomForestRegressor\n",
    "\n",
    "# Setup random seed\n",
    "np.random.seed(42)\n",
    "\n",
    "# Create the data\n",
    "X = boston_df.drop(\"target\", axis=1)\n",
    "y = boston_df[\"target\"]\n",
    "\n",
    "# Split into train and test sets\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2)\n",
    "\n",
    "# Institate and fit the model (on the training set)\n",
    "model = RandomForestRegressor()\n",
    "model.fit(X_train, y_train)\n",
    "\n",
    "# Check the score of the model (on the test set)\n",
    "model.score(X_test, y_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Picking a machine learning model for a classification problem"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>age</th>\n",
       "      <th>sex</th>\n",
       "      <th>cp</th>\n",
       "      <th>trestbps</th>\n",
       "      <th>chol</th>\n",
       "      <th>fbs</th>\n",
       "      <th>restecg</th>\n",
       "      <th>thalach</th>\n",
       "      <th>exang</th>\n",
       "      <th>oldpeak</th>\n",
       "      <th>slope</th>\n",
       "      <th>ca</th>\n",
       "      <th>thal</th>\n",
       "      <th>target</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>63</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>145</td>\n",
       "      <td>233</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>150</td>\n",
       "      <td>0</td>\n",
       "      <td>2.3</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>37</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>130</td>\n",
       "      <td>250</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>187</td>\n",
       "      <td>0</td>\n",
       "      <td>3.5</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>41</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>130</td>\n",
       "      <td>204</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>172</td>\n",
       "      <td>0</td>\n",
       "      <td>1.4</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>56</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>120</td>\n",
       "      <td>236</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>178</td>\n",
       "      <td>0</td>\n",
       "      <td>0.8</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>57</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>120</td>\n",
       "      <td>354</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>163</td>\n",
       "      <td>1</td>\n",
       "      <td>0.6</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   age  sex  cp  trestbps  chol  fbs  restecg  thalach  exang  oldpeak  slope  \\\n",
       "0   63    1   3       145   233    1        0      150      0      2.3      0   \n",
       "1   37    1   2       130   250    0        1      187      0      3.5      0   \n",
       "2   41    0   1       130   204    0        0      172      0      1.4      2   \n",
       "3   56    1   1       120   236    0        1      178      0      0.8      2   \n",
       "4   57    0   0       120   354    0        1      163      1      0.6      2   \n",
       "\n",
       "   ca  thal  target  \n",
       "0   0     1       1  \n",
       "1   0     2       1  \n",
       "2   0     2       1  \n",
       "3   0     2       1  \n",
       "4   0     2       1  "
      ]
     },
     "execution_count": 50,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "heart_disease = pd.read_csv(\"data/heart-disease.csv\")\n",
    "heart_disease.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "303"
      ]
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# How many samples are there?\n",
    "len(heart_disease)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 303 entries, 0 to 302\n",
      "Data columns (total 14 columns):\n",
      " #   Column    Non-Null Count  Dtype  \n",
      "---  ------    --------------  -----  \n",
      " 0   age       303 non-null    int64  \n",
      " 1   sex       303 non-null    int64  \n",
      " 2   cp        303 non-null    int64  \n",
      " 3   trestbps  303 non-null    int64  \n",
      " 4   chol      303 non-null    int64  \n",
      " 5   fbs       303 non-null    int64  \n",
      " 6   restecg   303 non-null    int64  \n",
      " 7   thalach   303 non-null    int64  \n",
      " 8   exang     303 non-null    int64  \n",
      " 9   oldpeak   303 non-null    float64\n",
      " 10  slope     303 non-null    int64  \n",
      " 11  ca        303 non-null    int64  \n",
      " 12  thal      303 non-null    int64  \n",
      " 13  target    303 non-null    int64  \n",
      "dtypes: float64(1), int64(13)\n",
      "memory usage: 33.3 KB\n"
     ]
    }
   ],
   "source": [
    "heart_disease.info()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "* We've got 303 samples (1 row = 1 sample) and we're trying to predict whether or not a patient has heart disease.\n",
    "\n",
    "* Because we're trying to predict whether each sample is one thing or another, we've got a classification problem.\n",
    "\n",
    "* Let's see how it lines up with our Scikit-Learn algorithm cheat-sheet."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\John\\anaconda3\\lib\\site-packages\\sklearn\\svm\\_base.py:976: ConvergenceWarning: Liblinear failed to converge, increase the number of iterations.\n",
      "  warnings.warn(\"Liblinear failed to converge, increase \"\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0.8688524590163934"
      ]
     },
     "execution_count": 53,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Import LinearSVC from the svm module\n",
    "from sklearn.svm import LinearSVC\n",
    "\n",
    "# Setup random seed\n",
    "np.random.seed(42)\n",
    "\n",
    "# Split the data into X (features/data) and y (target/labels)\n",
    "X = heart_disease.drop(\"target\", axis=1)\n",
    "y = heart_disease[\"target\"]\n",
    "\n",
    "# Split into train and test sets\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2)\n",
    "\n",
    "# Instantiate and fit the model (on the training set)\n",
    "clf = LinearSVC(max_iter=1000)\n",
    "clf.fit(X_train, y_train)\n",
    "\n",
    "# Check the score of the model (on the test set)\n",
    "clf.score(X_test, y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.8524590163934426"
      ]
     },
     "execution_count": 54,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Import the RandomForestClassifier model class from the ensemble module\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "\n",
    "# Setup random seed\n",
    "np.random.seed(42)\n",
    "\n",
    "# Split the data into X (features/data) and y (target/labels)\n",
    "X = heart_disease.drop(\"target\", axis=1)\n",
    "y = heart_disease[\"target\"]\n",
    "\n",
    "# Split into train and test sets\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2)\n",
    "\n",
    "# Instantiate and fit the model (on the training set)\n",
    "clf = RandomForestClassifier()\n",
    "clf.fit(X_train, y_train)\n",
    "\n",
    "# Check the score of the model (on the test set)\n",
    "clf.score(X_test, y_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 3. Fit the model to data and using it to make predictions"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Scikit-Learn enables this in several ways. Two of the most common and useful are predict() and predict_proba()."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0, 1, 1, 0, 1, 1, 1, 0, 0, 1, 1, 0, 1, 0, 1, 1, 1, 0, 0, 0, 0, 0,\n",
       "       1, 1, 1, 1, 1, 1, 0, 1, 0, 0, 0, 0, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1,\n",
       "       1, 0, 1, 1, 0, 0, 0, 0, 1, 1, 0, 0, 0, 1, 0, 0, 0], dtype=int64)"
      ]
     },
     "execution_count": 55,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Use a trained model to make predictions\n",
    "clf.predict(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.8524590163934426"
      ]
     },
     "execution_count": 56,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Compare predictions to truth\n",
    "y_preds = clf.predict(X_test)\n",
    "np.mean(y_preds == y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.8524590163934426"
      ]
     },
     "execution_count": 57,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.metrics import accuracy_score\n",
    "accuracy_score(y_test, y_preds)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0.89, 0.11],\n",
       "       [0.49, 0.51],\n",
       "       [0.43, 0.57],\n",
       "       [0.84, 0.16],\n",
       "       [0.18, 0.82]])"
      ]
     },
     "execution_count": 58,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Return probabilities rather than labels\n",
    "clf.predict_proba(X_test[:5])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Regression "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Import the RandomForestRegressor model class from the ensemble module\n",
    "from sklearn.ensemble import RandomForestRegressor\n",
    "\n",
    "# Setup random seed\n",
    "np.random.seed(42)\n",
    "\n",
    "# Create the data\n",
    "X = boston_df.drop(\"target\", axis=1)\n",
    "y = boston_df[\"target\"]\n",
    "\n",
    "# Split into train and test sets\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2)\n",
    "\n",
    "# Institate and fit the model (on the training set)\n",
    "model = RandomForestRegressor()\n",
    "model.fit(X_train, y_train)\n",
    "\n",
    "# Make predictions\n",
    "y_preds = model.predict(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2.136382352941176"
      ]
     },
     "execution_count": 60,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Compare the predictions to the truth\n",
    "from sklearn.metrics import mean_absolute_error\n",
    "mean_absolute_error(y_test, y_preds)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 4. Evaluating a model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Scikit-Learn implements 3 different methods of evaluating models.\n",
    "\n",
    "* The score() method. Calling score() on a model instance will return a metric assosciated with the type of model you're using. The metric depends on which model you're using.\n",
    "* The scoring parameter. This parameter can be passed to methods such as cross_val_score() or GridSearchCV() to tell Scikit-Learn to use a specific type of scoring metric.\n",
    "* Problem-specific metric functions. Similar to how the scoring parameter can be passed different scoring functions, Scikit-Learn implements these as stand alone functions."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.8654448653350507"
      ]
     },
     "execution_count": 61,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Import the RandomForestRegressor model class from the ensemble module\n",
    "from sklearn.ensemble import RandomForestRegressor\n",
    "\n",
    "# Setup random seed\n",
    "np.random.seed(42)\n",
    "\n",
    "# Create the data\n",
    "X = boston_df.drop(\"target\", axis=1)\n",
    "y = boston_df[\"target\"]\n",
    "\n",
    "# Split into train and test sets\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2)\n",
    "\n",
    "# Institate and fit the model (on the training set)\n",
    "model = RandomForestRegressor()\n",
    "model.fit(X_train, y_train);\n",
    "\n",
    "# Check the score of the model (on the test set)\n",
    "model.score(X_test, y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Import cross_val_score from the model_selection module\n",
    "from sklearn.model_selection import cross_val_score\n",
    "\n",
    "# Import the RandomForestClassifier model class from the ensemble module\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "\n",
    "# Setup random seed\n",
    "np.random.seed(42)\n",
    "\n",
    "# Split the data into X (features/data) and y (target/labels)\n",
    "X = heart_disease.drop(\"target\", axis=1)\n",
    "y = heart_disease[\"target\"]\n",
    "\n",
    "# Split into train and test sets\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2)\n",
    "\n",
    "# Instantiate the model (on the training set)\n",
    "clf = RandomForestClassifier()\n",
    "\n",
    "# Call the fit method on the model and pass it training data\n",
    "clf.fit(X_train, y_train);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.8524590163934426"
      ]
     },
     "execution_count": 63,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Using score()\n",
    "clf.score(X_test, y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0.81967213, 0.86885246, 0.81967213, 0.78333333, 0.76666667])"
      ]
     },
     "execution_count": 64,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Using cross_val_score()\n",
    "cross_val_score(clf, X, y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(0.8524590163934426, 0.8248087431693989)"
      ]
     },
     "execution_count": 65,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.random.seed(42)\n",
    "\n",
    "# Single training and test split score\n",
    "clf_single_score = clf.score(X_test, y_test)\n",
    "\n",
    "# Take mean of 5-fold cross-validation\n",
    "clf_cross_val_score = np.mean(cross_val_score(clf, X, y, cv=5))\n",
    "\n",
    "clf_single_score, clf_cross_val_score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0.78688525, 0.86885246, 0.80327869, 0.78333333, 0.76666667])"
      ]
     },
     "execution_count": 66,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cross_val_score(clf, X, y, cv=5, scoring=None) # default scoring"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Classification model evaluation metrics"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Four of the main evaluation metrics/methods you'll come across for classification models are:\n",
    "\n",
    "* Accuracy\n",
    "* Area under ROC curve\n",
    "* Confusion matrix\n",
    "* Classification report"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Accuracy\n",
    "\n",
    "Accuracy is the default metric for the score() function within each of Scikit-Learn's classifier models. And it's probably the metric you'll see most often used for classification problems.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Heart Disease Classifier Accuracy: 85.25%\n"
     ]
    }
   ],
   "source": [
    "# Accuracy as percentage\n",
    "print(f\"Heart Disease Classifier Accuracy: {clf.score(X_test, y_test) * 100:.2f}%\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "\n",
    "##### Area Under Receiver Operating Characteristic (ROC) Curve\n",
    "\n",
    "ROC curves are a comparison of true postive rate (tpr) versus false positive rate (fpr).\n",
    "\n",
    "For clarity:\n",
    "\n",
    "* True positive = model predicts 1 when truth is 1\n",
    "* False positive = model predicts 1 when truth is 0\n",
    "* True negative = model predicts 0 when truth is 0\n",
    "* False negative = model predicts 0 when truth is 1\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.03448276, 0.03448276, 0.03448276, 0.03448276, 0.06896552,\n",
       "       0.06896552, 0.10344828, 0.13793103, 0.13793103, 0.17241379,\n",
       "       0.17241379, 0.27586207, 0.4137931 , 0.48275862, 0.55172414,\n",
       "       0.65517241, 0.72413793, 0.72413793, 0.82758621, 1.        ])"
      ]
     },
     "execution_count": 68,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.metrics import roc_curve\n",
    "\n",
    "# Make predictions with probabilities\n",
    "y_probs = clf.predict_proba(X_test)\n",
    "\n",
    "# Keep the probabilites of the positive class only\n",
    "y_probs = y_probs[:, 1]\n",
    "\n",
    "# Calculate fpr, tpr and thresholds\n",
    "fpr, tpr, thresholds = roc_curve(y_test, y_probs)\n",
    "\n",
    "# Check the false positive rate\n",
    "fpr"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEWCAYAAABrDZDcAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8vihELAAAACXBIWXMAAAsTAAALEwEAmpwYAAA76UlEQVR4nO3dd3gUVffA8e+hhAQIHRTpIDUgLTRRBAFFUEFFfiAWVEQU9FVs2EGwoyhVsYAKyKsoSBMQXhEBkWboRUSBKCC9JqSd3x8zwSVukgWy2Wz2fJ4nT3Z27sycO7M7Z+bO7B1RVYwxxoSuPIEOwBhjTGBZIjDGmBBnicAYY0KcJQJjjAlxlgiMMSbEWSIwxpgQZ4kghxGRjSLSOtBx5BQi8oyIfBigZU8QkaGBWHZWE5GeIjL/PKc978+kiCwVkYbnM+35EpGHReS17FxmsLNEkAER+UNE4kTkhIjsdXcMhf25TFWNUtVF/lxGKhEpICKvisgut56/isgTIiLZsXwv8bQWkVjP91T1FVXt7aflibvT2CAiJ0UkVkS+FJF6/lje+RKRQSIy8ULmoaqTVPUaH5b1r+R3vp9JEbkBOK6qv7jDg0Qk0f0+HRGRZSLSIs00xURkrPt9OyUi60Xkbi/zvk1EVrnz2iMi34rIFe7occDtIlImg9iCYttnF0sEmbtBVQsDDYCGwNOBDefciUi+dEZ9CbQFOgKRwB1AH+BdP8QgIpLTPm/vAv8BHgZKADWA6UCnrF5QBtvA7wK47L7AZ2ne+6/7fSoFfI/zGQRARMKABUAloAVQFHgCeE1EBniUGwC8A7wCXARUBMYAnQFUNR74Frgzg9iybNsHcttmGVW1v3T+gD+Adh7DbwCzPYabA8uAI8BaoLXHuBLAeOAv4DAw3WPc9UCMO90y4LK0ywQuAeKAEh7jGgIHgPzu8D3AZnf+84BKHmUV6Af8CvzupW5tgXigQpr3mwHJwKXu8CLgVWAFcBT4Jk1MGa2DRcDLwFK3LpcCd7sxHwd2APe7ZQu5ZVKAE+7fJcAgYKJbprJbr7uAXe66eNZjeRHAJ+762Aw8CcSms22ru/VsmsH2nwCMBma78f4MVPMY/y6wGzgGrAau9Bg3CJgKTHTH9waaAj+562oPMAoI85gmCvgOOATsA54BOgAJQKK7Tta6ZYsCH7nz+RMYCuR1x/Vy1/lwd15D3feWuOPFHfe3u03XAXVxDgIS3eWdAGam/R4Aed24fnPXyWrSfIbccmHu9iyfZp1M9Biu427P0u7wvW5MhdLM6//ceIq49T4B3JrJd7cn8P0FbPtFQG+P4TPrz9v3C3gPGJZmHt8AA9zXlwBfAfvd8g8Hev92VqyBDiAn/6X5ApQH1gPvusPlgIM4R9N5gPbucOqHejbwX6A4kB+4yn2/kfthb+Z+qe5yl1PAyzL/B9znEc+bwHvu6y7AdqA2kA94DliW5oP6HU5CivBSt9eAH9Kp907+2UEvwtnR1MXZWX/FPzvmzNbBIpwddpQbY36cI65qODujq4BTQCO3fGvS7Ljxngg+wNnp1wdOA7U96+Su8/I4O7j0EkFfYGcm238Czo60qRv/JGCKx/jbgZLuuMeAvUC4R9yJ7nbK48bbGCdx5nPrshl4xC0fibNTfwwId4ebpV0HHsueDrzvbpMyOIk6dZv1ApKAh9xlRXB2IrgWZwdezN0OtYGyHnUemsH34Amc70FNd9r6QEkv6y4KOJnBtgxzt9cBIJ/73hTgEy/zyufW51qcxJiUOk0G264RcOgCtv0iMk8EZ75fQCucgwJxxxfHSYSXuNt/NfCCW++qOAdB1wZ6H5f6l9NO1XOi6SJyHGcj/w286L5/OzBHVeeoaoqqfgesAjqKSFngOqCvqh5W1URV/cGd7j7gfVX9WVWTVfUTnJ1Zcy/Lngz0AKdpBejuvgdwP/Cqqm5W1SSc0+QGIlLJY/pXVfWQqsZ5mXcpnB2PN3vc8ak+U9UNqnoSeB7oJiJ5M1oHHtNOUNWNqprkrofZqvqbOn4A5gNXphNHegarapyqrsU5C6nvvt8NeMVd57HAiAzmUTKD+nv6WlVXuOt4Ek4TIQCqOlFVD7p1ewsogLODTPWTqk53102cqq5W1eVu+T9wduRXuWWvB/aq6luqGq+qx1X1Z28BichFOJ+vR1T1pKr+jXOE392j2F+qOtJdVtrtn4iTaGrh7Lg2q6ov6wKcM5vnVHWruw3XqupBL+WK4ZwxpNVNRI7g7CTvA7q66xbS+Uy64w+440sCBzymSc9xnLMHb3zd9pnx/H79iJMcUj/LXXG2/19AE5yDo5dUNUFVd+AczHT3OtcAsESQuS6qGolztFqLf3aQlYBb3YteR9wP9xVAWaACztHIYS/zqwQ8lma6CjhHDmlNBVqIyCU4RxyK84FLnc+7HvM4hHOEVs5j+t0Z1OuAG6s3Zd3x3uazE+fIvhQZrwOvMYjIdSKyXEQOueU7cnbS8cVej9engNQL+JekWV5G9T9I+vX3ZVmIyGMisllEjrp1KcrZdUlb9xoiMsu9EHoMJ3mnlq+A09zii0o422CPx3p/H+fMwOuyPanq/3CapUYD+0RknIgU8XHZvsZ5GCfZpPWFqhbDadvfgHOWlMrrZ9Jtgy/ljj8IlPKhXT4Sp9nLG1+3fWbOrGN1TgOm4B64AbfhHDiAs70uSfM9eQZnHeQIlgh85B69TgCGuW/txjlSLubxV0hVX3PHlRCRYl5mtRt4Oc10BVX1cy/LPIJzxNwN54P1ufuBS53P/WnmE6GqyzxnkUGVFgDNRKSC55si0hTny/4/j7c9y1TEOaI8kMk6+FcMIlIAp2lpGHCRu0OYg5PAMovXF3twmoS8xZ3WQqC8iESfz4JE5ErgKZxtU9yty1H+qQv8uz5jgS1AdVUtgrMzSC2/G6fJzJu089mNcxZZymO9F1HVqAymOXuGqiNUtTFOE04NnCafTKfLJE5Pv+KcyJbzNlJVD+Cc1Q5yz6DB+UxeJyKF0hS/Bae+y3GuscTjNLllpDbO2aI3vmz7k0BBj+GLvZRJu64+B7q6Z+XNcD7r4Kyz39N8TyJVtSM5hCWCc/MO0F5EGuBcBLxBRK4VkbwiEu7e/ljePc3+FhgjIsVFJL+ItHLn8QHQV0SauXfSFBKRTiLi7egJnKagO3G+DJM93n8PeFpEogBEpKiI3OprRVR1Ac4X4isRiXLr0BznKGasqv7qUfx2EakjIgWBl4Cpqpqc0TpIZ7FhOM0n+4EkEbkO8LylcR9QUkTSO6XPzBc466S4uwPqn15Bt35jgM/dmMPc+LuLyEAflhWJ01a9H8gnIi/gXMzMbJpjwAkRqQU84DFuFnCxiDwizm29kSLSzB23D6iceteV+/maD7wlIkVEJI+IVBORq/CBiDRxP3/5cXZ48TgXT1OXVTWDyT8EhohIdffze5mIlExbSFUTcXbs6cakqltwbnJ40n3rMyAW+FJEKrvfm2txmvgGqepRVT2K09Y+WkS6iEhBt9x1IvKGx+yvwvkOeluuL9s+BrjZnf+lOBeyM6TObbL73XU0zz2QA+f6zTEReUpEItzvSl0RaZLZPLOLJYJzoKr7gU+B51V1N87tas/gbPzdOEdVqev0Dpwj5y041xYeceexCqdtdBTO6fN2nAtR6ZmBc5fDPrdNPDWWacDrwBS3mWEDTrvxubgF5xa+uTh3YkzEuRPloTTlPsM5G9qLcyHzYTeGzNbBWVT1uDvtFzh1v82tX+r4LThHVTvcU2hvzWUZeQlnR/I7zk5oKs6RZHoe5p8mkiM4TR43ATN9WNY8nB3NNpzmsngybooCeBynzsdxDgj+mzrCXTftgRtw1vOvQBt3dOotlgdFZI37+k6cxLoJZ11OxffmjiLu8g+7sR/knzPdj4A67vqf7mXat3G233ycpPYRzsVSb97H+R5k5E2gj4iUUdXTOHfM7ca5Q+uYu7xnVfXN1AlU9W1gAM4NEqmfu/44F9ARkXCcJsdPMlhuZtt+OM7dU/vc+Uz69yy8+tytw5mDNveg6Qac60u/45xNf0j61zCyXeoVbmO8EpFFOHd6BOTXvRdCRB4AuquqT0fKJuuJyBLgIfdoObuW+RDOLa1PZlrYAM5tWcbkCm5bc1WcduTqOLdijgpoUCFOVa/IvFSWL3Nkdi8z2FkiMLlJGE5zRBWc0/0pOG3BxpgMWNOQMcaEOLtYbIwxIS7omoZKlSqllStXDnQYxhgTVFavXn1AVUt7Gxd0iaBy5cqsWrUq0GEYY0xQEZGd6Y2zpiFjjAlxlgiMMSbEWSIwxpgQZ4nAGGNCnCUCY4wJcX5LBCLysYj8LSIb0hkvIjJCRLaLyDoRaeSvWIwxxqTPn2cEE3AeK5ee63D6g6mO86zUsX6MxRhjTDr89jsCVV0sIpUzKNIZ+NR90MpyESkmImXP4ZF5xpicbPfXcDgm0FHkComJ8PufeajR5HIoe03mE5yjQP6grBxn998e6773r0QgIn1wzhqoWLFitgRnjLlAK+6H0wc4+6Ft5lz98scl3DPuVv4+Vphts7+nUC5LBN4+HV57wFPVccA4gOjoaOslz5hgoMlQ4yGIHhHoSIJSfHwSgwcv4803V1KqVARjPmxHoRY1/LKsQCaCWM5+pmx54K8AxWKMMTlKly7TmTfvD+6+uy5vvdWa4sXD/basQN4+OgO40717qDlw1K4PGGNC2fHjCcTHJwEwcGBT5s/vyscfd/BrEgA/nhGIyOdAa6CUiMQCLwL5AVT1PWAOznNFtwOngLv9FYsxxuR08+b9Tp8+87n99jq8/PKVtG6dfddD/XnXUI9MxivQz1/LDwlHt8BRrz/TMCbwUhICHUFQOHQojgEDFvHJJxupVasEnTpVzfYYgq4bauNhaTc4sj7QURiTvrASgY4gR1u4cCc9e87m4MF4nn22Oc8915zw8OzfLVsiCGZJcXBJR2jweqAjMcYLgSK1Ah1EjlamTEGqVCnK3LldadCgTMDisEQQ7PIXg2J1Ax2FMcYHqsonn2xkzZp9jBjRlnr1SrNs2W2IBPa3FtbpnDHGZIPffz/CtddO5e675xITs5+4uESAgCcBsDMCY4zxq+TkFEaPjuHppxeTJ48wZkw77r+/PnnyBD4BpLJEYIwxfnTgQBwvvLCUq66qwHvvtadixSKBDulfLBEYY0wWS0xMZtKkzdx5ZxQXXVSINWvuoEqVojmiGcgbSwTGGJOFVq/eyz33zGPduv2ULVuIa6+tQtWqxQIdVobsYrExxmSBuLhEBg5cTLNmk9i//xTTpnXm2murBDosn9gZgTHGZIEuXb5h/vw/6N27Hm++eRXFivm3f6CsZInAGGPO07FjpwkLy0t4eD6eeaYZTz7ZhLZtKwU6rHNmTUPGGHMe5szZQd26E3jppZ8AuOqqCkGZBMASgTHGnJMDB05xxx1z6NTpayIjw7jxxmqBDumCWdOQMcb46Lvv/qBnz9kcPnyaF15owTPPNKNAgeDfjQZ/DYwxJpuULVuIGjVKMHZsO+rVKx3ocLKMNQ0ZY0w6VJUPP1xHv34LAKhbtzQ//tg9VyUBsERgjDFe7dhxhHbtvuS+++azadPBHNVJXFazRGCMMR6Sk1MYPnwVdetOYOXKvbz/fnsWLuxGRET+QIfmN3aNIKc5thV+vheS4zMve2oXlGzq/5iMCSEHDsQxePBPtG1bkbFj21O+fGSgQ/I7SwQ5zaHVsH8plLkK8hXOuOzFF0Pl27InLmNysYSEZCZO3ESvXnW56KJCxMTcSaVKRXJlM5A3lghyqqbjoEiNQEdhTK63cuUe7rlnHhs2HKB8+UiuuaYylSsXDXRY2cquERhjQtKpU4k8/vgimjefzOHD8cyYcRPXXFM50GEFhJ0RGGNCUufO01mwYCd9+lzGG29cRdGiBQIdUsBYIjDGhIyjR09ToIDTSdzzzzfnmWea0aZNxUCHFXDWNGSMCQmzZv1GVNR4Bg9eBkCrVhUsCbgsERhjcrX9+09x222zuOGGaZQoEc7NN1cPdEg5jjUNGWNyrfnznU7ijh49zeDBlzNwYDPCwvIGOqwcxxKBMSbXKleuMLVrl2Ts2HZERZUKdDg5ljUNGWNyjZQUZdy4tTzwwHcAREWVYvHi7pYEMmGJwBiTK2zffpi2bb/g/vu/Y+vWQ2c6iTOZs0RgjAlqyckpvPXWSi677BPWrNnHBx9ck+s7ictqfk0EItJBRLaKyHYRGehlfFERmSkia0Vko4jc7c94jDG5z4EDcQwdupz27SuxadPd9O59Wcj0EZRV/JYIRCQvMBq4DqgD9BCROmmK9QM2qWp9oDXwloiE+SsmY0zucPp0Eh98sI6UFD3TSdz06V0oVy739xTqD/48I2gKbFfVHaqaAEwBOqcpo0CkOOm7MHAISPJjTMaYIPfzz3to3Pgz+vSZz4IFOwGoVKmonQVcAH8mgnLAbo/hWPc9T6OA2sBfwHrgP6qaknZGItJHRFaJyKr9+/f7K15jTA528mQCAwZ8T4sWkzh6NIHZs28O2U7ispo/E4G39Kxphq8FYoBLgAbAKBEp8q+JVMeparSqRpcunbueFWqM8U2XLt8wfPhq+vatz8aNvejYsWqgQ8o1/JkIYoEKHsPlcY78Pd0NfK2O7cDvQC0/xmSMCSJHjsSfuQ30hRda8MMP/8eYMe0pUiR0ewr1B38mgpVAdRGp4l4A7g7MSFNmF9AWQEQuAmoCO/wYkzEmSMyYsZ2oqAkMHvwTAFdeWZ5WrSpkMpU5H35LBKqaBPQH5gGbgS9UdaOI9BWRvm6xIcDlIrIeWAg8paoH/BWTMSbn+/vvk3TvPpPOnadTqlQEXbvak/r8za99DanqHGBOmvfe83j9F3CNP2MwxgSPuXN/p2fP2Zw4kciQIS156qmm5M9vncT5m3U6Z4zJMSpUiKRevVKMGdOOOnWsf6DsYl1MGGMCJiVFGTs2hvvvnw84ncQtWtTdkkA2szOC7JKcAAmHMy+XeNT/sRiTA2zbdojevefz44+xtG9fifj4JMLDbZcUCLbW/U1TYMd4iHkaTp/Dj+HyWE8bJndKSnI6iXvxxWVERORj/PgO3HVXlP0yOIAsEfjTgZ9hVX84tApKt4RKg8CXD3uBUlCokt/DMyYQDh6M4/XXV9KxY1VGj25L2bKFAx1SyLNE4A9x+2DtQNgxASLKQouJUPk235KAMbnQ6dNJTJiwkfvuu4yLLirE2rV3UqHCvzoRMAFiiSArpSTC1pGwYTAkx0HtJ6Huc5DfekQ0oeunn/7i3nvnsnnzIapVK0a7dpUsCeQwlgiyyp7vYPV/4NhmKNsBGr8DRWoGOipjAubEiQSee24JI0asoUKFSObOvYV27azJMyeyRHChTvwBawZA7DQoXBVazYBy11szkAl5XbpMZ+HCXfTv35BXXrmSyEi7ASKnEtW0HYLmbNHR0bpq1apAh+E4uhnmNgLyQNQzUPsxyBse6KiMCZjDh+MJD89LRER+liyJBeCKK8oHOCoDICKrVTXa2ziff1AmIoWyLqRc4sg6SI6HqxdA3WctCZiQ9vXX26hTZzyDBi0DnARgSSA4ZJoIRORyEdmE03EcIlJfRMb4PbJgElYs0BEYEzB7956ka9dvuOWWGVx8cSG6d7ee5IONL9cIhuM8QGYGgKquFZFWfo3KGBMUvv12Bz17zuHUqUReeeVKHn882jqJC0I+XSxW1d1pfvWX7J9wjDHBpFKlIjRsWIbRo9tSq1bJQIdjzpMv1wh2i8jlgIpImIg8jttMZIwJLSkpyqhRa7jvvnkA1KlTioULu1kSCHK+nBH0Bd7FefB8LDAfeNCfQQXc5mHw19zMy8Xv9X8sxuQQW7ce4t5757F06Z9ce21l6yQuF/FlK9ZU1Z6eb4hIS2Cpf0LKAX77GE7/DUUyuegVVgzK3QiFKmdHVMYERGJiMsOGrWLw4GUULJifCRM6cOed1klcbuJLIhgJNPLhvdzloqvhii8CHYUxAXf4cDxvvrmSG26oxsiRbbn4YruTPLdJNxGISAvgcqC0iAzwGFUEsNsCjMnF4uOT+Pjj9fTt24AyZQqxbt1dlC9vfWblVhmdEYQBhd0ynp+AY0BXfwZljAmcJUtiuffeeWzbdpgaNUrQrl0lSwK5XLqJQFV/AH4QkQmqujMbYzLGBMDx4wk8/fRiRo+OoXLlIsyf39U6iQsRvlwjOCUibwJRwJk+FFT1ar9FZYzJdl26TOf773fxn/80YujQKyhc2DqJCxW+JIJJwH+B63FuJb0LOIdnLhpjcqpDh+IID89HwYL5GTKkJSJX0KLFJYEOy2QzX35QVlJVPwISVfUHVb0HaO7nuIwxfjZ16lZq1/6nk7jLLy9nSSBE+ZIIEt3/e0Skk4g0BKxLQWOC1J49J7j55m+49daZVKgQSc+etQMdkgkwX5qGhopIUeAxnN8PFAEe8WdQxhj/mD37N26/fQ7x8cm8/norBgyIJl8+n3ujN7lUpolAVWe5L48CbeDML4uNMUGmatViNGlyMaNGtaVGjRKBDsfkEBn9oCwv0A2nj6G5qrpBRK4HngEigIbZE6Ix5nwlJ6cwatQvrFu3n48+6kDt2iWZP//WQIdlcpiMzgg+AioAK4ARIrITaAEMVNXp2RCbMeYCbNp0gN695/PTT3/RsWMV6yTOpCujT0U0cJmqpohIOHAAuFRVrctNY3KwhIRk3nhjBUOGLCcyMoyJEzty2221rZM4k66MrhIlqGoKgKrGA9vONQmISAcR2Soi20VkYDplWotIjIhsFJEfzmX+xph/O3IknuHDV3PTTZeyaVMvevasY0nAZCijM4JaIrLOfS1ANXdYAFXVyzKasXuNYTTQHuc5BitFZIaqbvIoUwwYA3RQ1V0iUub8q2JM6IqLS+Sjj9bz4IMNKVOmEOvX9+KSSwoHOiwTJDJKBBd6c3FTYLuq7gAQkSlAZ2CTR5nbgK9VdReAqv59gcs0JuQsXryb3r3n8+uvh6lduyRt21ayJGDOSbpNQ6q6M6M/H+ZdDtjtMRzrvuepBlBcRBaJyGoRudPbjESkj4isEpFV+/db7xbGABw7dpoHH/yOq676L0lJKSxYcCtt21oncebc+fMWAm+Nkupl+Y2Btji3pP4kIstVddtZE6mOA8YBREdHp52HMSGpS5fpLFq0m0cfbcyQIS0pVMg6iTPnx5+JIBbn9tNU5YG/vJQ5oKongZMishioD2zDGPMvBw6comDB/BQsmJ+XX74SEWje3PoHMhfGp9+Wi0iEiNQ8x3mvBKqLSBURCQO6AzPSlPkGuFJE8olIQaAZsPkcl2NMrqeqTJmyhdq1x/Pii87jwlu0uMSSgMkSmSYCEbkBiAHmusMNRCTtDv1fVDUJ6A/Mw9m5f6GqG0Wkr4j0dctsdue7DueHax+q6obzrIsxudKffx6nS5fp9OgxiypVinLnnVGBDsnkMr40DQ3CuQNoEYCqxohIZV9mrqpzgDlp3nsvzfCbwJu+zM+YUDNr1m/07DmbxMQUhg27ikceaUzevNZJnMlaviSCJFU9aj9IMSb7XXppMS6//BJGjmzLpZcWD3Q4Jpfy5dBig4jcBuQVkeoiMhJY5ue4jAlJyckpDB++il69vgWgVq2SfPttV0sCxq98SQQP4Tyv+DQwGac76kf8GJMxIWnjxgO0bPk5AwYs4sCBOOLjkwIdkgkRvjQN1VTVZ4Fn/R2MMaEoISGZ1177maFDl1O0aAEmT+5E9+61rH8gk218SQRvi0hZ4Etgiqpu9HNMxoSUI0fiGTHiF269tSbvvNOG0qULBjokE2IybRpS1TZAa2A/ME5E1ovIc/4OzJjc7NSpRN59dzXJySluJ3F3MWlSJ0sCJiB8ug9NVfeq6gigL85vCl7wZ1DG5Gbff7+LevUm8Mgj37NokdMdV9my1kmcCRxfflBWW0QGicgGYBTOHUPl/R6ZMbnM0aOnuf/++Vx99ReICN9/3806iTM5gi/XCMYDnwPXqGravoKMMT7q0mU6ixfH8sQTTRg06HIKFswf6JCMAXxIBKraPDsCMSY32r//FIUKOZ3EvfrqleTNKzRpUjbQYRlzlnSbhkTkC/f/ehFZ5/G33uPJZcYYL1SVyZM3n9VJXPPml1gSMDlSRmcE/3H/X58dgRiTW8TGHueBB75j1qwdNGtWll696gY6JGMylNETyva4Lx/08nSyB7MnPGOCy4wZ26lTZzz/+98uhg9vw9KlPYiKKhXosIzJkC+3j7b38t51WR2IMblBjRrFueKKcqxf38t6CjVBI92mIRF5AOfIv2qaawKRwFJ/B2ZMMEhKSuGdd1azbt1+Pv20I7VqlWTOnFsCHZYx5ySjawSTgW+BV4GBHu8fV9VDfo3KmCCwbt1+7r13LqtW7aNz50uJj08iPNyfT381xj8y+tSqqv4hIv3SjhCREpYMTKg6fTqJV175mVde+ZkSJcL54osb6Nq1hnUSZ4JWZmcE1wOrAQU8P+UKVPVjXMbkWMeOJTBmTAw9etRi+PA2lCwZEeiQjLkg6SYCVb3e/V8l+8IxJmc6eTKBcePW8fDDjShduiAbNvTioosKBTosY7KEL30NtRSRQu7r20XkbRGp6P/QjMkZFi7cSb16nzBgwCJ++CEWwJKAyVV8ubdtLHBKROoDTwI7gc/8GpUxOcCRI/H07j2Pdu2+JF++PPzww/9x9dV2DGRyH18fXq8i0hl4V1U/EpG7/B2YMYF2003f8OOPsTz1VFNefLEFERHWSZzJnXxJBMdF5GngDuBKEckL2DfC5Er79p2kcOH8FCoUxmuvtSJfPqFx44sDHZYxfuVL09D/4Ty4/h5V3QuUA970a1TGZDNV5bPPNlKnznhefHEZAM2albUkYEKCL4+q3AtMAoqKyPVAvKp+6vfIjMkmu3Ydo1Onr7nzzm+pWbME995bL9AhGZOtMm0aEpFuOGcAi3B+SzBSRJ5Q1al+ji1rpSTC3oWQHJd52cRj/o/H5AjffLOd22+fjSqMGHE1Dz7YwPoHMiHHl2sEzwJNVPVvABEpDSwAgisR7JkHP9zge/mw4v6LxQScqiIi1KpVgtatKzByZFsqVy4a6LCMCQhfEkGe1CTgOoiPD73PUVLPBK6YCpGXZl6+SC3/xmMCIikphbfeWsn69QeYOLETNWuWYObMmwMdljEB5UsimCsi83CeWwzOxeM5/gvJz4rUgmJRgY7CBMDatX9zzz3zWLNmHzfdVN06iTPG5cszi58QkZuBK3CuEYxT1Wl+j8yYLBIfn8TQoct5/fUVlCwZztSpN3LLLTUCHZYxOUZGzyOoDgwDqgHrgcdV9c/sCsyYrHL8eALvv7+Wnj1r8/bbrSlRwjqJM8ZTRm39HwOzgFtweiAdea4zF5EOIrJVRLaLyMAMyjURkWQR6XquyzDGmxMnEhg2bCXJySmULl2QTZvuZsKE6ywJGONFRk1Dkar6gft6q4isOZcZu79AHo3zqMtYYKWIzFDVTV7KvQ7MO5f5G5Oe+fP/oE+f+ezadYzGjS+iTZuKlC5dMNBhGZNjZXRGEC4iDUWkkYg0AiLSDGemKbBdVXeoagIwBejspdxDwFfA317GGeOzQ4fiuPvub7n22qmEh+fjxx970KaNdRJnTGYyOiPYA7ztMbzXY1iBqzOZdzlgt8dwLNDMs4CIlANucufVJL0ZiUgfoA9AxYr2xTbe3XTTNyxd+ifPPNOM559vYXcEGeOjjB5M0+YC5+3tuX2aZvgd4ClVTc7oMX+qOg4YBxAdHZ12HiaE7d17kshIp5O4N9+8irCwvDRoUCbQYRkTVPz5w7BYoILHcHngrzRlooEpIvIH0BUYIyJd/BiTySVUlQkTNlCnznheeGEpAE2blrUkYMx58Oe580qguohUAf4EugO3eRbwfAymiEwAZqnqdD/GZHKBP/44yv33f8f8+X9wxRXl6NOnfqBDMiao+S0RqGqSiPTHuRsoL/Cxqm4Ukb7u+Pf8tWyTe02b9it33DEHERg1qi0PPNCAPHnSb1Y0xmTOl95HBegJVFXVl9znFV+sqisym1ZV55CmO4r0EoCq9vIpYhOSUjuJi4oqSbt2lXj33TZUqmSdxBmTFXy5RjAGaAH0cIeP4/w+wBi/S0xM5pVXltOz52wAatQowfTpXSwJGJOFfEkEzVS1HxAPoKqHgTC/RmUMsGbNPpo2ncSzzy4hOVk5fTop0CEZkyv5kggS3V//Kpx5HkGKX6MyIS0uLpGnn15M06YT2bv3JNOmdea//72BAgXsdwHG+IMv36wRwDSgjIi8jHOb53N+jcqEtJMnE/noo/XcdVcUw4a1pnjx8ECHZEyu5ks31JNEZDXQFudHYl1UdbPfIzMh5fjxBMaOjeGxx6IpVcrpJK5UKesfyJjs4MtdQxWBU8BMz/dUdZc/AzOhY+7c37n//vns3n2cpk0vpnXripYEjMlGvjQNzca5PiBAOFAF2ArYY77MBTl4MI4BA77n0083Ubt2CZYuvY0WLS4JdFjGhBxfmobqeQ67PY/e77eITMi4+eZvWLbsL55/vjnPPtvcLgYbEyDn/M1T1TUikm5PocZkZM+eE0RGhlG4cBjDhjmdxNWvb/0DGRNIvlwjGOAxmAdoBOz3W0QmV1JVxo/fwIABi7jnnrq8/XYbmjQpG+iwjDH4dkYQ6fE6CeeawVf+CcfkRjt2HOH++79jwYKdtGpVnr59rZM4Y3KSDBOB+0Oywqr6RDbFY3KZr7/exh13zCFv3jyMHduOPn3qWydxxuQw6SYCEcnn9iDqy2MpjTlLaidx9eqVpkOHKrzzThsqVCgS6LCMMV5kdEawAud6QIyIzAC+BE6mjlTVr/0cmwlCCQnJvPHGCjZuPMjkyZ2oXr04X33l7VHVxpicwpdrBCWAgzjPFU79PYEClgjMWVat2su9985j3br9dO9ei4SEZLsl1JggkNG3tIx7x9AG/kkAqey5weaMuLhEXnxxGW+9tYqLLy7EN9904cYbLw10WMYYH2WUCPIChfHtIfQmhJ08mciECRu49956vPFGK4oVs07ijAkmGSWCPar6UrZFYoLKsWOnGTMmhieeaEKpUgXZvPkeSpaMCHRYxpjzkFEisHv8jFezZ/9G374L+OuvEzRvXpbWrStaEjAmiGX0YJq22RaFCQr795+iZ8/ZXH/9NIoWDWPZstto3bpioMMyxlygdM8IVPVQdgZicr5bbpnB8uV/MWjQ5Tz9dDPCwvIGOiRjTBawe/tMhv788zhFixagcOEwhg9vTYECealbt3SgwzLGZCFfnllsQpCq8sEH66hTZzwvvLAUgMaNL7YkYEwuZGcE5l9+++0I9903j++/302bNhXo169hoEMyxviRJQJzlqlTt3Lnnd+SP38exo27ht696yFiN5AZk5tZIjDAP53E1a9fhk6dqjJ8eBvKl4/MfEJjTNCzawQhLiEhmcGDl9G9+yxUlerVi/PllzdaEjAmhFgiCGErVuyhcePPGDRoGfny5SEhITnQIRljAsASQQg6dSqRxx9fRIsWkzl8OJ6ZM29i0qRO1lOoMSHKvvkhKC4uiYkTN9Gnz2W8/norihQpEOiQjDEB5NczAhHpICJbRWS7iAz0Mr6niKxz/5aJiD3M1k+OHj3Nyy8vJykphZIlI9i8+R7Gjm1vScAY479E4D7veDRwHVAH6CEiddIU+x24SlUvA4YA4/wVTyibOfO3Mz8MW7IkFoDixa2raGOMw59nBE2B7aq6Q1UTgCnAWc8sVNVlqnrYHVwOlPdjPCFn//5T9OgxixtvnEbJkuH8/HNP6yTOGPMv/rxGUA7Y7TEcCzTLoPy9wLfeRohIH6APQMWKtiPzVWoncS+91JKnnmpqncQZY7zyZyLw+clmItIGJxFc4W28qo7DbTaKjo62p6NlIDb2OMWKOZ3EvfNOGwoUyEtUVKlAh2WMycH82TQUC1TwGC4P/JW2kIhcBnwIdFbVg36MJ1dLSVHef38tdeqM5/nnnU7iGjW6yJKAMSZT/jwjWAlUF5EqwJ9Ad+A2zwIiUhH4GrhDVbf5MZZc7ddfD3PfffP44YdY2ratyEMPWSdxxhjf+S0RqGqSiPQH5gF5gY9VdaOI9HXHvwe8AJQExrgdmyWparS/YsqNvvzS6SSuQIG8fPTRtdx9d13rJM4Yc078+oMyVZ0DzEnz3nser3sDvf0ZQ26V2klcw4Zl6Ny5Gm+/3YZLLikc6LCMMUHIupgIMqdPJ/HCC0vo1m0mqsqllxZnypQbLAkYY86bJYIgsnz5XzRq9BlDhiwnIiKfdRJnjMkSlgiCwMmTCTz66Pdcfvlkjh9PYM6cm/n0047WSZwxJkvYniQIxMcnM2XKFh58sAGvvtqKyMiwQIdkjMlFLBHkUEeOxDNy5C88/XQzt5O4uylWzPoHMsZkPWsayoGmT/+VOnXGM3jwMpYt+xPAkoAxxm/sjCAH2bfvJA89tJAvv9xG/fqlmTnzJho3vjjQYRmTbRITE4mNjSU+Pj7QoQSt8PBwypcvT/78+X2exhJBDtK16wxWrNjL0KFX8OSTTcif3zqJM6ElNjaWyMhIKleubD+MPA+qysGDB4mNjaVKlSo+T2eJIMB27TpG8eLhREaGMWLE1RQokJc6dax/IBOa4uPjLQlcABGhZMmS7N+//5yms2sEAZKSoowe/QtRUc4DYwAaNrzIkoAJeZYELsz5rD87IwiArVsP0bv3PJYs+ZP27Svxn/80CnRIxpgQZmcE2eyLL7ZQv/4nbNhwgPHjOzBvXlcqVy4a6LCMMa68efPSoEED6tatyw033MCRI0fOjNu4cSNXX301NWrUoHr16gwZMgTVfx6R8u233xIdHU3t2rWpVasWjz/+eABqcO4sEWST1A9L48YXc/PN1dm8+R569bKeQo3JaSIiIoiJiWHDhg2UKFGC0aNHAxAXF8eNN97IwIED2bZtG2vXrmXZsmWMGTMGgA0bNtC/f38mTpzI5s2b2bBhA1WrVg1kVXxmTUN+Fh+fxJAhP7FlyyGmTr2RatWKMXny9YEOy5icb/UjcDgma+dZvAE0fsfn4i1atGDdunUATJ48mZYtW3LNNdcAULBgQUaNGkXr1q3p168fb7zxBs8++yy1atUCIF++fDz44INZG7+f2BmBHy1b9icNG37KK6/8TGRkmHUSZ0wQSU5OZuHChdx4442A0yzUuHHjs8pUq1aNEydOcOzYMTZs2PCv8cHCzgj84MSJBJ555kdGjfqFChUimTv3Fq691vd7eo0xnNORe1aKi4ujQYMG/PHHHzRu3Jj27dsD/zwDxJtgb+K1MwI/SEhIZurUbfTr15ANG+62JGBMEEm9RrBz504SEhLOXCOIiopi1apVZ5XdsWMHhQsXJjIykqioKFavXh2IkC+YJYIscuhQHIMGLSUpKYUSJSLYvPkeRo5saz2FGhOkihYtyogRIxg2bBiJiYn07NmTJUuWsGDBAsA5c3j44Yd58sknAXjiiSd45ZVX2LbNefx6SkoKb7/9dsDiPxeWCLLAV19to06d8QwduvxMJ3FFixYIcFTGmAvVsGFD6tevz5QpU4iIiOCbb75h6NCh1KxZk3r16tGkSRP69+8PwGWXXcY777xDjx49qF27NnXr1mXPnj0BroFv7BrBBdiz5wT9+y/k669/pWHDMsyd25UGDcoEOixjzAU4ceLEWcMzZ84887pevXosWrQo3Wmvv/56rr8++O4KtERwAbp1m8nKlXt57bUreeyxJuTLZydYxpjgY4ngHO3ceZQSJSKIjAxj5Mi2RETko2bNEoEOyxhjzpsdwvooJUUZOXINUVETeP75JQA0aFDGkoAxJujZGYEPtmw5SO/e81m69E86dKjMo48G549GjDHGG0sEmZgyZQt33fUthQvn59NPr+P22+sE/Y9HjDHGkyWCdKSkKHnyCE2aXMytt9bgrbdac9FFhQIdljHGZDm7RpBGXFwiAwcu5pZbvkFVqVatGBMndrIkYEyI2LdvH7fddhtVq1alcePGtGjRgmnTpvl1matWreLhhx/26zIyYonAw48/xtKgwae8/voKSpaMIDExJdAhGWOykarSpUsXWrVqxY4dO1i9ejVTpkwhNjbWr8uNjo5mxIgRfl1GRqxpCDh+PIGBAxczZkwMVaoU5bvvbqVdu0qBDsuYkNe69ZR/vdetW00efLAhp04l0rHjV/8a36tXXXr1qsuBA6fo2nXGWeMWLeqe4fL+97//ERYWRt++fc+8V6lSJR566CEmTJjAqlWrGDVqFOD8eOzxxx+ndevWzJ8/nxdffJHTp09TrVo1xo8fT+HChRk4cCAzZswgX758XHPNNQwbNowvv/ySwYMHkzdvXooWLcrixYtZtGgRw4YNY9asWQwaNIhdu3axY8cOdu3axSOPPHLmbGHIkCFMmjSJChUqUKpUKRo3bpwlD7+xRAAkJiYzffp2HnmkMUOHtqRQIesfyJhQtHHjRho1OrdHxx44cIChQ4eyYMECChUqxOuvv87bb79N//79mTZtGlu2bEFEzjzp7KWXXmLevHmUK1furKefedqyZQvff/89x48fp2bNmjzwwAOsXbuWr776il9++YWkpCQaNWqUZd1eh2wiOHgwjnffXc0LL1xOiRIRbNlyj3UQZ0wOk9ERfMGC+TMcX6pUwUzPADLTr18/lixZQlhYGP369fNaZvny5WzatImWLVsCkJCQQIsWLShSpAjh4eH07t2bTp06nel6omXLlvTq1Ytu3bpx8803e51np06dKFCgAAUKFKBMmTLs27ePJUuW0LlzZyIiIgC44YYbLqhunvx6jUBEOojIVhHZLiIDvYwXERnhjl8nIn5/iruq8uWXW6lTZzyvvrqCn376C8CSgDGGqKgo1qxZc2Z49OjRLFy4kP3795MvXz5SUv65bhgfHw84+5T27dsTExNDTEwMmzZt4qOPPiJfvnysWLGCW265henTp9OhQwcA3nvvPYYOHcru3btp0KABBw8e/FccBQr802ll3rx5SUpKOuvZyFnNb4lARPICo4HrgDpADxGpk6bYdUB1968PMNZf8QD8dbgIN9/xC926zaRChUhWrbqdK68s789FGmOCyNVXX018fDxjx/6zKzp16hQAlStXJiYmhpSUFHbv3s2KFSsAaN68OUuXLmX79u1nym/bto0TJ05w9OhROnbsyDvvvENMTAwAv/32G82aNeOll16iVKlS7N6926fYrrjiCmbOnEl8fDwnTpxg9uzZWVZvfzYNNQW2q+oOABGZAnQGNnmU6Qx8qk6qWy4ixUSkrKr6pe/WbiNuZ/Wu/bzxRisefTTaOokzxpxFRJg+fTqPPvoob7zxBqVLlz7T7t+yZUuqVKlCvXr1qFu37plrCaVLl2bChAn06NGD06dPAzB06FAiIyPp3Lkz8fHxqCrDhw8HnOcW/Prrr6gqbdu2pX79+vzwww+ZxtakSRNuvPFG6tevT6VKlYiOjqZo0aJZU29/nW6ISFegg6r2dofvAJqpan+PMrOA11R1iTu8EHhKVVelmVcfnDMGKlas2Hjnzp3nHtD+n1g76z0iGj5FjQZpT0yMMTnB5s2bqV27dqDDyLFOnDhB4cKFOXXqFK1atWLcuHFeL257W48islpVo73N159nBN76YUibdXwpg6qOA8YBREdHn1/mKt2C+ne3OK9JjTEmJ+jTpw+bNm0iPj6eu+6665zvcEqPPxNBLFDBY7g88Nd5lDHGGANMnjzZL/P1ZyP5SqC6iFQRkTCgOzAjTZkZwJ3u3UPNgaP+uj5gjAkO/rw7JhScz/rz2xmBqiaJSH9gHpAX+FhVN4pIX3f8e8AcoCOwHTgF3O2veIwxOV94eDgHDx6kZMmS1svveVBVDh48SHh4+DlN57eLxf4SHR2tq1atyrygMSboJCYmEhsbe+YefXPuwsPDKV++PPnz5z/r/UBdLDbGmHOSP39+qlSpEugwQo7dSG+MMSHOEoExxoQ4SwTGGBPigu5isYjsB87jp8UAlAIOZGE4wcDqHBqszqHhQupcSVVLexsRdIngQojIqvSumudWVufQYHUODf6qszUNGWNMiLNEYIwxIS7UEsG4QAcQAFbn0GB1Dg1+qXNIXSMwxhjzb6F2RmCMMSYNSwTGGBPicmUiEJEOIrJVRLaLyEAv40VERrjj14lI1jzdIYB8qHNPt67rRGSZiNQPRJxZKbM6e5RrIiLJ7lPzgpovdRaR1iISIyIbRSTzZyDmcD58touKyEwRWevWOah7MRaRj0XkbxHZkM74rN9/qWqu+sPp8vo3oCoQBqwF6qQp0xH4FucJac2BnwMddzbU+XKguPv6ulCos0e5/+F0ed410HFnw3YuhvNc8IrucJlAx50NdX4GeN19XRo4BIQFOvYLqHMroBGwIZ3xWb7/yo1nBE2B7aq6Q1UTgClA5zRlOgOfqmM5UExEymZ3oFko0zqr6jJVPewOLsd5Glww82U7AzwEfAX8nZ3B+Ykvdb4N+FpVdwGoarDX25c6KxApzgMMCuMkgqTsDTPrqOpinDqkJ8v3X7kxEZQDdnsMx7rvnWuZYHKu9bkX54gimGVaZxEpB9wEvJeNcfmTL9u5BlBcRBaJyGoRuTPbovMPX+o8CqiN85jb9cB/VDUle8ILiCzff+XG5xF4e6xR2ntkfSkTTHyuj4i0wUkEV/g1Iv/zpc7vAE+panIuedqVL3XOBzQG2gIRwE8islxVt/k7OD/xpc7XAjHA1UA14DsR+VFVj/k5tkDJ8v1XbkwEsUAFj+HyOEcK51ommPhUHxG5DPgQuE5VD2ZTbP7iS52jgSluEigFdBSRJFWdni0RZj1fP9sHVPUkcFJEFgP1gWBNBL7U+W7gNXUa0LeLyO9ALWBF9oSY7bJ8/5Ubm4ZWAtVFpIqIhAHdgRlpyswA7nSvvjcHjqrqnuwONAtlWmcRqQh8DdwRxEeHnjKts6pWUdXKqloZmAo8GMRJAHz7bH8DXCki+USkINAM2JzNcWYlX+q8C+cMCBG5CKgJ7MjWKLNXlu+/ct0ZgaomiUh/YB7OHQcfq+pGEenrjn8P5w6SjsB24BTOEUXQ8rHOLwAlgTHuEXKSBnHPjT7WOVfxpc6qullE5gLrgBTgQ1X1ehtiMPBxOw8BJojIepxmk6dUNWi7pxaRz4HWQCkRiQVeBPKD//Zf1sWEMcaEuNzYNGSMMeYcWCIwxpgQZ4nAGGNCnCUCY4wJcZYIjDEmxFkiMDmS21tojMdf5QzKnsiC5U0Qkd/dZa0RkRbnMY8PRaSO+/qZNOOWXWiM7nxS18sGt8fNYpmUbyAiHbNi2Sb3sttHTY4kIidUtXBWl81gHhOAWao6VUSuAYap6mUXML8Ljimz+YrIJ8A2VX05g/K9gGhV7Z/VsZjcw84ITFAQkcIistA9Wl8vIv/qaVREyorIYo8j5ivd968RkZ/cab8Ukcx20IuBS91pB7jz2iAij7jvFRKR2W7/9xtE5P/c9xeJSLSIvAZEuHFMcsedcP//1/MI3T0TuUVE8orImyKyUpw+5u/3YbX8hNvZmIg0Fec5E7+4/2u6v8R9Cfg/N5b/c2P/2F3OL97WowlBge572/7sz9sfkIzTkVgMMA3nV/BF3HGlcH5VmXpGe8L9/xjwrPs6LxDpll0MFHLffwp4wcvyJuA+rwC4FfgZp/O29UAhnO6NNwINgVuADzymLer+X4Rz9H0mJo8yqTHeBHzivg7D6UUyAugDPOe+XwBYBVTxEucJj/p9CXRwh4sA+dzX7YCv3Ne9gFEe078C3O6+LobTB1GhQG9v+wvsX67rYsLkGnGq2iB1QETyA6+ISCucrhPKARcBez2mWQl87JadrqoxInIVUAdY6natEYZzJO3NmyLyHLAfp4fWtsA0dTpwQ0S+Bq4E5gLDROR1nOakH8+hXt8CI0SkANABWKyqcW5z1GXyz1PUigLVgd/TTB8hIjFAZWA18J1H+U9EpDpOT5T501n+NcCNIvK4OxwOVCS4+yMyF8gSgQkWPXGePtVYVRNF5A+cndgZqrrYTRSdgM9E5E3gMPCdqvbwYRlPqOrU1AERaeetkKpuE5HGOP29vCoi81X1JV8qoarxIrIIp+vk/wM+T10c8JCqzstkFnGq2kBEigKzgH7ACJz+dr5X1ZvcC+uL0plegFtUdasv8ZrQYNcITLAoCvztJoE2QKW0BUSkklvmA+AjnMf9LQdaikhqm39BEanh4zIXA13caQrhNOv8KCKXAKdUdSIwzF1OWonumYk3U3A6CrsSpzM13P8PpE4jIjXcZXqlqkeBh4HH3WmKAn+6o3t5FD2O00SWah7wkLinRyLSML1lmNBhicAEi0lAtIiswjk72OKlTGsgRkR+wWnHf1dV9+PsGD8XkXU4iaGWLwtU1TU41w5W4Fwz+FBVfwHqASvcJppngaFeJh8HrEu9WJzGfJzn0i5Q5/GL4DwnYhOwRpyHlr9PJmfsbixrcbpmfgPn7GQpzvWDVN8DdVIvFuOcOeR3Y9vgDpsQZ7ePGmNMiLMzAmOMCXGWCIwxJsRZIjDGmBBnicAYY0KcJQJjjAlxlgiMMSbEWSIwxpgQ9/9oFExm6C0vmQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "def plot_roc_curve(fpr, tpr):\n",
    "    \"\"\"\n",
    "    Plots a ROC curve given the false positve rate (fpr) and \n",
    "    true postive rate (tpr) of a classifier.\n",
    "    \"\"\"\n",
    "    # Plot ROC curve\n",
    "    plt.plot(fpr, tpr, color='orange', label='ROC')\n",
    "    # Plot line with no predictive power (baseline)\n",
    "    plt.plot([0, 1], [0, 1], color='darkblue', linestyle='--', label='Guessing')\n",
    "    # Customize the plot\n",
    "    plt.xlabel('False Positive Rate')\n",
    "    plt.ylabel('True Positive Rate')\n",
    "    plt.title('Receiver Operating Characteristic (ROC) Curve')\n",
    "    plt.legend()\n",
    "    plt.show()\n",
    "    \n",
    "plot_roc_curve(fpr, tpr)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.9304956896551724"
      ]
     },
     "execution_count": 70,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.metrics import roc_auc_score\n",
    "\n",
    "roc_auc_score(y_test, y_probs)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "\n",
    "* The most ideal position for a ROC curve to run along the top left corner of the plot.\n",
    "\n",
    "* This would mean the model predicts only true positives and no false positives. And would result in a ROC AUC score of 1.0.\n",
    "\n",
    "* You can see this by creating a ROC curve using only the y_test labels."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEWCAYAAABrDZDcAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8vihELAAAACXBIWXMAAAsTAAALEwEAmpwYAAA6bElEQVR4nO3de5wN9f/A8dc7rF2sOyV3cl1C1i0SUYmKSr6ki0pS1Ld8u6h+leJbKUVyKV3oQr6lkluIkpBcat0lKWxR635de3v//phZHdvZ3YOdPXv2vJ+Pxz72zJnPzLw/M+fMe+Yzcz4jqooxxpjwdU6wAzDGGBNclgiMMSbMWSIwxpgwZ4nAGGPCnCUCY4wJc5YIjDEmzFkiyGNEZIOItAt2HHmFiDwuIm8FadmTRGRYMJad00Skt4jMP8Npz/gzKSJLRaTJmUx7pkTkfhF5ITeXGeosEWRBRH4TkeMickREdrs7hmJeLlNVY1R1kZfLSCcihUXkeRHZ4dbzZxF5WEQkN5bvJ552IhLv+56qPqeqfT1anrg7jfUiclRE4kXkYxFp6MXyzpSIDBGRD85mHqo6WVWvCGBZ/0h+Z/qZFJFrgMOq+qM7PEREkt3v0wERWSYirTJMU1JExrvft2Misk5Ebvcz75tEZJU7r10i8oWItHFHTwBuFpHyWcQWEts+t1giyN41qloMaAw0AR4LbjinT0QKZjLqY6AD0BmIBm4B+gGvehCDiEhe+7y9CvwbuB8oDdQGpgNdcnpBWWwDzwVx2f2B9zO89z/3+1QW+BrnMwiAiEQAC4CqQCugBPAw8IKIDPIpNwgYBTwHnAtUAcYBXQFUNRH4Arg1i9hybNsHc9vmGFW1v0z+gN+Ajj7DLwKzfYZbAsuAA8AaoJ3PuNLAROAPYD8w3Wfc1UCcO90y4MKMywTOB44DpX3GNQH2AIXc4TuATe785wFVfcoqMAD4GfjVT906AIlA5QzvtwBSgQvc4UXA88AK4CDweYaYsloHi4D/AkvdulwA3O7GfBjYBtztli3qlkkDjrh/5wNDgA/cMtXcet0G7HDXxRM+y4sC3nXXxybgESA+k21by61n8yy2/yRgLDDbjfd7oKbP+FeBncAhYDVwic+4IcA04AN3fF+gOfCdu652AWOACJ9pYoAvgX3An8DjQCcgCUh218kat2wJ4G13Pr8Dw4AC7rg+7jof6c5rmPveEne8uOP+crfpWqABzkFAsru8I8DMjN8DoIAb1y/uOllNhs+QWy7C3Z6VMqyTD3yG67vbs5w7fKcbU9EM8/qXG09xt95HgBuz+e72Br4+i22/COjrM3xy/fn7fgGvAyMyzONzYJD7+nzgEyDBLX9/sPdvp8Qa7ADy8l+GL0AlYB3wqjtcEdiLczR9DnC5O5z+oZ4N/A8oBRQCLnXfv8j9sLdwv1S3ucsp7GeZXwF3+cTzEvC6+7obsBWoBxQE/g9YluGD+iVOQoryU7cXgG8yqfd2/t5BL8LZ0TTA2Vl/wt875uzWwSKcHXaMG2MhnCOumjg7o0uBY8BFbvl2ZNhx4z8RvImz028EnADq+dbJXeeVcHZwmSWC/sD2bLb/JJwdaXM3/snAVJ/xNwNl3HH/AXYDkT5xJ7vb6Rw33qY4ibOgW5dNwANu+Wicnfp/gEh3uEXGdeCz7OnAG+42KY+TqNO3WR8gBbjPXVYUpyaCK3F24CXd7VAPqOBT52FZfA8exvke1HGnbQSU8bPuYoCjWWzLCHd77QEKuu9NBd71M6+Cbn2uxEmMKenTZLHtLgL2ncW2X0T2ieDk9wtoi3NQIO74UjiJ8Hx3+68GnnLrXQPnIOjKYO/j0v/y2ql6XjRdRA7jbOS/gKfd928G5qjqHFVNU9UvgVVAZxGpAFwF9FfV/aqarKrfuNPdBbyhqt+raqqqvouzM2vpZ9lTgF7gNK0APd33AO4GnlfVTaqagnOa3FhEqvpM/7yq7lPV437mXRZnx+PPLnd8uvdVdb2qHgWeBHqISIGs1oHPtJNUdYOqprjrYbaq/qKOb4D5wCWZxJGZZ1T1uKquwTkLaeS+3wN4zl3n8cDoLOZRJov6+/pUVVe463gyThMhAKr6garudev2MlAYZweZ7jtVne6um+OqulpVl7vlf8PZkV/qlr0a2K2qL6tqoqoeVtXv/QUkIufifL4eUNWjqvoXzhF+T59if6jqa+6yMm7/ZJxEUxdnx7VJVQNZF+Cc2fyfqv7kbsM1qrrXT7mSOGcMGfUQkQM4O8m7gO7uuoVMPpPu+D3u+DLAHp9pMnMY5+zBn0C3fXZ8v1/f4iSH9M9yd5zt/wfQDOfg6FlVTVLVbTgHMz39zjUILBFkr5uqRuMcrdbl7x1kVeBG96LXAffD3QaoAFTGORrZ72d+VYH/ZJiuMs6RQ0bTgFYicj7OEYfifODS5/Oqzzz24RyhVfSZfmcW9drjxupPBXe8v/lsxzmyL0vW68BvDCJylYgsF5F9bvnOnJp0ArHb5/UxIP0C/vkZlpdV/feSef0DWRYi8h8R2SQiB926lODUumSse20RmeVeCD2Ek7zTy1fGaW4JRFWcbbDLZ72/gXNm4HfZvlT1K5xmqbHAnyIyQUSKB7jsQOPcj5NsMvpIVUvitO2vxzlLSuf3M+m2wZd1x+8FygbQLh+N0+zlT6DbPjsn17E6pwFTcQ/cgJtwDhzA2V7nZ/iePI6zDvIESwQBco9eJwEj3Ld24hwpl/T5K6qqL7jjSotIST+z2gn8N8N0RVT1Qz/LPIBzxNwD54P1ofuBS5/P3RnmE6Wqy3xnkUWVFgAtRKSy75si0hzny/6Vz9u+ZargHFHuyWYd/CMGESmM07Q0AjjX3SHMwUlg2cUbiF04TUL+4s5oIVBJRGLPZEEicgnwKM62KeXW5SB/1wX+WZ/xwGaglqoWx9kZpJffidNk5k/G+ezEOYss67Pei6tqTBbTnDpD1dGq2hSnCac2TpNPttNlE6evn3FOZCv6G6mqe3DOaoe4Z9DgfCavEpGiGYrfgFPf5TjXWBJxmtyyUg/nbNGfQLb9UaCIz/B5fspkXFcfAt3ds/IWOJ91cNbZrxm+J9Gq2pk8whLB6RkFXC4ijXEuAl4jIleKSAERiXRvf6zknmZ/AYwTkVIiUkhE2rrzeBPoLyIt3DtpiopIFxHxd/QETlPQrThfhik+778OPCYiMQAiUkJEbgy0Iqq6AOcL8YmIxLh1aIlzFDNeVX/2KX6ziNQXkSLAs8A0VU3Nah1kstgInOaTBCBFRK4CfG9p/BMoIyKZndJn5yOcdVLK3QENzKygW79xwIduzBFu/D1FZHAAy4rGaatOAAqKyFM4FzOzm+YQcERE6gL3+IybBZwnIg+Ic1tvtIi0cMf9CVRLv+vK/XzNB14WkeIico6I1BSRSwmAiDRzP3+FcHZ4iTgXT9OXVSOLyd8ChopILffze6GIlMlYSFWTcXbsmcakqptxbnJ4xH3rfSAe+FhEqrnfmytxmviGqOpBVT2I09Y+VkS6iUgRt9xVIvKiz+wvxfkO+ltuINs+Drjenf8FOBeys6TObbIJ7jqa5x7IgXP95pCIPCoiUe53pYGINMtunrnFEsFpUNUE4D3gSVXdiXO72uM4G38nzlFV+jq9BefIeTPOtYUH3HmswmkbHYNz+rwV50JUZmbg3OXwp9smnh7LZ8BwYKrbzLAep934dNyAcwvfXJw7MT7AuRPlvgzl3sc5G9qNcyHzfjeG7NbBKVT1sDvtRzh1v8mtX/r4zThHVdvcU2h/zWVZeRZnR/Irzk5oGs6RZGbu5+8mkgM4TR7XATMDWNY8nB3NFpzmskSybooCeAinzodxDgj+lz7CXTeXA9fgrOefgfbu6PRbLPeKyA/u61txEutGnHU5jcCbO4q7y9/vxr6Xv8903wbqu+t/up9pX8HZfvNxktrbOBdL/XkD53uQlZeAfiJSXlVP4NwxtxPnDq1D7vKeUNWX0idQ1VeAQTg3SKR/7gbiXEBHRCJxmhzfzWK52W37kTh3T/3pzmfyP2fh14duHU4etLkHTdfgXF/6Feds+i0yv4aR69KvcBvjl4gswrnTIyi/7j0bInIP0FNVAzpSNjlPRJYA97lHy7m1zPtwbml9JNvCBnBuyzImX3DbmmvgtCPXwrkVc0xQgwpzqtom+1I5vszXcnuZoc4SgclPInCaI6rjnO5PxWkLNsZkwZqGjDEmzNnFYmOMCXMh1zRUtmxZrVatWrDDMMaYkLJ69eo9qlrO37iQSwTVqlVj1apVwQ7DGGNCiohsz2ycNQ0ZY0yYs0RgjDFhzhKBMcaEOUsExhgT5iwRGGNMmPMsEYjIOyLyl4isz2S8iMhoEdkqImtF5CKvYjHGGJM5L88IJuE8Vi4zV+H0B1ML51mp4z2MxRhjTCY8+x2Bqi4WkWpZFOkKvOc+aGW5iJQUkQqn8ci807N1Avw2JftyxhiTxySnCL/uLkLthrWg6agcn38wrxFU5NT+2+M59TGLJ4lIPxFZJSKrEhISzmxpv02B/XFnNq0xxgTJj1uL03xgG9o/1IqjxyX7Cc5AMH9Z7K9GfnvAU9UJwASA2NjYM+8lr1Rj6LjojCc3xpjckpiYwjPPLOOll1ZStmwU4yZ0pGib2p4sK5iJIJ5TnylbCfgjSLEYY0ye0q3bdObN+43bb2/Ayy+3o1SpSM+WFcymoRnAre7dQy2Bg55dHzDGmBBw+HASiYkpAAwe3Jz587vzzjudPE0C4OEZgYh8CLQDyopIPPA0UAhAVV8H5uA8V3QrcAy43atYjDEmr5s371f69ZvPzTfX57//vYR27ark2rK9vGuoVzbjFRjg1fKNMSYU7Nt3nEGDFvHuuxuoW7c0XbrUyPUYQq4bamOMyS8WLtxO796z2bs3kSeeaMn//V9LIiNzf7dsicAYY4KkfPkiVK9egrlzu9O4cfmgxWF9DRljTC5RVSZNWs/99y8EoGHDcixbdlNQkwBYIjDGmFzx668HuPLKadx++1zi4hI4fjwZABFvfiR2OqxpyBhjPJSamsbYsXE89thizjlHGDeuI3ff3Yhzzgl+AkhnicAYYzy0Z89xnnpqKZdeWpnXX7+cKlWKBzukf7BEYIwxOSw5OZXJkzdx660xnHtuUX744RaqVy+RJ5qB/LFEYIwxOWj16t3cccc81q5NoEKFolx5ZXVq1CgZ7LCyZBeLjTEmBxw/nszgwYtp0WIyCQnH+Oyzrlx5ZfVghxUQOyMwxpgc0K3b58yf/xt9+zbkpZcupWRJb/sHykmWCIwx5gwdOnSCiIgCREYW5PHHW/DII83o0KFqsMM6bdY0ZIwxZ2DOnG00aDCJZ5/9DoBLL60ckkkALBEYY8xp2bPnGLfcMocuXT4lOjqCa6+tGeyQzpo1DRljTIC+/PI3eveezf79J3jqqVY8/ngLChcO/d1o6NfAGGNySYUKRalduzTjx3ekYcNywQ4nx1jTkDHGZEJVeeuttQwYsACABg3K8e23PfNVEgBLBMYY49e2bQfo2PFj7rprPhs37s1TncTlNEsExhjjIzU1jZEjV9GgwSRWrtzNG29czsKFPYiKKhTs0Dxj1wiMMcbHnj3HeeaZ7+jQoQrjx19OpUrRwQ7Jc5YIjDFhLykplQ8+2EifPg0499yixMXdStWqxfNlM5A/lgiMMWFt5cpd3HHHPNav30OlStFccUU1qlUrEeywcpVdIzDGhKVjx5J56KFFtGw5hf37E5kx4zquuKJasMMKCjsjMMaEpa5dp7NgwXb69buQF1+8lBIlCgc7pKCxRGCMCRsHD56gcGGnk7gnn2zJ44+3oH37KsEOK+isacgYExZmzfqFmJiJPPPMMgDatq1sScBlicAYk68lJBzjpptmcc01n1G6dCTXX18r2CHlOdY0ZIzJt+bPdzqJO3jwBM88czGDB7cgIqJAsMPKcywRGGPyrYoVi1GvXhnGj+9ITEzZYIeTZ1nTkDEm30hLUyZMWMM993wJQExMWRYv7mlJIBuWCIwx+cLWrfvp0OEj7r77S376ad/JTuJM9iwRGGNCWmpqGi+/vJILL3yXH374kzffvCLfdxKX0zxNBCLSSUR+EpGtIjLYz/gSIjJTRNaIyAYRud3LeIwx+c+ePccZNmw5l19elY0bb6dv3wvDpo+gnOJZIhCRAsBY4CqgPtBLROpnKDYA2KiqjYB2wMsiEuFVTMaY/OHEiRTefHMtaWl6spO46dO7UbFi/u8p1AtenhE0B7aq6jZVTQKmAl0zlFEgWpz0XQzYB6R4GJMxJsR9//0umjZ9n3795rNgwXYAqlYtYWcBZ8HLRFAR2OkzHO++52sMUA/4A1gH/FtV0zLOSET6icgqEVmVkJDgVbzGmDzs6NEkBg36mlatJnPwYBKzZ18ftp3E5TQvE4G/9KwZhq8E4oDzgcbAGBEp/o+JVCeoaqyqxpYrl7+eFWqMCUy3bp8zcuRq+vdvxIYNfejcuUawQ8o3vEwE8UBln+FKOEf+vm4HPlXHVuBXoK6HMRljQsiBA4knbwN96qlWfPPNvxg37nKKFw/fnkK94GUiWAnUEpHq7gXgnsCMDGV2AB0ARORcoA6wzcOYjDEhYsaMrcTETOKZZ74D4JJLKtG2beVspjJnwrNEoKopwEBgHrAJ+EhVN4hIfxHp7xYbClwsIuuAhcCjqrrHq5iMMXnfX38dpWfPmXTtOp2yZaPo3r12sEPK9zzta0hV5wBzMrz3us/rP4ArvIzBGBM65s79ld69Z3PkSDJDh7bm0UebU6iQdRLnNet0zhiTZ1SuHE3DhmUZN64j9etb/0C5xbqYMMYETVqaMn58HHffPR9wOolbtKinJYFcZonAGBMUW7bso127/3HvvQv49deDJCbab0mDxRKBMSZXpaSkMXz491x44busW5fAxImdmDevO5GR1lIdLLbmjTG5au/e4wwfvpLOnWswdmwHKlQoFuyQwp4lAmOM506cSGHSpA3cddeFnHtuUdasuZXKlf/RiYAJEksExhhPfffdH9x551w2bdpHzZol6dixqiWBPMauERhjPHHkSBIPPPAVrVtP4ejRZObOvYGOHasGOyzjh50RGGM80a3bdBYu3MHAgU147rlLiI62R43kVZYIjDE5Zv/+RCIjCxAVVYghQy5myJCLadOmUrDDMtkIuGlIRIp6GYgxJrR9+ukW6tefyJAhywBo06aSJYEQkW0iEJGLRWQjTsdxiEgjERnneWTGmJCwe/dRunf/nBtumMF55xWlZ0/rST7UBNI0NBLnATIzAFR1jYi09TQqY0xI+OKLbfTuPYdjx5J57rlLeOihWOskLgQFdI1AVXdmeB5oqjfhGGNCSdWqxWnSpDxjx3agbt0ywQ7HnKFArhHsFJGLARWRCBF5CLeZyBgTXtLSlDFjfuCuu+YBUL9+WRYu7GFJIMQFkgj6AwNwHjwfj/Ns4Xs9jMkYkwf99NM+2radyn33fcXOnYetk7h8JJCmoTqq2tv3DRFpDSz1JiRjTF6SnJzKiBGreOaZZRQpUohJkzpx660xZGguNiEskDOC1wJ8zxiTD+3fn8hLL63kmmtqsnHj7dx2WwNLAvlMpmcEItIKuBgoJyKDfEYVB+y2AGPyscTEFN55Zx39+zemfPmirF17G5UqRQc7LOORrJqGIoBibhnfT8AhoLuXQRljgmfJknjuvHMeW7bsp3bt0nTsWNWSQD6XaSJQ1W+Ab0Rkkqpuz8WYjDFBcPhwEo89tpixY+OoVq048+d3t07iwkQgF4uPichLQAwQmf6mql7mWVTGmFzXrdt0vv56B//+90UMG9aGYsWsk7hwEUgimAz8D7ga51bS24AEL4MyxuSOffuOExlZkCJFCjF0aGtE2tCq1fnBDsvkskDuGiqjqm8Dyar6jareAbT0OC5jjMemTfuJevX+7iTu4osrWhIIU4EkgmT3/y4R6SIiTQDrUtCYELVr1xGuv/5zbrxxJpUrR9O7d71gh2SCLJCmoWEiUgL4D87vB4oDD3gZlDHGG7Nn/8LNN88hMTGV4cPbMmhQLAUL2oMKw122iUBVZ7kvDwLt4eQvi40xIaZGjZI0a3YeY8Z0oHbt0sEOx+QRWf2grADQA6ePobmqul5ErgYeB6KAJrkTojHmTKWmpjFmzI+sXZvA2293ol69Msyff2OwwzJ5TFZnBG8DlYEVwGgR2Q60Agar6vRciM0YcxY2btxD377z+e67P+jcuTqJiSlERtrTac0/ZfWpiAUuVNU0EYkE9gAXqOru3AnNGHMmkpJSefHFFQwdupzo6Ag++KAzN91Uz/oHMpnK6ipRkqqmAahqIrDldJOAiHQSkZ9EZKuIDM6kTDsRiRORDSLyzenM3xjzTwcOJDJy5Gquu+4CNm7sQ+/e9S0JmCxldUZQV0TWuq8FqOkOC6CqemFWM3avMYwFLsd5jsFKEZmhqht9ypQExgGdVHWHiJQ/86oYE76OH0/m7bfXce+9TShfvijr1vXh/POLBTssEyKySgRne3Nxc2Crqm4DEJGpQFdgo0+Zm4BPVXUHgKr+dZbLNCbsLF68k7595/Pzz/upV68MHTpUtSRgTkumTUOquj2rvwDmXRHY6TMc777nqzZQSkQWichqEbnV34xEpJ+IrBKRVQkJ1ruFMQCHDp3g3nu/5NJL/0dKShoLFtxIhw7WSZw5fV7eQuCvUVL9LL8p0AHnltTvRGS5qm45ZSLVCcAEgNjY2IzzMCYsdes2nUWLdvLgg00ZOrQ1RYtaJ3HmzHiZCOJxbj9NVwn4w0+ZPap6FDgqIouBRsAWjDH/sGfPMYoUKUSRIoX4738vQQRatrT+gczZCei35SISJSJ1TnPeK4FaIlJdRCKAnsCMDGU+By4RkYIiUgRoAWw6zeUYk++pKlOnbqZevYk8/bTzuPBWrc63JGByRLaJQESuAeKAue5wYxHJuEP/B1VNAQYC83B27h+p6gYR6S8i/d0ym9z5rsX54dpbqrr+DOtiTL70+++H6dZtOr16zaJ69RLcemtMsEMy+UwgTUNDcO4AWgSgqnEiUi2QmavqHGBOhvdezzD8EvBSIPMzJtzMmvULvXvPJjk5jREjLuWBB5pSoIB1EmdyViCJIEVVD9oPUozJfRdcUJKLLz6f117rwAUXlAp2OCafCuTQYr2I3AQUEJFaIvIasMzjuIwJS6mpaYwcuYo+fb4AoG7dMnzxRXdLAsZTgSSC+3CeV3wCmILTHfUDHsZkTFjasGEPrVt/yKBBi9iz5ziJiSnBDsmEiUCahuqo6hPAE14HY0w4SkpK5YUXvmfYsOWUKFGYKVO60LNnXesfyOSaQBLBKyJSAfgYmKqqGzyOyZiwcuBAIqNH/8iNN9Zh1Kj2lCtXJNghmTCTbdOQqrYH2gEJwAQRWSci/+d1YMbkZ8eOJfPqq6tJTU1zO4m7jcmTu1gSMEER0H1oqrpbVUcD/XF+U/CUl0EZk599/fUOGjacxAMPfM2iRU53XBUqWCdxJngC+UFZPREZIiLrgTE4dwxV8jwyY/KZgwdPcPfd87nsso8QEb7+uod1EmfyhECuEUwEPgSuUNWMfQUZYwLUrdt0Fi+O5+GHmzFkyMUUKVIo2CEZAwSQCFS1ZW4EYkx+lJBwjKJFnU7inn/+EgoUEJo1qxDssIw5RaZNQyLykft/nYis9flb5/PkMmOMH6rKlCmbTukkrmXL8y0JmDwpqzOCf7v/r86NQIzJL+LjD3PPPV8ya9Y2WrSoQJ8+DYIdkjFZyuoJZbvcl/f6eTrZvbkTnjGhZcaMrdSvP5GvvtrByJHtWbq0FzExZYMdljFZCuT20cv9vHdVTgdiTH5Qu3Yp2rSpyLp1faynUBMyMm0aEpF7cI78a2S4JhANLPU6MGNCQUpKGqNGrWbt2gTee68zdeuWYc6cG4IdljGnJatrBFOAL4DngcE+7x9W1X2eRmVMCFi7NoE775zLqlV/0rXrBSQmphAZ6eXTX43xRlafWlXV30RkQMYRIlLakoEJVydOpPDcc9/z3HPfU7p0JB99dA3du9e2TuJMyMrujOBqYDWggO+nXIEaHsZlTJ516FAS48bF0atXXUaObE+ZMlHBDsmYs5JpIlDVq93/1XMvHGPypqNHk5gwYS33338R5coVYf36Ppx7btFgh2VMjgikr6HWIlLUfX2ziLwiIlW8D82YvGHhwu00bPgugwYt4ptv4gEsCZh8JZB728YDx0SkEfAIsB1439OojMkDDhxIpG/feXTs+DEFC57DN9/8i8sus2Mgk/8E+vB6FZGuwKuq+raI3OZ1YMYE23XXfc6338bz6KPNefrpVkRFWSdxJn8KJBEcFpHHgFuAS0SkAGDfCJMv/fnnUYoVK0TRohG88EJbChYUmjY9L9hhGeOpQJqG/oXz4Po7VHU3UBF4ydOojMllqsr772+gfv2JPP30MgBatKhgScCEhUAeVbkbmAyUEJGrgURVfc/zyIzJJTt2HKJLl0+59dYvqFOnNHfe2TDYIRmTqwK5a6gHsAK4EegBfC8i3b0OzJjc8PnnW4mJmcjixfGMHn0Z337bk3r1ygQ7LGNyVSDXCJ4AmqnqXwAiUg5YAEzzMjBjvKSqiAh165amXbvKvPZaB6pVKxHssIwJikCuEZyTngRcewOczpg8JyUljeHDv+eWW+YAUKdOaWbOvN6SgAlrgZwRzBWReTjPLQbn4vEc70Iyxhtr1vzFHXfM44cf/uS662pZJ3HGuAJ5ZvHDInI90Aanv6EJqvqZ55EZk0MSE1MYNmw5w4evoEyZSKZNu5Ybbqgd7LCMyTOyeh5BLWAEUBNYBzykqr/nVmDG5JTDh5N444019O5dj1deaUfp0tZJnDG+smrrfweYBdyA0wPpa6c7cxHpJCI/ichWERmcRblmIpJqdyOZnHLkSBIjRqwkNTWNcuWKsHHj7UyadJUlAWP8yKppKFpV33Rf/yQiP5zOjN1fII/FedRlPLBSRGao6kY/5YYD805n/sZkZv783+jXbz47dhyiadNzad++CuXKFQl2WMbkWVmdEUSKSBMRuUhELgKiMgxnpzmwVVW3qWoSMBXo6qfcfcAnwF9+xhkTsH37jnP77V9w5ZXTiIwsyLff9qJ9e+skzpjsZHVGsAt4xWd4t8+wApdlM++KwE6f4XighW8BEakIXOfOq1lmMxKRfkA/gCpV7Itt/Lvuus9ZuvR3Hn+8BU8+2cruCDImQFk9mKb9Wc7b33P7NMPwKOBRVU3N6jF/qjoBmAAQGxubcR4mjO3efZToaKeTuJdeupSIiAI0blw+2GEZE1K8/GFYPFDZZ7gS8EeGMrHAVBH5DegOjBORbh7GZPIJVWXSpPXUrz+Rp55aCkDz5hUsCRhzBrw8d14J1BKR6sDvQE/gJt8Cvo/BFJFJwCxVne5hTCYf+O23g9x995fMn/8bbdpUpF+/RsEOyZiQ5lkiUNUUERmIczdQAeAdVd0gIv3d8a97tWyTf3322c/ccsscRGDMmA7cc09jzjkn82ZFY0z2sk0E4jTe9wZqqOqz7vOKz1PVFdlNq6pzyNAdRWYJQFX7BBSxCUvpncTFxJShY8eqvPpqe6pWtf6BjMkJgVwjGAe0Anq5w4dxfh9gjOeSk1N57rnl9O49G4DatUszfXo3SwLG5KBAEkELVR0AJAKo6n4gwtOojAF++OFPmjefzBNPLCE1VTlxIiXYIRmTLwWSCJLdX/8qnHweQZqnUZmwdvx4Mo89tpjmzT9g9+6jfPZZV/73v2soXNh+F2CMFwL5Zo0GPgPKi8h/cW7z/D9PozJh7ejRZN5+ex233RbDiBHtKFUqMtghGZOvBdIN9WQRWQ10wPmRWDdV3eR5ZCasHD6cxPjxcfznP7GULet0Ele2rPUPZExuCOSuoSrAMWCm73uqusPLwEz4mDv3V+6+ez47dx6mefPzaNeuiiUBY3JRIE1Ds3GuDwgQCVQHfgJiPIzLhIG9e48zaNDXvPfeRurVK83SpTfRqtX5wQ7LmLATSNNQQ99ht+fRuz2LyISN66//nGXL/uDJJ1vyxBMt7WKwMUFy2t88Vf1BRDLtKdSYrOzadYTo6AiKFYtgxAink7hGjax/IGOCKZBrBIN8Bs8BLgISPIvI5EuqysSJ6xk0aBF33NGAV15pT7NmFYIdljGGwM4Ion1ep+BcM/jEm3BMfrRt2wHuvvtLFizYTtu2lejf3zqJMyYvyTIRuD8kK6aqD+dSPCaf+fTTLdxyyxwKFDiH8eM70q9fI+skzpg8JtNEICIF3R5EA3kspTGnSO8krmHDcnTqVJ1Ro9pTuXLxYIdljPEjqzOCFTjXA+JEZAbwMXA0faSqfupxbCYEJSWl8uKLK9iwYS9TpnShVq1SfPKJv0dVG2PyikCuEZQG9uI8Vzj99wQKWCIwp1i1ajd33jmPtWsT6NmzLklJqXZLqDEhIKtvaXn3jqH1/J0A0tlzg81Jx48n8/TTy3j55VWcd15RPv+8G9dee0GwwzLGBCirRFAAKEZgD6E3Yezo0WQmTVrPnXc25MUX21KypHUSZ0woySoR7FLVZ3MtEhNSDh06wbhxcTz8cDPKli3Cpk13UKZMVLDDMsacgawSgd3jZ/yaPfsX+vdfwB9/HKFlywq0a1fFkoAxISyrB9N0yLUoTEhISDhG796zufrqzyhRIoJly26iXbsqwQ7LGHOWMj0jUNV9uRmIyftuuGEGy5f/wZAhF/PYYy2IiCgQ7JCMMTnA7u0zWfr998OUKFGYYsUiGDmyHYULF6BBg3LBDssYk4MCeWaxCUOqyptvrqV+/Yk89dRSAJo2Pc+SgDH5kJ0RmH/45ZcD3HXXPL7+eift21dmwIAmwQ7JGOMhSwTmFNOm/cStt35BoULnMGHCFfTt2xARu4HMmPzMEoEB/u4krlGj8nTpUoORI9tTqVJ09hMaY0KeXSMIc0lJqTzzzDJ69pyFqlKrVik+/vhaSwLGhBFLBGFsxYpdNG36PkOGLKNgwXNISkoNdkjGmCCwRBCGjh1L5qGHFtGq1RT2709k5szrmDy5i/UUakyYsm9+GDp+PIUPPthIv34XMnx4W4oXLxzskIwxQeTpGYGIdBKRn0Rkq4gM9jO+t4isdf+WiYg9zNYjBw+e4L//XU5KShplykSxadMdjB9/uSUBY4x3icB93vFY4CqgPtBLROpnKPYrcKmqXggMBSZ4FU84mznzl5M/DFuyJB6AUqWsq2hjjMPLM4LmwFZV3aaqScBU4JRnFqrqMlXd7w4uByp5GE/YSUg4Rq9es7j22s8oUyaS77/vbZ3EGWP+wctrBBWBnT7D8UCLLMrfCXzhb4SI9AP6AVSpYjuyQKV3Evfss6159NHm1kmcMcYvLxNBwE82E5H2OImgjb/xqjoBt9koNjbWno6Whfj4w5Qs6XQSN2pUewoXLkBMTNlgh2WMycO8bBqKByr7DFcC/shYSEQuBN4CuqrqXg/jydfS0pQ33lhD/foTefJJp5O4iy4615KAMSZbXp4RrARqiUh14HegJ3CTbwERqQJ8Ctyiqls8jCVf+/nn/dx11zy++SaeDh2qcN991kmcMSZwniUCVU0RkYHAPKAA8I6qbhCR/u7414GngDLAOLdjsxRVjfUqpvzo44+dTuIKFy7A229fye23N7BO4owxp8XTH5Sp6hxgTob3Xvd53Rfo62UM+VV6J3FNmpSna9eavPJKe84/v1iwwzLGhCDrYiLEnDiRwlNPLaFHj5moKhdcUIqpU6+xJGCMOWOWCELI8uV/cNFF7zN06HKiogpaJ3HGmBxhiSAEHD2axIMPfs3FF0/h8OEk5sy5nvfe62ydxBljcoTtSUJAYmIqU6du5t57G/P8822Jjo4IdkjGmHzEEkEedeBAIq+99iOPPdbC7STudkqWtP6BjDE5z5qG8qDp03+mfv2JPPPMMpYt+x3AkoAxxjN2RpCH/PnnUe67byEff7yFRo3KMXPmdTRtel6wwzIm1yQnJxMfH09iYmKwQwlZkZGRVKpUiUKFCgU8jSWCPKR79xmsWLGbYcPa8MgjzShUyDqJM+ElPj6e6OhoqlWrZj+MPAOqyt69e4mPj6d69eoBT2eJIMh27DhEqVKRREdHMHr0ZRQuXID69a1/IBOeEhMTLQmcBRGhTJkyJCQknNZ0do0gSNLSlLFjfyQmxnlgDECTJudaEjBhz5LA2TmT9WdnBEHw00/76Nt3HkuW/M7ll1fl3/++KNghGWPCmJ0R5LKPPtpMo0bvsn79HiZO7MS8ed2pVq1EsMMyxrgKFChA48aNadCgAddccw0HDhw4OW7Dhg1cdtll1K5dm1q1ajF06FBU/35EyhdffEFsbCz16tWjbt26PPTQQ0GowemzRJBL0j8sTZuex/XX12LTpjvo08d6CjUmr4mKiiIuLo7169dTunRpxo4dC8Dx48e59tprGTx4MFu2bGHNmjUsW7aMcePGAbB+/XoGDhzIBx98wKZNm1i/fj01atQIZlUCZk1DHktMTGHo0O/YvHkf06ZdS82aJZky5epgh2VM3rf6Adgfl7PzLNUYmo4KuHirVq1Yu3YtAFOmTKF169ZcccUVABQpUoQxY8bQrl07BgwYwIsvvsgTTzxB3bp1AShYsCD33ntvzsbvETsj8NCyZb/TpMl7PPfc90RHR1gnccaEkNTUVBYuXMi1114LOM1CTZs2PaVMzZo1OXLkCIcOHWL9+vX/GB8q7IzAA0eOJPH4498yZsyPVK4czdy5N3DllYHf02uM4bSO3HPS8ePHady4Mb/99htNmzbl8ssvB/5+Bog/od7Ea2cEHkhKSmXatC0MGNCE9etvtyRgTAhJv0awfft2kpKSTl4jiImJYdWqVaeU3bZtG8WKFSM6OpqYmBhWr14djJDPmiWCHLJv33GGDFlKSkoapUtHsWnTHbz2WgfrKdSYEFWiRAlGjx7NiBEjSE5Opnfv3ixZsoQFCxYAzpnD/fffzyOPPALAww8/zHPPPceWLc7j19PS0njllVeCFv/psESQAz75ZAv1609k2LDlJzuJK1GicJCjMsacrSZNmtCoUSOmTp1KVFQUn3/+OcOGDaNOnTo0bNiQZs2aMXDgQAAuvPBCRo0aRa9evahXrx4NGjRg165dQa5BYOwawVnYtesIAwcu5NNPf6ZJk/LMndudxo3LBzssY8xZOHLkyCnDM2fOPPm6YcOGLFq0KNNpr776aq6+OvTuCrREcBZ69JjJypW7eeGFS/jPf5pRsKCdYBljQo8lgtO0fftBSpeOIjo6gtde60BUVEHq1Ckd7LCMMeaM2SFsgNLSlNde+4GYmEk8+eQSABo3Lm9JwBgT8uyMIACbN++lb9/5LF36O506VePBB0PzRyPGGOOPJYJsTJ26mdtu+4JixQrx3ntXcfPN9UP+xyPGGOPLEkEm0tKUc84RmjU7jxtvrM3LL7fj3HOLBjssY4zJcXaNIIPjx5MZPHgxN9zwOapKzZol+eCDLpYEjAkTf/75JzfddBM1atSgadOmtGrVis8++8zTZa5atYr777/f02VkxRKBj2+/jadx4/cYPnwFZcpEkZycFuyQjDG5SFXp1q0bbdu2Zdu2baxevZqpU6cSHx/v6XJjY2MZPXq0p8vIijUNAYcPJzF48GLGjYujevUSfPnljXTsWDXYYRkT9tq1m/qP93r0qMO99zbh2LFkOnf+5B/j+/RpQJ8+Ddiz5xjdu884ZdyiRT2zXN5XX31FREQE/fv3P/le1apVue+++5g0aRKrVq1izJgxgPPjsYceeoh27doxf/58nn76aU6cOEHNmjWZOHEixYoVY/DgwcyYMYOCBQtyxRVXMGLECD7++GOeeeYZChQoQIkSJVi8eDGLFi1ixIgRzJo1iyFDhrBjxw62bdvGjh07eOCBB06eLQwdOpTJkydTuXJlypYtS9OmTXPk4TeWCIDk5FSmT9/KAw80Zdiw1hQtav0DGROONmzYwEUXnd6jY/fs2cOwYcNYsGABRYsWZfjw4bzyyisMHDiQzz77jM2bNyMiJ5909uyzzzJv3jwqVqx4ytPPfG3evJmvv/6aw4cPU6dOHe655x7WrFnDJ598wo8//khKSgoXXXRRjnV7HbaJYO/e47z66mqeeupiSpeOYvPmO6yDOGPymKyO4IsUKZTl+LJli2R7BpCdAQMGsGTJEiIiIhgwYIDfMsuXL2fjxo20bt0agKSkJFq1akXx4sWJjIykb9++dOnS5WTXE61bt6ZPnz706NGD66+/3u88u3TpQuHChSlcuDDly5fnzz//ZMmSJXTt2pWoqCgArrnmmrOqmy9PrxGISCcR+UlEtorIYD/jRURGu+PXiojnT3FXVT7++Cfq15/I88+v4Lvv/gCwJGCMISYmhh9++OHk8NixY1m4cCEJCQkULFiQtLS/rxsmJiYCzj7l8ssvJy4ujri4ODZu3Mjbb79NwYIFWbFiBTfccAPTp0+nU6dOALz++usMGzaMnTt30rhxY/bu3fuPOAoX/rvTygIFCpCSknLKs5FzmmeJQEQKAGOBq4D6QC8RqZ+h2FVALfevHzDeq3gA/thTmOuv/5wePWZSuXI0q1bdzCWXVPJykcaYEHLZZZeRmJjI+PF/74qOHTsGQLVq1YiLiyMtLY2dO3eyYsUKAFq2bMnSpUvZunXryfJbtmzhyJEjHDx4kM6dOzNq1Cji4uIA+OWXX2jRogXPPvssZcuWZefOnQHF1qZNG2bOnEliYiJHjhxh9uzZOVZvL5uGmgNbVXUbgIhMBboCG33KdAXeUyfVLReRkiJSQVU96bu1x7CmrP7lN158sS0PPhhrncQZY04hIkyfPp0HH3yQF198kXLlyp1s92/dujXVq1enYcOGNGjQ4OS1hHLlyjFp0iR69erFiRMnABg2bBjR0dF07dqVxMREVJWRI0cCznMLfv75Z1SVDh060KhRI7755ptsY2vWrBnXXnstjRo1omrVqsTGxlKiRImcqbdXpxsi0h3opKp93eFbgBaqOtCnzCzgBVVd4g4vBB5V1VUZ5tUP54yBKlWqNN2+ffvpB7T6AdZsKURU08eoXdv6BzImL9q0aRP16tULdhh51pEjRyhWrBjHjh2jbdu2TJgwwe/FbX/rUURWq2qsv/l6eUbgrx+GjFknkDKo6gRgAkBsbOyZZa6mo2hkXQQZY0JYv3792LhxI4mJidx2222nfYdTZrxMBPFAZZ/hSsAfZ1DGGGMMMGXKFE/m62Uj+UqglohUF5EIoCcwI0OZGcCt7t1DLYGDXl0fMMaEBi/vjgkHZ7L+PDsjUNUUERkIzAMKAO+o6gYR6e+Ofx2YA3QGtgLHgNu9iscYk/dFRkayd+9eypQpY738ngFVZe/evURGRp7WdJ5dLPZKbGysrlq1KvuCxpiQk5ycTHx8/Ml79M3pi4yMpFKlShQqVOiU94N1sdgYY05LoUKFqF69erDDCDt2I70xxoQ5SwTGGBPmLBEYY0yYC7mLxSKSAJzBT4sBKAvsycFwQoHVOTxYncPD2dS5qqqW8zci5BLB2RCRVZldNc+vrM7hweocHryqszUNGWNMmLNEYIwxYS7cEsGEYAcQBFbn8GB1Dg+e1DmsrhEYY4z5p3A7IzDGGJOBJQJjjAlz+TIRiEgnEflJRLaKyGA/40VERrvj14pIzjzdIYgCqHNvt65rRWSZiDQKRpw5Kbs6+5RrJiKp7lPzQlogdRaRdiISJyIbRCT7ZyDmcQF8tkuIyEwRWePWOaR7MRaRd0TkLxFZn8n4nN9/qWq++sPp8voXoAYQAawB6mco0xn4AucJaS2B74Mddy7U+WKglPv6qnCos0+5r3C6PO8e7LhzYTuXxHkueBV3uHyw486FOj8ODHdflwP2ARHBjv0s6twWuAhYn8n4HN9/5cczgubAVlXdpqpJwFSga4YyXYH31LEcKCkiFXI70ByUbZ1VdZmq7ncHl+M8DS6UBbKdAe4DPgH+ys3gPBJInW8CPlXVHQCqGur1DqTOCkSL8wCDYjiJICV3w8w5qroYpw6ZyfH9V35MBBWBnT7D8e57p1smlJxufe7EOaIIZdnWWUQqAtcBr+diXF4KZDvXBkqJyCIRWS0it+ZadN4IpM5jgHo4j7ldB/xbVdNyJ7ygyPH9V358HoG/xxplvEc2kDKhJOD6iEh7nETQxtOIvBdInUcBj6pqaj552lUgdS4INAU6AFHAdyKyXFW3eB2cRwKp85VAHHAZUBP4UkS+VdVDHscWLDm+/8qPiSAeqOwzXAnnSOF0y4SSgOojIhcCbwFXqereXIrNK4HUORaY6iaBskBnEUlR1em5EmHOC/SzvUdVjwJHRWQx0AgI1UQQSJ1vB15QpwF9q4j8CtQFVuROiLkux/df+bFpaCVQS0Sqi0gE0BOYkaHMDOBW9+p7S+Cgqu7K7UBzULZ1FpEqwKfALSF8dOgr2zqranVVraaq1YBpwL0hnAQgsM/258AlIlJQRIoALYBNuRxnTgqkzjtwzoAQkXOBOsC2XI0yd+X4/ivfnRGoaoqIDATm4dxx8I6qbhCR/u7413HuIOkMbAWO4RxRhKwA6/wUUAYY5x4hp2gI99wYYJ3zlUDqrKqbRGQusBZIA95SVb+3IYaCALfzUGCSiKzDaTZ5VFVDtntqEfkQaAeUFZF44GmgEHi3/7IuJowxJszlx6YhY4wxp8ESgTHGhDlLBMYYE+YsERhjTJizRGCMMWHOEoHJk9zeQuN8/qplUfZIDixvkoj86i7rBxFpdQbzeEtE6ruvH88wbtnZxujOJ329rHd73CyZTfnGItI5J5Zt8i+7fdTkSSJyRFWL5XTZLOYxCZilqtNE5ApghKpeeBbzO+uYspuviLwLbFHV/2ZRvg8Qq6oDczoWk3/YGYEJCSJSTEQWukfr60TkHz2NikgFEVnsc8R8ifv+FSLynTvtxyKS3Q56MXCBO+0gd17rReQB972iIjLb7f9+vYj8y31/kYjEisgLQJQbx2R33BH3//98j9DdM5EbRKSAiLwkIivF6WP+7gBWy3e4nY2JSHNxnjPxo/u/jvtL3GeBf7mx/MuN/R13OT/6W48mDAW77237sz9/f0AqTkdiccBnOL+CL+6OK4vzq8r0M9oj7v//AE+4rwsA0W7ZxUBR9/1Hgaf8LG8S7vMKgBuB73E6b1sHFMXp3ngD0AS4AXjTZ9oS7v9FOEffJ2PyKZMe43XAu+7rCJxeJKOAfsD/ue8XBlYB1f3EecSnfh8Dndzh4kBB93VH4BP3dR9gjM/0zwE3u69L4vRBVDTY29v+gvuX77qYMPnGcVVtnD4gIoWA50SkLU7XCRWBc4HdPtOsBN5xy05X1TgRuRSoDyx1u9aIwDmS9uclEfk/IAGnh9YOwGfqdOCGiHwKXALMBUaIyHCc5qRvT6NeXwCjRaQw0AlYrKrH3eaoC+Xvp6iVAGoBv2aYPkpE4oBqwGrgS5/y74pILZyeKAtlsvwrgGtF5CF3OBKoQmj3R2TOkiUCEyp64zx9qqmqJovIbzg7sZNUdbGbKLoA74vIS8B+4EtV7RXAMh5W1WnpAyLS0V8hVd0iIk1x+nt5XkTmq+qzgVRCVRNFZBFO18n/Aj5MXxxwn6rOy2YWx1W1sYiUAGYBA4DROP3tfK2q17kX1hdlMr0AN6jqT4HEa8KDXSMwoaIE8JebBNoDVTMWEJGqbpk3gbdxHve3HGgtIult/kVEpHaAy1wMdHOnKYrTrPOtiJwPHFPVD4AR7nIySnbPTPyZitNR2CU4nanh/r8nfRoRqe0u0y9VPQjcDzzkTlMC+N0d3cen6GGcJrJ084D7xD09EpEmmS3DhA9LBCZUTAZiRWQVztnBZj9l2gFxIvIjTjv+q6qagLNj/FBE1uIkhrqBLFBVf8C5drAC55rBW6r6I9AQWOE20TwBDPMz+QRgbfrF4gzm4zyXdoE6j18E5zkRG4EfxHlo+Rtkc8buxrIGp2vmF3HOTpbiXD9I9zVQP/1iMc6ZQyE3tvXusAlzdvuoMcaEOTsjMMaYMGeJwBhjwpwlAmOMCXOWCIwxJsxZIjDGmDBnicAYY8KcJQJjjAlz/w9BBNbIg3C3NwAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Plot perfect ROC curve\n",
    "fpr, tpr, thresholds = roc_curve(y_test, y_test)\n",
    "plot_roc_curve(fpr, tpr)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1.0"
      ]
     },
     "execution_count": 72,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Perfect ROC AUC score\n",
    "roc_auc_score(y_test, y_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Confusion matrix\n",
    "\n",
    "A confusion matrix is a quick way to compare the labels a model predicts and the actual labels it was supposed to predict. In essence, giving you an idea of where the model is getting confused.\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[24,  5],\n",
       "       [ 4, 28]], dtype=int64)"
      ]
     },
     "execution_count": 73,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.metrics import confusion_matrix\n",
    "\n",
    "y_preds = clf.predict(X_test)\n",
    "\n",
    "confusion_matrix(y_test, y_preds)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th>Predicted Label</th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Actual Label</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>24</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>4</td>\n",
       "      <td>28</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Predicted Label   0   1\n",
       "Actual Label           \n",
       "0                24   5\n",
       "1                 4  28"
      ]
     },
     "execution_count": 74,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.crosstab(y_test, \n",
    "            y_preds, \n",
    "            rownames=[\"Actual Label\"], \n",
    "            colnames=[\"Predicted Label\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAANQAAADQCAYAAAB/T08OAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8vihELAAAACXBIWXMAAAsTAAALEwEAmpwYAAAOEklEQVR4nO3deZAW9Z3H8fdnBkFBULmySNSAIorxCJ4YNQZdI1i7xiMeISl0zYLGSKLR4Kpr4pHEipo1Xlmz6uLGBeMBXrgYl4CQeIGIB2qiMdl4IShYOgOlDHz3j6dHx6mZ4WH49fT0+HlVTdFPdz9Pf2bKj308fSgiMLM0aooOYNaVuFBmCblQZgm5UGYJuVBmCblQZgl1KzpAa1bfeamP5yey9UlTio7Q5ayse1ktjfcayiwhF8osIRfKLCEXyiwhF8osIRfKLCEXyiwhF8osIRfKLCEXyiwhF8osIRfKLCEXyiwhF8osIRfKLCEXyiwhF8osIRfKLCEXyiwhF8osIRfKLCEXyiwhF8osIRfKLCEXyiwhF8osIRfKLCEXyiwhF8osIRfKLCEXyiwhF8osIRfKLCEXyiwhF8osoU77jN3OZum79Vxw5x94p241kjhm72GM23/nj6bfMn8J/zZrEXPO+xpb9dq0wKTl9PSSudTV1bN27VoaGtYy+qCjio7ULi5UlWprxPfH7MnOg/tR/8EaTrxuJvvtMIjtB27J0nfreezlNxm0Za+iY5baP4z9BiveWVl0jI2S2yafpJ0kTZZ0taRfZMM7r/+dndOAPj3ZeXA/AHr12IShA7Zg2XurALjigYV87/CRRcazTiKXQkmaDNwGCHgCWJANT5N0bh7L7Eivr6zjxTdXsOtn+zP3hVcZ0Kcnwwf1LTpWqUUE0++Zwpz5dzP+5OOLjtNueW3ynQLsEhFrmo6U9HNgCXBZS2+SNAGYAHDNhH/klL/fO6d47bfqgzWcPfVhzjlib2prarhx7rP88uRDi45VeocfejxLly6j/4C+zLj3Fl760ys88ocFRcfaYHlt8q0Dtm5h/KBsWosi4lcRsVdE7NUZy7Rm7Tq+P/Vhxu4+hEN22ZbXVrzP6yvrOO6a+xlz+XSWvbeKE6+bydvvry46auksXboMgLeXr+D++x5i5J67FZyoffJaQ30PmC3pJeDVbNy2wA7Ad3JaZq4igoumP8qQgVvwzQNGADDs77ZiznnHfTTPmMunM/XbY32UbwP17LkZNTU11NXV07PnZowefQA/u+zaomO1Sy6FiohZknYE9gEGU9l/eg1YEBFr81hm3hb/33LuX/wKwz6zJcddcz8AZxz2BQ4cPrjgZOU3YGB/bp12PQC13bpx1+33Mvt/5xWcqn0UEUVnaNHqOy/tnMFKaOuTphQdoctZWfeyWhrvMyXMEnKhzBJyocwScqHMEnKhzBJyocwScqHMEnKhzBJyocwScqHMEnKhzBJyocwScqHMEnKhzBJq9XooSdcArV5CERGTcklkVmJtXWC4sMNSmHURrRYqIm5p+lpSr4iozz+SWXmtdx9K0ihJzwMvZK93l3R97snMSqiagxJXAV8B3gGIiKeBg3LMZFZaVR3li4hXm40q5Y1WzPJWzV2PXpW0PxCSugOTyDb/zOyTqllDnQqcTuV2YK8De2SvzayZ9a6hIuJtYFwHZDErvWqO8g2VdJ+k5ZKWSbpH0tCOCGdWNtVs8k0FbqdyX/KtgTuAaXmGMiuragqliPh1RDRkP7fSxilJZp9mbZ3L1/jAoznZM51uo1Kk44GZHZDNrHTaOijxJJUCNd7DeWKTaQFcklcos7Jq61y+IR0ZxKwrqOpxNpI+D4wAPnrwUUT8V16hzMpqvYWS9EPgYCqFegAYA/wecKHMmqnmKN+xwCHA0og4Gdgd6JFrKrOSqqZQqyNiHdAgqQ+wDPAXu2YtqGYfaqGkLYH/oHLkrw54Is9QZmVVzbl8384G/13SLKBPRDyTbyyzcmrri92RbU2LiEX5RDIrr1YfWi1pThvvi4gYnU+kim7dB/v0pkRWvzG/6Ahdzib9h7b40Oq2vtj9cn5xzLom3+jSLCEXyiwhF8osoWqu2JWkb0i6MHu9raR98o9mVj7VrKGuB0YBJ2av3weuyy2RWYlVc6bEvhExUtJTABGxMrudmJk1U80aao2kWrLL3iUNANblmsqspKop1NXADGCgpB9TuXTjJ7mmMiupas7l+29JT1K5hEPAVyPCd441a0E1FxhuC6wC7ms6LiL+lmcwszKq5qDETD6+WcumwBDgj8AuOeYyK6VqNvl2bfo6Owt9Yiuzm32qbfCZEtllG3vnkMWs9KrZhzqrycsaYCSwPLdEZiVWzT5U7ybDDVT2qe7KJ45ZubVZqOwL3c0j4pwOymNWaq3uQ0nqFhFrqWzimVkV2lpDPUGlTIsl3UvlMTb1jRMjYnrO2cxKp5p9qL5UngA/mo+/jwrAhTJrpq1CDcyO8D3HJ5/CAX4+lFmL2ipULbA5nyxSIxfKrAVtFerNiLi4w5KYdQFtnSnR4n3HzKx1bRXqkA5LYdZFtFqoiFjRkUHMugLfRswsIRfKLCEXyiwhF8osIRfKLCEXyiwhF8osIRfKLCEXyiwhF8osoWouMLRW1NTU8Phj/8Mbry/lyKPGFx2nVN58aznnXXIFb69YSY3EsUeO4ZvHfZUX//RnLr78Gj74cA21tbX869mns+uI4UXHrZoLtREmnfEtXnzxJfr07r3+me0TutXWcs4Z/8yI4TtQX7+K406ZxP57f4Err7+J0/5pHAeO2pt5jzzBldffxJRrf1Z03Kp5k6+dBg8exNgxh3DzzdOKjlJKA/r3ZcTwHQDo1asnQ7fbhreWv4Mk6upXAVBXv4qB/fsVGXODdfgaStLJEfGfHb3c1H5+5UWc+y+X0rv35kVHKb3X33yLF176M7vtMpzJ353IxLMu4IrrbiTWBbfecGXR8TZIEWuoi1qbIGmCpIWSFq5bV9/abIU7YuyhLFv2NoueerboKKW3atVqzjz/UiZPmsjmvXrxmxkzmXzGBGbP+DU/mDSBC396VdERN4gi0t8eQtIzrU0CdoyIHuv7jG7dB3fa+1b8+NJzGff1Y2loaGDTTXvQp09vZtz9AONPmlR0tBatfmN+0RFatKahgdPP+SFf3HdPxp9wNAD7HXYMjz54J5KICPY77Bgef6jz3WBrk/5DW7yiPa9Nvs8AXwFWNhsv4JGcltlhzr/gMs6/4DIAvnTQKM4689ROW6bOKiK48KdXMXS7bT4qE8CA/v1Y8NSz7DNyNx5/cjHbbTO4wJQbLq9C3U/lFs6Lm0+QNDenZVqJPPXMEu6bNZth23+OY8afDsB3J47nosmTuOwXN9Cwdi09unfnhz8o1/+octnkS6Ezb/KVTWfd5Cuz1jb5fNjcLCEXyiwhF8osIRfKLCEXyiwhF8osIRfKLCEXyiwhF8osIRfKLCEXyiwhF8osIRfKLCEXyiwhF8osIRfKLCEXyiwhF8osIRfKLCEXyiwhF8osIRfKLCEXyiwhF8osIRfKLCEXyiwhF8osIRfKLCEXyiwhF8osIRfKLCEXyiwhF8osIRfKLCEXyiwhF8osoU770OqykDQhIn5VdI6uoux/T6+hNt6EogN0MaX+e7pQZgm5UGYJuVAbr7Tb+51Uqf+ePihhlpDXUGYJuVDtJOlwSX+U9LKkc4vOU3aSbpa0TNJzRWfZGC5UO0iqBa4DxgAjgBMljSg2VelNAQ4vOsTGcqHaZx/g5Yh4JSI+BG4Djiw4U6lFxDxgRdE5NpYL1T6DgVebvH4tG2efci5U+6iFcT5cai5UO70GbNPk9WeBNwrKYp2IC9U+C4BhkoZI6g6cANxbcCbrBFyodoiIBuA7wIPAC8DtEbGk2FTlJmka8CgwXNJrkk4pOlN7+EwJs4S8hjJLyIUyS8iFMkvIhTJLyIUyS8iFyomktZIWS3pO0h2Sem7EZ02RdGw2fGNbJ+JKOljS/u1Yxl8l9a92fLN56jZwWT+SdPaGZiwDFyo/qyNij4j4PPAhcGrTidkZ6xssIr4VEc+3McvBwAYXytJwoTrGfGCHbO0xR9JU4FlJtZIul7RA0jOSJgKo4lpJz0uaCQxs/CBJcyXtlQ0fLmmRpKclzZb0OSrFPTNbOx4oaYCku7JlLJD0xey9/ST9VtJTkm6g5fMTP0HS3ZKelLRE0oRm067MssyWNCAbt72kWdl75kvaKclfszOLCP/k8APUZf92A+4BTqOy9qgHhmTTJgAXZMM9gIXAEOBo4CGgFtgaeBc4NptvLrAXMIDKGe+Nn9U3+/dHwNlNckwFDsiGtwVeyIavBi7Mho+gcnJv/xZ+j782jm+yjM2A54B+2esAxmXDFwLXZsOzgWHZ8L7A71rK2JV+urWvhlaFzSQtzobnAzdR2RR7IiL+ko0/DNitcf8I2AIYBhwETIuItcAbkn7XwufvB8xr/KyIaO1aokOBEdJHK6A+knpnyzg6e+9MSSur+J0mSToqG94my/oOsA74TTb+VmC6pM2z3/eOJsvuUcUySs2Fys/qiNij6YjsP6z6pqOAMyLiwWbzjWX9l4Ooinmgslk/KiJWt5Cl6vPOJB1MpZyjImKVpLnApq3MHtly323+N+jqvA9VrAeB0yRtAiBpR0m9gHnACdk+1iDgyy2891HgS5KGZO/tm41/H+jdZL7fUjmRl2y+PbLBecC4bNwYYKv1ZN0CWJmVaScqa8hGNUDjWvbrwO8j4j3gL5K+li1DknZfzzJKz4Uq1o3A88Ci7OYkN1DZapgBvAQ8C/wSeLj5GyNiOZV9sOmSnubjTa77gKMaD0oAk4C9soMez/Px0caLgIMkLaKy6fm39WSdBXST9AxwCfBYk2n1wC6SngRGAxdn48cBp2T5lvApuE2AzzY3S8hrKLOEXCizhFwos4RcKLOEXCizhFwos4RcKLOEXCizhP4fGmY1Cwbqd6sAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 216x216 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "import seaborn as sns \n",
    "def plot_conf_mat(conf_mat):\n",
    "    \"\"\"\n",
    "    Plots a confusion matrix using Seaborn's heatmap().\n",
    "    \"\"\"\n",
    "    fig, ax = plt.subplots(figsize=(3, 3))\n",
    "    ax = sns.heatmap(conf_mat,\n",
    "                     annot=True, # Annotate the boxes \n",
    "                     cbar=False)\n",
    "    plt.xlabel('Predicted label')\n",
    "    plt.ylabel('True label');\n",
    "\n",
    "conf_mat=confusion_matrix(y_test, y_preds)\n",
    "\n",
    "plot_conf_mat(conf_mat)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAANQAAADQCAYAAAB/T08OAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8vihELAAAACXBIWXMAAAsTAAALEwEAmpwYAAAOdElEQVR4nO3deXRW9YHG8e+TNypoQEfFJYAHKHZwqStgUevgUlmmra2jjBS1x9pi3RlHR8/oWLV22qMdz5RRWq1anFpROEpR4aiViqgjhai4AFVrQUmMG6Ite5bf/PHeYIxJeAm/m5sbn8857+Eu75v75D0+3iV3UQgBM4ujLOsAZt2JC2UWkQtlFpELZRaRC2UWkQtlFlF51gHasmHhDB/Pj6Ti6ElZR+h26jfVqLXpXkOZReRCmUXkQplF5EKZReRCmUXkQplF5EKZReRCmUXkQplF5EKZReRCmUXkQplF5EKZReRCmUXkQplF5EKZReRCmUXkQplF5EKZReRCmUXkQplF5EKZReRCmUXkQplF5EKZReRCmUXkQplF5EKZReRCmUXkQplF5EKZReRCmUXkQplF5EKZReRCmUXUZZ+x29W8s+ojrrz1flZ9vAZJnHLsUCaMOpJX36zl+qkPsm7DJip334WfnHcqFT17ZB03d0adOJKbbrqOQlkZd/56GjfceEvWkTrEhSpRoVDg0m+PYb8Blaxdv5HTrp7Clw8czLV3/I5Lxo9m6H4Dmfnkc0yd/TQXnHJC1nFzpaysjMk//zGjx46nurqWBc/O4aGHH2PZstezjrbVUtvkkzRE0uWSJkv6eTK8X1rLS1ufXXqx34BKAHbquQODKvvw3od/ZUXtBxw+ZAAAIw78AnMXLckwZT4NH3Yob7yxguXL36Kuro7p02fxja+PyjpWh6RSKEmXA/cCAhYCi5LhaZKuSGOZnanm/dX86c1avjS4H4P77cG85/8EwGMLl/DOhx9nnC5/KvvuxcrqtzePV9fUUlm5V4aJOi6tNdTZwLAQwk9DCHcnr58Cw5N5rZI0UVKVpKo7Zj6eUrRts27DRv518jQumzCWip49uPb7J3Pv4ws47T+msG79RrYrL2QdMXckfWZaCCGDJNsurX2oRqASeLPF9L2Tea0KIdwG3AawYeGMLveN1tU3cMnkaYw98mBOGHYAAAMr+3Dr5WcBsKL2A+a/+GqWEXOpprqW/v0qN4/367s3tbXvZpio49Iq1CRgrqTXgZXJtH2AwcAFKS0zVSEErrl9JoMq+3DmmKM2T1/18Rp227mCxsZGfjVrHqceNzzDlPm0qGoxgwcPZMCA/tTUvMO4cSdxxpnnZx2rQ1IpVAjhEUlfpLiJ15fi/lM1sCiE0JDGMtP2wmtv8vAzi9m3/56Mu/JmAC489au89e4q7n38jwAcP3R/vnnMYVnGzKWGhgYunnQVc2bfQ6GsjKl33cfSpa9lHatD1FW3VbviJl9eVRw9KesI3U79pprP7vjhMyXMonKhzCJyocwicqHMInKhzCJyocwicqHMInKhzCJyocwicqHMInKhzCJyocwicqHMInKhzCJq83ooSf8DtHkJRQjholQSmeVYexcYVnVaCrNuos1ChRDuaj4uaacQwtr0I5nl1xb3oSSNkLQUWJaMHyxpSurJzHKolIMS/w2MAlYBhBBeBI5JMZNZbpV0lC+EsLLFpFzeaMUsbaXc9WilpCOBIGl74CKSzT8z+7RS1lA/AM6neDuwGuCQZNzMWtjiGiqE8AEwoROymOVeKUf5Bkl6SNL7kt6TNEvSoM4IZ5Y3pWzy3QNMp3hf8kpgBjAtzVBmeVVKoRRC+E0IoT553U07pySZfZ61dy7frsngE8kzne6lWKR/BmZ3Qjaz3GnvoMRzFAvUdA/nc5rNC8CP0gplllftncs3sDODmHUHJT3ORtKBwP7A5sebhxD+N61QZnm1xUJJ+iEwkmKh5gBjgKcBF8qshVKO8p0CHA+8E0I4CzgY2CHVVGY5VUqh1ocQGoF6Sb2B9wD/YdesFaXsQ1VJ2gX4FcUjf2uAhWmGMsurUs7lOy8Z/KWkR4DeIYSX0o1llk/t/WG3zacvSzoshPB8OpHM8qvNh1ZLeqKdz4UQwnHpRCoq376vT2+KZP3bT2UdodvZbvdBrT60ur0/7B6bXhyz7sk3ujSLyIUyi8iFMouolCt2Jel0SVcn4/tIGp5+NLP8KWUNNQUYAYxPxv8G3JJaIrMcK+VMiSNCCIdJegEghLA6uZ2YmbVQyhqqTlKB5LJ3SX2AxlRTmeVUKYWaDMwE9pD0Y4qXbvxnqqnMcqqUc/l+K+k5ipdwCPhmCMF3jjVrRSkXGO4DrAMeaj4thPBWmsHM8qiUgxKz+eRmLT2AgcCrwAEp5jLLpVI2+b7UfDw5C/2cNt5u9rm21WdKJJdtDEshi1nulbIPdUmz0TLgMOD91BKZ5Vgp+1C9mg3XU9ynuj+dOGb51m6hkj/oVoQQLuukPGa51uY+lKTyEEIDxU08MytBe2uohRTLtFjSgxQfY7O2aWYI4YGUs5nlTin7ULtSfAL8cXzy96gAuFBmLbRXqD2SI3yv8OmncICfD2XWqvYKVQAq+HSRmrhQZq1or1C1IYTrOi2JWTfQ3pkSrd53zMza1l6hju+0FGbdRJuFCiF82JlBzLoD30bMLCIXyiwiF8osIhfKLCIXyiwiF8osIhfKLCIXyiwiF8osIhfKLKJSLjC0Vow6cSQ33XQdhbIy7vz1NG640U/42RobN27iO+dfxqa6OhrqG/jqsUdzwffO4Gc3386Tz/yR8u3K6d93b67/90vo3asi67gla/Mp8Fnryk+BLysrY9mSpxg9djzV1bUseHYOp59xHsuWvZ51tFZ1xafAhxBYv34DO+7Yk7r6es4891KuuPgc1qxdxxGHH0J5eYGbptwBwCXnnZ1x2s9q6ynw3uTrgOHDDuWNN1awfPlb1NXVMX36LL7x9VFZx8oVSey4Y08A6uvrqa+vRxJHHXE45eUFAA46YAjvvvdBljG3WqcXStJZnb3M2Cr77sXK6rc3j1fX1FJZuVeGifKpoaGBf/rO+RzztfGMGHYoBx0w5FPzZ85+jKNH5OsmxVmsoa5ta4akiZKqJFU1Nq5t622Zkz67tu+qm85dWaFQ4P67bmHuzN/w8tLXeP0vKzbPu/WuaRQKBb524rHZBeyAVA5KSHqprVnAnm19LoRwG3AbdO19qJrqWvr3q9w83q/v3tTWvpthonzr3auCYYcdxNMLqth30ABmzfk9859ZyO2Tf9Lq/7y6srSO8u0JjAJWt5gu4P9SWmanWVS1mMGDBzJgQH9qat5h3LiTOOPM87OOlSsfrv6I8vJyeveqYMPGjSxY9ALfPf1Unl5QxR2/ncHUm2+gZ48eWcfcamkV6mGKt3Be3HKGpHkpLbPTNDQ0cPGkq5gz+x4KZWVMves+li59LetYufL+qtVcef3PaGhsJDQGRh33FUYedQRjxn2XTXV1fH/SlUDxwMQP/+3CjNOWzofNPwe64mHzvPNhc7NO4EKZReRCmUXkQplF5EKZReRCmUXkQplF5EKZReRCmUXkQplF5EKZReRCmUXkQplF5EKZReRCmUXkQplF5EKZReRCmUXkQplF5EKZReRCmUXkQplF5EKZReRCmUXkQplF5EKZReRCmUXkQplF5EKZReRCmUXkQplF5EKZReRCmUXkQplF5EKZReRCmUXUZR9anReSJoYQbss6R3eR9+/Ta6htNzHrAN1Mrr9PF8osIhfKLCIXatvldnu/i8r19+mDEmYReQ1lFpEL1UGSRkt6VdKfJV2RdZ68k3SnpPckvZJ1lm3hQnWApAJwCzAG2B8YL2n/bFPl3lRgdNYhtpUL1THDgT+HEP4SQtgE3AuclHGmXAshzAc+zDrHtnKhOqYvsLLZeHUyzT7nXKiOUSvTfLjUXKgOqgb6NxvvB7ydURbrQlyojlkE7CtpoKTtgdOABzPOZF2AC9UBIYR64ALgUWAZMD2EsCTbVPkmaRrwLPD3kqolnZ11po7wmRJmEXkNZRaRC2UWkQtlFpELZRaRC2UWkQuVEkkNkhZLekXSDEk7bsPPmirplGT49vZOxJU0UtKRHVjGCkm7lzq9xXvWbOWyrpF06dZmzAMXKj3rQwiHhBAOBDYBP2g+MzljfauFEL4XQljazltGAltdKIvDheocTwGDk7XHE5LuAV6WVJB0o6RFkl6SdA6Aim6WtFTSbGCPph8kaZ6kocnwaEnPS3pR0lxJAygW91+SteNXJPWRdH+yjEWSjko+u5ukxyS9IOlWWj8/8VMk/U7Sc5KWSJrYYt5/JVnmSuqTTPuCpEeSzzwlaUiUb7MrCyH4lcILWJP8Ww7MAs6luPZYCwxM5k0ErkqGdwCqgIHAycDvgQJQCXwEnJK8bx4wFOhD8Yz3pp+1a/LvNcClzXLcAxydDO8DLEuGJwNXJ8P/SPHk3t1b+T1WNE1vtoyewCvAbsl4ACYkw1cDNyfDc4F9k+EjgD+0lrE7vco7VkMrQU9Ji5Php4A7KG6KLQwhLE+mnwgc1LR/BOwM7AscA0wLITQAb0v6Qys//8vA/KafFUJo61qiE4D9pc0roN6SeiXLODn57GxJq0v4nS6S9K1kuH+SdRXQCNyXTL8beEBSRfL7zmi27B1KWEauuVDpWR9COKT5hOQ/rLXNJwEXhhAebfG+sWz5chCV8B4obtaPCCGsbyVLyeedSRpJsZwjQgjrJM0DerTx9pAs96OW30F3532obD0KnCtpOwBJX5S0EzAfOC3Zx9obOLaVzz4L/IOkgclnd02m/w3o1ex9j1E8kZfkfYckg/OBCcm0McDfbSHrzsDqpExDKK4hm5QBTWvZbwNPhxD+CiyXdGqyDEk6eAvLyD0XKlu3A0uB55Obk9xKcathJvA68DLwC+DJlh8MIbxPcR/sAUkv8skm10PAt5oOSgAXAUOTgx5L+eRo47XAMZKep7jp+dYWsj4ClEt6CfgRsKDZvLXAAZKeA44DrkumTwDOTvIt4XNwmwCfbW4WkddQZhG5UGYRuVBmEblQZhG5UGYRuVBmEblQZhG5UGYR/T9pFW19enOSBgAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 216x216 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Create perfect confusion matrix\n",
    "perfect_conf_mat = confusion_matrix(y_test, y_test)\n",
    "plot_conf_mat(perfect_conf_mat)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<sklearn.metrics._plot.confusion_matrix.ConfusionMatrixDisplay at 0x187e3f04040>"
      ]
     },
     "execution_count": 77,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAATgAAAEJCAYAAAAAWTtiAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8vihELAAAACXBIWXMAAAsTAAALEwEAmpwYAAAaC0lEQVR4nO3de5hV1X3/8feHQS6CF8iIGRHvqEGNJiGoSWMxppHEPmrb2GBiy8+Sn9F4SXMx0aQtrSmJT9OktUm0pUolVTEkMZHcRCVaND8VEa9g+IFKEEFxRDECAjPz7R97jx5w5szeh3M45+z5vHz2M2evvc/aX5iHr2uvtfZeigjMzIpoQL0DMDOrFSc4MyssJzgzKywnODMrLCc4MyssJzgzKywnODOrG0kzJa2T9MQO5RdLWiZpiaR/Kim/XNKK9NipfdU/sBZBm5lldD3wXeD73QWSTgbOAN4ZEVskjUrLxwGTgaOA/YA7JR0eEZ29Vd5QCW7YiEExYvTQeodhOby6pKXeIVgOr7ORrbFFO1PHqScPi5fW95pTtvPQY1vmRcSk3o5HxAJJB+1QfAFwZURsSc9Zl5afAdyclj8jaQUwAbivt/obKsGNGD2Ui+ecWO8wLIc7j9mz3iFYDg903bnTdby0vpOF8w7IdG5L2/LWCi5xOPABSdOB14EvRsSDwGjg/pLzVqdlvWqoBGdmjS+ALrqynt4qaVHJ/oyImNHHdwYCI4ATgPcCcyQdAvTU8iz7rKkTnJnlEgTbeu/22lF7RIzPeYnVwC2RPCi/UFIX0JqWjyk5b39gTbmKPIpqZrl1ZfyvQj8FPggg6XBgENAOzAUmSxos6WBgLLCwXEVuwZlZLkHQWaW3EEmaDUwkuZVdDUwDZgIz06kjW4EpaWtuiaQ5wFKgA7iw3AgqOMGZWQW6ynd9ZRYRZ/dy6Jxezp8OTM9avxOcmeUSQGeVElytOcGZWW7VasHVmhOcmeUSwLYmeRO4E5yZ5RKEb1HNrKACOpsjvznBmVk+yZMMzcEJzsxyEp09PjXVeJzgzCyXZJDBCc7MCiiZB+cEZ2YF1eUWnJkVkVtwZlZYgehskhcROcGZWW6+RTWzQgrE1miOtTic4Mwsl2Sir29RzaygPMhgZoUUITrDLTgzK6gut+DMrIiSQYbmSB3N0c40s4bRPciQZeuLpJmS1qULzOx47IuSQlJrSdnlklZIWibp1L7qd4Izs9w6Q5m2DK4HJu1YKGkM8EfAqpKyccBk4Kj0O1dLKjtfxQnOzHLpfpIhy9ZnXRELgPU9HPoX4Etsv3L9GcDNEbElIp4BVgATytXfHDfSZtZQurKPorZKWlSyPyMiZpT7gqTTgeci4lFpu1bgaOD+kv3VaVmvnODMLJfkYfvMCa49IsZnPVnS7sBXgQ/3dLiXcHrlBGdmuQRiW+0e1ToUOBjobr3tDyyWNIGkxTam5Nz9gTXlKnOCM7NcIqjZRN+IeBwY1b0vaSUwPiLaJc0FbpL0bWA/YCywsFx9HmQws5xEV8atz5qk2cB9wBGSVkua2tu5EbEEmAMsBW4DLoyIznL1uwVnZrkE1WvBRcTZfRw/aIf96cD0rPU7wZlZbn7hpZkVUiC/8NLMiilZNrA5UkdzRGlmDcQLP5tZQQW5nmSoKyc4M8vNLTgzK6QIuQVnZsWUDDJ4VS0zKySvyWBmBZUMMrgPzswKyk8ymFkh+UkGMys0r2xvZoUUAdu6nODMrICSW1QnuH5hyd8MoX1BC4NGBif+dBMAT31nEC/+eiAMgEEjg6Omv87gUcGGxwfw5N8PSb4YcMhntjLqQx11jN52NOv+JWx+rYWuLujsEBd/9Ih6h9SQ/CQDIGkScBXQAlwbEVfW8nr1sN+Z2xjzia0s+cqQN8oOPHcrh168FYBVN+zG09cM4h3TtjD8sC4m/GATAwbClhfF/X+2O60TOxjg/800lC+ddRivvuxfSm88TQRIF2T9HsnirauBByXNjYiltbpmPYwY38nm57b/ZQ8c/ubnzs16Yy2glqFvlndt6XmJILPG51tUSBZkXRERTwNIuplk4dZCJbjerLhqEGvn7sbAPYL3zNz8RvmGxwaw9G+H8PqaARz1jdfdems0Ib4++ykI+MUNb+NXN7bWO6KGlGW9hUZQyzQ8Gni2ZL/PRVqL5LDPbuUD8zfy9tM6ePam3d4o3+udXZx46yYm3LyJldcOonNLHYO0t/jcmWO5aNIRfPWcQzj9/7Rz9PGv1TukhpOMorZk2voiaaakdZKeKCn7pqTfSnpM0k8k7V1y7HJJKyQtk3RqX/XXMsFlWqRV0nmSFklatHH91hqGUx9vP20b6+58azNt2KFdtAwNNi5vjqZ+f7H+heR/Rhte2o3f/GovjjxuU50jajzdE32zbBlcD0zaoewO4OiIeCfw/4HLASSNAyYDR6XfuTrtCutVLf91ZVqkNSJmRMT4iBg/bOSgGoaz62z63Zu/2BfvGsiwg7sA2LxadKWDppvXiI0rBzBkdNmFuW0XGjy0k6HDOt/4/J4//D0rlw3p41v9U7WWDYyIBcD6Hcpuj4ju6QX3k+QOSLq4bo6ILRHxDLCCpCusV7XsAXoQGCvpYOA5ksz7iRpery4ev3QILz/YwrZXxD2nDOOQz2yl/Z4WNq0cgARD9guO/LvXAXhlcQsrrxuEBoIGwJF/s4VBI5zgGsWIfTqYdt0zALS0wF0/3ZtFd+9Z56gaT85R1FZJi0r2Z0TEjByX+yvgB+nn0SQJr1uf3V41S3AR0SHpImAeyTSRmenCrYVyzDdff0vZ6D/b1uO5bad30Ha65701qudXDeaCPzqy3mE0hRyjqO0RMb6Sa0j6KtAB3Nhd1MNpZVsINR3Di4hfAr+s5TXMbNeKEB01niYiaQrwx8ApEdGdxDJ1e5VyD7eZ5VbFQYa3SB8Q+DJwekSUjvLMBSZLGpx2fY0FFpary7OwzCyXaj7JIGk2MJGkr241MI1k1HQwcIckgPsj4vyIWCJpDslc2g7gwojoLFe/E5yZ5VatBBcRZ/dQfF2Z86cD07PW7wRnZrn4hZdmVmjN8qiWE5yZ5RIBHX7hpZkVlW9RzayQ3AdnZoUWTnBmVlQeZDCzQopwH5yZFZbo9CiqmRWV++DMrJC8qpaZFVck/XDNwAnOzHLzKKqZFVJ4kMHMisy3qGZWWB5FNbNCinCCM7MCa5ZpIs3RU2hmDSUi29YXSTMlrZP0REnZSEl3SFqe/hxRcuxySSskLZN0al/1O8GZWS6B6OoakGnL4Hpg0g5llwHzI2IsMD/dR9I4kgXkj0q/c7WklnKVO8GZWW6RceuznogFwPodis8AZqWfZwFnlpTfHBFbIuIZYAUwoVz9TnBmlk86yJBlq9C+EbEWIP05Ki0fDTxbct7qtKxXHmQws/yyz4NrlbSoZH9GRMyo8Ko9ZcyykTjBmVluOVpn7RExPmf1L0hqi4i1ktqAdWn5amBMyXn7A2vKVdRrgpP0Hcpkx4i4JHu8ZlYUAXR11XSayFxgCnBl+vPWkvKbJH0b2A8YCywsV1G5FtyiMsfMrL8KoErz4CTNBiaS3MquBqaRJLY5kqYCq4CzACJiiaQ5wFKgA7gwIjrL1d9rgouIWaX7koZFxMad+LOYWUFU61nUiDi7l0On9HL+dGB61vr7HEWVdKKkpcCT6f6xkq7OegEzK6BqzROpsSzTRP4VOBV4CSAiHgVOqmFMZtbQsk0RaYTnVTONokbEs9J2wZa97zWzgmuA1lkWWRLcs5LeB4SkQcAlpLerZtYPBURtR1GrJsst6vnAhSQzhp8Djkv3zazfUsatvvpswUVEO/DJXRCLmTWLJrlFzTKKeoikn0l6MX2tya2SDtkVwZlZgyrQKOpNwBygjWT28A+B2bUMyswaWPdE3yxbnWVJcIqI/46IjnS7gYbIzWZWL9V64WWtlXsWdWT68S5JlwE3kyS2jwO/2AWxmVmjapJR1HKDDA+RJLTuP8mnS44F8LVaBWVmjU0N0DrLotyzqAfvykDMrEk0yABCFpmeZJB0NDAOGNJdFhHfr1VQZtbIGmMAIYs+E5ykaSSvMxkH/BL4CHAv4ARn1l81SQsuyyjqx0heXfJ8RJwLHAsMrmlUZtbYujJudZblFnVzRHRJ6pC0J8nrgz3R16y/quILL2stS4JbJGlv4D9JRlZfo4/XBJtZsTX9KGq3iPhM+vHfJd0G7BkRj9U2LDNraM2e4CS9u9yxiFhcm5DMzKqjXAvuW2WOBfDBKsfCq0tauPPoPapdrdXQvDUP1zsEy2HCqZuqUk+1blElfQ74FElOeRw4F9gd+AFwELAS+POIeLmS+stN9D25kgrNrOCCqjyqJWk0yQt0x0XE5nTFrMkkU9LmR8SV6WOilwFfruQaWaaJmJltr3qvSxoIDJU0kKTltgY4A+he1W8WcGalYTrBmVluimxbORHxHPDPJGufrgU2RMTtwL4RsTY9Zy0wqtI4neDMLL/sLbhWSYtKtvO6q5A0gqS1djDJuyaHSTqnmmFmeVRLJK8sPyQirpB0APD2iPBcOLP+KvsgQ3tEjO/l2IeAZyLiRQBJtwDvA16Q1BYRayW1kTxcUJEsLbirgROB7hWofw98r9ILmllzy3p7mmGkdRVwgqTd04bUKSQr9s0FpqTnTAFurTTWLE8yHB8R75b0MEBEvJwuH2hm/VUVRlEj4gFJPwIWAx3Aw8AMYDgwR9JUkiR4VqXXyJLgtklqIW2UStqHhniM1szqpVrz4CJiGjBth+ItJK25nZblFvXfgJ8AoyRNJ3lV0tercXEza1JNsqpWlmdRb5T0EElGFXBmRHhle7P+Klv/WkPIMop6ALAJ+FlpWUSsqmVgZtbAipLgSFbQ6l58ZgjJnJVlwFE1jMvMGpiapBc+yy3qMaX76VtGPt3L6WZmDSPTojOlImKxpPfWIhgzaxJFuUWV9PmS3QHAu4EXaxaRmTW2Ig0yAKUvaOsg6ZP7cW3CMbOmUIQEl07wHR4Rl+6ieMysGTR7gpM0MCI6yr263Mz6H1GMUdSFJP1tj0iaC/wQ2Nh9MCJuqXFsZtaICtYHNxJ4iWQNhu75cAE4wZn1VwVIcKPSEdQneDOxdWuSP56Z1USTZIByCa6F5LUlPb0XpUn+eGZWC0W4RV0bEVfsskjMrHkUIMHt/BvtzKx4ohijqFV54ZyZFVCzt+AiYv2uDMTMmkcR+uDMzHrWJAnO66KaWT5ZX1eeIQlK2lvSjyT9VtKTkk6UNFLSHZKWpz9HVBqqE5yZ5SKqtmwgwFXAbRFxJHAsybKBlwHzI2IsMD/dr4gTnJnlVo0EJ2lP4CTgOoCI2BoRr5Csdj8rPW0WcGalcTrBmVl+2W9RWyUtKtnOK6nlEJJ3S/6XpIclXStpGLBvRKwFSH+OqjRMDzKYWX7ZBxnaI2J8L8cGkrzQ4+J0Eeir2Inb0Z64BWdm+WS8Pc3QB7caWB0RD6T7PyJJeC9IagNIf66rNFQnODPLrwqjqBHxPPCspCPSolOApcBcYEpaNgW4tdIwfYtqZrlV8VGti4EbJQ0CngbOJWl4zZE0FVgFnFVp5U5wZpZbtZ5kiIhHgJ766KryqKgTnJnlk3ESbyNwgjOz/JzgzKyIup9kaAZOcGaWm7qaI8M5wZlZPu6DM7Mi8y2qmRWXE5yZFZVbcGZWXE5wZlZIBVlVy8zsLTwPzsyKLZojwznBmVluzdKC8/vgamzAgOB7ty/jillP1zsUS33rc2P482OO4ryTj9iu/NbrWpn6B0fyfycewbVfawPg1fUtXPqxQznjsGP47ldG1yPcxlPFVbVqrWYtOEkzgT8G1kXE0bW6TqM781PtPLt8CLsP76x3KJb68MfXc/q57Xzzswe8UfbIb4bz/+btxTXzlzFocPBKe/JPY9CQYMqlz7Ny2RBW/nZIvUJuOM0yyFDLFtz1wKQa1t/wWtu2MuGUV/nVTSPrHYqVOOaEjewxYvv/4fz8+2/j4xe9wKDBSbNj79YOAIbs3sXRx298o9wS6sq21VvNElxELADW16r+ZnD+P6zh2n9sI7pU71CsD889NYQnHhjOJaeN5Yt/ehjLHhla75AaV5AMMmTZ6qzufXCSzuteUmwbW+odTtUc/6FXeaV9ICse373eoVgGnZ3w2oYWrvr5cj71t2uY/umDGuHfZ8Oq4sLPNVX3BBcRMyJifESM343B9Q6nasa9dyMnfPhVZj2wlMuv+R3H/sFrfOk7v6t3WNaL1rZtvP+jG5DgyHdtYsAA2LC+pd5hNa4qDjJIaknXRf15uj9S0h2Slqc/R1QaZt0TXFH91zfaOGf8OKYcP45vXHAgj947nH+6+MB6h2W9eN+kDTxy73AAVj81mG1bxV4jPTDUk+6JvlVswX0WeLJk/zJgfkSMBeazE2uleh6c9TvfuOBAHrtvOBvWD+ST7xnHX3zheU6dvJ5vf34M5518BLvtFlx61SqUdp3+5YRxbHxtAB1bxX3z9uLrs5/iwMOL052SW0TVXngpaX/gNGA68Pm0+AxgYvp5FnA38OVK6q/lNJHZJEG2SloNTIuI62p1vUb22H3Deey+4fUOw1KXX9NzV8GXv7uqx/LvL1xay3CaU/X61/4V+BKwR0nZvhGxFiAi1koaVWnlNUtwEXF2reo2s/rKcfvZKmlRyf6MiJgBIKl7nuxDkiZWNcCUb1HNLJ8Ast+itkdET+ueArwfOF3SR4EhwJ6SbgBekNSWtt7agHWVhupBBjPLrwqjqBFxeUTsHxEHAZOBX0fEOcBcYEp62hTg1krDdAvOzHKr8Ry3K4E5kqYCq4CzKq3ICc7Mcqv2soERcTfJaCkR8RJwSjXqdYIzs3wa5E0hWTjBmVkuyUTf5shwTnBmll8DvCkkCyc4M8vNLTgzKyb3wZlZcVXvWdRac4Izs/x8i2pmheSFn82s0NyCM7PCao785gRnZvmpqznuUZ3gzCyfwBN9zayYRHiir5kVmBOcmRWWE5yZFZL74MysyDyKamYFFb5FNbOCCpomwXlVLTPLryvjVoakMZLukvSkpCWSPpuWj5R0h6Tl6c8RlYbpBGdmuSki09aHDuALEfEO4ATgQknjgMuA+RExFpif7lfECc7M8ovItpWtItZGxOL08++BJ4HRwBnArPS0WcCZlYbpPjgzyycCOqs7iirpIOBdwAPAvhGxNrlUrJU0qtJ6neDMLL/sgwytkhaV7M+IiBmlJ0gaDvwY+OuIeFVSlYJ0gjOzSmRPcO0RMb63g5J2I0luN0bELWnxC5La0tZbG7Cu0jDdB2dm+QTQFdm2MpQ01a4DnoyIb5ccmgtMST9PAW6tNFS34Mwsp4CoSh/c+4G/AB6X9Eha9hXgSmCOpKnAKuCsSi/gBGdm+QRVGWSIiHuB3jrcTtnpC+AEZ2aVaJInGZzgzCw/JzgzKyY/bG9mRRWAX5dkZoXlFpyZFVP1H9WqFSc4M8snIKozD67mnODMLL8+nlJoFE5wZpaf++DMrJAiPIpqZgXmFpyZFVMQnZ31DiITJzgzy6f7dUlNwAnOzPLzNBEzK6IAwi04MyukqNoLL2vOCc7McmuWQQZFAw33SnoR+F2946iBVqC93kFYLkX9nR0YEfvsTAWSbiP5+8miPSIm7cz1dkZDJbiikrSo3MpC1nj8OysGr6plZoXlBGdmheUEt2vM6PsUazD+nRWA++DMrLDcgjOzwnKCqyFJkyQtk7RC0mX1jsf6JmmmpHWSnqh3LLbznOBqRFIL8D3gI8A44GxJ4+oblWVwPVC3eVtWXU5wtTMBWBERT0fEVuBm4Iw6x2R9iIgFwPp6x2HV4QRXO6OBZ0v2V6dlZraLOMHVjnoo85C12S7kBFc7q4ExJfv7A2vqFItZv+QEVzsPAmMlHSxpEDAZmFvnmMz6FSe4GomIDuAiYB7wJDAnIpbUNyrri6TZwH3AEZJWS5pa75iscn6SwcwKyy04MyssJzgzKywnODMrLCc4MyssJzgzKywnuCYiqVPSI5KekPRDSbvvRF3XS/pY+vnaci8CkDRR0vsquMZKSW9ZnKS38h3OeS3ntf5e0hfzxmjF5gTXXDZHxHERcTSwFTi/9GD6BpPcIuJTEbG0zCkTgdwJzqzenOCa1z3AYWnr6i5JNwGPS2qR9E1JD0p6TNKnAZT4rqSlkn4BjOquSNLdksannydJWizpUUnzJR1Ekkg/l7YePyBpH0k/Tq/xoKT3p999m6TbJT0s6T/o+Xnc7Uj6qaSHJC2RdN4Ox76VxjJf0j5p2aGSbku/c4+kI6vyt2mF5IWfm5CkgSTvmbstLZoAHB0Rz6RJYkNEvFfSYOA3km4H3gUcARwD7AssBWbuUO8+wH8CJ6V1jYyI9ZL+HXgtIv45Pe8m4F8i4l5JB5A8rfEOYBpwb0RcIek0YLuE1Yu/Sq8xFHhQ0o8j4iVgGLA4Ir4g6e/Sui8iWSvh/IhYLul44GrggxX8NVo/4ATXXIZKeiT9fA9wHcmt48KIeCYt/zDwzu7+NWAvYCxwEjA7IjqBNZJ+3UP9JwALuuuKiN7ei/YhYJz0RgNtT0l7pNf40/S7v5D0coY/0yWS/iT9PCaN9SWgC/hBWn4DcIuk4emf94cl1x6c4RrWTznBNZfNEXFcaUH6D31jaRFwcUTM2+G8j9L365qU4RxIujZOjIjNPcSS+dk/SRNJkuWJEbFJ0t3AkF5Oj/S6r+z4d2DWG/fBFc884AJJuwFIOlzSMGABMDnto2sDTu7hu/cBfyjp4PS7I9Py3wN7lJx3O8ntIul5x6UfFwCfTMs+AozoI9a9gJfT5HYkSQuy2wCguxX6CZJb31eBZySdlV5Dko7t4xrWjznBFc+1JP1ri9OFU/6DpKX+E2A58DhwDfA/O34xIl4k6Te7RdKjvHmL+DPgT7oHGYBLgPHpIMZS3hzN/QfgJEmLSW6VV/UR623AQEmPAV8D7i85thE4StJDJH1sV6TlnwSmpvEtwa+BtzL8NhEzKyy34MyssJzgzKywnODMrLCc4MyssJzgzKywnODMrLCc4MyssJzgzKyw/hdzL/ffaV9nywAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Returns an error.... (at time of writing)\n",
    "from sklearn.metrics import plot_confusion_matrix\n",
    "\n",
    "plot_confusion_matrix(clf, X, y)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Classification report¶"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.86      0.83      0.84        29\n",
      "           1       0.85      0.88      0.86        32\n",
      "\n",
      "    accuracy                           0.85        61\n",
      "   macro avg       0.85      0.85      0.85        61\n",
      "weighted avg       0.85      0.85      0.85        61\n",
      "\n"
     ]
    }
   ],
   "source": [
    "from sklearn.metrics import classification_report\n",
    "\n",
    "print(classification_report(y_test, y_preds))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Each term measures something slightly different:\n",
    "\n",
    "* Precision - Indicates the proportion of positive identifications (model predicted class 1) which were actually correct. A model which produces no false positives has a precision of 1.0.\n",
    "* Recall - Indicates the proportion of actual positives which were correctly classified. A model which produces no false negatives has a recall of 1.0.\n",
    "* F1 score - A combination of precision and recall. A perfect model achieves an F1 score of 1.0.\n",
    "* Support - The number of samples each metric was calculated on.\n",
    "* Accuracy - The accuracy of the model in decimal form. Perfect accuracy is equal to 1.0, in other words, getting the prediction right 100% of the time.\n",
    "* Macro avg - Short for macro average, the average precision, recall and F1 score between classes. Macro avg doesn't take class imbalance into effect. So if you do have class imbalances (more examples of one class than another), you should pay attention to this.\n",
    "* Weighted avg - Short for weighted average, the weighted average precision, recall and F1 score between classes. Weighted means each metric is calculated with respect to how many samples there are in each class. This metric will favour the majority class (e.g. it will give a high value when one class out performs another due to having more samples)."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "When should you use each?\n",
    "\n",
    "*It can be tempting to base your classification models perfomance only on accuracy. And accuracy is a good metric to report, except when you have very imbalanced classes.\n",
    "\n",
    "*For example, let's say there were 10,000 people. And 1 of them had a disease. You're asked to build a model to predict who has it.\n",
    "\n",
    "*You build the model and find your model to be 99.99% accurate. Which sounds great! ...until you realise, all its doing is predicting no one has the disease, in other words all 10,000 predictions are false.\n",
    "\n",
    "*In this case, you'd want to turn to metrics such as precision, recall and F1 score."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\John\\anaconda3\\lib\\site-packages\\sklearn\\metrics\\_classification.py:1221: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0.0</th>\n",
       "      <th>1.0</th>\n",
       "      <th>accuracy</th>\n",
       "      <th>macro avg</th>\n",
       "      <th>weighted avg</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>precision</th>\n",
       "      <td>0.99990</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.9999</td>\n",
       "      <td>0.499950</td>\n",
       "      <td>0.99980</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>recall</th>\n",
       "      <td>1.00000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.9999</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>0.99990</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>f1-score</th>\n",
       "      <td>0.99995</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.9999</td>\n",
       "      <td>0.499975</td>\n",
       "      <td>0.99985</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>support</th>\n",
       "      <td>9999.00000</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.9999</td>\n",
       "      <td>10000.000000</td>\n",
       "      <td>10000.00000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                  0.0  1.0  accuracy     macro avg  weighted avg\n",
       "precision     0.99990  0.0    0.9999      0.499950       0.99980\n",
       "recall        1.00000  0.0    0.9999      0.500000       0.99990\n",
       "f1-score      0.99995  0.0    0.9999      0.499975       0.99985\n",
       "support    9999.00000  1.0    0.9999  10000.000000   10000.00000"
      ]
     },
     "execution_count": 79,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Where precision and recall become valuable\n",
    "disease_true = np.zeros(10000)\n",
    "disease_true[0] = 1 # only one case\n",
    "\n",
    "disease_preds = np.zeros(10000) # every prediction is 0\n",
    "\n",
    "pd.DataFrame(classification_report(disease_true, \n",
    "                                   disease_preds, \n",
    "                                   output_dict=True))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "You can see here, we've got an accuracy of 0.9999 (99.99%), great precision and recall on class 0.0 but nothing for class 1.0.\n",
    "\n",
    "Ask yourself, although the model achieves 99.99% accuracy, is it useful?\n",
    "\n",
    "To summarize:\n",
    "\n",
    "* Accuracy is a good measure to start with if all classes are balanced (e.g. same amount of samples which are labelled with 0 or 1)\n",
    "* Precision and recall become more important when classes are imbalanced.\n",
    "* If false positive predictions are worse than false negatives, aim for higher precision.\n",
    "* If false negative predictions are worse than false positives, aim for higher recall."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Regression model evaluation metrics¶"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We'll check out the following.\n",
    "\n",
    "* R^2 (pronounced r-squared) or coefficient of determination - Compares your models predictions to the mean of the targets. Values can range from negative infinity (a very poor model) to 1. For example, if all your model does is predict the mean of the targets, its R^2 value would be 0. And if your model perfectly predicts a range of numbers it's R^2 value would be 1.\n",
    "* Mean absolute error (MAE) - The average of the absolute differences between predictions and actual values. It gives you an idea of how wrong your predictions were.\n",
    "* Mean squared error (MSE) - The average squared differences between predictions and actual values. Squaring the errors removes negative errors. It also amplifies outliers (samples which have larger errors)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Import the RandomForestRegressor model class from the ensemble module\n",
    "from sklearn.ensemble import RandomForestRegressor\n",
    "\n",
    "# Setup random seed\n",
    "np.random.seed(42)\n",
    "\n",
    "# Create the data\n",
    "X = boston_df.drop(\"target\", axis=1)\n",
    "y = boston_df[\"target\"]\n",
    "\n",
    "# Split into train and test sets\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2)\n",
    "\n",
    "# Institate and fit the model (on the training set)\n",
    "model = RandomForestRegressor()\n",
    "model.fit(X_train, y_train);"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### R^2 Score (coefficient of determination)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.8654448653350507"
      ]
     },
     "execution_count": 81,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\n",
    "# Calculate the models R^2 score\n",
    "model.score(X_test, y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2.220446049250313e-16"
      ]
     },
     "execution_count": 82,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.metrics import r2_score\n",
    "\n",
    "# Fill an array with y_test mean\n",
    "y_test_mean = np.full(len(y_test), y_test.mean())\n",
    "\n",
    "r2_score(y_test, y_test_mean)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1.0"
      ]
     },
     "execution_count": 83,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#And a perfect model would get a score of 1.\n",
    "r2_score(y_test, y_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Mean Absolute Error (MAE)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2.136382352941176"
      ]
     },
     "execution_count": 84,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Mean absolute error\n",
    "from sklearn.metrics import mean_absolute_error\n",
    "\n",
    "y_preds = model.predict(X_test)\n",
    "mae = mean_absolute_error(y_test, y_preds)\n",
    "mae\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "\n",
    "Our model achieves an MAE of 2.203. This means, on average our models predictions are 2.203 units away from the actual value.\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>actual values</th>\n",
       "      <th>predictions</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>173</th>\n",
       "      <td>23.6</td>\n",
       "      <td>23.081</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>274</th>\n",
       "      <td>32.4</td>\n",
       "      <td>30.574</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>491</th>\n",
       "      <td>13.6</td>\n",
       "      <td>16.759</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>72</th>\n",
       "      <td>22.8</td>\n",
       "      <td>23.460</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>452</th>\n",
       "      <td>16.1</td>\n",
       "      <td>16.893</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>76</th>\n",
       "      <td>20.0</td>\n",
       "      <td>21.644</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>316</th>\n",
       "      <td>17.8</td>\n",
       "      <td>19.113</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     actual values  predictions\n",
       "173           23.6       23.081\n",
       "274           32.4       30.574\n",
       "491           13.6       16.759\n",
       "72            22.8       23.460\n",
       "452           16.1       16.893\n",
       "76            20.0       21.644\n",
       "316           17.8       19.113"
      ]
     },
     "execution_count": 85,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.DataFrame(data={\"actual values\": y_test, \n",
    "                   \"predictions\": y_preds})\n",
    "\n",
    "df.head(7)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAd4AAAD4CAYAAABCHIdcAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8vihELAAAACXBIWXMAAAsTAAALEwEAmpwYAAAu9ElEQVR4nO3de3Qc1Z0n8O+vJb/kByBZ4Ieibh6WZNmCOPLxYOIzAxMgMJnExB4SHEHIbEDYDCwxm0OYKMd74jNiAywhm2GDo4RxjNWbhASIGThLMGYIj6yZkSF+y5iMJb+xsY3xE1vqu39Ut2nJXd1VXa9b1d/POXXaXVZ33aquur+6r7qilAIRERH5IxZ0AoiIiEoJAy8REZGPGHiJiIh8xMBLRETkIwZeIiIiH5X7ubGxY8eqRCLh5yaJiEJvzZo1HyilqoNOB7nD18CbSCTQ1dXl5yaJiEJPRHqDTgO5h1XNREREPmLgJSIi8hEDLxERkY8YeImIiHzEwEtEROQjS4FXRHpEZL2I/ElEutLrKkVkpYhsTb+e521SoyWZBBIJIBYzXpPJoFOkHx4j0gHPQ3KbnRLvVUqpTyulpqff3w9glVJqEoBV6fdkQTIJtLYCvb2AUsZraysv6Gw8RqQDnofkBbEyLaCI9ACYrpT6IGvdFgBXKqX2iMh4AK8qperzfc/06dMVx/Ead829OUblxeNAT4/fqdETjxHpQJfzUETWZBV6KOSsBt5tAA4BUAB+qpTqEJEPlVLnZv3NIaXUWdXNItIKoBUAamtrm3tzncUlJhYz7p4HEwFSKf/ToyMeI9KBLuchA2+0WK1q/qxS6jMArgfwDyLyl1Y3oJTqUEpNV0pNr67mE88AoLbW3vpSxGNEOuB5SF6wFHiVUrvTr/sAPAtgBoD301XMSL/u8yqRUdPeDlRUDFxXUWGsJwOPEemA5yF5oWDgFZGRIjI6828A1wLYAOA5ALem/+xWACu8SmTUtLQAHR1GO5GI8drRYawnA48R6YDnIXmhYBuviFwEo5QLGJMq/B+lVLuIVAF4CkAtgO0AblRKHcz3XexcRRQtySTQ1gZs325Uv7a3Myh5gW280VJwdiKl1H8CuCzH+gMAPudFoohIf5mhNsePG+8zQ20ABl+ifPjkKiIqSlvbJ0E34/hxYz0RmWPgJaKibN9ubz0RGRh4iagoHGpDVBwGXiIqCofaEBWHgZeIisKhNkTFKdirmYjITEsLAy2RXSzxEhER+YiBl4iIyEcMvERERD5i4CUiIvIRAy8REZGPGHiJiIh8xMBL+komgUQCiMWM12SS6dENjwmRbQy8QWGGlV9m6pveXkCpT6a+Ceo46ZYeHfCYEBWl4Hy8buJ8vGmD51MDjGft8bE/n0gkjIx8sHgc6OnxOzX6pUcHPCa+4Xy80cLAGwRmWIXFYkYpajARIJVienTAY+IbBt5oYVVzEDifWmG6TX2jW3p0wGNCVBQG3iAwwypMt6lvdEuPDnhMiIrCwBsEZliF6Tb1jW7p0QGPCVFR2MYblGQSaGszqpdra42gywyLiHJgG2+0sMQblJYWoyNVKmW8MuhqjyPAiMgNnI+XyILBI8AyQ1YB3jMRkT0s8RJZ0NY2cNg1YLxvawsmPUQUXgy8RBZwBBgRuYWBl8gCjgAjIrcw8BJZwBFgROQWBl4iCzhklYjcwl7NRBa1tDDQEpFzLPESERH5iIGXiIjIRwy8REREPmLgJSIi8hEDLxERkY8YeImIiHzEwEtEROQjy4FXRMpE5B0ReT79vlJEVorI1vTred4lk4iIKBrslHjvAbA56/39AFYppSYBWJV+T0RERHlYCrwiUgPgCwB+nrV6NoBl6X8vA3CDqykjIiKKIKsl3h8BuA9AKmvdBUqpPQCQfj0/1wdFpFVEukSka//+/U7SSkREFHoFA6+I/C2AfUqpNcVsQCnVoZSarpSaXl1dXcxXEBERRYaVSRI+C+BLIvI3AIYDGCMinQDeF5HxSqk9IjIewD4vE0pERBQFBUu8Sql/VErVKKUSAG4C8IpS6mYAzwG4Nf1ntwJY4VkqiYiCkkwCiQQQixmvyWTQKaKQczIt4A8APCUi3wSwHcCN7iSJiEgTySTQ2gocP2687+013gOcI5KKJkop3zY2ffp01dXV5dv2iIgcSSSMYDtYPA709PiWDBFZo5Sa7tsGyVN8chURkZnt2+2tJ7KAgZeIyExtrb31RBYw8BIRmWlvByoqBq6rqDDWExWJgZeIyExLC9DRYbTpihivHR3sWEWOOOnVTEQUfS0tDLTkKpZ4iYiIfMTAS0RE5CMGXiIiIh8x8BIREfmIgZeIiMhHDLxEREQ+YuAlQnAT0HDiG6LSw3G8VPKCmoCGE98QlSbOTkQlL6gJaDSZ+IZCgLMTRQurmsl1b9yZxM7yBFISw87yBN64U+/606AmoOHEN0SliYGXXPXGnUlMe7wVNf29iEGhpr8X0x5v1Tr4BjUBDSe+ISpNbOMlVyU62jASxwesG4njSHS0AT/Rs+GyvX1gWyvgzwQ07e3Ay3+fxH8/3YZabMd21OL7Q9pxdbuex4mCU11dXb527drnAEwFC0xhkAKwoa+v77bm5uZ9g/+TgZdcNaE/dz2p2XodtLQA8TeTSHS0YUL/duwuq0XPre2Y5XEPpxYk8VVpRXn6RiWBXvxMWtMXJYMvfaK9vT0xbty4kdXV1YdisZh/HXOoKKlUSvbv39+4d+/enwP40uD/550TuWp3We56UrP1WkgmMWvZwOrxWctavR/b09aG8lMDawfKTx0H2tq83S6FTiKRGFFdXf0Rg244xGIxVV1dfRhGDcXZ/+9zeijielrbcQwDJw4/hgr0tGo8cXhb28B6ZsB473UAZO8qskhEwKAbLunfK2eMZeAlV836SQveWdCBnWVxpCDYWRbHOws6MEvT9l0AwQVA9q4iKkkMvOS6WT9pQU1fD2IqhZq+Hr2DLhBcAGxvN3pxZfOjVxdRkZ588slzRaT5nXfeGV7sdzz//POjV65cOdJJOioqKqYNXjdjxoz6p59+ekz2usWLF59/8803m17IM2bMqH/ttdcqzP7fKwy8VFjUn2sYVABsaQE6OownZogYrx0dfGwVObZkCSonTEBTLIbmCRPQtGQJKt343l/96leVn/nMZ44uX7686O975ZVXRr/++uuj3EhPthtvvPHAL3/5ywHpevrppytvvvnmg25vyykGXsov81zD3l5AqU+eaxil4BtkAGxpMR5TlUoZrwy65NCSJahcuBDxPXswVClgzx4MXbgQcafB9/Dhw7Gurq5RS5cu7Xn22WfPy6zv6+tDa2trTV1dXWNdXV1je3v7+QAwceLEpj179pQDwGuvvVYxY8aM+i1btgx98sknq5csWXJBQ0ND44svvjhq7ty5iaVLl575vkxp9vDhw7GZM2fWNTY2Tq6rq2vs7Ow8N1/6brnllkOrVq0658SJEwIAW7ZsGbpv374h11577dGWlpbaqVOnTr7kkkumLFy4cEKuz2eXopcuXXre3LlzEwCwe/fu8s9//vMXT506dfLUqVMnv/TSSyMB4IUXXhjV0NDQ2NDQ0Dh58uTGQ4cOWY6nDLyUX1Adj/zGAEgRsXgxJp48OTBvP3kSscWLMdHJ9yaTyXOvvPLKw5deeunH5557bv8bb7xRAQCPPPJIdW9v77CNGzduevfddzfddtttB8y+o76+/tTXv/71/fPnz3+/u7t703XXXXfU7G8rKipSL7zwwnubNm3a/Ic//OHd7373uzWpVMo0fePGjeu/7LLLjj399NPnAMCyZcsqv/SlLx2KxWL44Q9/uGvDhg2bu7u7N7755puj33rrrRFW9/uOO+741L333vv+hg0bNj/77LN/nj9/fiK93+N+/OMf93Z3d29avXp196hRo8wTN0ioAm/Uazy1xJ639vAkpYDt3YuhdtZb9dRTT1XOmzfvEADMnTv3YKa6+ZVXXhkzf/78/UOGDAEAXHDBBf1OtpORSqXkW9/6Vk1dXV3jVVddVbdv376hO3fuzPvsia985SsHf/3rX58HAM8880zlLbfcchAwgnBjY+PkxsbGxq1btw5fu3at5TbqN998c8w999xT29DQ0PjFL37xkqNHj5YdOnQodvnllx/99re//al/+qd/Ov+DDz4oy+y/FaF5gAZncglIbW3uJ/mz5+3ZeJKSBsaNw6k9e84OsuPG4VSx37l3796y1atXj3n33XdH3HXXXejv7xcRUY8//vhOpRRE5KyhTmVlZSpTQj1x4oRpIa+8vFz19xuxOpVK4fTp0wIAP/3pTysPHDhQvn79+s3Dhg1TEydObMr3PQDQ0tLy4fe+971PvfHGGxUnT56MzZo163h3d/fQxx577II1a9Zsrq6u7p87d27i5MmTZ32PiJz5d6a6GgCUUujq6to8atSoAfv4wAMP7L3hhhsOr1ix4pwrrrhi8osvvvjutGnTTuZLX0ZoSrylUuOpHfa8tY4nKWlg0SLsGj4cA6o9hw9HatEi7Cr2O5cvX37enDlzDuzevXv9rl271u/du3ddTU3NqZdeemnU1Vdf/dGSJUuqT58+DQB4//33ywCgpqbm1JtvvlkBAE899dSZNtzRo0f3HzlypCzzPh6Pn1qzZk0FYFRn9/X1CQAcPny4bOzYsaeHDRum/vVf/3X07t27C5bYzznnnNTll19+5LbbbkvMmTPnIAAcOnSobMSIEanKysr+HTt2lL/66qvn5PpsVVXV6bfffnt4f38/VqxYcSa9s2bN+ujBBx88P/P+j3/84wgA2Lhx47AZM2acaG9v39vU1HRsw4YNlkvRoQm8rPEMCHveWseTlDQwfz4OPvooesePxykRYPx4nHr0UfTOn4+ie/f+5je/qZozZ86h7HWzZ88+tHz58sqFCxfur6mpOdXQ0DClvr6+8YknnqgEgEWLFu2+7777apubm+vLysrOlBbnzp374QsvvHBupnPV3Xffvf+Pf/zj6KampsmrV68eOWLEiBQA3HbbbQfXrl07curUqZM7OzsrL7zwQkulyZtuuungli1bRmSqmWfOnHli6tSpxydNmjTllltuSTQ3N+dsV/7+97+/a/bs2ZfMnDmz/oILLjidWd/R0bHj7bffHllXV9d48cUXT3nssceqAeChhx46f9KkSVPq6+sbR4wYkfq7v/u7w1aPZ2jm4+XcpaQ9hydpMmkUjrdvN2ry29t5f0OGl19++dTVV1+9Puh0kD1r164de9lllyUGrw9NiZc1nqQ9BydpKYzaIiJDaAIvazxJew5OUjYPE5WO0ARegEMtKQSKPEndbB7miCYivYUq8DJHoahy63HRrLIm0l94Ai9zFIowt/owsMqaSH8FA6+IDBeRfxeRtSKyUUS+n15fKSIrRWRr+vW8Qt/lCHMUijC3+jBwRBOR/qyUeD8G8NdKqcsAfBrAdSJyOYD7AaxSSk0CsCr93jshylFYI+5ACR88N/owcIpf8kpZWVlzQ0ND46RJk6Zcf/31Fx05cqToGtPsiRG++tWvxtesWWP68InB0wg+9NBD1Y899lhVsdvWQcEDpwyZAcdD0osCMBvAsvT6ZQBu8CKBZ4QkR2GNuAM8eI5x2B0BAJYsqcSECU2IxZoxYUITlixxPC3gsGHDUt3d3Zu2bt26cciQIeqRRx6pzv7/vr6+or7317/+dW9zc7PpwzEGTyN433337b/rrrtMJ2IIA0t3LCJSJiJ/ArAPwEql1FsALlBK7QGA9Ov5Jp9tFZEuEenav39/8SkNSY7CGnEHePAc47A7wpIllVi4MI49e4bCmBdwKBYujLsRfDNmzZp19L333hv2/PPPj/6Lv/iLui9+8YsX1tfXT+nr68Mdd9xRM3Xq1Ml1dXWNDz/88FjAeAbz17/+9dqLL754ypVXXnnJBx98cGaegOzJ6H/729+OaWxsnFxfX984c+bMulzTCN57770TFi1adAFgPL7xsssua6irq2u85pprLt6/f39Z5jsXLFgwsampaXIikZj64osvjgKArq6u4U1NTZMbGhoa6+rqGtevXz/MrWNih6XAq5TqV0p9GkANgBkiMtXqBpRSHUqp6Uqp6dXV1YU/YCYkOQqHhTgQouYEnUVp2F3JXQNuWLx4IgZPAnDyZAyLFzuaFjDj9OnT+P3vfz+mqanpBACsW7du5MMPP7zrz3/+88Yf/ehHY88555z+DRs2bF67du3mZcuWVXd3dw9dvnz5ue+9996wLVu2bPzFL37R+/bbb48a/L27d+8uv+uuuxLPPPPMn7ds2bLpd7/73Z8LTSP4jW9848IHHnhg57vvvrtpypQpJ77zne+cmWu3r69P1q9fv/nBBx/csXjx4gkA8M///M/Vd9555/vd3d2b1q1bt/nCCy8seuIIJ2zV0SulPgTwKoDrALwvIuMBIP26z+3EnSUEOQqHhTgQkuYEzzDKDFCS14Ab9u7NPZmA2XqLPv7441hDQ0NjU1NTY01Nzal77rnnAwC49NJLjzU0NJwCgJdffnnMU089VdXQ0NA4bdq0yYcOHSrftGnT8D/84Q+jv/KVrxwsLy9HIpE4PXPmzCODv//VV18dOWPGjCOZ7yo0veCBAwfKjhw5UvaFL3zhKADcfvvtB1avXn0moN94442HAOCKK644tnPnzqEAMHPmzGOPPPLI+La2tnFbt24dOnjGIb9Y6dVcLSLnpv89AsDVALoBPAfg1vSf3QpghUdpDBUOC3EgJM0JnnAzykQkgJfkNeCGceNyl+LM1luUaePt7u7etGzZsh3Dhw9XgDFhfeZvlFLyyCOPbM/83a5du9bPmTPnI2DgtHu5pKcXdJLEATLpKy8vR39/vwDA/PnzD65YseK9ESNGpK6//vq65557brRrG7TBSol3PIB/E5F1AP4DRhvv8wB+AOAaEdkK4Jr0+5LHYSEOhKQ5IZ+iY55bUSZCxcSSvAbcsGjRLgwfPmBaQAwfnsKiRUVPC2jVNddcc/jxxx+v/vjjjwUA1q1bN+yjjz6K/dVf/dWR3/zmN5V9fX3o7e0dsnr16rMC3lVXXXXsrbfeGt3d3T0U+GR6wcHTCGZUVVX1jxkzpj/TfvvEE09UzZw5M+fMQxmbNm0aOnny5I+/973v7bv22ms//NOf/jTCjf22y0qv5nVKqWlKqUuVUlOVUovT6w8opT6nlJqUfi16yqmo4bCQ4iXRggR6EEMKCfQgCZeDroelQUcxz60oE6FiYqleA47Nn38Qjz7ai/HjT8GYF/AUHn20F/Pne55HL1y48IOGhoaTTU1NkydNmjTl9ttvj58+fVpuueWWDy+66KKP6+vrp3zzm9+snTFjxllVzRMmTOj78Y9/3PPlL3/5kvr6+sYvf/nLFwFnTyOY/ZmlS5du+853vlNTV1fXuG7duhE/+MEPdudL3/Llyyvr6uqmNDQ0NG7dunX4HXfcEUjv6NBMC1hqMpl4dh5aURG6AqAtnu+zxxtwNCugW/NexmJG1B9MxLgTzAjBHISleA2Y4bSA4RT6aQFLTQRqXW3zvLDm8QYcFVrdat+2UkwMSXV0KV4DVBoYeDUWgk7crvK8Tc/jDTiqGnUrylgJ4CGqjvbzGohInzQKAQZe0obnbXoeb8BxodWNKGMlgLPX0ll0rwRQSiGVSrnX5Zc8l/69Urn+j4GXtOH5aCKPN+BZ1ajdolihAM5eS2fRvRKgp6fnxP79+89h8A2HVCol+/fvPwfAhlz/z85VpBXP+/yEoFPRAF70MApTryWffi+rfdKCcv75569duXLldgBTwQJTGKQAbOjr67utubn57IdLKaV8W5qbmxVRmHR2KhWPKyVivHZ2+pyAeFwpIyYMXOJxZ98b+I5Z0NmpVEXFwP2uqPAkrV4dZrcA6FI+5tVcvF1Y4iUyoUXBUPeimJfcGmJlgRa/dR4iskYpNT3odJA7WGVBZKKtDZh9PIltSKAfMWxDArOPJ/1t9yvl9lgfO4Fx6BL5iYHXCxyXEAmf7U3iZ2hFAr2IQSGBXvwMrfhsr4+/Zyk/v9rnm45SG75HwWHgdZvu4xLIsgfL2jASA7u6jsRxPFjmY5G3lItipXzTQZHGwOs23cclkGUT+3NXaZqtt8zt4UFRVco3HRRpDLxuc9guxVpqfUg8d5Wm2XpLWCNiT6nedFCkMfC6zUG7FPNkzXhR1ckaEVdk36COHWssvFmlsGDgdZuDzJp5smY8qOpUvblrPszWW1VKNSWDb1APHDAW3qxSaPg5aLhkHqBR5MMJRJSah061DXHVD1HbEFfz0KlEPE1tNGn6gIgdZXGV60kNO8riRX+nj8+Z0ILZwy50fPCFW8AHaERq8XVjJRN4i3R3Vac6ioE56FFUqLuropGDZsfCu6s61ZGquDeBUeNI9DXk/o2/huLTpvtTl9wmknt/s5eo3awy8EZr8XVjDLz5HamKq1y5yJGqeNBJcyw7Fs7LEXxcDYx5IlHQBeF4PHetRr4gWSjNZoEoasEngyVeLmFffN0YA28B+W7lfY4Sbgeo7MxyG+Lm++iCFHIfxxTE14JwrmNotzBu5e9LrcSb65hoWLnxCRcuJgbeaC2+bqyYwBt0CcVXhW7lfcpRvKipzb6n6DcJjG4V0czaUXsQ9y1A5TuGds5pK0G1mN8r7NdVdvpvH9mptseMGoQdZXH1+gKNdsali4mBN1qLrxuzG3gtn7Nhz0UyCt3K+1SM8aIE5WeJ16wddR46fauSdesYWq1GtnMJaNwEbp/uO+PSicDAG63F143ZDbyW2sN0v/DsyuSgZoHXw4a77E273bu6UBvvUVS4VlIxO2/KynIfUi/uZdxqd/X6Jsjnezr3FLpONGjPV0q5diIw8EZr8XVjdgOvpR6gkchFcsi3Xx7kKFYCo9Pe1YUCu1s/mdm92IIF/t2juXVaelHrE/rOWBZqhpy257t2ibHEyyXH4uvG7AZeS2MeQ5+LmPA5elipCnard7UfP5lZxulXKcjNipiCaba5sdDfq1ro1ryjLF70PrpaicY2Xi45Fl83Zjfw5uudmhHlITg5c1yPck0/Oz853QUtqhAt8C2dNg9o6FtnCg3krTBqxYo9hd24xNwes87AG63F143Z7tVs4QqI+kMnBrNyM1IMPzs/Ocn4OzuV+saQgdXU3xjSqWfQ8CvyFlGFYClput7hFGjbVZ2dxQXP9P5mN38Uc9/pxY0NA2+0Fl83ZjvwWjiDS+0xi148clApnx9woYrP00Nzo+VnsdKLWhCdi8UW0mY7+Tk+MLgnvNXD6cXPwcAbrcXXjRX1AI0COXSg7VUBlAi8eORghtvVY14wK41vQ9zfhBT67f08Mb0Ikro3BFu49mxdnib7uy099tvO4fSiDwMDb7QWXzfmxZOr8uY5DgKj2x1a3GJpiFUButYgWmHW/tzvsKrdFiu/vd+d/tz6UQMczuY6O8fE5Pfqh9g+nCzxcim0+Loxrx4ZmfP6chAYLX00oBKB03ivcw2iFVp0prPy2+teYszFqwe4BHGnZ/dEd/H3yrXpTFwvdvcZeKO1+LoxX5/VbPVCypEpWPpogMOYLOdjxe6bzjo71emhA3O100O9GqdjwspvH8Y7nELDdIpJf1DHwe6J7nI6sysOBp8uxXwtA2+0Fl835mvgdZA5WhmKoEXJKx8H++ZpmtysDnXyPQ4yWsu/fdjq9L2YpCNPAPT08BRzY+xBgty60WXgjdbi68a0K/Ga/E0KyDmcIPuj2veuNdm3wQ8WyG4vtpLZFJ036VYCdJAjav/bFytPB6Ni41BgM0VpUrXjVsUYA2+0Fl835mvgddIBJiszzQTfXB/VehiTyb5lZ3h2hw05GkOrSUZ4hoMcsajfXofSb4E0vL4g/+QSxQTGwGaK0uRGjyVeLrkWXzfm+3y8xQ75sHC3r1scOYuFKj67D8pwVNLT7dGeDn5A2x8tIgi4HqctpMGsx7yT8zvfTFGe37gGebOT3nYKonpl4HFkGy+Xwn8AfArAvwHYDGAjgHvS6ysBrASwNf16XqHv8jvwFjUkyGJg0OSG2pRZ6WXADEA2g6GjMbQB3qkE0us9m819z/f9RccSC2ko9CTGzKnx+oJOtaPM2vy3ZsG8RfLcxOlQO+BEjh/wmBj9K9irmYtS1gLveACfSf97NIB3ATQCeAjA/en19wN4sNB3+Rl4LWeOhcYtZjKnHFeLpfwhoEzE0nhfmwHB0RjaPD+Il4co73ngYMO2PmrzBsfsZ6mqKv5mz8qjRi1UAKnbR9qb0tHs+H8wKvfGTox0sJNucnJSenCTycAbrcX+B4AVAK4BsAXA+PS68QC2FPqsn4HX7miir6FTHZM8pd9CA/FyXagBFost5fU2BxwW05M7+7DkehqW7RskmxmhFk0CNhNhpeRpd1+sPGq0UAVQRYVS22OFv2ewnD+dFztpUcESu9Pr1oNmFQbeaC32/hhIANgOYAyADwf93yGTz7QC6ALQVVtbq/xSzGiieehUvRJXqUKZwOCL0OxCrarK/Xkfcv28eX12TlhV9Uk6Cw04zDOGttj7DksxyUFGqEXTss30Wyl52t0Xq48azXVqZP+mrj05zIudtMBSE4zTuzWWeLkUWKz/ITAKwBoAc9LvPxz0/4cKfUeQJd5cw2bMrg/TafHMLqKAMpF8zPL61xc4vEnIypmPVMXV3VWdpjG7qqpwdbelwOhnRyiv2Cixe3EfZ6npwQLXJukw2ckTo3LvpFvj432Z49uDmi4G3mgt1v4IGALg9wDuzVqndVVz9rlvNmzG7GESpr19zS7CAKvNCh2Ds/J6l24SrPRLy3XcB5eyLAUEBxmhJx2VfJCrOtRJfu5WLLBUYrQqxw/g9Rhps5vqVOa6zNfnw8516/LJxcAbraXwHwAC4EkAPxq0/uFBnaseKvRdQfVqNgukgx8mkVnuriocVQbcgZtcqCdGVrmXSbnFpZsEK/E733HP8KPqz+0g5jmPOqK5FQvs9Gq2y+vx8WYl3gHHecEC7U4OBt5oLYX/AJgFQAFYB+BP6eVvAFQBWJUeTrQKQGWh7/J9HG+GhYdJnHV9pXOpfuCsu+Sz7sBN2j7vrso9VtH3as5sJkFsH6rO7lyWJ7OxEr/NSxdSMD0DDpIHRb1MFbmTQk32JgoFNFtBT5v6cf95veu5bvRybkyz6hAG3mgtvm4ssMCb52oudH1ZuQPv7Mz9RCez6zrQp1vlCETZDzQwOpcVzmwKlXgrKsx7Qdtv5FXFZ4QmCd1m8uQku7+NlXsC2/cNWvQIC4YfNRGZErtpJ0oNjzMDb7QWXzcWWOB1cDVbuQM3+5uyssKfDcSZ0nzxTyeyNBLJtW7NDuSZZ9WNzTo5P0y3VcIlXqV8LGwWcZyDKggz8EZr8XVjgQVeVXy7lJXYka/aVbOmogHc6LxZMBMq9EdeF3FMMtcjVXFXNmvlGNo+zlo3QGvOTmS0eZyD/FkYeKO1+LqxoAKv0wum0LWc78ZZs6aiAbQpWFk4SEUfR486KmV4UuJ1tMMlrJgL3cZxDvJ6YeCN1uLrxoIKvF5fMGEtoIQl3Y7TaTeIuTDm1lEbr5v7ohlPk+9x1XGhmgsv942BN1qLrxsLKvD60VclrPlhGNLta0mjiCjpeq9mF9OmE8+Tb/NCt5ueQjVbXu4bA2+0Fl835mrgNcvJcqy3mnGHIQiVIl87+WpT/56DzmmzwOvk232WuN305AuuXu8bA2+0Fl835lrgNbsCTAa+v76g09/qQHKVr/FG56E8OqfNAq+Tb/epV8Wkx+zm3Ot9Y+CN1uLrxlwLvEWM33HSQYqC5etNkc4ngs5ps8Bp8t0Yc+9merz6rlwYeKO1+Lox1wKv3cceWrjtDHlhIvJ8awbQuepD57RZ4CT5XgwJd/Nwso2Xi53F1415XeLtj+Uu8VqZ2STkhYngRamBXOd98bCHth+KTY6V69Pj0UQFsVczF6uLrxvzuo33FyMXFD2zScgLE8HiwdNThH4Xr58sqjsG3mgtvm7M617NTmc2iepF6zldqguC+gF1PXF0+V1cEKFdKQoDb7QWXzfm9TheyxenrhllWOnQQO5i6c7W6aFzqVKH38UlnZ25JyLR4TD7gYE3WouvG/M68FrKA3XOKIthMUo4udco+NkAiyOZtJnN/Ws3DbZPD52LYjqnza7O3FNvhva6tYmBN1qLrxvz48lVOgcJU8VGRYtRwuvepEHdzGRv1mzuX7ulO9unh86lyijdZOp43fqIgTdai68bC3J2ojN0yyitZo65gnO+8cxFPLkrF52r77PT5laJ1/bpoXtAiEqzim7Xrc8YeKO1+LoxLQKvbhmlk3ESuT43eKmoUF8bNN+unTwrb34XcKaenbZ5OPupRcWU7myfHlEqVepMt+vWZwy80Vp83ZgWgVe3tiIrd/J2n9Q1aNlRFs/Z29tJiffuquADzuC0Ze9jsTcCRcXRqJQqdVbiNzgMvNFafN2YDoFXt96Rlh7sbhKcU8DZpTyLf3cUxjOsCzHL78zS7WcJxKvfknFUUyX8wzDwRmvxdWNuBl4vn4DjJ0sPdjdJ9OCS7GnkLgGbPdHL6k7nPNY6tLnpVnvhtxIORFry8Pdg4I3W4uvG3Aq8TmqddIgXg9NT8KEfJjs8uO02VzvnUVSolFlp2MlO63AHo0MaglLiVa/a8fj3YOCN1uLrxtwKvL700vWJk17DuT6bK4i71eP3rPQEnfGb3EX1Q6JfANTtRC4VZqVaj38PBt5oLb5uzK3A66TUqkO8cCs9uT6ba/GsI1TQVZ0mmd02xAP/Xb2WMhm3nIImw2uCPje8kO9i9bgqjYE3WouvG9OhxKuUfnmClfSY/U32+qoqpYYOVbnjq2477YYcGeFRVKh5WVXwUS0A7iiL57wIdpTFB/xdID+7bne3bsmX8bDEy8XG4uvGdGjjDSM7+xvF+JpXeoezq9Y9KHBo52sm7flfwyc/eGDXSVSrwfPNA573rtc5Bt5oLb5uTIdezWEU1XzMTaV2jOLx3O352fsb2DHJV+0a5gvX7IBmliFDjADMXs1cCixi/Kb+mD59uurq6vJte1ERixlX9mAiQCrlf3p0lEwCra3A8eOfrKuoADo6gJaW4NLlFSv7G9h5k0gAvb1nr6+qAk6cCO+PlOugDxaPAz09rm9aRNYopaa7/sUUiFjQCaDCamvtrS9FLS1G/h2PG4ElHg9Pfl4MK/sb2HnT3m4E1GyZ94OD1vHjQFubxwlySfZBN7N9u3/podBi4A0Bs3ysvT2Y9OiqpcUobKRSxmtUg25Gof0N7Lwxuys4eDD334cpWGUOulnw5d0wWcDAGwKlVpojdwR63uS6K4hS1Q3vhskBtvESkT+i1hCfTBrV5Nu3GzcP7e2e7QfbeKOFJV6iEpdMGv2hYjHjNZks7nveuDOJneUJpCSGneUJvHHnoC+KWtVNqbVtkGtY4iUqYW4VQt+4M4lpj7diJD75omOowDsLOjDrJwxITrHEGy0s8RKVsLY2dzoaJzraBgRdABiJ40h06NFj2a1SPZEbGHh9xIufdLN9OzAPSWxDAv2IYRsSmIek7Y7GE/pzf8BsvZ8ypfreXmNcc2+v8Z7XHwWlYOAVkX8RkX0isiFrXaWIrBSRrenX87xNZvjx4icd3VWZxM/QigR6EYNCAr34GVpxV6W9E3N3We6eyWbr/eRWqZ7ILVZKvL8AcN2gdfcDWKWUmgRgVfo95cGLn3T0AHJXET8AeydmT2s7jmHg8JpjqEBPq7XhNV7WBpmV3sM0fJiipWDgVUq9BmDwyPfZAJal/70MwA3uJit6ePGTjkYdzH0Cmq03M+snLXhnQQd2lsWRgmBnWdxyxyqva4O8Gj7MpiMqlqVezSKSAPC8Umpq+v2HSqlzs/7/kFIqZ3WziLQCaAWA2tra5t5cz3AtAWaPr/Xo0a5E1mhwYnqdBC+GD/s9JJm9mqPF885VSqkOpdR0pdT06upqrzenLT7ohrSkwYnpVgcvM14MH2bTETlRXuTn3heR8UqpPSIyHsA+NxMVRZmL3KcH3RBZo8GJeVdlEv/jwCdjgDMdvMZWAoA76WhpcXeX2HREThRb4n0OwK3pf98KYIU7yQkBBw07fNANaSngE9OtDl5+itJjp8l/VoYT/RLA/wNQLyI7ReSbAH4A4BoR2QrgmvT76OOYICLXudXBy08a1NBTiFnp1TxPKTVeKTVEKVWjlHpCKXVAKfU5pdSk9KvJfF8R47Rhh90gic4WwuJj1B47Tf7ik6vscNKww9IyUW5+FB89uOll0xEVi4HXDid35uwGSZSb18VH3vSSZjg7kR1OBu/FYsZFP5iIcctMRN7QYKyyUxzHGy0s8drh5M48hO1YRGeEuX8Cx/6QZhh4bUqiBQn0IIYUEuhB0uo4Q3aDpLAqoqpWqzjNm17SjVLKt6W5uVmFWWenUhUVShm5j7FUVBjrLX9BPK6UiPFq+YNEAYrHB570mSUez/nnjq8Tt2mXIPsAdCkf82ou3i5s47UhAk1FRPbZ7J+g5XWSTIb6sXFs440WBl4b2D+KSpLNSMrrxH0MvNHCNl4b2FREJclm/wReJ0T5MfDawP5RVJJs9ubndUKUHwOvDXxMHJUsG49p4nVClB/beImINMc23mhhiZeIiMhHDLxEVJBWD8QgCrnyoBNARHob/IjyzIOrALbbEhWDJV4iyosTaxG5i4GXiPLiHANE7mLgJaK8+EAMIncx8BJRXnwgBpG7GHiJKC8+EIPIXezVTEQFtbQw0BK5hSVeIiIiHzHwEhER+YiBl4iIyEcMvERERD5i4CUiIvKRr9MCish+AL1FfnwsgA9cTI7uSm1/gdLbZ+5v9Lm1z3GlVLUL30Ma8DXwOiEiXaU0H2Wp7S9QevvM/Y2+UtxnKoxVzURERD5i4CUiIvJRmAJvR9AJ8Fmp7S9QevvM/Y2+UtxnKiA0bbxERERREKYSLxERUegx8BIREfkoFIFXRK4TkS0i8p6I3B90etwmIp8SkX8Tkc0islFE7kmvrxSRlSKyNf16XtBpdZOIlInIOyLyfPp9ZPdXRM4Vkd+KSHf6d54Z8f1dmD6XN4jIL0VkeNT2V0T+RUT2iciGrHWm+ygi/5jOw7aIyOeDSTXpQPvAKyJlAP43gOsBNAKYJyKNwabKdX0A/ptSajKAywH8Q3of7wewSik1CcCq9PsouQfA5qz3Ud7f/wXgRaVUA4DLYOx3JPdXRCYC+K8ApiulpgIoA3ATore/vwBw3aB1OfcxfT3fBGBK+jM/SedtVIK0D7wAZgB4Tyn1n0qpUwB+BWB2wGlylVJqj1Lq7fS/j8DIlCfC2M9l6T9bBuCGQBLoARGpAfAFAD/PWh3J/RWRMQD+EsATAKCUOqWU+hAR3d+0cgAjRKQcQAWA3YjY/iqlXgNwcNBqs32cDeBXSqmPlVLbALwHI2+jEhSGwDsRwI6s9zvT6yJJRBIApgF4C8AFSqk9gBGcAZwfYNLc9iMA9wFIZa2L6v5eBGA/gKXpqvWfi8hIRHR/lVK7APxPANsB7AFwWCn1EiK6v4OY7WNJ5WOUXxgCr+RYF8kxUCIyCsDTAL6llPoo6PR4RUT+FsA+pdSaoNPik3IAnwHwuFJqGoBjCH81q6l0u+ZsABcCmABgpIjcHGyqAlcy+RgVFobAuxPAp7Le18CotooUERkCI+gmlVLPpFe/LyLj0/8/HsC+oNLnss8C+JKI9MBoOvhrEelEdPd3J4CdSqm30u9/CyMQR3V/rwawTSm1Xyl1GsAzAK5AdPc3m9k+lkQ+RtaEIfD+B4BJInKhiAyF0UHhuYDT5CoRERjtf5uVUj/M+q/nANya/vetAFb4nTYvKKX+USlVo5RKwPg9X1FK3Yzo7u9eADtEpD696nMANiGi+wujivlyEalIn9ufg9FvIar7m81sH58DcJOIDBORCwFMAvDvAaSPNBCKJ1eJyN/AaBMsA/AvSqn2YFPkLhGZBeB1AOvxSZvnd2G08z4FoBZGZnajUmpwZ45QE5ErAXxbKfW3IlKFiO6viHwaRkeyoQD+E8Dfw7jxjer+fh/AV2H02H8HwG0ARiFC+ysivwRwJYyp/94H8N8B/A4m+ygibQD+C4xj8i2l1P/1P9Wkg1AEXiIioqgIQ1UzERFRZDDwEhER+YiBl4iIyEcMvERERD5i4CUiIvIRAy8REZGPGHiJiIh89P8BIMvvB9sOXLUAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "fig, ax = plt.subplots()\n",
    "x = np.arange(0, len(df), 1)\n",
    "ax.scatter(x, df[\"actual values\"], c='b', label=\"Acutual Values\")\n",
    "ax.scatter(x, df[\"predictions\"], c='r', label=\"Predictions\")\n",
    "ax.legend(loc=(1, 0.5));"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Mean Squared Error (MSE)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "9.867437068627442"
      ]
     },
     "execution_count": 87,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Mean squared error\n",
    "from sklearn.metrics import mean_squared_error\n",
    "\n",
    "mse = mean_squared_error(y_test, y_preds)\n",
    "mse"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "MSE will always be higher than MAE because is squares the errors rather than only taking the absolute difference into account.\n",
    "\n",
    "Now you might be thinking, which regression evaluation metric should you use?\n",
    "\n",
    "* R^2 is similar to accuracy. It gives you a quick indication of how well your model might be doing. Generally, the closer your R^2 value is to 1.0, the better the model. But it doesn't really tell exactly how wrong your model is in terms of how far off each prediction is.\n",
    "*MAE gives a better indication of how far off each of your model's predictions are on average.\n",
    "* As for MAE or MSE, because of the way MSE is calculated, squaring the differences between predicted values and actual values, it amplifies larger differences. Let's say we're predicting the value of houses (which we are).\n",
    "Pay more attention to MAE: When being \\$10,000 off is ***twice*** as bad as being \\$5,000 off.\n",
    "Pay more attention to MSE: When being \\$10,000 off is ***more than twice*** as bad as being \\$5,000 off."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### using the scoring parameter"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import cross_val_score\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "\n",
    "np.random.seed(42)\n",
    "\n",
    "X = heart_disease.drop(\"target\", axis=1)\n",
    "y = heart_disease[\"target\"]\n",
    "\n",
    "clf = RandomForestClassifier(n_estimators=100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The cross-validated accuracy is: 82.48%\n"
     ]
    }
   ],
   "source": [
    "np.random.seed(42)\n",
    "cv_acc = cross_val_score(clf, X, y, cv=5, scoring=\"accuracy\")\n",
    "print(f\"The cross-validated accuracy is: {np.mean(cv_acc)*100:.2f}%\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The cross-validated precision is: 0.83\n"
     ]
    }
   ],
   "source": [
    "np.random.seed(42)\n",
    "cv_precision = cross_val_score(clf, X, y, cv=5, scoring=\"precision\")\n",
    "print(f\"The cross-validated precision is: {np.mean(cv_precision):.2f}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Using different evaluation metrics with Scikit-Learn\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Classifier metrics on the test set:\n",
      "Accuracy: 85.25%\n",
      "Precision: 0.85\n",
      "Recall: 0.88\n",
      "F1: 0.86\n"
     ]
    }
   ],
   "source": [
    "from sklearn.metrics import accuracy_score, precision_score, recall_score, f1_score\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "np.random.seed(42)\n",
    "\n",
    "X = heart_disease.drop(\"target\", axis=1)\n",
    "y = heart_disease[\"target\"]\n",
    "\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2)\n",
    "\n",
    "clf = RandomForestClassifier(n_estimators=100)\n",
    "clf.fit(X_train, y_train)\n",
    "\n",
    "# Make predictions\n",
    "y_preds = clf.predict(X_test)\n",
    "\n",
    "# Evaluate the classifier\n",
    "print(\"Classifier metrics on the test set:\")\n",
    "print(f\"Accuracy: {accuracy_score(y_test, y_preds) * 100:.2f}%\")\n",
    "print(f\"Precision: {precision_score(y_test, y_preds):.2f}\")\n",
    "print(f\"Recall: {recall_score(y_test, y_preds):.2f}\")\n",
    "print(f\"F1: {f1_score(y_test, y_preds):.2f}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Regression model metrics on the test set:\n",
      "R^2: 0.87\n",
      "MAE: 2.14\n",
      "MSE: 9.87\n"
     ]
    }
   ],
   "source": [
    "from sklearn.metrics import r2_score, mean_absolute_error, mean_squared_error\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "np.random.seed(42)\n",
    "\n",
    "X = boston_df.drop(\"target\", axis=1)\n",
    "y = boston_df[\"target\"]\n",
    "\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, \n",
    "                                                    y, \n",
    "                                                    test_size=0.2)\n",
    "\n",
    "model = RandomForestRegressor(n_estimators=100)\n",
    "model.fit(X_train, y_train)\n",
    "\n",
    "# Make predictions\n",
    "y_preds = model.predict(X_test)\n",
    "\n",
    "# Evaluate the model\n",
    "print(\"Regression model metrics on the test set:\")\n",
    "print(f\"R^2: {r2_score(y_test, y_preds):.2f}\")\n",
    "print(f\"MAE: {mean_absolute_error(y_test, y_preds):.2f}\")\n",
    "print(f\"MSE: {mean_squared_error(y_test, y_preds):.2f}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Improving model predictions through experimentation (hyperparameter tuning)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "* Your next goal is to improve upon these baseline metrics.\n",
    "* Could we collect more data? In machine learning, more data is generally better, as it gives a model more opportunities to learn patterns.\n",
    "* Could we improve our data? This could mean filling in misisng values or finding a better encoding (turning things into numbers) strategy."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "* Patterns in data are also often referred to as data parameters. The difference between parameters and hyperparameters is a machine learning model seeks to find parameters in data on its own, where as, hyperparameters are settings on a model which a user (you) can adjust."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.ensemble import RandomForestClassifier\n",
    "\n",
    "clf = RandomForestClassifier()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'bootstrap': True,\n",
       " 'ccp_alpha': 0.0,\n",
       " 'class_weight': None,\n",
       " 'criterion': 'gini',\n",
       " 'max_depth': None,\n",
       " 'max_features': 'auto',\n",
       " 'max_leaf_nodes': None,\n",
       " 'max_samples': None,\n",
       " 'min_impurity_decrease': 0.0,\n",
       " 'min_impurity_split': None,\n",
       " 'min_samples_leaf': 1,\n",
       " 'min_samples_split': 2,\n",
       " 'min_weight_fraction_leaf': 0.0,\n",
       " 'n_estimators': 100,\n",
       " 'n_jobs': None,\n",
       " 'oob_score': False,\n",
       " 'random_state': None,\n",
       " 'verbose': 0,\n",
       " 'warm_start': False}"
      ]
     },
     "execution_count": 94,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "clf.get_params()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "To get familar with hyparameter tuning, we'll take our RandomForestClassifier and adjust its hyperparameters in 3 ways.\n",
    "\n",
    "* By hand\n",
    "* Randomly with RandomSearchCV\n",
    "* Exhaustively with GridSearchCV"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Tuning hyperparameters by hand"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "metadata": {},
   "outputs": [],
   "source": [
    "def evaluate_preds(y_true, y_preds):\n",
    "    \"\"\"\n",
    "    Performs evaluation comparison on y_true labels vs. y_pred labels.\n",
    "    \"\"\"\n",
    "    accuracy = accuracy_score(y_true, y_preds)\n",
    "    precision = precision_score(y_true, y_preds)\n",
    "    recall = recall_score(y_true, y_preds)\n",
    "    f1 = f1_score(y_true, y_preds)\n",
    "    metric_dict = {\"accuracy\": round(accuracy, 2),\n",
    "                   \"precision\": round(precision, 2), \n",
    "                   \"recall\": round(recall, 2),\n",
    "                   \"f1\": round(f1, 2)}\n",
    "    print(f\"Acc: {accuracy * 100:.2f}%\")\n",
    "    print(f\"Precision: {precision:.2f}\")\n",
    "    print(f\"Recall: {recall:.2f}\")\n",
    "    print(f\"F1 score: {f1:.2f}\")\n",
    "\n",
    "    return metric_dict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Acc: 82.22%\n",
      "Precision: 0.81\n",
      "Recall: 0.88\n",
      "F1 score: 0.85\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'accuracy': 0.82, 'precision': 0.81, 'recall': 0.88, 'f1': 0.85}"
      ]
     },
     "execution_count": 96,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.ensemble import RandomForestClassifier\n",
    "\n",
    "np.random.seed(42)\n",
    "\n",
    "# Shuffle the data\n",
    "heart_disease = heart_disease.sample(frac=1)\n",
    "\n",
    "# Split into X & y\n",
    "X = heart_disease.drop(\"target\", axis=1)\n",
    "y = heart_disease[\"target\"]\n",
    "\n",
    "# Split the data into train, validation & test sets\n",
    "train_split = round(0.7 * len(heart_disease)) # 70% of data\n",
    "valid_split = round(train_split + 0.15 * len(heart_disease)) # 15% of data\n",
    "X_train, y_train = X[:train_split], y[:train_split]\n",
    "X_valid, y_valid = X[train_split:valid_split], y[train_split:valid_split]\n",
    "X_test, y_test = X[valid_split:], y[valid_split:]\n",
    "\n",
    "clf = RandomForestClassifier()\n",
    "clf.fit(X_train, y_train)\n",
    "\n",
    "# Make predictions\n",
    "y_preds = clf.predict(X_valid)\n",
    "\n",
    "# Evaluate the classifier\n",
    "baseline_metrics = evaluate_preds(y_valid, y_preds)\n",
    "baseline_metrics"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We'll change 1 of the hyperparameters, n_estimators to 100 and see if it improves on the validation set."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Acc: 82.22%\n",
      "Precision: 0.81\n",
      "Recall: 0.88\n",
      "F1 score: 0.85\n"
     ]
    }
   ],
   "source": [
    "np.random.seed(42)\n",
    "\n",
    "# Create a second classifier\n",
    "clf_2 = RandomForestClassifier(n_estimators=600)\n",
    "clf_2.fit(X_train, y_train)\n",
    "\n",
    "# Make predictions\n",
    "y_preds_2 = clf_2.predict(X_valid)\n",
    "\n",
    "# Evaluate the 2nd classifier\n",
    "clf_2_metrics = evaluate_preds(y_valid, y_preds_2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Hyperparameter tuning with RandomizedSearchCV"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Scikit-Learn's RandomizedSearchCV allows us to randomly search across different hyperparameters to see which work best. It also stores details about the ones which work best!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Hyperparameter grid RandomizedSearchCV will search over\n",
    "grid = {\"n_estimators\": [10, 100, 200, 500, 1000, 1200],\n",
    "        \"max_depth\": [None, 5, 10, 20, 30],\n",
    "        \"max_features\": [\"auto\", \"sqrt\"],\n",
    "        \"min_samples_split\": [2, 4, 6],\n",
    "        \"min_samples_leaf\": [1, 2, 4]}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 5 folds for each of 20 candidates, totalling 100 fits\n",
      "[CV] n_estimators=1200, min_samples_split=6, min_samples_leaf=2, max_features=sqrt, max_depth=5 \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV]  n_estimators=1200, min_samples_split=6, min_samples_leaf=2, max_features=sqrt, max_depth=5, total=   1.6s\n",
      "[CV] n_estimators=1200, min_samples_split=6, min_samples_leaf=2, max_features=sqrt, max_depth=5 \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done   1 out of   1 | elapsed:    1.5s remaining:    0.0s\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV]  n_estimators=1200, min_samples_split=6, min_samples_leaf=2, max_features=sqrt, max_depth=5, total=   1.6s\n",
      "[CV] n_estimators=1200, min_samples_split=6, min_samples_leaf=2, max_features=sqrt, max_depth=5 \n",
      "[CV]  n_estimators=1200, min_samples_split=6, min_samples_leaf=2, max_features=sqrt, max_depth=5, total=   1.6s\n",
      "[CV] n_estimators=1200, min_samples_split=6, min_samples_leaf=2, max_features=sqrt, max_depth=5 \n",
      "[CV]  n_estimators=1200, min_samples_split=6, min_samples_leaf=2, max_features=sqrt, max_depth=5, total=   1.6s\n",
      "[CV] n_estimators=1200, min_samples_split=6, min_samples_leaf=2, max_features=sqrt, max_depth=5 \n",
      "[CV]  n_estimators=1200, min_samples_split=6, min_samples_leaf=2, max_features=sqrt, max_depth=5, total=   1.6s\n",
      "[CV] n_estimators=100, min_samples_split=4, min_samples_leaf=2, max_features=auto, max_depth=30 \n",
      "[CV]  n_estimators=100, min_samples_split=4, min_samples_leaf=2, max_features=auto, max_depth=30, total=   0.1s\n",
      "[CV] n_estimators=100, min_samples_split=4, min_samples_leaf=2, max_features=auto, max_depth=30 \n",
      "[CV]  n_estimators=100, min_samples_split=4, min_samples_leaf=2, max_features=auto, max_depth=30, total=   0.2s\n",
      "[CV] n_estimators=100, min_samples_split=4, min_samples_leaf=2, max_features=auto, max_depth=30 \n",
      "[CV]  n_estimators=100, min_samples_split=4, min_samples_leaf=2, max_features=auto, max_depth=30, total=   0.1s\n",
      "[CV] n_estimators=100, min_samples_split=4, min_samples_leaf=2, max_features=auto, max_depth=30 \n",
      "[CV]  n_estimators=100, min_samples_split=4, min_samples_leaf=2, max_features=auto, max_depth=30, total=   0.1s\n",
      "[CV] n_estimators=100, min_samples_split=4, min_samples_leaf=2, max_features=auto, max_depth=30 \n",
      "[CV]  n_estimators=100, min_samples_split=4, min_samples_leaf=2, max_features=auto, max_depth=30, total=   0.1s\n",
      "[CV] n_estimators=200, min_samples_split=2, min_samples_leaf=2, max_features=sqrt, max_depth=10 \n",
      "[CV]  n_estimators=200, min_samples_split=2, min_samples_leaf=2, max_features=sqrt, max_depth=10, total=   0.3s\n",
      "[CV] n_estimators=200, min_samples_split=2, min_samples_leaf=2, max_features=sqrt, max_depth=10 \n",
      "[CV]  n_estimators=200, min_samples_split=2, min_samples_leaf=2, max_features=sqrt, max_depth=10, total=   0.3s\n",
      "[CV] n_estimators=200, min_samples_split=2, min_samples_leaf=2, max_features=sqrt, max_depth=10 \n",
      "[CV]  n_estimators=200, min_samples_split=2, min_samples_leaf=2, max_features=sqrt, max_depth=10, total=   0.3s\n",
      "[CV] n_estimators=200, min_samples_split=2, min_samples_leaf=2, max_features=sqrt, max_depth=10 \n",
      "[CV]  n_estimators=200, min_samples_split=2, min_samples_leaf=2, max_features=sqrt, max_depth=10, total=   0.3s\n",
      "[CV] n_estimators=200, min_samples_split=2, min_samples_leaf=2, max_features=sqrt, max_depth=10 \n",
      "[CV]  n_estimators=200, min_samples_split=2, min_samples_leaf=2, max_features=sqrt, max_depth=10, total=   0.3s\n",
      "[CV] n_estimators=100, min_samples_split=6, min_samples_leaf=1, max_features=auto, max_depth=20 \n",
      "[CV]  n_estimators=100, min_samples_split=6, min_samples_leaf=1, max_features=auto, max_depth=20, total=   0.2s\n",
      "[CV] n_estimators=100, min_samples_split=6, min_samples_leaf=1, max_features=auto, max_depth=20 \n",
      "[CV]  n_estimators=100, min_samples_split=6, min_samples_leaf=1, max_features=auto, max_depth=20, total=   0.2s\n",
      "[CV] n_estimators=100, min_samples_split=6, min_samples_leaf=1, max_features=auto, max_depth=20 \n",
      "[CV]  n_estimators=100, min_samples_split=6, min_samples_leaf=1, max_features=auto, max_depth=20, total=   0.1s\n",
      "[CV] n_estimators=100, min_samples_split=6, min_samples_leaf=1, max_features=auto, max_depth=20 \n",
      "[CV]  n_estimators=100, min_samples_split=6, min_samples_leaf=1, max_features=auto, max_depth=20, total=   0.1s\n",
      "[CV] n_estimators=100, min_samples_split=6, min_samples_leaf=1, max_features=auto, max_depth=20 \n",
      "[CV]  n_estimators=100, min_samples_split=6, min_samples_leaf=1, max_features=auto, max_depth=20, total=   0.1s\n",
      "[CV] n_estimators=10, min_samples_split=4, min_samples_leaf=1, max_features=sqrt, max_depth=5 \n",
      "[CV]  n_estimators=10, min_samples_split=4, min_samples_leaf=1, max_features=sqrt, max_depth=5, total=   0.0s\n",
      "[CV] n_estimators=10, min_samples_split=4, min_samples_leaf=1, max_features=sqrt, max_depth=5 \n",
      "[CV]  n_estimators=10, min_samples_split=4, min_samples_leaf=1, max_features=sqrt, max_depth=5, total=   0.0s\n",
      "[CV] n_estimators=10, min_samples_split=4, min_samples_leaf=1, max_features=sqrt, max_depth=5 \n",
      "[CV]  n_estimators=10, min_samples_split=4, min_samples_leaf=1, max_features=sqrt, max_depth=5, total=   0.0s\n",
      "[CV] n_estimators=10, min_samples_split=4, min_samples_leaf=1, max_features=sqrt, max_depth=5 \n",
      "[CV]  n_estimators=10, min_samples_split=4, min_samples_leaf=1, max_features=sqrt, max_depth=5, total=   0.0s\n",
      "[CV] n_estimators=10, min_samples_split=4, min_samples_leaf=1, max_features=sqrt, max_depth=5 \n",
      "[CV]  n_estimators=10, min_samples_split=4, min_samples_leaf=1, max_features=sqrt, max_depth=5, total=   0.0s\n",
      "[CV] n_estimators=10, min_samples_split=4, min_samples_leaf=2, max_features=auto, max_depth=10 \n",
      "[CV]  n_estimators=10, min_samples_split=4, min_samples_leaf=2, max_features=auto, max_depth=10, total=   0.0s\n",
      "[CV] n_estimators=10, min_samples_split=4, min_samples_leaf=2, max_features=auto, max_depth=10 \n",
      "[CV]  n_estimators=10, min_samples_split=4, min_samples_leaf=2, max_features=auto, max_depth=10, total=   0.0s\n",
      "[CV] n_estimators=10, min_samples_split=4, min_samples_leaf=2, max_features=auto, max_depth=10 \n",
      "[CV]  n_estimators=10, min_samples_split=4, min_samples_leaf=2, max_features=auto, max_depth=10, total=   0.0s\n",
      "[CV] n_estimators=10, min_samples_split=4, min_samples_leaf=2, max_features=auto, max_depth=10 \n",
      "[CV]  n_estimators=10, min_samples_split=4, min_samples_leaf=2, max_features=auto, max_depth=10, total=   0.0s\n",
      "[CV] n_estimators=10, min_samples_split=4, min_samples_leaf=2, max_features=auto, max_depth=10 \n",
      "[CV]  n_estimators=10, min_samples_split=4, min_samples_leaf=2, max_features=auto, max_depth=10, total=   0.0s\n",
      "[CV] n_estimators=500, min_samples_split=6, min_samples_leaf=2, max_features=sqrt, max_depth=None \n",
      "[CV]  n_estimators=500, min_samples_split=6, min_samples_leaf=2, max_features=sqrt, max_depth=None, total=   0.7s\n",
      "[CV] n_estimators=500, min_samples_split=6, min_samples_leaf=2, max_features=sqrt, max_depth=None \n",
      "[CV]  n_estimators=500, min_samples_split=6, min_samples_leaf=2, max_features=sqrt, max_depth=None, total=   0.7s\n",
      "[CV] n_estimators=500, min_samples_split=6, min_samples_leaf=2, max_features=sqrt, max_depth=None \n",
      "[CV]  n_estimators=500, min_samples_split=6, min_samples_leaf=2, max_features=sqrt, max_depth=None, total=   0.7s\n",
      "[CV] n_estimators=500, min_samples_split=6, min_samples_leaf=2, max_features=sqrt, max_depth=None \n",
      "[CV]  n_estimators=500, min_samples_split=6, min_samples_leaf=2, max_features=sqrt, max_depth=None, total=   0.7s\n",
      "[CV] n_estimators=500, min_samples_split=6, min_samples_leaf=2, max_features=sqrt, max_depth=None \n",
      "[CV]  n_estimators=500, min_samples_split=6, min_samples_leaf=2, max_features=sqrt, max_depth=None, total=   0.7s\n",
      "[CV] n_estimators=200, min_samples_split=6, min_samples_leaf=2, max_features=sqrt, max_depth=None \n",
      "[CV]  n_estimators=200, min_samples_split=6, min_samples_leaf=2, max_features=sqrt, max_depth=None, total=   0.3s\n",
      "[CV] n_estimators=200, min_samples_split=6, min_samples_leaf=2, max_features=sqrt, max_depth=None \n",
      "[CV]  n_estimators=200, min_samples_split=6, min_samples_leaf=2, max_features=sqrt, max_depth=None, total=   0.3s\n",
      "[CV] n_estimators=200, min_samples_split=6, min_samples_leaf=2, max_features=sqrt, max_depth=None \n",
      "[CV]  n_estimators=200, min_samples_split=6, min_samples_leaf=2, max_features=sqrt, max_depth=None, total=   0.3s\n",
      "[CV] n_estimators=200, min_samples_split=6, min_samples_leaf=2, max_features=sqrt, max_depth=None \n",
      "[CV]  n_estimators=200, min_samples_split=6, min_samples_leaf=2, max_features=sqrt, max_depth=None, total=   0.3s\n",
      "[CV] n_estimators=200, min_samples_split=6, min_samples_leaf=2, max_features=sqrt, max_depth=None \n",
      "[CV]  n_estimators=200, min_samples_split=6, min_samples_leaf=2, max_features=sqrt, max_depth=None, total=   0.3s\n",
      "[CV] n_estimators=200, min_samples_split=4, min_samples_leaf=4, max_features=auto, max_depth=10 \n",
      "[CV]  n_estimators=200, min_samples_split=4, min_samples_leaf=4, max_features=auto, max_depth=10, total=   0.3s\n",
      "[CV] n_estimators=200, min_samples_split=4, min_samples_leaf=4, max_features=auto, max_depth=10 \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV]  n_estimators=200, min_samples_split=4, min_samples_leaf=4, max_features=auto, max_depth=10, total=   0.3s\n",
      "[CV] n_estimators=200, min_samples_split=4, min_samples_leaf=4, max_features=auto, max_depth=10 \n",
      "[CV]  n_estimators=200, min_samples_split=4, min_samples_leaf=4, max_features=auto, max_depth=10, total=   0.3s\n",
      "[CV] n_estimators=200, min_samples_split=4, min_samples_leaf=4, max_features=auto, max_depth=10 \n",
      "[CV]  n_estimators=200, min_samples_split=4, min_samples_leaf=4, max_features=auto, max_depth=10, total=   0.3s\n",
      "[CV] n_estimators=200, min_samples_split=4, min_samples_leaf=4, max_features=auto, max_depth=10 \n",
      "[CV]  n_estimators=200, min_samples_split=4, min_samples_leaf=4, max_features=auto, max_depth=10, total=   0.3s\n",
      "[CV] n_estimators=1000, min_samples_split=4, min_samples_leaf=2, max_features=sqrt, max_depth=20 \n",
      "[CV]  n_estimators=1000, min_samples_split=4, min_samples_leaf=2, max_features=sqrt, max_depth=20, total=   1.3s\n",
      "[CV] n_estimators=1000, min_samples_split=4, min_samples_leaf=2, max_features=sqrt, max_depth=20 \n",
      "[CV]  n_estimators=1000, min_samples_split=4, min_samples_leaf=2, max_features=sqrt, max_depth=20, total=   1.4s\n",
      "[CV] n_estimators=1000, min_samples_split=4, min_samples_leaf=2, max_features=sqrt, max_depth=20 \n",
      "[CV]  n_estimators=1000, min_samples_split=4, min_samples_leaf=2, max_features=sqrt, max_depth=20, total=   1.4s\n",
      "[CV] n_estimators=1000, min_samples_split=4, min_samples_leaf=2, max_features=sqrt, max_depth=20 \n",
      "[CV]  n_estimators=1000, min_samples_split=4, min_samples_leaf=2, max_features=sqrt, max_depth=20, total=   1.4s\n",
      "[CV] n_estimators=1000, min_samples_split=4, min_samples_leaf=2, max_features=sqrt, max_depth=20 \n",
      "[CV]  n_estimators=1000, min_samples_split=4, min_samples_leaf=2, max_features=sqrt, max_depth=20, total=   1.4s\n",
      "[CV] n_estimators=1200, min_samples_split=2, min_samples_leaf=2, max_features=sqrt, max_depth=20 \n",
      "[CV]  n_estimators=1200, min_samples_split=2, min_samples_leaf=2, max_features=sqrt, max_depth=20, total=   1.7s\n",
      "[CV] n_estimators=1200, min_samples_split=2, min_samples_leaf=2, max_features=sqrt, max_depth=20 \n",
      "[CV]  n_estimators=1200, min_samples_split=2, min_samples_leaf=2, max_features=sqrt, max_depth=20, total=   1.7s\n",
      "[CV] n_estimators=1200, min_samples_split=2, min_samples_leaf=2, max_features=sqrt, max_depth=20 \n",
      "[CV]  n_estimators=1200, min_samples_split=2, min_samples_leaf=2, max_features=sqrt, max_depth=20, total=   1.6s\n",
      "[CV] n_estimators=1200, min_samples_split=2, min_samples_leaf=2, max_features=sqrt, max_depth=20 \n",
      "[CV]  n_estimators=1200, min_samples_split=2, min_samples_leaf=2, max_features=sqrt, max_depth=20, total=   1.6s\n",
      "[CV] n_estimators=1200, min_samples_split=2, min_samples_leaf=2, max_features=sqrt, max_depth=20 \n",
      "[CV]  n_estimators=1200, min_samples_split=2, min_samples_leaf=2, max_features=sqrt, max_depth=20, total=   1.6s\n",
      "[CV] n_estimators=1200, min_samples_split=4, min_samples_leaf=2, max_features=auto, max_depth=5 \n",
      "[CV]  n_estimators=1200, min_samples_split=4, min_samples_leaf=2, max_features=auto, max_depth=5, total=   1.6s\n",
      "[CV] n_estimators=1200, min_samples_split=4, min_samples_leaf=2, max_features=auto, max_depth=5 \n",
      "[CV]  n_estimators=1200, min_samples_split=4, min_samples_leaf=2, max_features=auto, max_depth=5, total=   1.6s\n",
      "[CV] n_estimators=1200, min_samples_split=4, min_samples_leaf=2, max_features=auto, max_depth=5 \n",
      "[CV]  n_estimators=1200, min_samples_split=4, min_samples_leaf=2, max_features=auto, max_depth=5, total=   1.6s\n",
      "[CV] n_estimators=1200, min_samples_split=4, min_samples_leaf=2, max_features=auto, max_depth=5 \n",
      "[CV]  n_estimators=1200, min_samples_split=4, min_samples_leaf=2, max_features=auto, max_depth=5, total=   1.5s\n",
      "[CV] n_estimators=1200, min_samples_split=4, min_samples_leaf=2, max_features=auto, max_depth=5 \n",
      "[CV]  n_estimators=1200, min_samples_split=4, min_samples_leaf=2, max_features=auto, max_depth=5, total=   1.6s\n",
      "[CV] n_estimators=100, min_samples_split=6, min_samples_leaf=4, max_features=auto, max_depth=10 \n",
      "[CV]  n_estimators=100, min_samples_split=6, min_samples_leaf=4, max_features=auto, max_depth=10, total=   0.1s\n",
      "[CV] n_estimators=100, min_samples_split=6, min_samples_leaf=4, max_features=auto, max_depth=10 \n",
      "[CV]  n_estimators=100, min_samples_split=6, min_samples_leaf=4, max_features=auto, max_depth=10, total=   0.1s\n",
      "[CV] n_estimators=100, min_samples_split=6, min_samples_leaf=4, max_features=auto, max_depth=10 \n",
      "[CV]  n_estimators=100, min_samples_split=6, min_samples_leaf=4, max_features=auto, max_depth=10, total=   0.1s\n",
      "[CV] n_estimators=100, min_samples_split=6, min_samples_leaf=4, max_features=auto, max_depth=10 \n",
      "[CV]  n_estimators=100, min_samples_split=6, min_samples_leaf=4, max_features=auto, max_depth=10, total=   0.1s\n",
      "[CV] n_estimators=100, min_samples_split=6, min_samples_leaf=4, max_features=auto, max_depth=10 \n",
      "[CV]  n_estimators=100, min_samples_split=6, min_samples_leaf=4, max_features=auto, max_depth=10, total=   0.1s\n",
      "[CV] n_estimators=1200, min_samples_split=6, min_samples_leaf=4, max_features=sqrt, max_depth=5 \n",
      "[CV]  n_estimators=1200, min_samples_split=6, min_samples_leaf=4, max_features=sqrt, max_depth=5, total=   1.7s\n",
      "[CV] n_estimators=1200, min_samples_split=6, min_samples_leaf=4, max_features=sqrt, max_depth=5 \n",
      "[CV]  n_estimators=1200, min_samples_split=6, min_samples_leaf=4, max_features=sqrt, max_depth=5, total=   1.6s\n",
      "[CV] n_estimators=1200, min_samples_split=6, min_samples_leaf=4, max_features=sqrt, max_depth=5 \n",
      "[CV]  n_estimators=1200, min_samples_split=6, min_samples_leaf=4, max_features=sqrt, max_depth=5, total=   1.6s\n",
      "[CV] n_estimators=1200, min_samples_split=6, min_samples_leaf=4, max_features=sqrt, max_depth=5 \n",
      "[CV]  n_estimators=1200, min_samples_split=6, min_samples_leaf=4, max_features=sqrt, max_depth=5, total=   1.7s\n",
      "[CV] n_estimators=1200, min_samples_split=6, min_samples_leaf=4, max_features=sqrt, max_depth=5 \n",
      "[CV]  n_estimators=1200, min_samples_split=6, min_samples_leaf=4, max_features=sqrt, max_depth=5, total=   1.6s\n",
      "[CV] n_estimators=10, min_samples_split=6, min_samples_leaf=2, max_features=auto, max_depth=None \n",
      "[CV]  n_estimators=10, min_samples_split=6, min_samples_leaf=2, max_features=auto, max_depth=None, total=   0.0s\n",
      "[CV] n_estimators=10, min_samples_split=6, min_samples_leaf=2, max_features=auto, max_depth=None \n",
      "[CV]  n_estimators=10, min_samples_split=6, min_samples_leaf=2, max_features=auto, max_depth=None, total=   0.0s\n",
      "[CV] n_estimators=10, min_samples_split=6, min_samples_leaf=2, max_features=auto, max_depth=None \n",
      "[CV]  n_estimators=10, min_samples_split=6, min_samples_leaf=2, max_features=auto, max_depth=None, total=   0.0s\n",
      "[CV] n_estimators=10, min_samples_split=6, min_samples_leaf=2, max_features=auto, max_depth=None \n",
      "[CV]  n_estimators=10, min_samples_split=6, min_samples_leaf=2, max_features=auto, max_depth=None, total=   0.0s\n",
      "[CV] n_estimators=10, min_samples_split=6, min_samples_leaf=2, max_features=auto, max_depth=None \n",
      "[CV]  n_estimators=10, min_samples_split=6, min_samples_leaf=2, max_features=auto, max_depth=None, total=   0.0s\n",
      "[CV] n_estimators=500, min_samples_split=2, min_samples_leaf=1, max_features=sqrt, max_depth=20 \n",
      "[CV]  n_estimators=500, min_samples_split=2, min_samples_leaf=1, max_features=sqrt, max_depth=20, total=   0.7s\n",
      "[CV] n_estimators=500, min_samples_split=2, min_samples_leaf=1, max_features=sqrt, max_depth=20 \n",
      "[CV]  n_estimators=500, min_samples_split=2, min_samples_leaf=1, max_features=sqrt, max_depth=20, total=   0.7s\n",
      "[CV] n_estimators=500, min_samples_split=2, min_samples_leaf=1, max_features=sqrt, max_depth=20 \n",
      "[CV]  n_estimators=500, min_samples_split=2, min_samples_leaf=1, max_features=sqrt, max_depth=20, total=   0.7s\n",
      "[CV] n_estimators=500, min_samples_split=2, min_samples_leaf=1, max_features=sqrt, max_depth=20 \n",
      "[CV]  n_estimators=500, min_samples_split=2, min_samples_leaf=1, max_features=sqrt, max_depth=20, total=   0.7s\n",
      "[CV] n_estimators=500, min_samples_split=2, min_samples_leaf=1, max_features=sqrt, max_depth=20 \n",
      "[CV]  n_estimators=500, min_samples_split=2, min_samples_leaf=1, max_features=sqrt, max_depth=20, total=   0.6s\n",
      "[CV] n_estimators=100, min_samples_split=6, min_samples_leaf=2, max_features=auto, max_depth=10 \n",
      "[CV]  n_estimators=100, min_samples_split=6, min_samples_leaf=2, max_features=auto, max_depth=10, total=   0.1s\n",
      "[CV] n_estimators=100, min_samples_split=6, min_samples_leaf=2, max_features=auto, max_depth=10 \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV]  n_estimators=100, min_samples_split=6, min_samples_leaf=2, max_features=auto, max_depth=10, total=   0.1s\n",
      "[CV] n_estimators=100, min_samples_split=6, min_samples_leaf=2, max_features=auto, max_depth=10 \n",
      "[CV]  n_estimators=100, min_samples_split=6, min_samples_leaf=2, max_features=auto, max_depth=10, total=   0.1s\n",
      "[CV] n_estimators=100, min_samples_split=6, min_samples_leaf=2, max_features=auto, max_depth=10 \n",
      "[CV]  n_estimators=100, min_samples_split=6, min_samples_leaf=2, max_features=auto, max_depth=10, total=   0.1s\n",
      "[CV] n_estimators=100, min_samples_split=6, min_samples_leaf=2, max_features=auto, max_depth=10 \n",
      "[CV]  n_estimators=100, min_samples_split=6, min_samples_leaf=2, max_features=auto, max_depth=10, total=   0.1s\n",
      "[CV] n_estimators=100, min_samples_split=4, min_samples_leaf=4, max_features=auto, max_depth=10 \n",
      "[CV]  n_estimators=100, min_samples_split=4, min_samples_leaf=4, max_features=auto, max_depth=10, total=   0.1s\n",
      "[CV] n_estimators=100, min_samples_split=4, min_samples_leaf=4, max_features=auto, max_depth=10 \n",
      "[CV]  n_estimators=100, min_samples_split=4, min_samples_leaf=4, max_features=auto, max_depth=10, total=   0.1s\n",
      "[CV] n_estimators=100, min_samples_split=4, min_samples_leaf=4, max_features=auto, max_depth=10 \n",
      "[CV]  n_estimators=100, min_samples_split=4, min_samples_leaf=4, max_features=auto, max_depth=10, total=   0.1s\n",
      "[CV] n_estimators=100, min_samples_split=4, min_samples_leaf=4, max_features=auto, max_depth=10 \n",
      "[CV]  n_estimators=100, min_samples_split=4, min_samples_leaf=4, max_features=auto, max_depth=10, total=   0.1s\n",
      "[CV] n_estimators=100, min_samples_split=4, min_samples_leaf=4, max_features=auto, max_depth=10 \n",
      "[CV]  n_estimators=100, min_samples_split=4, min_samples_leaf=4, max_features=auto, max_depth=10, total=   0.1s\n",
      "[CV] n_estimators=500, min_samples_split=2, min_samples_leaf=1, max_features=sqrt, max_depth=10 \n",
      "[CV]  n_estimators=500, min_samples_split=2, min_samples_leaf=1, max_features=sqrt, max_depth=10, total=   0.7s\n",
      "[CV] n_estimators=500, min_samples_split=2, min_samples_leaf=1, max_features=sqrt, max_depth=10 \n",
      "[CV]  n_estimators=500, min_samples_split=2, min_samples_leaf=1, max_features=sqrt, max_depth=10, total=   0.8s\n",
      "[CV] n_estimators=500, min_samples_split=2, min_samples_leaf=1, max_features=sqrt, max_depth=10 \n",
      "[CV]  n_estimators=500, min_samples_split=2, min_samples_leaf=1, max_features=sqrt, max_depth=10, total=   0.8s\n",
      "[CV] n_estimators=500, min_samples_split=2, min_samples_leaf=1, max_features=sqrt, max_depth=10 \n",
      "[CV]  n_estimators=500, min_samples_split=2, min_samples_leaf=1, max_features=sqrt, max_depth=10, total=   0.8s\n",
      "[CV] n_estimators=500, min_samples_split=2, min_samples_leaf=1, max_features=sqrt, max_depth=10 \n",
      "[CV]  n_estimators=500, min_samples_split=2, min_samples_leaf=1, max_features=sqrt, max_depth=10, total=   0.7s\n",
      "[CV] n_estimators=500, min_samples_split=2, min_samples_leaf=2, max_features=auto, max_depth=None \n",
      "[CV]  n_estimators=500, min_samples_split=2, min_samples_leaf=2, max_features=auto, max_depth=None, total=   0.7s\n",
      "[CV] n_estimators=500, min_samples_split=2, min_samples_leaf=2, max_features=auto, max_depth=None \n",
      "[CV]  n_estimators=500, min_samples_split=2, min_samples_leaf=2, max_features=auto, max_depth=None, total=   0.7s\n",
      "[CV] n_estimators=500, min_samples_split=2, min_samples_leaf=2, max_features=auto, max_depth=None \n",
      "[CV]  n_estimators=500, min_samples_split=2, min_samples_leaf=2, max_features=auto, max_depth=None, total=   0.7s\n",
      "[CV] n_estimators=500, min_samples_split=2, min_samples_leaf=2, max_features=auto, max_depth=None \n",
      "[CV]  n_estimators=500, min_samples_split=2, min_samples_leaf=2, max_features=auto, max_depth=None, total=   0.7s\n",
      "[CV] n_estimators=500, min_samples_split=2, min_samples_leaf=2, max_features=auto, max_depth=None \n",
      "[CV]  n_estimators=500, min_samples_split=2, min_samples_leaf=2, max_features=auto, max_depth=None, total=   0.7s\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done 100 out of 100 | elapsed:  1.0min finished\n"
     ]
    }
   ],
   "source": [
    "from sklearn.model_selection import RandomizedSearchCV, train_test_split\n",
    "np.random.seed(42)\n",
    "\n",
    "# Split into X & y\n",
    "X = heart_disease.drop(\"target\", axis=1)\n",
    "y = heart_disease[\"target\"]\n",
    "\n",
    "# Split into train and test sets\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2)\n",
    "\n",
    "# Set n_jobs to -1 to use all cores (NOTE: n_jobs=-1 is broken as of 8 Dec 2019, using n_jobs=1 works)\n",
    "clf = RandomForestClassifier(n_jobs=1)\n",
    "\n",
    "# Setup RandomizedSearchCV\n",
    "rs_clf = RandomizedSearchCV(estimator=clf,\n",
    "                            param_distributions=grid,\n",
    "                            n_iter=20, # try 20 models total\n",
    "                            cv=5, # 5-fold cross-validation\n",
    "                            verbose=2) # print out results\n",
    "\n",
    "# Fit the RandomizedSearchCV version of clf\n",
    "rs_clf.fit(X_train, y_train);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'n_estimators': 100,\n",
       " 'min_samples_split': 6,\n",
       " 'min_samples_leaf': 4,\n",
       " 'max_features': 'auto',\n",
       " 'max_depth': 10}"
      ]
     },
     "execution_count": 100,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Find the best hyperparameters found by RandomizedSearchCV\n",
    "rs_clf.best_params_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Acc: 83.61%\n",
      "Precision: 0.78\n",
      "Recall: 0.89\n",
      "F1 score: 0.83\n"
     ]
    }
   ],
   "source": [
    "# Make predictions with the best hyperparameters\n",
    "rs_y_preds = rs_clf.predict(X_test)\n",
    "\n",
    "# Evaluate the predictions\n",
    "rs_metrics = evaluate_preds(y_test, rs_y_preds)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "###  Hyperparameter tuning with GridSearchCV¶"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "# Another hyperparameter grid similar to rs_clf.best_params_\n",
    "grid_2 = {'n_estimators': [1200, 1500, 2000],\n",
    "          'max_depth': [None, 5, 10],\n",
    "          'max_features': ['auto', 'sqrt'],\n",
    "          'min_samples_split': [4, 6],\n",
    "          'min_samples_leaf': [1, 2]}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 5 folds for each of 72 candidates, totalling 360 fits\n",
      "[CV] max_depth=None, max_features=auto, min_samples_leaf=1, min_samples_split=4, n_estimators=1200 \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV]  max_depth=None, max_features=auto, min_samples_leaf=1, min_samples_split=4, n_estimators=1200, total=   1.7s\n",
      "[CV] max_depth=None, max_features=auto, min_samples_leaf=1, min_samples_split=4, n_estimators=1200 \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done   1 out of   1 | elapsed:    1.6s remaining:    0.0s\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV]  max_depth=None, max_features=auto, min_samples_leaf=1, min_samples_split=4, n_estimators=1200, total=   1.7s\n",
      "[CV] max_depth=None, max_features=auto, min_samples_leaf=1, min_samples_split=4, n_estimators=1200 \n",
      "[CV]  max_depth=None, max_features=auto, min_samples_leaf=1, min_samples_split=4, n_estimators=1200, total=   1.7s\n",
      "[CV] max_depth=None, max_features=auto, min_samples_leaf=1, min_samples_split=4, n_estimators=1200 \n",
      "[CV]  max_depth=None, max_features=auto, min_samples_leaf=1, min_samples_split=4, n_estimators=1200, total=   1.7s\n",
      "[CV] max_depth=None, max_features=auto, min_samples_leaf=1, min_samples_split=4, n_estimators=1200 \n",
      "[CV]  max_depth=None, max_features=auto, min_samples_leaf=1, min_samples_split=4, n_estimators=1200, total=   1.7s\n",
      "[CV] max_depth=None, max_features=auto, min_samples_leaf=1, min_samples_split=4, n_estimators=1500 \n",
      "[CV]  max_depth=None, max_features=auto, min_samples_leaf=1, min_samples_split=4, n_estimators=1500, total=   2.0s\n",
      "[CV] max_depth=None, max_features=auto, min_samples_leaf=1, min_samples_split=4, n_estimators=1500 \n",
      "[CV]  max_depth=None, max_features=auto, min_samples_leaf=1, min_samples_split=4, n_estimators=1500, total=   2.0s\n",
      "[CV] max_depth=None, max_features=auto, min_samples_leaf=1, min_samples_split=4, n_estimators=1500 \n",
      "[CV]  max_depth=None, max_features=auto, min_samples_leaf=1, min_samples_split=4, n_estimators=1500, total=   2.0s\n",
      "[CV] max_depth=None, max_features=auto, min_samples_leaf=1, min_samples_split=4, n_estimators=1500 \n",
      "[CV]  max_depth=None, max_features=auto, min_samples_leaf=1, min_samples_split=4, n_estimators=1500, total=   2.1s\n",
      "[CV] max_depth=None, max_features=auto, min_samples_leaf=1, min_samples_split=4, n_estimators=1500 \n",
      "[CV]  max_depth=None, max_features=auto, min_samples_leaf=1, min_samples_split=4, n_estimators=1500, total=   2.1s\n",
      "[CV] max_depth=None, max_features=auto, min_samples_leaf=1, min_samples_split=4, n_estimators=2000 \n",
      "[CV]  max_depth=None, max_features=auto, min_samples_leaf=1, min_samples_split=4, n_estimators=2000, total=   2.7s\n",
      "[CV] max_depth=None, max_features=auto, min_samples_leaf=1, min_samples_split=4, n_estimators=2000 \n",
      "[CV]  max_depth=None, max_features=auto, min_samples_leaf=1, min_samples_split=4, n_estimators=2000, total=   2.7s\n",
      "[CV] max_depth=None, max_features=auto, min_samples_leaf=1, min_samples_split=4, n_estimators=2000 \n",
      "[CV]  max_depth=None, max_features=auto, min_samples_leaf=1, min_samples_split=4, n_estimators=2000, total=   2.8s\n",
      "[CV] max_depth=None, max_features=auto, min_samples_leaf=1, min_samples_split=4, n_estimators=2000 \n",
      "[CV]  max_depth=None, max_features=auto, min_samples_leaf=1, min_samples_split=4, n_estimators=2000, total=   2.7s\n",
      "[CV] max_depth=None, max_features=auto, min_samples_leaf=1, min_samples_split=4, n_estimators=2000 \n",
      "[CV]  max_depth=None, max_features=auto, min_samples_leaf=1, min_samples_split=4, n_estimators=2000, total=   2.7s\n",
      "[CV] max_depth=None, max_features=auto, min_samples_leaf=1, min_samples_split=6, n_estimators=1200 \n",
      "[CV]  max_depth=None, max_features=auto, min_samples_leaf=1, min_samples_split=6, n_estimators=1200, total=   1.7s\n",
      "[CV] max_depth=None, max_features=auto, min_samples_leaf=1, min_samples_split=6, n_estimators=1200 \n",
      "[CV]  max_depth=None, max_features=auto, min_samples_leaf=1, min_samples_split=6, n_estimators=1200, total=   1.6s\n",
      "[CV] max_depth=None, max_features=auto, min_samples_leaf=1, min_samples_split=6, n_estimators=1200 \n",
      "[CV]  max_depth=None, max_features=auto, min_samples_leaf=1, min_samples_split=6, n_estimators=1200, total=   1.6s\n",
      "[CV] max_depth=None, max_features=auto, min_samples_leaf=1, min_samples_split=6, n_estimators=1200 \n",
      "[CV]  max_depth=None, max_features=auto, min_samples_leaf=1, min_samples_split=6, n_estimators=1200, total=   1.7s\n",
      "[CV] max_depth=None, max_features=auto, min_samples_leaf=1, min_samples_split=6, n_estimators=1200 \n",
      "[CV]  max_depth=None, max_features=auto, min_samples_leaf=1, min_samples_split=6, n_estimators=1200, total=   1.7s\n",
      "[CV] max_depth=None, max_features=auto, min_samples_leaf=1, min_samples_split=6, n_estimators=1500 \n",
      "[CV]  max_depth=None, max_features=auto, min_samples_leaf=1, min_samples_split=6, n_estimators=1500, total=   2.1s\n",
      "[CV] max_depth=None, max_features=auto, min_samples_leaf=1, min_samples_split=6, n_estimators=1500 \n",
      "[CV]  max_depth=None, max_features=auto, min_samples_leaf=1, min_samples_split=6, n_estimators=1500, total=   2.1s\n",
      "[CV] max_depth=None, max_features=auto, min_samples_leaf=1, min_samples_split=6, n_estimators=1500 \n",
      "[CV]  max_depth=None, max_features=auto, min_samples_leaf=1, min_samples_split=6, n_estimators=1500, total=   2.1s\n",
      "[CV] max_depth=None, max_features=auto, min_samples_leaf=1, min_samples_split=6, n_estimators=1500 \n",
      "[CV]  max_depth=None, max_features=auto, min_samples_leaf=1, min_samples_split=6, n_estimators=1500, total=   2.2s\n",
      "[CV] max_depth=None, max_features=auto, min_samples_leaf=1, min_samples_split=6, n_estimators=1500 \n",
      "[CV]  max_depth=None, max_features=auto, min_samples_leaf=1, min_samples_split=6, n_estimators=1500, total=   2.0s\n",
      "[CV] max_depth=None, max_features=auto, min_samples_leaf=1, min_samples_split=6, n_estimators=2000 \n",
      "[CV]  max_depth=None, max_features=auto, min_samples_leaf=1, min_samples_split=6, n_estimators=2000, total=   2.6s\n",
      "[CV] max_depth=None, max_features=auto, min_samples_leaf=1, min_samples_split=6, n_estimators=2000 \n",
      "[CV]  max_depth=None, max_features=auto, min_samples_leaf=1, min_samples_split=6, n_estimators=2000, total=   2.6s\n",
      "[CV] max_depth=None, max_features=auto, min_samples_leaf=1, min_samples_split=6, n_estimators=2000 \n",
      "[CV]  max_depth=None, max_features=auto, min_samples_leaf=1, min_samples_split=6, n_estimators=2000, total=   2.7s\n",
      "[CV] max_depth=None, max_features=auto, min_samples_leaf=1, min_samples_split=6, n_estimators=2000 \n",
      "[CV]  max_depth=None, max_features=auto, min_samples_leaf=1, min_samples_split=6, n_estimators=2000, total=   2.7s\n",
      "[CV] max_depth=None, max_features=auto, min_samples_leaf=1, min_samples_split=6, n_estimators=2000 \n",
      "[CV]  max_depth=None, max_features=auto, min_samples_leaf=1, min_samples_split=6, n_estimators=2000, total=   2.7s\n",
      "[CV] max_depth=None, max_features=auto, min_samples_leaf=2, min_samples_split=4, n_estimators=1200 \n",
      "[CV]  max_depth=None, max_features=auto, min_samples_leaf=2, min_samples_split=4, n_estimators=1200, total=   1.7s\n",
      "[CV] max_depth=None, max_features=auto, min_samples_leaf=2, min_samples_split=4, n_estimators=1200 \n",
      "[CV]  max_depth=None, max_features=auto, min_samples_leaf=2, min_samples_split=4, n_estimators=1200, total=   1.8s\n",
      "[CV] max_depth=None, max_features=auto, min_samples_leaf=2, min_samples_split=4, n_estimators=1200 \n",
      "[CV]  max_depth=None, max_features=auto, min_samples_leaf=2, min_samples_split=4, n_estimators=1200, total=   1.7s\n",
      "[CV] max_depth=None, max_features=auto, min_samples_leaf=2, min_samples_split=4, n_estimators=1200 \n",
      "[CV]  max_depth=None, max_features=auto, min_samples_leaf=2, min_samples_split=4, n_estimators=1200, total=   1.6s\n",
      "[CV] max_depth=None, max_features=auto, min_samples_leaf=2, min_samples_split=4, n_estimators=1200 \n",
      "[CV]  max_depth=None, max_features=auto, min_samples_leaf=2, min_samples_split=4, n_estimators=1200, total=   1.6s\n",
      "[CV] max_depth=None, max_features=auto, min_samples_leaf=2, min_samples_split=4, n_estimators=1500 \n",
      "[CV]  max_depth=None, max_features=auto, min_samples_leaf=2, min_samples_split=4, n_estimators=1500, total=   2.1s\n",
      "[CV] max_depth=None, max_features=auto, min_samples_leaf=2, min_samples_split=4, n_estimators=1500 \n",
      "[CV]  max_depth=None, max_features=auto, min_samples_leaf=2, min_samples_split=4, n_estimators=1500, total=   2.0s\n",
      "[CV] max_depth=None, max_features=auto, min_samples_leaf=2, min_samples_split=4, n_estimators=1500 \n",
      "[CV]  max_depth=None, max_features=auto, min_samples_leaf=2, min_samples_split=4, n_estimators=1500, total=   2.1s\n",
      "[CV] max_depth=None, max_features=auto, min_samples_leaf=2, min_samples_split=4, n_estimators=1500 \n",
      "[CV]  max_depth=None, max_features=auto, min_samples_leaf=2, min_samples_split=4, n_estimators=1500, total=   2.0s\n",
      "[CV] max_depth=None, max_features=auto, min_samples_leaf=2, min_samples_split=4, n_estimators=1500 \n",
      "[CV]  max_depth=None, max_features=auto, min_samples_leaf=2, min_samples_split=4, n_estimators=1500, total=   2.0s\n",
      "[CV] max_depth=None, max_features=auto, min_samples_leaf=2, min_samples_split=4, n_estimators=2000 \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV]  max_depth=None, max_features=auto, min_samples_leaf=2, min_samples_split=4, n_estimators=2000, total=   2.7s\n",
      "[CV] max_depth=None, max_features=auto, min_samples_leaf=2, min_samples_split=4, n_estimators=2000 \n",
      "[CV]  max_depth=None, max_features=auto, min_samples_leaf=2, min_samples_split=4, n_estimators=2000, total=   2.8s\n",
      "[CV] max_depth=None, max_features=auto, min_samples_leaf=2, min_samples_split=4, n_estimators=2000 \n",
      "[CV]  max_depth=None, max_features=auto, min_samples_leaf=2, min_samples_split=4, n_estimators=2000, total=   2.7s\n",
      "[CV] max_depth=None, max_features=auto, min_samples_leaf=2, min_samples_split=4, n_estimators=2000 \n",
      "[CV]  max_depth=None, max_features=auto, min_samples_leaf=2, min_samples_split=4, n_estimators=2000, total=   2.7s\n",
      "[CV] max_depth=None, max_features=auto, min_samples_leaf=2, min_samples_split=4, n_estimators=2000 \n",
      "[CV]  max_depth=None, max_features=auto, min_samples_leaf=2, min_samples_split=4, n_estimators=2000, total=   2.6s\n",
      "[CV] max_depth=None, max_features=auto, min_samples_leaf=2, min_samples_split=6, n_estimators=1200 \n",
      "[CV]  max_depth=None, max_features=auto, min_samples_leaf=2, min_samples_split=6, n_estimators=1200, total=   1.6s\n",
      "[CV] max_depth=None, max_features=auto, min_samples_leaf=2, min_samples_split=6, n_estimators=1200 \n",
      "[CV]  max_depth=None, max_features=auto, min_samples_leaf=2, min_samples_split=6, n_estimators=1200, total=   1.6s\n",
      "[CV] max_depth=None, max_features=auto, min_samples_leaf=2, min_samples_split=6, n_estimators=1200 \n",
      "[CV]  max_depth=None, max_features=auto, min_samples_leaf=2, min_samples_split=6, n_estimators=1200, total=   1.6s\n",
      "[CV] max_depth=None, max_features=auto, min_samples_leaf=2, min_samples_split=6, n_estimators=1200 \n",
      "[CV]  max_depth=None, max_features=auto, min_samples_leaf=2, min_samples_split=6, n_estimators=1200, total=   1.6s\n",
      "[CV] max_depth=None, max_features=auto, min_samples_leaf=2, min_samples_split=6, n_estimators=1200 \n",
      "[CV]  max_depth=None, max_features=auto, min_samples_leaf=2, min_samples_split=6, n_estimators=1200, total=   1.6s\n",
      "[CV] max_depth=None, max_features=auto, min_samples_leaf=2, min_samples_split=6, n_estimators=1500 \n",
      "[CV]  max_depth=None, max_features=auto, min_samples_leaf=2, min_samples_split=6, n_estimators=1500, total=   2.0s\n",
      "[CV] max_depth=None, max_features=auto, min_samples_leaf=2, min_samples_split=6, n_estimators=1500 \n",
      "[CV]  max_depth=None, max_features=auto, min_samples_leaf=2, min_samples_split=6, n_estimators=1500, total=   2.0s\n",
      "[CV] max_depth=None, max_features=auto, min_samples_leaf=2, min_samples_split=6, n_estimators=1500 \n",
      "[CV]  max_depth=None, max_features=auto, min_samples_leaf=2, min_samples_split=6, n_estimators=1500, total=   2.0s\n",
      "[CV] max_depth=None, max_features=auto, min_samples_leaf=2, min_samples_split=6, n_estimators=1500 \n",
      "[CV]  max_depth=None, max_features=auto, min_samples_leaf=2, min_samples_split=6, n_estimators=1500, total=   2.0s\n",
      "[CV] max_depth=None, max_features=auto, min_samples_leaf=2, min_samples_split=6, n_estimators=1500 \n",
      "[CV]  max_depth=None, max_features=auto, min_samples_leaf=2, min_samples_split=6, n_estimators=1500, total=   2.0s\n",
      "[CV] max_depth=None, max_features=auto, min_samples_leaf=2, min_samples_split=6, n_estimators=2000 \n",
      "[CV]  max_depth=None, max_features=auto, min_samples_leaf=2, min_samples_split=6, n_estimators=2000, total=   2.6s\n",
      "[CV] max_depth=None, max_features=auto, min_samples_leaf=2, min_samples_split=6, n_estimators=2000 \n",
      "[CV]  max_depth=None, max_features=auto, min_samples_leaf=2, min_samples_split=6, n_estimators=2000, total=   2.7s\n",
      "[CV] max_depth=None, max_features=auto, min_samples_leaf=2, min_samples_split=6, n_estimators=2000 \n",
      "[CV]  max_depth=None, max_features=auto, min_samples_leaf=2, min_samples_split=6, n_estimators=2000, total=   2.7s\n",
      "[CV] max_depth=None, max_features=auto, min_samples_leaf=2, min_samples_split=6, n_estimators=2000 \n",
      "[CV]  max_depth=None, max_features=auto, min_samples_leaf=2, min_samples_split=6, n_estimators=2000, total=   2.6s\n",
      "[CV] max_depth=None, max_features=auto, min_samples_leaf=2, min_samples_split=6, n_estimators=2000 \n",
      "[CV]  max_depth=None, max_features=auto, min_samples_leaf=2, min_samples_split=6, n_estimators=2000, total=   2.6s\n",
      "[CV] max_depth=None, max_features=sqrt, min_samples_leaf=1, min_samples_split=4, n_estimators=1200 \n",
      "[CV]  max_depth=None, max_features=sqrt, min_samples_leaf=1, min_samples_split=4, n_estimators=1200, total=   1.6s\n",
      "[CV] max_depth=None, max_features=sqrt, min_samples_leaf=1, min_samples_split=4, n_estimators=1200 \n",
      "[CV]  max_depth=None, max_features=sqrt, min_samples_leaf=1, min_samples_split=4, n_estimators=1200, total=   1.6s\n",
      "[CV] max_depth=None, max_features=sqrt, min_samples_leaf=1, min_samples_split=4, n_estimators=1200 \n",
      "[CV]  max_depth=None, max_features=sqrt, min_samples_leaf=1, min_samples_split=4, n_estimators=1200, total=   1.6s\n",
      "[CV] max_depth=None, max_features=sqrt, min_samples_leaf=1, min_samples_split=4, n_estimators=1200 \n",
      "[CV]  max_depth=None, max_features=sqrt, min_samples_leaf=1, min_samples_split=4, n_estimators=1200, total=   1.6s\n",
      "[CV] max_depth=None, max_features=sqrt, min_samples_leaf=1, min_samples_split=4, n_estimators=1200 \n",
      "[CV]  max_depth=None, max_features=sqrt, min_samples_leaf=1, min_samples_split=4, n_estimators=1200, total=   1.6s\n",
      "[CV] max_depth=None, max_features=sqrt, min_samples_leaf=1, min_samples_split=4, n_estimators=1500 \n",
      "[CV]  max_depth=None, max_features=sqrt, min_samples_leaf=1, min_samples_split=4, n_estimators=1500, total=   2.0s\n",
      "[CV] max_depth=None, max_features=sqrt, min_samples_leaf=1, min_samples_split=4, n_estimators=1500 \n",
      "[CV]  max_depth=None, max_features=sqrt, min_samples_leaf=1, min_samples_split=4, n_estimators=1500, total=   2.0s\n",
      "[CV] max_depth=None, max_features=sqrt, min_samples_leaf=1, min_samples_split=4, n_estimators=1500 \n",
      "[CV]  max_depth=None, max_features=sqrt, min_samples_leaf=1, min_samples_split=4, n_estimators=1500, total=   2.1s\n",
      "[CV] max_depth=None, max_features=sqrt, min_samples_leaf=1, min_samples_split=4, n_estimators=1500 \n",
      "[CV]  max_depth=None, max_features=sqrt, min_samples_leaf=1, min_samples_split=4, n_estimators=1500, total=   2.0s\n",
      "[CV] max_depth=None, max_features=sqrt, min_samples_leaf=1, min_samples_split=4, n_estimators=1500 \n",
      "[CV]  max_depth=None, max_features=sqrt, min_samples_leaf=1, min_samples_split=4, n_estimators=1500, total=   2.0s\n",
      "[CV] max_depth=None, max_features=sqrt, min_samples_leaf=1, min_samples_split=4, n_estimators=2000 \n",
      "[CV]  max_depth=None, max_features=sqrt, min_samples_leaf=1, min_samples_split=4, n_estimators=2000, total=   2.6s\n",
      "[CV] max_depth=None, max_features=sqrt, min_samples_leaf=1, min_samples_split=4, n_estimators=2000 \n",
      "[CV]  max_depth=None, max_features=sqrt, min_samples_leaf=1, min_samples_split=4, n_estimators=2000, total=   2.6s\n",
      "[CV] max_depth=None, max_features=sqrt, min_samples_leaf=1, min_samples_split=4, n_estimators=2000 \n",
      "[CV]  max_depth=None, max_features=sqrt, min_samples_leaf=1, min_samples_split=4, n_estimators=2000, total=   2.7s\n",
      "[CV] max_depth=None, max_features=sqrt, min_samples_leaf=1, min_samples_split=4, n_estimators=2000 \n",
      "[CV]  max_depth=None, max_features=sqrt, min_samples_leaf=1, min_samples_split=4, n_estimators=2000, total=   2.7s\n",
      "[CV] max_depth=None, max_features=sqrt, min_samples_leaf=1, min_samples_split=4, n_estimators=2000 \n",
      "[CV]  max_depth=None, max_features=sqrt, min_samples_leaf=1, min_samples_split=4, n_estimators=2000, total=   2.7s\n",
      "[CV] max_depth=None, max_features=sqrt, min_samples_leaf=1, min_samples_split=6, n_estimators=1200 \n",
      "[CV]  max_depth=None, max_features=sqrt, min_samples_leaf=1, min_samples_split=6, n_estimators=1200, total=   1.6s\n",
      "[CV] max_depth=None, max_features=sqrt, min_samples_leaf=1, min_samples_split=6, n_estimators=1200 \n",
      "[CV]  max_depth=None, max_features=sqrt, min_samples_leaf=1, min_samples_split=6, n_estimators=1200, total=   1.6s\n",
      "[CV] max_depth=None, max_features=sqrt, min_samples_leaf=1, min_samples_split=6, n_estimators=1200 \n",
      "[CV]  max_depth=None, max_features=sqrt, min_samples_leaf=1, min_samples_split=6, n_estimators=1200, total=   1.6s\n",
      "[CV] max_depth=None, max_features=sqrt, min_samples_leaf=1, min_samples_split=6, n_estimators=1200 \n",
      "[CV]  max_depth=None, max_features=sqrt, min_samples_leaf=1, min_samples_split=6, n_estimators=1200, total=   1.6s\n",
      "[CV] max_depth=None, max_features=sqrt, min_samples_leaf=1, min_samples_split=6, n_estimators=1200 \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV]  max_depth=None, max_features=sqrt, min_samples_leaf=1, min_samples_split=6, n_estimators=1200, total=   1.6s\n",
      "[CV] max_depth=None, max_features=sqrt, min_samples_leaf=1, min_samples_split=6, n_estimators=1500 \n",
      "[CV]  max_depth=None, max_features=sqrt, min_samples_leaf=1, min_samples_split=6, n_estimators=1500, total=   2.0s\n",
      "[CV] max_depth=None, max_features=sqrt, min_samples_leaf=1, min_samples_split=6, n_estimators=1500 \n",
      "[CV]  max_depth=None, max_features=sqrt, min_samples_leaf=1, min_samples_split=6, n_estimators=1500, total=   2.0s\n",
      "[CV] max_depth=None, max_features=sqrt, min_samples_leaf=1, min_samples_split=6, n_estimators=1500 \n",
      "[CV]  max_depth=None, max_features=sqrt, min_samples_leaf=1, min_samples_split=6, n_estimators=1500, total=   2.0s\n",
      "[CV] max_depth=None, max_features=sqrt, min_samples_leaf=1, min_samples_split=6, n_estimators=1500 \n",
      "[CV]  max_depth=None, max_features=sqrt, min_samples_leaf=1, min_samples_split=6, n_estimators=1500, total=   2.0s\n",
      "[CV] max_depth=None, max_features=sqrt, min_samples_leaf=1, min_samples_split=6, n_estimators=1500 \n",
      "[CV]  max_depth=None, max_features=sqrt, min_samples_leaf=1, min_samples_split=6, n_estimators=1500, total=   2.0s\n",
      "[CV] max_depth=None, max_features=sqrt, min_samples_leaf=1, min_samples_split=6, n_estimators=2000 \n",
      "[CV]  max_depth=None, max_features=sqrt, min_samples_leaf=1, min_samples_split=6, n_estimators=2000, total=   2.6s\n",
      "[CV] max_depth=None, max_features=sqrt, min_samples_leaf=1, min_samples_split=6, n_estimators=2000 \n",
      "[CV]  max_depth=None, max_features=sqrt, min_samples_leaf=1, min_samples_split=6, n_estimators=2000, total=   2.8s\n",
      "[CV] max_depth=None, max_features=sqrt, min_samples_leaf=1, min_samples_split=6, n_estimators=2000 \n",
      "[CV]  max_depth=None, max_features=sqrt, min_samples_leaf=1, min_samples_split=6, n_estimators=2000, total=   2.7s\n",
      "[CV] max_depth=None, max_features=sqrt, min_samples_leaf=1, min_samples_split=6, n_estimators=2000 \n",
      "[CV]  max_depth=None, max_features=sqrt, min_samples_leaf=1, min_samples_split=6, n_estimators=2000, total=   2.8s\n",
      "[CV] max_depth=None, max_features=sqrt, min_samples_leaf=1, min_samples_split=6, n_estimators=2000 \n",
      "[CV]  max_depth=None, max_features=sqrt, min_samples_leaf=1, min_samples_split=6, n_estimators=2000, total=   2.8s\n",
      "[CV] max_depth=None, max_features=sqrt, min_samples_leaf=2, min_samples_split=4, n_estimators=1200 \n",
      "[CV]  max_depth=None, max_features=sqrt, min_samples_leaf=2, min_samples_split=4, n_estimators=1200, total=   1.7s\n",
      "[CV] max_depth=None, max_features=sqrt, min_samples_leaf=2, min_samples_split=4, n_estimators=1200 \n",
      "[CV]  max_depth=None, max_features=sqrt, min_samples_leaf=2, min_samples_split=4, n_estimators=1200, total=   1.7s\n",
      "[CV] max_depth=None, max_features=sqrt, min_samples_leaf=2, min_samples_split=4, n_estimators=1200 \n",
      "[CV]  max_depth=None, max_features=sqrt, min_samples_leaf=2, min_samples_split=4, n_estimators=1200, total=   1.7s\n",
      "[CV] max_depth=None, max_features=sqrt, min_samples_leaf=2, min_samples_split=4, n_estimators=1200 \n",
      "[CV]  max_depth=None, max_features=sqrt, min_samples_leaf=2, min_samples_split=4, n_estimators=1200, total=   1.6s\n",
      "[CV] max_depth=None, max_features=sqrt, min_samples_leaf=2, min_samples_split=4, n_estimators=1200 \n",
      "[CV]  max_depth=None, max_features=sqrt, min_samples_leaf=2, min_samples_split=4, n_estimators=1200, total=   1.6s\n",
      "[CV] max_depth=None, max_features=sqrt, min_samples_leaf=2, min_samples_split=4, n_estimators=1500 \n",
      "[CV]  max_depth=None, max_features=sqrt, min_samples_leaf=2, min_samples_split=4, n_estimators=1500, total=   2.1s\n",
      "[CV] max_depth=None, max_features=sqrt, min_samples_leaf=2, min_samples_split=4, n_estimators=1500 \n",
      "[CV]  max_depth=None, max_features=sqrt, min_samples_leaf=2, min_samples_split=4, n_estimators=1500, total=   2.1s\n",
      "[CV] max_depth=None, max_features=sqrt, min_samples_leaf=2, min_samples_split=4, n_estimators=1500 \n",
      "[CV]  max_depth=None, max_features=sqrt, min_samples_leaf=2, min_samples_split=4, n_estimators=1500, total=   2.0s\n",
      "[CV] max_depth=None, max_features=sqrt, min_samples_leaf=2, min_samples_split=4, n_estimators=1500 \n",
      "[CV]  max_depth=None, max_features=sqrt, min_samples_leaf=2, min_samples_split=4, n_estimators=1500, total=   2.0s\n",
      "[CV] max_depth=None, max_features=sqrt, min_samples_leaf=2, min_samples_split=4, n_estimators=1500 \n",
      "[CV]  max_depth=None, max_features=sqrt, min_samples_leaf=2, min_samples_split=4, n_estimators=1500, total=   2.0s\n",
      "[CV] max_depth=None, max_features=sqrt, min_samples_leaf=2, min_samples_split=4, n_estimators=2000 \n",
      "[CV]  max_depth=None, max_features=sqrt, min_samples_leaf=2, min_samples_split=4, n_estimators=2000, total=   2.7s\n",
      "[CV] max_depth=None, max_features=sqrt, min_samples_leaf=2, min_samples_split=4, n_estimators=2000 \n",
      "[CV]  max_depth=None, max_features=sqrt, min_samples_leaf=2, min_samples_split=4, n_estimators=2000, total=   2.8s\n",
      "[CV] max_depth=None, max_features=sqrt, min_samples_leaf=2, min_samples_split=4, n_estimators=2000 \n",
      "[CV]  max_depth=None, max_features=sqrt, min_samples_leaf=2, min_samples_split=4, n_estimators=2000, total=   2.7s\n",
      "[CV] max_depth=None, max_features=sqrt, min_samples_leaf=2, min_samples_split=4, n_estimators=2000 \n",
      "[CV]  max_depth=None, max_features=sqrt, min_samples_leaf=2, min_samples_split=4, n_estimators=2000, total=   2.9s\n",
      "[CV] max_depth=None, max_features=sqrt, min_samples_leaf=2, min_samples_split=4, n_estimators=2000 \n",
      "[CV]  max_depth=None, max_features=sqrt, min_samples_leaf=2, min_samples_split=4, n_estimators=2000, total=   2.7s\n",
      "[CV] max_depth=None, max_features=sqrt, min_samples_leaf=2, min_samples_split=6, n_estimators=1200 \n",
      "[CV]  max_depth=None, max_features=sqrt, min_samples_leaf=2, min_samples_split=6, n_estimators=1200, total=   1.6s\n",
      "[CV] max_depth=None, max_features=sqrt, min_samples_leaf=2, min_samples_split=6, n_estimators=1200 \n",
      "[CV]  max_depth=None, max_features=sqrt, min_samples_leaf=2, min_samples_split=6, n_estimators=1200, total=   1.6s\n",
      "[CV] max_depth=None, max_features=sqrt, min_samples_leaf=2, min_samples_split=6, n_estimators=1200 \n",
      "[CV]  max_depth=None, max_features=sqrt, min_samples_leaf=2, min_samples_split=6, n_estimators=1200, total=   1.6s\n",
      "[CV] max_depth=None, max_features=sqrt, min_samples_leaf=2, min_samples_split=6, n_estimators=1200 \n",
      "[CV]  max_depth=None, max_features=sqrt, min_samples_leaf=2, min_samples_split=6, n_estimators=1200, total=   1.6s\n",
      "[CV] max_depth=None, max_features=sqrt, min_samples_leaf=2, min_samples_split=6, n_estimators=1200 \n",
      "[CV]  max_depth=None, max_features=sqrt, min_samples_leaf=2, min_samples_split=6, n_estimators=1200, total=   1.7s\n",
      "[CV] max_depth=None, max_features=sqrt, min_samples_leaf=2, min_samples_split=6, n_estimators=1500 \n",
      "[CV]  max_depth=None, max_features=sqrt, min_samples_leaf=2, min_samples_split=6, n_estimators=1500, total=   2.0s\n",
      "[CV] max_depth=None, max_features=sqrt, min_samples_leaf=2, min_samples_split=6, n_estimators=1500 \n",
      "[CV]  max_depth=None, max_features=sqrt, min_samples_leaf=2, min_samples_split=6, n_estimators=1500, total=   2.0s\n",
      "[CV] max_depth=None, max_features=sqrt, min_samples_leaf=2, min_samples_split=6, n_estimators=1500 \n",
      "[CV]  max_depth=None, max_features=sqrt, min_samples_leaf=2, min_samples_split=6, n_estimators=1500, total=   2.0s\n",
      "[CV] max_depth=None, max_features=sqrt, min_samples_leaf=2, min_samples_split=6, n_estimators=1500 \n",
      "[CV]  max_depth=None, max_features=sqrt, min_samples_leaf=2, min_samples_split=6, n_estimators=1500, total=   2.0s\n",
      "[CV] max_depth=None, max_features=sqrt, min_samples_leaf=2, min_samples_split=6, n_estimators=1500 \n",
      "[CV]  max_depth=None, max_features=sqrt, min_samples_leaf=2, min_samples_split=6, n_estimators=1500, total=   2.0s\n",
      "[CV] max_depth=None, max_features=sqrt, min_samples_leaf=2, min_samples_split=6, n_estimators=2000 \n",
      "[CV]  max_depth=None, max_features=sqrt, min_samples_leaf=2, min_samples_split=6, n_estimators=2000, total=   2.6s\n",
      "[CV] max_depth=None, max_features=sqrt, min_samples_leaf=2, min_samples_split=6, n_estimators=2000 \n",
      "[CV]  max_depth=None, max_features=sqrt, min_samples_leaf=2, min_samples_split=6, n_estimators=2000, total=   2.7s\n",
      "[CV] max_depth=None, max_features=sqrt, min_samples_leaf=2, min_samples_split=6, n_estimators=2000 \n",
      "[CV]  max_depth=None, max_features=sqrt, min_samples_leaf=2, min_samples_split=6, n_estimators=2000, total=   2.7s\n",
      "[CV] max_depth=None, max_features=sqrt, min_samples_leaf=2, min_samples_split=6, n_estimators=2000 \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV]  max_depth=None, max_features=sqrt, min_samples_leaf=2, min_samples_split=6, n_estimators=2000, total=   2.7s\n",
      "[CV] max_depth=None, max_features=sqrt, min_samples_leaf=2, min_samples_split=6, n_estimators=2000 \n",
      "[CV]  max_depth=None, max_features=sqrt, min_samples_leaf=2, min_samples_split=6, n_estimators=2000, total=   2.7s\n",
      "[CV] max_depth=5, max_features=auto, min_samples_leaf=1, min_samples_split=4, n_estimators=1200 \n",
      "[CV]  max_depth=5, max_features=auto, min_samples_leaf=1, min_samples_split=4, n_estimators=1200, total=   1.6s\n",
      "[CV] max_depth=5, max_features=auto, min_samples_leaf=1, min_samples_split=4, n_estimators=1200 \n",
      "[CV]  max_depth=5, max_features=auto, min_samples_leaf=1, min_samples_split=4, n_estimators=1200, total=   1.6s\n",
      "[CV] max_depth=5, max_features=auto, min_samples_leaf=1, min_samples_split=4, n_estimators=1200 \n",
      "[CV]  max_depth=5, max_features=auto, min_samples_leaf=1, min_samples_split=4, n_estimators=1200, total=   1.5s\n",
      "[CV] max_depth=5, max_features=auto, min_samples_leaf=1, min_samples_split=4, n_estimators=1200 \n",
      "[CV]  max_depth=5, max_features=auto, min_samples_leaf=1, min_samples_split=4, n_estimators=1200, total=   1.6s\n",
      "[CV] max_depth=5, max_features=auto, min_samples_leaf=1, min_samples_split=4, n_estimators=1200 \n",
      "[CV]  max_depth=5, max_features=auto, min_samples_leaf=1, min_samples_split=4, n_estimators=1200, total=   1.6s\n",
      "[CV] max_depth=5, max_features=auto, min_samples_leaf=1, min_samples_split=4, n_estimators=1500 \n",
      "[CV]  max_depth=5, max_features=auto, min_samples_leaf=1, min_samples_split=4, n_estimators=1500, total=   1.9s\n",
      "[CV] max_depth=5, max_features=auto, min_samples_leaf=1, min_samples_split=4, n_estimators=1500 \n",
      "[CV]  max_depth=5, max_features=auto, min_samples_leaf=1, min_samples_split=4, n_estimators=1500, total=   2.0s\n",
      "[CV] max_depth=5, max_features=auto, min_samples_leaf=1, min_samples_split=4, n_estimators=1500 \n",
      "[CV]  max_depth=5, max_features=auto, min_samples_leaf=1, min_samples_split=4, n_estimators=1500, total=   2.0s\n",
      "[CV] max_depth=5, max_features=auto, min_samples_leaf=1, min_samples_split=4, n_estimators=1500 \n",
      "[CV]  max_depth=5, max_features=auto, min_samples_leaf=1, min_samples_split=4, n_estimators=1500, total=   1.9s\n",
      "[CV] max_depth=5, max_features=auto, min_samples_leaf=1, min_samples_split=4, n_estimators=1500 \n",
      "[CV]  max_depth=5, max_features=auto, min_samples_leaf=1, min_samples_split=4, n_estimators=1500, total=   1.9s\n",
      "[CV] max_depth=5, max_features=auto, min_samples_leaf=1, min_samples_split=4, n_estimators=2000 \n",
      "[CV]  max_depth=5, max_features=auto, min_samples_leaf=1, min_samples_split=4, n_estimators=2000, total=   2.6s\n",
      "[CV] max_depth=5, max_features=auto, min_samples_leaf=1, min_samples_split=4, n_estimators=2000 \n",
      "[CV]  max_depth=5, max_features=auto, min_samples_leaf=1, min_samples_split=4, n_estimators=2000, total=   2.6s\n",
      "[CV] max_depth=5, max_features=auto, min_samples_leaf=1, min_samples_split=4, n_estimators=2000 \n",
      "[CV]  max_depth=5, max_features=auto, min_samples_leaf=1, min_samples_split=4, n_estimators=2000, total=   2.6s\n",
      "[CV] max_depth=5, max_features=auto, min_samples_leaf=1, min_samples_split=4, n_estimators=2000 \n",
      "[CV]  max_depth=5, max_features=auto, min_samples_leaf=1, min_samples_split=4, n_estimators=2000, total=   2.6s\n",
      "[CV] max_depth=5, max_features=auto, min_samples_leaf=1, min_samples_split=4, n_estimators=2000 \n",
      "[CV]  max_depth=5, max_features=auto, min_samples_leaf=1, min_samples_split=4, n_estimators=2000, total=   2.6s\n",
      "[CV] max_depth=5, max_features=auto, min_samples_leaf=1, min_samples_split=6, n_estimators=1200 \n",
      "[CV]  max_depth=5, max_features=auto, min_samples_leaf=1, min_samples_split=6, n_estimators=1200, total=   1.5s\n",
      "[CV] max_depth=5, max_features=auto, min_samples_leaf=1, min_samples_split=6, n_estimators=1200 \n",
      "[CV]  max_depth=5, max_features=auto, min_samples_leaf=1, min_samples_split=6, n_estimators=1200, total=   1.6s\n",
      "[CV] max_depth=5, max_features=auto, min_samples_leaf=1, min_samples_split=6, n_estimators=1200 \n",
      "[CV]  max_depth=5, max_features=auto, min_samples_leaf=1, min_samples_split=6, n_estimators=1200, total=   1.6s\n",
      "[CV] max_depth=5, max_features=auto, min_samples_leaf=1, min_samples_split=6, n_estimators=1200 \n",
      "[CV]  max_depth=5, max_features=auto, min_samples_leaf=1, min_samples_split=6, n_estimators=1200, total=   1.6s\n",
      "[CV] max_depth=5, max_features=auto, min_samples_leaf=1, min_samples_split=6, n_estimators=1200 \n",
      "[CV]  max_depth=5, max_features=auto, min_samples_leaf=1, min_samples_split=6, n_estimators=1200, total=   1.6s\n",
      "[CV] max_depth=5, max_features=auto, min_samples_leaf=1, min_samples_split=6, n_estimators=1500 \n",
      "[CV]  max_depth=5, max_features=auto, min_samples_leaf=1, min_samples_split=6, n_estimators=1500, total=   1.9s\n",
      "[CV] max_depth=5, max_features=auto, min_samples_leaf=1, min_samples_split=6, n_estimators=1500 \n",
      "[CV]  max_depth=5, max_features=auto, min_samples_leaf=1, min_samples_split=6, n_estimators=1500, total=   1.9s\n",
      "[CV] max_depth=5, max_features=auto, min_samples_leaf=1, min_samples_split=6, n_estimators=1500 \n",
      "[CV]  max_depth=5, max_features=auto, min_samples_leaf=1, min_samples_split=6, n_estimators=1500, total=   2.0s\n",
      "[CV] max_depth=5, max_features=auto, min_samples_leaf=1, min_samples_split=6, n_estimators=1500 \n",
      "[CV]  max_depth=5, max_features=auto, min_samples_leaf=1, min_samples_split=6, n_estimators=1500, total=   1.9s\n",
      "[CV] max_depth=5, max_features=auto, min_samples_leaf=1, min_samples_split=6, n_estimators=1500 \n",
      "[CV]  max_depth=5, max_features=auto, min_samples_leaf=1, min_samples_split=6, n_estimators=1500, total=   2.0s\n",
      "[CV] max_depth=5, max_features=auto, min_samples_leaf=1, min_samples_split=6, n_estimators=2000 \n",
      "[CV]  max_depth=5, max_features=auto, min_samples_leaf=1, min_samples_split=6, n_estimators=2000, total=   2.5s\n",
      "[CV] max_depth=5, max_features=auto, min_samples_leaf=1, min_samples_split=6, n_estimators=2000 \n",
      "[CV]  max_depth=5, max_features=auto, min_samples_leaf=1, min_samples_split=6, n_estimators=2000, total=   2.5s\n",
      "[CV] max_depth=5, max_features=auto, min_samples_leaf=1, min_samples_split=6, n_estimators=2000 \n",
      "[CV]  max_depth=5, max_features=auto, min_samples_leaf=1, min_samples_split=6, n_estimators=2000, total=   2.5s\n",
      "[CV] max_depth=5, max_features=auto, min_samples_leaf=1, min_samples_split=6, n_estimators=2000 \n",
      "[CV]  max_depth=5, max_features=auto, min_samples_leaf=1, min_samples_split=6, n_estimators=2000, total=   2.5s\n",
      "[CV] max_depth=5, max_features=auto, min_samples_leaf=1, min_samples_split=6, n_estimators=2000 \n",
      "[CV]  max_depth=5, max_features=auto, min_samples_leaf=1, min_samples_split=6, n_estimators=2000, total=   2.6s\n",
      "[CV] max_depth=5, max_features=auto, min_samples_leaf=2, min_samples_split=4, n_estimators=1200 \n",
      "[CV]  max_depth=5, max_features=auto, min_samples_leaf=2, min_samples_split=4, n_estimators=1200, total=   1.5s\n",
      "[CV] max_depth=5, max_features=auto, min_samples_leaf=2, min_samples_split=4, n_estimators=1200 \n",
      "[CV]  max_depth=5, max_features=auto, min_samples_leaf=2, min_samples_split=4, n_estimators=1200, total=   1.5s\n",
      "[CV] max_depth=5, max_features=auto, min_samples_leaf=2, min_samples_split=4, n_estimators=1200 \n",
      "[CV]  max_depth=5, max_features=auto, min_samples_leaf=2, min_samples_split=4, n_estimators=1200, total=   1.6s\n",
      "[CV] max_depth=5, max_features=auto, min_samples_leaf=2, min_samples_split=4, n_estimators=1200 \n",
      "[CV]  max_depth=5, max_features=auto, min_samples_leaf=2, min_samples_split=4, n_estimators=1200, total=   1.5s\n",
      "[CV] max_depth=5, max_features=auto, min_samples_leaf=2, min_samples_split=4, n_estimators=1200 \n",
      "[CV]  max_depth=5, max_features=auto, min_samples_leaf=2, min_samples_split=4, n_estimators=1200, total=   1.6s\n",
      "[CV] max_depth=5, max_features=auto, min_samples_leaf=2, min_samples_split=4, n_estimators=1500 \n",
      "[CV]  max_depth=5, max_features=auto, min_samples_leaf=2, min_samples_split=4, n_estimators=1500, total=   2.0s\n",
      "[CV] max_depth=5, max_features=auto, min_samples_leaf=2, min_samples_split=4, n_estimators=1500 \n",
      "[CV]  max_depth=5, max_features=auto, min_samples_leaf=2, min_samples_split=4, n_estimators=1500, total=   2.0s\n",
      "[CV] max_depth=5, max_features=auto, min_samples_leaf=2, min_samples_split=4, n_estimators=1500 \n",
      "[CV]  max_depth=5, max_features=auto, min_samples_leaf=2, min_samples_split=4, n_estimators=1500, total=   2.0s\n",
      "[CV] max_depth=5, max_features=auto, min_samples_leaf=2, min_samples_split=4, n_estimators=1500 \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV]  max_depth=5, max_features=auto, min_samples_leaf=2, min_samples_split=4, n_estimators=1500, total=   2.0s\n",
      "[CV] max_depth=5, max_features=auto, min_samples_leaf=2, min_samples_split=4, n_estimators=1500 \n",
      "[CV]  max_depth=5, max_features=auto, min_samples_leaf=2, min_samples_split=4, n_estimators=1500, total=   2.0s\n",
      "[CV] max_depth=5, max_features=auto, min_samples_leaf=2, min_samples_split=4, n_estimators=2000 \n",
      "[CV]  max_depth=5, max_features=auto, min_samples_leaf=2, min_samples_split=4, n_estimators=2000, total=   2.7s\n",
      "[CV] max_depth=5, max_features=auto, min_samples_leaf=2, min_samples_split=4, n_estimators=2000 \n",
      "[CV]  max_depth=5, max_features=auto, min_samples_leaf=2, min_samples_split=4, n_estimators=2000, total=   2.6s\n",
      "[CV] max_depth=5, max_features=auto, min_samples_leaf=2, min_samples_split=4, n_estimators=2000 \n",
      "[CV]  max_depth=5, max_features=auto, min_samples_leaf=2, min_samples_split=4, n_estimators=2000, total=   2.5s\n",
      "[CV] max_depth=5, max_features=auto, min_samples_leaf=2, min_samples_split=4, n_estimators=2000 \n",
      "[CV]  max_depth=5, max_features=auto, min_samples_leaf=2, min_samples_split=4, n_estimators=2000, total=   2.6s\n",
      "[CV] max_depth=5, max_features=auto, min_samples_leaf=2, min_samples_split=4, n_estimators=2000 \n",
      "[CV]  max_depth=5, max_features=auto, min_samples_leaf=2, min_samples_split=4, n_estimators=2000, total=   2.6s\n",
      "[CV] max_depth=5, max_features=auto, min_samples_leaf=2, min_samples_split=6, n_estimators=1200 \n",
      "[CV]  max_depth=5, max_features=auto, min_samples_leaf=2, min_samples_split=6, n_estimators=1200, total=   1.5s\n",
      "[CV] max_depth=5, max_features=auto, min_samples_leaf=2, min_samples_split=6, n_estimators=1200 \n",
      "[CV]  max_depth=5, max_features=auto, min_samples_leaf=2, min_samples_split=6, n_estimators=1200, total=   1.5s\n",
      "[CV] max_depth=5, max_features=auto, min_samples_leaf=2, min_samples_split=6, n_estimators=1200 \n",
      "[CV]  max_depth=5, max_features=auto, min_samples_leaf=2, min_samples_split=6, n_estimators=1200, total=   1.6s\n",
      "[CV] max_depth=5, max_features=auto, min_samples_leaf=2, min_samples_split=6, n_estimators=1200 \n",
      "[CV]  max_depth=5, max_features=auto, min_samples_leaf=2, min_samples_split=6, n_estimators=1200, total=   1.5s\n",
      "[CV] max_depth=5, max_features=auto, min_samples_leaf=2, min_samples_split=6, n_estimators=1200 \n",
      "[CV]  max_depth=5, max_features=auto, min_samples_leaf=2, min_samples_split=6, n_estimators=1200, total=   1.5s\n",
      "[CV] max_depth=5, max_features=auto, min_samples_leaf=2, min_samples_split=6, n_estimators=1500 \n",
      "[CV]  max_depth=5, max_features=auto, min_samples_leaf=2, min_samples_split=6, n_estimators=1500, total=   1.9s\n",
      "[CV] max_depth=5, max_features=auto, min_samples_leaf=2, min_samples_split=6, n_estimators=1500 \n",
      "[CV]  max_depth=5, max_features=auto, min_samples_leaf=2, min_samples_split=6, n_estimators=1500, total=   1.9s\n",
      "[CV] max_depth=5, max_features=auto, min_samples_leaf=2, min_samples_split=6, n_estimators=1500 \n",
      "[CV]  max_depth=5, max_features=auto, min_samples_leaf=2, min_samples_split=6, n_estimators=1500, total=   2.0s\n",
      "[CV] max_depth=5, max_features=auto, min_samples_leaf=2, min_samples_split=6, n_estimators=1500 \n",
      "[CV]  max_depth=5, max_features=auto, min_samples_leaf=2, min_samples_split=6, n_estimators=1500, total=   1.9s\n",
      "[CV] max_depth=5, max_features=auto, min_samples_leaf=2, min_samples_split=6, n_estimators=1500 \n",
      "[CV]  max_depth=5, max_features=auto, min_samples_leaf=2, min_samples_split=6, n_estimators=1500, total=   2.0s\n",
      "[CV] max_depth=5, max_features=auto, min_samples_leaf=2, min_samples_split=6, n_estimators=2000 \n",
      "[CV]  max_depth=5, max_features=auto, min_samples_leaf=2, min_samples_split=6, n_estimators=2000, total=   2.6s\n",
      "[CV] max_depth=5, max_features=auto, min_samples_leaf=2, min_samples_split=6, n_estimators=2000 \n",
      "[CV]  max_depth=5, max_features=auto, min_samples_leaf=2, min_samples_split=6, n_estimators=2000, total=   2.6s\n",
      "[CV] max_depth=5, max_features=auto, min_samples_leaf=2, min_samples_split=6, n_estimators=2000 \n",
      "[CV]  max_depth=5, max_features=auto, min_samples_leaf=2, min_samples_split=6, n_estimators=2000, total=   2.6s\n",
      "[CV] max_depth=5, max_features=auto, min_samples_leaf=2, min_samples_split=6, n_estimators=2000 \n",
      "[CV]  max_depth=5, max_features=auto, min_samples_leaf=2, min_samples_split=6, n_estimators=2000, total=   2.6s\n",
      "[CV] max_depth=5, max_features=auto, min_samples_leaf=2, min_samples_split=6, n_estimators=2000 \n",
      "[CV]  max_depth=5, max_features=auto, min_samples_leaf=2, min_samples_split=6, n_estimators=2000, total=   2.5s\n",
      "[CV] max_depth=5, max_features=sqrt, min_samples_leaf=1, min_samples_split=4, n_estimators=1200 \n",
      "[CV]  max_depth=5, max_features=sqrt, min_samples_leaf=1, min_samples_split=4, n_estimators=1200, total=   1.6s\n",
      "[CV] max_depth=5, max_features=sqrt, min_samples_leaf=1, min_samples_split=4, n_estimators=1200 \n",
      "[CV]  max_depth=5, max_features=sqrt, min_samples_leaf=1, min_samples_split=4, n_estimators=1200, total=   1.6s\n",
      "[CV] max_depth=5, max_features=sqrt, min_samples_leaf=1, min_samples_split=4, n_estimators=1200 \n",
      "[CV]  max_depth=5, max_features=sqrt, min_samples_leaf=1, min_samples_split=4, n_estimators=1200, total=   1.6s\n",
      "[CV] max_depth=5, max_features=sqrt, min_samples_leaf=1, min_samples_split=4, n_estimators=1200 \n",
      "[CV]  max_depth=5, max_features=sqrt, min_samples_leaf=1, min_samples_split=4, n_estimators=1200, total=   1.6s\n",
      "[CV] max_depth=5, max_features=sqrt, min_samples_leaf=1, min_samples_split=4, n_estimators=1200 \n",
      "[CV]  max_depth=5, max_features=sqrt, min_samples_leaf=1, min_samples_split=4, n_estimators=1200, total=   1.6s\n",
      "[CV] max_depth=5, max_features=sqrt, min_samples_leaf=1, min_samples_split=4, n_estimators=1500 \n",
      "[CV]  max_depth=5, max_features=sqrt, min_samples_leaf=1, min_samples_split=4, n_estimators=1500, total=   1.9s\n",
      "[CV] max_depth=5, max_features=sqrt, min_samples_leaf=1, min_samples_split=4, n_estimators=1500 \n",
      "[CV]  max_depth=5, max_features=sqrt, min_samples_leaf=1, min_samples_split=4, n_estimators=1500, total=   2.0s\n",
      "[CV] max_depth=5, max_features=sqrt, min_samples_leaf=1, min_samples_split=4, n_estimators=1500 \n",
      "[CV]  max_depth=5, max_features=sqrt, min_samples_leaf=1, min_samples_split=4, n_estimators=1500, total=   1.9s\n",
      "[CV] max_depth=5, max_features=sqrt, min_samples_leaf=1, min_samples_split=4, n_estimators=1500 \n",
      "[CV]  max_depth=5, max_features=sqrt, min_samples_leaf=1, min_samples_split=4, n_estimators=1500, total=   2.0s\n",
      "[CV] max_depth=5, max_features=sqrt, min_samples_leaf=1, min_samples_split=4, n_estimators=1500 \n",
      "[CV]  max_depth=5, max_features=sqrt, min_samples_leaf=1, min_samples_split=4, n_estimators=1500, total=   2.0s\n",
      "[CV] max_depth=5, max_features=sqrt, min_samples_leaf=1, min_samples_split=4, n_estimators=2000 \n",
      "[CV]  max_depth=5, max_features=sqrt, min_samples_leaf=1, min_samples_split=4, n_estimators=2000, total=   2.6s\n",
      "[CV] max_depth=5, max_features=sqrt, min_samples_leaf=1, min_samples_split=4, n_estimators=2000 \n",
      "[CV]  max_depth=5, max_features=sqrt, min_samples_leaf=1, min_samples_split=4, n_estimators=2000, total=   2.6s\n",
      "[CV] max_depth=5, max_features=sqrt, min_samples_leaf=1, min_samples_split=4, n_estimators=2000 \n",
      "[CV]  max_depth=5, max_features=sqrt, min_samples_leaf=1, min_samples_split=4, n_estimators=2000, total=   2.6s\n",
      "[CV] max_depth=5, max_features=sqrt, min_samples_leaf=1, min_samples_split=4, n_estimators=2000 \n",
      "[CV]  max_depth=5, max_features=sqrt, min_samples_leaf=1, min_samples_split=4, n_estimators=2000, total=   2.5s\n",
      "[CV] max_depth=5, max_features=sqrt, min_samples_leaf=1, min_samples_split=4, n_estimators=2000 \n",
      "[CV]  max_depth=5, max_features=sqrt, min_samples_leaf=1, min_samples_split=4, n_estimators=2000, total=   2.6s\n",
      "[CV] max_depth=5, max_features=sqrt, min_samples_leaf=1, min_samples_split=6, n_estimators=1200 \n",
      "[CV]  max_depth=5, max_features=sqrt, min_samples_leaf=1, min_samples_split=6, n_estimators=1200, total=   1.5s\n",
      "[CV] max_depth=5, max_features=sqrt, min_samples_leaf=1, min_samples_split=6, n_estimators=1200 \n",
      "[CV]  max_depth=5, max_features=sqrt, min_samples_leaf=1, min_samples_split=6, n_estimators=1200, total=   1.6s\n",
      "[CV] max_depth=5, max_features=sqrt, min_samples_leaf=1, min_samples_split=6, n_estimators=1200 \n",
      "[CV]  max_depth=5, max_features=sqrt, min_samples_leaf=1, min_samples_split=6, n_estimators=1200, total=   1.5s\n",
      "[CV] max_depth=5, max_features=sqrt, min_samples_leaf=1, min_samples_split=6, n_estimators=1200 \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV]  max_depth=5, max_features=sqrt, min_samples_leaf=1, min_samples_split=6, n_estimators=1200, total=   1.5s\n",
      "[CV] max_depth=5, max_features=sqrt, min_samples_leaf=1, min_samples_split=6, n_estimators=1200 \n",
      "[CV]  max_depth=5, max_features=sqrt, min_samples_leaf=1, min_samples_split=6, n_estimators=1200, total=   1.5s\n",
      "[CV] max_depth=5, max_features=sqrt, min_samples_leaf=1, min_samples_split=6, n_estimators=1500 \n",
      "[CV]  max_depth=5, max_features=sqrt, min_samples_leaf=1, min_samples_split=6, n_estimators=1500, total=   2.0s\n",
      "[CV] max_depth=5, max_features=sqrt, min_samples_leaf=1, min_samples_split=6, n_estimators=1500 \n",
      "[CV]  max_depth=5, max_features=sqrt, min_samples_leaf=1, min_samples_split=6, n_estimators=1500, total=   1.9s\n",
      "[CV] max_depth=5, max_features=sqrt, min_samples_leaf=1, min_samples_split=6, n_estimators=1500 \n",
      "[CV]  max_depth=5, max_features=sqrt, min_samples_leaf=1, min_samples_split=6, n_estimators=1500, total=   1.9s\n",
      "[CV] max_depth=5, max_features=sqrt, min_samples_leaf=1, min_samples_split=6, n_estimators=1500 \n",
      "[CV]  max_depth=5, max_features=sqrt, min_samples_leaf=1, min_samples_split=6, n_estimators=1500, total=   1.9s\n",
      "[CV] max_depth=5, max_features=sqrt, min_samples_leaf=1, min_samples_split=6, n_estimators=1500 \n",
      "[CV]  max_depth=5, max_features=sqrt, min_samples_leaf=1, min_samples_split=6, n_estimators=1500, total=   1.9s\n",
      "[CV] max_depth=5, max_features=sqrt, min_samples_leaf=1, min_samples_split=6, n_estimators=2000 \n",
      "[CV]  max_depth=5, max_features=sqrt, min_samples_leaf=1, min_samples_split=6, n_estimators=2000, total=   2.6s\n",
      "[CV] max_depth=5, max_features=sqrt, min_samples_leaf=1, min_samples_split=6, n_estimators=2000 \n",
      "[CV]  max_depth=5, max_features=sqrt, min_samples_leaf=1, min_samples_split=6, n_estimators=2000, total=   2.6s\n",
      "[CV] max_depth=5, max_features=sqrt, min_samples_leaf=1, min_samples_split=6, n_estimators=2000 \n",
      "[CV]  max_depth=5, max_features=sqrt, min_samples_leaf=1, min_samples_split=6, n_estimators=2000, total=   2.5s\n",
      "[CV] max_depth=5, max_features=sqrt, min_samples_leaf=1, min_samples_split=6, n_estimators=2000 \n",
      "[CV]  max_depth=5, max_features=sqrt, min_samples_leaf=1, min_samples_split=6, n_estimators=2000, total=   2.6s\n",
      "[CV] max_depth=5, max_features=sqrt, min_samples_leaf=1, min_samples_split=6, n_estimators=2000 \n",
      "[CV]  max_depth=5, max_features=sqrt, min_samples_leaf=1, min_samples_split=6, n_estimators=2000, total=   2.6s\n",
      "[CV] max_depth=5, max_features=sqrt, min_samples_leaf=2, min_samples_split=4, n_estimators=1200 \n",
      "[CV]  max_depth=5, max_features=sqrt, min_samples_leaf=2, min_samples_split=4, n_estimators=1200, total=   1.5s\n",
      "[CV] max_depth=5, max_features=sqrt, min_samples_leaf=2, min_samples_split=4, n_estimators=1200 \n",
      "[CV]  max_depth=5, max_features=sqrt, min_samples_leaf=2, min_samples_split=4, n_estimators=1200, total=   1.5s\n",
      "[CV] max_depth=5, max_features=sqrt, min_samples_leaf=2, min_samples_split=4, n_estimators=1200 \n",
      "[CV]  max_depth=5, max_features=sqrt, min_samples_leaf=2, min_samples_split=4, n_estimators=1200, total=   1.6s\n",
      "[CV] max_depth=5, max_features=sqrt, min_samples_leaf=2, min_samples_split=4, n_estimators=1200 \n",
      "[CV]  max_depth=5, max_features=sqrt, min_samples_leaf=2, min_samples_split=4, n_estimators=1200, total=   1.6s\n",
      "[CV] max_depth=5, max_features=sqrt, min_samples_leaf=2, min_samples_split=4, n_estimators=1200 \n",
      "[CV]  max_depth=5, max_features=sqrt, min_samples_leaf=2, min_samples_split=4, n_estimators=1200, total=   1.6s\n",
      "[CV] max_depth=5, max_features=sqrt, min_samples_leaf=2, min_samples_split=4, n_estimators=1500 \n",
      "[CV]  max_depth=5, max_features=sqrt, min_samples_leaf=2, min_samples_split=4, n_estimators=1500, total=   2.0s\n",
      "[CV] max_depth=5, max_features=sqrt, min_samples_leaf=2, min_samples_split=4, n_estimators=1500 \n",
      "[CV]  max_depth=5, max_features=sqrt, min_samples_leaf=2, min_samples_split=4, n_estimators=1500, total=   2.0s\n",
      "[CV] max_depth=5, max_features=sqrt, min_samples_leaf=2, min_samples_split=4, n_estimators=1500 \n",
      "[CV]  max_depth=5, max_features=sqrt, min_samples_leaf=2, min_samples_split=4, n_estimators=1500, total=   2.0s\n",
      "[CV] max_depth=5, max_features=sqrt, min_samples_leaf=2, min_samples_split=4, n_estimators=1500 \n",
      "[CV]  max_depth=5, max_features=sqrt, min_samples_leaf=2, min_samples_split=4, n_estimators=1500, total=   2.0s\n",
      "[CV] max_depth=5, max_features=sqrt, min_samples_leaf=2, min_samples_split=4, n_estimators=1500 \n",
      "[CV]  max_depth=5, max_features=sqrt, min_samples_leaf=2, min_samples_split=4, n_estimators=1500, total=   2.0s\n",
      "[CV] max_depth=5, max_features=sqrt, min_samples_leaf=2, min_samples_split=4, n_estimators=2000 \n",
      "[CV]  max_depth=5, max_features=sqrt, min_samples_leaf=2, min_samples_split=4, n_estimators=2000, total=   2.6s\n",
      "[CV] max_depth=5, max_features=sqrt, min_samples_leaf=2, min_samples_split=4, n_estimators=2000 \n",
      "[CV]  max_depth=5, max_features=sqrt, min_samples_leaf=2, min_samples_split=4, n_estimators=2000, total=   2.6s\n",
      "[CV] max_depth=5, max_features=sqrt, min_samples_leaf=2, min_samples_split=4, n_estimators=2000 \n",
      "[CV]  max_depth=5, max_features=sqrt, min_samples_leaf=2, min_samples_split=4, n_estimators=2000, total=   2.5s\n",
      "[CV] max_depth=5, max_features=sqrt, min_samples_leaf=2, min_samples_split=4, n_estimators=2000 \n",
      "[CV]  max_depth=5, max_features=sqrt, min_samples_leaf=2, min_samples_split=4, n_estimators=2000, total=   2.6s\n",
      "[CV] max_depth=5, max_features=sqrt, min_samples_leaf=2, min_samples_split=4, n_estimators=2000 \n",
      "[CV]  max_depth=5, max_features=sqrt, min_samples_leaf=2, min_samples_split=4, n_estimators=2000, total=   2.5s\n",
      "[CV] max_depth=5, max_features=sqrt, min_samples_leaf=2, min_samples_split=6, n_estimators=1200 \n",
      "[CV]  max_depth=5, max_features=sqrt, min_samples_leaf=2, min_samples_split=6, n_estimators=1200, total=   1.5s\n",
      "[CV] max_depth=5, max_features=sqrt, min_samples_leaf=2, min_samples_split=6, n_estimators=1200 \n",
      "[CV]  max_depth=5, max_features=sqrt, min_samples_leaf=2, min_samples_split=6, n_estimators=1200, total=   1.5s\n",
      "[CV] max_depth=5, max_features=sqrt, min_samples_leaf=2, min_samples_split=6, n_estimators=1200 \n",
      "[CV]  max_depth=5, max_features=sqrt, min_samples_leaf=2, min_samples_split=6, n_estimators=1200, total=   1.5s\n",
      "[CV] max_depth=5, max_features=sqrt, min_samples_leaf=2, min_samples_split=6, n_estimators=1200 \n",
      "[CV]  max_depth=5, max_features=sqrt, min_samples_leaf=2, min_samples_split=6, n_estimators=1200, total=   1.6s\n",
      "[CV] max_depth=5, max_features=sqrt, min_samples_leaf=2, min_samples_split=6, n_estimators=1200 \n",
      "[CV]  max_depth=5, max_features=sqrt, min_samples_leaf=2, min_samples_split=6, n_estimators=1200, total=   1.5s\n",
      "[CV] max_depth=5, max_features=sqrt, min_samples_leaf=2, min_samples_split=6, n_estimators=1500 \n",
      "[CV]  max_depth=5, max_features=sqrt, min_samples_leaf=2, min_samples_split=6, n_estimators=1500, total=   1.9s\n",
      "[CV] max_depth=5, max_features=sqrt, min_samples_leaf=2, min_samples_split=6, n_estimators=1500 \n",
      "[CV]  max_depth=5, max_features=sqrt, min_samples_leaf=2, min_samples_split=6, n_estimators=1500, total=   2.0s\n",
      "[CV] max_depth=5, max_features=sqrt, min_samples_leaf=2, min_samples_split=6, n_estimators=1500 \n",
      "[CV]  max_depth=5, max_features=sqrt, min_samples_leaf=2, min_samples_split=6, n_estimators=1500, total=   1.9s\n",
      "[CV] max_depth=5, max_features=sqrt, min_samples_leaf=2, min_samples_split=6, n_estimators=1500 \n",
      "[CV]  max_depth=5, max_features=sqrt, min_samples_leaf=2, min_samples_split=6, n_estimators=1500, total=   2.0s\n",
      "[CV] max_depth=5, max_features=sqrt, min_samples_leaf=2, min_samples_split=6, n_estimators=1500 \n",
      "[CV]  max_depth=5, max_features=sqrt, min_samples_leaf=2, min_samples_split=6, n_estimators=1500, total=   1.9s\n",
      "[CV] max_depth=5, max_features=sqrt, min_samples_leaf=2, min_samples_split=6, n_estimators=2000 \n",
      "[CV]  max_depth=5, max_features=sqrt, min_samples_leaf=2, min_samples_split=6, n_estimators=2000, total=   2.6s\n",
      "[CV] max_depth=5, max_features=sqrt, min_samples_leaf=2, min_samples_split=6, n_estimators=2000 \n",
      "[CV]  max_depth=5, max_features=sqrt, min_samples_leaf=2, min_samples_split=6, n_estimators=2000, total=   2.6s\n",
      "[CV] max_depth=5, max_features=sqrt, min_samples_leaf=2, min_samples_split=6, n_estimators=2000 \n",
      "[CV]  max_depth=5, max_features=sqrt, min_samples_leaf=2, min_samples_split=6, n_estimators=2000, total=   2.6s\n",
      "[CV] max_depth=5, max_features=sqrt, min_samples_leaf=2, min_samples_split=6, n_estimators=2000 \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV]  max_depth=5, max_features=sqrt, min_samples_leaf=2, min_samples_split=6, n_estimators=2000, total=   2.6s\n",
      "[CV] max_depth=5, max_features=sqrt, min_samples_leaf=2, min_samples_split=6, n_estimators=2000 \n",
      "[CV]  max_depth=5, max_features=sqrt, min_samples_leaf=2, min_samples_split=6, n_estimators=2000, total=   2.6s\n",
      "[CV] max_depth=10, max_features=auto, min_samples_leaf=1, min_samples_split=4, n_estimators=1200 \n",
      "[CV]  max_depth=10, max_features=auto, min_samples_leaf=1, min_samples_split=4, n_estimators=1200, total=   1.6s\n",
      "[CV] max_depth=10, max_features=auto, min_samples_leaf=1, min_samples_split=4, n_estimators=1200 \n",
      "[CV]  max_depth=10, max_features=auto, min_samples_leaf=1, min_samples_split=4, n_estimators=1200, total=   1.6s\n",
      "[CV] max_depth=10, max_features=auto, min_samples_leaf=1, min_samples_split=4, n_estimators=1200 \n",
      "[CV]  max_depth=10, max_features=auto, min_samples_leaf=1, min_samples_split=4, n_estimators=1200, total=   1.6s\n",
      "[CV] max_depth=10, max_features=auto, min_samples_leaf=1, min_samples_split=4, n_estimators=1200 \n",
      "[CV]  max_depth=10, max_features=auto, min_samples_leaf=1, min_samples_split=4, n_estimators=1200, total=   1.6s\n",
      "[CV] max_depth=10, max_features=auto, min_samples_leaf=1, min_samples_split=4, n_estimators=1200 \n",
      "[CV]  max_depth=10, max_features=auto, min_samples_leaf=1, min_samples_split=4, n_estimators=1200, total=   1.6s\n",
      "[CV] max_depth=10, max_features=auto, min_samples_leaf=1, min_samples_split=4, n_estimators=1500 \n",
      "[CV]  max_depth=10, max_features=auto, min_samples_leaf=1, min_samples_split=4, n_estimators=1500, total=   2.0s\n",
      "[CV] max_depth=10, max_features=auto, min_samples_leaf=1, min_samples_split=4, n_estimators=1500 \n",
      "[CV]  max_depth=10, max_features=auto, min_samples_leaf=1, min_samples_split=4, n_estimators=1500, total=   2.0s\n",
      "[CV] max_depth=10, max_features=auto, min_samples_leaf=1, min_samples_split=4, n_estimators=1500 \n",
      "[CV]  max_depth=10, max_features=auto, min_samples_leaf=1, min_samples_split=4, n_estimators=1500, total=   2.0s\n",
      "[CV] max_depth=10, max_features=auto, min_samples_leaf=1, min_samples_split=4, n_estimators=1500 \n",
      "[CV]  max_depth=10, max_features=auto, min_samples_leaf=1, min_samples_split=4, n_estimators=1500, total=   2.0s\n",
      "[CV] max_depth=10, max_features=auto, min_samples_leaf=1, min_samples_split=4, n_estimators=1500 \n",
      "[CV]  max_depth=10, max_features=auto, min_samples_leaf=1, min_samples_split=4, n_estimators=1500, total=   2.0s\n",
      "[CV] max_depth=10, max_features=auto, min_samples_leaf=1, min_samples_split=4, n_estimators=2000 \n",
      "[CV]  max_depth=10, max_features=auto, min_samples_leaf=1, min_samples_split=4, n_estimators=2000, total=   2.6s\n",
      "[CV] max_depth=10, max_features=auto, min_samples_leaf=1, min_samples_split=4, n_estimators=2000 \n",
      "[CV]  max_depth=10, max_features=auto, min_samples_leaf=1, min_samples_split=4, n_estimators=2000, total=   2.6s\n",
      "[CV] max_depth=10, max_features=auto, min_samples_leaf=1, min_samples_split=4, n_estimators=2000 \n",
      "[CV]  max_depth=10, max_features=auto, min_samples_leaf=1, min_samples_split=4, n_estimators=2000, total=   2.6s\n",
      "[CV] max_depth=10, max_features=auto, min_samples_leaf=1, min_samples_split=4, n_estimators=2000 \n",
      "[CV]  max_depth=10, max_features=auto, min_samples_leaf=1, min_samples_split=4, n_estimators=2000, total=   2.6s\n",
      "[CV] max_depth=10, max_features=auto, min_samples_leaf=1, min_samples_split=4, n_estimators=2000 \n",
      "[CV]  max_depth=10, max_features=auto, min_samples_leaf=1, min_samples_split=4, n_estimators=2000, total=   2.6s\n",
      "[CV] max_depth=10, max_features=auto, min_samples_leaf=1, min_samples_split=6, n_estimators=1200 \n",
      "[CV]  max_depth=10, max_features=auto, min_samples_leaf=1, min_samples_split=6, n_estimators=1200, total=   1.5s\n",
      "[CV] max_depth=10, max_features=auto, min_samples_leaf=1, min_samples_split=6, n_estimators=1200 \n",
      "[CV]  max_depth=10, max_features=auto, min_samples_leaf=1, min_samples_split=6, n_estimators=1200, total=   1.6s\n",
      "[CV] max_depth=10, max_features=auto, min_samples_leaf=1, min_samples_split=6, n_estimators=1200 \n",
      "[CV]  max_depth=10, max_features=auto, min_samples_leaf=1, min_samples_split=6, n_estimators=1200, total=   1.5s\n",
      "[CV] max_depth=10, max_features=auto, min_samples_leaf=1, min_samples_split=6, n_estimators=1200 \n",
      "[CV]  max_depth=10, max_features=auto, min_samples_leaf=1, min_samples_split=6, n_estimators=1200, total=   1.6s\n",
      "[CV] max_depth=10, max_features=auto, min_samples_leaf=1, min_samples_split=6, n_estimators=1200 \n",
      "[CV]  max_depth=10, max_features=auto, min_samples_leaf=1, min_samples_split=6, n_estimators=1200, total=   1.6s\n",
      "[CV] max_depth=10, max_features=auto, min_samples_leaf=1, min_samples_split=6, n_estimators=1500 \n",
      "[CV]  max_depth=10, max_features=auto, min_samples_leaf=1, min_samples_split=6, n_estimators=1500, total=   2.0s\n",
      "[CV] max_depth=10, max_features=auto, min_samples_leaf=1, min_samples_split=6, n_estimators=1500 \n",
      "[CV]  max_depth=10, max_features=auto, min_samples_leaf=1, min_samples_split=6, n_estimators=1500, total=   1.9s\n",
      "[CV] max_depth=10, max_features=auto, min_samples_leaf=1, min_samples_split=6, n_estimators=1500 \n",
      "[CV]  max_depth=10, max_features=auto, min_samples_leaf=1, min_samples_split=6, n_estimators=1500, total=   1.9s\n",
      "[CV] max_depth=10, max_features=auto, min_samples_leaf=1, min_samples_split=6, n_estimators=1500 \n",
      "[CV]  max_depth=10, max_features=auto, min_samples_leaf=1, min_samples_split=6, n_estimators=1500, total=   1.9s\n",
      "[CV] max_depth=10, max_features=auto, min_samples_leaf=1, min_samples_split=6, n_estimators=1500 \n",
      "[CV]  max_depth=10, max_features=auto, min_samples_leaf=1, min_samples_split=6, n_estimators=1500, total=   2.0s\n",
      "[CV] max_depth=10, max_features=auto, min_samples_leaf=1, min_samples_split=6, n_estimators=2000 \n",
      "[CV]  max_depth=10, max_features=auto, min_samples_leaf=1, min_samples_split=6, n_estimators=2000, total=   2.6s\n",
      "[CV] max_depth=10, max_features=auto, min_samples_leaf=1, min_samples_split=6, n_estimators=2000 \n",
      "[CV]  max_depth=10, max_features=auto, min_samples_leaf=1, min_samples_split=6, n_estimators=2000, total=   2.6s\n",
      "[CV] max_depth=10, max_features=auto, min_samples_leaf=1, min_samples_split=6, n_estimators=2000 \n",
      "[CV]  max_depth=10, max_features=auto, min_samples_leaf=1, min_samples_split=6, n_estimators=2000, total=   2.6s\n",
      "[CV] max_depth=10, max_features=auto, min_samples_leaf=1, min_samples_split=6, n_estimators=2000 \n",
      "[CV]  max_depth=10, max_features=auto, min_samples_leaf=1, min_samples_split=6, n_estimators=2000, total=   2.6s\n",
      "[CV] max_depth=10, max_features=auto, min_samples_leaf=1, min_samples_split=6, n_estimators=2000 \n",
      "[CV]  max_depth=10, max_features=auto, min_samples_leaf=1, min_samples_split=6, n_estimators=2000, total=   2.6s\n",
      "[CV] max_depth=10, max_features=auto, min_samples_leaf=2, min_samples_split=4, n_estimators=1200 \n",
      "[CV]  max_depth=10, max_features=auto, min_samples_leaf=2, min_samples_split=4, n_estimators=1200, total=   1.5s\n",
      "[CV] max_depth=10, max_features=auto, min_samples_leaf=2, min_samples_split=4, n_estimators=1200 \n",
      "[CV]  max_depth=10, max_features=auto, min_samples_leaf=2, min_samples_split=4, n_estimators=1200, total=   1.6s\n",
      "[CV] max_depth=10, max_features=auto, min_samples_leaf=2, min_samples_split=4, n_estimators=1200 \n",
      "[CV]  max_depth=10, max_features=auto, min_samples_leaf=2, min_samples_split=4, n_estimators=1200, total=   1.6s\n",
      "[CV] max_depth=10, max_features=auto, min_samples_leaf=2, min_samples_split=4, n_estimators=1200 \n",
      "[CV]  max_depth=10, max_features=auto, min_samples_leaf=2, min_samples_split=4, n_estimators=1200, total=   1.6s\n",
      "[CV] max_depth=10, max_features=auto, min_samples_leaf=2, min_samples_split=4, n_estimators=1200 \n",
      "[CV]  max_depth=10, max_features=auto, min_samples_leaf=2, min_samples_split=4, n_estimators=1200, total=   1.6s\n",
      "[CV] max_depth=10, max_features=auto, min_samples_leaf=2, min_samples_split=4, n_estimators=1500 \n",
      "[CV]  max_depth=10, max_features=auto, min_samples_leaf=2, min_samples_split=4, n_estimators=1500, total=   2.0s\n",
      "[CV] max_depth=10, max_features=auto, min_samples_leaf=2, min_samples_split=4, n_estimators=1500 \n",
      "[CV]  max_depth=10, max_features=auto, min_samples_leaf=2, min_samples_split=4, n_estimators=1500, total=   2.0s\n",
      "[CV] max_depth=10, max_features=auto, min_samples_leaf=2, min_samples_split=4, n_estimators=1500 \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV]  max_depth=10, max_features=auto, min_samples_leaf=2, min_samples_split=4, n_estimators=1500, total=   2.0s\n",
      "[CV] max_depth=10, max_features=auto, min_samples_leaf=2, min_samples_split=4, n_estimators=1500 \n",
      "[CV]  max_depth=10, max_features=auto, min_samples_leaf=2, min_samples_split=4, n_estimators=1500, total=   2.0s\n",
      "[CV] max_depth=10, max_features=auto, min_samples_leaf=2, min_samples_split=4, n_estimators=1500 \n",
      "[CV]  max_depth=10, max_features=auto, min_samples_leaf=2, min_samples_split=4, n_estimators=1500, total=   2.0s\n",
      "[CV] max_depth=10, max_features=auto, min_samples_leaf=2, min_samples_split=4, n_estimators=2000 \n",
      "[CV]  max_depth=10, max_features=auto, min_samples_leaf=2, min_samples_split=4, n_estimators=2000, total=   2.6s\n",
      "[CV] max_depth=10, max_features=auto, min_samples_leaf=2, min_samples_split=4, n_estimators=2000 \n",
      "[CV]  max_depth=10, max_features=auto, min_samples_leaf=2, min_samples_split=4, n_estimators=2000, total=   2.6s\n",
      "[CV] max_depth=10, max_features=auto, min_samples_leaf=2, min_samples_split=4, n_estimators=2000 \n",
      "[CV]  max_depth=10, max_features=auto, min_samples_leaf=2, min_samples_split=4, n_estimators=2000, total=   2.6s\n",
      "[CV] max_depth=10, max_features=auto, min_samples_leaf=2, min_samples_split=4, n_estimators=2000 \n",
      "[CV]  max_depth=10, max_features=auto, min_samples_leaf=2, min_samples_split=4, n_estimators=2000, total=   2.6s\n",
      "[CV] max_depth=10, max_features=auto, min_samples_leaf=2, min_samples_split=4, n_estimators=2000 \n",
      "[CV]  max_depth=10, max_features=auto, min_samples_leaf=2, min_samples_split=4, n_estimators=2000, total=   2.6s\n",
      "[CV] max_depth=10, max_features=auto, min_samples_leaf=2, min_samples_split=6, n_estimators=1200 \n",
      "[CV]  max_depth=10, max_features=auto, min_samples_leaf=2, min_samples_split=6, n_estimators=1200, total=   1.5s\n",
      "[CV] max_depth=10, max_features=auto, min_samples_leaf=2, min_samples_split=6, n_estimators=1200 \n",
      "[CV]  max_depth=10, max_features=auto, min_samples_leaf=2, min_samples_split=6, n_estimators=1200, total=   1.6s\n",
      "[CV] max_depth=10, max_features=auto, min_samples_leaf=2, min_samples_split=6, n_estimators=1200 \n",
      "[CV]  max_depth=10, max_features=auto, min_samples_leaf=2, min_samples_split=6, n_estimators=1200, total=   1.6s\n",
      "[CV] max_depth=10, max_features=auto, min_samples_leaf=2, min_samples_split=6, n_estimators=1200 \n",
      "[CV]  max_depth=10, max_features=auto, min_samples_leaf=2, min_samples_split=6, n_estimators=1200, total=   1.5s\n",
      "[CV] max_depth=10, max_features=auto, min_samples_leaf=2, min_samples_split=6, n_estimators=1200 \n",
      "[CV]  max_depth=10, max_features=auto, min_samples_leaf=2, min_samples_split=6, n_estimators=1200, total=   1.5s\n",
      "[CV] max_depth=10, max_features=auto, min_samples_leaf=2, min_samples_split=6, n_estimators=1500 \n",
      "[CV]  max_depth=10, max_features=auto, min_samples_leaf=2, min_samples_split=6, n_estimators=1500, total=   1.9s\n",
      "[CV] max_depth=10, max_features=auto, min_samples_leaf=2, min_samples_split=6, n_estimators=1500 \n",
      "[CV]  max_depth=10, max_features=auto, min_samples_leaf=2, min_samples_split=6, n_estimators=1500, total=   2.0s\n",
      "[CV] max_depth=10, max_features=auto, min_samples_leaf=2, min_samples_split=6, n_estimators=1500 \n",
      "[CV]  max_depth=10, max_features=auto, min_samples_leaf=2, min_samples_split=6, n_estimators=1500, total=   1.9s\n",
      "[CV] max_depth=10, max_features=auto, min_samples_leaf=2, min_samples_split=6, n_estimators=1500 \n",
      "[CV]  max_depth=10, max_features=auto, min_samples_leaf=2, min_samples_split=6, n_estimators=1500, total=   1.9s\n",
      "[CV] max_depth=10, max_features=auto, min_samples_leaf=2, min_samples_split=6, n_estimators=1500 \n",
      "[CV]  max_depth=10, max_features=auto, min_samples_leaf=2, min_samples_split=6, n_estimators=1500, total=   1.9s\n",
      "[CV] max_depth=10, max_features=auto, min_samples_leaf=2, min_samples_split=6, n_estimators=2000 \n",
      "[CV]  max_depth=10, max_features=auto, min_samples_leaf=2, min_samples_split=6, n_estimators=2000, total=   2.6s\n",
      "[CV] max_depth=10, max_features=auto, min_samples_leaf=2, min_samples_split=6, n_estimators=2000 \n",
      "[CV]  max_depth=10, max_features=auto, min_samples_leaf=2, min_samples_split=6, n_estimators=2000, total=   2.6s\n",
      "[CV] max_depth=10, max_features=auto, min_samples_leaf=2, min_samples_split=6, n_estimators=2000 \n",
      "[CV]  max_depth=10, max_features=auto, min_samples_leaf=2, min_samples_split=6, n_estimators=2000, total=   2.5s\n",
      "[CV] max_depth=10, max_features=auto, min_samples_leaf=2, min_samples_split=6, n_estimators=2000 \n",
      "[CV]  max_depth=10, max_features=auto, min_samples_leaf=2, min_samples_split=6, n_estimators=2000, total=   2.6s\n",
      "[CV] max_depth=10, max_features=auto, min_samples_leaf=2, min_samples_split=6, n_estimators=2000 \n",
      "[CV]  max_depth=10, max_features=auto, min_samples_leaf=2, min_samples_split=6, n_estimators=2000, total=   2.6s\n",
      "[CV] max_depth=10, max_features=sqrt, min_samples_leaf=1, min_samples_split=4, n_estimators=1200 \n",
      "[CV]  max_depth=10, max_features=sqrt, min_samples_leaf=1, min_samples_split=4, n_estimators=1200, total=   1.6s\n",
      "[CV] max_depth=10, max_features=sqrt, min_samples_leaf=1, min_samples_split=4, n_estimators=1200 \n",
      "[CV]  max_depth=10, max_features=sqrt, min_samples_leaf=1, min_samples_split=4, n_estimators=1200, total=   1.6s\n",
      "[CV] max_depth=10, max_features=sqrt, min_samples_leaf=1, min_samples_split=4, n_estimators=1200 \n",
      "[CV]  max_depth=10, max_features=sqrt, min_samples_leaf=1, min_samples_split=4, n_estimators=1200, total=   1.6s\n",
      "[CV] max_depth=10, max_features=sqrt, min_samples_leaf=1, min_samples_split=4, n_estimators=1200 \n",
      "[CV]  max_depth=10, max_features=sqrt, min_samples_leaf=1, min_samples_split=4, n_estimators=1200, total=   1.7s\n",
      "[CV] max_depth=10, max_features=sqrt, min_samples_leaf=1, min_samples_split=4, n_estimators=1200 \n",
      "[CV]  max_depth=10, max_features=sqrt, min_samples_leaf=1, min_samples_split=4, n_estimators=1200, total=   1.6s\n",
      "[CV] max_depth=10, max_features=sqrt, min_samples_leaf=1, min_samples_split=4, n_estimators=1500 \n",
      "[CV]  max_depth=10, max_features=sqrt, min_samples_leaf=1, min_samples_split=4, n_estimators=1500, total=   2.0s\n",
      "[CV] max_depth=10, max_features=sqrt, min_samples_leaf=1, min_samples_split=4, n_estimators=1500 \n",
      "[CV]  max_depth=10, max_features=sqrt, min_samples_leaf=1, min_samples_split=4, n_estimators=1500, total=   2.0s\n",
      "[CV] max_depth=10, max_features=sqrt, min_samples_leaf=1, min_samples_split=4, n_estimators=1500 \n",
      "[CV]  max_depth=10, max_features=sqrt, min_samples_leaf=1, min_samples_split=4, n_estimators=1500, total=   2.0s\n",
      "[CV] max_depth=10, max_features=sqrt, min_samples_leaf=1, min_samples_split=4, n_estimators=1500 \n",
      "[CV]  max_depth=10, max_features=sqrt, min_samples_leaf=1, min_samples_split=4, n_estimators=1500, total=   2.0s\n",
      "[CV] max_depth=10, max_features=sqrt, min_samples_leaf=1, min_samples_split=4, n_estimators=1500 \n",
      "[CV]  max_depth=10, max_features=sqrt, min_samples_leaf=1, min_samples_split=4, n_estimators=1500, total=   2.0s\n",
      "[CV] max_depth=10, max_features=sqrt, min_samples_leaf=1, min_samples_split=4, n_estimators=2000 \n",
      "[CV]  max_depth=10, max_features=sqrt, min_samples_leaf=1, min_samples_split=4, n_estimators=2000, total=   2.6s\n",
      "[CV] max_depth=10, max_features=sqrt, min_samples_leaf=1, min_samples_split=4, n_estimators=2000 \n",
      "[CV]  max_depth=10, max_features=sqrt, min_samples_leaf=1, min_samples_split=4, n_estimators=2000, total=   2.6s\n",
      "[CV] max_depth=10, max_features=sqrt, min_samples_leaf=1, min_samples_split=4, n_estimators=2000 \n",
      "[CV]  max_depth=10, max_features=sqrt, min_samples_leaf=1, min_samples_split=4, n_estimators=2000, total=   2.6s\n",
      "[CV] max_depth=10, max_features=sqrt, min_samples_leaf=1, min_samples_split=4, n_estimators=2000 \n",
      "[CV]  max_depth=10, max_features=sqrt, min_samples_leaf=1, min_samples_split=4, n_estimators=2000, total=   2.6s\n",
      "[CV] max_depth=10, max_features=sqrt, min_samples_leaf=1, min_samples_split=4, n_estimators=2000 \n",
      "[CV]  max_depth=10, max_features=sqrt, min_samples_leaf=1, min_samples_split=4, n_estimators=2000, total=   2.6s\n",
      "[CV] max_depth=10, max_features=sqrt, min_samples_leaf=1, min_samples_split=6, n_estimators=1200 \n",
      "[CV]  max_depth=10, max_features=sqrt, min_samples_leaf=1, min_samples_split=6, n_estimators=1200, total=   1.6s\n",
      "[CV] max_depth=10, max_features=sqrt, min_samples_leaf=1, min_samples_split=6, n_estimators=1200 \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV]  max_depth=10, max_features=sqrt, min_samples_leaf=1, min_samples_split=6, n_estimators=1200, total=   1.5s\n",
      "[CV] max_depth=10, max_features=sqrt, min_samples_leaf=1, min_samples_split=6, n_estimators=1200 \n",
      "[CV]  max_depth=10, max_features=sqrt, min_samples_leaf=1, min_samples_split=6, n_estimators=1200, total=   1.6s\n",
      "[CV] max_depth=10, max_features=sqrt, min_samples_leaf=1, min_samples_split=6, n_estimators=1200 \n",
      "[CV]  max_depth=10, max_features=sqrt, min_samples_leaf=1, min_samples_split=6, n_estimators=1200, total=   1.5s\n",
      "[CV] max_depth=10, max_features=sqrt, min_samples_leaf=1, min_samples_split=6, n_estimators=1200 \n",
      "[CV]  max_depth=10, max_features=sqrt, min_samples_leaf=1, min_samples_split=6, n_estimators=1200, total=   1.6s\n",
      "[CV] max_depth=10, max_features=sqrt, min_samples_leaf=1, min_samples_split=6, n_estimators=1500 \n",
      "[CV]  max_depth=10, max_features=sqrt, min_samples_leaf=1, min_samples_split=6, n_estimators=1500, total=   2.0s\n",
      "[CV] max_depth=10, max_features=sqrt, min_samples_leaf=1, min_samples_split=6, n_estimators=1500 \n",
      "[CV]  max_depth=10, max_features=sqrt, min_samples_leaf=1, min_samples_split=6, n_estimators=1500, total=   1.9s\n",
      "[CV] max_depth=10, max_features=sqrt, min_samples_leaf=1, min_samples_split=6, n_estimators=1500 \n",
      "[CV]  max_depth=10, max_features=sqrt, min_samples_leaf=1, min_samples_split=6, n_estimators=1500, total=   2.0s\n",
      "[CV] max_depth=10, max_features=sqrt, min_samples_leaf=1, min_samples_split=6, n_estimators=1500 \n",
      "[CV]  max_depth=10, max_features=sqrt, min_samples_leaf=1, min_samples_split=6, n_estimators=1500, total=   2.0s\n",
      "[CV] max_depth=10, max_features=sqrt, min_samples_leaf=1, min_samples_split=6, n_estimators=1500 \n",
      "[CV]  max_depth=10, max_features=sqrt, min_samples_leaf=1, min_samples_split=6, n_estimators=1500, total=   1.9s\n",
      "[CV] max_depth=10, max_features=sqrt, min_samples_leaf=1, min_samples_split=6, n_estimators=2000 \n",
      "[CV]  max_depth=10, max_features=sqrt, min_samples_leaf=1, min_samples_split=6, n_estimators=2000, total=   2.6s\n",
      "[CV] max_depth=10, max_features=sqrt, min_samples_leaf=1, min_samples_split=6, n_estimators=2000 \n",
      "[CV]  max_depth=10, max_features=sqrt, min_samples_leaf=1, min_samples_split=6, n_estimators=2000, total=   2.6s\n",
      "[CV] max_depth=10, max_features=sqrt, min_samples_leaf=1, min_samples_split=6, n_estimators=2000 \n",
      "[CV]  max_depth=10, max_features=sqrt, min_samples_leaf=1, min_samples_split=6, n_estimators=2000, total=   2.6s\n",
      "[CV] max_depth=10, max_features=sqrt, min_samples_leaf=1, min_samples_split=6, n_estimators=2000 \n",
      "[CV]  max_depth=10, max_features=sqrt, min_samples_leaf=1, min_samples_split=6, n_estimators=2000, total=   2.6s\n",
      "[CV] max_depth=10, max_features=sqrt, min_samples_leaf=1, min_samples_split=6, n_estimators=2000 \n",
      "[CV]  max_depth=10, max_features=sqrt, min_samples_leaf=1, min_samples_split=6, n_estimators=2000, total=   2.7s\n",
      "[CV] max_depth=10, max_features=sqrt, min_samples_leaf=2, min_samples_split=4, n_estimators=1200 \n",
      "[CV]  max_depth=10, max_features=sqrt, min_samples_leaf=2, min_samples_split=4, n_estimators=1200, total=   1.6s\n",
      "[CV] max_depth=10, max_features=sqrt, min_samples_leaf=2, min_samples_split=4, n_estimators=1200 \n",
      "[CV]  max_depth=10, max_features=sqrt, min_samples_leaf=2, min_samples_split=4, n_estimators=1200, total=   1.5s\n",
      "[CV] max_depth=10, max_features=sqrt, min_samples_leaf=2, min_samples_split=4, n_estimators=1200 \n",
      "[CV]  max_depth=10, max_features=sqrt, min_samples_leaf=2, min_samples_split=4, n_estimators=1200, total=   1.5s\n",
      "[CV] max_depth=10, max_features=sqrt, min_samples_leaf=2, min_samples_split=4, n_estimators=1200 \n",
      "[CV]  max_depth=10, max_features=sqrt, min_samples_leaf=2, min_samples_split=4, n_estimators=1200, total=   1.5s\n",
      "[CV] max_depth=10, max_features=sqrt, min_samples_leaf=2, min_samples_split=4, n_estimators=1200 \n",
      "[CV]  max_depth=10, max_features=sqrt, min_samples_leaf=2, min_samples_split=4, n_estimators=1200, total=   1.5s\n",
      "[CV] max_depth=10, max_features=sqrt, min_samples_leaf=2, min_samples_split=4, n_estimators=1500 \n",
      "[CV]  max_depth=10, max_features=sqrt, min_samples_leaf=2, min_samples_split=4, n_estimators=1500, total=   1.9s\n",
      "[CV] max_depth=10, max_features=sqrt, min_samples_leaf=2, min_samples_split=4, n_estimators=1500 \n",
      "[CV]  max_depth=10, max_features=sqrt, min_samples_leaf=2, min_samples_split=4, n_estimators=1500, total=   2.0s\n",
      "[CV] max_depth=10, max_features=sqrt, min_samples_leaf=2, min_samples_split=4, n_estimators=1500 \n",
      "[CV]  max_depth=10, max_features=sqrt, min_samples_leaf=2, min_samples_split=4, n_estimators=1500, total=   1.9s\n",
      "[CV] max_depth=10, max_features=sqrt, min_samples_leaf=2, min_samples_split=4, n_estimators=1500 \n",
      "[CV]  max_depth=10, max_features=sqrt, min_samples_leaf=2, min_samples_split=4, n_estimators=1500, total=   2.0s\n",
      "[CV] max_depth=10, max_features=sqrt, min_samples_leaf=2, min_samples_split=4, n_estimators=1500 \n",
      "[CV]  max_depth=10, max_features=sqrt, min_samples_leaf=2, min_samples_split=4, n_estimators=1500, total=   1.9s\n",
      "[CV] max_depth=10, max_features=sqrt, min_samples_leaf=2, min_samples_split=4, n_estimators=2000 \n",
      "[CV]  max_depth=10, max_features=sqrt, min_samples_leaf=2, min_samples_split=4, n_estimators=2000, total=   2.6s\n",
      "[CV] max_depth=10, max_features=sqrt, min_samples_leaf=2, min_samples_split=4, n_estimators=2000 \n",
      "[CV]  max_depth=10, max_features=sqrt, min_samples_leaf=2, min_samples_split=4, n_estimators=2000, total=   2.6s\n",
      "[CV] max_depth=10, max_features=sqrt, min_samples_leaf=2, min_samples_split=4, n_estimators=2000 \n",
      "[CV]  max_depth=10, max_features=sqrt, min_samples_leaf=2, min_samples_split=4, n_estimators=2000, total=   2.6s\n",
      "[CV] max_depth=10, max_features=sqrt, min_samples_leaf=2, min_samples_split=4, n_estimators=2000 \n",
      "[CV]  max_depth=10, max_features=sqrt, min_samples_leaf=2, min_samples_split=4, n_estimators=2000, total=   2.6s\n",
      "[CV] max_depth=10, max_features=sqrt, min_samples_leaf=2, min_samples_split=4, n_estimators=2000 \n",
      "[CV]  max_depth=10, max_features=sqrt, min_samples_leaf=2, min_samples_split=4, n_estimators=2000, total=   2.6s\n",
      "[CV] max_depth=10, max_features=sqrt, min_samples_leaf=2, min_samples_split=6, n_estimators=1200 \n",
      "[CV]  max_depth=10, max_features=sqrt, min_samples_leaf=2, min_samples_split=6, n_estimators=1200, total=   1.6s\n",
      "[CV] max_depth=10, max_features=sqrt, min_samples_leaf=2, min_samples_split=6, n_estimators=1200 \n",
      "[CV]  max_depth=10, max_features=sqrt, min_samples_leaf=2, min_samples_split=6, n_estimators=1200, total=   1.5s\n",
      "[CV] max_depth=10, max_features=sqrt, min_samples_leaf=2, min_samples_split=6, n_estimators=1200 \n",
      "[CV]  max_depth=10, max_features=sqrt, min_samples_leaf=2, min_samples_split=6, n_estimators=1200, total=   1.5s\n",
      "[CV] max_depth=10, max_features=sqrt, min_samples_leaf=2, min_samples_split=6, n_estimators=1200 \n",
      "[CV]  max_depth=10, max_features=sqrt, min_samples_leaf=2, min_samples_split=6, n_estimators=1200, total=   1.6s\n",
      "[CV] max_depth=10, max_features=sqrt, min_samples_leaf=2, min_samples_split=6, n_estimators=1200 \n",
      "[CV]  max_depth=10, max_features=sqrt, min_samples_leaf=2, min_samples_split=6, n_estimators=1200, total=   1.6s\n",
      "[CV] max_depth=10, max_features=sqrt, min_samples_leaf=2, min_samples_split=6, n_estimators=1500 \n",
      "[CV]  max_depth=10, max_features=sqrt, min_samples_leaf=2, min_samples_split=6, n_estimators=1500, total=   1.9s\n",
      "[CV] max_depth=10, max_features=sqrt, min_samples_leaf=2, min_samples_split=6, n_estimators=1500 \n",
      "[CV]  max_depth=10, max_features=sqrt, min_samples_leaf=2, min_samples_split=6, n_estimators=1500, total=   1.9s\n",
      "[CV] max_depth=10, max_features=sqrt, min_samples_leaf=2, min_samples_split=6, n_estimators=1500 \n",
      "[CV]  max_depth=10, max_features=sqrt, min_samples_leaf=2, min_samples_split=6, n_estimators=1500, total=   1.9s\n",
      "[CV] max_depth=10, max_features=sqrt, min_samples_leaf=2, min_samples_split=6, n_estimators=1500 \n",
      "[CV]  max_depth=10, max_features=sqrt, min_samples_leaf=2, min_samples_split=6, n_estimators=1500, total=   2.0s\n",
      "[CV] max_depth=10, max_features=sqrt, min_samples_leaf=2, min_samples_split=6, n_estimators=1500 \n",
      "[CV]  max_depth=10, max_features=sqrt, min_samples_leaf=2, min_samples_split=6, n_estimators=1500, total=   2.0s\n",
      "[CV] max_depth=10, max_features=sqrt, min_samples_leaf=2, min_samples_split=6, n_estimators=2000 \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV]  max_depth=10, max_features=sqrt, min_samples_leaf=2, min_samples_split=6, n_estimators=2000, total=   2.7s\n",
      "[CV] max_depth=10, max_features=sqrt, min_samples_leaf=2, min_samples_split=6, n_estimators=2000 \n",
      "[CV]  max_depth=10, max_features=sqrt, min_samples_leaf=2, min_samples_split=6, n_estimators=2000, total=   2.5s\n",
      "[CV] max_depth=10, max_features=sqrt, min_samples_leaf=2, min_samples_split=6, n_estimators=2000 \n",
      "[CV]  max_depth=10, max_features=sqrt, min_samples_leaf=2, min_samples_split=6, n_estimators=2000, total=   2.6s\n",
      "[CV] max_depth=10, max_features=sqrt, min_samples_leaf=2, min_samples_split=6, n_estimators=2000 \n",
      "[CV]  max_depth=10, max_features=sqrt, min_samples_leaf=2, min_samples_split=6, n_estimators=2000, total=   2.6s\n",
      "[CV] max_depth=10, max_features=sqrt, min_samples_leaf=2, min_samples_split=6, n_estimators=2000 \n",
      "[CV]  max_depth=10, max_features=sqrt, min_samples_leaf=2, min_samples_split=6, n_estimators=2000, total=   2.6s\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done 360 out of 360 | elapsed: 12.4min finished\n"
     ]
    }
   ],
   "source": [
    "from sklearn.model_selection import GridSearchCV, train_test_split\n",
    "\n",
    "np.random.seed(42)\n",
    "\n",
    "# Split into X & y\n",
    "X = heart_disease.drop(\"target\", axis=1)\n",
    "y = heart_disease[\"target\"]\n",
    "\n",
    "# Split into train and test sets\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2)\n",
    "\n",
    "# Set n_jobs to -1 to use all cores (NOTE: n_jobs=-1 is broken as of 8 Dec 2019, using n_jobs=1 works)\n",
    "clf = RandomForestClassifier(n_jobs=1)\n",
    "\n",
    "# Setup GridSearchCV\n",
    "gs_clf = GridSearchCV(estimator=clf,\n",
    "                      param_grid=grid_2,\n",
    "                      cv=2, # 2-fold cross-validation\n",
    "                      verbose=2) # print out progress\n",
    "\n",
    "# Fit the RandomizedSearchCV version of clf\n",
    "gs_clf.fit(X_train, y_train);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 104,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'max_depth': 5,\n",
       " 'max_features': 'sqrt',\n",
       " 'min_samples_leaf': 2,\n",
       " 'min_samples_split': 6,\n",
       " 'n_estimators': 1200}"
      ]
     },
     "execution_count": 104,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Check the best hyperparameters found with GridSearchCV\n",
    "gs_clf.best_params_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Acc: 83.61%\n",
      "Precision: 0.78\n",
      "Recall: 0.89\n",
      "F1 score: 0.83\n"
     ]
    }
   ],
   "source": [
    "# Max predictions with the GridSearchCV classifier\n",
    "gs_y_preds = gs_clf.predict(X_test)\n",
    "\n",
    "# Evaluate the predictions\n",
    "gs_metrics = evaluate_preds(y_test, gs_y_preds)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 106,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAlMAAAH1CAYAAADS7HuGAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8vihELAAAACXBIWXMAAAsTAAALEwEAmpwYAAAnJ0lEQVR4nO3deZBV5bmo8ecFEYwgDug5lqCghQpCK5M4oBi5ETUJHo0naoziiENQclJaMZUbIVETp3i8elU0Kac4RI8a4xiJI4ETTmiEi2FSoqgtxnkIk9r43T+66bRNCw3fhrVonl8VZe+111797nbbPqy19tqRUkKSJElrp03RA0iSJG3IjClJkqQMxpQkSVIGY0qSJCmDMSVJkpTBmJIkScqwSVHfuEuXLql79+5FfXtJkqQWmzZt2rsppW2bu6+wmOrevTvV1dVFfXtJkqQWi4hXv+w+D/NJkiRlMKYkSZIyGFOSJEkZCjtnSlJxPvvsM2pqali2bFnRo2gNdOjQga5du9KuXbuiR5HUiDElbYRqamro1KkT3bt3JyKKHkctkFLivffeo6amhh49ehQ9jqRGPMwnbYSWLVvGNttsY0htQCKCbbbZxr2JUgkZU9JGypDa8PjvTConY0pSIRYsWECfPn3WybafffZZvvGNbwDw0EMPcemll66T7yNJ4DlTkoDuFzxa0e0tuPTrFd1ejhEjRjBixIiix5DUirlnSlJhamtrGTlyJFVVVRx99NEsWbKEn/3sZwwaNIg+ffowatQoUkoAXHPNNfTu3ZuqqiqOPfZYABYvXswpp5zCoEGD6NevH7///e9X+h633noro0ePBuCkk07i3HPPZb/99mPnnXfmvvvua1jviiuuYNCgQVRVVTF27Nj18OwltRbGlKTCzJs3j1GjRjFz5ky22GILrr/+ekaPHs3UqVP561//ytKlS3nkkUcAuPTSS5k+fTozZ85k/PjxAFxyySUcfPDBTJ06lWeeeYbzzz+fxYsXr/J7vvnmm0yaNIlHHnmECy64AIAJEybw0ksv8Ze//IUZM2Ywbdo0Jk6cuG6fvKRWw5iSVJhu3bqx//77A/Dd736XSZMm8cwzzzB48GD69u3L008/zaxZswCoqqri+OOP54477mCTTerOUJgwYQKXXnope+21FwcddBDLli3jtddeW+X3/Ld/+zfatGlD7969eeuttxq2M2HCBPr160f//v2ZO3cuL7300jp85pJaE8+ZklSYpu9OiwjOPvtsqqur6datG+PGjWu4FMCjjz7KxIkTeeihh7jooouYNWsWKSXuv/9+dtttty9sZ0UkNad9+/YNX684hJhS4kc/+hFnnHFGpZ6apI2Ie6YkFea1117jz3/+MwB33303Q4YMAaBLly4sWrSo4Zymzz//nNdff52vfvWrXH755Xz44YcsWrSI4cOHc+211zZE0fTp09dqjuHDh3PzzTezaNEiAN544w3efvvt3KcnaSPhnilJhenVqxe33XYbZ5xxBj179uSss87igw8+oG/fvnTv3p1BgwYBsHz5cr773e/y0UcfkVLiP/7jP9hyyy35yU9+wve//32qqqpIKdG9e/eGc6zWxCGHHMKcOXPYd999AejYsSN33HEH2223XUWfr6TWKVb8jW59GzhwYKquri7ke0sbuzlz5tCrV6+ix9Ba8N+dVIyImJZSGtjcfR7mkyRJymBMSZIkZTCmJEmSMngCuiRtBPre1jd7G/f+orYCk0CvuXMqsh2pLNwzJUmSlMGYkiRJymBMSZIkZfCcKUkwrnOFt/fRmj9k3Dg6duzIeeedx9y5czn22GOJCO677z522WWXL6z7+uuvc+KJJ/L3v/+dNm3aMGrUKMaMGVOp6StmZs2HFdlOVZtX/nnjw7dh3D5rvpEeO1ZkFkkrc8+UpNJ58MEHOeKII5g+ffpKIQWwySab8Mtf/pI5c+YwZcoUrrvuOmbPnl3ApJLknilJBbn99tu58soriQiqqqoaoumxxx7j6quvpm3btkycOJFnnnlmpcduv/32bL/99gB06tSJXr168cYbb9C7d+/1+hwkCYwpSQWYNWsWl1xyCZMnT6ZLly68//77XHPNNQAcfvjhnHnmmQ2H/FZnwYIFTJ8+ncGDB6/rsSWpWR7mk7TePf300xx99NF06dIFgK233nqttrNo0SK+9a1vcfXVV7PFFltUckRJajFjStJ6l1IiIrK28dlnn/Gtb32L448/nqOOOqpCk0nSmjOmJK13w4YN49577+W9994D4P3331+jx6eUOPXUU+nVqxc/+MEP1sWIktRinjMlaa0uZZBjjz324Mc//jFDhw6lbdu29OvXj+7du7f48ZMnT+Y3v/kNffv2Za+99gLg5z//OYcffvi6GViSVsGYklSIkSNHMnLkyGbvGzdu3CofO2TIEFJK62AqSVpzHuaTJEnK4J4pSaX13nvvMWzYsJWWP/XUU2yzzTYFTCRJKzOmJJXWNttsw4wZM4oeQ5JWycN8kiRJGYwpSZKkDB7mkySpBLpf8Gj2NhZ0+E7+IOv5UimtgXumJLUK3bt359133y16jLV2w1WXctv4a4seQ9JacM+UJPre1rei23th5AstXjelREqJNm02jr/bLV++nLZt2xY9hqQK2jh+e0kqlQULFtCrVy/OPvts+vfvz+uvv85ZZ53FwIED2WOPPRg7dmzDut27d2fs2LH079+fvn37MnfuXKDusgmHHHII/fr144wzzvjCRTyvuuoq+vTpQ58+fbj66qsbvufuu+/OaaedRp8+fTj++ON58skn2X///enZsyd/+ctfVppz1qxZ7L333uy1115UVVXx0ksvAXDHHXc0LD/jjDNYvnw5wJc+h8P2rWL81Zcz8qhDmfDIg0x+5kmOOWwo/37IEE4/9oiG9f720jxO/fdvcPj+e3HnzTdW7gcuaZ0ypiQVYt68eZx44olMnz6dnXbaiUsuuYTq6mpmzpzJc889x8yZMxvW7dKlC88//zxnnXUWV155JQA//elPGTJkCNOnT2fEiBG89tprAEybNo1bbrmF//mf/2HKlCn86le/Yvr06QDMnz+fMWPGMHPmTObOnctdd93FpEmTuPLKK/n5z3++0ozjx49nzJgxzJgxg+rqarp27cqcOXO45557mDx5MjNmzKBt27bceeedACs9hxfn/LVhW+3bt+e2B/7A4CFD+ekPx/DLG2/nvyZM4sobbm1YZ8HfXuSGO+7nzoef4sb/vIzPPvus4j93SZXnYb6SqcThlnt/UVuBSaDX3DkV2Y7UnJ122ol99tmn4fa9997LTTfdRG1tLW+++SazZ8+mqqoKgKOOOgqAAQMG8MADDwAwceLEhq+//vWvs9VWWwEwadIkjjzySDbffPOGx/7pT39ixIgR9OjRg7596/4b22OPPRg2bBgRQd++fVmwYMFKM+67775ccskl1NTUcNRRR9GzZ0+eeuoppk2bxqBBgwBYunQp2223XbPP4W8vzmPXXn0AGP7NIwGY+fxUBgzej6477gRA5/q5AQ44+BA2bd+eTdu3Z+su2/L+u2/zL9vvkPujlrSOGVOSCrEidgBeeeUVrrzySqZOncpWW23FSSedxLJlyxrub9++PQBt27altvaff1mIiJW2u6rP7FuxHYA2bdo03G7Tps0XtrvCd77zHQYPHsyjjz7K8OHD+fWvf01KiZEjR/KLX/ziC+s29xw+/eSThvs3+8rmKwaEZuYG2HTTL85XW7v8S5+LpPLwMJ+kwn388cdsvvnmdO7cmbfeeovHH398tY858MADGw6vPf7443zwwQcNyx988EGWLFnC4sWL+d3vfscBBxywVnO9/PLL7Lzzzpx77rmMGDGCmTNnMmzYMO677z7efvttAN5//31effXVFj+HqgF7M23KZGpeexWAj+rnlrThcs+UpMLtueee9OvXjz322IOdd96Z/ffff7WPGTt2LMcddxz9+/dn6NCh7LjjjgD079+fk046ib333huA0047jX79+jV7GG917rnnHu644w7atWvHv/7rv3LhhRey9dZbc/HFF3PIIYfw+eef065dO6677jr22WefFj2HrbfpwoWXXc0PRp1A+vxztu6yLTfe9bs1nk1SecSqdomvSwMHDkzV1dWFfO8y85wprQ9z5syhV69eRY/R6s2s+bAi26lq80rD13NefZteT3x7jbfRt8eO2XP4u2XdKstFOyvxWoHKvF7K9FqJiGkppYHN3edhPkmSpAzGlCRJUgZjSpIkKYMxJUmSlMF381GZk/6gQp/WXaET/yRJ0vrhnilJkqQMxpSkUhs/fjy33377SssXLFhAnz59Cpjon2699VZGjx5d6AySiudhPknM2b2y15yq1LVhamtrOfPMMyuyrdw5NtnEX5eSmudvB0mFuOiii7jzzjvp1q0bXbp0YcCAAZx33nkcdNBB7LfffkyePJkRI0bwj3/8g44dO3Leeecxbdo0TjnlFL7yla8wZMiQZrf75ptvcswxx/Dxxx9TW1vLDTfcwAEHHMCECRMYO3Ysn3zyCbvssgu33HILHTt25Gc/+xkPP/wwS5cuZb/99uPGG28kIlaa48ADD2TMmDEsXryY9u3b89RTTwGwcOFCDj30UP72t79x5JFHcvnll6/PH6OkEjCmpDVUlqsUM+6j/G0UpLq6mvvvv5/p06dTW1tL//79GTBgQMP9H374Ic899xwA48aNa1h+8sknc+211zJ06FDOP//8Zrd91113MXz4cH784x+zfPlylixZwrvvvsvFF1/Mk08+yeabb85ll13GVVddxYUXXsjo0aO58MILATjhhBN45JFH+OY3v/mFOT799FN233137rnnHgYNGsTHH3/MZpttBsCMGTOYPn067du3Z7fdduOcc86hW7du6+LHJqmkWnTOVEQcGhHzImJ+RFzQzP2dI+LhiPh/ETErIk6u/KiSWotJkyZxxBFHsNlmm9GpU6eGeFnhmGOOWekxH330ER9++CFDhw4F6sKnOYMGDeKWW25h3LhxvPDCC3Tq1IkpU6Ywe/Zs9t9/f/baay9uu+02Xn217oOGn3nmGQYPHkzfvn15+umnmTVr1kpzzJs3j+23355BgwYBsMUWWzQc9hs2bBidO3emQ4cO9O7du2G7kjYeq90zFRFtgeuArwE1wNSIeCilNLvRat8DZqeUvhkR2wLzIuLOlNKn62RqSRu01X0m6Oabb97sYyJitds+8MADmThxIo8++ignnHAC559/PltttRVf+9rXuPvuu7+w7rJlyzj77LOprq6mW7dujBs3jmXLlq00x6q+d/v27Ru+btu2LbW1lfn8OkkbjpbsmdobmJ9Serk+jn4LHNFknQR0irrfNh2B9wF/o0hq1pAhQ3j44YdZtmwZixYt4tFHV3/odMstt6Rz585MmjQJgDvvvLPZ9V599VW22247Tj/9dE499VSef/559tlnHyZPnsz8+fMBWLJkCS+++GJDOHXp0oVFixZx3333NbvN3XffnYULFzJ16lQA/vGPfxhNkhq05JypHYDXG92uAQY3Wef/Ag8BC4FOwDEppc+bbigiRgGjAHbc0YtTShurQYMGMWLECPbcc0922mknBg4cSOfOnVf7uFtuuaXhBPThw4c3u86zzz7LFVdcQbt27ejYsSO333472267LbfeeivHHXccn3zyCQAXX3wxu+66K6effjp9+/ale/fuDYfxmtp000255557OOecc1i6dCmbbbYZTz755Nr/ACS1KrG63e0R8e/A8JTSafW3TwD2Timd02ido4H9gR8AuwB/BPZMKX38ZdsdOHBgqq6uzn8GFVCmK6D3rcAV0O/9RWX+xlypt7e3Nq3hBPQ5c+bQq1dlL4ewphYtWkTHjh1ZsmQJBx54IDfddBP9+/cvdKZKm1nzYUW2U9XmlYav57z6Nr2e+PYab8PfLeVXlt8tlXitQGVeL2V6rUTEtJTSwObua8meqRqg8VtTulK3B6qxk4FLU12ZzY+IV4Ddgb+sxbySNgKjRo1i9uzZLFu2jJEjR7a6kJK08WhJTE0FekZED+AN4Figafq+BgwD/hQR/wLsBrxcyUEltS533XVX0SNIUkWsNqZSSrURMRp4AmgL3JxSmhURZ9bfPx64CLg1Il4AAvhhSunddTi3JElSKbToop0ppceAx5osG9/o64XAIZUdTdK61NJLDag86s6kWPV5rpLWPz/oWNoIdejQgffee2+113tSeaSUeG9xLR0+8gwKqWz8OBlpI9S1a1dqamp45513ih6lVXvrg6UV2c6ceAdIdPjoZbo+f1lFtimpcowpaSPUrl07evToUfQYrd5hJbrsiqR1x8N8kiRJGdwzJW2g+t7WtyLbaW0X1pOk9c09U5IkSRmMKUmSpAzGlCRJUgZjSpIkKYMxJUmSlMGYkiRJymBMSZIkZTCmJEmSMhhTkiRJGYwpSZKkDMaUJElSBmNKkiQpgzElSZKUwZiSJEnKYExJkiRlMKYkSZIyGFOSJEkZjClJkqQMxpQkSVIGY0qSJCmDMSVJkpTBmJIkScpgTEmSJGUwpiRJkjIYU5IkSRmMKUmSpAzGlCRJUgZjSpIkKYMxJUmSlMGYkiRJymBMSZIkZTCmJEmSMhhTkiRJGYwpSZKkDMaUJElSBmNKkiQpgzElSZKUwZiSJEnKYExJkiRlMKYkSZIyGFOSJEkZjClJkqQMxpQkSVIGY0qSJCmDMSVJkpTBmJIkScpgTEmSJGUwpiRJkjIYU5IkSRmMKUmSpAzGlCRJUgZjSpIkKYMxJUmSlMGYkiRJymBMSZIkZTCmJEmSMhhTkiRJGYwpSZKkDMaUJElSBmNKkiQpgzElSZKUwZiSJEnKYExJkiRlMKYkSZIyGFOSJEkZjClJkqQMxpQkSVIGY0qSJCmDMSVJkpTBmJIkScpgTEmSJGUwpiRJkjIYU5IkSRmMKUmSpAzGlCRJUgZjSpIkKYMxJUmSlMGYkiRJymBMSZIkZWhRTEXEoRExLyLmR8QFX7LOQRExIyJmRcRzlR1TkiSpnDZZ3QoR0Ra4DvgaUANMjYiHUkqzG62zJXA9cGhK6bWI2G4dzStJklQqLdkztTcwP6X0ckrpU+C3wBFN1vkO8EBK6TWAlNLblR1TkiSpnFoSUzsArze6XVO/rLFdga0i4tmImBYRJza3oYgYFRHVEVH9zjvvrN3EkiRJJdKSmIpmlqUmtzcBBgBfB4YDP4mIXVd6UEo3pZQGppQGbrvttms8rCRJUtms9pwp6vZEdWt0uyuwsJl13k0pLQYWR8REYE/gxYpMKUmSVFIt2TM1FegZET0iYlPgWOChJuv8HjggIjaJiK8Ag4E5lR1VkiSpfFa7ZyqlVBsRo4EngLbAzSmlWRFxZv3941NKcyLiD8BM4HPg1ymlv67LwSVJksqgJYf5SCk9BjzWZNn4JrevAK6o3GiSJEnl5xXQJUmSMhhTkiRJGYwpSZKkDMaUJElSBmNKkiQpgzElSZKUwZiSJEnKYExJkiRlMKYkSZIyGFOSJEkZjClJkqQMxpQkSVIGY0qSJCmDMSVJkpTBmJIkScpgTEmSJGUwpiRJkjIYU5IkSRmMKUmSpAzGlCRJUgZjSpIkKYMxJUmSlMGYkiRJymBMSZIkZTCmJEmSMhhTkiRJGYwpSZKkDMaUJElSBmNKkiQpgzElSZKUwZiSJEnKYExJkiRlMKYkSZIyGFOSJEkZjClJkqQMxpQkSVIGY0qSJCmDMSVJkpTBmJIkScpgTEmSJGUwpiRJkjIYU5IkSRmMKUmSpAzGlCRJUgZjSpIkKYMxJUmSlMGYkiRJymBMSZIkZTCmJEmSMhhTkiRJGYwpSZKkDMaUJElSBmNKkiQpgzElSZKUwZiSJEnKYExJkiRlMKYkSZIyGFOSJEkZjClJkqQMxpQkSVIGY0qSJCmDMSVJkpTBmJIkScpgTEmSJGUwpiRJkjIYU5IkSRmMKUmSpAzGlCRJUgZjSpIkKYMxJUmSlMGYkiRJymBMSZIkZTCmJEmSMhhTkiRJGYwpSZKkDMaUJElSBmNKkiQpgzElSZKUwZiSJEnKYExJkiRlMKYkSZIyGFOSJEkZjClJkqQMxpQkSVKGFsVURBwaEfMiYn5EXLCK9QZFxPKIOLpyI0qSJJXXamMqItoC1wGHAb2B4yKi95esdxnwRKWHlCRJKquW7JnaG5ifUno5pfQp8FvgiGbWOwe4H3i7gvNJkiSVWktiagfg9Ua3a+qXNYiIHYAjgfGVG02SJKn8WhJT0cyy1OT21cAPU0rLV7mhiFERUR0R1e+8804LR5QkSSqvTVqwTg3QrdHtrsDCJusMBH4bEQBdgMMjojal9GDjlVJKNwE3AQwcOLBpkEmSJG1wWhJTU4GeEdEDeAM4FvhO4xVSSj1WfB0RtwKPNA0pSZKk1mi1MZVSqo2I0dS9S68tcHNKaVZEnFl/v+dJSZKkjVZL9kyRUnoMeKzJsmYjKqV0Uv5YkiRJGwavgC5JkpTBmJIkScpgTEmSJGUwpiRJkjIYU5IkSRmMKUmSpAzGlCRJUgZjSpIkKYMxJUmSlMGYkiRJymBMSZIkZTCmJEmSMhhTkiRJGYwpSZKkDMaUJElSBmNKkiQpgzElSZKUwZiSJEnKYExJkiRlMKYkSZIyGFOSJEkZjClJkqQMxpQkSVIGY0qSJCmDMSVJkpTBmJIkScpgTEmSJGUwpiRJkjIYU5IkSRmMKUmSpAzGlCRJUgZjSpIkKYMxJUmSlMGYkiRJymBMSZIkZTCmJEmSMhhTkiRJGYwpSZKkDMaUJElSBmNKkiQpgzElSZKUwZiSJEnKYExJkiRlMKYkSZIyGFOSJEkZjClJkqQMxpQkSVIGY0qSJCmDMSVJkpTBmJIkScpgTEmSJGUwpiRJkjIYU5IkSRmMKUmSpAzGlCRJUgZjSpIkKYMxJUmSlMGYkiRJymBMSZIkZTCmJEmSMhhTkiRJGYwpSZKkDMaUJElSBmNKkiQpgzElSZKUwZiSJEnKYExJkiRlMKYkSZIyGFOSJEkZjClJkqQMxpQkSVIGY0qSJCmDMSVJkpTBmJIkScpgTEmSJGUwpiRJkjIYU5IkSRmMKUmSpAzGlCRJUgZjSpIkKYMxJUmSlMGYkiRJymBMSZIkZTCmJEmSMrQopiLi0IiYFxHzI+KCZu4/PiJm1v/574jYs/KjSpIklc9qYyoi2gLXAYcBvYHjIqJ3k9VeAYamlKqAi4CbKj2oJElSGbVkz9TewPyU0ssppU+B3wJHNF4hpfTfKaUP6m9OAbpWdkxJkqRyaklM7QC83uh2Tf2yL3Mq8HjOUJIkSRuKTVqwTjSzLDW7YsRXqYupIV9y/yhgFMCOO+7YwhElSZLKqyV7pmqAbo1udwUWNl0pIqqAXwNHpJTea25DKaWbUkoDU0oDt91227WZV5IkqVRaElNTgZ4R0SMiNgWOBR5qvEJE7Ag8AJyQUnqx8mNKkiSV02oP86WUaiNiNPAE0Ba4OaU0KyLOrL9/PHAhsA1wfUQA1KaUBq67sSVJksqhJedMkVJ6DHisybLxjb4+DTitsqNJkiSVn1dAlyRJymBMSZIkZTCmJEmSMhhTkiRJGYwpSZKkDMaUJElSBmNKkiQpgzElSZKUwZiSJEnKYExJkiRlMKYkSZIyGFOSJEkZjClJkqQMxpQkSVIGY0qSJCmDMSVJkpTBmJIkScpgTEmSJGUwpiRJkjIYU5IkSRmMKUmSpAzGlCRJUgZjSpIkKYMxJUmSlMGYkiRJymBMSZIkZTCmJEmSMhhTkiRJGYwpSZKkDMaUJElSBmNKkiQpgzElSZKUwZiSJEnKYExJkiRlMKYkSZIyGFOSJEkZjClJkqQMxpQkSVIGY0qSJCmDMSVJkpTBmJIkScpgTEmSJGUwpiRJkjIYU5IkSRmMKUmSpAzGlCRJUgZjSpIkKYMxJUmSlMGYkiRJymBMSZIkZTCmJEmSMhhTkiRJGYwpSZKkDMaUJElSBmNKkiQpgzElSZKUwZiSJEnKYExJkiRlMKYkSZIyGFOSJEkZjClJkqQMxpQkSVIGY0qSJCmDMSVJkpTBmJIkScpgTEmSJGUwpiRJkjIYU5IkSRmMKUmSpAzGlCRJUgZjSpIkKYMxJUmSlMGYkiRJymBMSZIkZTCmJEmSMhhTkiRJGYwpSZKkDMaUJElSBmNKkiQpgzElSZKUwZiSJEnKYExJkiRlMKYkSZIyGFOSJEkZWhRTEXFoRMyLiPkRcUEz90dEXFN//8yI6F/5USVJkspntTEVEW2B64DDgN7AcRHRu8lqhwE96/+MAm6o8JySJEml1JI9U3sD81NKL6eUPgV+CxzRZJ0jgNtTnSnAlhGxfYVnlSRJKp2WxNQOwOuNbtfUL1vTdSRJklqdTVqwTjSzLK3FOkTEKOoOAwIsioh5Lfj+G4zmfghr7q9dgHdzttD0GOxai8o8I62sLK8VqNDrxdfKOlWW14u/W8qvLK8VaJW/W3b6sjtaElM1QLdGt7sCC9diHVJKNwE3teB7brQiojqlNLDoOVR+vla0Jny9qKV8ray5lhzmmwr0jIgeEbEpcCzwUJN1HgJOrH9X3z7ARymlNys8qyRJUumsds9USqk2IkYDTwBtgZtTSrMi4sz6+8cDjwGHA/OBJcDJ625kSZKk8mjJYT5SSo9RF0yNl41v9HUCvlfZ0TZaHgZVS/la0Zrw9aKW8rWyhqKugyRJkrQ2/DgZSZKkDMaUJElSBmNKkiQpQ4tOQNe6FRH3AzcDj6eUPi96HpVbROxA3cXjGv77TSlNLG4ilU1E/GBV96eUrlpfs2jDFREdU0qLip5jQ2BMlcMN1F1O4pqI+C/g1pTS3IJnUglFxGXAMcBsYHn94gQYU2qsU9EDqFWYDexY9BAbAt/NVyIR0Rk4DvgxdZ91+CvgjpTSZ4UOptKo/wimqpTSJ0XPImnDt4q9mAH8OKW09fqcZ0PlnqmSiIhtgO8CJwDTgTuBIcBI4KDiJlPJvAy0A4wpfamIuGZV96eUzl1fs6j0fg5cAdQ2c5/nVbeQMVUCEfEAsDvwG+CbjT6K556IqC5uMpXQEmBGRDxFo6Dyf45qYlrRA2iD8TzwYEpppddMRJxWwDwbJA/zlUBEHJxSerroOVR+ETGyueUppdvW9yySNnz15+meBRyfUvo/Te77l5TSW8VMtmFxz1Q59IqI51NKHwJExFbAcSml64sdS2WTUrqt/gPHd61fNM9z6vRlImJb4IdAb6DDiuUppYMLG0pl0xvYHDglIm6n7lypFfzd0kIeDy2H01eEFEBK6QPg9OLGUVlFxEHAS8B1wPXAixFxYJEzqdTuBOYAPYCfAguAqUUOpNK5EfgDdaeaTGvyx9NMWsjDfCUQETOBPes/MJqIaAvMTCntUexkKpuImAZ8J6U0r/72rsDdKaUBxU6mMoqIaSmlARExM6VUVb/suZTS0KJnU7lExA0ppbOKnmND5WG+cngCuDcixlN3zaAzqfubgtRUuxUhBZBSejEi2hU5kEptxWGaNyPi68BCoGuB86ikDKk87pkqgYhoA5wBDKPuePUE4NcppeWrfKA2OhFxM3XB/Zv6RccDm6SUTi5uKpVVRHwD+BPQDbgW2AL4aUrpoUIHk1oZY0ragEREe+B71F2DLKi78vn1XsRTkopjTJVARPQEfsHK77jZubChJG3wIuI2YEyTdwr/MqV0SqGDSa2M50yVwy3AWOA/ga9S9zl9scpHaKMSEfemlL4dES9Qd5jvC1acXCw1UdX0ncIR0a/AeaRWyZgqh81SSk9FRKSUXgXGRcSfqAssCWBM/T+/UegU2tC0iYit6i+3QkRsjb/3pYrzP6pyWFZ/EvpLETEaeAPYruCZVCKNPmLoXWBpSunz+ssi7A48XtxkKrlfAv8dEfdRt0fz28AlxY4ktT6eM1UCETGIugvrbQlcRN07bq5IKU0pci6VT/11pg4AtgKmUHdRvSUppeMLHUylFRG9gYOpO3XgqZTS7IJHklodr4BesPoLdH47pbQopVSTUjo5pfQtQ0pfIlJKS4CjgGtTSkdS98YF6ctsDSxOKV0LvBMRPYoeSGptjKmC1V9LakBEeMK5WiIiYl/qri/1aP0yD9erWRExlrrP5vtR/aJ2wB3FTSS1Tv4SLofpwO/rP7178YqFKaUHihtJJfV96v7H+LuU0qyI2Bl4ptiRVGJHAv2A5wFSSgsjolOxI0mtjzFVDlsD71F3XsMKCTCm9AUppeeA5xrdfhk4t7iJVHKfppRSRKz43M/Nix5Iao2MqRLwo0C0OhFxdUrp+xHxMM1fZ2pEAWOpxOpPHXgkIm4EtoyI04FTgF8VO5nU+vhuvhKIiFto/n+QXqVYAETEgJTStIgY2tz99XuspC+IiOepO2fqEOrezfdESumPxU4ltT7umSqHRxp93YG68xwWFjSLSiilNK3+y2rqrzMFDe8GbV/YYCq7PwMfppTOL3oQqTVzz1QJ1V/A88mU0sGrXVkblYiYAvyvlNKi+tsdgQkppf2KnUxlFBGzgV2BV/nim1v8+CGpgtwzVU49gR2LHkKl1GFFSAGklBZFxFeKHEildljRA0gbA2OqBCLiH3zxnKm/U3eeg9TU4ojon1J6HurOpQKWFjyTSqr+sz4lrWMe5pM2IPUfPfRb/nlO3fbAMY3OqZIkrWfGVAlExJHA0ymlj+pvbwkclFJ6sMi5VE4R0Q7Yjbp3Z81NKX1W8EiStFEzpkogImaklPZqsmx6SqlfQSOppOrPj/oBsFNK6fSI6AnsllJ6ZDUPlSStI342Xzk09+/B89nUnFuAT4F962/XABcXN44kyZgqh+qIuCoidomInSPiPwHPgVFzdkkpXQ58BpBSWkrd4T5JUkGMqXI4h7q9DfcA91L37qzvFTqRyurTiNiM+nd/RsQuwCfFjiRJGzfPmZI2IBHxNeB/A72BCcD+wEkppWeLnEuSNmbumSqBiPhj/Tv4VtzeKiKeKHAklVD9lfG3Ao4CTgLuBgYaUpJULPdMlUBz79zz3XxqTkRMTCkdWPQckqR/cs9UOXweEQ0fHxMR3fniFdGlFf4YEedFRLeI2HrFn6KHkqSNmXumSiAiDgVuAp6rX3QgMCql5KE+fUFEvEIzoZ1S2rmAcSRJGFOlERHbAaOAGUAH4O2U0sRCh1Lp1L+T72xgCHVR9SdgfP0lEiRJBTCmSiAiTgPGAF2pi6l9gD+nlA4uci6VT0TcC3wM3Fm/6Dhgy5TSt4ubSpI2bl5luxzGAIOAKSmlr0bE7sBPC55J5bRbSmnPRrefiYj/V9g0kiRPQC+JZSmlZQAR0T6lNJe6D7KVmpoeEfusuBERg4HJBc4jSRs990yVQ039daYepO7dWh8ACwudSGU1GDgxIl6rv70jMCciXgBSSqmquNEkaePkOVMlExFDgc7AH1JKnxY9j8olInZa1f0ppVfX1yySpDrGlCRJUgbPmZIkScpgTEmSJGUwpiRJkjIYU5IkSRmMKUmSpAz/H+pPHdG6VZmBAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 720x576 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "compare_metrics = pd.DataFrame({\"baseline\": baseline_metrics,\n",
    "                                \"clf_2\": clf_2_metrics,\n",
    "                                \"random search\": rs_metrics,\n",
    "                                \"grid search\": gs_metrics})\n",
    "compare_metrics.plot.bar(figsize=(10, 8));"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Saving and loading trained machine learning models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 107,
   "metadata": {},
   "outputs": [],
   "source": [
    "### Saving and loading a model with pickle"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 108,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pickle\n",
    "\n",
    "# Save an existing model to file\n",
    "pickle.dump(gs_clf, open(\"gs_random_forest_model_1.pkl\", \"wb\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 109,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Load a saved model\n",
    "loaded_pickle_model = pickle.load(open(\"gs_random_forest_model_1.pkl\", \"rb\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 110,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Acc: 83.61%\n",
      "Precision: 0.78\n",
      "Recall: 0.89\n",
      "F1 score: 0.83\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'accuracy': 0.84, 'precision': 0.78, 'recall': 0.89, 'f1': 0.83}"
      ]
     },
     "execution_count": 110,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Make predictions and evaluate the loaded model\n",
    "pickle_y_preds = loaded_pickle_model.predict(X_test)\n",
    "evaluate_preds(y_test, pickle_y_preds)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Saving and loading a model with joblib"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 111,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['gs_random_forest_model_1.joblib']"
      ]
     },
     "execution_count": 111,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from joblib import dump, load\n",
    "\n",
    "# Save a model to file\n",
    "dump(gs_clf, filename=\"gs_random_forest_model_1.joblib\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 112,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Import a saved joblib model\n",
    "loaded_joblib_model = load(filename=\"gs_random_forest_model_1.joblib\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 113,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Acc: 83.61%\n",
      "Precision: 0.78\n",
      "Recall: 0.89\n",
      "F1 score: 0.83\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'accuracy': 0.84, 'precision': 0.78, 'recall': 0.89, 'f1': 0.83}"
      ]
     },
     "execution_count": 113,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Make and evaluate joblib predictions \n",
    "joblib_y_preds = loaded_joblib_model.predict(X_test)\n",
    "evaluate_preds(y_test, joblib_y_preds)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "According to Scikit-Learn's documentation, they suggest it may be more efficient to use joblib as it's more efficient with large numpy array (which is what may be contained in trained/fitted Scikit-Learn models)."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# pipeline one more time, knowing what we know now"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 114,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Make</th>\n",
       "      <th>Colour</th>\n",
       "      <th>Odometer (KM)</th>\n",
       "      <th>Doors</th>\n",
       "      <th>Price</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Honda</td>\n",
       "      <td>White</td>\n",
       "      <td>35431.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>15323.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>BMW</td>\n",
       "      <td>Blue</td>\n",
       "      <td>192714.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>19943.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Honda</td>\n",
       "      <td>White</td>\n",
       "      <td>84714.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>28343.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Toyota</td>\n",
       "      <td>White</td>\n",
       "      <td>154365.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>13434.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Nissan</td>\n",
       "      <td>Blue</td>\n",
       "      <td>181577.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>14043.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     Make Colour  Odometer (KM)  Doors    Price\n",
       "0   Honda  White        35431.0    4.0  15323.0\n",
       "1     BMW   Blue       192714.0    5.0  19943.0\n",
       "2   Honda  White        84714.0    4.0  28343.0\n",
       "3  Toyota  White       154365.0    4.0  13434.0\n",
       "4  Nissan   Blue       181577.0    3.0  14043.0"
      ]
     },
     "execution_count": 114,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data = pd.read_csv(\"data/car-sales-extended-missing-data.csv\")\n",
    "data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 115,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Make              object\n",
       "Colour            object\n",
       "Odometer (KM)    float64\n",
       "Doors            float64\n",
       "Price            float64\n",
       "dtype: object"
      ]
     },
     "execution_count": 115,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.dtypes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 116,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Make             49\n",
       "Colour           50\n",
       "Odometer (KM)    50\n",
       "Doors            50\n",
       "Price            50\n",
       "dtype: int64"
      ]
     },
     "execution_count": 116,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.isna().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 117,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.22188417408787875"
      ]
     },
     "execution_count": 117,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "from sklearn.compose import ColumnTransformer\n",
    "from sklearn.pipeline import Pipeline\n",
    "from sklearn.impute import SimpleImputer\n",
    "from sklearn.preprocessing import OneHotEncoder\n",
    "\n",
    "# Modelling\n",
    "from sklearn.ensemble import RandomForestRegressor\n",
    "from sklearn.model_selection import train_test_split, GridSearchCV\n",
    "\n",
    "# Setup random seed\n",
    "import numpy as np\n",
    "np.random.seed(42)\n",
    "\n",
    "# Import data and drop the rows with missing labels\n",
    "data = pd.read_csv(\"data/car-sales-extended-missing-data.csv\")\n",
    "data.dropna(subset=[\"Price\"], inplace=True)\n",
    "\n",
    "# Define different features and transformer pipelines\n",
    "categorical_features = [\"Make\", \"Colour\"]\n",
    "categorical_transformer = Pipeline(steps=[\n",
    "    (\"imputer\", SimpleImputer(strategy=\"constant\", fill_value=\"missing\")),\n",
    "    (\"onehot\", OneHotEncoder(handle_unknown=\"ignore\"))])\n",
    "\n",
    "door_feature = [\"Doors\"]\n",
    "door_transformer = Pipeline(steps=[\n",
    "    (\"imputer\", SimpleImputer(strategy=\"constant\", fill_value=4))])\n",
    "\n",
    "numeric_features = [\"Odometer (KM)\"]\n",
    "numeric_transformer = Pipeline(steps=[\n",
    "    (\"imputer\", SimpleImputer(strategy=\"mean\"))\n",
    "])\n",
    "\n",
    "# Setup preprocessing steps (fill missing values, then convert to numbers)\n",
    "preprocessor = ColumnTransformer(\n",
    "    transformers=[\n",
    "        (\"cat\", categorical_transformer, categorical_features),\n",
    "        (\"door\", door_transformer, door_feature),\n",
    "        (\"num\", numeric_transformer, numeric_features)])\n",
    "\n",
    "# Create a preprocessing and modelling pipeline\n",
    "model = Pipeline(steps=[(\"preprocessor\", preprocessor),\n",
    "                        (\"model\", RandomForestRegressor())])\n",
    "\n",
    "# Split data\n",
    "X = data.drop(\"Price\", axis=1)\n",
    "y = data[\"Price\"]\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2)\n",
    "\n",
    "# Fit and score the model\n",
    "model.fit(X_train, y_train)\n",
    "model.score(X_test, y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 118,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 5 folds for each of 32 candidates, totalling 160 fits\n",
      "[CV] model__max_depth=None, model__max_features=auto, model__min_samples_split=2, model__n_estimators=100, preprocessor__num__imputer__strategy=mean \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV]  model__max_depth=None, model__max_features=auto, model__min_samples_split=2, model__n_estimators=100, preprocessor__num__imputer__strategy=mean, total=   0.3s\n",
      "[CV] model__max_depth=None, model__max_features=auto, model__min_samples_split=2, model__n_estimators=100, preprocessor__num__imputer__strategy=mean \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done   1 out of   1 | elapsed:    0.2s remaining:    0.0s\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV]  model__max_depth=None, model__max_features=auto, model__min_samples_split=2, model__n_estimators=100, preprocessor__num__imputer__strategy=mean, total=   0.2s\n",
      "[CV] model__max_depth=None, model__max_features=auto, model__min_samples_split=2, model__n_estimators=100, preprocessor__num__imputer__strategy=mean \n",
      "[CV]  model__max_depth=None, model__max_features=auto, model__min_samples_split=2, model__n_estimators=100, preprocessor__num__imputer__strategy=mean, total=   0.2s\n",
      "[CV] model__max_depth=None, model__max_features=auto, model__min_samples_split=2, model__n_estimators=100, preprocessor__num__imputer__strategy=mean \n",
      "[CV]  model__max_depth=None, model__max_features=auto, model__min_samples_split=2, model__n_estimators=100, preprocessor__num__imputer__strategy=mean, total=   0.2s\n",
      "[CV] model__max_depth=None, model__max_features=auto, model__min_samples_split=2, model__n_estimators=100, preprocessor__num__imputer__strategy=mean \n",
      "[CV]  model__max_depth=None, model__max_features=auto, model__min_samples_split=2, model__n_estimators=100, preprocessor__num__imputer__strategy=mean, total=   0.2s\n",
      "[CV] model__max_depth=None, model__max_features=auto, model__min_samples_split=2, model__n_estimators=100, preprocessor__num__imputer__strategy=median \n",
      "[CV]  model__max_depth=None, model__max_features=auto, model__min_samples_split=2, model__n_estimators=100, preprocessor__num__imputer__strategy=median, total=   0.2s\n",
      "[CV] model__max_depth=None, model__max_features=auto, model__min_samples_split=2, model__n_estimators=100, preprocessor__num__imputer__strategy=median \n",
      "[CV]  model__max_depth=None, model__max_features=auto, model__min_samples_split=2, model__n_estimators=100, preprocessor__num__imputer__strategy=median, total=   0.2s\n",
      "[CV] model__max_depth=None, model__max_features=auto, model__min_samples_split=2, model__n_estimators=100, preprocessor__num__imputer__strategy=median \n",
      "[CV]  model__max_depth=None, model__max_features=auto, model__min_samples_split=2, model__n_estimators=100, preprocessor__num__imputer__strategy=median, total=   0.2s\n",
      "[CV] model__max_depth=None, model__max_features=auto, model__min_samples_split=2, model__n_estimators=100, preprocessor__num__imputer__strategy=median \n",
      "[CV]  model__max_depth=None, model__max_features=auto, model__min_samples_split=2, model__n_estimators=100, preprocessor__num__imputer__strategy=median, total=   0.2s\n",
      "[CV] model__max_depth=None, model__max_features=auto, model__min_samples_split=2, model__n_estimators=100, preprocessor__num__imputer__strategy=median \n",
      "[CV]  model__max_depth=None, model__max_features=auto, model__min_samples_split=2, model__n_estimators=100, preprocessor__num__imputer__strategy=median, total=   0.2s\n",
      "[CV] model__max_depth=None, model__max_features=auto, model__min_samples_split=2, model__n_estimators=1000, preprocessor__num__imputer__strategy=mean \n",
      "[CV]  model__max_depth=None, model__max_features=auto, model__min_samples_split=2, model__n_estimators=1000, preprocessor__num__imputer__strategy=mean, total=   2.2s\n",
      "[CV] model__max_depth=None, model__max_features=auto, model__min_samples_split=2, model__n_estimators=1000, preprocessor__num__imputer__strategy=mean \n",
      "[CV]  model__max_depth=None, model__max_features=auto, model__min_samples_split=2, model__n_estimators=1000, preprocessor__num__imputer__strategy=mean, total=   2.2s\n",
      "[CV] model__max_depth=None, model__max_features=auto, model__min_samples_split=2, model__n_estimators=1000, preprocessor__num__imputer__strategy=mean \n",
      "[CV]  model__max_depth=None, model__max_features=auto, model__min_samples_split=2, model__n_estimators=1000, preprocessor__num__imputer__strategy=mean, total=   2.2s\n",
      "[CV] model__max_depth=None, model__max_features=auto, model__min_samples_split=2, model__n_estimators=1000, preprocessor__num__imputer__strategy=mean \n",
      "[CV]  model__max_depth=None, model__max_features=auto, model__min_samples_split=2, model__n_estimators=1000, preprocessor__num__imputer__strategy=mean, total=   2.1s\n",
      "[CV] model__max_depth=None, model__max_features=auto, model__min_samples_split=2, model__n_estimators=1000, preprocessor__num__imputer__strategy=mean \n",
      "[CV]  model__max_depth=None, model__max_features=auto, model__min_samples_split=2, model__n_estimators=1000, preprocessor__num__imputer__strategy=mean, total=   2.2s\n",
      "[CV] model__max_depth=None, model__max_features=auto, model__min_samples_split=2, model__n_estimators=1000, preprocessor__num__imputer__strategy=median \n",
      "[CV]  model__max_depth=None, model__max_features=auto, model__min_samples_split=2, model__n_estimators=1000, preprocessor__num__imputer__strategy=median, total=   2.1s\n",
      "[CV] model__max_depth=None, model__max_features=auto, model__min_samples_split=2, model__n_estimators=1000, preprocessor__num__imputer__strategy=median \n",
      "[CV]  model__max_depth=None, model__max_features=auto, model__min_samples_split=2, model__n_estimators=1000, preprocessor__num__imputer__strategy=median, total=   2.1s\n",
      "[CV] model__max_depth=None, model__max_features=auto, model__min_samples_split=2, model__n_estimators=1000, preprocessor__num__imputer__strategy=median \n",
      "[CV]  model__max_depth=None, model__max_features=auto, model__min_samples_split=2, model__n_estimators=1000, preprocessor__num__imputer__strategy=median, total=   2.1s\n",
      "[CV] model__max_depth=None, model__max_features=auto, model__min_samples_split=2, model__n_estimators=1000, preprocessor__num__imputer__strategy=median \n",
      "[CV]  model__max_depth=None, model__max_features=auto, model__min_samples_split=2, model__n_estimators=1000, preprocessor__num__imputer__strategy=median, total=   2.1s\n",
      "[CV] model__max_depth=None, model__max_features=auto, model__min_samples_split=2, model__n_estimators=1000, preprocessor__num__imputer__strategy=median \n",
      "[CV]  model__max_depth=None, model__max_features=auto, model__min_samples_split=2, model__n_estimators=1000, preprocessor__num__imputer__strategy=median, total=   2.1s\n",
      "[CV] model__max_depth=None, model__max_features=auto, model__min_samples_split=4, model__n_estimators=100, preprocessor__num__imputer__strategy=mean \n",
      "[CV]  model__max_depth=None, model__max_features=auto, model__min_samples_split=4, model__n_estimators=100, preprocessor__num__imputer__strategy=mean, total=   0.2s\n",
      "[CV] model__max_depth=None, model__max_features=auto, model__min_samples_split=4, model__n_estimators=100, preprocessor__num__imputer__strategy=mean \n",
      "[CV]  model__max_depth=None, model__max_features=auto, model__min_samples_split=4, model__n_estimators=100, preprocessor__num__imputer__strategy=mean, total=   0.2s\n",
      "[CV] model__max_depth=None, model__max_features=auto, model__min_samples_split=4, model__n_estimators=100, preprocessor__num__imputer__strategy=mean \n",
      "[CV]  model__max_depth=None, model__max_features=auto, model__min_samples_split=4, model__n_estimators=100, preprocessor__num__imputer__strategy=mean, total=   0.2s\n",
      "[CV] model__max_depth=None, model__max_features=auto, model__min_samples_split=4, model__n_estimators=100, preprocessor__num__imputer__strategy=mean \n",
      "[CV]  model__max_depth=None, model__max_features=auto, model__min_samples_split=4, model__n_estimators=100, preprocessor__num__imputer__strategy=mean, total=   0.2s\n",
      "[CV] model__max_depth=None, model__max_features=auto, model__min_samples_split=4, model__n_estimators=100, preprocessor__num__imputer__strategy=mean \n",
      "[CV]  model__max_depth=None, model__max_features=auto, model__min_samples_split=4, model__n_estimators=100, preprocessor__num__imputer__strategy=mean, total=   0.2s\n",
      "[CV] model__max_depth=None, model__max_features=auto, model__min_samples_split=4, model__n_estimators=100, preprocessor__num__imputer__strategy=median \n",
      "[CV]  model__max_depth=None, model__max_features=auto, model__min_samples_split=4, model__n_estimators=100, preprocessor__num__imputer__strategy=median, total=   0.2s\n",
      "[CV] model__max_depth=None, model__max_features=auto, model__min_samples_split=4, model__n_estimators=100, preprocessor__num__imputer__strategy=median \n",
      "[CV]  model__max_depth=None, model__max_features=auto, model__min_samples_split=4, model__n_estimators=100, preprocessor__num__imputer__strategy=median, total=   0.2s\n",
      "[CV] model__max_depth=None, model__max_features=auto, model__min_samples_split=4, model__n_estimators=100, preprocessor__num__imputer__strategy=median \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV]  model__max_depth=None, model__max_features=auto, model__min_samples_split=4, model__n_estimators=100, preprocessor__num__imputer__strategy=median, total=   0.2s\n",
      "[CV] model__max_depth=None, model__max_features=auto, model__min_samples_split=4, model__n_estimators=100, preprocessor__num__imputer__strategy=median \n",
      "[CV]  model__max_depth=None, model__max_features=auto, model__min_samples_split=4, model__n_estimators=100, preprocessor__num__imputer__strategy=median, total=   0.2s\n",
      "[CV] model__max_depth=None, model__max_features=auto, model__min_samples_split=4, model__n_estimators=100, preprocessor__num__imputer__strategy=median \n",
      "[CV]  model__max_depth=None, model__max_features=auto, model__min_samples_split=4, model__n_estimators=100, preprocessor__num__imputer__strategy=median, total=   0.2s\n",
      "[CV] model__max_depth=None, model__max_features=auto, model__min_samples_split=4, model__n_estimators=1000, preprocessor__num__imputer__strategy=mean \n",
      "[CV]  model__max_depth=None, model__max_features=auto, model__min_samples_split=4, model__n_estimators=1000, preprocessor__num__imputer__strategy=mean, total=   1.9s\n",
      "[CV] model__max_depth=None, model__max_features=auto, model__min_samples_split=4, model__n_estimators=1000, preprocessor__num__imputer__strategy=mean \n",
      "[CV]  model__max_depth=None, model__max_features=auto, model__min_samples_split=4, model__n_estimators=1000, preprocessor__num__imputer__strategy=mean, total=   1.9s\n",
      "[CV] model__max_depth=None, model__max_features=auto, model__min_samples_split=4, model__n_estimators=1000, preprocessor__num__imputer__strategy=mean \n",
      "[CV]  model__max_depth=None, model__max_features=auto, model__min_samples_split=4, model__n_estimators=1000, preprocessor__num__imputer__strategy=mean, total=   1.9s\n",
      "[CV] model__max_depth=None, model__max_features=auto, model__min_samples_split=4, model__n_estimators=1000, preprocessor__num__imputer__strategy=mean \n",
      "[CV]  model__max_depth=None, model__max_features=auto, model__min_samples_split=4, model__n_estimators=1000, preprocessor__num__imputer__strategy=mean, total=   1.9s\n",
      "[CV] model__max_depth=None, model__max_features=auto, model__min_samples_split=4, model__n_estimators=1000, preprocessor__num__imputer__strategy=mean \n",
      "[CV]  model__max_depth=None, model__max_features=auto, model__min_samples_split=4, model__n_estimators=1000, preprocessor__num__imputer__strategy=mean, total=   1.9s\n",
      "[CV] model__max_depth=None, model__max_features=auto, model__min_samples_split=4, model__n_estimators=1000, preprocessor__num__imputer__strategy=median \n",
      "[CV]  model__max_depth=None, model__max_features=auto, model__min_samples_split=4, model__n_estimators=1000, preprocessor__num__imputer__strategy=median, total=   1.9s\n",
      "[CV] model__max_depth=None, model__max_features=auto, model__min_samples_split=4, model__n_estimators=1000, preprocessor__num__imputer__strategy=median \n",
      "[CV]  model__max_depth=None, model__max_features=auto, model__min_samples_split=4, model__n_estimators=1000, preprocessor__num__imputer__strategy=median, total=   1.9s\n",
      "[CV] model__max_depth=None, model__max_features=auto, model__min_samples_split=4, model__n_estimators=1000, preprocessor__num__imputer__strategy=median \n",
      "[CV]  model__max_depth=None, model__max_features=auto, model__min_samples_split=4, model__n_estimators=1000, preprocessor__num__imputer__strategy=median, total=   1.9s\n",
      "[CV] model__max_depth=None, model__max_features=auto, model__min_samples_split=4, model__n_estimators=1000, preprocessor__num__imputer__strategy=median \n",
      "[CV]  model__max_depth=None, model__max_features=auto, model__min_samples_split=4, model__n_estimators=1000, preprocessor__num__imputer__strategy=median, total=   1.9s\n",
      "[CV] model__max_depth=None, model__max_features=auto, model__min_samples_split=4, model__n_estimators=1000, preprocessor__num__imputer__strategy=median \n",
      "[CV]  model__max_depth=None, model__max_features=auto, model__min_samples_split=4, model__n_estimators=1000, preprocessor__num__imputer__strategy=median, total=   2.0s\n",
      "[CV] model__max_depth=None, model__max_features=sqrt, model__min_samples_split=2, model__n_estimators=100, preprocessor__num__imputer__strategy=mean \n",
      "[CV]  model__max_depth=None, model__max_features=sqrt, model__min_samples_split=2, model__n_estimators=100, preprocessor__num__imputer__strategy=mean, total=   0.2s\n",
      "[CV] model__max_depth=None, model__max_features=sqrt, model__min_samples_split=2, model__n_estimators=100, preprocessor__num__imputer__strategy=mean \n",
      "[CV]  model__max_depth=None, model__max_features=sqrt, model__min_samples_split=2, model__n_estimators=100, preprocessor__num__imputer__strategy=mean, total=   0.2s\n",
      "[CV] model__max_depth=None, model__max_features=sqrt, model__min_samples_split=2, model__n_estimators=100, preprocessor__num__imputer__strategy=mean \n",
      "[CV]  model__max_depth=None, model__max_features=sqrt, model__min_samples_split=2, model__n_estimators=100, preprocessor__num__imputer__strategy=mean, total=   0.2s\n",
      "[CV] model__max_depth=None, model__max_features=sqrt, model__min_samples_split=2, model__n_estimators=100, preprocessor__num__imputer__strategy=mean \n",
      "[CV]  model__max_depth=None, model__max_features=sqrt, model__min_samples_split=2, model__n_estimators=100, preprocessor__num__imputer__strategy=mean, total=   0.2s\n",
      "[CV] model__max_depth=None, model__max_features=sqrt, model__min_samples_split=2, model__n_estimators=100, preprocessor__num__imputer__strategy=mean \n",
      "[CV]  model__max_depth=None, model__max_features=sqrt, model__min_samples_split=2, model__n_estimators=100, preprocessor__num__imputer__strategy=mean, total=   0.2s\n",
      "[CV] model__max_depth=None, model__max_features=sqrt, model__min_samples_split=2, model__n_estimators=100, preprocessor__num__imputer__strategy=median \n",
      "[CV]  model__max_depth=None, model__max_features=sqrt, model__min_samples_split=2, model__n_estimators=100, preprocessor__num__imputer__strategy=median, total=   0.2s\n",
      "[CV] model__max_depth=None, model__max_features=sqrt, model__min_samples_split=2, model__n_estimators=100, preprocessor__num__imputer__strategy=median \n",
      "[CV]  model__max_depth=None, model__max_features=sqrt, model__min_samples_split=2, model__n_estimators=100, preprocessor__num__imputer__strategy=median, total=   0.2s\n",
      "[CV] model__max_depth=None, model__max_features=sqrt, model__min_samples_split=2, model__n_estimators=100, preprocessor__num__imputer__strategy=median \n",
      "[CV]  model__max_depth=None, model__max_features=sqrt, model__min_samples_split=2, model__n_estimators=100, preprocessor__num__imputer__strategy=median, total=   0.2s\n",
      "[CV] model__max_depth=None, model__max_features=sqrt, model__min_samples_split=2, model__n_estimators=100, preprocessor__num__imputer__strategy=median \n",
      "[CV]  model__max_depth=None, model__max_features=sqrt, model__min_samples_split=2, model__n_estimators=100, preprocessor__num__imputer__strategy=median, total=   0.2s\n",
      "[CV] model__max_depth=None, model__max_features=sqrt, model__min_samples_split=2, model__n_estimators=100, preprocessor__num__imputer__strategy=median \n",
      "[CV]  model__max_depth=None, model__max_features=sqrt, model__min_samples_split=2, model__n_estimators=100, preprocessor__num__imputer__strategy=median, total=   0.2s\n",
      "[CV] model__max_depth=None, model__max_features=sqrt, model__min_samples_split=2, model__n_estimators=1000, preprocessor__num__imputer__strategy=mean \n",
      "[CV]  model__max_depth=None, model__max_features=sqrt, model__min_samples_split=2, model__n_estimators=1000, preprocessor__num__imputer__strategy=mean, total=   1.6s\n",
      "[CV] model__max_depth=None, model__max_features=sqrt, model__min_samples_split=2, model__n_estimators=1000, preprocessor__num__imputer__strategy=mean \n",
      "[CV]  model__max_depth=None, model__max_features=sqrt, model__min_samples_split=2, model__n_estimators=1000, preprocessor__num__imputer__strategy=mean, total=   1.6s\n",
      "[CV] model__max_depth=None, model__max_features=sqrt, model__min_samples_split=2, model__n_estimators=1000, preprocessor__num__imputer__strategy=mean \n",
      "[CV]  model__max_depth=None, model__max_features=sqrt, model__min_samples_split=2, model__n_estimators=1000, preprocessor__num__imputer__strategy=mean, total=   1.7s\n",
      "[CV] model__max_depth=None, model__max_features=sqrt, model__min_samples_split=2, model__n_estimators=1000, preprocessor__num__imputer__strategy=mean \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV]  model__max_depth=None, model__max_features=sqrt, model__min_samples_split=2, model__n_estimators=1000, preprocessor__num__imputer__strategy=mean, total=   1.7s\n",
      "[CV] model__max_depth=None, model__max_features=sqrt, model__min_samples_split=2, model__n_estimators=1000, preprocessor__num__imputer__strategy=mean \n",
      "[CV]  model__max_depth=None, model__max_features=sqrt, model__min_samples_split=2, model__n_estimators=1000, preprocessor__num__imputer__strategy=mean, total=   1.7s\n",
      "[CV] model__max_depth=None, model__max_features=sqrt, model__min_samples_split=2, model__n_estimators=1000, preprocessor__num__imputer__strategy=median \n",
      "[CV]  model__max_depth=None, model__max_features=sqrt, model__min_samples_split=2, model__n_estimators=1000, preprocessor__num__imputer__strategy=median, total=   1.7s\n",
      "[CV] model__max_depth=None, model__max_features=sqrt, model__min_samples_split=2, model__n_estimators=1000, preprocessor__num__imputer__strategy=median \n",
      "[CV]  model__max_depth=None, model__max_features=sqrt, model__min_samples_split=2, model__n_estimators=1000, preprocessor__num__imputer__strategy=median, total=   1.7s\n",
      "[CV] model__max_depth=None, model__max_features=sqrt, model__min_samples_split=2, model__n_estimators=1000, preprocessor__num__imputer__strategy=median \n",
      "[CV]  model__max_depth=None, model__max_features=sqrt, model__min_samples_split=2, model__n_estimators=1000, preprocessor__num__imputer__strategy=median, total=   1.7s\n",
      "[CV] model__max_depth=None, model__max_features=sqrt, model__min_samples_split=2, model__n_estimators=1000, preprocessor__num__imputer__strategy=median \n",
      "[CV]  model__max_depth=None, model__max_features=sqrt, model__min_samples_split=2, model__n_estimators=1000, preprocessor__num__imputer__strategy=median, total=   1.7s\n",
      "[CV] model__max_depth=None, model__max_features=sqrt, model__min_samples_split=2, model__n_estimators=1000, preprocessor__num__imputer__strategy=median \n",
      "[CV]  model__max_depth=None, model__max_features=sqrt, model__min_samples_split=2, model__n_estimators=1000, preprocessor__num__imputer__strategy=median, total=   1.7s\n",
      "[CV] model__max_depth=None, model__max_features=sqrt, model__min_samples_split=4, model__n_estimators=100, preprocessor__num__imputer__strategy=mean \n",
      "[CV]  model__max_depth=None, model__max_features=sqrt, model__min_samples_split=4, model__n_estimators=100, preprocessor__num__imputer__strategy=mean, total=   0.2s\n",
      "[CV] model__max_depth=None, model__max_features=sqrt, model__min_samples_split=4, model__n_estimators=100, preprocessor__num__imputer__strategy=mean \n",
      "[CV]  model__max_depth=None, model__max_features=sqrt, model__min_samples_split=4, model__n_estimators=100, preprocessor__num__imputer__strategy=mean, total=   0.2s\n",
      "[CV] model__max_depth=None, model__max_features=sqrt, model__min_samples_split=4, model__n_estimators=100, preprocessor__num__imputer__strategy=mean \n",
      "[CV]  model__max_depth=None, model__max_features=sqrt, model__min_samples_split=4, model__n_estimators=100, preprocessor__num__imputer__strategy=mean, total=   0.2s\n",
      "[CV] model__max_depth=None, model__max_features=sqrt, model__min_samples_split=4, model__n_estimators=100, preprocessor__num__imputer__strategy=mean \n",
      "[CV]  model__max_depth=None, model__max_features=sqrt, model__min_samples_split=4, model__n_estimators=100, preprocessor__num__imputer__strategy=mean, total=   0.2s\n",
      "[CV] model__max_depth=None, model__max_features=sqrt, model__min_samples_split=4, model__n_estimators=100, preprocessor__num__imputer__strategy=mean \n",
      "[CV]  model__max_depth=None, model__max_features=sqrt, model__min_samples_split=4, model__n_estimators=100, preprocessor__num__imputer__strategy=mean, total=   0.2s\n",
      "[CV] model__max_depth=None, model__max_features=sqrt, model__min_samples_split=4, model__n_estimators=100, preprocessor__num__imputer__strategy=median \n",
      "[CV]  model__max_depth=None, model__max_features=sqrt, model__min_samples_split=4, model__n_estimators=100, preprocessor__num__imputer__strategy=median, total=   0.2s\n",
      "[CV] model__max_depth=None, model__max_features=sqrt, model__min_samples_split=4, model__n_estimators=100, preprocessor__num__imputer__strategy=median \n",
      "[CV]  model__max_depth=None, model__max_features=sqrt, model__min_samples_split=4, model__n_estimators=100, preprocessor__num__imputer__strategy=median, total=   0.2s\n",
      "[CV] model__max_depth=None, model__max_features=sqrt, model__min_samples_split=4, model__n_estimators=100, preprocessor__num__imputer__strategy=median \n",
      "[CV]  model__max_depth=None, model__max_features=sqrt, model__min_samples_split=4, model__n_estimators=100, preprocessor__num__imputer__strategy=median, total=   0.2s\n",
      "[CV] model__max_depth=None, model__max_features=sqrt, model__min_samples_split=4, model__n_estimators=100, preprocessor__num__imputer__strategy=median \n",
      "[CV]  model__max_depth=None, model__max_features=sqrt, model__min_samples_split=4, model__n_estimators=100, preprocessor__num__imputer__strategy=median, total=   0.2s\n",
      "[CV] model__max_depth=None, model__max_features=sqrt, model__min_samples_split=4, model__n_estimators=100, preprocessor__num__imputer__strategy=median \n",
      "[CV]  model__max_depth=None, model__max_features=sqrt, model__min_samples_split=4, model__n_estimators=100, preprocessor__num__imputer__strategy=median, total=   0.2s\n",
      "[CV] model__max_depth=None, model__max_features=sqrt, model__min_samples_split=4, model__n_estimators=1000, preprocessor__num__imputer__strategy=mean \n",
      "[CV]  model__max_depth=None, model__max_features=sqrt, model__min_samples_split=4, model__n_estimators=1000, preprocessor__num__imputer__strategy=mean, total=   1.5s\n",
      "[CV] model__max_depth=None, model__max_features=sqrt, model__min_samples_split=4, model__n_estimators=1000, preprocessor__num__imputer__strategy=mean \n",
      "[CV]  model__max_depth=None, model__max_features=sqrt, model__min_samples_split=4, model__n_estimators=1000, preprocessor__num__imputer__strategy=mean, total=   1.5s\n",
      "[CV] model__max_depth=None, model__max_features=sqrt, model__min_samples_split=4, model__n_estimators=1000, preprocessor__num__imputer__strategy=mean \n",
      "[CV]  model__max_depth=None, model__max_features=sqrt, model__min_samples_split=4, model__n_estimators=1000, preprocessor__num__imputer__strategy=mean, total=   1.6s\n",
      "[CV] model__max_depth=None, model__max_features=sqrt, model__min_samples_split=4, model__n_estimators=1000, preprocessor__num__imputer__strategy=mean \n",
      "[CV]  model__max_depth=None, model__max_features=sqrt, model__min_samples_split=4, model__n_estimators=1000, preprocessor__num__imputer__strategy=mean, total=   1.6s\n",
      "[CV] model__max_depth=None, model__max_features=sqrt, model__min_samples_split=4, model__n_estimators=1000, preprocessor__num__imputer__strategy=mean \n",
      "[CV]  model__max_depth=None, model__max_features=sqrt, model__min_samples_split=4, model__n_estimators=1000, preprocessor__num__imputer__strategy=mean, total=   1.5s\n",
      "[CV] model__max_depth=None, model__max_features=sqrt, model__min_samples_split=4, model__n_estimators=1000, preprocessor__num__imputer__strategy=median \n",
      "[CV]  model__max_depth=None, model__max_features=sqrt, model__min_samples_split=4, model__n_estimators=1000, preprocessor__num__imputer__strategy=median, total=   1.6s\n",
      "[CV] model__max_depth=None, model__max_features=sqrt, model__min_samples_split=4, model__n_estimators=1000, preprocessor__num__imputer__strategy=median \n",
      "[CV]  model__max_depth=None, model__max_features=sqrt, model__min_samples_split=4, model__n_estimators=1000, preprocessor__num__imputer__strategy=median, total=   1.6s\n",
      "[CV] model__max_depth=None, model__max_features=sqrt, model__min_samples_split=4, model__n_estimators=1000, preprocessor__num__imputer__strategy=median \n",
      "[CV]  model__max_depth=None, model__max_features=sqrt, model__min_samples_split=4, model__n_estimators=1000, preprocessor__num__imputer__strategy=median, total=   1.5s\n",
      "[CV] model__max_depth=None, model__max_features=sqrt, model__min_samples_split=4, model__n_estimators=1000, preprocessor__num__imputer__strategy=median \n",
      "[CV]  model__max_depth=None, model__max_features=sqrt, model__min_samples_split=4, model__n_estimators=1000, preprocessor__num__imputer__strategy=median, total=   1.6s\n",
      "[CV] model__max_depth=None, model__max_features=sqrt, model__min_samples_split=4, model__n_estimators=1000, preprocessor__num__imputer__strategy=median \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV]  model__max_depth=None, model__max_features=sqrt, model__min_samples_split=4, model__n_estimators=1000, preprocessor__num__imputer__strategy=median, total=   1.6s\n",
      "[CV] model__max_depth=5, model__max_features=auto, model__min_samples_split=2, model__n_estimators=100, preprocessor__num__imputer__strategy=mean \n",
      "[CV]  model__max_depth=5, model__max_features=auto, model__min_samples_split=2, model__n_estimators=100, preprocessor__num__imputer__strategy=mean, total=   0.2s\n",
      "[CV] model__max_depth=5, model__max_features=auto, model__min_samples_split=2, model__n_estimators=100, preprocessor__num__imputer__strategy=mean \n",
      "[CV]  model__max_depth=5, model__max_features=auto, model__min_samples_split=2, model__n_estimators=100, preprocessor__num__imputer__strategy=mean, total=   0.2s\n",
      "[CV] model__max_depth=5, model__max_features=auto, model__min_samples_split=2, model__n_estimators=100, preprocessor__num__imputer__strategy=mean \n",
      "[CV]  model__max_depth=5, model__max_features=auto, model__min_samples_split=2, model__n_estimators=100, preprocessor__num__imputer__strategy=mean, total=   0.2s\n",
      "[CV] model__max_depth=5, model__max_features=auto, model__min_samples_split=2, model__n_estimators=100, preprocessor__num__imputer__strategy=mean \n",
      "[CV]  model__max_depth=5, model__max_features=auto, model__min_samples_split=2, model__n_estimators=100, preprocessor__num__imputer__strategy=mean, total=   0.2s\n",
      "[CV] model__max_depth=5, model__max_features=auto, model__min_samples_split=2, model__n_estimators=100, preprocessor__num__imputer__strategy=mean \n",
      "[CV]  model__max_depth=5, model__max_features=auto, model__min_samples_split=2, model__n_estimators=100, preprocessor__num__imputer__strategy=mean, total=   0.2s\n",
      "[CV] model__max_depth=5, model__max_features=auto, model__min_samples_split=2, model__n_estimators=100, preprocessor__num__imputer__strategy=median \n",
      "[CV]  model__max_depth=5, model__max_features=auto, model__min_samples_split=2, model__n_estimators=100, preprocessor__num__imputer__strategy=median, total=   0.2s\n",
      "[CV] model__max_depth=5, model__max_features=auto, model__min_samples_split=2, model__n_estimators=100, preprocessor__num__imputer__strategy=median \n",
      "[CV]  model__max_depth=5, model__max_features=auto, model__min_samples_split=2, model__n_estimators=100, preprocessor__num__imputer__strategy=median, total=   0.2s\n",
      "[CV] model__max_depth=5, model__max_features=auto, model__min_samples_split=2, model__n_estimators=100, preprocessor__num__imputer__strategy=median \n",
      "[CV]  model__max_depth=5, model__max_features=auto, model__min_samples_split=2, model__n_estimators=100, preprocessor__num__imputer__strategy=median, total=   0.2s\n",
      "[CV] model__max_depth=5, model__max_features=auto, model__min_samples_split=2, model__n_estimators=100, preprocessor__num__imputer__strategy=median \n",
      "[CV]  model__max_depth=5, model__max_features=auto, model__min_samples_split=2, model__n_estimators=100, preprocessor__num__imputer__strategy=median, total=   0.2s\n",
      "[CV] model__max_depth=5, model__max_features=auto, model__min_samples_split=2, model__n_estimators=100, preprocessor__num__imputer__strategy=median \n",
      "[CV]  model__max_depth=5, model__max_features=auto, model__min_samples_split=2, model__n_estimators=100, preprocessor__num__imputer__strategy=median, total=   0.2s\n",
      "[CV] model__max_depth=5, model__max_features=auto, model__min_samples_split=2, model__n_estimators=1000, preprocessor__num__imputer__strategy=mean \n",
      "[CV]  model__max_depth=5, model__max_features=auto, model__min_samples_split=2, model__n_estimators=1000, preprocessor__num__imputer__strategy=mean, total=   1.5s\n",
      "[CV] model__max_depth=5, model__max_features=auto, model__min_samples_split=2, model__n_estimators=1000, preprocessor__num__imputer__strategy=mean \n",
      "[CV]  model__max_depth=5, model__max_features=auto, model__min_samples_split=2, model__n_estimators=1000, preprocessor__num__imputer__strategy=mean, total=   1.5s\n",
      "[CV] model__max_depth=5, model__max_features=auto, model__min_samples_split=2, model__n_estimators=1000, preprocessor__num__imputer__strategy=mean \n",
      "[CV]  model__max_depth=5, model__max_features=auto, model__min_samples_split=2, model__n_estimators=1000, preprocessor__num__imputer__strategy=mean, total=   1.5s\n",
      "[CV] model__max_depth=5, model__max_features=auto, model__min_samples_split=2, model__n_estimators=1000, preprocessor__num__imputer__strategy=mean \n",
      "[CV]  model__max_depth=5, model__max_features=auto, model__min_samples_split=2, model__n_estimators=1000, preprocessor__num__imputer__strategy=mean, total=   1.4s\n",
      "[CV] model__max_depth=5, model__max_features=auto, model__min_samples_split=2, model__n_estimators=1000, preprocessor__num__imputer__strategy=mean \n",
      "[CV]  model__max_depth=5, model__max_features=auto, model__min_samples_split=2, model__n_estimators=1000, preprocessor__num__imputer__strategy=mean, total=   1.5s\n",
      "[CV] model__max_depth=5, model__max_features=auto, model__min_samples_split=2, model__n_estimators=1000, preprocessor__num__imputer__strategy=median \n",
      "[CV]  model__max_depth=5, model__max_features=auto, model__min_samples_split=2, model__n_estimators=1000, preprocessor__num__imputer__strategy=median, total=   1.4s\n",
      "[CV] model__max_depth=5, model__max_features=auto, model__min_samples_split=2, model__n_estimators=1000, preprocessor__num__imputer__strategy=median \n",
      "[CV]  model__max_depth=5, model__max_features=auto, model__min_samples_split=2, model__n_estimators=1000, preprocessor__num__imputer__strategy=median, total=   1.5s\n",
      "[CV] model__max_depth=5, model__max_features=auto, model__min_samples_split=2, model__n_estimators=1000, preprocessor__num__imputer__strategy=median \n",
      "[CV]  model__max_depth=5, model__max_features=auto, model__min_samples_split=2, model__n_estimators=1000, preprocessor__num__imputer__strategy=median, total=   1.4s\n",
      "[CV] model__max_depth=5, model__max_features=auto, model__min_samples_split=2, model__n_estimators=1000, preprocessor__num__imputer__strategy=median \n",
      "[CV]  model__max_depth=5, model__max_features=auto, model__min_samples_split=2, model__n_estimators=1000, preprocessor__num__imputer__strategy=median, total=   1.4s\n",
      "[CV] model__max_depth=5, model__max_features=auto, model__min_samples_split=2, model__n_estimators=1000, preprocessor__num__imputer__strategy=median \n",
      "[CV]  model__max_depth=5, model__max_features=auto, model__min_samples_split=2, model__n_estimators=1000, preprocessor__num__imputer__strategy=median, total=   1.4s\n",
      "[CV] model__max_depth=5, model__max_features=auto, model__min_samples_split=4, model__n_estimators=100, preprocessor__num__imputer__strategy=mean \n",
      "[CV]  model__max_depth=5, model__max_features=auto, model__min_samples_split=4, model__n_estimators=100, preprocessor__num__imputer__strategy=mean, total=   0.2s\n",
      "[CV] model__max_depth=5, model__max_features=auto, model__min_samples_split=4, model__n_estimators=100, preprocessor__num__imputer__strategy=mean \n",
      "[CV]  model__max_depth=5, model__max_features=auto, model__min_samples_split=4, model__n_estimators=100, preprocessor__num__imputer__strategy=mean, total=   0.2s\n",
      "[CV] model__max_depth=5, model__max_features=auto, model__min_samples_split=4, model__n_estimators=100, preprocessor__num__imputer__strategy=mean \n",
      "[CV]  model__max_depth=5, model__max_features=auto, model__min_samples_split=4, model__n_estimators=100, preprocessor__num__imputer__strategy=mean, total=   0.2s\n",
      "[CV] model__max_depth=5, model__max_features=auto, model__min_samples_split=4, model__n_estimators=100, preprocessor__num__imputer__strategy=mean \n",
      "[CV]  model__max_depth=5, model__max_features=auto, model__min_samples_split=4, model__n_estimators=100, preprocessor__num__imputer__strategy=mean, total=   0.2s\n",
      "[CV] model__max_depth=5, model__max_features=auto, model__min_samples_split=4, model__n_estimators=100, preprocessor__num__imputer__strategy=mean \n",
      "[CV]  model__max_depth=5, model__max_features=auto, model__min_samples_split=4, model__n_estimators=100, preprocessor__num__imputer__strategy=mean, total=   0.2s\n",
      "[CV] model__max_depth=5, model__max_features=auto, model__min_samples_split=4, model__n_estimators=100, preprocessor__num__imputer__strategy=median \n",
      "[CV]  model__max_depth=5, model__max_features=auto, model__min_samples_split=4, model__n_estimators=100, preprocessor__num__imputer__strategy=median, total=   0.2s\n",
      "[CV] model__max_depth=5, model__max_features=auto, model__min_samples_split=4, model__n_estimators=100, preprocessor__num__imputer__strategy=median \n",
      "[CV]  model__max_depth=5, model__max_features=auto, model__min_samples_split=4, model__n_estimators=100, preprocessor__num__imputer__strategy=median, total=   0.2s\n",
      "[CV] model__max_depth=5, model__max_features=auto, model__min_samples_split=4, model__n_estimators=100, preprocessor__num__imputer__strategy=median \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV]  model__max_depth=5, model__max_features=auto, model__min_samples_split=4, model__n_estimators=100, preprocessor__num__imputer__strategy=median, total=   0.2s\n",
      "[CV] model__max_depth=5, model__max_features=auto, model__min_samples_split=4, model__n_estimators=100, preprocessor__num__imputer__strategy=median \n",
      "[CV]  model__max_depth=5, model__max_features=auto, model__min_samples_split=4, model__n_estimators=100, preprocessor__num__imputer__strategy=median, total=   0.2s\n",
      "[CV] model__max_depth=5, model__max_features=auto, model__min_samples_split=4, model__n_estimators=100, preprocessor__num__imputer__strategy=median \n",
      "[CV]  model__max_depth=5, model__max_features=auto, model__min_samples_split=4, model__n_estimators=100, preprocessor__num__imputer__strategy=median, total=   0.2s\n",
      "[CV] model__max_depth=5, model__max_features=auto, model__min_samples_split=4, model__n_estimators=1000, preprocessor__num__imputer__strategy=mean \n",
      "[CV]  model__max_depth=5, model__max_features=auto, model__min_samples_split=4, model__n_estimators=1000, preprocessor__num__imputer__strategy=mean, total=   1.4s\n",
      "[CV] model__max_depth=5, model__max_features=auto, model__min_samples_split=4, model__n_estimators=1000, preprocessor__num__imputer__strategy=mean \n",
      "[CV]  model__max_depth=5, model__max_features=auto, model__min_samples_split=4, model__n_estimators=1000, preprocessor__num__imputer__strategy=mean, total=   1.4s\n",
      "[CV] model__max_depth=5, model__max_features=auto, model__min_samples_split=4, model__n_estimators=1000, preprocessor__num__imputer__strategy=mean \n",
      "[CV]  model__max_depth=5, model__max_features=auto, model__min_samples_split=4, model__n_estimators=1000, preprocessor__num__imputer__strategy=mean, total=   1.4s\n",
      "[CV] model__max_depth=5, model__max_features=auto, model__min_samples_split=4, model__n_estimators=1000, preprocessor__num__imputer__strategy=mean \n",
      "[CV]  model__max_depth=5, model__max_features=auto, model__min_samples_split=4, model__n_estimators=1000, preprocessor__num__imputer__strategy=mean, total=   1.4s\n",
      "[CV] model__max_depth=5, model__max_features=auto, model__min_samples_split=4, model__n_estimators=1000, preprocessor__num__imputer__strategy=mean \n",
      "[CV]  model__max_depth=5, model__max_features=auto, model__min_samples_split=4, model__n_estimators=1000, preprocessor__num__imputer__strategy=mean, total=   1.4s\n",
      "[CV] model__max_depth=5, model__max_features=auto, model__min_samples_split=4, model__n_estimators=1000, preprocessor__num__imputer__strategy=median \n",
      "[CV]  model__max_depth=5, model__max_features=auto, model__min_samples_split=4, model__n_estimators=1000, preprocessor__num__imputer__strategy=median, total=   1.4s\n",
      "[CV] model__max_depth=5, model__max_features=auto, model__min_samples_split=4, model__n_estimators=1000, preprocessor__num__imputer__strategy=median \n",
      "[CV]  model__max_depth=5, model__max_features=auto, model__min_samples_split=4, model__n_estimators=1000, preprocessor__num__imputer__strategy=median, total=   1.5s\n",
      "[CV] model__max_depth=5, model__max_features=auto, model__min_samples_split=4, model__n_estimators=1000, preprocessor__num__imputer__strategy=median \n",
      "[CV]  model__max_depth=5, model__max_features=auto, model__min_samples_split=4, model__n_estimators=1000, preprocessor__num__imputer__strategy=median, total=   1.4s\n",
      "[CV] model__max_depth=5, model__max_features=auto, model__min_samples_split=4, model__n_estimators=1000, preprocessor__num__imputer__strategy=median \n",
      "[CV]  model__max_depth=5, model__max_features=auto, model__min_samples_split=4, model__n_estimators=1000, preprocessor__num__imputer__strategy=median, total=   1.4s\n",
      "[CV] model__max_depth=5, model__max_features=auto, model__min_samples_split=4, model__n_estimators=1000, preprocessor__num__imputer__strategy=median \n",
      "[CV]  model__max_depth=5, model__max_features=auto, model__min_samples_split=4, model__n_estimators=1000, preprocessor__num__imputer__strategy=median, total=   1.4s\n",
      "[CV] model__max_depth=5, model__max_features=sqrt, model__min_samples_split=2, model__n_estimators=100, preprocessor__num__imputer__strategy=mean \n",
      "[CV]  model__max_depth=5, model__max_features=sqrt, model__min_samples_split=2, model__n_estimators=100, preprocessor__num__imputer__strategy=mean, total=   0.1s\n",
      "[CV] model__max_depth=5, model__max_features=sqrt, model__min_samples_split=2, model__n_estimators=100, preprocessor__num__imputer__strategy=mean \n",
      "[CV]  model__max_depth=5, model__max_features=sqrt, model__min_samples_split=2, model__n_estimators=100, preprocessor__num__imputer__strategy=mean, total=   0.1s\n",
      "[CV] model__max_depth=5, model__max_features=sqrt, model__min_samples_split=2, model__n_estimators=100, preprocessor__num__imputer__strategy=mean \n",
      "[CV]  model__max_depth=5, model__max_features=sqrt, model__min_samples_split=2, model__n_estimators=100, preprocessor__num__imputer__strategy=mean, total=   0.1s\n",
      "[CV] model__max_depth=5, model__max_features=sqrt, model__min_samples_split=2, model__n_estimators=100, preprocessor__num__imputer__strategy=mean \n",
      "[CV]  model__max_depth=5, model__max_features=sqrt, model__min_samples_split=2, model__n_estimators=100, preprocessor__num__imputer__strategy=mean, total=   0.1s\n",
      "[CV] model__max_depth=5, model__max_features=sqrt, model__min_samples_split=2, model__n_estimators=100, preprocessor__num__imputer__strategy=mean \n",
      "[CV]  model__max_depth=5, model__max_features=sqrt, model__min_samples_split=2, model__n_estimators=100, preprocessor__num__imputer__strategy=mean, total=   0.1s\n",
      "[CV] model__max_depth=5, model__max_features=sqrt, model__min_samples_split=2, model__n_estimators=100, preprocessor__num__imputer__strategy=median \n",
      "[CV]  model__max_depth=5, model__max_features=sqrt, model__min_samples_split=2, model__n_estimators=100, preprocessor__num__imputer__strategy=median, total=   0.1s\n",
      "[CV] model__max_depth=5, model__max_features=sqrt, model__min_samples_split=2, model__n_estimators=100, preprocessor__num__imputer__strategy=median \n",
      "[CV]  model__max_depth=5, model__max_features=sqrt, model__min_samples_split=2, model__n_estimators=100, preprocessor__num__imputer__strategy=median, total=   0.1s\n",
      "[CV] model__max_depth=5, model__max_features=sqrt, model__min_samples_split=2, model__n_estimators=100, preprocessor__num__imputer__strategy=median \n",
      "[CV]  model__max_depth=5, model__max_features=sqrt, model__min_samples_split=2, model__n_estimators=100, preprocessor__num__imputer__strategy=median, total=   0.1s\n",
      "[CV] model__max_depth=5, model__max_features=sqrt, model__min_samples_split=2, model__n_estimators=100, preprocessor__num__imputer__strategy=median \n",
      "[CV]  model__max_depth=5, model__max_features=sqrt, model__min_samples_split=2, model__n_estimators=100, preprocessor__num__imputer__strategy=median, total=   0.1s\n",
      "[CV] model__max_depth=5, model__max_features=sqrt, model__min_samples_split=2, model__n_estimators=100, preprocessor__num__imputer__strategy=median \n",
      "[CV]  model__max_depth=5, model__max_features=sqrt, model__min_samples_split=2, model__n_estimators=100, preprocessor__num__imputer__strategy=median, total=   0.1s\n",
      "[CV] model__max_depth=5, model__max_features=sqrt, model__min_samples_split=2, model__n_estimators=1000, preprocessor__num__imputer__strategy=mean \n",
      "[CV]  model__max_depth=5, model__max_features=sqrt, model__min_samples_split=2, model__n_estimators=1000, preprocessor__num__imputer__strategy=mean, total=   1.2s\n",
      "[CV] model__max_depth=5, model__max_features=sqrt, model__min_samples_split=2, model__n_estimators=1000, preprocessor__num__imputer__strategy=mean \n",
      "[CV]  model__max_depth=5, model__max_features=sqrt, model__min_samples_split=2, model__n_estimators=1000, preprocessor__num__imputer__strategy=mean, total=   1.2s\n",
      "[CV] model__max_depth=5, model__max_features=sqrt, model__min_samples_split=2, model__n_estimators=1000, preprocessor__num__imputer__strategy=mean \n",
      "[CV]  model__max_depth=5, model__max_features=sqrt, model__min_samples_split=2, model__n_estimators=1000, preprocessor__num__imputer__strategy=mean, total=   1.2s\n",
      "[CV] model__max_depth=5, model__max_features=sqrt, model__min_samples_split=2, model__n_estimators=1000, preprocessor__num__imputer__strategy=mean \n",
      "[CV]  model__max_depth=5, model__max_features=sqrt, model__min_samples_split=2, model__n_estimators=1000, preprocessor__num__imputer__strategy=mean, total=   1.2s\n",
      "[CV] model__max_depth=5, model__max_features=sqrt, model__min_samples_split=2, model__n_estimators=1000, preprocessor__num__imputer__strategy=mean \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV]  model__max_depth=5, model__max_features=sqrt, model__min_samples_split=2, model__n_estimators=1000, preprocessor__num__imputer__strategy=mean, total=   1.2s\n",
      "[CV] model__max_depth=5, model__max_features=sqrt, model__min_samples_split=2, model__n_estimators=1000, preprocessor__num__imputer__strategy=median \n",
      "[CV]  model__max_depth=5, model__max_features=sqrt, model__min_samples_split=2, model__n_estimators=1000, preprocessor__num__imputer__strategy=median, total=   1.2s\n",
      "[CV] model__max_depth=5, model__max_features=sqrt, model__min_samples_split=2, model__n_estimators=1000, preprocessor__num__imputer__strategy=median \n",
      "[CV]  model__max_depth=5, model__max_features=sqrt, model__min_samples_split=2, model__n_estimators=1000, preprocessor__num__imputer__strategy=median, total=   1.3s\n",
      "[CV] model__max_depth=5, model__max_features=sqrt, model__min_samples_split=2, model__n_estimators=1000, preprocessor__num__imputer__strategy=median \n",
      "[CV]  model__max_depth=5, model__max_features=sqrt, model__min_samples_split=2, model__n_estimators=1000, preprocessor__num__imputer__strategy=median, total=   1.3s\n",
      "[CV] model__max_depth=5, model__max_features=sqrt, model__min_samples_split=2, model__n_estimators=1000, preprocessor__num__imputer__strategy=median \n",
      "[CV]  model__max_depth=5, model__max_features=sqrt, model__min_samples_split=2, model__n_estimators=1000, preprocessor__num__imputer__strategy=median, total=   1.2s\n",
      "[CV] model__max_depth=5, model__max_features=sqrt, model__min_samples_split=2, model__n_estimators=1000, preprocessor__num__imputer__strategy=median \n",
      "[CV]  model__max_depth=5, model__max_features=sqrt, model__min_samples_split=2, model__n_estimators=1000, preprocessor__num__imputer__strategy=median, total=   1.2s\n",
      "[CV] model__max_depth=5, model__max_features=sqrt, model__min_samples_split=4, model__n_estimators=100, preprocessor__num__imputer__strategy=mean \n",
      "[CV]  model__max_depth=5, model__max_features=sqrt, model__min_samples_split=4, model__n_estimators=100, preprocessor__num__imputer__strategy=mean, total=   0.1s\n",
      "[CV] model__max_depth=5, model__max_features=sqrt, model__min_samples_split=4, model__n_estimators=100, preprocessor__num__imputer__strategy=mean \n",
      "[CV]  model__max_depth=5, model__max_features=sqrt, model__min_samples_split=4, model__n_estimators=100, preprocessor__num__imputer__strategy=mean, total=   0.1s\n",
      "[CV] model__max_depth=5, model__max_features=sqrt, model__min_samples_split=4, model__n_estimators=100, preprocessor__num__imputer__strategy=mean \n",
      "[CV]  model__max_depth=5, model__max_features=sqrt, model__min_samples_split=4, model__n_estimators=100, preprocessor__num__imputer__strategy=mean, total=   0.1s\n",
      "[CV] model__max_depth=5, model__max_features=sqrt, model__min_samples_split=4, model__n_estimators=100, preprocessor__num__imputer__strategy=mean \n",
      "[CV]  model__max_depth=5, model__max_features=sqrt, model__min_samples_split=4, model__n_estimators=100, preprocessor__num__imputer__strategy=mean, total=   0.1s\n",
      "[CV] model__max_depth=5, model__max_features=sqrt, model__min_samples_split=4, model__n_estimators=100, preprocessor__num__imputer__strategy=mean \n",
      "[CV]  model__max_depth=5, model__max_features=sqrt, model__min_samples_split=4, model__n_estimators=100, preprocessor__num__imputer__strategy=mean, total=   0.1s\n",
      "[CV] model__max_depth=5, model__max_features=sqrt, model__min_samples_split=4, model__n_estimators=100, preprocessor__num__imputer__strategy=median \n",
      "[CV]  model__max_depth=5, model__max_features=sqrt, model__min_samples_split=4, model__n_estimators=100, preprocessor__num__imputer__strategy=median, total=   0.1s\n",
      "[CV] model__max_depth=5, model__max_features=sqrt, model__min_samples_split=4, model__n_estimators=100, preprocessor__num__imputer__strategy=median \n",
      "[CV]  model__max_depth=5, model__max_features=sqrt, model__min_samples_split=4, model__n_estimators=100, preprocessor__num__imputer__strategy=median, total=   0.1s\n",
      "[CV] model__max_depth=5, model__max_features=sqrt, model__min_samples_split=4, model__n_estimators=100, preprocessor__num__imputer__strategy=median \n",
      "[CV]  model__max_depth=5, model__max_features=sqrt, model__min_samples_split=4, model__n_estimators=100, preprocessor__num__imputer__strategy=median, total=   0.1s\n",
      "[CV] model__max_depth=5, model__max_features=sqrt, model__min_samples_split=4, model__n_estimators=100, preprocessor__num__imputer__strategy=median \n",
      "[CV]  model__max_depth=5, model__max_features=sqrt, model__min_samples_split=4, model__n_estimators=100, preprocessor__num__imputer__strategy=median, total=   0.1s\n",
      "[CV] model__max_depth=5, model__max_features=sqrt, model__min_samples_split=4, model__n_estimators=100, preprocessor__num__imputer__strategy=median \n",
      "[CV]  model__max_depth=5, model__max_features=sqrt, model__min_samples_split=4, model__n_estimators=100, preprocessor__num__imputer__strategy=median, total=   0.1s\n",
      "[CV] model__max_depth=5, model__max_features=sqrt, model__min_samples_split=4, model__n_estimators=1000, preprocessor__num__imputer__strategy=mean \n",
      "[CV]  model__max_depth=5, model__max_features=sqrt, model__min_samples_split=4, model__n_estimators=1000, preprocessor__num__imputer__strategy=mean, total=   1.2s\n",
      "[CV] model__max_depth=5, model__max_features=sqrt, model__min_samples_split=4, model__n_estimators=1000, preprocessor__num__imputer__strategy=mean \n",
      "[CV]  model__max_depth=5, model__max_features=sqrt, model__min_samples_split=4, model__n_estimators=1000, preprocessor__num__imputer__strategy=mean, total=   1.2s\n",
      "[CV] model__max_depth=5, model__max_features=sqrt, model__min_samples_split=4, model__n_estimators=1000, preprocessor__num__imputer__strategy=mean \n",
      "[CV]  model__max_depth=5, model__max_features=sqrt, model__min_samples_split=4, model__n_estimators=1000, preprocessor__num__imputer__strategy=mean, total=   1.2s\n",
      "[CV] model__max_depth=5, model__max_features=sqrt, model__min_samples_split=4, model__n_estimators=1000, preprocessor__num__imputer__strategy=mean \n",
      "[CV]  model__max_depth=5, model__max_features=sqrt, model__min_samples_split=4, model__n_estimators=1000, preprocessor__num__imputer__strategy=mean, total=   1.2s\n",
      "[CV] model__max_depth=5, model__max_features=sqrt, model__min_samples_split=4, model__n_estimators=1000, preprocessor__num__imputer__strategy=mean \n",
      "[CV]  model__max_depth=5, model__max_features=sqrt, model__min_samples_split=4, model__n_estimators=1000, preprocessor__num__imputer__strategy=mean, total=   1.2s\n",
      "[CV] model__max_depth=5, model__max_features=sqrt, model__min_samples_split=4, model__n_estimators=1000, preprocessor__num__imputer__strategy=median \n",
      "[CV]  model__max_depth=5, model__max_features=sqrt, model__min_samples_split=4, model__n_estimators=1000, preprocessor__num__imputer__strategy=median, total=   1.2s\n",
      "[CV] model__max_depth=5, model__max_features=sqrt, model__min_samples_split=4, model__n_estimators=1000, preprocessor__num__imputer__strategy=median \n",
      "[CV]  model__max_depth=5, model__max_features=sqrt, model__min_samples_split=4, model__n_estimators=1000, preprocessor__num__imputer__strategy=median, total=   1.2s\n",
      "[CV] model__max_depth=5, model__max_features=sqrt, model__min_samples_split=4, model__n_estimators=1000, preprocessor__num__imputer__strategy=median \n",
      "[CV]  model__max_depth=5, model__max_features=sqrt, model__min_samples_split=4, model__n_estimators=1000, preprocessor__num__imputer__strategy=median, total=   1.2s\n",
      "[CV] model__max_depth=5, model__max_features=sqrt, model__min_samples_split=4, model__n_estimators=1000, preprocessor__num__imputer__strategy=median \n",
      "[CV]  model__max_depth=5, model__max_features=sqrt, model__min_samples_split=4, model__n_estimators=1000, preprocessor__num__imputer__strategy=median, total=   1.2s\n",
      "[CV] model__max_depth=5, model__max_features=sqrt, model__min_samples_split=4, model__n_estimators=1000, preprocessor__num__imputer__strategy=median \n",
      "[CV]  model__max_depth=5, model__max_features=sqrt, model__min_samples_split=4, model__n_estimators=1000, preprocessor__num__imputer__strategy=median, total=   1.3s\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done 160 out of 160 | elapsed:  2.4min finished\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "GridSearchCV(cv=5,\n",
       "             estimator=Pipeline(steps=[('preprocessor',\n",
       "                                        ColumnTransformer(transformers=[('cat',\n",
       "                                                                         Pipeline(steps=[('imputer',\n",
       "                                                                                          SimpleImputer(fill_value='missing',\n",
       "                                                                                                        strategy='constant')),\n",
       "                                                                                         ('onehot',\n",
       "                                                                                          OneHotEncoder(handle_unknown='ignore'))]),\n",
       "                                                                         ['Make',\n",
       "                                                                          'Colour']),\n",
       "                                                                        ('door',\n",
       "                                                                         Pipeline(steps=[('imputer',\n",
       "                                                                                          SimpleImputer(fill_value=4,\n",
       "                                                                                                        strategy='constant'))]),\n",
       "                                                                         ['Doors']),\n",
       "                                                                        ('num',\n",
       "                                                                         Pipeline(steps=[('imputer',\n",
       "                                                                                          SimpleImputer())]),\n",
       "                                                                         ['Odometer '\n",
       "                                                                          '(KM)'])])),\n",
       "                                       ('model', RandomForestRegressor())]),\n",
       "             param_grid={'model__max_depth': [None, 5],\n",
       "                         'model__max_features': ['auto', 'sqrt'],\n",
       "                         'model__min_samples_split': [2, 4],\n",
       "                         'model__n_estimators': [100, 1000],\n",
       "                         'preprocessor__num__imputer__strategy': ['mean',\n",
       "                                                                  'median']},\n",
       "             verbose=2)"
      ]
     },
     "execution_count": 118,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Using grid search with pipeline\n",
    "pipe_grid = {\n",
    "    \"preprocessor__num__imputer__strategy\": [\"mean\", \"median\"],\n",
    "    \"model__n_estimators\": [100, 1000],\n",
    "    \"model__max_depth\": [None, 5],\n",
    "    \"model__max_features\": [\"auto\", \"sqrt\"],\n",
    "    \"model__min_samples_split\": [2, 4]\n",
    "}\n",
    "\n",
    "gs_model = GridSearchCV(model, pipe_grid, cv=5, verbose=2)\n",
    "gs_model.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 119,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.292308819012865"
      ]
     },
     "execution_count": 119,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Score the best model\n",
    "gs_model.score(X_test, y_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Done"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}