Premier League Dataset.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Premier League Classification"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### This notebook takes the historical data on English Premier League matches from 2000 to 2018 and aims to build an efficient model that predicts, for each game, if the home team is going to win or not. "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Data importing and preprocessing with Pandas"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from IPython.core.interactiveshell import InteractiveShell\n",
    "InteractiveShell.ast_node_interactivity = \"all\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 6461 entries, 0 to 6460\n",
      "Data columns (total 23 columns):\n",
      "Date        6460 non-null object\n",
      "HomeTeam    6460 non-null object\n",
      "AwayTeam    6460 non-null object\n",
      "FTHG        6460 non-null float64\n",
      "FTAG        6460 non-null float64\n",
      "FTR         6460 non-null object\n",
      "HTHG        6460 non-null float64\n",
      "HTAG        6460 non-null float64\n",
      "HTR         6460 non-null object\n",
      "Referee     6460 non-null object\n",
      "HS          6460 non-null float64\n",
      "AS          6460 non-null float64\n",
      "HST         6460 non-null float64\n",
      "AST         6460 non-null float64\n",
      "HF          6460 non-null float64\n",
      "AF          6460 non-null float64\n",
      "HC          6460 non-null float64\n",
      "AC          6460 non-null float64\n",
      "HY          6460 non-null float64\n",
      "AY          6460 non-null float64\n",
      "HR          6460 non-null float64\n",
      "AR          6460 non-null float64\n",
      "Season      6461 non-null object\n",
      "dtypes: float64(16), object(7)\n",
      "memory usage: 1.1+ MB\n"
     ]
    }
   ],
   "source": [
    "df=pd.read_csv('dataset.csv')\n",
    "df.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Date</th>\n",
       "      <th>HomeTeam</th>\n",
       "      <th>AwayTeam</th>\n",
       "      <th>FTHG</th>\n",
       "      <th>FTAG</th>\n",
       "      <th>FTR</th>\n",
       "      <th>HTHG</th>\n",
       "      <th>HTAG</th>\n",
       "      <th>HTR</th>\n",
       "      <th>Referee</th>\n",
       "      <th>...</th>\n",
       "      <th>AST</th>\n",
       "      <th>HF</th>\n",
       "      <th>AF</th>\n",
       "      <th>HC</th>\n",
       "      <th>AC</th>\n",
       "      <th>HY</th>\n",
       "      <th>AY</th>\n",
       "      <th>HR</th>\n",
       "      <th>AR</th>\n",
       "      <th>Season</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>19/08/00</td>\n",
       "      <td>Charlton</td>\n",
       "      <td>Man City</td>\n",
       "      <td>4.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>H</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>H</td>\n",
       "      <td>Rob Harris</td>\n",
       "      <td>...</td>\n",
       "      <td>4.0</td>\n",
       "      <td>13.0</td>\n",
       "      <td>12.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>00-01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>19/08/00</td>\n",
       "      <td>Chelsea</td>\n",
       "      <td>West Ham</td>\n",
       "      <td>4.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>H</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>H</td>\n",
       "      <td>Graham Barber</td>\n",
       "      <td>...</td>\n",
       "      <td>5.0</td>\n",
       "      <td>19.0</td>\n",
       "      <td>14.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>00-01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>19/08/00</td>\n",
       "      <td>Coventry</td>\n",
       "      <td>Middlesbrough</td>\n",
       "      <td>1.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>A</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>D</td>\n",
       "      <td>Barry Knight</td>\n",
       "      <td>...</td>\n",
       "      <td>9.0</td>\n",
       "      <td>15.0</td>\n",
       "      <td>21.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>00-01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>19/08/00</td>\n",
       "      <td>Derby</td>\n",
       "      <td>Southampton</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>D</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>A</td>\n",
       "      <td>Andy D'Urso</td>\n",
       "      <td>...</td>\n",
       "      <td>6.0</td>\n",
       "      <td>11.0</td>\n",
       "      <td>13.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>00-01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>19/08/00</td>\n",
       "      <td>Leeds</td>\n",
       "      <td>Everton</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>H</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>H</td>\n",
       "      <td>Dermot Gallagher</td>\n",
       "      <td>...</td>\n",
       "      <td>6.0</td>\n",
       "      <td>21.0</td>\n",
       "      <td>20.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>00-01</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 23 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       Date  HomeTeam       AwayTeam  FTHG  FTAG FTR  HTHG  HTAG HTR  \\\n",
       "0  19/08/00  Charlton       Man City   4.0   0.0   H   2.0   0.0   H   \n",
       "1  19/08/00   Chelsea       West Ham   4.0   2.0   H   1.0   0.0   H   \n",
       "2  19/08/00  Coventry  Middlesbrough   1.0   3.0   A   1.0   1.0   D   \n",
       "3  19/08/00     Derby    Southampton   2.0   2.0   D   1.0   2.0   A   \n",
       "4  19/08/00     Leeds        Everton   2.0   0.0   H   2.0   0.0   H   \n",
       "\n",
       "            Referee  ...  AST    HF    AF   HC   AC   HY   AY   HR   AR  \\\n",
       "0        Rob Harris  ...  4.0  13.0  12.0  6.0  6.0  1.0  2.0  0.0  0.0   \n",
       "1     Graham Barber  ...  5.0  19.0  14.0  7.0  7.0  1.0  2.0  0.0  0.0   \n",
       "2      Barry Knight  ...  9.0  15.0  21.0  8.0  4.0  5.0  3.0  1.0  0.0   \n",
       "3       Andy D'Urso  ...  6.0  11.0  13.0  5.0  8.0  1.0  1.0  0.0  0.0   \n",
       "4  Dermot Gallagher  ...  6.0  21.0  20.0  6.0  4.0  1.0  3.0  0.0  0.0   \n",
       "\n",
       "   Season  \n",
       "0   00-01  \n",
       "1   00-01  \n",
       "2   00-01  \n",
       "3   00-01  \n",
       "4   00-01  \n",
       "\n",
       "[5 rows x 23 columns]"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "Date         object\n",
       "HomeTeam     object\n",
       "AwayTeam     object\n",
       "FTHG        float64\n",
       "FTAG        float64\n",
       "FTR          object\n",
       "HTHG        float64\n",
       "HTAG        float64\n",
       "HTR          object\n",
       "Referee      object\n",
       "HS          float64\n",
       "AS          float64\n",
       "HST         float64\n",
       "AST         float64\n",
       "HF          float64\n",
       "AF          float64\n",
       "HC          float64\n",
       "AC          float64\n",
       "HY          float64\n",
       "AY          float64\n",
       "HR          float64\n",
       "AR          float64\n",
       "Season       object\n",
       "dtype: object"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head() #show first 5 columns\n",
    "df.dtypes # show datatypes"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### The variables present in the data frame are the following:\n",
    "\n",
    "- **DateMatch:** Date of the match(dd/mm/yy)\n",
    "- **HomeTeam:** Home Team\n",
    "- **AwayTeam:** Away Team\n",
    "- **FTHG:** Full Time Home Team Goals\n",
    "- **FTAG:** Full Time Away Team Goals\n",
    "- **FTR:** Full Time Result (H=Home Win, D=Draw, A=Away Win)\n",
    "- **HTHG:** Half Time Home Team Goals\n",
    "- **HTAG:** Half Time Away Team Goals\n",
    "- **HTR:** Half Time Result (H=Home Win, D=Draw, A=Away Win)\n",
    "- **Referee:** Match Referee\n",
    "- **HS:** Home Team Shots\n",
    "- **AS:** Away Team Shots\n",
    "- **HST:** Home Team Shots on Target\n",
    "- **AST:** Away Team Shots on Target\n",
    "- **HF:** Home Team Fouls Committed\n",
    "- **AF:** Away Team Fouls Committed\n",
    "- **HC:** Home Team Corners\n",
    "- **AC:** Away Team Corners\n",
    "- **HY:** Home Team Yellow Cards\n",
    "- **AY:** Away Team Yellow Cards\n",
    "- **HR:** Home Team Red Cards\n",
    "- **AR:** Away Team Red Cards\n",
    "- **Season:** Season (00-01 means the season 2000-2001 and so on)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Create new column for home-win"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Date</th>\n",
       "      <th>HomeTeam</th>\n",
       "      <th>AwayTeam</th>\n",
       "      <th>FTHG</th>\n",
       "      <th>FTAG</th>\n",
       "      <th>FTR</th>\n",
       "      <th>HTHG</th>\n",
       "      <th>HTAG</th>\n",
       "      <th>HTR</th>\n",
       "      <th>Referee</th>\n",
       "      <th>...</th>\n",
       "      <th>HF</th>\n",
       "      <th>AF</th>\n",
       "      <th>HC</th>\n",
       "      <th>AC</th>\n",
       "      <th>HY</th>\n",
       "      <th>AY</th>\n",
       "      <th>HR</th>\n",
       "      <th>AR</th>\n",
       "      <th>Season</th>\n",
       "      <th>HW</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>19/08/00</td>\n",
       "      <td>Charlton</td>\n",
       "      <td>Man City</td>\n",
       "      <td>4.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>H</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>H</td>\n",
       "      <td>Rob Harris</td>\n",
       "      <td>...</td>\n",
       "      <td>13.0</td>\n",
       "      <td>12.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>00-01</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>19/08/00</td>\n",
       "      <td>Chelsea</td>\n",
       "      <td>West Ham</td>\n",
       "      <td>4.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>H</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>H</td>\n",
       "      <td>Graham Barber</td>\n",
       "      <td>...</td>\n",
       "      <td>19.0</td>\n",
       "      <td>14.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>00-01</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>19/08/00</td>\n",
       "      <td>Coventry</td>\n",
       "      <td>Middlesbrough</td>\n",
       "      <td>1.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>A</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>D</td>\n",
       "      <td>Barry Knight</td>\n",
       "      <td>...</td>\n",
       "      <td>15.0</td>\n",
       "      <td>21.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>00-01</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>19/08/00</td>\n",
       "      <td>Derby</td>\n",
       "      <td>Southampton</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>D</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>A</td>\n",
       "      <td>Andy D'Urso</td>\n",
       "      <td>...</td>\n",
       "      <td>11.0</td>\n",
       "      <td>13.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>00-01</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>19/08/00</td>\n",
       "      <td>Leeds</td>\n",
       "      <td>Everton</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>H</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>H</td>\n",
       "      <td>Dermot Gallagher</td>\n",
       "      <td>...</td>\n",
       "      <td>21.0</td>\n",
       "      <td>20.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>00-01</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 24 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       Date  HomeTeam       AwayTeam  FTHG  FTAG FTR  HTHG  HTAG HTR  \\\n",
       "0  19/08/00  Charlton       Man City   4.0   0.0   H   2.0   0.0   H   \n",
       "1  19/08/00   Chelsea       West Ham   4.0   2.0   H   1.0   0.0   H   \n",
       "2  19/08/00  Coventry  Middlesbrough   1.0   3.0   A   1.0   1.0   D   \n",
       "3  19/08/00     Derby    Southampton   2.0   2.0   D   1.0   2.0   A   \n",
       "4  19/08/00     Leeds        Everton   2.0   0.0   H   2.0   0.0   H   \n",
       "\n",
       "            Referee  ...    HF    AF   HC   AC   HY   AY   HR   AR  Season  \\\n",
       "0        Rob Harris  ...  13.0  12.0  6.0  6.0  1.0  2.0  0.0  0.0   00-01   \n",
       "1     Graham Barber  ...  19.0  14.0  7.0  7.0  1.0  2.0  0.0  0.0   00-01   \n",
       "2      Barry Knight  ...  15.0  21.0  8.0  4.0  5.0  3.0  1.0  0.0   00-01   \n",
       "3       Andy D'Urso  ...  11.0  13.0  5.0  8.0  1.0  1.0  0.0  0.0   00-01   \n",
       "4  Dermot Gallagher  ...  21.0  20.0  6.0  4.0  1.0  3.0  0.0  0.0   00-01   \n",
       "\n",
       "    HW  \n",
       "0  1.0  \n",
       "1  1.0  \n",
       "2  0.0  \n",
       "3  0.0  \n",
       "4  1.0  \n",
       "\n",
       "[5 rows x 24 columns]"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.loc[df.FTR == 'H', 'HW'] = 1 #for winning the game\n",
    "df.loc[df.FTR != 'H', 'HW'] = 0 #for loosing the game\n",
    "df.head() #show first columns"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Checking for missing values in the dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "Date        1\n",
       "HomeTeam    1\n",
       "AwayTeam    1\n",
       "FTHG        1\n",
       "FTAG        1\n",
       "FTR         1\n",
       "HTHG        1\n",
       "HTAG        1\n",
       "HTR         1\n",
       "Referee     1\n",
       "HS          1\n",
       "AS          1\n",
       "HST         1\n",
       "AST         1\n",
       "HF          1\n",
       "AF          1\n",
       "HC          1\n",
       "AC          1\n",
       "HY          1\n",
       "AY          1\n",
       "HR          1\n",
       "AR          1\n",
       "Season      0\n",
       "HW          0\n",
       "dtype: int64"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.isnull().values.any()\n",
    "df.isnull().sum()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "As there is only 1 missing value in the dataset of 6461, the Pandas method .dropna() is being applied"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "False"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_clean = df.dropna()\n",
    "df_clean.isnull().values.any()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of home wins:(3011, 24)\n"
     ]
    }
   ],
   "source": [
    "print('Number of home wins:' + str(df_clean.loc[df.HW == 1].shape))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Removing rows based on subsetting"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "In order to go further on preprocessing the data, rows were removed based on subsetting. Data from Season 03-04 was removed, because during this season Arsenal won all of their games and were unbeatable, which makes it kind of an outlier and that's those rows were removed."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(6080, 24)"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Date</th>\n",
       "      <th>HomeTeam</th>\n",
       "      <th>AwayTeam</th>\n",
       "      <th>FTHG</th>\n",
       "      <th>FTAG</th>\n",
       "      <th>FTR</th>\n",
       "      <th>HTHG</th>\n",
       "      <th>HTAG</th>\n",
       "      <th>HTR</th>\n",
       "      <th>Referee</th>\n",
       "      <th>...</th>\n",
       "      <th>HF</th>\n",
       "      <th>AF</th>\n",
       "      <th>HC</th>\n",
       "      <th>AC</th>\n",
       "      <th>HY</th>\n",
       "      <th>AY</th>\n",
       "      <th>HR</th>\n",
       "      <th>AR</th>\n",
       "      <th>Season</th>\n",
       "      <th>HW</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>19/08/00</td>\n",
       "      <td>Charlton</td>\n",
       "      <td>Man City</td>\n",
       "      <td>4.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>H</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>H</td>\n",
       "      <td>Rob Harris</td>\n",
       "      <td>...</td>\n",
       "      <td>13.0</td>\n",
       "      <td>12.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>00-01</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>19/08/00</td>\n",
       "      <td>Chelsea</td>\n",
       "      <td>West Ham</td>\n",
       "      <td>4.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>H</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>H</td>\n",
       "      <td>Graham Barber</td>\n",
       "      <td>...</td>\n",
       "      <td>19.0</td>\n",
       "      <td>14.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>00-01</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>19/08/00</td>\n",
       "      <td>Coventry</td>\n",
       "      <td>Middlesbrough</td>\n",
       "      <td>1.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>A</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>D</td>\n",
       "      <td>Barry Knight</td>\n",
       "      <td>...</td>\n",
       "      <td>15.0</td>\n",
       "      <td>21.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>00-01</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>19/08/00</td>\n",
       "      <td>Derby</td>\n",
       "      <td>Southampton</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>D</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>A</td>\n",
       "      <td>Andy D'Urso</td>\n",
       "      <td>...</td>\n",
       "      <td>11.0</td>\n",
       "      <td>13.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>00-01</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>19/08/00</td>\n",
       "      <td>Leeds</td>\n",
       "      <td>Everton</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>H</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>H</td>\n",
       "      <td>Dermot Gallagher</td>\n",
       "      <td>...</td>\n",
       "      <td>21.0</td>\n",
       "      <td>20.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>00-01</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 24 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       Date  HomeTeam       AwayTeam  FTHG  FTAG FTR  HTHG  HTAG HTR  \\\n",
       "0  19/08/00  Charlton       Man City   4.0   0.0   H   2.0   0.0   H   \n",
       "1  19/08/00   Chelsea       West Ham   4.0   2.0   H   1.0   0.0   H   \n",
       "2  19/08/00  Coventry  Middlesbrough   1.0   3.0   A   1.0   1.0   D   \n",
       "3  19/08/00     Derby    Southampton   2.0   2.0   D   1.0   2.0   A   \n",
       "4  19/08/00     Leeds        Everton   2.0   0.0   H   2.0   0.0   H   \n",
       "\n",
       "            Referee  ...    HF    AF   HC   AC   HY   AY   HR   AR  Season  \\\n",
       "0        Rob Harris  ...  13.0  12.0  6.0  6.0  1.0  2.0  0.0  0.0   00-01   \n",
       "1     Graham Barber  ...  19.0  14.0  7.0  7.0  1.0  2.0  0.0  0.0   00-01   \n",
       "2      Barry Knight  ...  15.0  21.0  8.0  4.0  5.0  3.0  1.0  0.0   00-01   \n",
       "3       Andy D'Urso  ...  11.0  13.0  5.0  8.0  1.0  1.0  0.0  0.0   00-01   \n",
       "4  Dermot Gallagher  ...  21.0  20.0  6.0  4.0  1.0  3.0  0.0  0.0   00-01   \n",
       "\n",
       "    HW  \n",
       "0  1.0  \n",
       "1  1.0  \n",
       "2  0.0  \n",
       "3  0.0  \n",
       "4  1.0  \n",
       "\n",
       "[5 rows x 24 columns]"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_model=df_clean[df_clean.Season!='03-04']\n",
    "df_model.shape\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Model Building"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The **target variable** of the model is Home Win (HW), where is being aimed to predict if there will be a Home Win for a team based on a given set of explanatory variables. \n",
    "\n",
    "First, the creation of arrays needs to be proceded for the features and the targeted variable. Not all possible variables were taking into account, but just some of them. \n",
    "\n",
    "- Not all Full Time Home Team Goals (FTHG), Full Time Away Team Goals (FTAG) and Full Time Result (FTR) were considered as these would explain 100 % of the targeted variable. \n",
    "\n",
    "- Half Time Result (HTR) was not considered, which is already given by Half Time Home Team Goals (HTHG) and Halt Time Away Team Goals (HTAG). \n",
    "\n",
    "- Home Team Yellow Cards (HY) and Away Team Yellow Cards (AY) were not considered, because these variables are having almost no impact on the model.\n",
    "\n",
    "- Also variables were not inclued, where it made no sense to include them, i.e. DateMatch, HomeTeam, AwayTeam, Referee and Season.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "y = df_clean['HW'].values\n",
    "X = df_clean[['HS','AS','HST','AST','HF','AF','HC','AC','HTHG','HTAG']].values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[17.,  8., 14., ...,  6.,  2.,  0.],\n",
       "       [17., 12., 10., ...,  7.,  1.,  0.],\n",
       "       [ 6., 16.,  3., ...,  4.,  1.,  1.],\n",
       "       ...,\n",
       "       [26.,  8., 11., ...,  0.,  1.,  2.],\n",
       "       [14., 16.,  6., ...,  4.,  1.,  2.],\n",
       "       [15., 14.,  4., ...,  6.,  1.,  0.]])"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([1., 1., 0., ..., 0., 1., 1.])"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Classification"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "from sklearn.metrics import classification_report, confusion_matrix"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Train Test Split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size = 0.3, random_state=20)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 1. Logistic Regression"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "As a first type of algorithm Logistic Regression was selected."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "logreg = LogisticRegression(solver='lbfgs')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "LogisticRegression(C=1.0, class_weight=None, dual=False, fit_intercept=True,\n",
       "                   intercept_scaling=1, l1_ratio=None, max_iter=100,\n",
       "                   multi_class='warn', n_jobs=None, penalty='l2',\n",
       "                   random_state=None, solver='lbfgs', tol=0.0001, verbose=0,\n",
       "                   warm_start=False)"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "logreg.fit(X_train,y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_pred_logreg = logreg.predict(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[843 203]\n",
      " [236 656]]\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "         0.0       0.78      0.81      0.79      1046\n",
      "         1.0       0.76      0.74      0.75       892\n",
      "\n",
      "    accuracy                           0.77      1938\n",
      "   macro avg       0.77      0.77      0.77      1938\n",
      "weighted avg       0.77      0.77      0.77      1938\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(confusion_matrix(y_test, y_pred_logreg))\n",
    "print(classification_report(y_test, y_pred_logreg))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Several settings of Train Test Split were selected and also selecting different explanatory variables and this current setting gave the best results. The algorithm has an precision of ~78% for predicting 0 (Home Team draw or loss) and ~76% for predicting 1 (Home Team wins). The model has 77% accuracy."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 2. Random Forest"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "As a second type of algorithm Random Forest was selected."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "rfc = RandomForestClassifier(n_estimators=10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "RandomForestClassifier(bootstrap=True, class_weight=None, criterion='gini',\n",
       "                       max_depth=None, max_features='auto', max_leaf_nodes=None,\n",
       "                       min_impurity_decrease=0.0, min_impurity_split=None,\n",
       "                       min_samples_leaf=1, min_samples_split=2,\n",
       "                       min_weight_fraction_leaf=0.0, n_estimators=10,\n",
       "                       n_jobs=None, oob_score=False, random_state=None,\n",
       "                       verbose=0, warm_start=False)"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rfc.fit(X_train,y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[848 198]\n",
      " [293 599]]\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "         0.0       0.74      0.81      0.78      1046\n",
      "         1.0       0.75      0.67      0.71       892\n",
      "\n",
      "    accuracy                           0.75      1938\n",
      "   macro avg       0.75      0.74      0.74      1938\n",
      "weighted avg       0.75      0.75      0.75      1938\n",
      "\n"
     ]
    }
   ],
   "source": [
    "y_pred_rfc = rfc.predict(X_test)\n",
    "print(confusion_matrix(y_test, y_pred_rfc))\n",
    "print(classification_report(y_test, y_pred_rfc))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Several settings of Random Forest (n_estimators) were treid and this current setting gave the best results. The algorithm has a precision of ~75% for predicting 0 (Home Team draw or loss) and ~75% for predicting 1 (Home Team wins). The model has 75% accuracy."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 3. K-Nearest Neighbors"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "As a third type algorithm K-Nearest Neighbors was selected."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "knn = KNeighborsClassifier(n_neighbors=10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "KNeighborsClassifier(algorithm='auto', leaf_size=30, metric='minkowski',\n",
       "                     metric_params=None, n_jobs=None, n_neighbors=10, p=2,\n",
       "                     weights='uniform')"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "knn.fit(X_train,y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[789 257]\n",
      " [419 473]]\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "         0.0       0.65      0.75      0.70      1046\n",
      "         1.0       0.65      0.53      0.58       892\n",
      "\n",
      "    accuracy                           0.65      1938\n",
      "   macro avg       0.65      0.64      0.64      1938\n",
      "weighted avg       0.65      0.65      0.65      1938\n",
      "\n"
     ]
    }
   ],
   "source": [
    "y_pred_knn = knn.predict(X_test)\n",
    "print(confusion_matrix(y_test, y_pred_knn))\n",
    "print(classification_report(y_test, y_pred_knn))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "K-Nearest Neighbors algorithm gives a precision of 65% for predicting 0 (Home Team draw or loss) and 65% for predicting 1 (Home Team wins). The model has 65% accuracy. This result is the worst out of the three algorithms."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Cross-validation"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "In the following part a Cross-validation with cross_val_score method and was performed followd by kfold split."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0.76256767, 0.77012384, 0.76470588, 0.77941176, 0.77691712])"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.model_selection import cross_val_score\n",
    "scores = cross_val_score(logreg, X, y, cv=5)\n",
    "scores"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.77 (+/- 0.01)\n"
     ]
    }
   ],
   "source": [
    "print(\"Accuracy: %0.2f (+/- %0.2f)\" % (scores.mean(), scores.std() * 2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[17. 12. 10.  5. 19. 14.  7.  7.  1.  0.]\n",
      " [ 6. 16.  3.  9. 15. 21.  8.  4.  1.  1.]\n",
      " [ 5.  5.  4.  3. 12. 12.  5.  4.  0.  0.]\n",
      " [16.  3. 10.  2.  8.  8.  6.  1.  0.  0.]\n",
      " [ 8. 14.  2.  7. 10. 21.  2.  9.  0.  0.]]\n",
      "[[17.  8. 14.  4. 13. 12.  6.  6.  2.  0.]\n",
      " [17. 12. 10.  5. 19. 14.  7.  7.  1.  0.]\n",
      " [ 6. 16.  3.  9. 15. 21.  8.  4.  1.  1.]\n",
      " [ 6. 13.  4.  6. 11. 13.  5.  8.  1.  2.]\n",
      " [17. 12.  8.  6. 21. 20.  6.  4.  2.  0.]]\n",
      "[[17.  8. 14.  4. 13. 12.  6.  6.  2.  0.]\n",
      " [ 6. 13.  4.  6. 11. 13.  5.  8.  1.  2.]\n",
      " [17. 12.  8.  6. 21. 20.  6.  4.  2.  0.]\n",
      " [ 5.  5.  4.  3. 12. 12.  5.  4.  0.  0.]\n",
      " [ 8. 14.  2.  7. 10. 21.  2.  9.  0.  0.]]\n"
     ]
    }
   ],
   "source": [
    "from sklearn.model_selection import KFold\n",
    "\n",
    "kfold = KFold(n_splits=3, shuffle=True, random_state=12)\n",
    "\n",
    "for train, test in kfold.split(X):\n",
    "    X_train, X_test = X[train], X[test]\n",
    "    y_train, y_test = y[train], y[test]\n",
    "    print(X_train[:5])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Hyperparameter tuning"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "In order to further improve the accuracy of the two most accurate classifiers, Logistic Regression and Random Forest, Hyperparameter tuning was applied in order to find the optimal set of hyperparameters for the learning algorithms."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Hyperparameter tuning for Random Forest"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import GridSearchCV\n",
    "from sklearn.svm import SVC\n",
    "from sklearn.model_selection import RandomizedSearchCV\n",
    "from pprint import pprint"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'bootstrap': [True, False],\n",
      " 'max_depth': [10, 20, 30, 40, 50, 60, 70, 80, 90, 100, 110, None],\n",
      " 'max_features': ['auto', 'sqrt'],\n",
      " 'min_samples_leaf': [1, 2, 4],\n",
      " 'min_samples_split': [2, 5, 10],\n",
      " 'n_estimators': [200, 400, 600, 800, 1000, 1200, 1400, 1600, 1800, 2000]}\n"
     ]
    }
   ],
   "source": [
    "# Number of trees in random forest\n",
    "n_estimators = [int(x) for x in np.linspace(start = 200, stop = 2000, num = 10)]\n",
    "# Number of features to consider at every split\n",
    "max_features = ['auto', 'sqrt']\n",
    "# Maximum number of levels in tree\n",
    "max_depth = [int(x) for x in np.linspace(10, 110, num = 11)]\n",
    "max_depth.append(None)\n",
    "# Minimum number of samples required to split a node\n",
    "min_samples_split = [2, 5, 10]\n",
    "# Minimum number of samples required at each leaf node\n",
    "min_samples_leaf = [1, 2, 4]\n",
    "# Method of selecting samples for training each tree\n",
    "bootstrap = [True, False]\n",
    "# Create the random grid\n",
    "random_grid = {'n_estimators': n_estimators,\n",
    "               'max_features': max_features,\n",
    "               'max_depth': max_depth,\n",
    "               'min_samples_split': min_samples_split,\n",
    "               'min_samples_leaf': min_samples_leaf,\n",
    "               'bootstrap': bootstrap}\n",
    "pprint(random_grid)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 3 folds for each of 100 candidates, totalling 300 fits\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Using backend LokyBackend with 4 concurrent workers.\n",
      "[Parallel(n_jobs=-1)]: Done  33 tasks      | elapsed:  2.6min\n",
      "[Parallel(n_jobs=-1)]: Done 154 tasks      | elapsed: 10.9min\n",
      "[Parallel(n_jobs=-1)]: Done 300 out of 300 | elapsed: 20.2min finished\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "RandomizedSearchCV(cv=3, error_score='raise-deprecating',\n",
       "                   estimator=RandomForestRegressor(bootstrap=True,\n",
       "                                                   criterion='mse',\n",
       "                                                   max_depth=None,\n",
       "                                                   max_features='auto',\n",
       "                                                   max_leaf_nodes=None,\n",
       "                                                   min_impurity_decrease=0.0,\n",
       "                                                   min_impurity_split=None,\n",
       "                                                   min_samples_leaf=1,\n",
       "                                                   min_samples_split=2,\n",
       "                                                   min_weight_fraction_leaf=0.0,\n",
       "                                                   n_estimators='warn',\n",
       "                                                   n_jobs=None, oob_score=False,\n",
       "                                                   random_sta...\n",
       "                   param_distributions={'bootstrap': [True, False],\n",
       "                                        'max_depth': [10, 20, 30, 40, 50, 60,\n",
       "                                                      70, 80, 90, 100, 110,\n",
       "                                                      None],\n",
       "                                        'max_features': ['auto', 'sqrt'],\n",
       "                                        'min_samples_leaf': [1, 2, 4],\n",
       "                                        'min_samples_split': [2, 5, 10],\n",
       "                                        'n_estimators': [200, 400, 600, 800,\n",
       "                                                         1000, 1200, 1400, 1600,\n",
       "                                                         1800, 2000]},\n",
       "                   pre_dispatch='2*n_jobs', random_state=42, refit=True,\n",
       "                   return_train_score=False, scoring=None, verbose=2)"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Use the random grid to search for best hyperparameters\n",
    "# Importing RandomForestRegressor\n",
    "from sklearn.ensemble import RandomForestRegressor\n",
    "# First create the base model to tune\n",
    "rf = RandomForestRegressor()\n",
    "# Random search of parameters, using 3 fold cross validation, \n",
    "# search across 100 different combinations, and use all available cores\n",
    "rf_random = RandomizedSearchCV(estimator = rf, param_distributions = random_grid, n_iter = 100, cv = 3, verbose=2, random_state=42, n_jobs = -1)\n",
    "# Fit the random search model\n",
    "rf_random.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'n_estimators': 400,\n",
       " 'min_samples_split': 2,\n",
       " 'min_samples_leaf': 4,\n",
       " 'max_features': 'sqrt',\n",
       " 'max_depth': 10,\n",
       " 'bootstrap': True}"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rf_random.best_params_"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### After founding the best parameters for Random Forest,  the algorithm was rerun."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "rfc1 = RandomForestClassifier(n_estimators = 1600, min_samples_split = 2, min_samples_leaf = 4, max_features = 'sqrt', max_depth = 10, bootstrap = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "RandomForestClassifier(bootstrap=True, class_weight=None, criterion='gini',\n",
       "                       max_depth=10, max_features='sqrt', max_leaf_nodes=None,\n",
       "                       min_impurity_decrease=0.0, min_impurity_split=None,\n",
       "                       min_samples_leaf=4, min_samples_split=2,\n",
       "                       min_weight_fraction_leaf=0.0, n_estimators=1600,\n",
       "                       n_jobs=None, oob_score=False, random_state=None,\n",
       "                       verbose=0, warm_start=False)"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rfc1.fit(X_train,y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[918 225]\n",
      " [287 723]]\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "         0.0       0.76      0.80      0.78      1143\n",
      "         1.0       0.76      0.72      0.74      1010\n",
      "\n",
      "    accuracy                           0.76      2153\n",
      "   macro avg       0.76      0.76      0.76      2153\n",
      "weighted avg       0.76      0.76      0.76      2153\n",
      "\n"
     ]
    }
   ],
   "source": [
    "y_pred_rfc1 = rfc1.predict(X_test)\n",
    "print(confusion_matrix(y_test, y_pred_rfc1))\n",
    "print(classification_report(y_test, y_pred_rfc1))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "With setting found parameters, Random Forest algorithm gets a slighty better result with a precision of ~76% for predicting 0 (Home Team draw or loss) and ~77% for predicting 1 (Home Team wins). The model has 76% accuracy."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Hyperparameter tuning for Logistic Regression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "c_space = np.logspace(-5, 8, 15)\n",
    "param_grid1 = {'C': c_space}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "GridSearchCV(cv=5, error_score='raise-deprecating',\n",
       "             estimator=LogisticRegression(C=1.0, class_weight=None, dual=False,\n",
       "                                          fit_intercept=True,\n",
       "                                          intercept_scaling=1, l1_ratio=None,\n",
       "                                          max_iter=100, multi_class='warn',\n",
       "                                          n_jobs=None, penalty='l2',\n",
       "                                          random_state=None, solver='lbfgs',\n",
       "                                          tol=0.0001, verbose=0,\n",
       "                                          warm_start=False),\n",
       "             iid='warn', n_jobs=None,\n",
       "             param_grid={'C': array([1.00000000e-05, 8.48342898e-05, 7.19685673e-04, 6.10540230e-03,\n",
       "       5.17947468e-02, 4.39397056e-01, 3.72759372e+00, 3.16227766e+01,\n",
       "       2.68269580e+02, 2.27584593e+03, 1.93069773e+04, 1.63789371e+05,\n",
       "       1.38949549e+06, 1.17876863e+07, 1.00000000e+08])},\n",
       "             pre_dispatch='2*n_jobs', refit=True, return_train_score=False,\n",
       "             scoring=None, verbose=0)"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Tuned Logistic Regression Parameter: {'C': 3.727593720314938}\n",
      "Tuned Logistic Regression Accuracy: 0.7761783143719526\n"
     ]
    }
   ],
   "source": [
    "logreg_cv1 = GridSearchCV(logreg,param_grid1,cv=5)\n",
    "logreg_cv1.fit(X_train,y_train)\n",
    "print(\"Tuned Logistic Regression Parameter: {}\".format(logreg_cv1.best_params_))\n",
    "print(\"Tuned Logistic Regression Accuracy: {}\".format(logreg_cv1.best_score_))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### After founding the best parameter C, the algorithm was rerun."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "logreg1 = LogisticRegression(solver='lbfgs', C = 3.727593720314938)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "LogisticRegression(C=3.727593720314938, class_weight=None, dual=False,\n",
       "                   fit_intercept=True, intercept_scaling=1, l1_ratio=None,\n",
       "                   max_iter=100, multi_class='warn', n_jobs=None, penalty='l2',\n",
       "                   random_state=None, solver='lbfgs', tol=0.0001, verbose=0,\n",
       "                   warm_start=False)"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "logreg1.fit(X_train,y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_pred_logreg1 = logreg1.predict(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[907 236]\n",
      " [278 732]]\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "         0.0       0.77      0.79      0.78      1143\n",
      "         1.0       0.76      0.72      0.74      1010\n",
      "\n",
      "    accuracy                           0.76      2153\n",
      "   macro avg       0.76      0.76      0.76      2153\n",
      "weighted avg       0.76      0.76      0.76      2153\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(confusion_matrix(y_test, y_pred_logreg1))\n",
    "print(classification_report(y_test, y_pred_logreg1))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "With setting found parameter C, almost the same results as without specifically setting the parameter C were presented with a precision of ~77% for predicting 0 (Home Team draw or loss) and ~76% for predicting 1 (Home Team wins). The model has 76% accuracy."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## ROC Curve"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "A Receiver Operator Characteristic (ROC) curve is a graphical plot used to show the diagnostic ability of binary classifiers. Therefore, it is being applied to the best two algorithms used, Logistic Regression and Random Forest, to better visualize their performance."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Logistic Regression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import roc_curve\n",
    "from sklearn.metrics import roc_auc_score\n",
    "from sklearn.model_selection import cross_val_score\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<matplotlib.lines.Line2D at 0x1a245b6048>]"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "[<matplotlib.lines.Line2D at 0x1a222a87f0>]"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "Text(0.5, 0, 'False Positive Rate')"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "Text(0, 0.5, 'True Positive Rate')"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "Text(0.5, 1.0, 'ROC Curve')"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEWCAYAAABrDZDcAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjAsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+17YcXAAAgAElEQVR4nO3deZgU5dX38e9xxn0hCmrYd4EB0ZARAgioKIqCa1CUoOjACIioaNxF5VECCKIoq6ggbqgBxYQnxEdj9DUCoggCiozsuLAIKC4Iw3n/6B7TjrP0wFTXdPfvc11z2VVd3XVqGOvUfddd5zZ3R0RE0td+YQcgIiLhUiIQEUlzSgQiImlOiUBEJM0pEYiIpDklAhGRNKdEICKS5pQIJKWY2Woz+8HMdpjZl2Y2xcwOK7RNGzN7w8y+NbPtZvaqmWUV2uYIM3vIzNZGvysvulylmP2amQ00syVm9p2ZrTezF83s+CCPV6Q8KBFIKurq7ocBJwK/A24reMPMWgP/BF4BqgF1gUXAO2ZWL7rNAcDrQFPgLOAIoA2wBWhZzD4fBq4DBgJHAccBLwPnlDV4M8ss62dE9oXpyWJJJWa2Gujt7v8XXR4BNHX3c6LLbwMfuXv/Qp/7X2CTu19uZr2B+4H67r4jjn02BD4BWrv7/GK2eRN42t0nR5d7ReM8ObrswADgeiATmAPscPebYr7jFeDf7v6gmVUDHgHaAzuA0e4+Jo5fkcivqEUgKcvMagCdgbzo8iFEruxfLGLzF4Azoq9PB/4RTxKI6gisLy4JlMH5QCsgC3gWuMTMDMDMjgQ6Ac+b2X7Aq0RaMtWj+7/ezM7cx/1LmlIikFT0spl9C6wDNgJ3R9cfReRv/osiPvMFUND/X7mYbYpT1u2L8xd3/9rdfwDeBhxoF33vj8C77v45cBJwtLsPcfef3H0l8BjQvRxikDSkRCCp6Hx3Pxw4BWjMf0/wW4E9QNUiPlMV2Bx9vaWYbYpT1u2Ls67ghUf6bJ8HLo2uugx4Jvq6NlDNzLYV/AC3A8eWQwyShpQIJGW5+7+BKcDI6PJ3wLtAtyI2v5jIDWKA/wPONLND49zV60ANM8suYZvvgENiln9bVMiFlp8D/mhmtYl0Gf01un4dsMrdfxPzc7i7nx1nvCK/oEQgqe4h4AwzOzG6fCtwRXSo5+FmdqSZ3Qe0Bu6NbjONyMn2r2bW2Mz2M7PKZna7mf3qZOvuK4BxwHNmdoqZHWBmB5lZdzO7NbrZh8CFZnaImTUAckoL3N0XApuAycAcd98WfWs+8I2Z3WJmB5tZhpk1M7OT9uYXJKJEICnN3TcBTwF3RZf/H3AmcCGRfv01RIaYnhw9oePuO4ncMP4EeA34hsjJtwowr5hdDQQeBcYC24DPgAuI3NQFGA38BHwFTOW/3TyleS4ay7Mxx5QPdCUyPHYVkS6tyUClOL9T5Bc0fFREJM2pRSAikuaUCERE0pwSgYhImlMiEBFJc0lX3KpKlSpep06dsMMQEUkq77///mZ3P7qo95IuEdSpU4cFCxaEHYaISFIxszXFvaeuIRGRNKdEICKS5pQIRETSnBKBiEiaUyIQEUlzgSUCM3vCzDaa2ZJi3jczGxOdFHyxmbUIKhYRESlekC2CKUQm/i5OZ6Bh9CcXGB9gLCIiUozAniNw97fMrE4Jm5wHPBWdiWmumf3GzKq6e3lM+SciUm6enbeWVz7cENr+9+zJ56efdtGi3jHc3bVpuX9/mA+UVSdmaj5gfXTdrxKBmeUSaTVQq1athAQnIhVD2CdhgHmrvgagVd2jEr7vbdu2sXz5cjIzM/ld3SIfDN5nYSYCK2JdkZMjuPskYBJAdna2JlAQSVJ7c1IP8yRcoFXdozjvxOpc1ipxF6Lbtm3jz3/+My9MnkyDBg2YPHkyHTo0C2RfYSaC9UDNmOUawOchxSIi5aC0E/3enNTDOAmHLT8/nzZt2rB8+XJuvvlm7rnnHg4++ODA9hdmIpgFDDCz54lMzL1d9wdEKp6yXMWXdqJPx5N6WWzZsoWjjjqKjIwM7r//fmrWrEl2dnbg+w0sEZjZc8ApQBUzWw/cDewP4O4TgNnA2UAe8D1wZVCxiEjR4jnJl+UqXif6vePuPPPMM1x33XUMGzaMPn36cMEFFyRs/0GOGrq0lPcduCao/YtI0WJP/vGc5HVyD9a6devo27cvs2fP5g9/+ANt27ZNeAxJV4ZaRP5rX2++6iQfrueee46rr76a/Px8HnroIQYMGEBGRkbC41AiEEkyZb2iL0wn/4rjyCOPpFWrVkyaNIm6deuGFodFemiSR3Z2tmtiGkkXRV3xFz7566SePHbv3s3o0aP56aefuOOOO4DI/QGzokbTly8ze9/di7zzrBaBSAVUkACKuuLXFX1yWrRoETk5Obz//vtcfPHFPyeARCSB0igRiFQAha/8YxOATvrJbefOndx3330MGzaMo446ihdffJGLLrqoQiSAAkoEIiEo6cRf8F8lgNSwYsUKhg8fzmWXXcaDDz5I5cqVww7pV5QIRMrJvjx4pRN/atmxYwevvPIKPXr0oFmzZnzyySfUq1cv7LCKpUQgUg6enbeW22d+BOjBq3T32muvkZuby5o1a2jRogVNmjSp0EkAlAhEyqS4q/6CK/yhFxyvk3ua2rp1KzfddBNPPPEExx13HP/+979p0qRJ2GHFRYlApBjxDN0soCv89Jafn0/btm359NNPue222xg8eDAHHXRQ2GHFTYlApAjFdfXohC+xNm/e/HORuKFDh1KrVi1atEi+WXeVCESiinpiV109UhR3Z9q0aVx//fUMGzaM3Nxczj///LDD2mtKBCL8ugWgK38pzpo1a7j66quZM2cObdq0oX379mGHtM+UCCRtqQUgZfX000/Tr18/3J1HHnmE/v37s99++4Ud1j5TIpC0UlzBNrUAJB5HH300bdu2ZeLEidSuXTvscMqNis5JSiptmKcKtkk8du3axahRo9i1axd33XUXkLgiceVNReckrZT0cJeu/CVeCxcuJCcnh4ULF9K9e/cKVSSuvCkRSMooXLFT/f2yN3788UeGDBnCiBEjqFKlCn/961+58MILww4rUEoEkpRKe9hLV/2yt/Ly8hg5ciSXX345o0aN4sgjjww7pMApEUhSKK1aZ8FrJQDZGzt27GDmzJn07NmTZs2asXz58lBnDEs0JQKpsEqaklEnfSkvc+bMITc3l3Xr1pGdnU2TJk3SKgmAEoFUMBreKYmyZcsWBg0axFNPPUXjxo15++23k6ZIXHlTIpDQlNbPr5O/BKWgSFxeXh533HEHd955Z1IViStvSgSSMOrnl7Bt2rSJypUrk5GRwfDhw6lduzYnnnhi2GGFTolAAlfcROw66UuiuDtTpkxh0KBBDBs2jKuvvprzzjsv7LAqDCUCCdwrH25g2Rff6MQvoVi9ejW5ubm89tprtGvXjlNPPTXskCocJQIJRGw30LIvviGr6hFMv7p1yFFJupk2bRr9+vXDzBg3bhxXX311ShSJK29KBFKuiuoGyqp6BOedWD3kyCQdHXvssbRv354JEyZQq5ZaosVRIpByUVQCUDeQJNquXbsYMWIE+fn5DB48mE6dOtGpU6eww6rwlAhknxUu8qYEIGH44IMPuOqqq1i0aBGXXXZZ0lYJDYMSgeyT2CSgIm8Shh9++IF7772XkSNHcvTRRzNz5syknjYyDIEmAjM7C3gYyAAmu/uwQu/XAqYCv4luc6u7zw4yJikfqvQpFcXKlSt58MEH6dWrFw888EBaFIkrb4ElAjPLAMYCZwDrgffMbJa7L4vZ7E7gBXcfb2ZZwGygTlAxSfnRkFAJ0zfffMOMGTPo1asXTZs2ZcWKFSk1Y1iiBTmOqiWQ5+4r3f0n4Hmg8BMcDhwRfV0J+DzAeKScPDtvLfNWff3zkFAlAUmk2bNn06xZM3Jycvj4448BlAT2UZBdQ9WBdTHL64FWhba5B/inmV0LHAqcXtQXmVkukAtoCFiICncHaUioJNLmzZu54YYbePrpp8nKyuKdd95J2yJx5S3IRFDU7frCEyRfCkxx91Fm1hqYZmbN3H3PLz7kPgmYBJE5iwOJVoqloaEStoIicStXrmTw4MHcfvvtHHjggWGHlTKCTATrgZoxyzX4dddPDnAWgLu/a2YHAVWAjQHGJWWgoaESpq+++oqjjz6ajIwMRo4cSe3atWnevHnYYaWcIO8RvAc0NLO6ZnYA0B2YVWibtUBHADNrAhwEbAowJonTs/PWcsnEd38xNFT3AyRR3J3HH3+cRo0aMWnSJAC6du2qJBCQwFoE7r7bzAYAc4gMDX3C3Zea2RBggbvPAm4EHjOzG4h0G/Vyd3X9hEytAAnTypUr6dOnD2+88QYdOnTg9NOLvHUo5SjQ5wiizwTMLrRucMzrZUDbIGOQ+OnZAAnb1KlT6d+/PxkZGUyYMIE+ffqoSFwC6MliAdQKkIqhWrVqnHbaaYwfP54aNWqEHU7aUCJIc2oFSJh++uknhg0bxp49e7jnnns444wzOOOMM8IOK+0oEaQxtQIkTO+99x5XXXUVS5YsoWfPnioSFyIlgjQTO2GMWgEShu+//57BgwczevRoqlatyqxZs+jatWvYYaU1JYI0UrgFoFaAhGHVqlU88sgj9OnTh+HDh1OpUqWwQ0p7SgRpQuWiJUzbt29nxowZXHnllTRt2pS8vDxq1qxZ+gclITQuK00UdAcpCUii/f3vf6dp06b07t2bTz75BEBJoIJRIkgDBdVCW9U9SklAEmbTpk306NGDLl26cOSRR/Luu+/SuHHjsMOSIqhrKMXFdgmpWqgkSn5+PieffDKrVq3i3nvv5dZbb+WAAw4IOywpRlyJIForqJa75wUcj5QTPR8gYfjyyy855phjyMjIYNSoUdSpU4dmzZqFHZaUotSuITM7B/gIeC26fKKZzQw6MNl7Ba2Agu4gJQEJ2p49e5g4cSLHHXccEydOBKBLly5KAkkinhbBECITyvwLwN0/NLMGgUYl+0Q3hiWR8vLy6NOnD2+++SannXYaZ555ZtghSRnFkwh2ufu2Qk/8qUJoBVTQHVQwl7CSgATtySefpH///hxwwAE89thj5OTk6OngJBRPIvjYzC4G9jOzusB1wNxgw5J4FfWkcMGDYiJBq1WrFmeeeSZjx46lenX9zSUrK638v5kdCgwGOkVXzQHudfcfAo6tSNnZ2b5gwYIwdl2hFDV9JKAnhSVQO3fu5C9/+Qt79uxhyJAhYYcjZWBm77t7dlHvxdMiONPdbwFuifnCC4EZ5RSfxCH2yh9+ffWvk78Ebd68eeTk5LB06VKuuOIKFYlLIfEkgjv59Un/jiLWSUAK1wgq+K8SgCTCd999x1133cVDDz1E9erV+dvf/sY555wTdlhSjopNBGZ2JpGJ5aub2YMxbx0B7Ak6MNGzAFIxrFmzhnHjxtG3b1+GDRvGEUccEXZIUs5KahFsBJYAPwJLY9Z/C9waZFDprqj+f139SyJt27aNl156id69e5OVlUVeXp5mDEthxSYCd18ILDSzZ9z9xwTGlNY0WYyE7ZVXXqFfv35s3LiRk08+mcaNGysJpLh47hFUN7P7gSzgoIKV7n5cYFGlGU0WIxXBxo0bGThwINOnT6d58+bMmjVLReLSRDyJYApwHzAS6Axcie4RlKuCh8Cyqh6hVoCEIj8/n7Zt27J27Vruu+8+br75Zvbff/+ww5IEiScRHOLuc8xspLt/BtxpZm8HHVg6iH0SOKvqEUy/unXYIUma+fzzz/ntb39LRkYGDz/8MHXq1CErKyvssCTB4pmPYKdFBgt/ZmZ9zawrcEzAcaW82MJwWVWP0JPAklB79uxh/PjxNG7cmAkTJgBw9tlnKwmkqXhaBDcAhwEDgfuBSsBVQQaV6jRtpITp008/pU+fPrz11lucfvrpdO7cOeyQJGSlJgJ3nxd9+S3QE8DMNIRgH6g6qITl8ccfZ8CAARx00EE88cQT9OrVS08HS8ldQ2Z2kpmdb2ZVostNzewpVHRur2naSAlTnTp16Ny5M8uWLePKK69UEhCghERgZn8BngF6AP8wszuIzEmwCNDQ0b2gaSMl0Xbu3Mmdd97JnXfeCUDHjh2ZMWMGVatWDTkyqUhK6ho6DzjB3X8ws6OAz6PLyxMTWupRl5Ak0n/+8x9ycnL45JNPuOqqq1QkTopVUtfQjwWlpt39a+ATJYG9py4hSZQdO3Zw3XXXcfLJJ/P999/zj3/8g8cff1xJQIpVUiKoZ2Yzoj8zgToxy3FVHjWzs8xsuZnlmVmR9YnM7GIzW2ZmS83s2b05iGRQ0BpQl5AEbe3atUycOJFrrrmGJUuWaOpIKVVJXUMXFVp+tCxfbGYZwFjgDGA98J6ZzXL3ZTHbNARuA9q6+1YzS7nnEzR9pCTC1q1befHFF8nNzSUrK4uVK1dSrVq1sMOSJFFS0bnX9/G7WwJ57r4SwMyeJ3LfYVnMNn2Ase6+NbrPjfu4zwon9slhtQYkCDNnzqR///5s2rSJDh060KhRIyUBKZN4nizeW9WBdTHL66PrYh0HHGdm75jZXDM7q6gvMrNcM1tgZgs2bdoUULjl69l5a7lk4ru/KB+h1oCUpy+//JJu3bpx4YUX8tvf/pb58+fTqFGjsMOSJBTPk8V7q6g7U4UnSM4EGgKnADWAt82smbtv+8WH3CcBkyAyZ3H5h1q+iiolLVKe8vPzadeuHevWrWPo0KHcdNNNKhIney3uRGBmB7r7zjJ893qgZsxyDSJDUAtvM9fddwGrzGw5kcTwXhn2U+FomKgEZf369VSrVo2MjAzGjBlD3bp1VSpa9lmpXUNm1tLMPgJWRJdPMLNH4vju94CGZlbXzA4AugOzCm3zMnBq9HurEOkqWlmG+Css3RiW8rRnzx4eeeQRGjduzPjx4wHo3LmzkoCUi3juEYwBugBbANx9EdGTd0ncfTcwAJgDfAy84O5LzWyImZ0b3WwOsMXMlhF5avnP7r6l7IdRcRQ8LyBSXj755BPat2/PwIEDOfnkk+nSpUvYIUmKiadraD93X1PoYZT8eL7c3WcDswutGxzz2oFB0Z+kpxISUt4mT57MgAEDOOSQQ5g6dSo9e/bUg2FS7uJJBOvMrCXg0WcDrgU+DTas5KR7A1Le6tevT9euXXn00Uc59thjww5HUlQ8iaAfke6hWsBXwP9F10kMlZCQ8vDjjz8yZMgQAIYOHcqpp57KqaeW2hMrsk/iSQS73b174JEkMXUJSXl45513yMnJYfny5fTu3VtF4iRh4rlZ/J6ZzTazK8zs8MAjSkLqEpJ98e2333LttdfSrl07du7cyZw5c3jssceUBCRhSk0E7l4fuA/4PfCRmb1sZmohFKIuIdlb69evZ/LkyVx77bV89NFHdOrUKeyQJM3EVWLC3f/j7gOBFsA3RCasETRcVPbOli1bfn4eoEmTJqxcuZKHH36Yww47LOTIJB3F80DZYWbWw8xeBeYDm4A2gUeWBHRvQMrK3XnppZfIyspi4MCBLF8emeJDM4ZJmOJpESwB/gCMcPcG7n5jzIT2aU33BqQsvvjiCy666CK6detGzZo1WbBggYrESYUQz6iheu6+J/BIkpTuDUg8CorEbdiwgREjRnDDDTeQmRlkzUeR+BX7l2hmo9z9RuCvZvarip/ufmGgkVVwsc8NiBRn3bp1VK9enYyMDMaOHUvdunU57rjjwg5L5BdKuiSZHv1vmWYmSwe6NyClyc/PZ+zYsdx2222MGDGCa665RlNGSoVV0gxl86Mvm7j7L5KBmQ0A9nUGs6SlewNSko8//picnBzeffddOnfuTNeuXcMOSaRE8dwsvqqIdTnlHUiyUCkJKcmkSZM48cQT+fTTT5k2bRp///vfqVVLfydSsZV0j+ASInMI1DWzGTFvHQ5sK/pTqU1dQlKahg0bcsEFFzBmzBiOOeaYsMMRiUtJ9wjmE5mDoAYwNmb9t8DCIIOqqNQlJIX98MMP3HPPPZgZw4YNU5E4SUol3SNYBawiUm1UotQlJAXeeustevfuzYoVK+jbt6+KxEnSKvYegZn9O/rfrWb2dczPVjNTTQVJW9988w39+/enQ4cO5Ofn8/rrrzN+/HglAUlaJXUNFbRvqyQiEJFk8fnnnzNlyhQGDRrEkCFDOPTQQ8MOSWSfFNsiiHmauCaQ4e75QGvgaiDt/vJVXC69bd68mXHjxgHQuHFjVq1axahRo5QEJCXEM3z0ZSLTVNYHngKaAM8GGlUFo9FC6cvdmT59OllZWVx//fV8+mlkllZNGympJJ5EsMfddwEXAg+5+7VAWp0NNVooPX3++eecf/75dO/endq1a/P++++rPISkpLimqjSzbkBP4Pzouv2DC6li0QNk6Sk/P5/27duzYcMGRo4cyXXXXacicZKy4vnLvgroT6QM9Uozqws8F2xYFUdBa0BdQulhzZo11KhRg4yMDMaNG0e9evVo0KBB2GGJBCqeqSqXAAOBBWbWGFjn7vcHHlkFotZA6svPz+fBBx+kSZMmP88c1qlTJyUBSQultgjMrB0wDdgAGPBbM+vp7u8EHZxIIixZsoScnBzmz59Ply5dOP/880v/kEgKiadraDRwtrsvAzCzJkQSQ3aQgYkkwoQJExg4cCCVKlXi2WefpXv37nowTNJOPKOGDihIAgDu/jFwQHAhiQTPPTLXUpMmTejWrRvLli3j0ksvVRKQtBRPi+ADM5tIpBUA0IM0LTonye/7779n8ODBZGRkMHz4cDp06ECHDh3CDkskVPG0CPoCnwE3A7cAK4k8XZzy9DRxannzzTdp3rw5o0aNYseOHT+3CkTSXYktAjM7HqgPzHT3EYkJqeLQ0NHUsH37dm6++WYmTZpE/fr1eeONN1QqWiRGSdVHbydSXqIH8JqZFTVTWcrSg2Sp44svvuDpp5/mpptuYvHixUoCIoWU1DXUA2ju7t2Ak4B+Zf1yMzvLzJabWZ6Z3VrCdn80MzezCjESSbWFkt+mTZt45JFHgEiRuNWrV/PAAw9wyCGHhByZSMVTUiLY6e7fAbj7plK2/RUzyyAys1lnIAu41MyyitjucCIPrM0ry/cHSbWFkpe78+yzz9KkSRNuvPHGn4vEHX300SFHJlJxlXRyr2dmM6I/M4H6McszSvhcgZZAnruvdPefgOeB84rY7n+AEcCPZY4+QOoSSj7r1q2ja9eu9OjRgwYNGrBw4UIViROJQ0k3iy8qtPxoGb+7OrAuZnk90Cp2AzP7HVDT3f9mZjcV90VmlgvkAtSqFezJOfbegCSP3bt3c8opp/Dll18yevRorr32WjIyMsIOSyQplDRn8ev7+N1FPZnz83g9M9uPyFPLvUr7InefBEwCyM7ODnTMn0YKJZfVq1dTs2ZNMjMzmThxIvXq1aNevXphhyWSVMrU719G64nMblagBvB5zPLhQDPgTTNbDfwBmBXmDWONFEoeu3fvZuTIkTRp0uTnmcNOP/10JQGRvRBkgfX3gIbRstUbgO7AZQVvuvt2YuZDNrM3gZvcfUGAMZVIrYHksHjxYnJycliwYAHnnXceF11UuBdTRMoi7haBmR1Yli92993AAGAO8DHwgrsvNbMhZnZu2cJMHLUGKrZx48bx+9//njVr1jB9+nRmzpxJtWrVwg5LJKnFU4a6JfA4UAmoZWYnAL2jU1aWyN1nA7MLrRtczLanxBOwpCd3x8xo1qwZ3bt3Z/To0VSpUqX0D4pIqeLpGhoDdCHylDHuvsjM9GimJMR3333HnXfeSWZmJg888ADt27enffv2YYclklLi6Rraz93XFFqXH0QwIrFef/11jj/+eB566CF27typInEiAYknEayLdg+5mWWY2fXApwHHlXCqNFpxbNu2jd69e3P66aeTmZnJW2+9xZgxYzRXgEhA4kkE/YBBQC3gKyLDPMtcd6ii04ihiuOrr77i+eef55ZbbmHRokW0a9cu7JBEUlqp9wjcfSORoZ8pTyOGwlNw8r/uuuto1KgRq1ev1s1gkQSJZ9TQY8Q8EVzA3XMDiSgEKisRHnfnmWee4brrrmPHjh2cffbZNGzYUElAJIHi6Rr6P+D16M87wDHAziCDSiSVnA7P2rVrOeecc+jZsyeNGjXiww8/pGHDhmGHJZJ24ukamh67bGbTgNcCiyjBVHI6HAVF4jZu3MiYMWPo37+/isSJhGRvSkzUBWqXdyBh0r2BxFm5ciW1a9cmMzOTxx57jPr161OnTp2wwxJJa6V2DZnZVjP7OvqzjUhr4PbgQwuehowmzu7duxk+fDhZWVmMHTsWgI4dOyoJiFQApU1eb8AJRIrGAezxFHqqR0NGE+PDDz8kJyeHDz74gAsuuIBu3bqFHZKIxCixRRA96c909/zoT8okgQLqFgrWo48+ykknncSGDRt46aWXmDFjBlWrVg07LBGJEc+ooflm1iLwSCSlFFwzNG/enB49erBs2TKVixapoIrtGjKzzGgp6ZOBPmb2GfAdkZnH3N2VHORXduzYwR133MH+++/PyJEjVSROJAmUdI9gPtACOD9BsUiS++c//0lubi5r167l2muv/bl0tIhUbCUlAgNw988SFIskqa1btzJo0CCmTJlCo0aNeOuttzj55JPDDktE4lRSIjjazAYV96a7PxhAPJKENm7cyEsvvcRtt93G4MGDOeigg8IOSUTKoKREkAEcRrRlIBLryy+/5LnnnuOGG274uUhc5cqVww5LRPZCSYngC3cfkrBIEkyF5vaOu/PUU09xww038P3339OlSxcaNmyoJCCSxEoaPprSLQE9TFZ2q1ev5qyzzqJXr15kZWWpSJxIiiipRdAxYVGERA+TxW/37t2ceuqpbN68mbFjx9K3b1/22y+ex1BEpKIrNhG4u4rwCHl5edStW5fMzEyeeOIJ6tWrR+3aKVVzUCTtpeUlnYrNlW7Xrl0MHTqUpk2b/lwk7tRTT1USEElBe1OGOunp/kDJPvjgA3Jycvjwww/p1q0bl1xySdghiUiA0rJFALo/UJwxY8bQsmVLvvzyS2bMmMELL7zAscceG3ZYIhKgtE0E8ksFReJ+940IwKQAAA9qSURBVLvfcfnll7Ns2TIuuOCCkKMSkURIy64h+a9vv/2W2267jQMPPJBRo0bRrl072rVrF3ZYIpJAahGksX/84x80a9aMcePG4e6k4HQTIhIHJYI0tGXLFq644go6d+7MoYceyjvvvMODDz6oSqEiaSrtEoGGjkYSwcyZM7nrrrtYuHAhrVu3DjskEQlRoInAzM4ys+Vmlmdmtxbx/iAzW2Zmi83sdTMLfJB6ug4d/eKLLxg5ciTuznHHHceaNWsYMmQIBx54YNihiUjIAksEZpYBjAU6A1nApWaWVWizhUC2uzcHXgJGBBVPrHQaOuruPPHEEzRp0oS77rqLvLw8AI488siQIxORiiLIFkFLIM/dV7r7T8DzwHmxG7j7v9z9++jiXKBGgPGkXbfQqlWr6NSpEzk5OZxwwgksWrRIReJE5FeCHD5aHVgXs7weaFXC9jnA/xb1hpnlArkAtWrt/ZV8OnUL7d69m9NOO40tW7Ywfvx4cnNzVSRORIoUZCIoaghKkeMTzexPQDbQoaj33X0SMAkgOzt7n8Y4pnq30IoVK6hXrx6ZmZk8+eST1K9fn5o1a4YdlohUYEFeIq4HYs9ANYDPC29kZqcDdwDnuvvOAONJabt27eK+++6jWbNmPProowCccsopSgIiUqogWwTvAQ3NrC6wAegOXBa7gZn9DpgInOXuGwOMJaUtWLCAnJwcFi9eTPfu3bn00kvDDklEkkhgLQJ33w0MAOYAHwMvuPtSMxtiZudGN3uAyLzIL5rZh2Y2K6h4UtXDDz9Mq1at2Lx5M6+88grPPfccxxxzTNhhiUgSCbTWkLvPBmYXWjc45vXpQe4/lbk7ZkZ2djY5OTmMGDGC3/zmN2GHJSJJSEXnksw333zDLbfcwkEHHcTo0aNp27Ytbdu2DTssEUliGk+YRGbPnk3Tpk2ZNGkSmZmZKhInIuVCiSAJbN68mT/96U+cc845VKpUif/85z888MADKhInIuVCiSAJbN26lVdffZW7776bDz74gFatSnouT0SkbHSPoILasGEDzzzzDH/+859p2LAha9as0c1gEQmEWgQVjLvz2GOPkZWVxT333MNnn30GoCQgIoFRIqhAPvvsMzp27Ehubi4tWrRg8eLFNGjQIOywRCTFqWuogti9ezcdO3bk66+/ZuLEifTu3VtF4kQkIZQIQrZ8+XLq169PZmYmU6dOpX79+tSoEWg1bhGRX9AlZ0h++ukn7r33Xo4//njGjh0LQIcOHZQERCTh1CIIwfz588nJyWHJkiVcdtll9OjRI+yQRCSNqUWQYA899BCtW7f++dmAZ555hipVqoQdloikMSWCBCkoB9GyZUv69OnD0qVL6dKlS8hRiYioayhw27dv5+abb+bggw/moYceok2bNrRp0ybssEREfqYWQYBeffVVsrKymDx5MgceeKCKxIlIhaREEIBNmzZx2WWXce6551K5cmXmzp3L8OHDVSRORCokJYIAbN++ndmzZ3PvvfeyYMECTjrppLBDEhEplu4RlJN169bx9NNPc+utt9KgQQPWrFlDpUqVwg5LRKRUahHsoz179jBhwgSaNm3Kfffd93OROCUBEUkWaZMInp23lnmrvi7X71yxYgWnnXYa/fr1o2XLlnz00UcqEiciSSdtuoZe+XADAOedWL1cvm/37t2cccYZbNu2jccff5wrr7xSN4NFJCmlTSIAaFX3KC5rVWufvuPjjz+mYcOGZGZmMm3aNOrXr0+1atXKKUIRkcRLm66hfbVz507uvvtumjdvzqOPPgpAu3btlAREJOmlVYtgb82dO5ecnByWLVtGz5496dmzZ9ghiYiUG7UISjFq1CjatGnDt99+y+zZs3nqqaeoXLly2GGJiJQbJYJi7NmzB4DWrVvTt29flixZQufOnUOOSkSk/KlrqJBt27Zx4403csghh/DII4+oSJyIpDy1CGK8/PLLZGVlMXXqVA4//HAViRORtKBEAGzcuJGLL76YCy64gGOPPZb58+czdOhQPRcgImlBiQD45ptveO2117j//vuZP38+LVq0CDskEZGESdt7BGvXrmXatGncfvvtNGjQgLVr13L44YeHHZaISMIF2iIws7PMbLmZ5ZnZrUW8f6CZTY++P8/M6gQZD0RGA40bN46mTZsydOjQn4vEKQmISLoKLBGYWQYwFugMZAGXmllWoc1ygK3u3gAYDQwPKh6AH374nlNOOYVrrrmG1q1bs3TpUhWJE5G0F2SLoCWQ5+4r3f0n4HngvELbnAdMjb5+CehoAd2hdXcWL17MRx99xJNPPsmcOXOoU6dOELsSEUkqQd4jqA6si1leD7Qqbht3321m24HKwObYjcwsF8gFqFVr74rGNa1eiSNbNeOe+5dRtWrVvfoOEZFUFGQiKOrKvvDA/Hi2wd0nAZMAsrOz92pw/91dmwJN9+ajIiIpLciuofVAzZjlGsDnxW1jZplAJaB8Z48REZESBZkI3gMamlldMzsA6A7MKrTNLOCK6Os/Am+4HucVEUmowLqGon3+A4A5QAbwhLsvNbMhwAJ3nwU8DkwzszwiLYHuQcUjIiJFC/SBMnefDcwutG5wzOsfgW5BxiAiIiVTiQkRkTSnRCAikuaUCERE0pwSgYhImrNkG61pZpuANXv58SoUemo5DeiY04OOOT3syzHXdveji3oj6RLBvjCzBe6eHXYciaRjTg865vQQ1DGra0hEJM0pEYiIpLl0SwSTwg4gBDrm9KBjTg+BHHNa3SMQEZFfS7cWgYiIFKJEICKS5lIyEZjZWWa23MzyzOzWIt4/0MymR9+fZ2Z1Eh9l+YrjmAeZ2TIzW2xmr5tZ7TDiLE+lHXPMdn80MzezpB9qGM8xm9nF0X/rpWb2bKJjLG9x/G3XMrN/mdnC6N/32WHEWV7M7Akz22hmS4p538xsTPT3sdjMWuzzTt09pX6IlLz+DKgHHAAsArIKbdMfmBB93R2YHnbcCTjmU4FDoq/7pcMxR7c7HHgLmAtkhx13Av6dGwILgSOjy8eEHXcCjnkS0C/6OgtYHXbc+3jM7YEWwJJi3j8b+F8iMzz+AZi3r/tMxRZBSyDP3Ve6+0/A88B5hbY5D5gaff0S0NHMipo2M1mUeszu/i93/z66OJfIjHHJLJ5/Z4D/AUYAPyYyuIDEc8x9gLHuvhXA3TcmOMbyFs8xO3BE9HUlfj0TYlJx97coeabG84CnPGIu8Bsz26eJ2FMxEVQH1sUsr4+uK3Ibd98NbAcqJyS6YMRzzLFyiFxRJLNSj9nMfgfUdPe/JTKwAMXz73wccJyZvWNmc83srIRFF4x4jvke4E9mtp7I/CfXJia00JT1//dSBToxTUiKurIvPEY2nm2SSdzHY2Z/ArKBDoFGFLwSj9nM9gNGA70SFVACxPPvnEmke+gUIq2+t82smbtvCzi2oMRzzJcCU9x9lJm1JjLrYTN33xN8eKEo9/NXKrYI1gM1Y5Zr8Oum4s/bmFkmkeZkSU2xii6eY8bMTgfuAM51950Jii0opR3z4UAz4E0zW02kL3VWkt8wjvdv+xV33+Xuq4DlRBJDsornmHOAFwDc/V3gICLF2VJVXP+/l0UqJoL3gIZmVtfMDiByM3hWoW1mAVdEX/8ReMOjd2GSVKnHHO0mmUgkCSR7vzGUcszuvt3dq7h7HXevQ+S+yLnuviCccMtFPH/bLxMZGICZVSHSVbQyoVGWr3iOeS3QEcDMmhBJBJsSGmVizQIuj44e+gOw3d2/2JcvTLmuIXffbWYDgDlERhw84e5LzWwIsMDdZwGPE2k+5hFpCXQPL+J9F+cxPwAcBrwYvS++1t3PDS3ofRTnMaeUOI95DtDJzJYB+cCf3X1LeFHvmziP+UbgMTO7gUgXSa9kvrAzs+eIdO1Vid73uBvYH8DdJxC5D3I2kAd8D1y5z/tM4t+XiIiUg1TsGhIRkTJQIhARSXNKBCIiaU6JQEQkzSkRiIikOSUCqXDMLN/MPoz5qVPCtnWKq9JYxn2+Ga1wuShanqHRXnxHXzO7PPq6l5lVi3lvsplllXOc75nZiXF85nozO2Rf9y2pS4lAKqIf3P3EmJ/VCdpvD3c/gUhBwgfK+mF3n+DuT0UXewHVYt7r7e7LyiXK/8Y5jvjivB5QIpBiKRFIUohe+b9tZh9Ef9oUsU1TM5sfbUUsNrOG0fV/ilk/0cwyStndW0CD6Gc7RuvcfxStE39gdP0w++/8DiOj6+4xs5vM7I9E6jk9E93nwdEr+Wwz62dmI2Ji7mVmj+xlnO8SU2zMzMab2QKLzENwb3TdQCIJ6V9m9q/ouk5m9m709/iimR1Wyn4kxSkRSEV0cEy30Mzouo3AGe7eArgEGFPE5/oCD7v7iUROxOujJQcuAdpG1+cDPUrZf1fgIzM7CJgCXOLuxxN5Er+fmR0FXAA0dffmwH2xH3b3l4AFRK7cT3T3H2Lefgm4MGb5EmD6XsZ5FpGSEgXucPdsoDnQwcyau/sYInVoTnX3U6NlJ+4ETo/+LhcAg0rZj6S4lCsxISnhh+jJMNb+wKPRPvF8IjV0CnsXuMPMagAz3H2FmXUEfg+8Fy2tcTCRpFKUZ8zsB2A1kVLGjYBV7v5p9P2pwDXAo0TmN5hsZn8H4i5z7e6bzGxltEbMiug+3ol+b1niPJRIyYXY2akuNrNcIv9fVyUyScviQp/9Q3T9O9H9HEDk9yZpTIlAksUNwFfACURasr+aaMbdnzWzecA5wBwz602kZO9Ud78tjn30iC1KZ2ZFzlERrX/Tkkihs+7AAOC0MhzLdOBi4BNgpru7Rc7KccdJZKauYcBY4EIzqwvcBJzk7lvNbAqR4muFGfCau19ahnglxalrSJJFJeCLaI35nkSuhn/BzOoBK6PdIbOIdJG8DvzRzI6JbnOUxT9f8ydAHTNrEF3uCfw72qdeyd1nE7kRW9TInW+JlMIuygzgfCJ19KdH15UpTnffRaSL5w/RbqUjgO+A7WZ2LNC5mFjmAm0LjsnMDjGzolpXkkaUCCRZjAOuMLO5RLqFvitim0uAJWb2IdCYyHR+y4icMP9pZouB14h0m5TK3X8kUtnxRTP7CNgDTCByUv1b9Pv+TaS1UtgUYELBzeJC37sVWAbUdvf50XVljjN672EUcJO7LyIyV/FS4Aki3U0FJgH/a2b/cvdNREY0PRfdz1wivytJY6o+KiKS5tQiEBFJc0oEIiJpTolARCTNKRGIiKQ5JQIRkTSnRCAikuaUCERE0tz/Byq9cQCXaqeMAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "y_pred_logreg_prob = logreg1.predict_proba(X_test)[:,1]\n",
    "\n",
    "# Generate ROC curve values: fpr, tpr, thresholds\n",
    "fpr, tpr, thresholds = roc_curve(y_test, y_pred_logreg_prob)\n",
    "\n",
    "# Plot ROC curve\n",
    "plt.plot([0, 1], [0, 1], 'k--')\n",
    "plt.plot(fpr, tpr)\n",
    "plt.xlabel('False Positive Rate')\n",
    "plt.ylabel('True Positive Rate')\n",
    "plt.title('ROC Curve')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "AUC: 0.8560969482775049\n"
     ]
    }
   ],
   "source": [
    "print(\"AUC: {}\".format(roc_auc_score(y_test, y_pred_logreg_prob)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Random Forest"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<matplotlib.lines.Line2D at 0x1a24764048>]"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "[<matplotlib.lines.Line2D at 0x1a24764198>]"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "Text(0.5, 0, 'False Positive Rate')"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "Text(0, 0.5, 'True Positive Rate')"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "Text(0.5, 1.0, 'ROC Curve')"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEWCAYAAABrDZDcAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjAsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+17YcXAAAgAElEQVR4nO3dd5xU9fX/8ddxV8VKFNRIrwILoiErSLeC2NGgKEGJC0sRG2JXVIIEECwoIEVEURQ1oJjwDTEaoz8jIEoRUWSlY6EIKBaE5fz+mFmyrltmYe/cnZn38/HYh3Nn7sw9d1nvmU+552PujoiIpK4Dwg5ARETCpUQgIpLilAhERFKcEoGISIpTIhARSXFKBCIiKU6JQEQkxSkRSFIxs9Vm9qOZ7TCzr8xsipkdXmCfVmb2ppl9Z2bbzew1M8sosM+RZvaIma2NflZOdLtyEcc1M7vezJaa2fdmtt7MXjKzE4M8X5GyoEQgyegCdz8cOBn4HXBH3gtm1hL4J/AqUAWoDSwG3jWzOtF9DgLeABoD5wBHAq2ALUDzIo75KHADcD1wNHAC8ApwXmmDN7P00r5HZH+Y7iyWZGJmq4Ge7v6v6PYIoLG7nxfdfgf4yN37FXjf/wGb3P0qM+sJPADUdfcdMRyzPvAp0NLd5xexz1vAs+4+KbrdIxpnm+i2A/2BG4F0YA6ww90H5vuMV4H/uPtDZlYFeAxoB+wAHnb30TH8ikR+RS0CSVpmVg3oBOREtw8l8s3+pUJ2fxE4O/r4LOAfsSSBqDOB9UUlgVK4GGgBZADTgMvNzADM7CigA/CCmR0AvEakJVM1evwbzazjfh5fUpQSgSSjV8zsO2AdsBG4N/r80UT+5r8s5D1fAnn9/5WK2Kcopd2/KH9x92/c/UfgHcCBttHX/gC85+5fAKcAx7j7YHf/2d1XAhOBrmUQg6QgJQJJRhe7+xHAaUBD/neB3wrsAY4v5D3HA5ujj7cUsU9RSrt/UdblPfBIn+0LwBXRp64Enos+rglUMbNteT/AncBxZRCDpCAlAkla7v4fYAowMrr9PfAe0KWQ3S8jMkAM8C+go5kdFuOh3gCqmVlmMft8Dxyab/u3hYVcYPt54A9mVpNIl9Ffo8+vA1a5+2/y/Rzh7ufGGK/ILygRSLJ7BDjbzE6Obt8OXB2d6nmEmR1lZkOAlsD90X2mErnY/tXMGprZAWZWyczuNLNfXWzdfQUwFnjezE4zs4PMrIKZdTWz26O7LQIuMbNDzawekFVS4O6+ENgETALmuPu26EvzgW/N7DYzO8TM0sysiZmdsi+/IBElAklq7r4JeAa4J7r9/4COwCVE+vXXEJli2iZ6QcfddxIZMP4UeB34lsjFtzIwr4hDXQ88DowBtgGfA52JDOoCPAz8DHwNPM3/unlK8nw0lmn5zikXuIDI9NhVRLq0JgEVY/xMkV/Q9FERkRSnFoGISIpTIhARSXFKBCIiKU6JQEQkxSVccavKlSt7rVq1wg5DRCShfPDBB5vd/ZjCXku4RFCrVi0WLFgQdhgiIgnFzNYU9Zq6hkREUpwSgYhIilMiEBFJcUoEIiIpTolARCTFBZYIzGyymW00s6VFvG5mNjq6KPgSM2sWVCwiIlK0IFsEU4gs/F2UTkD96E82MC7AWEREpAiB3Ufg7m+bWa1idrkIeCa6EtNcM/uNmR3v7mWx5J+ISLkxbd5aXl20YZ/fv2dPLj//vItmdY7l3gsal2FkEWHeUFaVfEvzAeujz/0qEZhZNpFWAzVq1IhLcCKSGPb3IhsP81Z9A0CL2keX+r3btm1j+fLlpKen87vahd4YvN/CTARWyHOFLo7g7hOACQCZmZlaQEEkAQV1wd6fi2y8tKh9NBedXJUrW8T+RXbbtm3ccsstvDhpEvXq1WPSpEm0b98kkPjCTATrger5tqsBX4QUi4jsg9Jc3IO6YO/LRba8y83NpVWrVixfvpxbb72V++67j0MOOSSw44WZCGYB/c3sBSILc2/X+IBI+VLShb40F/dkvGCXtS1btnD00UeTlpbGAw88QPXq1cnMzAz8uIElAjN7HjgNqGxm64F7gQMB3P0JYDZwLpAD/AD8KahYRKR08hJASRd6XdzLhrvz3HPPccMNNzBs2DB69epF586d43b8IGcNXVHC6w5cG9TxRSQ2hX3rz58AdKEP1rp16+jTpw+zZ8/m1FNPpXXr1nGPIeHKUItI8Uo7KFvYt34lgPh4/vnn6d27N7m5uTzyyCP079+ftLS0uMehRCCSRKbNW8udMz8CYh+U1UU/PEcddRQtWrRgwoQJ1K5dO7Q4LNJDkzgyMzNdC9NIqop18HZo5xN1YS+Hdu/ezcMPP8zPP//MXXfdBUTGB8wKm01ftszsA3cvdORZLQKRci7/xV+Dt4lr8eLFZGVl8cEHH3DZZZftTQDxSAIlUSIQKQeK+6af/+KvC33i2blzJ0OGDGHYsGEcffTRvPTSS1x66aXlIgHkUSIQiZNYL/YF6eKf2FasWMHw4cO58soreeihh6hUqVLYIf2KEoFIwGKZk6+LfXLZsWMHr776Kt26daNJkyZ8+umn1KlTJ+ywiqREIFIGYv22r4t98nv99dfJzs5mzZo1NGvWjEaNGpXrJABKBCL7Rd/2Jc/WrVsZOHAgkydP5oQTTuA///kPjRo1CjusmCgRiMSgqG/8+rYvECkS17p1az777DPuuOMOBg0aRIUKFcIOK2ZKBCLFKOkbvxJAatu8efPeInFDhw6lRo0aNGuWeKvuKhGIFKKwBKALvuRxd6ZOncqNN97IsGHDyM7O5uKLLw47rH2mRCCSjxKAlGTNmjX07t2bOXPm0KpVK9q1axd2SPtNiUBSXlF37ioBSEHPPvssffv2xd157LHH6NevHwcccEDYYe03JQJJWYV9+1cCkOIcc8wxtG7dmvHjx1OzZs2wwykzSgSS9DTjR/bVrl27GDVqFLt27eKee+6hY8eOdOjQoVyVhygLSgSSdApe+DXjR/bFwoULycrKYuHChXTt2rVcFYkra0oEkhSKq9CpC76Uxk8//cTgwYMZMWIElStX5q9//SuXXHJJ2GEFSolAEk5JSyvqwi/7Iycnh5EjR3LVVVcxatQojjrqqLBDCpwSgSSEkmry6+Iv+2PHjh3MnDmT7t2706RJE5YvXx7qimHxpkQg5Zpm9kjQ5syZQ3Z2NuvWrSMzM5NGjRqlVBIAJQIpp3RjlwRty5YtDBgwgGeeeYaGDRvyzjvvJEyRuLKmRCDlTsEF2JUApKzlFYnLycnhrrvu4u67706oInFlTYlAyo2CrQAtwC5lbdOmTVSqVIm0tDSGDx9OzZo1Ofnkk8MOK3RKBBKa4ub7qxUgZcndmTJlCgMGDGDYsGH07t2biy66KOywyg0lAgnNq4s2sOzLb8k4/khACUCCsXr1arKzs3n99ddp27Ytp59+etghlTtKBBKKafPWMm/VN7SofTTTe7cMOxxJUlOnTqVv376YGWPHjqV3795JUSSurCkRSFwVHAe46OSqIUckyey4446jXbt2PPHEE9SooZZmUZQIJC40HVTiYdeuXYwYMYLc3FwGDRpEhw4d6NChQ9hhlXtKBBI4TQeVePjwww+55pprWLx4MVdeeeXeInFSMiUCCYymg0o8/Pjjj9x///2MHDmSY445hpkzZyb0spFhCDQRmNk5wKNAGjDJ3YcVeL0G8DTwm+g+t7v77CBjkuCpG0jiaeXKlTz00EP06NGDBx98MCWKxJW1wBKBmaUBY4CzgfXA+2Y2y92X5dvtbuBFdx9nZhnAbKBWUDFJsJQAJF6+/fZbZsyYQY8ePWjcuDErVqxIqhXD4i3IFkFzIMfdVwKY2QvARUD+RODAkdHHFYEvAoxHAqD1fiXeZs+eTZ8+fdiwYQMtWrSgUaNGSgL7KchEUBVYl297PdCiwD73Af80s+uAw4CzCvsgM8sGsgFNASsnVBVU4m3z5s3cdNNNPPvss2RkZPDuu++mbJG4shZkIihsuN4LbF8BTHH3UWbWEphqZk3cfc8v3uQ+AZgAkJmZWfAzJI7U/SNhyCsSt3LlSgYNGsSdd97JwQcfHHZYSSPIRLAeqJ5vuxq/7vrJAs4BcPf3zKwCUBnYGGBcso80DVTi7euvv+aYY44hLS2NkSNHUrNmTZo2bRp2WEknyHut3wfqm1ltMzsI6ArMKrDPWuBMADNrBFQANgUYk+yDafPWcvn49/YmgaGdT2R675ZKAhIYd+fJJ5+kQYMGTJgwAYALLrhASSAggbUI3H23mfUH5hCZGjrZ3T82s8HAAnefBdwMTDSzm4h0G/Vwd3X9lAMaBJawrFy5kl69evHmm2/Svn17zjqr0KFDKUOWaNfdzMxMX7BgQdhhJK3CxgAAJQCJi6effpp+/fqRlpbGgw8+SK9evVQkroyY2QfunlnYa7qzWAANAkv5UKVKFc444wzGjRtHtWrVwg4nZSgRiAaBJTQ///wzw4YNY8+ePdx3332cffbZnH322WGHlXKUCFJUYWMAqgUk8fT+++9zzTXXsHTpUrp3764icSFS51uKylsdDCKtACUBiZcffviBgQMHcuqpp7J161ZmzZrFM888oyQQIrUIUpBWB5MwrVq1iscee4xevXoxfPhwKlasGHZIKU+JIIVodTAJy/bt25kxYwZ/+tOfaNy4MTk5OVSvXr3kN0pcKBGkAM0IkjD9/e9/p3fv3nz55Ze0bNmShg0bKgmUM0oEKSBvPEAJQOJp06ZN3HjjjUybNo0mTZowY8YMGjZsGHZYUgglgiSn8QAJQ25uLm3atGHVqlXcf//93H777Rx00EFhhyVFiCkRRGsF1XD3nIDjkTKi8QAJw1dffcWxxx5LWloao0aNolatWjRp0iTssKQEJSYCMzsPeAg4CKhtZicD97p756CDk9JRfSAJy549e5g4cSK33HILw4cPp2/fvpx//vlhhyUxiqVFMJjIgjL/BnD3RWZWL9CopNQK3h2sBCDxkpOTQ69evXjrrbc444wz6NixY9ghSSnFkgh2ufu2Ajd7JFaluiSXPwnoxjCJp6eeeop+/fpx0EEHMXHiRLKysnRjWAKKJRF8YmaXAQeYWW3gBmBusGFJrJQEJEw1atSgY8eOjBkzhqpVNQ6VqEosQ21mhwGDgA7Rp+YA97v7jwHHViiVoY4oOBisJCDxsHPnTv7yl7+wZ88eBg8eHHY4Ugr7W4a6o7vfBtyW7wMvAWaUUXxSSqoWKmGYN28eWVlZfPzxx1x99dUqEpdEYik6d3chz91V1oFIbAp2BWnJSAna999/z4ABA2jZsiXbt2/nb3/7G1OmTFESSCJFtgjMrCORheWrmtlD+V46EtgTdGBSuLzpoeoKknhZs2YNY8eOpU+fPgwbNowjjzwy7JCkjBXXNbQRWAr8BHyc7/nvgNuDDEoKl/8uYSUBCdK2bdt4+eWX6dmzJxkZGeTk5GjFsCRWZCJw94XAQjN7zt1/imNMUoDuEpZ4evXVV+nbty8bN26kTZs2NGzYUEkgycUyWFzVzB4AMoAKeU+6+wmBRSV7aWBY4mXjxo1cf/31TJ8+naZNmzJr1iwViUsRsSSCKcAQYCTQCfgTGiMInKaHSjzl5ubSunVr1q5dy5AhQ7j11ls58MADww5L4iSWRHCou88xs5Hu/jlwt5m9E3RgqUytAImXL774gt/+9rekpaXx6KOPUqtWLTIyMsIOS+IslumjOy0yT+xzM+tjZhcAxwYcV8rS9FCJhz179jBu3DgaNmzIE088AcC5556rJJCiYmkR3AQcDlwPPABUBK4JMqhUpumhErTPPvuMXr168fbbb3PWWWfRqVOnsEOSkJWYCNx9XvThd0B3ADPTFIIyljcmkLeSmJKABOHJJ5+kf//+VKhQgcmTJ9OjRw/dGCbFJwIzOwWoCvw/d99sZo2JlJo4A1AyKCOFjQmIBKFWrVp06tSJMWPGcPzxx4cdjpQTRRadM7O/AJcCi4HawEwilUeHA+Pc/Yd4BZlfshWdU/VQCdLOnTv585//DMCQIUNCjkbCtK9F5y4CTnL3H83saOCL6PbyIIJMVRoTkKD897//JSsri08//ZRrrrlGReKkSMXNGvopr9S0u38DfKokUHamzVvL5ePf05iAlLkdO3Zwww030KZNG3744Qf+8Y9/8OSTTyoJSJGKaxHUMbO8UtMG1Mq3jbtfUtKHm9k5wKNAGjDJ3YcVss9lwH1EVj1b7O5Xxh5+YtKYgARp7dq1jB8/nmuvvZahQ4dyxBFHhB2SlHPFJYJLC2w/XpoPNrM0YAxwNrAeeN/MZrn7snz71AfuAFq7+1YzS4n7E9QdJGVt69atvPTSS2RnZ5ORkcHKlSupUqVK2GFJgiiu6Nwb+/nZzYEcd18JYGYvEBl3WJZvn17AGHffGj3mxv08ZrmnCqJS1mbOnEm/fv3YtGkT7du3p0GDBkoCUiqx3Fm8r6oC6/Jtr48+l98JwAlm9q6ZzY12Jf2KmWWb2QIzW7Bp06aAwg1e/i4hdQfJ/vrqq6/o0qULl1xyCb/97W+ZP38+DRo0CDssSUCx3Fm8rwobmSo4VzUdqA+cRuS+hHfMrIm7b/vFm9wnABMgMn207EMNnqaJSlnKzc2lbdu2rFu3jqFDhzJw4EAViZN9FnMiMLOD3X1nKT57PVA933Y1IlNQC+4z1913AavMbDmRxPB+KY5T7ikJSFlZv349VapUIS0tjdGjR1O7dm2Vipb9VmLXkJk1N7OPgBXR7ZPM7LEYPvt9oL6Z1Tazg4CuwKwC+7wCnB793MpEuopWliL+ck9JQMrCnj17eOyxx2jYsCHjxo0DoFOnTkoCUiZiGSMYDZwPbAFw98VEL97FcffdQH9gDvAJ8KK7f2xmg83swuhuc4AtZrYM+Ddwi7tvKf1plF+aIST769NPP6Vdu3Zcf/31tGnThvPPPz/skCTJxNI1dIC7rylwM0puLB/u7rOB2QWeG5TvsQMDoj9JRzOEZH9NmjSJ/v37c+ihh/L000/TvXt33RgmZS6WRLDOzJoDHr034Drgs2DDSg55rQHNEJJ9VbduXS644AIef/xxjjvuuLDDkSQVSyLoS6R7qAbwNfCv6HNSDLUGZF/89NNPDB48GIChQ4dy+umnc/rpJfbEiuyXWBLBbnfvGngkSUT3C8i+ePfdd8nKymL58uX07NlTReIkbmIZLH7fzGab2dVmpqIlMdAAsZTGd999x3XXXUfbtm3ZuXMnc+bMYeLEiUoCEjclJgJ3rwsMAX4PfGRmr5iZWghFUJeQlNb69euZNGkS1113HR999BEdOnQIOyRJMTGVmHD3/7r79UAz4FvguUCjSlDqEpJYbdmyZe/9AI0aNWLlypU8+uijHH744SFHJqkolhvKDjezbmb2GjAf2AS0CjyyBKQuISmJu/Pyyy+TkZHB9ddfz/LlkSU+tGykhCmWFsFS4FRghLvXc/eb8y1oL1HqEpKSfPnll1x66aV06dKF6tWrs2DBAhWJk3IhlllDddx9T+CRJKhp89by6qINzFv1DaAuISlcXpG4DRs2MGLECG666SbS04Os+SgSuyL/Es1slLvfDPzVzH5V8TOWFcpSwauLNuxdbvKik6uqNSC/sG7dOqpWrUpaWhpjxoyhdu3anHDCCWGHJfILxX0lmR79b6lWJktFGccfyfTeLcMOQ8qR3NxcxowZwx133MGIESO49tpr6dixY9hhiRSquBXK5kcfNnL3XyQDM+sP7O8KZiJJ6ZNPPiErK4v33nuPTp06ccEFF4QdkkixYhksvqaQ57LKOpBElDdALJJnwoQJnHzyyXz22WdMnTqVv//979Sooe5CKd+KGyO4nMgaArXNbEa+l44AthX+rtSionJSUP369encuTOjR4/m2GOPDTsckZgUN0Ywn8gaBNWAMfme/w5YGGRQiUDTRQXgxx9/5L777sPMGDZsmIrESUIqboxgFbCKSLVRKUCtAXn77bfp2bMnK1asoE+fPioSJwmryDECM/tP9L9bzeybfD9bzSylO8bVGkht3377Lf369aN9+/bk5ubyxhtvMG7cOCUBSVjFdQ3ltW8rxyOQRKLWQGr74osvmDJlCgMGDGDw4MEcdthhYYcksl+K6xrKu5u4OvCFu/9sZm2ApsCzRIrPpZS8u4jzbiBTayB1bN68mRdffJF+/frRsGFDVq1apRXDJGnEMn30FSLLVNYFngEaAdMCjaqcyksCGccfqdZAinB3pk+fTkZGBjfeeCOffRZZpVVJQJJJLIlgj7vvAi4BHnH364CUuwrmjQvk3UWs1kDy++KLL7j44ovp2rUrNWvW5IMPPlB5CElKMS1VaWZdgO7AxdHnDgwupPJJ4wKpJTc3l3bt2rFhwwZGjhzJDTfcoCJxkrRi+cu+BuhHpAz1SjOrDTwfbFjlk8YFkt+aNWuoVq0aaWlpjB07ljp16lCvXr2wwxIJVCxLVS4FrgcWmFlDYJ27PxB4ZCJxlJuby0MPPUSjRo32rhzWoUMHJQFJCSW2CMysLTAV2AAY8Fsz6+7u7wYdnEg8LF26lKysLObPn8/555/PxRdfXPKbRJJILIPFDwPnuntrd28FnAc8GmxY5YuKyyWvJ554gmbNmrFy5UqmTZvGrFmzqFatWthhicRVLIngIHdflrfh7p8ABwUXUvmjgeLk4x5Za6lRo0Z06dKFZcuWccUVV+juYElJsQwWf2hm44l0DwF0IwWLzmmgODn88MMPDBo0iLS0NIYPH0779u1p37592GGJhCqWFkEf4HPgVuA2YCXQO8igyhN1CyWPt956i6ZNmzJq1Ch27Nixt1UgkuqKbRGY2YlAXWCmu4+IT0jli7qFEt/27du59dZbmTBhAnXr1uXNN99UqWiRfIqrPnonkfIS3YDXzaywlcqSmqqMJocvv/ySZ599loEDB7JkyRIlAZECiusa6gY0dfcuwClA39J+uJmdY2bLzSzHzG4vZr8/mJmbWWZpjxGUafPWcufMjwC1BhLRpk2beOyxxwBo2LAhq1ev5sEHH+TQQw8NOTKR8qe4RLDT3b8HcPdNJez7K2aWRmRls05ABnCFmWUUst8RRG5Ym1eazw9aXpfQ0M4nqjWQQNydadOm0ahRI26++ea9ReKOOeaYkCMTKb+Ku7jXMbMZ0Z+ZQN182zOKeV+e5kCOu69095+BF4CLCtnvz8AI4KdSRx8wdQkllnXr1nHBBRfQrVs36tWrx8KFC1UkTiQGxQ0WX1pg+/FSfnZVYF2+7fVAi/w7mNnvgOru/jczG1jUB5lZNpANUKNG8Bfm/GMDkhh2797NaaedxldffcXDDz/MddddR1paWthhiSSE4hameWM/P7uwO3P2ztczswOI3LXco6QPcvcJwASAzMzMwOf8aaZQ4li9ejXVq1cnPT2d8ePHU6dOHerUqRN2WCIJpVT9/qW0nsjqZnmqAV/k2z4CaAK8ZWargVOBWeVlwFjdQuXb7t27GTlyJI0aNWLs2LEAnHXWWUoCIvsgyETwPlDfzGqb2UFAV2BW3ovuvt3dK7t7LXevBcwFLnT3BQHGVCLdQFb+LVmyhJYtW3LLLbfQsWNHLr20YC+miJRGzInAzA4uzQe7+26gPzAH+AR40d0/NrPBZnZh6cKMH3ULlW9jx47l97//PWvWrGH69OnMnDmTKlWqhB2WSEKLpQx1c+BJoCJQw8xOAnpGl6wslrvPBmYXeG5QEfueFkvA8aBuofLH3TEzmjRpQteuXXn44YepXLly2GGJJIVYis6NBs4ncpcx7r7YzHRrpsTF999/z9133016ejoPPvgg7dq1o127dmGHJZJUYukaOsDd1xR4LjeIYMKm8YHy5Y033uDEE0/kkUceYefOnSoSJxKQWBLBumj3kJtZmpndCHwWcFxxp5IS5ce2bdvo2bMnZ511Funp6bz99tuMHj1aawWIBCSWRNAXGADUAL4mMs2z1HWHyjuVlCg/vv76a1544QVuu+02Fi9eTNu2bcMOSSSplThG4O4biUz9THoaJA5P3sX/hhtuoEGDBqxevVqDwSJxEsusoYnkuyM4j7tnBxKRpBR357nnnuOGG25gx44dnHvuudSvX19JQCSOYuka+hfwRvTnXeBYYGeQQUlqWLt2Leeddx7du3enQYMGLFq0iPr164cdlkjKiaVraHr+bTObCrweWEQhUJG5+MsrErdx40ZGjx5Nv379VCROJCSx3EdQUG2gZlkHEibdTRw/K1eupGbNmqSnpzNx4kTq1q1LrVq1wg5LJKWV2DVkZlvN7JvozzYirYE7gw8tvjRQHKzdu3czfPhwMjIyGDNmDABnnnmmkoBIOVDS4vUGnARsiD61x3VXj5TSokWLyMrK4sMPP6Rz58506dIl7JBEJJ9iWwTRi/5Md8+N/igJSKk8/vjjnHLKKWzYsIGXX36ZGTNmcPzxx4cdlojkE8usoflm1izwSEKishLByPvO0LRpU7p168ayZctULlqknCqya8jM0qOlpNsAvczsc+B7IiuPubsnRXLQQHHZ2rFjB3fddRcHHnggI0eOVJE4kQRQ3BjBfKAZcHGcYgmNBorLxj//+U+ys7NZu3Yt11133d7S0SJSvhWXCAzA3T+PUyySoLZu3cqAAQOYMmUKDRo04O2336ZNmzZhhyUiMSouERxjZgOKetHdHwogHklAGzdu5OWXX+aOO+5g0KBBVKhQIeyQRKQUiksEacDhRFsGIvl99dVXPP/889x00017i8RVqlQp7LBEZB8Ulwi+dPfBcYtEEoK788wzz3DTTTfxww8/cP7551O/fn0lAZEEVtz00aRvCWjqaOmsXr2ac845hx49epCRkaEicSJJorgWwZlxiyIkmjoau927d3P66aezefNmxowZQ58+fTjggFhuQxGR8q7IRODuKfFVWVNHi5eTk0Pt2rVJT09n8uTJ1KlTh5o1k6rmoEjK01c6KdSuXbsYOnQojRs33lsk7vTTT1cSEElC+1KGWpLchx9+SFZWFosWLaJLly5cfvnlYYckIgFSi0B+YfTo0TRv3pyvvvqKGTNm8OKLL3LccceFHZaIBEiJQID/FT5k334AAA9hSURBVIn73e9+x1VXXcWyZcvo3LlzyFGJSDykbCLQ1NGI7777jv79+zNw4EAA2rZty+TJkznqqKNCjkxE4iVlE4GmjsI//vEPmjRpwtixY3F3tNyESGpKyUSQf7H6VJw6umXLFq6++mo6derEYYcdxrvvvstDDz2kSqEiKSolE0Gqtwa2bNnCzJkzueeee1i4cCEtW7YMOyQRCVGgicDMzjGz5WaWY2a3F/L6ADNbZmZLzOwNMwt8knqqtga+/PJLRo4cibtzwgknsGbNGgYPHszBBx8cdmgiErLAEoGZpQFjgE5ABnCFmWUU2G0hkOnuTYGXgRFBxZMn1VoD7s7kyZNp1KgR99xzDzk5OQAaDBaRvYJsETQHctx9pbv/DLwAXJR/B3f/t7v/EN2cC1QLMJ69UqU1sGrVKjp06EBWVhYnnXQSixcvVpE4EfmVIO8srgqsy7e9HmhRzP5ZwP8V9oKZZQPZADVqJP8FvCzs3r2bM844gy1btjBu3Diys7NVJE5EChVkIihsCkqh8xPN7I9AJtC+sNfdfQIwASAzM1NzHIuxYsUK6tSpQ3p6Ok899RR169alevXqYYclIuVYkF8R1wP5r0DVgC8K7mRmZwF3ARe6+84A40lqu3btYsiQITRp0oTHH38cgNNOO01JQERKFGSL4H2gvpnVBjYAXYEr8+9gZr8DxgPnuPvGAGNJagsWLCArK4slS5bQtWtXrrjiirBDEpEEEliLwN13A/2BOcAnwIvu/rGZDTazC6O7PUhkXeSXzGyRmc0KKp5k9eijj9KiRQs2b97Mq6++yvPPP8+xxx4bdlgikkACLUPt7rOB2QWeG5Tv8VlBHj+ZuTtmRmZmJllZWYwYMYLf/OY3YYclIglI6xEkmG+//ZbbbruNChUq8PDDD9O6dWtat24ddlgiksA0nzCBzJ49m8aNGzNhwgTS09NVJE5EyoQSQQLYvHkzf/zjHznvvPOoWLEi//3vf3nwwQdVJE5EyoQSQQLYunUrr732Gvfeey8ffvghLVoUd1+eiEjpaIygnNqwYQPPPfcct9xyC/Xr12fNmjUaDBaRQKRUiyARViVzdyZOnEhGRgb33Xcfn3/+OYCSgIgEJqUSQXmvPPr5559z5plnkp2dTbNmzViyZAn16tULOywRSXIp1zVUXiuP7t69mzPPPJNvvvmG8ePH07NnTxWJE5G4SLlEUN4sX76cunXrkp6eztNPP03dunWpVi0u1bhFRIAU6xoqT37++Wfuv/9+TjzxRMaMGQNA+/btlQREJO7UIgjB/PnzycrKYunSpVx55ZV069Yt7JBEJIWpRRBnjzzyCC1bttx7b8Bzzz1H5cqVww5LRFKYEkGc5JWDaN68Ob169eLjjz/m/PPPDzkqERF1DQVu+/bt3HrrrRxyyCE88sgjtGrVilatWoUdlojIXmoRBOi1114jIyODSZMmcfDBB6tInIiUS0oEAdi0aRNXXnklF154IZUqVWLu3LkMHz5cReJEpFxSIgjA9u3bmT17Nvfffz8LFizglFNOCTskEZEiaYygjKxbt45nn32W22+/nXr16rFmzRoqVqwYdlgiIiVSi2A/7dmzhyeeeILGjRszZMiQvUXilAREJFEoEeyHFStWcMYZZ9C3b1+aN2/ORx99pCJxIpJw1DW0j3bv3s3ZZ5/Ntm3bePLJJ/nTn/6kwWARSUhKBKX0ySefUL9+fdLT05k6dSp169alSpUqYYclIrLP1DUUo507d3LvvffStGlTHn/8cQDatm2rJCAiCU8tghjMnTuXrKwsli1bRvfu3enevXvYIYmIlBm1CEowatQoWrVqxXfffcfs2bN55plnqFSpUthhiYiUGSWCIuzZsweAli1b0qdPH5YuXUqnTp1CjkpEpOypa6iAbdu2cfPNN3PooYfy2GOPqUiciCQ9tQjyeeWVV8jIyODpp5/miCOOUJE4EUkJSgTAxo0bueyyy+jcuTPHHXcc8+fPZ+jQobovQERSghIB8O233/L666/zwAMPMH/+fJo1axZ2SCIicZOyYwRr165l6tSp3HnnndSrV4+1a9dyxBFHhB2WiEjcBdoiMLNzzGy5meWY2e2FvH6wmU2Pvj7PzGoFFcu0eWuZt+obAMaOHUvjxo0ZOnTo3iJxSgIikqoCSwRmlgaMAToBGcAVZpZRYLcsYKu71wMeBoYHFc+rizYAsOJf07j22mtp2bIlH3/8sYrEiUjKC7JF0BzIcfeV7v4z8AJwUYF9LgKejj5+GTjTAhqhdXfY+Bmr/vUsTz31FHPmzKFWrVpBHEpEJKEEOUZQFViXb3s90KKofdx9t5ltByoBm/PvZGbZQDZAjRo19imYxlUrclSLJtz3wDKOP/74ffoMEZFkFGQiKOybfcGJ+bHsg7tPACYAZGZm7tPk/nsvaAw03pe3iogktSC7htYD1fNtVwO+KGofM0sHKgLfBBiTiIgUEGQieB+ob2a1zewgoCswq8A+s4Cro4//ALzpup1XRCSuAusaivb59wfmAGnAZHf/2MwGAwvcfRbwJDDVzHKItAS6BhWPiIgULtAbytx9NjC7wHOD8j3+CegSZAwiIlI8lZgQEUlxSgQiIilOiUBEJMUpEYiIpDhLtNmaZrYJWLOPb69MgbuWU4DOOTXonFPD/pxzTXc/prAXEi4R7A8zW+DumWHHEU8659Sgc04NQZ2zuoZERFKcEoGISIpLtUQwIewAQqBzTg0659QQyDmn1BiBiIj8Wqq1CEREpAAlAhGRFJeUicDMzjGz5WaWY2a3F/L6wWY2Pfr6PDOrFf8oy1YM5zzAzJaZ2RIze8PMaoYRZ1kq6Zzz7fcHM3MzS/iphrGcs5ldFv23/tjMpsU7xrIWw992DTP7t5ktjP59nxtGnGXFzCab2UYzW1rE62Zmo6O/jyVm1my/D+ruSfVDpOT150Ad4CBgMZBRYJ9+wBPRx12B6WHHHYdzPh04NPq4byqcc3S/I4C3gblAZthxx+HfuT6wEDgqun1s2HHH4ZwnAH2jjzOA1WHHvZ/n3A5oBiwt4vVzgf8jssLjqcC8/T1mMrYImgM57r7S3X8GXgAuKrDPRcDT0ccvA2eaWWHLZiaKEs/Z3f/t7j9EN+cSWTEukcXy7wzwZ2AE8FM8gwtILOfcCxjj7lsB3H1jnGMsa7GcswNHRh9X5NcrISYUd3+b4ldqvAh4xiPmAr8xs/1aiD0ZE0FVYF2+7fXR5wrdx913A9uBSnGJLhixnHN+WUS+USSyEs/ZzH4HVHf3v8UzsADF8u98AnCCmb1rZnPN7Jy4RReMWM75PuCPZraeyPon18UntNCU9v/3EgW6ME1ICvtmX3CObCz7JJKYz8fM/ghkAu0DjSh4xZ6zmR0APAz0iFdAcRDLv3M6ke6h04i0+t4xsybuvi3g2IISyzlfAUxx91Fm1pLIqodN3H1P8OGFosyvX8nYIlgPVM+3XY1fNxX37mNm6USak8U1xcq7WM4ZMzsLuAu40N13xim2oJR0zkcATYC3zGw1kb7UWQk+YBzr3/ar7r7L3VcBy4kkhkQVyzlnAS8CuPt7QAUixdmSVUz/v5dGMiaC94H6ZlbbzA4iMhg8q8A+s4Cro4//ALzp0VGYBFXiOUe7ScYTSQKJ3m8MJZyzu29398ruXsvdaxEZF7nQ3ReEE26ZiOVv+xUiEwMws8pEuopWxjXKshXLOa8FzgQws0ZEEsGmuEYZX7OAq6Kzh04Ftrv7l/vzgUnXNeTuu82sPzCHyIyDye7+sZkNBha4+yzgSSLNxxwiLYGu4UW8/2I85weBw4GXouPia939wtCC3k8xnnNSifGc5wAdzGwZkAvc4u5bwot6/8R4zjcDE83sJiJdJD0S+YudmT1PpGuvcnTc417gQAB3f4LIOMi5QA7wA/Cn/T5mAv++RESkDCRj15CIiJSCEoGISIpTIhARSXFKBCIiKU6JQEQkxSkRSLljZrlmtijfT61i9q1VVJXGUh7zrWiFy8XR8gwN9uEz+pjZVdHHPcysSr7XJplZRhnH+b6ZnRzDe240s0P399iSvJQIpDz60d1PzvezOk7H7ebuJxEpSPhgad/s7k+4+zPRzR5AlXyv9XT3ZWUS5f/iHEtscd4IKBFIkZQIJCFEv/m/Y2YfRn9aFbJPYzObH21FLDGz+tHn/5jv+fFmllbC4d4G6kXfe2a0zv1H0TrxB0efH2b/W99hZPS5+8xsoJn9gUg9p+eixzwk+k0+08z6mtmIfDH3MLPH9jHO98hXbMzMxpnZAousQ3B/9LnriSSkf5vZv6PPdTCz96K/x5fM7PASjiNJTolAyqND8nULzYw+txE4292bAZcDowt5Xx/gUXc/mciFeH205MDlQOvo87lAtxKOfwHwkZlVAKYAl7v7iUTuxO9rZkcDnYHG7t4UGJL/ze7+MrCAyDf3k939x3wvvwxckm/7cmD6PsZ5DpGSEnnucvdMoCnQ3syauvtoInVoTnf306NlJ+4Gzor+LhcAA0o4jiS5pCsxIUnhx+jFML8DgcejfeK5RGroFPQecJeZVQNmuPsKMzsT+D3wfrS0xiFEkkphnjOzH4HVREoZNwBWuftn0defBq4FHieyvsEkM/s7EHOZa3ffZGYrozViVkSP8W70c0sT52FESi7kX53qMjPLJvL/9fFEFmlZUuC9p0affzd6nIOI/N4khSkRSKK4CfgaOIlIS/ZXC824+zQzmwecB8wxs55ESvY+7e53xHCMbvmL0plZoWtUROvfNCdS6Kwr0B84oxTnMh24DPgUmOnubpGrcsxxElmpaxgwBrjEzGoDA4FT3H2rmU0hUnytIANed/crShGvJDl1DUmiqAh8Ga0x353It+FfMLM6wMpod8gsIl0kbwB/MLNjo/scbbGv1/wpUMvM6kW3uwP/ifapV3T32UQGYgubufMdkVLYhZkBXEykjv706HOlitPddxHp4jk12q10JPA9sN3MjgM6FRHLXKB13jmZ2aFmVljrSlKIEoEkirHA1WY2l0i30PeF7HM5sNTMFgENiSznt4zIBfOfZrYEeJ1It0mJ3P0nIpUdXzKzj4A9wBNELqp/i37ef4i0VgqaAjyRN1hc4HO3AsuAmu4+P/pcqeOMjj2MAga6+2IiaxV/DEwm0t2UZwLwf2b2b3ffRGRG0/PR48wl8ruSFKbqoyIiKU4tAhGRFKdEICKS4pQIRERSnBKBiEiKUyIQEUlxSgQiIilOiUBEJMX9f+kztkA1mgqpAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "y_pred_rfc_prob = rfc1.predict_proba(X_test)[:,1]\n",
    "\n",
    "# Generate ROC curve values: fpr, tpr, thresholds\n",
    "fpr, tpr, thresholds = roc_curve(y_test, y_pred_rfc_prob)\n",
    "\n",
    "# Plot ROC curve\n",
    "plt.plot([0, 1], [0, 1], 'k--')\n",
    "plt.plot(fpr, tpr)\n",
    "plt.xlabel('False Positive Rate')\n",
    "plt.ylabel('True Positive Rate')\n",
    "plt.title('ROC Curve')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "AUC: 0.8491411345858996\n"
     ]
    }
   ],
   "source": [
    "print(\"AUC: {}\".format(roc_auc_score(y_test, y_pred_rfc_prob)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Conclusion"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "To compare different classifiers, it can be useful to calculate the area under the ROC curve (AUC). The AUC for Logistic Regression is slighty better than Random Forest, after doing hyperparameter tuning for both classifiers. This indicates that Logistic Regression algorithm is slighty more accurate when predicting the outcome."
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}