3-Modelling_Repeater.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Capstone Hotel reservation prediction\n",
    "\n",
    "![Logo](img/hotel-logo.png)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Modelling Repeater\n",
    "#### (Folgebuchung)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Import required libraries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-08-24T20:32:40.668090Z",
     "start_time": "2020-08-24T20:32:40.640602Z"
    }
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "pd.set_option(\"display.max_columns\", None)\n",
    "\n",
    "#modelling\n",
    "from sklearn import preprocessing\n",
    "from sklearn.datasets import make_classification\n",
    "from sklearn.ensemble import AdaBoostClassifier, RandomForestClassifier\n",
    "from sklearn.inspection import permutation_importance\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.metrics import accuracy_score, classification_report, confusion_matrix, f1_score, precision_score, recall_score, roc_auc_score, roc_curve\n",
    "from sklearn.model_selection import cross_val_score, cross_validate, cross_val_predict, RandomizedSearchCV, train_test_split\n",
    "from sklearn.naive_bayes import GaussianNB\n",
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "from sklearn.svm import SVC\n",
    "from sklearn.tree import DecisionTreeClassifier\n",
    "from sklearn.utils import resample\n",
    "\n",
    "#visualisations\n",
    "import seaborn as sns\n",
    "import matplotlib.pyplot as plt\n",
    "sns.set(style=\"whitegrid\")\n",
    "\n",
    "import visuals as vs\n",
    "\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Loading cleaned dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-08-24T19:15:37.535427Z",
     "start_time": "2020-08-24T19:15:35.978012Z"
    }
   },
   "outputs": [],
   "source": [
    "df = pd.read_csv(\"data/dataset_w_feature.csv\", index_col=0, encoding=\"iso-8859-15\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-08-24T19:15:37.692515Z",
     "start_time": "2020-08-24T19:15:37.537423Z"
    }
   },
   "outputs": [],
   "source": [
    "df[\"buchungsdatum\"]      = pd.to_datetime(df[\"buchungsdatum\"])\n",
    "df[\"anreisedatum\"]       = pd.to_datetime(df[\"anreisedatum\"])\n",
    "df[\"abreisedatum\"]       = pd.to_datetime(df[\"abreisedatum\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-08-24T19:15:37.740387Z",
     "start_time": "2020-08-24T19:15:37.693514Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>buchungsdatum</th>\n",
       "      <th>anreisedatum</th>\n",
       "      <th>abreisedatum</th>\n",
       "      <th>naechte_insgesamt</th>\n",
       "      <th>gebuchte_naechte</th>\n",
       "      <th>personen_anzahl</th>\n",
       "      <th>gesamtpreis</th>\n",
       "      <th>saison</th>\n",
       "      <th>buchung_telefon</th>\n",
       "      <th>buchung_online</th>\n",
       "      <th>buchung_reisebuero</th>\n",
       "      <th>zimmerarten</th>\n",
       "      <th>reise_adventure</th>\n",
       "      <th>reise_relax</th>\n",
       "      <th>reise_standard</th>\n",
       "      <th>reise_genuss</th>\n",
       "      <th>kind</th>\n",
       "      <th>entfernung</th>\n",
       "      <th>zahler</th>\n",
       "      <th>kunden_id</th>\n",
       "      <th>buchungsnr</th>\n",
       "      <th>buchungsnr_reisebuero</th>\n",
       "      <th>menue_code</th>\n",
       "      <th>zusatzleistung</th>\n",
       "      <th>destination</th>\n",
       "      <th>anrede</th>\n",
       "      <th>plz</th>\n",
       "      <th>ort</th>\n",
       "      <th>kundenclub</th>\n",
       "      <th>jahrgangsalter_klassiert</th>\n",
       "      <th>bundesland</th>\n",
       "      <th>einwohner_adjusted_plz</th>\n",
       "      <th>bev_m_plz</th>\n",
       "      <th>bev_w_plz</th>\n",
       "      <th>bev_insgesamt_ort</th>\n",
       "      <th>bev_weiblich_ort</th>\n",
       "      <th>bev_maennlich_ort</th>\n",
       "      <th>qkm_plz</th>\n",
       "      <th>flaeche_ort</th>\n",
       "      <th>einwohnerdichte_prokm2_plz</th>\n",
       "      <th>haushalte_regiog_2017_plz</th>\n",
       "      <th>kaufkraft_einwohner_in_eur_regio</th>\n",
       "      <th>kk_einwohner_index_regiog_2017</th>\n",
       "      <th>kk_hh_index_regiog_2017</th>\n",
       "      <th>gemeindetyp_regiostar_2016</th>\n",
       "      <th>wirtschaftskraftmr1</th>\n",
       "      <th>wirtschaftskraftmr2</th>\n",
       "      <th>buchungsjahr</th>\n",
       "      <th>buchungsquartal</th>\n",
       "      <th>buchungsmonat</th>\n",
       "      <th>buchungswoche</th>\n",
       "      <th>buchungstag</th>\n",
       "      <th>buchung_wochentag</th>\n",
       "      <th>anreisejahr</th>\n",
       "      <th>anreisequartal</th>\n",
       "      <th>anreisemonat</th>\n",
       "      <th>anreisewoche</th>\n",
       "      <th>anreisetag</th>\n",
       "      <th>anreise_wochentag</th>\n",
       "      <th>abreisejahr</th>\n",
       "      <th>abreisequartal</th>\n",
       "      <th>abreisemonat</th>\n",
       "      <th>abreisewoche</th>\n",
       "      <th>abreisetag</th>\n",
       "      <th>abreise_wochentag</th>\n",
       "      <th>buchungsmonat-jahr</th>\n",
       "      <th>buchungsquartal-jahr</th>\n",
       "      <th>anreisemonat-jahr</th>\n",
       "      <th>anreisequartal-jahr</th>\n",
       "      <th>abreisemonat-jahr</th>\n",
       "      <th>abreisequartal-jahr</th>\n",
       "      <th>buchungskanal</th>\n",
       "      <th>werbe_ban</th>\n",
       "      <th>anz_werbe_ban</th>\n",
       "      <th>aufenthalt</th>\n",
       "      <th>naechte_ausserhalb</th>\n",
       "      <th>anz_buch</th>\n",
       "      <th>buchungsvorlauf</th>\n",
       "      <th>preis_per</th>\n",
       "      <th>preis_nacht</th>\n",
       "      <th>preis_nacht_per</th>\n",
       "      <th>VIP</th>\n",
       "      <th>buch_zeilen</th>\n",
       "      <th>erstbuchung</th>\n",
       "      <th>folgebuchung</th>\n",
       "      <th>repeater</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2010-01-29</td>\n",
       "      <td>2010-04-02</td>\n",
       "      <td>2010-04-10</td>\n",
       "      <td>8</td>\n",
       "      <td>8</td>\n",
       "      <td>2</td>\n",
       "      <td>2982</td>\n",
       "      <td>2010</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>classic</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>535.48</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1151</td>\n",
       "      <td>456</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>linz</td>\n",
       "      <td>Herr</td>\n",
       "      <td>29664</td>\n",
       "      <td>Walsrode</td>\n",
       "      <td>0</td>\n",
       "      <td>55-64</td>\n",
       "      <td>Niedersachsen</td>\n",
       "      <td>25173</td>\n",
       "      <td>12540</td>\n",
       "      <td>12633</td>\n",
       "      <td>25173</td>\n",
       "      <td>11577</td>\n",
       "      <td>11491</td>\n",
       "      <td>344.326464</td>\n",
       "      <td>344.326464</td>\n",
       "      <td>73.108</td>\n",
       "      <td>11453</td>\n",
       "      <td>20641.89</td>\n",
       "      <td>92.819</td>\n",
       "      <td>98.104</td>\n",
       "      <td>Landregion - Zentrale Stadt</td>\n",
       "      <td>-0.333392</td>\n",
       "      <td>-0.388738</td>\n",
       "      <td>2010</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>29</td>\n",
       "      <td>Freitag</td>\n",
       "      <td>2010</td>\n",
       "      <td>2</td>\n",
       "      <td>4</td>\n",
       "      <td>13</td>\n",
       "      <td>2</td>\n",
       "      <td>Freitag</td>\n",
       "      <td>2010</td>\n",
       "      <td>2</td>\n",
       "      <td>4</td>\n",
       "      <td>14</td>\n",
       "      <td>10</td>\n",
       "      <td>Samstag</td>\n",
       "      <td>20101</td>\n",
       "      <td>20101</td>\n",
       "      <td>20104</td>\n",
       "      <td>20102</td>\n",
       "      <td>20104</td>\n",
       "      <td>20102</td>\n",
       "      <td>reisebuero</td>\n",
       "      <td>keine</td>\n",
       "      <td>0</td>\n",
       "      <td>8</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>63</td>\n",
       "      <td>1491.00</td>\n",
       "      <td>372.75</td>\n",
       "      <td>186.38</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2010-01-14</td>\n",
       "      <td>2010-04-05</td>\n",
       "      <td>2010-04-10</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>2</td>\n",
       "      <td>1437</td>\n",
       "      <td>2010</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>classic</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>544.72</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>40</td>\n",
       "      <td>6413</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>linz</td>\n",
       "      <td>Frau</td>\n",
       "      <td>50226</td>\n",
       "      <td>Frechen</td>\n",
       "      <td>0</td>\n",
       "      <td>55-64</td>\n",
       "      <td>Nordrhein-Westfalen</td>\n",
       "      <td>52473</td>\n",
       "      <td>25645</td>\n",
       "      <td>26828</td>\n",
       "      <td>52473</td>\n",
       "      <td>26828</td>\n",
       "      <td>25645</td>\n",
       "      <td>45.118960</td>\n",
       "      <td>45.118960</td>\n",
       "      <td>1162.992</td>\n",
       "      <td>26227</td>\n",
       "      <td>25268.58</td>\n",
       "      <td>113.623</td>\n",
       "      <td>111.379</td>\n",
       "      <td>Landregion - Zentrale Stadt</td>\n",
       "      <td>0.853802</td>\n",
       "      <td>-0.345112</td>\n",
       "      <td>2010</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>14</td>\n",
       "      <td>Donnerstag</td>\n",
       "      <td>2010</td>\n",
       "      <td>2</td>\n",
       "      <td>4</td>\n",
       "      <td>14</td>\n",
       "      <td>5</td>\n",
       "      <td>Montag</td>\n",
       "      <td>2010</td>\n",
       "      <td>2</td>\n",
       "      <td>4</td>\n",
       "      <td>14</td>\n",
       "      <td>10</td>\n",
       "      <td>Samstag</td>\n",
       "      <td>20101</td>\n",
       "      <td>20101</td>\n",
       "      <td>20104</td>\n",
       "      <td>20102</td>\n",
       "      <td>20104</td>\n",
       "      <td>20102</td>\n",
       "      <td>reisebuero</td>\n",
       "      <td>generell</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>81</td>\n",
       "      <td>718.50</td>\n",
       "      <td>287.40</td>\n",
       "      <td>143.70</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2010-03-01</td>\n",
       "      <td>2010-04-05</td>\n",
       "      <td>2010-04-10</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>2</td>\n",
       "      <td>1737</td>\n",
       "      <td>2010</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>classic</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>606.63</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>2134</td>\n",
       "      <td>4322</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>linz</td>\n",
       "      <td>Herr</td>\n",
       "      <td>23970</td>\n",
       "      <td>Wismar</td>\n",
       "      <td>0</td>\n",
       "      <td>75-84</td>\n",
       "      <td>Mecklenburg-Vorpommern</td>\n",
       "      <td>1423</td>\n",
       "      <td>692</td>\n",
       "      <td>731</td>\n",
       "      <td>46442</td>\n",
       "      <td>21844</td>\n",
       "      <td>20706</td>\n",
       "      <td>50.563027</td>\n",
       "      <td>83.131346</td>\n",
       "      <td>17.117</td>\n",
       "      <td>6355</td>\n",
       "      <td>18697.65</td>\n",
       "      <td>84.076</td>\n",
       "      <td>73.665</td>\n",
       "      <td>Stadtregion - Kleinstädtischer, dörflicher Raum</td>\n",
       "      <td>-1.012536</td>\n",
       "      <td>-1.012536</td>\n",
       "      <td>2010</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>9</td>\n",
       "      <td>1</td>\n",
       "      <td>Montag</td>\n",
       "      <td>2010</td>\n",
       "      <td>2</td>\n",
       "      <td>4</td>\n",
       "      <td>14</td>\n",
       "      <td>5</td>\n",
       "      <td>Montag</td>\n",
       "      <td>2010</td>\n",
       "      <td>2</td>\n",
       "      <td>4</td>\n",
       "      <td>14</td>\n",
       "      <td>10</td>\n",
       "      <td>Samstag</td>\n",
       "      <td>20103</td>\n",
       "      <td>20101</td>\n",
       "      <td>20104</td>\n",
       "      <td>20102</td>\n",
       "      <td>20104</td>\n",
       "      <td>20102</td>\n",
       "      <td>reisebuero</td>\n",
       "      <td>keine</td>\n",
       "      <td>0</td>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>35</td>\n",
       "      <td>868.50</td>\n",
       "      <td>347.40</td>\n",
       "      <td>173.70</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2010-01-25</td>\n",
       "      <td>2010-03-21</td>\n",
       "      <td>2010-03-25</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>2</td>\n",
       "      <td>1047</td>\n",
       "      <td>2010</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>classic</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>421.59</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>776</td>\n",
       "      <td>6414</td>\n",
       "      <td>69</td>\n",
       "      <td>9</td>\n",
       "      <td>linz</td>\n",
       "      <td>Frau</td>\n",
       "      <td>15732</td>\n",
       "      <td>Eichwalde</td>\n",
       "      <td>0</td>\n",
       "      <td>65-74</td>\n",
       "      <td>Brandenburg</td>\n",
       "      <td>14690</td>\n",
       "      <td>7302</td>\n",
       "      <td>7388</td>\n",
       "      <td>14690</td>\n",
       "      <td>4135</td>\n",
       "      <td>4087</td>\n",
       "      <td>11.965427</td>\n",
       "      <td>11.965427</td>\n",
       "      <td>1227.704</td>\n",
       "      <td>7159</td>\n",
       "      <td>23442.06</td>\n",
       "      <td>105.410</td>\n",
       "      <td>104.087</td>\n",
       "      <td>Landregion - Städtischer Raum</td>\n",
       "      <td>0.417226</td>\n",
       "      <td>0.417226</td>\n",
       "      <td>2010</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>25</td>\n",
       "      <td>Montag</td>\n",
       "      <td>2010</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>11</td>\n",
       "      <td>21</td>\n",
       "      <td>Sonntag</td>\n",
       "      <td>2010</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>12</td>\n",
       "      <td>25</td>\n",
       "      <td>Donnerstag</td>\n",
       "      <td>20101</td>\n",
       "      <td>20101</td>\n",
       "      <td>20103</td>\n",
       "      <td>20101</td>\n",
       "      <td>20103</td>\n",
       "      <td>20101</td>\n",
       "      <td>reisebuero</td>\n",
       "      <td>keine</td>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>55</td>\n",
       "      <td>523.50</td>\n",
       "      <td>261.75</td>\n",
       "      <td>130.88</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2010-01-26</td>\n",
       "      <td>2010-03-07</td>\n",
       "      <td>2010-03-09</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>20</td>\n",
       "      <td>5279</td>\n",
       "      <td>2010</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>classic</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>37.88</td>\n",
       "      <td>0</td>\n",
       "      <td>5</td>\n",
       "      <td>813</td>\n",
       "      <td>2938</td>\n",
       "      <td>113</td>\n",
       "      <td>2</td>\n",
       "      <td>duesseldorf</td>\n",
       "      <td>Frau</td>\n",
       "      <td>53881</td>\n",
       "      <td>Euskirchen</td>\n",
       "      <td>1</td>\n",
       "      <td>45-54</td>\n",
       "      <td>Nordrhein-Westfalen</td>\n",
       "      <td>28531</td>\n",
       "      <td>13975</td>\n",
       "      <td>14556</td>\n",
       "      <td>57975</td>\n",
       "      <td>29578</td>\n",
       "      <td>28397</td>\n",
       "      <td>127.679129</td>\n",
       "      <td>139.430913</td>\n",
       "      <td>204.625</td>\n",
       "      <td>11683</td>\n",
       "      <td>23328.82</td>\n",
       "      <td>104.901</td>\n",
       "      <td>120.110</td>\n",
       "      <td>Stadtregion - Kleinstädtischer, dörflicher Raum</td>\n",
       "      <td>0.212335</td>\n",
       "      <td>-0.579173</td>\n",
       "      <td>2010</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>26</td>\n",
       "      <td>Dienstag</td>\n",
       "      <td>2010</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>9</td>\n",
       "      <td>7</td>\n",
       "      <td>Sonntag</td>\n",
       "      <td>2010</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>10</td>\n",
       "      <td>9</td>\n",
       "      <td>Dienstag</td>\n",
       "      <td>20101</td>\n",
       "      <td>20101</td>\n",
       "      <td>20103</td>\n",
       "      <td>20101</td>\n",
       "      <td>20103</td>\n",
       "      <td>20101</td>\n",
       "      <td>reisebuero</td>\n",
       "      <td>keine</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>40</td>\n",
       "      <td>263.95</td>\n",
       "      <td>2639.50</td>\n",
       "      <td>131.98</td>\n",
       "      <td>0</td>\n",
       "      <td>7</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  buchungsdatum anreisedatum abreisedatum  naechte_insgesamt  \\\n",
       "0    2010-01-29   2010-04-02   2010-04-10                  8   \n",
       "1    2010-01-14   2010-04-05   2010-04-10                  5   \n",
       "2    2010-03-01   2010-04-05   2010-04-10                  5   \n",
       "3    2010-01-25   2010-03-21   2010-03-25                  4   \n",
       "4    2010-01-26   2010-03-07   2010-03-09                  2   \n",
       "\n",
       "   gebuchte_naechte  personen_anzahl  gesamtpreis  saison  buchung_telefon  \\\n",
       "0                 8                2         2982    2010                0   \n",
       "1                 5                2         1437    2010                0   \n",
       "2                 5                2         1737    2010                0   \n",
       "3                 4                2         1047    2010                0   \n",
       "4                 2               20         5279    2010                0   \n",
       "\n",
       "   buchung_online  buchung_reisebuero zimmerarten  reise_adventure  \\\n",
       "0               0                   1     classic                0   \n",
       "1               0                   1     classic                0   \n",
       "2               0                   1     classic                0   \n",
       "3               0                   1     classic                0   \n",
       "4               0                   1     classic                1   \n",
       "\n",
       "   reise_relax  reise_standard  reise_genuss  kind  entfernung  zahler  \\\n",
       "0            0               0             1     0      535.48       1   \n",
       "1            0               1             0     0      544.72       0   \n",
       "2            0               1             0     0      606.63       0   \n",
       "3            1               0             0     0      421.59       1   \n",
       "4            0               0             0     0       37.88       0   \n",
       "\n",
       "   kunden_id  buchungsnr  buchungsnr_reisebuero  menue_code  zusatzleistung  \\\n",
       "0          1        1151                    456           5               5   \n",
       "1          2          40                   6413           5               5   \n",
       "2          3        2134                   4322           5               5   \n",
       "3          4         776                   6414          69               9   \n",
       "4          5         813                   2938         113               2   \n",
       "\n",
       "   destination anrede    plz         ort  kundenclub jahrgangsalter_klassiert  \\\n",
       "0         linz   Herr  29664    Walsrode           0                    55-64   \n",
       "1         linz   Frau  50226     Frechen           0                    55-64   \n",
       "2         linz   Herr  23970      Wismar           0                    75-84   \n",
       "3         linz   Frau  15732   Eichwalde           0                    65-74   \n",
       "4  duesseldorf   Frau  53881  Euskirchen           1                    45-54   \n",
       "\n",
       "               bundesland  einwohner_adjusted_plz  bev_m_plz  bev_w_plz  \\\n",
       "0           Niedersachsen                   25173      12540      12633   \n",
       "1     Nordrhein-Westfalen                   52473      25645      26828   \n",
       "2  Mecklenburg-Vorpommern                    1423        692        731   \n",
       "3             Brandenburg                   14690       7302       7388   \n",
       "4     Nordrhein-Westfalen                   28531      13975      14556   \n",
       "\n",
       "   bev_insgesamt_ort  bev_weiblich_ort  bev_maennlich_ort     qkm_plz  \\\n",
       "0              25173             11577              11491  344.326464   \n",
       "1              52473             26828              25645   45.118960   \n",
       "2              46442             21844              20706   50.563027   \n",
       "3              14690              4135               4087   11.965427   \n",
       "4              57975             29578              28397  127.679129   \n",
       "\n",
       "   flaeche_ort  einwohnerdichte_prokm2_plz  haushalte_regiog_2017_plz  \\\n",
       "0   344.326464                      73.108                      11453   \n",
       "1    45.118960                    1162.992                      26227   \n",
       "2    83.131346                      17.117                       6355   \n",
       "3    11.965427                    1227.704                       7159   \n",
       "4   139.430913                     204.625                      11683   \n",
       "\n",
       "   kaufkraft_einwohner_in_eur_regio  kk_einwohner_index_regiog_2017  \\\n",
       "0                          20641.89                          92.819   \n",
       "1                          25268.58                         113.623   \n",
       "2                          18697.65                          84.076   \n",
       "3                          23442.06                         105.410   \n",
       "4                          23328.82                         104.901   \n",
       "\n",
       "   kk_hh_index_regiog_2017                       gemeindetyp_regiostar_2016  \\\n",
       "0                   98.104                      Landregion - Zentrale Stadt   \n",
       "1                  111.379                      Landregion - Zentrale Stadt   \n",
       "2                   73.665  Stadtregion - Kleinstädtischer, dörflicher Raum   \n",
       "3                  104.087                    Landregion - Städtischer Raum   \n",
       "4                  120.110  Stadtregion - Kleinstädtischer, dörflicher Raum   \n",
       "\n",
       "   wirtschaftskraftmr1  wirtschaftskraftmr2  buchungsjahr  buchungsquartal  \\\n",
       "0            -0.333392            -0.388738          2010                1   \n",
       "1             0.853802            -0.345112          2010                1   \n",
       "2            -1.012536            -1.012536          2010                1   \n",
       "3             0.417226             0.417226          2010                1   \n",
       "4             0.212335            -0.579173          2010                1   \n",
       "\n",
       "   buchungsmonat  buchungswoche  buchungstag buchung_wochentag  anreisejahr  \\\n",
       "0              1              4           29           Freitag         2010   \n",
       "1              1              2           14        Donnerstag         2010   \n",
       "2              3              9            1            Montag         2010   \n",
       "3              1              4           25            Montag         2010   \n",
       "4              1              4           26          Dienstag         2010   \n",
       "\n",
       "   anreisequartal  anreisemonat  anreisewoche  anreisetag anreise_wochentag  \\\n",
       "0               2             4            13           2           Freitag   \n",
       "1               2             4            14           5            Montag   \n",
       "2               2             4            14           5            Montag   \n",
       "3               1             3            11          21           Sonntag   \n",
       "4               1             3             9           7           Sonntag   \n",
       "\n",
       "   abreisejahr  abreisequartal  abreisemonat  abreisewoche  abreisetag  \\\n",
       "0         2010               2             4            14          10   \n",
       "1         2010               2             4            14          10   \n",
       "2         2010               2             4            14          10   \n",
       "3         2010               1             3            12          25   \n",
       "4         2010               1             3            10           9   \n",
       "\n",
       "  abreise_wochentag  buchungsmonat-jahr  buchungsquartal-jahr  \\\n",
       "0           Samstag               20101                 20101   \n",
       "1           Samstag               20101                 20101   \n",
       "2           Samstag               20103                 20101   \n",
       "3        Donnerstag               20101                 20101   \n",
       "4          Dienstag               20101                 20101   \n",
       "\n",
       "   anreisemonat-jahr  anreisequartal-jahr  abreisemonat-jahr  \\\n",
       "0              20104                20102              20104   \n",
       "1              20104                20102              20104   \n",
       "2              20104                20102              20104   \n",
       "3              20103                20101              20103   \n",
       "4              20103                20101              20103   \n",
       "\n",
       "   abreisequartal-jahr buchungskanal werbe_ban  anz_werbe_ban  aufenthalt  \\\n",
       "0                20102    reisebuero     keine              0           8   \n",
       "1                20102    reisebuero  generell              5           5   \n",
       "2                20102    reisebuero     keine              0           5   \n",
       "3                20101    reisebuero     keine              0           4   \n",
       "4                20101    reisebuero     keine              0           2   \n",
       "\n",
       "   naechte_ausserhalb  anz_buch  buchungsvorlauf  preis_per  preis_nacht  \\\n",
       "0                   0         1               63    1491.00       372.75   \n",
       "1                   0         1               81     718.50       287.40   \n",
       "2                   0         1               35     868.50       347.40   \n",
       "3                   0         1               55     523.50       261.75   \n",
       "4                   0         1               40     263.95      2639.50   \n",
       "\n",
       "   preis_nacht_per  VIP  buch_zeilen  erstbuchung  folgebuchung  repeater  \n",
       "0           186.38    0            2            1             0         0  \n",
       "1           143.70    0            2            1             0         0  \n",
       "2           173.70    0            2            1             0         0  \n",
       "3           130.88    0            1            1             0         0  \n",
       "4           131.98    0            7            1             0         1  "
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Preparation"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Dropping redundant variables"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-08-24T19:15:37.835134Z",
     "start_time": "2020-08-24T19:15:37.741385Z"
    }
   },
   "outputs": [],
   "source": [
    "df.drop(\"buchungskanal\", axis=1, inplace=True)\n",
    "df.drop(\"werbe_ban\", axis=1, inplace=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Grouping variables"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-08-24T19:15:37.882517Z",
     "start_time": "2020-08-24T19:15:37.837130Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "categorial = list(df.columns[df.dtypes==object])\n",
    "datetime64 = ['buchungsdatum', 'anreisedatum', 'abreisedatum']\n",
    "numeric    = list(df.select_dtypes(include=['int64', 'float64']).columns)\n",
    "\n",
    "#to prevent data leakage\n",
    "for entry in ['erstbuchung', 'repeater', 'kunden_id', 'folgebuchung', 'menue_code', 'zusatzleistung', 'plz', 'kundenclub', 'anz_buch']: \n",
    "    numeric.remove(entry)\n",
    "    \n",
    "for entry in ['menue_code', 'zusatzleistung']: \n",
    "    categorial.append(entry)    \n",
    "\n",
    "categorial.remove('ort')\n",
    "\n",
    "# 8 = erstbuchung, repeater, kunden_id, folgebuchung, plz, kundenclub, anz_buch, ort\n",
    "df.shape[1] == len(categorial) + len(datetime64) + len(numeric) + 8"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### renaming destinations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-08-24T19:15:37.928394Z",
     "start_time": "2020-08-24T19:15:37.883514Z"
    }
   },
   "outputs": [],
   "source": [
    "aenderung = { \"saint etienne\": \"saint-etienne\",\n",
    "              \"viana do castelo\": \"viana-do-castelo\",\n",
    "            }\n",
    "\n",
    "df.destination.replace(aenderung, inplace=True)\n",
    "\n",
    "aenderung2 = {\n",
    "\"Landregion - Kleinstädtischer, dörflicher Raum\": \"Kleinstädt_(Land)\",\n",
    "\"Landregion - Städtischer Raum\": \"Stadt_(Land)\",\n",
    "\"Landregion - Zentrale Stadt\": \"zen_Stadt_(Land)\",\n",
    "\"Stadtregion - Kleinstädtischer, dörflicher Raum\": \"Kleinstädt_(Stadt)\",\n",
    "\"Stadtregion - Metropole\": \"Metropole\",\n",
    "\"Stadtregion - Mittelstadt, städtischer Raum\": \"Mittelstadt\",\n",
    "\"Stadtregion -Regiopole und Großstadt\": \"Regiopole\"\n",
    "    }\n",
    "\n",
    "df.gemeindetyp_regiostar_2016.replace(aenderung2, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-08-24T19:15:38.149812Z",
     "start_time": "2020-08-24T19:15:37.930389Z"
    }
   },
   "outputs": [],
   "source": [
    "df.menue_code     = df.menue_code.astype(\"str\")\n",
    "df.zusatzleistung = df.zusatzleistung.astype(\"str\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Normalisation and dummy encoding"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-08-24T19:15:38.807560Z",
     "start_time": "2020-08-24T19:15:38.150809Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(196989, 283)"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = df.reset_index()\n",
    "x = df[numeric]\n",
    "min_max_scaler = preprocessing.MinMaxScaler()\n",
    "x_scaled = min_max_scaler.fit_transform(x)\n",
    "columns = df[numeric].columns\n",
    "df_num = pd.DataFrame(x_scaled)\n",
    "df_num.columns = columns\n",
    "\n",
    "df_dum = pd.get_dummies(df[categorial])\n",
    "df_dum.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Logistic Regression\n",
    "Basic (only numeric features)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-08-24T19:15:38.822593Z",
     "start_time": "2020-08-24T19:15:38.808557Z"
    }
   },
   "outputs": [],
   "source": [
    "model, acc, pre, rec, f1, roc = [], [], [], [], [], []"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-08-24T19:16:07.849789Z",
     "start_time": "2020-08-24T19:15:38.823518Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Using backend LokyBackend with 8 concurrent workers.\n",
      "[Parallel(n_jobs=-1)]: Done   2 out of   5 | elapsed:   26.4s remaining:   39.7s\n",
      "[Parallel(n_jobs=-1)]: Done   5 out of   5 | elapsed:   28.1s finished\n"
     ]
    }
   ],
   "source": [
    "X = df[numeric]\n",
    "y = df.folgebuchung\n",
    "\n",
    "#X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.10)\n",
    "\n",
    "lr1 = LogisticRegression(solver='liblinear', max_iter=10_000, random_state=1337)\n",
    "\n",
    "y_pred1 = cross_val_predict(lr1, X, y, cv=5, n_jobs=-1, verbose=5)\n",
    "\n",
    "model.append(\"Basic Logistic Regression\")\n",
    "acc.append(accuracy_score(y, y_pred1)*100)\n",
    "pre.append(precision_score(y, y_pred1)*100)\n",
    "rec.append(recall_score(y, y_pred1)*100)\n",
    "f1.append(f1_score(y, y_pred1)*100)\n",
    "#roc1 = roc_curve(y, y_pred1)*100\n",
    "roc.append(roc_auc_score(y, y_pred1)*100) #ändern test oder train?\n",
    "\n",
    "conf_matrix1 = confusion_matrix(y, y_pred1)\n",
    "class_rep1 = classification_report(y, y_pred1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-08-09T18:09:45.250801Z",
     "start_time": "2020-08-09T18:09:16.781424Z"
    }
   },
   "source": [
    "only numeric features but normalised"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-08-24T19:16:23.092240Z",
     "start_time": "2020-08-24T19:16:07.850787Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Using backend LokyBackend with 8 concurrent workers.\n",
      "[Parallel(n_jobs=-1)]: Done   2 out of   5 | elapsed:   13.3s remaining:   19.9s\n",
      "[Parallel(n_jobs=-1)]: Done   5 out of   5 | elapsed:   14.2s finished\n"
     ]
    }
   ],
   "source": [
    "X = df_num\n",
    "y = df.folgebuchung\n",
    "\n",
    "#X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.10, shuffle = True, stratify = y)\n",
    "\n",
    "lr2 = LogisticRegression(solver='liblinear', max_iter=10_000, random_state=1337)\n",
    "\n",
    "y_pred2 = cross_val_predict(lr2, X, y, cv=5, n_jobs=-1, verbose=5)\n",
    "\n",
    "model.append(\"Normalised Logistic Regression\")\n",
    "acc.append(accuracy_score(y, y_pred2)*100)\n",
    "pre.append(precision_score(y, y_pred2)*100)\n",
    "rec.append(recall_score(y, y_pred2)*100)\n",
    "f1.append(f1_score(y, y_pred2)*100)\n",
    "#roc2 = roc_curve(y, y_pred2)*100\n",
    "roc.append(roc_auc_score(y, y_pred2)*100)\n",
    "\n",
    "conf_matrix2 = confusion_matrix(y, y_pred2)\n",
    "class_rep2 = classification_report(y, y_pred2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### extended Logistic Regression\n",
    "(categorial variables as dummies)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-08-24T19:16:50.533797Z",
     "start_time": "2020-08-24T19:16:23.093238Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Using backend LokyBackend with 8 concurrent workers.\n",
      "[Parallel(n_jobs=-1)]: Done   2 out of   5 | elapsed:   25.5s remaining:   38.3s\n",
      "[Parallel(n_jobs=-1)]: Done   5 out of   5 | elapsed:   26.4s finished\n"
     ]
    }
   ],
   "source": [
    "X = pd.concat([df_num, df_dum], axis=1)\n",
    "y = df.folgebuchung\n",
    "\n",
    "#X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.10)\n",
    "\n",
    "lr3 = LogisticRegression(solver='liblinear', max_iter=10_000, random_state=1337)\n",
    "\n",
    "y_pred3 = cross_val_predict(lr3, X, y, cv=5, n_jobs=-1, verbose=5)\n",
    "\n",
    "model.append(\"Logistic Regression (w. dummies)\")\n",
    "acc.append(accuracy_score(y, y_pred3)*100)\n",
    "pre.append(precision_score(y, y_pred3)*100)\n",
    "rec.append(recall_score(y, y_pred3)*100)\n",
    "f1.append(f1_score(y, y_pred3)*100)\n",
    "#roc3 = roc_curve(y, y_pred3)*100\n",
    "roc.append(roc_auc_score(y, y_pred3)*100)\n",
    "\n",
    "conf_matrix3 = confusion_matrix(y, y_pred3)\n",
    "class_rep3 = classification_report(y, y_pred3)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Dealing with imbalace \n",
    "in target feature folgebuchung by resampling"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-08-24T19:16:50.548758Z",
     "start_time": "2020-08-24T19:16:50.534795Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    157855\n",
       "1     39134\n",
       "Name: folgebuchung, dtype: int64"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.folgebuchung.value_counts()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "New train/test split without normalisation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-08-24T19:16:50.642507Z",
     "start_time": "2020-08-24T19:16:50.550753Z"
    }
   },
   "outputs": [],
   "source": [
    "X = pd.concat([df[numeric], df_dum], axis=1)\n",
    "#X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.10, random_state=1337)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Upsampling"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-08-24T19:16:51.135697Z",
     "start_time": "2020-08-24T19:16:50.643505Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1    157855\n",
       "0    157855\n",
       "Name: folgebuchung, dtype: int64"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# concatenate our training data back together\n",
    "X = pd.concat([X, y], axis=1)\n",
    "\n",
    "# separate minority and majority classes\n",
    "erstbuchung  = X[X.folgebuchung==0]\n",
    "folgebuchung = X[X.folgebuchung==1]\n",
    "\n",
    "# upsample minority\n",
    "folgebuchung_upsampled = resample(folgebuchung, replace=True, \n",
    "                          n_samples=len(erstbuchung), # match number in majority class\n",
    "                          random_state=1337) \n",
    "\n",
    "# combine majority and upsampled minority\n",
    "upsampled = pd.concat([erstbuchung, folgebuchung_upsampled])\n",
    "\n",
    "upsampled.folgebuchung.value_counts()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Downsampling"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-08-24T19:28:53.206676Z",
     "start_time": "2020-08-24T19:28:53.102443Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1    39134\n",
       "0    39134\n",
       "Name: folgebuchung, dtype: int64"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# downsample majority\n",
    "erstbuchung_downsampled = resample(erstbuchung, replace=False, \n",
    "                          n_samples=len(folgebuchung), # match minority n\n",
    "                          random_state=1337) \n",
    "\n",
    "# combine majority and upsampled minority\n",
    "downsampled = pd.concat([folgebuchung, erstbuchung_downsampled])\n",
    "\n",
    "downsampled.folgebuchung.value_counts()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### balanced Logistic Regression\n",
    "with upsampled data and additional dummy variables"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-08-24T19:32:49.305553Z",
     "start_time": "2020-08-24T19:29:00.201704Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Using backend LokyBackend with 8 concurrent workers.\n",
      "[Parallel(n_jobs=-1)]: Done   2 out of   5 | elapsed:  1.8min remaining:  2.8min\n",
      "[Parallel(n_jobs=-1)]: Done   5 out of   5 | elapsed:  3.8min finished\n"
     ]
    }
   ],
   "source": [
    "y = upsampled.folgebuchung\n",
    "X = upsampled.drop('folgebuchung', axis=1)\n",
    "\n",
    "lr4 = LogisticRegression(solver='liblinear', max_iter=10_000, n_jobs=-1, random_state=1337)\n",
    "\n",
    "y_pred4 = cross_val_predict(lr4, X, y, cv=5, n_jobs=-1, verbose=5)\n",
    "\n",
    "model.append(\"Balanced Logistic Regression (upsampled)\")\n",
    "acc.append(accuracy_score(y, y_pred4)*100)\n",
    "pre.append(precision_score(y, y_pred4)*100)\n",
    "rec.append(recall_score(y, y_pred4)*100)\n",
    "f1.append(f1_score(y, y_pred4)*100)\n",
    "#roc4 = roc_curve(y, y_pred4)*100\n",
    "roc.append(roc_auc_score(y, y_pred4)*100)\n",
    "\n",
    "conf_matrix4 = confusion_matrix(y, y_pred4)\n",
    "class_rep4 = classification_report(y, y_pred4)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2nd balanced Logistic Regression\n",
    "downsampled"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-08-24T19:33:22.102615Z",
     "start_time": "2020-08-24T19:32:53.804112Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Using backend LokyBackend with 8 concurrent workers.\n",
      "[Parallel(n_jobs=-1)]: Done   2 out of   5 | elapsed:   21.2s remaining:   31.9s\n",
      "[Parallel(n_jobs=-1)]: Done   5 out of   5 | elapsed:   27.7s finished\n"
     ]
    }
   ],
   "source": [
    "y = downsampled.folgebuchung\n",
    "X = downsampled.drop('folgebuchung', axis=1)\n",
    "\n",
    "lr5 = LogisticRegression(solver='liblinear', max_iter=10_000, n_jobs=-1, random_state=1337)\n",
    "\n",
    "y_pred5 = cross_val_predict(lr5, X, y, cv=5, n_jobs=-1, verbose=5)\n",
    "\n",
    "model.append(\"Balanced Logistic Regression (downsampled)\")\n",
    "acc.append(accuracy_score(y, y_pred5)*100)\n",
    "pre.append(precision_score(y, y_pred5)*100)\n",
    "rec.append(recall_score(y, y_pred5)*100)\n",
    "f1.append(f1_score(y, y_pred5)*100)\n",
    "#roc5 = roc_curve(y, y_pred5)*100\n",
    "roc.append(roc_auc_score(y, y_pred5)*100)\n",
    "\n",
    "conf_matrix5 = confusion_matrix(y, y_pred5)\n",
    "class_rep5 = classification_report(y, y_pred5)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Advanced models\n",
    "## Random forest\n",
    "fixed settings after reandom search"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-08-24T20:02:48.572591Z",
     "start_time": "2020-08-24T19:33:26.662477Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Using backend LokyBackend with 8 concurrent workers.\n",
      "[Parallel(n_jobs=-1)]: Done   2 out of   5 | elapsed: 28.5min remaining: 42.8min\n",
      "[Parallel(n_jobs=-1)]: Done   5 out of   5 | elapsed: 29.3min finished\n"
     ]
    }
   ],
   "source": [
    "X = pd.concat([df[numeric], df_dum], axis=1)\n",
    "y = df.folgebuchung\n",
    "\n",
    "#X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.10, random_state=1337)\n",
    "\n",
    "lr6 = RandomForestClassifier(n_estimators= 120, min_samples_split= 5,\n",
    "                             min_samples_leaf= 6, max_features= None,\n",
    "                             max_depth= None, random_state= 1337)\n",
    "\n",
    "y_pred6 = cross_val_predict(lr6, X, y, cv=5, n_jobs=-1, verbose=5)\n",
    "\n",
    "model.append(\"Random Forest\")\n",
    "acc.append(accuracy_score(y, y_pred6)*100)\n",
    "pre.append(precision_score(y, y_pred6)*100)\n",
    "rec.append(recall_score(y, y_pred6)*100)\n",
    "f1.append(f1_score(y, y_pred6)*100)\n",
    "#roc6 = roc_curve(y, y_pred6)*100\n",
    "roc.append(roc_auc_score(y, y_pred6)*100)\n",
    "\n",
    "conf_matrix6 = confusion_matrix(y, y_pred6)\n",
    "class_rep6 = classification_report(y, y_pred6)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### GaussianNB"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-08-24T20:09:30.224790Z",
     "start_time": "2020-08-24T20:09:24.776208Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Using backend LokyBackend with 8 concurrent workers.\n",
      "[Parallel(n_jobs=-1)]: Done   2 out of   5 | elapsed:    4.5s remaining:    6.9s\n",
      "[Parallel(n_jobs=-1)]: Done   5 out of   5 | elapsed:    4.6s finished\n"
     ]
    }
   ],
   "source": [
    "X = pd.concat([df[numeric], df_dum], axis=1)\n",
    "y = df.folgebuchung\n",
    "\n",
    "#X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.10, random_state=1337)\n",
    "\n",
    "lr7 = GaussianNB()\n",
    "\n",
    "y_pred7 = cross_val_predict(lr7, X, y, cv=5, n_jobs=-1, verbose=5)\n",
    "\n",
    "model.append(\"GaussianNB\")\n",
    "acc.append(accuracy_score(y, y_pred7)*100)\n",
    "pre.append(precision_score(y, y_pred7)*100)\n",
    "rec.append(recall_score(y, y_pred7)*100)\n",
    "f1.append(f1_score(y, y_pred7)*100)\n",
    "#roc7 = roc_curve(y, y_pred7)*100\n",
    "roc.append(roc_auc_score(y, y_pred7)*100)\n",
    "\n",
    "conf_matrix7 = confusion_matrix(y, y_pred7)\n",
    "class_rep7 = classification_report(y, y_pred7)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### KNeighborsClassifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-08-24T20:12:54.058218Z",
     "start_time": "2020-08-24T20:11:39.826263Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Using backend LokyBackend with 8 concurrent workers.\n",
      "[Parallel(n_jobs=-1)]: Done   2 out of   5 | elapsed:  1.2min remaining:  1.8min\n",
      "[Parallel(n_jobs=-1)]: Done   5 out of   5 | elapsed:  1.2min finished\n"
     ]
    }
   ],
   "source": [
    "lr8 = KNeighborsClassifier(n_neighbors=3)\n",
    "\n",
    "y_pred8 = cross_val_predict(lr8, X, y, cv=5, n_jobs=-1, verbose=5)\n",
    "\n",
    "model.append(\"KNeighborsClassifier\")\n",
    "acc.append(accuracy_score(y, y_pred8)*100)\n",
    "pre.append(precision_score(y, y_pred8)*100)\n",
    "rec.append(recall_score(y, y_pred8)*100)\n",
    "f1.append(f1_score(y, y_pred8)*100)\n",
    "#roc8 = roc_curve(y, y_pred8)*100\n",
    "roc.append(roc_auc_score(y, y_pred8)*100)\n",
    "\n",
    "conf_matrix8 = confusion_matrix(y, y_pred8)\n",
    "class_rep8 = classification_report(y, y_pred8)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### DecisionTreeClassifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-08-24T20:14:28.046065Z",
     "start_time": "2020-08-24T20:14:09.051646Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Using backend LokyBackend with 8 concurrent workers.\n",
      "[Parallel(n_jobs=-1)]: Done   2 out of   5 | elapsed:   17.5s remaining:   26.4s\n",
      "[Parallel(n_jobs=-1)]: Done   5 out of   5 | elapsed:   18.2s finished\n"
     ]
    }
   ],
   "source": [
    "lr9 = DecisionTreeClassifier(random_state=1337)\n",
    "\n",
    "y_pred9 = cross_val_predict(lr9, X, y, cv=5, n_jobs=-1, verbose=5)\n",
    "\n",
    "model.append(\"DecisionTreeClassifier\")\n",
    "acc.append(accuracy_score(y, y_pred9)*100)\n",
    "pre.append(precision_score(y, y_pred9)*100)\n",
    "rec.append(recall_score(y, y_pred9)*100)\n",
    "f1.append(f1_score(y, y_pred9)*100)\n",
    "#roc9 = roc_curve(y, y_pred9)*100\n",
    "roc.append(roc_auc_score(y, y_pred9)*100)\n",
    "\n",
    "conf_matrix9 = confusion_matrix(y, y_pred9)\n",
    "class_rep9 = classification_report(y, y_pred9)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### AdaBoost"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-08-24T20:29:24.423956Z",
     "start_time": "2020-08-24T20:14:31.829596Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Using backend LokyBackend with 8 concurrent workers.\n",
      "[Parallel(n_jobs=-1)]: Done   2 out of   5 | elapsed: 13.4min remaining: 20.0min\n",
      "[Parallel(n_jobs=-1)]: Done   5 out of   5 | elapsed: 14.9min finished\n"
     ]
    }
   ],
   "source": [
    "lr10 = AdaBoostClassifier(DecisionTreeClassifier(random_state=1337))\n",
    "\n",
    "y_pred10 = cross_val_predict(lr10, X, y, cv=5, n_jobs=-1, verbose=5)\n",
    "\n",
    "model.append(\"AdaBoost\")\n",
    "acc.append(accuracy_score(y, y_pred10)*100)\n",
    "pre.append(precision_score(y, y_pred10)*100)\n",
    "rec.append(recall_score(y, y_pred10)*100)\n",
    "f1.append(f1_score(y, y_pred10)*100)\n",
    "#roc10 = roc_curve(y, y_pred10)*100\n",
    "roc.append(roc_auc_score(y, y_pred10)*100)\n",
    "\n",
    "conf_matrix10 = confusion_matrix(y, y_pred10)\n",
    "class_rep10 = classification_report(y, y_pred10)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Model comparsion"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Naive Predictor Performace\n",
    "\n",
    "What if we chose a model that always predicted an individual as repeater, what would that model's accuracy, precision, recall and F-score be on this dataset?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-08-24T20:51:59.858874Z",
     "start_time": "2020-08-24T20:51:59.676361Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Naive Predictor:\n",
      "\tAccuracy score:\t19.87\n",
      "\tPrecision score:19.87\n",
      "\tRecall score:\t100.00\n",
      "\tF-score:\t23.66\n"
     ]
    }
   ],
   "source": [
    "# Calculate accuracy\n",
    "fol_c = df.query(\"folgebuchung == 1\").count()[0]\n",
    "n_fol_c = df.query(\"folgebuchung == 0\").count()[0]\n",
    "accuracy = fol_c / df.shape[0]\n",
    "\n",
    "# Calculating precision\n",
    "precision = fol_c / (fol_c + n_fol_c)\n",
    "\n",
    "#Calculating recall\n",
    "recall = fol_c / (fol_c + 0)\n",
    "\n",
    "# Calculate F-score using the formula above for beta = 0.5\n",
    "fscore =  (1  + (0.5*0.5)) * ( precision * recall / (( 0.5*0.5 * (precision))+ recall))\n",
    "\n",
    "# Print the results \n",
    "print (f\"Naive Predictor:\\n\\tAccuracy score:\\t{accuracy*100:.2f}\\n\\tPrecision score:{precision*100:.2f}\\n\\tRecall score:\\t{recall*100:.2f}\\n\\tF-score:\\t{fscore*100:.2f}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Confusion Matrix & Classification Report\n",
    "<img src=\"img/conf_matrix.png\" alt=\"Drawing\" style=\"width: 400px;\"/>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-08-24T20:52:03.962600Z",
     "start_time": "2020-08-24T20:52:03.948637Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model1 Basic Logistic Regression:\n",
      "\tAccuracy:\t78.13\n",
      "\tPrecision:\t25.9\n",
      "\tRecall:\t\t5.43\n",
      "\tF1:\t\t8.98\n",
      "\tROC AUC:\t50.79\n",
      "\n",
      "\n",
      "Model2 Normalised Logistic Regression:\n",
      "\tAccuracy:\t79.68\n",
      "\tPrecision:\t47.03\n",
      "\tRecall:\t\t18.16\n",
      "\tF1:\t\t26.2\n",
      "\tROC AUC:\t56.54\n",
      "\n",
      "\n",
      "Model3 Logistic Regression (w. dummies):\n",
      "\tAccuracy:\t79.59\n",
      "\tPrecision:\t47.53\n",
      "\tRecall:\t\t26.23\n",
      "\tF1:\t\t33.8\n",
      "\tROC AUC:\t59.52\n",
      "\n",
      "\n",
      "Model4 Balanced Logistic Regression (upsampled):\n",
      "\tAccuracy:\t58.99\n",
      "\tPrecision:\t58.24\n",
      "\tRecall:\t\t63.51\n",
      "\tF1:\t\t60.76\n",
      "\tROC AUC:\t58.99\n",
      "\n",
      "\n",
      "Model5 Balanced Logistic Regression (downsampled):\n",
      "\tAccuracy:\t59.82\n",
      "\tPrecision:\t59.97\n",
      "\tRecall:\t\t59.04\n",
      "\tF1:\t\t59.5\n",
      "\tROC AUC:\t59.82\n",
      "\n",
      "\n",
      "Model6 Random Forest:\n",
      "\tAccuracy:\t64.51\n",
      "\tPrecision:\t27.18\n",
      "\tRecall:\t\t46.82\n",
      "\tF1:\t\t34.39\n",
      "\tROC AUC:\t57.86\n",
      "\n",
      "\n",
      "Model7 GaussianNB:\n",
      "\tAccuracy:\t60.65\n",
      "\tPrecision:\t24.8\n",
      "\tRecall:\t\t48.26\n",
      "\tF1:\t\t32.76\n",
      "\tROC AUC:\t55.99\n",
      "\n",
      "\n",
      "Model8 KNeighborsClassifier:\n",
      "\tAccuracy:\t61.41\n",
      "\tPrecision:\t22.55\n",
      "\tRecall:\t\t38.74\n",
      "\tF1:\t\t28.51\n",
      "\tROC AUC:\t52.88\n",
      "\n",
      "\n",
      "Model9 DecisionTreeClassifier:\n",
      "\tAccuracy:\t63.88\n",
      "\tPrecision:\t28.07\n",
      "\tRecall:\t\t52.36\n",
      "\tF1:\t\t36.54\n",
      "\tROC AUC:\t59.55\n",
      "\n",
      "\n",
      "Model10 AdaBoost:\n",
      "\tAccuracy:\t71.89\n",
      "\tPrecision:\t35.28\n",
      "\tRecall:\t\t49.74\n",
      "\tF1:\t\t41.28\n",
      "\tROC AUC:\t63.56\n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "for entry in range(0,len(acc)):\n",
    "    print(f\"Model{entry+1} {model[entry]}:\\n\\tAccuracy:\\t{round(acc[entry],2)}\")\n",
    "    print(f\"\\tPrecision:\\t{round(pre[entry],2)}\")\n",
    "    print(f\"\\tRecall:\\t\\t{round(rec[entry],2)}\")\n",
    "    print(f\"\\tF1:\\t\\t{round(f1[entry],2)}\")\n",
    "    print(f\"\\tROC AUC:\\t{round(roc[entry],2)}\\n\\n\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Best results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-08-24T20:52:21.336065Z",
     "start_time": "2020-08-24T20:52:21.327090Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "LR6 Random Forest\n",
      "\n",
      "[[122148  35707]\n",
      " [ 19668  19466]]\n",
      "\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.86      0.77      0.82    157855\n",
      "           1       0.35      0.50      0.41     39134\n",
      "\n",
      "    accuracy                           0.72    196989\n",
      "   macro avg       0.61      0.64      0.61    196989\n",
      "weighted avg       0.76      0.72      0.74    196989\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(\"LR6 Random Forest\\n\")\n",
    "print(conf_matrix10)\n",
    "print(f\"\\n{class_rep10}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-08-06T11:38:46.979303Z",
     "start_time": "2020-08-06T11:34:46.984Z"
    }
   },
   "source": [
    "### Feature Importance"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-08-24T20:53:59.724917Z",
     "start_time": "2020-08-24T20:53:11.427384Z"
    }
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAoAAAAFgCAYAAAArYcg8AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nOzdeXhU5fn/8XcIm2hFyqKApdpabpe4BpHNioJ1YxFRqCKiX5W2Ql3auoKKFjcQd7Rfv/hTWqWIFsF9XxEFiWuq3rWKyFaooggoYUl+fzxn4iSZJBPIIZDzeV0XF5mz3nOemTP3eZZzckpKShARERGR5GhQ1wGIiIiIyJalBFBEREQkYZQAioiIiCSMEkARERGRhFECKCIiIpIwSgBFJCtmllPXMWwtdCyqV5+PUX1+b1D/358EDes6ANl6mdnLwCHAfu7+Sbl5BwDvAIe7+8tbPrrqmdnnwOPuPtLMegIvAQe7+7wY93kf0Mnd8zLMywM+AE519wfSpjcEVgA/An7u7p+lzesMzAGOdfenstj/y8Bqd++TZbw9qea4mFkTYFy03IxstlvJdk4FrgdaAle6+7hN3VaGbX8O/LSS2XPcvYuZjQH+5O47bMZ+dgUmAacCX1ayzOnAvVVs5hh3f9rMSoAL3f3GTY2nOtE+AH7v7ndkmH8Y8DKwZnOOS4bt9geOBX5TxTIvA4eVm/w98Anwf5niraXYXib6jpjZbsB84CR3fziLdcuUf03X34yYq7pf24Pu/uta3Nc+wO3AEbW1Tdk6KQGU6jQF7jazI9x9W75p5NtAV+CjOozhn8B/ozgeSJt+CNAM+BY4EvjftHk9gPXAa1nu4xxg42ZHWlZb4NwaxFCZW4H3gDHAZ1UvukkeBiZkmL4q+n8S8MRm7qM3cFSWyx4NrMwwPfUZ7Aos2Mx4slECDAQyJVQnxrTPC4DVWSz3OvCntNc7AKcDt5sZcSWBaZYSyuFfWS5fvvxruv7muB2YkmF6xguRzXAS0LmWtylbISWAUp2VQE/gTMIP6DbJ3b8F3qzjGErM7BXCD0a63oTEaCEVE8DuhBqsbH5McfcPayPWmPwYeMrdX41p+8vcvdIydvdFwKKY9p1JgbtX+uNcVay1bDZwqJm1dvf/piaaWQNCYvg+8PMtFEt535Q/Dmb2ItAJGEnmpLXWuHsRm3Fe2Nz1a+iLLfiZkQRQAijVmUWoQRhvZo+7+38qWzBqDhlHaNZpBrxIaHL7JJo/BuhDqEk6g1AjNoTQhDKAcMLvBvyHUCvwMSEZygccGO7ub0XbagSMBk4mNP19R2iiPM/dF2aIrWc0/2DCFfP8St7GZHc/PVrnSGAssB/wFfD/gKvcfWM0v2E0//To/U4Ccis7PpGXgOPNrJm7fxdNO5LQDLcAGGNmDdy9OJrXHfhL2vvIJxzjrsAaYCpwcWpb5ZuAozK5FTicUCNzE6FmalHqfUYOMLNbCD+8S4Br3X1SWhMXwENm9oq79zQzA26J4mhASDIucvf3y7/htGMPMM7Mxrl7TtTP6CxC7eIehORsorvfkrZuCTCK8DnZBfjNpja1lW8CzrRt4KnoeB0H7ESorRvr7tPLNe3+18yucvcxmxJL2v4vJNRczgdOcfe/p83vAzwG7OHun5rZHsCNQC9CLe9jwAVVJZmRp4EDgP6UvYjrTuh28FfCdy+132zK5RBgPHAgoYb6BcKxXZDetBu9x93d/fNsj4u7F5vZe0C/aBs9CZ+f3xJqjzcAh7r752Z2MnAZ8AtgMXCLu9+eFucOwM2ERLc4ipm0+btRrgk32t/VwEHAN8CDhM/JrylX/sB9qfUJ55WXgG7u/kbaPkYSvrNt3H11dd/hzZHF+SGHUK5nEY7ZekIC+wd3/yD6jlwZLVtCOE9/ToZuImb2DeF4j4m+GzcCNwAXE86XB7r7d2Z2LvB7oAPwb+Bqd38wbTvHEo733oRz1OOEz9KKzT0eUjUNApFsnEO4WLi9sgWivjFzCSeVcwgnjt2BWWbWLm3R/QlJ2EDg2rTp9wDPEk76iwg/So8QTmCnADsC96ctfzPhpHI98CvCCboXISmpTqrZJv3f3YQf1b9F76cXIRlIJafjgT8Ct6Vt5xbCyfR6QiK6PzC4mn2/RDiWB0f7+RGhueU54HmgBSEJI/rB35nw44qZ7Q28SkjIBxFOtIOBaZl2ZGZNo3WNUB4XAucRmpXLuxl4iJCgv09o9t8vOlYnRMtcRihbCH0BG0b7/zXQCnjCzDIlwKnmdwifodTf1wJ3ATMJyclDwI1mNrbc+mMINUG/pepm6Bwza1juX3UJeflt30jo+3QuIQn8kJD47kVoPk7FdjTV14jnZoinQuf6KDl6g5BEpBsEzI2Sv50JF2M/BU6L4u0KPGtmjauJYy3wJD+UY8pJhCRybbnpVZaLmTWLtrc4mn82IVmaGq1/DqF/8OtRjEuriS+TX1DxIu1yYDhwWZT8DSM0ib5COG9MBm42swvT1plK+P7+idCKcTLhIjOjqM/tc4SWj8GEZOgswne9uvJ/hXDuylSOj0XJX42+w+U0qOrzneW2/0hI0iYRmrJ/T0i87ovmTyKci78nlF1NukzsRDjPDAFGRcnflYRuGVOBvoRj+3czOymK+afAdMJn5dgovr7AxBrsVzaRagClWu6+0MxGAbeaWT93fzTDYhcA2wFHpmokopqAzwhf6j9GyzUEzk9dSUZX4ADTUoMCopPa08AD7j4xmtYSmGRmO7n7N0BrwlXi/4vWfyWqlRqSxfsp02xjZl0JtXhXuPsL0eSxwJtpnaufNrMVwH1mNp7QX++3hBPdLdF2XqCaPl3u/pGZLSOcXF8hNK+XAK9GJ8wlhBrBuYREbQ1hEAiEH8BlwHHRe8DMPgFeNbNfZmhaHUpIGPZ0939Hy38MZBrscY273xot8zbhCr6nu79vZu9Ey3zi7h+aWWtgT8KV/DPROl8QEvUdKNfvLdX8HoonNGNF5fkHYLy7j44WfTZKkC40s1vSaraedfe7qjqukXP4IUFNWRPFVJky2zazXwLPuftD0etZhGPe0N3/a2afRotW2bwbyVRbfiEhySxvCqGWfYcoUWhMSGrGRPPPJ/THTf9+zSEMmPg14YKpKg8DfzOz5u6+MjrOAwk1f/unFsqmXAgXdj8GbkvVdJnZl8ARUe31h2b2LaEmuromy5yoJh0gh9Df9HeEmsULyi17i7s/Fu2vASFRfcDdU7WXz0a1Vpeb2Z2EZu3jgF+napzMbC6V1/4DXBrNPz6tpn87YBhhoFaZ8k87f6W6eDwIDDKzP0av2xJqWgdGi9X0O5zuhuhfun8CqQFn2Wz7J8CfU991wnmzBXBT9NlbZGaLgOJU2UXf22zkEs6hqXPCTsAlwA3ufnm0zLPRRe/1hAuLzkAT4Hp3Xxqtt5rKB3RJLVICKNm6g/ADPzFK7Mr7JfBS+o9idIJ8gYoj/TINxJib9vey6P/0ROWr6P+dCP2GBgNEtYt7AnsREqYmWb2bSLT+Pwi1j9dF05oRTkyj0n6cICSlDQjNqf8hnPBKR+a6+1oze5Kodq8KL/NDLVhvYHZa888LhJrMa6L385q7r4vmHU6oeduYFtcbhGS0F+HqP11PoDCV/EUxFphZph/A2WnLrDCzVYRjnclXhE7v/2dmvQm1Qc+4+2VVvelyugCNCT8C6aYSfjS6EJqCINTCZWMa5Zr4qH5ATPltzwbOjn64HyeMIv9jxdWy0puKg0AqdE+ITCPUwvYhHIOjCc2zqaaywwll/U1a2S+M4u9F9QngE4Rj0YcwAKk7oVb9KdISQLIrl1cIydBjZjY12vaL7v5KNTFkciyhGTLd94RjUb7/X3pZdQTaEWqd07+jTxGaEzsTzgupaQC4+1Ize4PKdQP+nkr+onXuSMWSRTI0hXCxewjhIvMkwvczFUNNv8PpbqVsKwiEY5VS7bbd/bzofaQu4vYk1LhBOHdm1de4Cull1IVw0ZKpjP7HzHYHCoAiYG7aZ+nR9OMv8VECKFmJ+uWcTWjOuw74v3KLtADezbDqMmCftNdr3H1NhuVWZZhWaZ8YM+tGaKbaj/Aj+w7hZJj1/auiWpZ/ROsN9R9GObcgJHrXRf/Ka0s4aUHFEXiV9pFM8xLw5+jvIyk7Ivh54C9RbN0JzTEpLQn91DLdWqNthmmtCKOOy1uWYVr5Y11MJV1Eos9Cb0Lt1PHA/wDfRzWjYzy70eItKokl9XrHtGnLs9gewH+95rf4Kb/tcwl9IIcSfhiLzewR4H+imsyaeC+LWkIA3H15dLF0EiHZGgS8nKoVIZT9IVRMliCLz5y7rzGzZwjNwA8QRv8+Hl20pC9abbm4+6qopvQKQs3YCEJielmWNbXpZvFDTV8JIQH5zN0zvc/0smoZ/T+FzCNj2xLey/oM5fYfQnKdyY/J/vNWgbu/bWZOKMc3CeU4PVUjR82/w+kWVfP5rnbbZrYn4dzdg/Cdf4+QIEINzp1VyFRGszMtCLR199kWuttcSmiO/hPwHzM7x90fqYV4pApKACVrUSfh8YSagH+Wm72C0F+tvF34ofauVphZc0LtzCxgYFrz5jhCZ/dsTYyW7xY1K6ekTohjCf2gylsC7Bv93SZ6ndKy4uIVvAy0jvob7UVI+lKeJzSl9yL03Xshbd7KKJ5MP7KZEo0lhKa08loTBtVsMg8Dbc6MLgq6EPpJXUGoAXiwqnUjqQ7eOxP6kqXsEv1fq5+ZbLn794R+X1dGXQpOJDSt3UBomozT34G7oia5vvzQbQJC2T9FOMblZbp4yuQfwP+a2faEJslzMyyTVbm4+z+BwdGFyqGEvqV3mtnb7j6H7K3chKQdfqhZHUHZ1oOU+YSya5TWbSSlJbAuwzqp7bZOn2BmPyYMRHs9y9j+DpxhZjcRahTHlNt+Tb7DNVHltqNm88cIZbgv8GF0MXcOVd/aKHVBV3pBGHUJ2D6LeCD0wcw0+t4B3P11oE/U8tILuIjQ77aDuy/JsJ7UEg0CkZq6mjCSq3zN2CzgcDNrlZoQ/d2L7E+c2dqTcHV/S1ry14BQm5bVVayZ/Y6QtIxw93fS57n7KsKV8c/dfV7qH+FH4zpCP5o3CLWAJ6Rts2EUQ5Xc3QnJ2XmEUYbz0uYtITSRn034MX4vbdVZ0XsvSItpIaE/TYUbTxOak/KippZUjHnAz6qLsZwyzTFmtp+ZLTWzg9y92N1nR/FuIIz0y8ZcQm1W+Q7zg6PtZPpRj5WZ5ZpZoZmdD6Gc3P0aQlmn3lecTVPTCZ/fawhNZ/9Im5cq+w/Syr6QkFxkGtSTyWOEi/5RhOb9pzMsU225mNnRZrbcwm1l1kX9Zn8fLbcljhOEOwR8Bexa7jvaklC73pxwoQVlv6MtCBcslZkNHBOdT1IGEy44c8nufU0h9GEbRag5fSltXk2/wzVR3bZbE0Z13+3uhf7DnQaOjv5PnTvLv8fUBXH6YL4uVF+BNIfwWWpTrozyCBcyOWZ2lpl9ZmaN3P27qI/naMKxbp/9W5dNoRpAqZGoyeg3hFu8pLuZMJDiOTP7M+FkMpqQNGUzMrcmPibUelxuYcDIdoSagP2BEjPLqaoZ0sKgj1sJJ/V3LNzSInXyK4oSwiuAGWa2kjAauRWhRrCY8CP8Xao21MzWEprGf0eoKfmU6r1M+JF9PEN/l+cJA0xmpp2kIfywzQammdn/IyQJlxMS0neo6H7Cj9DjZnYF4aR6LeGKvjjD8pVJXcn3jjqV/5Pwo/BXC7eNWEFoCiwmy1GDUf/Q2wgDCzYQktVfEgZJTHD3r2sQX61w943RwIorozL9mPBDdyg/NKulapJOMLNnvQa3N8li/6vM7HHCSNcnyh2Dmwijf58ys1sJP6x/JPQlHV1hY5m3v9LMnic0s/0jqu0sv0y15RINpMgBppvZDYTv+PmEY5NKdr4h3FqoJ+E+lhX2tTncfUP02bspasJ+gTA45TrCwJj50SCM+4FbLIyI/4LQ1Nioik1fSxgN/rCZ3U34bl0D3BGVT5nyryS2T8xsHqEcJ5b7ftf0O1wTVW7b3ZdZGKx1voWBaBsJ39vUU4OaRf9/AzSz8DSXuYS7AiwG/mxm6wndM64m803OS3kYNHUbMCFKvOcSWlyuIZzbvjWzVwl3BnjIwsCdxoTP83wydymSWqQaQKkxd3+JcE+89GkLCT+USwgd0u8hjIjt6uEGvLW5/5WEJqwWwKOEptwvCQlVA0JfqaocRfgR6ENI3N4k1PK8QUj28DDSuT/hliyPEpLYNwiPvkv1l7uCUANzDqG2ZiXhdjLZeCmK4fkM856P5pVJst29gHCLktbR/u4hnJh7uvvi8huJ+lEdRagF+Fv0Hm6LXmfd2TvqQ3UDoV/c39x9A6Hz/ieE5qYnCDUPfbxmN6K+iB/uw/c4ob/UnwhdDOrKuYRjNQp4htC/8Y/unuqL+UI0/XbKPsGitkwhJOp/T5/o7l/wQ7+t+wn9BBsAvd29Jj+U/yB8tsoP8khXZbl4uD/b0YTbx/yN8J1pGsWSasa8iTCo4Gkyd0PYbNHgjN8SRks/SUhKHiKMgk1dAKZuYH8Voe/jHEJNaGXbfJNwW6m2hAEVownfmdRnMtvyr6wca/Qdrokst30C4bs/jXBPw2aEwUrww8C0qYTBGQ8R+kZvJHwG1hJqqa8kfEZKB5dV4SJCYno24bNwHuE8dHoU878I3R3aEEaqTyHUmh5ZST9QqUU5JSXb8tO9RKQyZrYv4Sa8j6ZN25HQUfsid7+t0pVFRKReUxOwSP21EzDTzK4l3IB1R8KIy1X8cNNeERFJINUAitRjZjaE0FTVkdBX61XCo6E+rtPARESkTikBFBEREUmYetUEXFBQ0ITwFIalxH8bAhEREZGtWS5hUNNb+fn5Rekz6lUCSEj+qnpYvIiIiEjSHEq4V2Sp+pYALgXo2LEjjRs3rutYYlVYWEhe3ubeN1S2JirT+kXlWb+oPOuXpJTnunXr+Ne//gVRfpSuviWAGwEaN25MkyZN6jqW2CXhPSaNyrR+UXnWLyrP+iVh5VmhW1ysCaCZnUK4kWYjwmO7JpabP4Bwg85c4C1guLuvM7NhhMfXpB5A/oS7j4ozVhEREZGkiC0BNLP2hEe+5BOemTrbzF5KPSkgeiD5HcBB0SNqphLuDn434ekLf3D3v2fcuIiIiIhssjgfBdcbeNHdV7j7GsJjXk5MzYym7RYlf80Ij4JJPfvyYGCYmX1gZvdHzxEUERERkVoQZxNwO8p2OlwKdE5fwN3Xm9kxhGdbLgaeTVv2RsKDra8l1BQOyXbHhYWFmx71NqSgoKCuQ5BaVldl2rBhQ0pKSsjJyamT/ddHDRs25L333qvrMOqNkpISSkpKKC4urrMYdM6tX5JennEmgA2A9LtM5wAVvrnu/hTQMnpc1V3AKe4+IDXfzMYBn9Zkx3l5efW+c2dBQQH5+fl1HYbUoroo0y+++IKcnBx23nlnGjVqpASwFq1Zs4btt9++rsOoN0pKSvjqq69YtWoVu++++xbfv8659UtSyrOoqKjSSrE4m4AXEW4+mLILsCT1wsx+bGa/Spv/ALCfmTU3swvSpucAG2KMUySx1qxZQ/v27WncuLGSP9mq5eTk0LJlS9auXVvXoYjUC3EmgM8DvcysddTHbyDwdNr8HOB+M+sQvT6JcJPC1cBFZnZINH0k8EiMcYokWoMGcZ4GRGqPLlJEak9sZ353XwyMAl4C3gWmuPtcM3vSzDq5+1fAcOBxM3sPMMJD6jcCg4C7zOwjwijii+KKU0RERCRpYr0PoLtPAaaUm3Zs2t8zgBkZ1nsNOCjO2EQks7Ub19I0t2mdbHfRokUcffTR/PznPy8z/S9/+Qtt27atZK3MFi5cyF133cW1115b41ircsYZZzBkyBB69+4NwA033MDUqVOZM2dO6ROIevTowb333ssvfvGLjNvo378/M2fOrHQf06dPZ+7cuVx//fVlpq9atYpLLrmEiRMnVrKmiEh26tuTQERkMzXNbUrO5NpvaisZVlL9QkCbNm2qTI6ytWTJEhYuXLjZ2ymvS5cuFBQUlCaAs2fP5oADDqCgoICuXbuyYMECmjVrRrt27Srdxqa+v5UrV/LRRx9t0roiIunU+UdEtglffvkl55xzDieccAIDBw5k9uzZACxbtowzzzyTQYMG0bNnT2699VYAxo4dS2FhIVdddRVz5sxh6NChpdu65JJLmD59emmN48knn8wZZ5zBxo0bue666xgwYAD9+vXjvvvuqxBH165deeedd0r33bhxY4466ihmzQrPWZ83bx7du3cHYMaMGQwYMID+/ftz2WWXUVRUBICZAaFG73e/+x3HHXccv/3tbzn++ONZtGgRAAsWLGDo0KH06tWL0aNHl76n5cuXM2LEiNo+vCKSMEoARWSrsnz5cvr371/6b9KkSQBcc801DBw4kOnTp3PXXXdxxRVXsHr1ah5//HH69OnDtGnTeOyxx5g8eTIrVqxg9OjR5OXlceWVV1a5v/nz5zN+/Hjuvfdepk2bBsAjjzzCww8/zAsvvMC8efPKLL/PPvvwxRdfUFRUxKxZs+jevTvdu3evkAB++umnTJs2jalTpzJz5kxatmzJPffcU2ZbEydOZPfdd+eJJ55gxIgRqYe2A7B06VJuv/12nnrqKV599VU++eQTRo8eTZs2bdQELFuNtRsrPGJ2m9i2qAlYRLYylTUBz549m88++4zbbrsNgA0bNrBw4ULOPPNM3nzzTe655x4++eQT1q9fz/fff5/1/lq2bMmuu+4KwBtvvMFHH33Em2++CcB3332Hu9OpU6fS5XNzc9l///354IMPmDVrFkOGDOEnP/kJa9euZeXKlbzzzjuMGjWKBx98kAULFjBo0CAA1q9fz957711m36+//jo33ngjAPvuuy8dO3YsndepUyd22mknADp06MDXX3/Ndtttl/X7EtkSmubmkjN5cizbLhk2LJbtSqAEUES2CcXFxUyePLk0KVq+fDktW7bk+uuvZ+HChfTp04fevXsze/ZsSkrK9jfMyckpM239+vWlfzdt+sPAlI0bN3LhhRfyq1+FW5SuWLEi482cu3Tpwttvv83777/P+PHjgdA0/MILL9CiRQt22GEHiouLOeaYY0qbb9esWcPGcjUaubm5FWJNadjwh9Nz+fhFRDaXmoBFZJvQpUsXpkwJNxX497//Td++ffn+++95/fXXOfPMMznmmGOYP38+y5Yto7i4mNzcXDZsCPeQb9GiBQsXLqSoqIhvvvmm0kdAdenShWnTprF+/XrWrFnDKaecwrvvvlthua5duzJz5kw6duxYmqh1796de++9t7T/X6dOnXjuuef46quvKCkpYcyYMUwuV1PStWtXHnvsMQDcnU8++aTKe901bNiw9D2JiGwO1QCKSBlrN67NesRuTbe7ObeXGT16NFdccQV9+/YFYNy4ceywww785je/4aKLLqJp06bssssu5OXlsWjRIvbaay9WrVrFhRdeyPjx4znssMM47rjjaN++faWPgPr1r3/NggULGDBgABs2bOCEE07gkEMOqbBcx44d+eabbzjllFNKp3Xp0oXzzz+fbt26lS4zcuRIhg0bRnFxMXvttRfDhw8vs50RI0Zw6aWX0rdvXzp06ECrVq3K1EiW17JlS9q1a8fQoUP529/+VuNjKCKSklOfmhUKCgp2A+brWcCyLaqLMv3oo4/Ya6+9tug+kyKbZwHPnDmTXXfdlfz8fJYsWcKpp57K888/r6ezVKGuPrM651ZuW+wDmJTyTHsW8O75+fmfp89TDaCISB352c9+xpVXXklxcTENGjTg6quvVvInIluEEkARkTqy7777Mn369LoOQ0QSSJeaIiIiIgmjBFBEREQkYZQAioiIiCSMEkARERGRhFECKCJlxPX8TT3XU0Rk66FRwCJSRlzP9sz2nl6rV69mwoQJvPXWW+Tm5rLjjjtyySWXsM8++9R6TClDhw5l5MiRGW/6nL5M6ubL/fv3z/i84pru8z//+Q/NmjUrnTZo0CCGDBlS421deumljBw5kvbt229WTOkeffRRnn76ae68804A/vWvf9G3b1/Gjx9Pv379AJgwYQKNGzfm97//fcZt3HrrreTl5dGrV69K92NmuHuF6bfddhvdunUr8xxmEak9qgEUka1GcXExZ599Ns2bN2fGjBnMnDmTESNGcPbZZ/P111/XaWxz584t/Xtzk7+UsWPHMnPmzNJ/m5L8AcyZM6fWnxXcpUsX3nnnndLXs2bNokePHsyaNat02rx580qffJLJeeedV2XyV5W33nqrwrOTRaT2KAEUka3GnDlzWLp0Keeee27pM3a7dOnCddddR3FxMXPmzGHo0KGly19yySVMnz6dRYsW0b9/fy644AL69u3LxRdfzNSpUxk8eDBHH300n376KQBHHHEEixYtKt1X+rYANmzYwOjRoxk8eDC9evXinHPOYe3atYwdOxaAk046CQi1Vhs2bKBHjx58+eWXAHzzzTf06NGD9evX8+qrrzJ06FCOP/54Ro4cWePk9e6772bAgAH069ePcePGlSZ3N998M4MGDeKoo45i6NChfPnll9x9990sX76c4cOH8/XXX1f6HlO1nEcddRQfffQRr776KieeeGKlMbZp04YWLVowf/58ICSA5513HnPnzqWkpISioiI+//xz9t9/fxYsWMAZZ5zBgAEDOPnkk/nwww/LlA/AX//6V371q18xcOBALrzwQm6//fbSfV1xxRX069ePfv36sWDBAmbMmEFhYSGjR4/OWDsoIptPCaCIbDU+/PBD9txzzwpPwzjssMNo2bJlleu6O2effTYzZ87k7bffZvHixTz44IP06dOHBx98MKv9v/POOzRq1IgHH3yQ5557jlWrVvHKK68wevRoAB566KHSZRs2bMjRRx/N008/DcCzzz7LkUceyapVq5gwYQITJ05kxowZ9OjRgxtvvDHj/kaPHk3//v3p379/6XOFX331VQoLC3n44YeZMWMGy5Yt49FHH2XBggV89tlnTJ06lWeeeYa2bdvy6KOPMnz4cNq0acPdd99NixYtqnx/ZsYzzzzDzp8xddoAACAASURBVDvvzIQJE7jnnnuqjLFLly68/fbbrF27lkWLFrHffvux66678vHHH/Pee+9x4IEH0rBhQy6++GIuvPBCHnnkEf785z9zwQUXlNnOxx9/zAMPPMD06dOZMmUKCxYsKDO/W7duPProo3Tv3p2pU6dy/PHHk5eXx9ixYzGzakpNRDaF+gCKyFajQYMGm/wc71atWrH33nsDsMsuu9C1a1cA2rVrV1ojVp2DDz6YnXbaiQceeIDPPvuMzz//nO+++67S5fv168d1113HqaeeyuOPP84FF1zAe++9x9KlSxk+fDgNGjSguLiY5s2bZ1x/7NixFfodvvHGG7z//vuccMIJAKxdu5Z27drRv39/Lr74Yh566CHmz5/Pu+++S4cOHbJ6Xyn77bcfQGmMp512GkClMXbt2pWXX36Z1q1bl/bF69atG3PmzOG7776je/furFmzhsLCQi699NLS9b777rsyNYpvvPEGhx9+ODvssAMAxx13HN9++23p/N69ewOwxx57MG/evBq9JxHZNEoARWSrkZeXx5QpUygpKSEnJ6d0+k033US3bt1o0KBBmb5u69evL/27cePGZbaVm5ubcR+p9Tds2FBh3gsvvMBtt93GaaedxgknnMDXX39dZd+6/fbbj5UrV/L++++zbNkyDjzwQJ5//nkOOuggJkyYwPbbb09RURFr1qzJ7gAAGzduZNiwYZxxxhkAfPvtt+Tm5lJYWMgf//hHTj/9dI466qgKxyKb99i0adPSfRx00EH85S9/Aag0xs6dO3Pbbbexww470KNHDwB69OjBfffdx8qVK7n88sspLi6mcePGZfpF/uc//2GnnXYqfZ1KhCuTau7Pycmp9b6MIpKZmoBFZKvRqVMnWrZsyR133FE6AOC1115j+vTp7LHHHrRo0YKFCxdSVFTEN998Q0FBQY2236JFC/79738DIdkr74033uCYY45h4MCB7LjjjsyZM6c0jtzc3IxJY9++fbnyyis57rjjANh///159913S5s577zzTsaNG5d1jF26dGHmzJmsWbOGDRs2MGLECJ555hneeustOnfuzMknn8xuu+3Gyy+/XCa21N/Vvcf0GFP9+yqLsXnz5jRt2pTXXnuttEY1Ly+Pzz77jOXLl7Pbbrvxox/9iN122600AXz99dcrDGbp2rUrr7zyCqtXr2bdunU8++yzZRL8TNLfk4jUPtUAikgZazduzPqWLTXdbtNKauVScnJyuPPOO7nuuuvo06cPDRs2pEWLFtx99920atWKVq1acdhhh3HcccfRvn178vPzaxTDueeey5///GfuuOOO0hqtdCeddBJ/+tOfeOKJJ2jUqBEHHXRQafNxr1696N+/f+mghpR+/fpx6623cvPNNwPQunVrrr32Wi6++GIAdt55Z8aPH591jEcccQQff/wxgwYNYuPGjRx66KEMGDCA5cuXM3LkSPr27QuERCwVW8+ePRk+fDiTJk2q9j2mx3j++edTXFxcZYydO3fmzTffLO1f2KBBAzp06FCmyXj8+PGMGTOGSZMm0ahRI26++eYyCV7Hjh057bTTGDx4MM2aNaNFixbVNvUfeuihXHnlldxwww0cdNBBWR8/EclOTn2qbi8oKNgNmJ+Xl7fJ/Yi2FQUFBTX+8ZOtW12U6UcffcRee+21RfeZFGvWrGH77bev6zC2CvPnz+eVV17h9NNPB+B3v/sdJ510EkcccUSNt1VXn1mdcysXx31DIft7h26KpJRnUVERhYWFALvn5+d/nj5PNYAiIhKr9u3b88EHH9CnTx9ycnLo0aMHhx9+eF2HJZJoSgBFRCRWjRs3ZsKECXUdhoik0SAQkYSranSmyNakPnVZEqlrSgBFEmz77bdn8eLFrFu3Tj+uslUrKSnhq6++Kr2VjYhsHjUBiyTYrrvuypdffsmCBQsy3uJENt26desq3JtQNk/Tpk3Zdddd6zoMkXpBCaBIgjVo0IA2bdrQpk2bug6l3ikoKGD//fev6zBERDJSE7CIiIhIwsRaA2hmpwCjgUbALe4+sdz8AcBVQC7wFjDc3deZWQfgfqAN4MAQd18dZ6wiIiIiSRFbDaCZtQeuAXoABwDDzWzvtPnbA3cAR7r7PkBT4PRo9p3Ane6+JzAPuDyuOEVERESSJs4m4N7Ai+6+wt3XAA8DJ6ZmRtN2c/dlZtaMUNv3tZk1An4ZLQ9wH3BSjHGKiIiIJEqcTcDtgKVpr5cCndMXcPf1ZnYMobl3MfAs0Ar41t03pK1Xo2Ff0WNP6r2CgoK6DkFqmcq0flF5VtRxn334UUy3clm1di3/+uc/Y9k2qDwziftxanEe86SXZ5wJYAMg/cZiOUCFO866+1NASzO7FrgLuLDcemRaryp6FrBsi1Sm9YvKs3JxPjs2rmOu8qwbKs/Nk/Ys4AribAJeBLRNe70LsCT1wsx+bGa/Spv/ALAfsBxobma50fS26euJiIiIyOaJMwF8HuhlZq2jPn4DgafT5ucA90cjfiH085vl7uuB14DB0fTTgKdijFNEREQkUWJLAN19MTAKeAl4F5ji7nPN7Ekz6+TuXwHDgcfN7D3AgIuj1c8hjBr+EDiUcCsZEREREakFsd4H0N2nAFPKTTs27e8ZwIwM6y0AesYZm4iIiEhS6UkgIiIiIgmjBFBEREQkYZQAioiIiCSMEkARERGRhFECKCIiIpIwSgBFREREEkYJoIiIiEjCKAEUERERSRglgCIiIiIJowRQREREJGGUAIqIiIgkjBJAERERkYRRAigiIiKSMEoARURERBJGCaCIiIhIwigBFBEREUkYJYAiIiIiCaMEUERERCRhlACKiIiIJIwSQBEREZGEUQIoIiIikjBKAEVEREQSRgmgiIiISMIoARQRERFJGCWAIiIiIgmjBFBEREQkYZQAioiIiCSMEkARERGRhFECKCIiIpIwSgBFREREEkYJoIiIiEjCNIxz42Z2CjAaaATc4u4Ty83vD1wF5ADzgTPc/WszGwZcDyyLFn3C3UfFGauIiIhIUsSWAJpZe+AaIB8oAmab2Uvu/mE0f0fgLuBgd19sZlcDY4DzgE7AH9z973HFJyIiIpJUcTYB9wZedPcV7r4GeBg4MW1+I2CEuy+OXr8PdIj+PhgYZmYfmNn9ZtYixjhFREREEiXOJuB2wNK010uBzqkX7v4V8AiAmW0HXALcnrbsjcBs4FrgDmBItjsuLCzcnLi3GQUFBXUdgtQylWn9ovKsKD8/P9btx3nMVZ4VqTy3XXEmgA2AkrTXOUBx+YXMrDkhEXzP3ScDuPuAtPnjgE9rsuO8vDyaNGmyKTFvMwoKCmL/4smWpTKtX1SedSOuY67yrBsqz81TVFRUaaVYnE3Ai4C2aa93AZakL2BmbYHXCM2/Z0XTmpvZBWmL5QAbYoxTREREJFHiTACfB3qZWWszawYMBJ5OzTSzXOAxYJq7n+/uqdrC1cBFZnZI9HokUVOxiIiIiGy+2JqAo5G9o4CXgMbAJHefa2ZPAlcAPwEOAhqaWWpwyDx3P8vMBgF3RX0D/wWcFlecIiIiIkkT630A3X0KMKXctGOjP+dRSQ2ku79GSA5FREREpJbpSSAiIiIiCaMEUERERCRhlACKyFZr7caN2/T2RUS2VrH2ARQR2RxNc3PJmTw5tu2XDBsW27ZFRLZmqgEUERERSRglgCIiIiIJowRQREREJGGUAIqIiIgkjBJAERERkYRRAigiIiKSMEoARURERBJGCaCIiIhIwigBFBEREUkYJYAiIiIiCaMEUERERCRhlACKiIiIJIwSQBEREZGEUQIoIiIikjBKAEVEREQSRgmgiIiISMIoARQRERFJGCWAIiIiIgmjBFBEREQkYZQAioiIiCSMEkARERGRhFECKCIiIpIwSgBFREREEkYJoIiIiEjCKAEUERERSRglgCIiIiIJowRQREREJGEaxrlxMzsFGA00Am5x94nl5vcHrgJygPnAGe7+tZl1AO4H2gAODHH31XHGKiIiIpIUsdUAmll74BqgB3AAMNzM9k6bvyNwF3Ccu+8PvA+MiWbfCdzp7nsC84DL44pTREREJGnibALuDbzo7ivcfQ3wMHBi2vxGwAh3Xxy9fh/oYGaNgF9GywPcB5wUY5wiIiIiiRJnE3A7YGna66VA59QLd/8KeATAzLYDLgFuB1oB37r7hrT1do0xThEREZFEiTMBbACUpL3OAYrLL2RmzQmJ4HvuPjlqOi4pt1iF9apSWFhYw1C3TQUFBXUdgtQylWlZ+fn5se8jzmOu8qwo7jJVeW5ZKs9tV7UJoJk95e7HlJv2prt3qWbVRcChaa93AZaU205b4BngReCCaPJyoLmZ5br7RqBt+fWqk5eXR5MmTWqyyjanoKBgi/w4ypajMq0bcR1zlWfdUHnWLyrPzVNUVFRppVilCaCZPQx0BH5uZu+nzWoEFGWx3+eBMWbWGlgDDASGp20/F3gMmObuY1PT3X29mb0GDAamAKcBT2WxPxERERHJQlU1gH8CdgP+D/h92vQNwIfVbdjdF5vZKOAloDEwyd3nmtmTwBXAT4CDgIZmlhocMs/dzwLOASab2WjgC+DkGr0rEREREalUpQmgu38OfG5m5u416oOXto0phFq89GnHRn/Oo5JRyO6+AOi5KfsUERERkaplMwikv5mNB1oTBnLkACXuvmOskYmIiIhILLJJAMcRmoPfpeLoXBERERHZxmSTAH7r7jNij0REREREtohsngQy18yOjz0SEREREdkiqroNzCpCk29D4DdmVgSsR30ARURERLZpVTUB522xKERERERki6nqNjALAMzstHKzSoDvzGw7d/84zuBEREREpPZlMwhkKNAVeAHYCPQCPgN2MrPr3P3uGOMTERERkVqWzSCQEqCTu/d39xOAA4DFwP7AiDiDExEREZHal00CuEt6U6+7zwfau/u3hBpBEREREdmGZNME/LWZDQfuIYwA/h/gKzPrSHYJpIiIiIhsRbJJ4M4ETgeKgO+BU6JpJwJjY4tMZBOs3RhfpXSc2xYREdmSqq0BdPd/A93MbCdgg7uvjmZdG2tkIpugaW4uOZMnx7LtkmHDYtmuiIjIllbVjaBvcffzzewx0p4BbGYAuHu/+MMTERERkdpWVQ3gC9H/D2+JQERERERky6jqRtCPRf9PNrNdgf2AZwgjgL/YQvGJiIiISC2rdhCImR0LzAYmAm2AD82sf9yBiYiIiEg8shkFfCVwCPCNuy8FegBXxxqViIiIiMQmmwQwN0r8AHD3d0kbFCIiIiIi25ZsEsDvzKwDUdJnZocCa2ONSkRERERiU2kCaGY/iv68GHgW+LmZvQE8Ek0TERERkW1QVbeB+a+ZzQKeAIYBPwZygTfd/cstEZyIiIiI1L6qmoB/AvwfsBcwBbgN6A0caGaNtkBsIiIiIhKDqu4D+F/gwegfZvZTQgI4DtgD+FFl64qIiIjI1qvaZwGb2e5AP+BXwEFAAXB3zHGJiIiISEyqehbwWKA/oabvKeBO4AV31whgERERkW1YVTWAlwGPAte7+5tbKB4RERERiVlVCaABfYHrzKwj8BzwOPCMu6/aEsGJiIiISO2rdBSwu3/i7je5++HAPsAzwADCs4Cf21IBioiIiEjtyuZJIAAdgNZAU2AdsDG2iEREREQkVlUNAvk9cDhwGLCCMBDkHuAld/9+y4QnIiIiIrWtqj6AxxKSvovd/ZMtFI+IiIiIxKyqG0Efs7kbN7NTgNFAI+AWd59YyXJ/BV509/ui18OA64Fl0SJPuPuozY1HRERERLK4EfSmMrP2wDVAPlAEzDazl9z9w7Rl2gH/C/QCXkxbvRPwB3f/e1zxiYiIiCRVtoNANkVvQq3eCndfAzwMnFhumSHATGBauekHA8PM7AMzu9/MWsQYp4iIiEiixFYDCLQDlqa9Xgp0Tl/A3ccDmFmPcusuBW4EZgPXAncQksWsFBYWbkK4256CgoK6DmGrk5+fH+v24z7mKtOy4i5PiPeYqzwr2pa/oyrPilSe2644E8AGQEna6xygOJsV3X1A6m8zGwd8WpMd5+Xl0aRJk5qsss0pKCjYIj+OUlacx1xlWjfiOuYqz7qh8qxfVJ6bp6ioqNJKsTibgBcBbdNe7wIsqW4lM2tuZhekTcoBNtRybCIiIiKJFWcC+DzQy8xam1kzYCDwdBbrrQYuMrNDotcjgUdiilFEREQkcWJLAN19MTAKeAl4F5ji7nPN7Ekz61TFehuBQcBdZvYRYRTxRXHFKSIiIpI0cfYBxN2nAFPKTTs2w3Knl3v9GnBQnLGJiIiIJFWcTcAiIiIishVSAigiIiKSMEoARURERBJGCaCIiIhIwigBFBEREUkYJYAiIiIiCaMEUERERCRhlACKiIiIJIwSQBEREZGEUQIoIiIikjBKAEVEREQSRgmgiIiISMIoARQRERFJGCWAIiIiIgmjBFBEREQkYZQAioiIiCSMEkARERGRhFECKCIiIpIwiU8A127cuE1uOz8/P7Zt19TajWvrOgQRERGpgYZ1HUBda5qbS87kybFsu2TYMHIm58Sy7a1JybCSug5BREREaiDxNYAiIiIiSaMEUERERCRhlACKiIiIJIwSQBEREZGEUQIoIiIikjBKAEVEREQSRgmgiIiISMIoARQRERFJGCWAIiIiIgmjBFBEREQkYZQAioiIiCRMrM8CNrNTgNFAI+AWd59YyXJ/BV509/ui1x2A+4E2gAND3H11nLGKiIiIJEVsNYBm1h64BugBHAAMN7O9yy3TzsweA04st/qdwJ3uvicwD7g8rjhFREREkibOJuDehFq9Fe6+BniYioneEGAmMC01wcwaAb+Mlge4DzgpxjhFREREEiXOJuB2wNK010uBzukLuPt4ADPrkTa5FfCtu29IW2/Xmuy4sLAw62Xz8/NrsmmpREFBQV2HAMRfnnG/z63lOG4ttsT3M85jrvKsaFv+jqo8K1J5brviTAAbACVpr3OA4k1YjyzXK5WXl0eTJk1qsopspqQk0nG+z4KCgsQcx61JXMdc5Vk3VJ71i8pz8xQVFVVaKRZnE/AioG3a612AJVmstxxobma50eu2Wa4nIiIiIlmIMwF8HuhlZq3NrBkwEHi6upXcfT3wGjA4mnQa8FRsUYqIiIgkTGwJoLsvBkYBLwHvAlPcfa6ZPWlmnapZ/RzCqOEPgUMJt5IRERERkVoQ630A3X0KMKXctGMzLHd6udcLgJ5xxiYiIiKSVHoSiIiIiEjCKAEUERERSRglgCIiIiIJowRQREREJGGUAIqIiIgkjBJAERERkYRRAigiIiKSMEoARURERBJGCaCIiIhIwigBFBEREUkYJYAiIiIiCaMEUERERCRhlACKiIiIJIwSQBEREZGEUQIoIiIikjBKAEVEREQSRgmgiIiISMIoARQRERFJGCWAIiIiIgmjBFBEREQkYZQAioiIiCSMEkARERGRhFECKCIiIpIwSgBFREREEkYJoIiIiEjCKAEUERERSRglgCJbifz8/LoOodTajWvrOoRtnsqzflF5Sn3TsK4DENmW5EzOqesQtoiSYSV1HcIWk4QyVXnWL0kqT4mPagBFREREEkYJoIiIiEjCKAEUERERSZhY+wCa2SnAaKARcIu7Tyw3/wBgErAj8CrwW3ffYGbDgOuBZdGiT7j7qDhjFREREUmK2GoAzaw9cA3QAzgAGG5me5db7H5gpLt3BHKAs6PpnYA/uPsB0T8lfyIiIiK1JM4m4N7Ai+6+wt3XAA8DJ6ZmmtlPge3c/c1o0n3ASdHfBwPDzOwDM7vfzFrEGKeIiIhIosTZBNwOWJr2einQuZr5u6b9fSMwG7gWuAMYku2OCwsLsw5ya7q307asoKCgrkMAVJ61aWsoU5Vn7dkayhNUprUlKeUZ5/vcWo5hXYkzAWwApN+sKAcozma+uw9ITTSzccCnNdlxXl4eTZo0qWm8shl0Uq9/VKb1i8qzfklKecb1PgsKChJxDIuKiiqtFIuzCXgR0Dbt9S7Akurmm1lzM7sgbXoOsCG2KEVEREQSJs4E8Hmgl5m1NrNmwEDg6dRMd18ArDWz7tGkocBTwGrgIjM7JJo+EngkxjhFREREEiW2BNDdFwOjgJeAd4Ep7j7XzJ40s07RYkOAm83sY2AH4DZ33wgMAu4ys4+AfOCiuOIUERERSZpY7wPo7lOAKeWmHZv293uUHRiSmv4acFCcsYmIiIgklZ4EIiIiIpIwSgBFREREEkYJoIiIiCTK1nQLmLUb19bJfmPtAygiIiKyqXIm59R1CLErGVZS/UIxUA2giIiISMIoARQRERFJGCWAIiIiIgmjBFBEREQkYZQAioiIiCSMEkARERGRhFECKCIiIpIwSgBFREREEkYJoIiIiEjCKAEUERERSRglgCIiIiIJowRQREREJGGUAIqIiIgkjBJAERERkYRRAigiIiKSMEoARURERBJGCaCIiIhIwigBFBEREUkYJYAiIiIiCaMEUERERCRhlACKiIiIJIwSQBEREZGEUQIoIiIikjBKAEVEREQSRgmgiIiISMIoARQRERFJGCWAIiIiIgnTMM6Nm9kpwGigEXCLu08sN/8AYBKwI/Aq8Ft332BmHYD7gTaAA0PcfXWcsYqIiIgkRWw1gGbWHrgG6AEcAAw3s73LLXY/MNLdOwI5wNnR9DuBO919T2AecHlccYqIiIgkTZw1gL2BF919BYCZPQycCFwdvf4psJ27vxktfx9wlZlNAn4JHJ82/RXg4iz2mQuwbt26GgXatlGjGi2fraKiIto2ahvLtrcmRUVFdR1CGSrPzbc1lWlc5QnJKdOtqTxB39HNpfKsX+Isz7R8KLf8vJySkpJYdmpmlwLbu/vo6PVZQGd3Hx697gqMd/ce0es9gCeBw4C33H3XaHpD4Dt3b1zdPgsKCnoAr8XxfkRERES2UYfm5+fPSp8QZw1gAyA9u8wBirOYX3465darylvAocBSYGNNghURERGpZ3KBtoT8qIw4E8BFhGQsZRdgSbn5bTPMXw40N7Ncd98YLZO+XqXy8/OLgFnVLigiIiKSDJ9mmhjnbWCeB3qZWWszawYMBJ5OzXT3BcBaM+seTRoKPOXu6wnNuIOj6acBT8UYp4iIiEiixJYAuvtiYBTwEvAuMMXd55rZk2bWKVpsCHCzmX0M7ADcFk0/hzBq+ENCLeLouOIUERERSZrYBoGIiIiIyNZJTwIRERERSRglgCIiIiIJowRQREREJGGUAIqIiIgkjBJAERERkYRRAriNM7MxZjamlralIeFVMLNO0bOq62r/GcvazHY3s3vqIKRtipn1NLOXN3Mbp5vZfbUTUXLo2GfPzK4ys0MrmfduNetuVccoqeVuZruZ2eebuG6umT1jZm5mPWs1sHLifBKISL3i7vOAs+o6jgx+Cvy8roMQkVpxGOH+uRW4+wFbOBbZ8toD+7p7u7h3pASwCmbWELgLyAN2Bt4HLgWmAIXAgcAy4CRgT+DOtNX3BQa7+8OVbHcJ8HN3X2Vms4GZ7n6DmZ0M9ADOBcYDPQnP8rvP3W+OrgjGRdMKgflAZzObQ7iZ9t3ufmu0n0uAQdGyzwAXu3uVtXxmdjfQGfgS+B93/8LMDgOuAZoBOwEXuPvM6KpsJZBP+NBe7e73Vndc61J0/C4H1gO7A3OBscBMwnv+Hjiayo/9GHfvaWZ/AIYRnlM9191/U80+LwO+A/YCPgBOcfd1ZnYN0Av4MeEzMdjdl5nZKYQboJcQnuF4drS5ztHnpT1wr7uPIdxA/WdmNtHdR1QSw27Ao8DHwD7AAuBUd19hZkcDVwONCJ+ns939q+gKdg5wAHCouy+v5vBuC1qZ2dOE4zcHGAGsdfccCLUNQE93P93MegMTCC0lC4BTom3sEdVqdABecPez0z8b0XbuA16O/j1CufNFdNwHEY77GuAdoGG03xuBIwmfrRnuflVU89se+AUh4Z/k7tdE8Q4DWgGPuftltXy8atO2fOw7AB2B1oRzYS/gEOA94NfuXmJmlwGnEp5D/yxwEfCTKmIYSXgC1vbAOuDkaJudgElmNsDdP0g/gGZW4u45ZtYeuIdwPm5HOEddUc0xKv3dcPdh2RRYLdnmyp1Qxl8AB0bn4x9H2/spcBThN6MB8Bnwm2iZz/nhfDk09ebNLA+4nfD73Aa4zt3/kmrNic7hROv3JJynW5nZPHdPPTQjFmoCrlo3YJ27dwX2IHzZjgX2B25y9zzgG2CIu8929wOiK7TJwJPAPzJt1N03AC8Ch5nZDoQP9WHR7KOBx4l+8N39IEJC1j+tWaAjcETal7gtcATQFRhpZgdEP+r5wMGEL0F7wpNXqvNK9B4eAW6Npv0eOCuK5SzChz/lJ4SntfQDbsxi+1uDbsB5hKS9KXAcYISE6EiqPvaYWS7hQqAT4Rg3jk7I1e1zJCEB7AAcZWZ7RDF0c/eOhBPOqdG2bgZ+5e77EE7ax0Xb+f/tnXvQXdMZxn9JJm4V90vrVqo8U7kYNEFLS4tOtIjWTKMTWpXqLTU1VBmSYkwxmmHUEJdSIkMxRam4jaSRSDIiLrn1QacYSS8yGVp1m5D+8a7j2zk553wRvuQ7Oe9vJpNvn7PP2mu9a+13P2utd6+1PXBoue4vJA0gOgtzmom/CoOBq0qai4DzJG0LXAx8zfY+REfhkspvJtvWeiL+IET/z4AhwADgR41OkrQhMAn4ru3BhGiv3W+7AN8k6nK4pIHdXHMVf1HsfjkhJIYSHQAkfRoYbntv4IvAXpI2KukMAY4gRMJZkrYon+9EPKh6s/iD9rb9YOLhfApwI3GPDAL2BYZIGk74wM8T/vazlfI1ysNmwAhC+AwifP4Y2zcDcwh/u5L4q+N44FbbB5S8/VzSNt3YqP65sbZou3onBsfuIAZ3ILayVxVj7wAACJtJREFUvQvYErgGGGF7CDADuLJy3cm2BVT95WjgQttDCd99aTd5PxpY0tPiD1IAtsT2NOAqST8lxNAehIr/t+2nymnzKQ0JQFJNQIzqZrTtfqIhfolo9AMl9SfE1BTgMODoEvMxm3Dyg7uy5tcrad1m+3+2/wPcS4jJw4gHxZPAXMIxdXfTvGV7Uvl7IuHwIHq1gySNBU4vNqjxUCnnSnbo5UxzsIIo51eIOn2xfN/K9th+D3icGJn7FTC+bH3Yivm2X7H9PiG+trL9AmHP0ZLGEwJ+0/L/DNuvlOudYPvuks5k2+/YXkqMWH4Ymz9ne2r5+6ZS7v0J5zqllHcM0c5rzP4Q6bcD02w/X+p+El1tvJ7BwGLbTwPYPtv2bytpLLP9DrHJ+jZN0qjRyF8cDMy0vbi0iZvK94uBtyTNIDopv7T9dvluiu13ixhfBmxePp9bOpW9nXa2/cPFxi8B/7C9sBwvJkTBVwlB9mb5/IbyWcM8FF/9HWCkpIuAo1jZr7bE9m+AlyWdQTybNiBGElvZqP65sbZo13q/BRhZzjm+HA8jZnxeLJ9fS1c9Q2N/eTqwkaSzicGT1a7nniYFYAskHU002DeJXt80wgG8XTltBVAbyt4DuB741mrcaJOJ3sChdO2XfDIwrzS+fsCZlVHFAwinAjFNWaXq/PsS05v9gMsrv9+fGNZuxXuVv/uUdAAeIxr+kyWNPpXz3gboRuz2NurttZyVbdrK9jVGAD8mbPFAmSZvxSptRtJ+xFRRX+BOoodZs/sH9pS0bem91uf9g7a3mjQqdz9geqWsQ4nebo36ttbuNLpXkFSzY//yf30dbC5ppwZp1Oqgvi76V/5u5C/eo4H/LeJhfyJMYWtgpqQ9W6QD7VNH7Wz7d5uUo1qeKn3oCrFqdO/vDMwkZpUmA7+vKwOSdpD0dPl3f91344mR/5cIUbG08vtmPmJdtZO2rHfbTwBbSRoK7GR7ZoPfV+sZGtv4duBYYCFwToN8Ncr/WiEFYGsOA253xLW9Roi1fo1OLEP6dwOn2l7UXcK2XyUay1HAdGJKeCwxFUA5/oGk/mWaeDohRBpxnKQNJW0JfIMQlI8CJ0jaVBFzeDdwXDfZ2rSIXoDvA4+U2Ic9gXGEozqGJjZoIw6StKOkvsCJRLmqtLR9EWMLCbE+jhBxQ9YgH18GptqeADxH1F0/YmTxAEmfLOddRti9GctZvXheSaoFkZ9ElHs2cGDlQTeW9pnKXxMOkrRLpe4fIR6eA8sDqdb+DWwnaa9yfCZNpq4KS4k4zI3KPdPwLc4KjwNDJX2qXHcksELSPsBfiBGPM4h2pg9fzF7J+mz7R4HjJW1c/O1JNHmRozAUeMH2ZcT9fixdfnU5EZu2pNYxs31k3e8PBy61fUfJ4470Xr/czvU+iZjyvbUczyZ8867l+BRa1zNEXY2zfQ8wHD4II1pKmZWTNIwI5VqrpABszXXETT2PiAeYQYjARowhptLOrfTaTusm/fuB12y/QTiQHYA/l+8mAM8TgapziID/qU3SeankbTrwa9uLbN9LxCDOJobAn6ZryLsZrwEjJD1DNNrTbC8jgo0XEFOXA4BNJH2ieTK9niXAzcSNvphwSFVa2r6I92uBJyQ9ScQR1o8Qrg5/APYu7WtqudZutpcQUxEPSppPdBRavVyzCNhC0sRurrcMOF/SAiIY+ULb/yTE/u0lH/sSUxbrKwuIuppH1P3vgLOIjtdM4iFEGYUfBdws6VkiLujiZonaXkDcuwsIX/FYq0yUNnQq8DAhAPoTIRhPlXzMlzSXaKP1HZR2Zb21ve37SjnmlHy8TAT+N+MhoK+khUSIzl+JWDmAB4AJkr7Q4vcXAROLfxhTrrtbi/PXJe1c77cQL3XcUtL4FyH67ip+9BBai1SA84Dppa4PBl4k6uo2YoRxIREj+VSzBHqKPitWtNPMXZJ8NFT35linUHqsU23vuo6zkgCStiYeRufbfl/SFcDzlZinpIdoV9tL2pyIkes1MWTtRLvWe0+Sy8D0IJK+TbwtugpeB+s5SdqY6Ok0YpztP63N/KxPKN4SbuZIjiyjej2dh91p8uY5vXP9wk5mGRH/NV/ScmIU6Lp1m6WOoe1sr3hTdRYxepasGW1X7z1NjgAmSZIkSZJ0GBkDmCRJkiRJ0mGkAEySJEmSJOkwUgAmSZIkSZJ0GPkSSJIkCbHPKrFkUnVB9Dm21+gFmrKA7Mm2u1smIkmSZK2TAjBJkqSLQ8s2ex8HA4ltBJMkSXodKQCTJEm6QdLniD1XtyZ2XLjC9g1ld4PLiJ1iBhBbO40mFgK+ANhc0o3EIuxXOjanr61HeaXtQZLOI/Z/3gF4xvYoSecQW/L1JRaO/cnaWEooSZLOIQVgkiRJF1MkVaeAjyDWD7sTOMH23LIg78yygn8fQrgdWBaXPQs4y/ZRksYBx9k+qQi+VnwaGGR7uaQTgcHAsHJ8CrHHeP12YEmSJGtMCsAkSZIuVpkCLnuT7g7cIH2wNezGwD62r5Z0LvDDshD3IcB/1+C6s8qm9BB7Qg8D5pTr9QM2WYM0kyRJmpICMEmSpDX9gNeru/dI2h54XdLXianh8cA9xJ6uoxqksYIYLayxQd33b9Rd7xLbV5drbQhs+VELkSRJUiWXgUmSJGmNgbckjQKQtDPxtvB+wOHAvUWszQFGEAIOYDmx4TzAq8AukraT1AcY2eJ6DwKjJW1Wji8AJn6M5UmSJEkBmCRJ0grb7wLHEKLsWeAhYKztGcAE4BBJ84i9Rf8G7FZeDpkFfEbSH20vBK4hROIs4O8tLnk9cB8wS9ICYAjwvR4pXJIkHUvuBZwkSZIkSdJh5AhgkiRJkiRJh5ECMEmSJEmSpMNIAZgkSZIkSdJhpABMkiRJkiTpMFIAJkmSJEmSdBgpAJMkSZIkSTqMFIBJkiRJkiQdxv8BHlTf66qjlWYAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 648x360 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# TODO: Import a supervised learning model that has 'feature_importances_'\n",
    "\n",
    "# TODO: Train the supervised model on the training set \n",
    "model = AdaBoostClassifier().fit(X,y)\n",
    "\n",
    "# TODO: Extract the feature importances\n",
    "importances = model.feature_importances_\n",
    "\n",
    "# Plot\n",
    "vs.feature_plot(importances, X, y)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "There are 38 features with importance, the first four have cumulative feature weight of 30%, each additional adds 2 more percent"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.3"
  },
  "latex_envs": {
   "LaTeX_envs_menu_present": true,
   "autoclose": false,
   "autocomplete": true,
   "bibliofile": "biblio.bib",
   "cite_by": "apalike",
   "current_citInitial": 1,
   "eqLabelWithNumbers": true,
   "eqNumInitial": 1,
   "hotkeys": {
    "equation": "Ctrl-E",
    "itemize": "Ctrl-I"
   },
   "labels_anchors": false,
   "latex_user_defs": false,
   "report_style_numbering": false,
   "user_envs_cfg": false
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}