examples/05_BestSingleModel.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-05-07T01:04:07.223678Z",
     "start_time": "2020-05-07T01:04:07.203114Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n",
      "\u001b[0m"
     ]
    }
   ],
   "source": [
    "# If you run this notebook on Google Colaboratory, uncomment the below to install automl_alex.\n",
    "!pip install -q -U automl_alex"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-05-07T01:05:50.878720Z",
     "start_time": "2020-05-07T01:05:50.874642Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/root/anaconda3/envs/automl_py310/lib/python3.10/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
      "  from .autonotebook import tqdm as notebook_tqdm\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "AutoML-Alex version: 2023.3.11\n"
     ]
    }
   ],
   "source": [
    "import sklearn\n",
    "import pandas as pd\n",
    "import time\n",
    "import automl_alex\n",
    "from automl_alex import DataPrepare\n",
    "from automl_alex import BestSingleModel, BestSingleModelClassifier, BestSingleModelRegressor\n",
    "print('AutoML-Alex version:', automl_alex.__version__)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-05-07T01:05:52.553896Z",
     "start_time": "2020-05-07T01:05:52.549132Z"
    }
   },
   "outputs": [],
   "source": [
    "RANDOM_SEED = 42"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Classifier"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-05-07T01:05:55.127924Z",
     "start_time": "2020-05-07T01:05:55.037616Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>checking_status</th>\n",
       "      <th>duration</th>\n",
       "      <th>credit_history</th>\n",
       "      <th>purpose</th>\n",
       "      <th>credit_amount</th>\n",
       "      <th>savings_status</th>\n",
       "      <th>employment</th>\n",
       "      <th>installment_commitment</th>\n",
       "      <th>personal_status</th>\n",
       "      <th>other_parties</th>\n",
       "      <th>residence_since</th>\n",
       "      <th>property_magnitude</th>\n",
       "      <th>age</th>\n",
       "      <th>other_payment_plans</th>\n",
       "      <th>housing</th>\n",
       "      <th>existing_credits</th>\n",
       "      <th>job</th>\n",
       "      <th>num_dependents</th>\n",
       "      <th>own_telephone</th>\n",
       "      <th>foreign_worker</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>&lt;0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>critical/other existing credit</td>\n",
       "      <td>radio/tv</td>\n",
       "      <td>1169.0</td>\n",
       "      <td>no known savings</td>\n",
       "      <td>&gt;=7</td>\n",
       "      <td>4.0</td>\n",
       "      <td>male single</td>\n",
       "      <td>none</td>\n",
       "      <td>4.0</td>\n",
       "      <td>real estate</td>\n",
       "      <td>67.0</td>\n",
       "      <td>none</td>\n",
       "      <td>own</td>\n",
       "      <td>2.0</td>\n",
       "      <td>skilled</td>\n",
       "      <td>1.0</td>\n",
       "      <td>yes</td>\n",
       "      <td>yes</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0&lt;=X&lt;200</td>\n",
       "      <td>48.0</td>\n",
       "      <td>existing paid</td>\n",
       "      <td>radio/tv</td>\n",
       "      <td>5951.0</td>\n",
       "      <td>&lt;100</td>\n",
       "      <td>1&lt;=X&lt;4</td>\n",
       "      <td>2.0</td>\n",
       "      <td>female div/dep/mar</td>\n",
       "      <td>none</td>\n",
       "      <td>2.0</td>\n",
       "      <td>real estate</td>\n",
       "      <td>22.0</td>\n",
       "      <td>none</td>\n",
       "      <td>own</td>\n",
       "      <td>1.0</td>\n",
       "      <td>skilled</td>\n",
       "      <td>1.0</td>\n",
       "      <td>none</td>\n",
       "      <td>yes</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>no checking</td>\n",
       "      <td>12.0</td>\n",
       "      <td>critical/other existing credit</td>\n",
       "      <td>education</td>\n",
       "      <td>2096.0</td>\n",
       "      <td>&lt;100</td>\n",
       "      <td>4&lt;=X&lt;7</td>\n",
       "      <td>2.0</td>\n",
       "      <td>male single</td>\n",
       "      <td>none</td>\n",
       "      <td>3.0</td>\n",
       "      <td>real estate</td>\n",
       "      <td>49.0</td>\n",
       "      <td>none</td>\n",
       "      <td>own</td>\n",
       "      <td>1.0</td>\n",
       "      <td>unskilled resident</td>\n",
       "      <td>2.0</td>\n",
       "      <td>none</td>\n",
       "      <td>yes</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>&lt;0</td>\n",
       "      <td>42.0</td>\n",
       "      <td>existing paid</td>\n",
       "      <td>furniture/equipment</td>\n",
       "      <td>7882.0</td>\n",
       "      <td>&lt;100</td>\n",
       "      <td>4&lt;=X&lt;7</td>\n",
       "      <td>2.0</td>\n",
       "      <td>male single</td>\n",
       "      <td>guarantor</td>\n",
       "      <td>4.0</td>\n",
       "      <td>life insurance</td>\n",
       "      <td>45.0</td>\n",
       "      <td>none</td>\n",
       "      <td>for free</td>\n",
       "      <td>1.0</td>\n",
       "      <td>skilled</td>\n",
       "      <td>2.0</td>\n",
       "      <td>none</td>\n",
       "      <td>yes</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>&lt;0</td>\n",
       "      <td>24.0</td>\n",
       "      <td>delayed previously</td>\n",
       "      <td>new car</td>\n",
       "      <td>4870.0</td>\n",
       "      <td>&lt;100</td>\n",
       "      <td>1&lt;=X&lt;4</td>\n",
       "      <td>3.0</td>\n",
       "      <td>male single</td>\n",
       "      <td>none</td>\n",
       "      <td>4.0</td>\n",
       "      <td>no known property</td>\n",
       "      <td>53.0</td>\n",
       "      <td>none</td>\n",
       "      <td>for free</td>\n",
       "      <td>2.0</td>\n",
       "      <td>skilled</td>\n",
       "      <td>2.0</td>\n",
       "      <td>none</td>\n",
       "      <td>yes</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  checking_status  duration                  credit_history  \\\n",
       "0              <0       6.0  critical/other existing credit   \n",
       "1        0<=X<200      48.0                   existing paid   \n",
       "2     no checking      12.0  critical/other existing credit   \n",
       "3              <0      42.0                   existing paid   \n",
       "4              <0      24.0              delayed previously   \n",
       "\n",
       "               purpose  credit_amount    savings_status employment  \\\n",
       "0             radio/tv         1169.0  no known savings        >=7   \n",
       "1             radio/tv         5951.0              <100     1<=X<4   \n",
       "2            education         2096.0              <100     4<=X<7   \n",
       "3  furniture/equipment         7882.0              <100     4<=X<7   \n",
       "4              new car         4870.0              <100     1<=X<4   \n",
       "\n",
       "   installment_commitment     personal_status other_parties  residence_since  \\\n",
       "0                     4.0         male single          none              4.0   \n",
       "1                     2.0  female div/dep/mar          none              2.0   \n",
       "2                     2.0         male single          none              3.0   \n",
       "3                     2.0         male single     guarantor              4.0   \n",
       "4                     3.0         male single          none              4.0   \n",
       "\n",
       "  property_magnitude   age other_payment_plans   housing  existing_credits  \\\n",
       "0        real estate  67.0                none       own               2.0   \n",
       "1        real estate  22.0                none       own               1.0   \n",
       "2        real estate  49.0                none       own               1.0   \n",
       "3     life insurance  45.0                none  for free               1.0   \n",
       "4  no known property  53.0                none  for free               2.0   \n",
       "\n",
       "                  job  num_dependents own_telephone foreign_worker  \n",
       "0             skilled             1.0           yes            yes  \n",
       "1             skilled             1.0          none            yes  \n",
       "2  unskilled resident             2.0          none            yes  \n",
       "3             skilled             2.0          none            yes  \n",
       "4             skilled             2.0          none            yes  "
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.datasets import fetch_openml\n",
    "from sklearn.model_selection import train_test_split\n",
    "# https://www.openml.org/d/31\n",
    "dataset = fetch_openml(data_id=31, as_frame=True)\n",
    "dataset.target = dataset.target.astype('category').cat.codes\n",
    "dataset.data.head(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-05-07T01:05:56.756465Z",
     "start_time": "2020-05-07T01:05:56.734220Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((800, 20), (200, 20))"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train, X_test, y_train, y_test = train_test_split(dataset.data, \n",
    "                                                    dataset.target,\n",
    "                                                    test_size=0.2, \n",
    "                                                    random_state=RANDOM_SEED,)\n",
    "X_train.shape, X_test.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## BestSingleModel"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[32m19:24:34\u001b[0m | \u001b[1mmetric is None! Default metric will be used. classifier: AUC, regression: MSE\u001b[0m\n"
     ]
    }
   ],
   "source": [
    "model = BestSingleModelClassifier(\n",
    "    clean_and_encod_data=True,\n",
    "    opt_data_prepare=True,\n",
    "    models_names = ['LightGBM', 'XGBoost', 'LinearModel'],\n",
    "    cat_encoder_names= [\n",
    "            \"HelmertEncoder\",\n",
    "            \"OneHotEncoder\",\n",
    "            \"CountEncoder\",\n",
    "            \"HashingEncoder\",\n",
    "            \"BackwardDifferenceEncoder\",\n",
    "        ],\n",
    "    clean_outliers = [True, False],\n",
    "    num_generator_select_operations = True,\n",
    "    num_generator_operations = [\"/\", \"*\", \"-\"],\n",
    "    auto_parameters=True,\n",
    "    feature_selection=True,\n",
    "    random_state=RANDOM_SEED,)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[32m19:24:34\u001b[0m | \u001b[1m##################################################\u001b[0m\n",
      "\u001b[32m19:24:34\u001b[0m | \u001b[1m> Step 1: calc parameters and pruned score: get test 10 trials\u001b[0m\n",
      "\u001b[32m19:24:53\u001b[0m | \u001b[1m One iteration ~ 1.8 sec\u001b[0m\n",
      "\u001b[32m19:24:53\u001b[0m | \u001b[1m Possible iters ~ 327.0\u001b[0m\n",
      "\u001b[32m19:24:53\u001b[0m | \u001b[1m--------------------------------------------------\u001b[0m\n",
      "\u001b[32m19:24:53\u001b[0m | \u001b[1m  Pruned Threshold Score: 0.6228\u001b[0m\n",
      "\u001b[32m19:24:53\u001b[0m | \u001b[1m##################################################\u001b[0m\n",
      "\u001b[32m19:24:53\u001b[0m | \u001b[1m> Step 2: Full opt with Threshold Score Pruner\u001b[0m\n",
      "\u001b[32m19:24:53\u001b[0m | \u001b[1m##################################################\u001b[0m\n",
      "\u001b[32m19:24:53\u001b[0m | \u001b[1m> Start optimization with the parameters:\u001b[0m\n",
      "\u001b[32m19:24:53\u001b[0m | \u001b[1mCV_Folds = 7\u001b[0m\n",
      "\u001b[32m19:24:53\u001b[0m | \u001b[1mScore_CV_Folds = 3\u001b[0m\n",
      "\u001b[32m19:24:53\u001b[0m | \u001b[1mFeature_Selection = True\u001b[0m\n",
      "\u001b[32m19:24:53\u001b[0m | \u001b[1mOpt_lvl = 2\u001b[0m\n",
      "\u001b[32m19:24:53\u001b[0m | \u001b[1mCold_start = 20\u001b[0m\n",
      "\u001b[32m19:24:53\u001b[0m | \u001b[1mEarly_stoping = 30\u001b[0m\n",
      "\u001b[32m19:24:53\u001b[0m | \u001b[1mMetric = roc_auc_score\u001b[0m\n",
      "\u001b[32m19:24:53\u001b[0m | \u001b[1mDirection = maximize\u001b[0m\n",
      "\u001b[32m19:24:53\u001b[0m | \u001b[1m##################################################\u001b[0m\n",
      "Optimize: : 89it [03:05,  2.09s/it, | Model: LinearModel | OptScore: 0.7457 | Best roc_auc_score: 0.746 ] \n",
      "\u001b[32m19:27:58\u001b[0m | \u001b[1m\n",
      " EarlyStopping Exceeded: Best Score: 0.746 roc_auc_score\u001b[0m\n",
      "\u001b[32m19:27:59\u001b[0m | \u001b[1m> Finish Opt!\u001b[0m\n",
      "\u001b[32m19:27:59\u001b[0m | \u001b[1mBest Score: 0.746 roc_auc_score\u001b[0m\n"
     ]
    }
   ],
   "source": [
    "history = model.opt(X_train, y_train, timeout=600, verbose=3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "predicts = model.predict(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test AUC:  0.8024\n"
     ]
    }
   ],
   "source": [
    "print('Test AUC: ', round(sklearn.metrics.roc_auc_score(y_test, predicts),4))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'LinearModel'"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.best_model_name"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'fit_intercept': False,\n",
       " 'C': 13.348092024903151,\n",
       " 'solver': 'liblinear',\n",
       " 'tol': 0.043777538481459946,\n",
       " 'class_weight': None,\n",
       " 'n_jobs': 1}"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.best_model_param"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "# select Features \n",
    "# if feature_selection=True\n",
    "#model.select_columns[:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'*': False,\n",
       " '-': True,\n",
       " '/': False,\n",
       " 'BackwardDifferenceEncoder': True,\n",
       " 'BackwardDifferenceEncoder_checking_status_0': True,\n",
       " 'BackwardDifferenceEncoder_checking_status_1': False,\n",
       " 'BackwardDifferenceEncoder_checking_status_2': True,\n",
       " 'BackwardDifferenceEncoder_credit_history_0': True,\n",
       " 'BackwardDifferenceEncoder_credit_history_1': True,\n",
       " 'BackwardDifferenceEncoder_credit_history_2': True,\n",
       " 'BackwardDifferenceEncoder_credit_history_3': True,\n",
       " 'BackwardDifferenceEncoder_employment_0': True,\n",
       " 'BackwardDifferenceEncoder_employment_1': True,\n",
       " 'BackwardDifferenceEncoder_employment_2': False,\n",
       " 'BackwardDifferenceEncoder_employment_3': True,\n",
       " 'BackwardDifferenceEncoder_existing_credits_0': False,\n",
       " 'BackwardDifferenceEncoder_existing_credits_1': False,\n",
       " 'BackwardDifferenceEncoder_existing_credits_2': True,\n",
       " 'BackwardDifferenceEncoder_housing_0': True,\n",
       " 'BackwardDifferenceEncoder_housing_1': True,\n",
       " 'BackwardDifferenceEncoder_installment_commitment_0': True,\n",
       " 'BackwardDifferenceEncoder_installment_commitment_1': True,\n",
       " 'BackwardDifferenceEncoder_installment_commitment_2': False,\n",
       " 'BackwardDifferenceEncoder_job_0': False,\n",
       " 'BackwardDifferenceEncoder_job_1': True,\n",
       " 'BackwardDifferenceEncoder_job_2': True,\n",
       " 'BackwardDifferenceEncoder_other_parties_0': True,\n",
       " 'BackwardDifferenceEncoder_other_parties_1': True,\n",
       " 'BackwardDifferenceEncoder_other_payment_plans_0': False,\n",
       " 'BackwardDifferenceEncoder_other_payment_plans_1': True,\n",
       " 'BackwardDifferenceEncoder_personal_status_0': True,\n",
       " 'BackwardDifferenceEncoder_personal_status_1': False,\n",
       " 'BackwardDifferenceEncoder_personal_status_2': False,\n",
       " 'BackwardDifferenceEncoder_property_magnitude_0': True,\n",
       " 'BackwardDifferenceEncoder_property_magnitude_1': True,\n",
       " 'BackwardDifferenceEncoder_property_magnitude_2': True,\n",
       " 'BackwardDifferenceEncoder_residence_since_0': True,\n",
       " 'BackwardDifferenceEncoder_residence_since_1': False,\n",
       " 'BackwardDifferenceEncoder_residence_since_2': True,\n",
       " 'BackwardDifferenceEncoder_savings_status_0': True,\n",
       " 'BackwardDifferenceEncoder_savings_status_1': False,\n",
       " 'BackwardDifferenceEncoder_savings_status_2': False,\n",
       " 'BackwardDifferenceEncoder_savings_status_3': True,\n",
       " 'CountEncoder': False,\n",
       " 'HashingEncoder': False,\n",
       " 'HelmertEncoder': False,\n",
       " 'OneHotEncoder': False,\n",
       " 'age': True,\n",
       " 'age_Is_Outliers_IQR': True,\n",
       " 'checking_status': False,\n",
       " 'credit_amount': False,\n",
       " 'credit_amount_Is_Outliers_IQR': False,\n",
       " 'credit_history': False,\n",
       " 'de_clean_outliers': True,\n",
       " 'duration': True,\n",
       " 'duration_Is_Outliers_IQR': True,\n",
       " 'employment': True,\n",
       " 'existing_credits': False,\n",
       " 'existing_credits_Is_Outliers_IQR': True,\n",
       " 'foreign_worker': True,\n",
       " 'housing': False,\n",
       " 'installment_commitment': True,\n",
       " 'installment_commitment_Is_Outliers_IQR': False,\n",
       " 'job': True,\n",
       " 'lr_C': 13.348092024903151,\n",
       " 'lr_class_weight': None,\n",
       " 'lr_fit_intercept': False,\n",
       " 'lr_solver': 'liblinear',\n",
       " 'lr_tol': 0.043777538481459946,\n",
       " 'model_name': 'LinearModel',\n",
       " 'num_dependents': True,\n",
       " 'other_parties': True,\n",
       " 'other_payment_plans': False,\n",
       " 'own_telephone': False,\n",
       " 'personal_status': False,\n",
       " 'property_magnitude': True,\n",
       " 'purpose': True,\n",
       " 'residence_since': False,\n",
       " 'residence_since_Is_Outliers_IQR': False,\n",
       " 'savings_status': False}"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.study.best_params"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>number</th>\n",
       "      <th>value</th>\n",
       "      <th>datetime_start</th>\n",
       "      <th>datetime_complete</th>\n",
       "      <th>duration</th>\n",
       "      <th>params_*</th>\n",
       "      <th>params_-</th>\n",
       "      <th>params_/</th>\n",
       "      <th>params_BackwardDifferenceEncoder</th>\n",
       "      <th>params_BackwardDifferenceEncoder_checking_status_0</th>\n",
       "      <th>...</th>\n",
       "      <th>params_purpose</th>\n",
       "      <th>params_residence_since</th>\n",
       "      <th>params_residence_since_Is_Outliers_IQR</th>\n",
       "      <th>params_savings_status</th>\n",
       "      <th>params_xgb_colsample_bytree</th>\n",
       "      <th>params_xgb_learning_rate</th>\n",
       "      <th>params_xgb_max_depth</th>\n",
       "      <th>params_xgb_min_child_weight</th>\n",
       "      <th>params_xgb_subsample</th>\n",
       "      <th>state</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>54</th>\n",
       "      <td>54</td>\n",
       "      <td>0.7460</td>\n",
       "      <td>2023-03-09 19:26:38.333078</td>\n",
       "      <td>2023-03-09 19:26:39.651560</td>\n",
       "      <td>0 days 00:00:01.318482</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>...</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>COMPLETE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>55</th>\n",
       "      <td>55</td>\n",
       "      <td>0.7459</td>\n",
       "      <td>2023-03-09 19:26:39.675467</td>\n",
       "      <td>2023-03-09 19:26:41.081892</td>\n",
       "      <td>0 days 00:00:01.406425</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>...</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>COMPLETE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>105</th>\n",
       "      <td>105</td>\n",
       "      <td>0.7457</td>\n",
       "      <td>2023-03-09 19:27:57.468501</td>\n",
       "      <td>2023-03-09 19:27:58.971463</td>\n",
       "      <td>0 days 00:00:01.502962</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>...</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>COMPLETE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>104</th>\n",
       "      <td>104</td>\n",
       "      <td>0.7454</td>\n",
       "      <td>2023-03-09 19:27:55.968897</td>\n",
       "      <td>2023-03-09 19:27:57.444682</td>\n",
       "      <td>0 days 00:00:01.475785</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>...</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>COMPLETE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>102</th>\n",
       "      <td>102</td>\n",
       "      <td>0.7453</td>\n",
       "      <td>2023-03-09 19:27:52.971664</td>\n",
       "      <td>2023-03-09 19:27:54.459649</td>\n",
       "      <td>0 days 00:00:01.487985</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>...</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>COMPLETE</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 249 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "     number   value             datetime_start          datetime_complete  \\\n",
       "54       54  0.7460 2023-03-09 19:26:38.333078 2023-03-09 19:26:39.651560   \n",
       "55       55  0.7459 2023-03-09 19:26:39.675467 2023-03-09 19:26:41.081892   \n",
       "105     105  0.7457 2023-03-09 19:27:57.468501 2023-03-09 19:27:58.971463   \n",
       "104     104  0.7454 2023-03-09 19:27:55.968897 2023-03-09 19:27:57.444682   \n",
       "102     102  0.7453 2023-03-09 19:27:52.971664 2023-03-09 19:27:54.459649   \n",
       "\n",
       "                  duration  params_*  params_-  params_/  \\\n",
       "54  0 days 00:00:01.318482     False      True     False   \n",
       "55  0 days 00:00:01.406425     False      True     False   \n",
       "105 0 days 00:00:01.502962     False      True     False   \n",
       "104 0 days 00:00:01.475785     False      True     False   \n",
       "102 0 days 00:00:01.487985     False      True     False   \n",
       "\n",
       "     params_BackwardDifferenceEncoder  \\\n",
       "54                               True   \n",
       "55                               True   \n",
       "105                              True   \n",
       "104                              True   \n",
       "102                              True   \n",
       "\n",
       "    params_BackwardDifferenceEncoder_checking_status_0  ... params_purpose  \\\n",
       "54                                                True  ...           True   \n",
       "55                                                True  ...           True   \n",
       "105                                               True  ...           True   \n",
       "104                                               True  ...           True   \n",
       "102                                               True  ...           True   \n",
       "\n",
       "    params_residence_since params_residence_since_Is_Outliers_IQR  \\\n",
       "54                   False                                  False   \n",
       "55                   False                                  False   \n",
       "105                  False                                  False   \n",
       "104                  False                                  False   \n",
       "102                  False                                  False   \n",
       "\n",
       "    params_savings_status params_xgb_colsample_bytree  \\\n",
       "54                  False                         NaN   \n",
       "55                  False                         NaN   \n",
       "105                 False                         NaN   \n",
       "104                 False                         NaN   \n",
       "102                 False                         NaN   \n",
       "\n",
       "    params_xgb_learning_rate params_xgb_max_depth params_xgb_min_child_weight  \\\n",
       "54                       NaN                  NaN                         NaN   \n",
       "55                       NaN                  NaN                         NaN   \n",
       "105                      NaN                  NaN                         NaN   \n",
       "104                      NaN                  NaN                         NaN   \n",
       "102                      NaN                  NaN                         NaN   \n",
       "\n",
       "    params_xgb_subsample     state  \n",
       "54                   NaN  COMPLETE  \n",
       "55                   NaN  COMPLETE  \n",
       "105                  NaN  COMPLETE  \n",
       "104                  NaN  COMPLETE  \n",
       "102                  NaN  COMPLETE  \n",
       "\n",
       "[5 rows x 249 columns]"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "history.sort_values('value', ascending=False).head(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'model_name': 'LinearModel',\n",
       " 'lr_fit_intercept': False,\n",
       " 'lr_C': 76.51183782805862,\n",
       " 'lr_solver': 'liblinear',\n",
       " 'lr_tol': 0.0018242400350534868,\n",
       " 'lr_class_weight': None,\n",
       " 'HelmertEncoder': False,\n",
       " 'OneHotEncoder': False,\n",
       " 'CountEncoder': False,\n",
       " 'HashingEncoder': True,\n",
       " 'BackwardDifferenceEncoder': True,\n",
       " 'de_clean_outliers': False,\n",
       " '/': True,\n",
       " '*': False,\n",
       " '-': True,\n",
       " 'num_dependents': True,\n",
       " 'own_telephone': False,\n",
       " 'foreign_worker': False,\n",
       " 'BackwardDifferenceEncoder_checking_status_0': True,\n",
       " 'BackwardDifferenceEncoder_checking_status_1': True,\n",
       " 'BackwardDifferenceEncoder_checking_status_2': True,\n",
       " 'BackwardDifferenceEncoder_credit_history_0': True,\n",
       " 'BackwardDifferenceEncoder_credit_history_1': False,\n",
       " 'BackwardDifferenceEncoder_credit_history_2': True,\n",
       " 'BackwardDifferenceEncoder_credit_history_3': False,\n",
       " 'BackwardDifferenceEncoder_savings_status_0': True,\n",
       " 'BackwardDifferenceEncoder_savings_status_1': True,\n",
       " 'BackwardDifferenceEncoder_savings_status_2': False,\n",
       " 'BackwardDifferenceEncoder_savings_status_3': True,\n",
       " 'BackwardDifferenceEncoder_employment_0': True,\n",
       " 'BackwardDifferenceEncoder_employment_1': True,\n",
       " 'BackwardDifferenceEncoder_employment_2': True,\n",
       " 'BackwardDifferenceEncoder_employment_3': True,\n",
       " 'BackwardDifferenceEncoder_installment_commitment_0': True,\n",
       " 'BackwardDifferenceEncoder_installment_commitment_1': False,\n",
       " 'BackwardDifferenceEncoder_installment_commitment_2': True,\n",
       " 'BackwardDifferenceEncoder_personal_status_0': False,\n",
       " 'BackwardDifferenceEncoder_personal_status_1': True,\n",
       " 'BackwardDifferenceEncoder_personal_status_2': False,\n",
       " 'BackwardDifferenceEncoder_other_parties_0': True,\n",
       " 'BackwardDifferenceEncoder_other_parties_1': True,\n",
       " 'BackwardDifferenceEncoder_residence_since_0': True,\n",
       " 'BackwardDifferenceEncoder_residence_since_1': False,\n",
       " 'BackwardDifferenceEncoder_residence_since_2': False,\n",
       " 'BackwardDifferenceEncoder_property_magnitude_0': True,\n",
       " 'BackwardDifferenceEncoder_property_magnitude_1': True,\n",
       " 'BackwardDifferenceEncoder_property_magnitude_2': True,\n",
       " 'BackwardDifferenceEncoder_other_payment_plans_0': False,\n",
       " 'BackwardDifferenceEncoder_other_payment_plans_1': False,\n",
       " 'BackwardDifferenceEncoder_housing_0': False,\n",
       " 'BackwardDifferenceEncoder_housing_1': False,\n",
       " 'BackwardDifferenceEncoder_existing_credits_0': False,\n",
       " 'BackwardDifferenceEncoder_existing_credits_1': False,\n",
       " 'BackwardDifferenceEncoder_existing_credits_2': True,\n",
       " 'BackwardDifferenceEncoder_job_0': True,\n",
       " 'BackwardDifferenceEncoder_job_1': False,\n",
       " 'BackwardDifferenceEncoder_job_2': True,\n",
       " 'checking_status': False,\n",
       " 'duration': True,\n",
       " 'credit_history': False,\n",
       " 'purpose': False,\n",
       " 'credit_amount': False,\n",
       " 'savings_status': False,\n",
       " 'employment': True,\n",
       " 'installment_commitment': False,\n",
       " 'personal_status': True,\n",
       " 'other_parties': True,\n",
       " 'residence_since': True,\n",
       " 'property_magnitude': False,\n",
       " 'age': False,\n",
       " 'other_payment_plans': False,\n",
       " 'housing': False,\n",
       " 'existing_credits': True,\n",
       " 'job': False,\n",
       " 'HashingEncoder_col_685': False,\n",
       " 'HashingEncoder_col_823': False,\n",
       " 'HashingEncoder_col_902': True,\n",
       " 'HashingEncoder_col_933': False,\n",
       " 'HashingEncoder_col_1236': True,\n",
       " 'HashingEncoder_col_1815': True,\n",
       " 'HashingEncoder_col_2407': True,\n",
       " 'HashingEncoder_col_2872': False,\n",
       " 'HashingEncoder_col_2873': False}"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.study.trials[8].params"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = model.get_model_from_iter(X_train, y_train, model.study.trials[8].params)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test AUC:  0.8066\n"
     ]
    }
   ],
   "source": [
    "predicts = model.predict(X_test)\n",
    "print('Test AUC: ', round(sklearn.metrics.roc_auc_score(y_test, predicts),4))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Save & Load"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Save model\n"
     ]
    }
   ],
   "source": [
    "model.save('model_1')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Load CrossValidation\n",
      "Load model\n"
     ]
    }
   ],
   "source": [
    "model_new = BestSingleModelClassifier(random_state=RANDOM_SEED,)\n",
    "model_new = model_new.load('model_1')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test AUC:  0.8066\n"
     ]
    }
   ],
   "source": [
    "predicts = model_new.predict(X_test)\n",
    "print('Test AUC: ', round(sklearn.metrics.roc_auc_score(y_test, predicts),4))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.9"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": false,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {
    "height": "calc(100% - 180px)",
    "left": "10px",
    "top": "150px",
    "width": "165px"
   },
   "toc_section_display": true,
   "toc_window_display": true
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}