return_cv_models.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "796b8e81-4c24-449f-80db-abc52afb7e5c",
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import seaborn as sns\n",
    "from collections import defaultdict, namedtuple\n",
    "from tqdm.auto import tqdm\n",
    "\n",
    "sns.set()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7a877536-4adb-47a9-bb0d-4d9b99358d14",
   "metadata": {},
   "source": [
    "# Libraries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "e9fe64d3-7b64-4b37-a1dc-c5e740ab929b",
   "metadata": {},
   "outputs": [],
   "source": [
    "import catboost\n",
    "import xgboost\n",
    "import lightgbm"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5ea69270-ea0e-4742-8281-5905dd800a5e",
   "metadata": {},
   "source": [
    "# Datasets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "9094b30d-84dc-4788-8aee-19cfb2348f98",
   "metadata": {},
   "outputs": [],
   "source": [
    "from catboost.datasets import (\n",
    "    adult,\n",
    "    amazon,\n",
    "    higgs\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 178,
   "id": "126f2016-f8f7-4cbd-b2ba-f646d7bd879e",
   "metadata": {},
   "outputs": [],
   "source": [
    "DataSource = namedtuple('DataSource', ['name', 'source', 'target'])\n",
    "datasets = [\n",
    "    DataSource('adult', adult(), 'income'),\n",
    "    DataSource('higgs', higgs(), 0),\n",
    "]\n",
    "\n",
    "catboost_params = {\n",
    "    'loss_function': 'Logloss',\n",
    "    'eval_metric': 'AUC',\n",
    "    'iterations': 100,\n",
    "    'learning_rate': 0.03,\n",
    "    'random_seed': 42,\n",
    "    'l2_leaf_reg': 3.0,\n",
    "    'depth': 6,\n",
    "    'max_leaves': 31,\n",
    "    'thread_count': 5\n",
    "}\n",
    "xgboost_params = {}\n",
    "lightgbm_params = {\n",
    "    'objective': 'cross_entropy',\n",
    "    'num_iterations': 100,\n",
    "    'learning_rate': 0.03,\n",
    "    'num_leaves': 31,\n",
    "    'max_depth': 6,\n",
    "    'lambda_l2': 3.0,\n",
    "    'num_threads': 5,\n",
    "    'seed': 42\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "id": "18cdabc3-9d75-4618-8fe2-d9481607eab9",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import roc_auc_score"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "bb68fbf0-28f8-467a-aae3-e9510b0d85af",
   "metadata": {},
   "source": [
    "# Catboost"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 150,
   "id": "9fca540f-68d9-45b9-a160-ee076721f3a6",
   "metadata": {},
   "outputs": [],
   "source": [
    "def prepare_pools(dataset):\n",
    "    train_df, test_df = dataset.source\n",
    "    train_df.fillna(-999, inplace=True)\n",
    "    test_df.fillna(-999, inplace=True)\n",
    "\n",
    "    if dataset.name == 'adult':\n",
    "        train_df[dataset.target] = train_df[dataset.target].map({'<=50K': 0, '>50K': 1}, na_action='ignore')\n",
    "\n",
    "    cat_features = [\n",
    "        col for col in train_df.columns[train_df.dtypes == object]\n",
    "        if col != dataset.target\n",
    "    ]\n",
    "    train_pool = catboost.Pool(train_df.drop(columns=[dataset.target]), train_df[dataset.target], cat_features=cat_features)\n",
    "    test_pool = catboost.Pool(test_df.drop(columns=[dataset.target]), test_df[dataset.target], cat_features=cat_features)\n",
    "    \n",
    "    return train_pool, test_pool"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 106,
   "id": "57f33eca-d278-4089-baef-38f597fc2ba0",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "4b56e2b8ccb04ea495a930db8d9d2643",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/1 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "catboost_results = defaultdict(list)\n",
    "\n",
    "for dataset in tqdm(datasets):\n",
    "    train_pool, test_pool = prepare_pools(dataset)\n",
    "    \n",
    "    cv_results, cv_models = catboost.cv(\n",
    "        train_pool,\n",
    "        catboost_params,\n",
    "        nfold=5,\n",
    "        partition_random_seed=42,\n",
    "        verbose=False,\n",
    "        return_models=True\n",
    "    )\n",
    "\n",
    "    single_model = catboost.CatBoost(catboost_params).fit(train_pool, verbose=False)\n",
    "    single_model_prediction = single_model.predict(test_pool)\n",
    "    cv_models_prediction = [model.predict(test_pool) for model in cv_models]\n",
    "    test_df = dataset.source[1]\n",
    "    catboost_results['name'].append(dataset.name)\n",
    "    catboost_results['cv_roc_auc_ensemble'].append(roc_auc_score(\n",
    "        test_df[dataset.target], np.mean(cv_models_prediction, axis=0)\n",
    "    ))\n",
    "    catboost_results['cv_roc_auc_mean'].append(\n",
    "        np.mean([roc_auc_score(test_df[dataset.target], prediction)\n",
    "                for prediction in cv_models_prediction], \n",
    "                axis=0)\n",
    "    )\n",
    "    catboost_results['cv_roc_auc_std'].append(\n",
    "        np.std([roc_auc_score(test_df[dataset.target], prediction)\n",
    "                for prediction in cv_models_prediction], \n",
    "                axis=0)\n",
    "    )\n",
    "    catboost_results['single_roc_auc'].append(\n",
    "        roc_auc_score(test_df[dataset.target], single_model_prediction)\n",
    "    )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 107,
   "id": "cbe2fd4e-ff76-4066-8855-aede30d2e71a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>name</th>\n",
       "      <th>cv_roc_auc_ensemble</th>\n",
       "      <th>cv_roc_auc_mean</th>\n",
       "      <th>cv_roc_auc_std</th>\n",
       "      <th>single_roc_auc</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>adult</td>\n",
       "      <td>0.916009</td>\n",
       "      <td>0.915398</td>\n",
       "      <td>0.000537</td>\n",
       "      <td>0.911577</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>higgs</td>\n",
       "      <td>0.778608</td>\n",
       "      <td>0.778434</td>\n",
       "      <td>0.000271</td>\n",
       "      <td>0.778236</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    name  cv_roc_auc_ensemble  cv_roc_auc_mean  cv_roc_auc_std  single_roc_auc\n",
       "0  adult             0.916009         0.915398        0.000537        0.911577\n",
       "1  higgs             0.778608         0.778434        0.000271        0.778236"
      ]
     },
     "execution_count": 107,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.DataFrame(catboost_results)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "031b2b3c-b41c-4b2d-8af3-531f460d3408",
   "metadata": {},
   "source": [
    "# Lightgbm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 134,
   "id": "e220bdbc-5f27-4ac1-a0ab-e58c600f0d4b",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.preprocessing import LabelEncoder"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 179,
   "id": "1daeea4b-7fc4-496f-b3fd-ef9d45a6dadc",
   "metadata": {},
   "outputs": [],
   "source": [
    "def prepare_lightgbm_dataset(dataset):\n",
    "    train_df, test_df = dataset.source\n",
    "\n",
    "    if dataset.name == 'adult':\n",
    "        train_df[dataset.target] = train_df[dataset.target].map({'<=50K': 0, '>50K': 1}, na_action='ignore')\n",
    "\n",
    "    cat_features = [\n",
    "        col for col in train_df.columns[train_df.dtypes == object]\n",
    "        if col != dataset.target\n",
    "    ]\n",
    "    encoders = [LabelEncoder() for _ in cat_features]\n",
    "    \n",
    "    for feature, encoder in zip(cat_features, encoders):\n",
    "        train_df[feature] = encoder.fit_transform(train_df[feature])\n",
    "        test_df[feature] = encoder.transform(test_df[feature])\n",
    "    train_df.fillna(-999, inplace=True)\n",
    "    test_df.fillna(-999, inplace=True)\n",
    "    \n",
    "    cat_features = [feature for feature in cat_features if feature != dataset.target]\n",
    "    \n",
    "    train_pool = lightgbm.Dataset(train_df.drop(columns=[dataset.target]), train_df[dataset.target], \n",
    "                                  categorical_feature=cat_features, free_raw_data=False)\n",
    "    test_pool = lightgbm.Dataset(test_df.drop(columns=[dataset.target]), test_df[dataset.target], \n",
    "                                 categorical_feature=cat_features, free_raw_data=False)\n",
    "    \n",
    "    return train_pool, test_pool, cat_features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 180,
   "id": "1c48a175-20cb-4075-86e2-de70de0c9b57",
   "metadata": {
    "scrolled": true,
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d592535c383b4401b20c81a0b1360ce9",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/2 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/vdkljukin/.local/lib/python3.8/site-packages/lightgbm/engine.py:527: UserWarning: Found `num_iterations` in params. Will use it instead of argument\n",
      "  _log_warning(\"Found `{}` in params. Will use it instead of argument\".format(alias))\n",
      "/home/vdkljukin/.local/lib/python3.8/site-packages/lightgbm/basic.py:1702: UserWarning: Using categorical_feature in Dataset.\n",
      "  _log_warning('Using categorical_feature in Dataset.')\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[LightGBM] [Warning] Accuracy may be bad since you didn't explicitly set num_leaves OR 2^max_depth > num_leaves. (num_leaves=31).\n",
      "[LightGBM] [Warning] Accuracy may be bad since you didn't explicitly set num_leaves OR 2^max_depth > num_leaves. (num_leaves=31).\n",
      "[LightGBM] [Warning] Accuracy may be bad since you didn't explicitly set num_leaves OR 2^max_depth > num_leaves. (num_leaves=31).\n",
      "[LightGBM] [Info] [cross_entropy:Init]: (objective) labels passed interval [0, 1] check\n",
      "[LightGBM] [Info] [cross_entropy:Init]: (metric) labels passed interval [0, 1] check\n",
      "[LightGBM] [Info] [cross_entropy:Init]: sum-of-weights = 26048.000000\n",
      "[LightGBM] [Warning] Auto-choosing row-wise multi-threading, the overhead of testing was 0.001468 seconds.\n",
      "You can set `force_row_wise=true` to remove the overhead.\n",
      "And if memory is not enough, you can set `force_col_wise=true`.\n",
      "[LightGBM] [Info] Total Bins 705\n",
      "[LightGBM] [Info] Number of data points in the train set: 26048, number of used features: 14\n",
      "[LightGBM] [Warning] Accuracy may be bad since you didn't explicitly set num_leaves OR 2^max_depth > num_leaves. (num_leaves=31).\n",
      "[LightGBM] [Info] [cross_entropy:Init]: (metric) labels passed interval [0, 1] check\n",
      "[LightGBM] [Info] [cross_entropy:Init]: sum-of-weights = 6513.000000\n",
      "[LightGBM] [Warning] Accuracy may be bad since you didn't explicitly set num_leaves OR 2^max_depth > num_leaves. (num_leaves=31).\n",
      "[LightGBM] [Warning] Accuracy may be bad since you didn't explicitly set num_leaves OR 2^max_depth > num_leaves. (num_leaves=31).\n",
      "[LightGBM] [Info] [cross_entropy:Init]: (objective) labels passed interval [0, 1] check\n",
      "[LightGBM] [Info] [cross_entropy:Init]: (metric) labels passed interval [0, 1] check\n",
      "[LightGBM] [Info] [cross_entropy:Init]: sum-of-weights = 26049.000000\n",
      "[LightGBM] [Warning] Auto-choosing row-wise multi-threading, the overhead of testing was 0.001054 seconds.\n",
      "You can set `force_row_wise=true` to remove the overhead.\n",
      "And if memory is not enough, you can set `force_col_wise=true`.\n",
      "[LightGBM] [Info] Total Bins 705\n",
      "[LightGBM] [Info] Number of data points in the train set: 26049, number of used features: 14\n",
      "[LightGBM] [Warning] Accuracy may be bad since you didn't explicitly set num_leaves OR 2^max_depth > num_leaves. (num_leaves=31).\n",
      "[LightGBM] [Info] [cross_entropy:Init]: (metric) labels passed interval [0, 1] check\n",
      "[LightGBM] [Info] [cross_entropy:Init]: sum-of-weights = 6512.000000\n",
      "[LightGBM] [Warning] Accuracy may be bad since you didn't explicitly set num_leaves OR 2^max_depth > num_leaves. (num_leaves=31).\n",
      "[LightGBM] [Warning] Accuracy may be bad since you didn't explicitly set num_leaves OR 2^max_depth > num_leaves. (num_leaves=31).\n",
      "[LightGBM] [Info] [cross_entropy:Init]: (objective) labels passed interval [0, 1] check\n",
      "[LightGBM] [Info] [cross_entropy:Init]: (metric) labels passed interval [0, 1] check\n",
      "[LightGBM] [Info] [cross_entropy:Init]: sum-of-weights = 26049.000000\n",
      "[LightGBM] [Warning] Auto-choosing col-wise multi-threading, the overhead of testing was 0.002790 seconds.\n",
      "You can set `force_col_wise=true` to remove the overhead.\n",
      "[LightGBM] [Info] Total Bins 705\n",
      "[LightGBM] [Info] Number of data points in the train set: 26049, number of used features: 14\n",
      "[LightGBM] [Warning] Accuracy may be bad since you didn't explicitly set num_leaves OR 2^max_depth > num_leaves. (num_leaves=31).\n",
      "[LightGBM] [Info] [cross_entropy:Init]: (metric) labels passed interval [0, 1] check\n",
      "[LightGBM] [Info] [cross_entropy:Init]: sum-of-weights = 6512.000000\n",
      "[LightGBM] [Warning] Accuracy may be bad since you didn't explicitly set num_leaves OR 2^max_depth > num_leaves. (num_leaves=31).\n",
      "[LightGBM] [Warning] Accuracy may be bad since you didn't explicitly set num_leaves OR 2^max_depth > num_leaves. (num_leaves=31).\n",
      "[LightGBM] [Info] [cross_entropy:Init]: (objective) labels passed interval [0, 1] check\n",
      "[LightGBM] [Info] [cross_entropy:Init]: (metric) labels passed interval [0, 1] check\n",
      "[LightGBM] [Info] [cross_entropy:Init]: sum-of-weights = 26049.000000\n",
      "[LightGBM] [Warning] Auto-choosing row-wise multi-threading, the overhead of testing was 0.001099 seconds.\n",
      "You can set `force_row_wise=true` to remove the overhead.\n",
      "And if memory is not enough, you can set `force_col_wise=true`.\n",
      "[LightGBM] [Info] Total Bins 705\n",
      "[LightGBM] [Info] Number of data points in the train set: 26049, number of used features: 14\n",
      "[LightGBM] [Warning] Accuracy may be bad since you didn't explicitly set num_leaves OR 2^max_depth > num_leaves. (num_leaves=31).\n",
      "[LightGBM] [Info] [cross_entropy:Init]: (metric) labels passed interval [0, 1] check\n",
      "[LightGBM] [Info] [cross_entropy:Init]: sum-of-weights = 6512.000000\n",
      "[LightGBM] [Warning] Accuracy may be bad since you didn't explicitly set num_leaves OR 2^max_depth > num_leaves. (num_leaves=31).\n",
      "[LightGBM] [Warning] Accuracy may be bad since you didn't explicitly set num_leaves OR 2^max_depth > num_leaves. (num_leaves=31).\n",
      "[LightGBM] [Info] [cross_entropy:Init]: (objective) labels passed interval [0, 1] check\n",
      "[LightGBM] [Info] [cross_entropy:Init]: (metric) labels passed interval [0, 1] check\n",
      "[LightGBM] [Info] [cross_entropy:Init]: sum-of-weights = 26049.000000\n",
      "[LightGBM] [Warning] Auto-choosing row-wise multi-threading, the overhead of testing was 0.001379 seconds.\n",
      "You can set `force_row_wise=true` to remove the overhead.\n",
      "And if memory is not enough, you can set `force_col_wise=true`.\n",
      "[LightGBM] [Info] Total Bins 705\n",
      "[LightGBM] [Info] Number of data points in the train set: 26049, number of used features: 14\n",
      "[LightGBM] [Warning] Accuracy may be bad since you didn't explicitly set num_leaves OR 2^max_depth > num_leaves. (num_leaves=31).\n",
      "[LightGBM] [Info] [cross_entropy:Init]: (metric) labels passed interval [0, 1] check\n",
      "[LightGBM] [Info] [cross_entropy:Init]: sum-of-weights = 6512.000000\n",
      "[LightGBM] [Info] [cross_entropy:BoostFromScore]: pavg = 0.240786 -> initscore = -1.148374\n",
      "[LightGBM] [Info] Start training from score -1.148374\n",
      "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
      "[LightGBM] [Info] [cross_entropy:BoostFromScore]: pavg = 0.240815 -> initscore = -1.148214\n",
      "[LightGBM] [Info] Start training from score -1.148214\n",
      "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
      "[LightGBM] [Info] [cross_entropy:BoostFromScore]: pavg = 0.240815 -> initscore = -1.148214\n",
      "[LightGBM] [Info] Start training from score -1.148214\n",
      "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
      "[LightGBM] [Info] [cross_entropy:BoostFromScore]: pavg = 0.240815 -> initscore = -1.148214\n",
      "[LightGBM] [Info] Start training from score -1.148214\n",
      "[LightGBM] [Info] [cross_entropy:BoostFromScore]: pavg = 0.240815 -> initscore = -1.148214\n",
      "[LightGBM] [Info] Start training from score -1.148214\n",
      "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
      "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
      "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
      "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
      "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
      "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
      "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
      "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
      "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
      "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
      "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
      "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
      "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
      "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
      "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
      "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
      "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
      "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
      "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
      "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
      "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
      "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
      "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
      "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
      "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
      "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
      "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
      "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
      "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
      "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
      "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
      "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
      "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
      "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
      "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
      "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
      "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
      "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
      "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
      "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
      "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
      "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
      "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
      "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
      "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
      "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
      "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
      "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
      "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
      "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
      "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
      "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
      "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
      "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
      "[LightGBM] [Warning] Accuracy may be bad since you didn't explicitly set num_leaves OR 2^max_depth > num_leaves. (num_leaves=31).\n",
      "[LightGBM] [Warning] Accuracy may be bad since you didn't explicitly set num_leaves OR 2^max_depth > num_leaves. (num_leaves=31).\n",
      "[LightGBM] [Warning] Accuracy may be bad since you didn't explicitly set num_leaves OR 2^max_depth > num_leaves. (num_leaves=31).\n",
      "[LightGBM] [Info] [cross_entropy:Init]: (objective) labels passed interval [0, 1] check\n",
      "[LightGBM] [Info] [cross_entropy:Init]: (metric) labels passed interval [0, 1] check\n",
      "[LightGBM] [Info] [cross_entropy:Init]: sum-of-weights = 32561.000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/vdkljukin/.local/lib/python3.8/site-packages/lightgbm/engine.py:148: UserWarning: Found `num_iterations` in params. Will use it instead of argument\n",
      "  _log_warning(\"Found `{}` in params. Will use it instead of argument\".format(alias))\n",
      "/home/vdkljukin/.local/lib/python3.8/site-packages/lightgbm/basic.py:1702: UserWarning: Using categorical_feature in Dataset.\n",
      "  _log_warning('Using categorical_feature in Dataset.')\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[LightGBM] [Warning] Auto-choosing row-wise multi-threading, the overhead of testing was 0.001220 seconds.\n",
      "You can set `force_row_wise=true` to remove the overhead.\n",
      "And if memory is not enough, you can set `force_col_wise=true`.\n",
      "[LightGBM] [Info] Total Bins 705\n",
      "[LightGBM] [Info] Number of data points in the train set: 32561, number of used features: 14\n",
      "[LightGBM] [Info] [cross_entropy:BoostFromScore]: pavg = 0.240810 -> initscore = -1.148246\n",
      "[LightGBM] [Info] Start training from score -1.148246\n",
      "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
      "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
      "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
      "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
      "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
      "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
      "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/vdkljukin/.local/lib/python3.8/site-packages/lightgbm/engine.py:527: UserWarning: Found `num_iterations` in params. Will use it instead of argument\n",
      "  _log_warning(\"Found `{}` in params. Will use it instead of argument\".format(alias))\n",
      "/home/vdkljukin/.local/lib/python3.8/site-packages/lightgbm/basic.py:1702: UserWarning: Using categorical_feature in Dataset.\n",
      "  _log_warning('Using categorical_feature in Dataset.')\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[LightGBM] [Warning] Accuracy may be bad since you didn't explicitly set num_leaves OR 2^max_depth > num_leaves. (num_leaves=31).\n",
      "[LightGBM] [Warning] Accuracy may be bad since you didn't explicitly set num_leaves OR 2^max_depth > num_leaves. (num_leaves=31).\n",
      "[LightGBM] [Warning] Accuracy may be bad since you didn't explicitly set num_leaves OR 2^max_depth > num_leaves. (num_leaves=31).\n",
      "[LightGBM] [Info] [cross_entropy:Init]: (objective) labels passed interval [0, 1] check\n",
      "[LightGBM] [Info] [cross_entropy:Init]: (metric) labels passed interval [0, 1] check\n",
      "[LightGBM] [Info] [cross_entropy:Init]: sum-of-weights = 8400000.000000\n",
      "[LightGBM] [Warning] Auto-choosing row-wise multi-threading, the overhead of testing was 0.557123 seconds.\n",
      "You can set `force_row_wise=true` to remove the overhead.\n",
      "And if memory is not enough, you can set `force_col_wise=true`.\n",
      "[LightGBM] [Info] Total Bins 6132\n",
      "[LightGBM] [Info] Number of data points in the train set: 8400000, number of used features: 28\n",
      "[LightGBM] [Warning] Accuracy may be bad since you didn't explicitly set num_leaves OR 2^max_depth > num_leaves. (num_leaves=31).\n",
      "[LightGBM] [Info] [cross_entropy:Init]: (metric) labels passed interval [0, 1] check\n",
      "[LightGBM] [Info] [cross_entropy:Init]: sum-of-weights = 2100000.000000\n",
      "[LightGBM] [Warning] Accuracy may be bad since you didn't explicitly set num_leaves OR 2^max_depth > num_leaves. (num_leaves=31).\n",
      "[LightGBM] [Warning] Accuracy may be bad since you didn't explicitly set num_leaves OR 2^max_depth > num_leaves. (num_leaves=31).\n",
      "[LightGBM] [Info] [cross_entropy:Init]: (objective) labels passed interval [0, 1] check\n",
      "[LightGBM] [Info] [cross_entropy:Init]: (metric) labels passed interval [0, 1] check\n",
      "[LightGBM] [Info] [cross_entropy:Init]: sum-of-weights = 8400000.000000\n",
      "[LightGBM] [Warning] Auto-choosing row-wise multi-threading, the overhead of testing was 0.224023 seconds.\n",
      "You can set `force_row_wise=true` to remove the overhead.\n",
      "And if memory is not enough, you can set `force_col_wise=true`.\n",
      "[LightGBM] [Info] Total Bins 6132\n",
      "[LightGBM] [Info] Number of data points in the train set: 8400000, number of used features: 28\n",
      "[LightGBM] [Warning] Accuracy may be bad since you didn't explicitly set num_leaves OR 2^max_depth > num_leaves. (num_leaves=31).\n",
      "[LightGBM] [Info] [cross_entropy:Init]: (metric) labels passed interval [0, 1] check\n",
      "[LightGBM] [Info] [cross_entropy:Init]: sum-of-weights = 2100000.000000\n",
      "[LightGBM] [Warning] Accuracy may be bad since you didn't explicitly set num_leaves OR 2^max_depth > num_leaves. (num_leaves=31).\n",
      "[LightGBM] [Warning] Accuracy may be bad since you didn't explicitly set num_leaves OR 2^max_depth > num_leaves. (num_leaves=31).\n",
      "[LightGBM] [Info] [cross_entropy:Init]: (objective) labels passed interval [0, 1] check\n",
      "[LightGBM] [Info] [cross_entropy:Init]: (metric) labels passed interval [0, 1] check\n",
      "[LightGBM] [Info] [cross_entropy:Init]: sum-of-weights = 8400000.000000\n",
      "[LightGBM] [Warning] Auto-choosing row-wise multi-threading, the overhead of testing was 0.159212 seconds.\n",
      "You can set `force_row_wise=true` to remove the overhead.\n",
      "And if memory is not enough, you can set `force_col_wise=true`.\n",
      "[LightGBM] [Info] Total Bins 6132\n",
      "[LightGBM] [Info] Number of data points in the train set: 8400000, number of used features: 28\n",
      "[LightGBM] [Warning] Accuracy may be bad since you didn't explicitly set num_leaves OR 2^max_depth > num_leaves. (num_leaves=31).\n",
      "[LightGBM] [Info] [cross_entropy:Init]: (metric) labels passed interval [0, 1] check\n",
      "[LightGBM] [Info] [cross_entropy:Init]: sum-of-weights = 2100000.000000\n",
      "[LightGBM] [Warning] Accuracy may be bad since you didn't explicitly set num_leaves OR 2^max_depth > num_leaves. (num_leaves=31).\n",
      "[LightGBM] [Warning] Accuracy may be bad since you didn't explicitly set num_leaves OR 2^max_depth > num_leaves. (num_leaves=31).\n",
      "[LightGBM] [Info] [cross_entropy:Init]: (objective) labels passed interval [0, 1] check\n",
      "[LightGBM] [Info] [cross_entropy:Init]: (metric) labels passed interval [0, 1] check\n",
      "[LightGBM] [Info] [cross_entropy:Init]: sum-of-weights = 8400000.000000\n",
      "[LightGBM] [Warning] Auto-choosing col-wise multi-threading, the overhead of testing was 1.319880 seconds.\n",
      "You can set `force_col_wise=true` to remove the overhead.\n",
      "[LightGBM] [Info] Total Bins 6132\n",
      "[LightGBM] [Info] Number of data points in the train set: 8400000, number of used features: 28\n",
      "[LightGBM] [Warning] Accuracy may be bad since you didn't explicitly set num_leaves OR 2^max_depth > num_leaves. (num_leaves=31).\n",
      "[LightGBM] [Info] [cross_entropy:Init]: (metric) labels passed interval [0, 1] check\n",
      "[LightGBM] [Info] [cross_entropy:Init]: sum-of-weights = 2100000.000000\n",
      "[LightGBM] [Warning] Accuracy may be bad since you didn't explicitly set num_leaves OR 2^max_depth > num_leaves. (num_leaves=31).\n",
      "[LightGBM] [Warning] Accuracy may be bad since you didn't explicitly set num_leaves OR 2^max_depth > num_leaves. (num_leaves=31).\n",
      "[LightGBM] [Info] [cross_entropy:Init]: (objective) labels passed interval [0, 1] check\n",
      "[LightGBM] [Info] [cross_entropy:Init]: (metric) labels passed interval [0, 1] check\n",
      "[LightGBM] [Info] [cross_entropy:Init]: sum-of-weights = 8400000.000000\n",
      "[LightGBM] [Warning] Auto-choosing row-wise multi-threading, the overhead of testing was 0.428335 seconds.\n",
      "You can set `force_row_wise=true` to remove the overhead.\n",
      "And if memory is not enough, you can set `force_col_wise=true`.\n",
      "[LightGBM] [Info] Total Bins 6132\n",
      "[LightGBM] [Info] Number of data points in the train set: 8400000, number of used features: 28\n",
      "[LightGBM] [Warning] Accuracy may be bad since you didn't explicitly set num_leaves OR 2^max_depth > num_leaves. (num_leaves=31).\n",
      "[LightGBM] [Info] [cross_entropy:Init]: (metric) labels passed interval [0, 1] check\n",
      "[LightGBM] [Info] [cross_entropy:Init]: sum-of-weights = 2100000.000000\n",
      "[LightGBM] [Info] [cross_entropy:BoostFromScore]: pavg = 0.529963 -> initscore = 0.119997\n",
      "[LightGBM] [Info] Start training from score 0.119997\n",
      "[LightGBM] [Info] [cross_entropy:BoostFromScore]: pavg = 0.529963 -> initscore = 0.119998\n",
      "[LightGBM] [Info] Start training from score 0.119998\n",
      "[LightGBM] [Info] [cross_entropy:BoostFromScore]: pavg = 0.529963 -> initscore = 0.119998\n",
      "[LightGBM] [Info] Start training from score 0.119998\n",
      "[LightGBM] [Info] [cross_entropy:BoostFromScore]: pavg = 0.529963 -> initscore = 0.119998\n",
      "[LightGBM] [Info] Start training from score 0.119998\n",
      "[LightGBM] [Info] [cross_entropy:BoostFromScore]: pavg = 0.529963 -> initscore = 0.119998\n",
      "[LightGBM] [Info] Start training from score 0.119998\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/vdkljukin/.local/lib/python3.8/site-packages/lightgbm/engine.py:148: UserWarning: Found `num_iterations` in params. Will use it instead of argument\n",
      "  _log_warning(\"Found `{}` in params. Will use it instead of argument\".format(alias))\n",
      "/home/vdkljukin/.local/lib/python3.8/site-packages/lightgbm/basic.py:1702: UserWarning: Using categorical_feature in Dataset.\n",
      "  _log_warning('Using categorical_feature in Dataset.')\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[LightGBM] [Warning] Accuracy may be bad since you didn't explicitly set num_leaves OR 2^max_depth > num_leaves. (num_leaves=31).\n",
      "[LightGBM] [Warning] Accuracy may be bad since you didn't explicitly set num_leaves OR 2^max_depth > num_leaves. (num_leaves=31).\n",
      "[LightGBM] [Warning] Accuracy may be bad since you didn't explicitly set num_leaves OR 2^max_depth > num_leaves. (num_leaves=31).\n",
      "[LightGBM] [Info] [cross_entropy:Init]: (objective) labels passed interval [0, 1] check\n",
      "[LightGBM] [Info] [cross_entropy:Init]: (metric) labels passed interval [0, 1] check\n",
      "[LightGBM] [Info] [cross_entropy:Init]: sum-of-weights = 10500000.000000\n",
      "[LightGBM] [Warning] Auto-choosing row-wise multi-threading, the overhead of testing was 0.260826 seconds.\n",
      "You can set `force_row_wise=true` to remove the overhead.\n",
      "And if memory is not enough, you can set `force_col_wise=true`.\n",
      "[LightGBM] [Info] Total Bins 6132\n",
      "[LightGBM] [Info] Number of data points in the train set: 10500000, number of used features: 28\n",
      "[LightGBM] [Info] [cross_entropy:BoostFromScore]: pavg = 0.529963 -> initscore = 0.119997\n",
      "[LightGBM] [Info] Start training from score 0.119997\n"
     ]
    }
   ],
   "source": [
    "lightgbm_results = defaultdict(list)\n",
    "\n",
    "for dataset in tqdm(datasets):\n",
    "    train_pool, test_pool, cat_features = prepare_lightgbm_dataset(dataset)\n",
    "    \n",
    "    cv_results = lightgbm.cv(\n",
    "        lightgbm_params,\n",
    "        train_pool,\n",
    "        nfold=5,\n",
    "        seed=42,\n",
    "        verbose_eval=False,\n",
    "        return_cvbooster=True\n",
    "    )\n",
    "    \n",
    "    cv_models = cv_results['cvbooster'].boosters\n",
    "\n",
    "    single_model = lightgbm.train(lightgbm_params, train_pool)\n",
    "    test_df = dataset.source[1]\n",
    "    single_model_prediction = single_model.predict(test_df.drop(columns=[dataset.target]), categorical_feature=cat_features)\n",
    "    cv_models_prediction = [model.predict(test_df.drop(columns=[dataset.target])) for model in cv_models]\n",
    "    lightgbm_results['name'].append(dataset.name)\n",
    "    lightgbm_results['cv_roc_auc_ensemble'].append(roc_auc_score(\n",
    "        test_df[dataset.target], np.mean(cv_models_prediction, axis=0)\n",
    "    ))\n",
    "    lightgbm_results['cv_roc_auc_mean'].append(\n",
    "        np.mean([roc_auc_score(test_df[dataset.target], prediction)\n",
    "                for prediction in cv_models_prediction], \n",
    "                axis=0)\n",
    "    )\n",
    "    lightgbm_results['cv_roc_auc_std'].append(\n",
    "        np.std([roc_auc_score(test_df[dataset.target], prediction)\n",
    "                for prediction in cv_models_prediction], \n",
    "                axis=0)\n",
    "    )\n",
    "    lightgbm_results['single_roc_auc'].append(\n",
    "        roc_auc_score(test_df[dataset.target], single_model_prediction)\n",
    "    )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 181,
   "id": "8c009638-4263-4ec9-8a0c-00555e5796aa",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>name</th>\n",
       "      <th>cv_roc_auc_ensemble</th>\n",
       "      <th>cv_roc_auc_mean</th>\n",
       "      <th>cv_roc_auc_std</th>\n",
       "      <th>single_roc_auc</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>adult</td>\n",
       "      <td>0.919005</td>\n",
       "      <td>0.918514</td>\n",
       "      <td>0.000467</td>\n",
       "      <td>0.918703</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>higgs</td>\n",
       "      <td>0.792647</td>\n",
       "      <td>0.792500</td>\n",
       "      <td>0.000099</td>\n",
       "      <td>0.792469</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    name  cv_roc_auc_ensemble  cv_roc_auc_mean  cv_roc_auc_std  single_roc_auc\n",
       "0  adult             0.919005         0.918514        0.000467        0.918703\n",
       "1  higgs             0.792647         0.792500        0.000099        0.792469"
      ]
     },
     "execution_count": 181,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.DataFrame(lightgbm_results)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7fb2b923-bd84-4d04-9f59-96e41ae737b4",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}