time_cv-classifier.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import random\n",
    "import os\n",
    "import matplotlib.pyplot as plt\n",
    "from sklearn.metrics import classification_report\n",
    "import lightgbm\n",
    "from sklearn.model_selection import StratifiedKFold, RandomizedSearchCV\n",
    "from sklearn.pipeline import Pipeline\n",
    "from sklearn.metrics import log_loss\n",
    "from sklearn.calibration import calibration_curve\n",
    "from sklearn.calibration import CalibratedClassifierCV"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "dir = \"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "wl_odds_player_df = pd.read_csv(dir + \"wl_odds_player_df.csv\")\n",
    "\n",
    "wl_odds_player_df = wl_odds_player_df.drop('Unnamed: 0', axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['league_id', 'season', 'date', 'stage', 'match_id', 'team_id',\n",
       "       'm_rating', 'win', 'lose', 'rating_diff', 'home', 'cumulative_win',\n",
       "       'cumulative_lose', 'match_num', 'match_bin', 'draw',\n",
       "       'avg_overall_rating', 'avg_volleys_imp', 'avg_long_passing',\n",
       "       'avg_finishing', 'avg_ball_control', 'avg_dribbling',\n",
       "       'avg_short_passing', 'avg_reactions', 'avg_vision_imp', 'avg_penalties',\n",
       "       'avg_agility_imp', 'result', 'cumulative_result'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "wl_odds_player_df.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "seasons = sorted(wl_odds_player_df[\"season\"].unique())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['2009/2010', '2010/2011']\n",
      "['2010/2011', '2011/2012']\n",
      "['2011/2012', '2012/2013']\n",
      "['2012/2013', '2013/2014']\n",
      "['2013/2014', '2014/2015']\n",
      "['2014/2015', '2015/2016']\n"
     ]
    }
   ],
   "source": [
    "for i in range(1, len(seasons)-1):\n",
    "    print(seasons[i:i+2])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Training period 1 season"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "ll_01 = list()\n",
    "\n",
    "for i in range(1, len(seasons)-1):\n",
    "    \n",
    "    train_df = wl_odds_player_df.loc[wl_odds_player_df[\"season\"].isin(seasons[i:i+2][:-1])].dropna()\n",
    "    test_df = wl_odds_player_df.loc[wl_odds_player_df[\"season\"].isin([seasons[i:i+2][-1]])].dropna()\n",
    "\n",
    "    train_dates = sorted(train_df[\"date\"].unique())\n",
    "\n",
    "    calib_dates = random.sample(train_dates, int(len(train_dates)*0.25))\n",
    "\n",
    "    calib_df = train_df.loc[train_df[\"date\"].isin(calib_dates)]\n",
    "\n",
    "    train_df = train_df.loc[~train_df[\"date\"].isin(calib_dates)]\n",
    "\n",
    "    feat_col = [c for c in train_df.columns if \"avg\" in c]\n",
    "    feat_col = feat_col + [\"rating_diff\"]\n",
    "\n",
    "    skf = StratifiedKFold(n_splits=6, shuffle=True)\n",
    "\n",
    "    lgb_param_grid = {'lgb__n_estimators': [100, 200, 300,],\n",
    "                     'lgb__num_leaves':  [2, 4, 6, 10, 15, 20],\n",
    "                     'lgb__subsample': [0.4, 0.6, 0.8, 1],\n",
    "                     'lgb__colsample_bytree': [0.4, 0.6, 0.8, 1],\n",
    "                     'lgb__is_unbalance': [False]}\n",
    "\n",
    "    lgb_pipe = Pipeline([(\"lgb\", lightgbm.LGBMClassifier())])\n",
    "\n",
    "    lgb_cv = RandomizedSearchCV(lgb_pipe, \n",
    "                                param_distributions=lgb_param_grid, \n",
    "                                n_jobs=-1,\n",
    "                                cv=skf, \n",
    "                                     n_iter=50,\n",
    "                                     refit=True,\n",
    "                                    )\n",
    "\n",
    "    lgb_cv.fit(train_df.drop([\"result\", \"win\", \"lose\", \"draw\", \"team_id\", \"m_rating\",\n",
    "                              \"cumulative_lose\", \"cumulative_win\",\n",
    "                              \"season\", \"date\", \"stage\", \"match_id\", 'match_num'], axis=1),\n",
    "                    train_df[\"win\"])\n",
    "\n",
    "\n",
    "    lgb_calib = CalibratedClassifierCV(lgb_cv, method='sigmoid', cv=5)\n",
    "\n",
    "    lgb_calib.fit(calib_df.drop([\"result\", \"win\", \"lose\", \"draw\", \"team_id\", \"m_rating\",\n",
    "                                 \"cumulative_lose\", \"cumulative_win\",\n",
    "                                 \"season\", \"date\", \"stage\", \"match_id\", 'match_num'], axis=1),\n",
    "                  calib_df[\"win\"])\n",
    "\n",
    "    lgb_calib_pred = lgb_calib.predict_proba(test_df.drop([\"result\", \"win\", \"lose\", \"draw\", \"team_id\", \"m_rating\",\n",
    "                                                           \"cumulative_lose\", \"cumulative_win\",\n",
    "                                                           \"season\", \"date\", \"stage\", \"match_id\", 'match_num'], axis=1))\n",
    "\n",
    "    ll_01.append(log_loss(test_df[\"win\"], lgb_calib_pred))\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Training period 2 season"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['2009/2010', '2010/2011', '2011/2012']\n",
      "['2010/2011', '2011/2012', '2012/2013']\n",
      "['2011/2012', '2012/2013', '2013/2014']\n",
      "['2012/2013', '2013/2014', '2014/2015']\n",
      "['2013/2014', '2014/2015', '2015/2016']\n"
     ]
    }
   ],
   "source": [
    "for i in range(1, len(seasons)-2):\n",
    "    print(seasons[i:i+3])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "ll_02 = list()\n",
    "\n",
    "for i in range(1, len(seasons)-2):\n",
    "    \n",
    "    train_df = wl_odds_player_df.loc[wl_odds_player_df[\"season\"].isin(seasons[i:i+3][:-1])].dropna()\n",
    "    test_df = wl_odds_player_df.loc[wl_odds_player_df[\"season\"].isin([seasons[i:i+3][-1]])].dropna()\n",
    "\n",
    "    train_dates = sorted(train_df[\"date\"].unique())\n",
    "\n",
    "    calib_dates = random.sample(train_dates, int(len(train_dates)*0.25))\n",
    "\n",
    "    calib_df = train_df.loc[train_df[\"date\"].isin(calib_dates)]\n",
    "\n",
    "    train_df = train_df.loc[~train_df[\"date\"].isin(calib_dates)]\n",
    "\n",
    "    feat_col = [c for c in train_df.columns if \"avg\" in c]\n",
    "    feat_col = feat_col + [\"rating_diff\"]\n",
    "\n",
    "    skf = StratifiedKFold(n_splits=6, shuffle=True)\n",
    "\n",
    "    lgb_param_grid = {'lgb__n_estimators': [100, 200, 300,],\n",
    "                     'lgb__num_leaves':  [2, 4, 6, 10, 15, 20],\n",
    "                     'lgb__subsample': [0.4, 0.6, 0.8, 1],\n",
    "                     'lgb__colsample_bytree': [0.4, 0.6, 0.8, 1],\n",
    "                     'lgb__is_unbalance': [False]}\n",
    "\n",
    "    lgb_pipe = Pipeline([(\"lgb\", lightgbm.LGBMClassifier())])\n",
    "\n",
    "    lgb_cv = RandomizedSearchCV(lgb_pipe, \n",
    "                                param_distributions=lgb_param_grid, \n",
    "                                n_jobs=-1,\n",
    "                                cv=skf, \n",
    "                                     n_iter=50,\n",
    "                                     refit=True,\n",
    "                                    )\n",
    "\n",
    "    lgb_cv.fit(train_df.drop([\"result\", \"win\", \"lose\", \"draw\", \"team_id\", \"m_rating\",\n",
    "                              \"cumulative_lose\", \"cumulative_win\",\n",
    "                              \"season\", \"date\", \"stage\", \"match_id\", 'match_num'], axis=1),\n",
    "                    train_df[\"win\"])\n",
    "\n",
    "\n",
    "    lgb_calib = CalibratedClassifierCV(lgb_cv, method='sigmoid', cv=5)\n",
    "\n",
    "    lgb_calib.fit(calib_df.drop([\"result\", \"win\", \"lose\", \"draw\", \"team_id\", \"m_rating\",\n",
    "                                 \"cumulative_lose\", \"cumulative_win\",\n",
    "                                 \"season\", \"date\", \"stage\", \"match_id\", 'match_num'], axis=1),\n",
    "                  calib_df[\"win\"])\n",
    "\n",
    "    lgb_calib_pred = lgb_calib.predict_proba(test_df.drop([\"result\", \"win\", \"lose\", \"draw\", \"team_id\", \"m_rating\",\n",
    "                                                           \"cumulative_lose\", \"cumulative_win\",\n",
    "                                                           \"season\", \"date\", \"stage\", \"match_id\", 'match_num'], axis=1))\n",
    "\n",
    "    ll_02.append(log_loss(test_df[\"win\"], lgb_calib_pred))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['2009/2010', '2010/2011', '2011/2012', '2012/2013']\n",
      "['2010/2011', '2011/2012', '2012/2013', '2013/2014']\n",
      "['2011/2012', '2012/2013', '2013/2014', '2014/2015']\n",
      "['2012/2013', '2013/2014', '2014/2015', '2015/2016']\n"
     ]
    }
   ],
   "source": [
    "for i in range(1, len(seasons)-3):\n",
    "    print(seasons[i:i+4])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Training period 3 seasons"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "ll_03 = list()\n",
    "\n",
    "for i in range(1, len(seasons)-3):\n",
    "    \n",
    "    train_df = wl_odds_player_df.loc[wl_odds_player_df[\"season\"].isin(seasons[i:i+4][:-1])].dropna()\n",
    "    test_df = wl_odds_player_df.loc[wl_odds_player_df[\"season\"].isin([seasons[i:i+4][-1]])].dropna()\n",
    "\n",
    "    train_dates = sorted(train_df[\"date\"].unique())\n",
    "\n",
    "    calib_dates = random.sample(train_dates, int(len(train_dates)*0.25))\n",
    "\n",
    "    calib_df = train_df.loc[train_df[\"date\"].isin(calib_dates)]\n",
    "\n",
    "    train_df = train_df.loc[~train_df[\"date\"].isin(calib_dates)]\n",
    "\n",
    "    feat_col = [c for c in train_df.columns if \"avg\" in c]\n",
    "    feat_col = feat_col + [\"rating_diff\"]\n",
    "\n",
    "    skf = StratifiedKFold(n_splits=6, shuffle=True)\n",
    "\n",
    "    lgb_param_grid = {'lgb__n_estimators': [100, 200, 300,],\n",
    "                     'lgb__num_leaves':  [2, 4, 6, 10, 15, 20],\n",
    "                     'lgb__subsample': [0.4, 0.6, 0.8, 1],\n",
    "                     'lgb__colsample_bytree': [0.4, 0.6, 0.8, 1],\n",
    "                     'lgb__is_unbalance': [False]}\n",
    "\n",
    "    lgb_pipe = Pipeline([(\"lgb\", lightgbm.LGBMClassifier())])\n",
    "\n",
    "    lgb_cv = RandomizedSearchCV(lgb_pipe, \n",
    "                                param_distributions=lgb_param_grid, \n",
    "                                n_jobs=-1,\n",
    "                                cv=skf, \n",
    "                                     n_iter=50,\n",
    "                                     refit=True,\n",
    "                                    )\n",
    "\n",
    "    lgb_cv.fit(train_df.drop([\"result\", \"win\", \"lose\", \"draw\", \"team_id\", \"m_rating\",\n",
    "                              \"cumulative_lose\", \"cumulative_win\",\n",
    "                              \"season\", \"date\", \"stage\", \"match_id\", 'match_num'], axis=1),\n",
    "                    train_df[\"win\"])\n",
    "\n",
    "\n",
    "    lgb_calib = CalibratedClassifierCV(lgb_cv, method='sigmoid', cv=5)\n",
    "\n",
    "    lgb_calib.fit(calib_df.drop([\"result\", \"win\", \"lose\", \"draw\", \"team_id\", \"m_rating\",\n",
    "                                 \"cumulative_lose\", \"cumulative_win\",\n",
    "                                 \"season\", \"date\", \"stage\", \"match_id\", 'match_num'], axis=1),\n",
    "                  calib_df[\"win\"])\n",
    "\n",
    "    lgb_calib_pred = lgb_calib.predict_proba(test_df.drop([\"result\", \"win\", \"lose\", \"draw\", \"team_id\", \"m_rating\",\n",
    "                                                           \"cumulative_lose\", \"cumulative_win\",\n",
    "                                                           \"season\", \"date\", \"stage\", \"match_id\", 'match_num'], axis=1))\n",
    "\n",
    "    ll_03.append(log_loss(test_df[\"win\"], lgb_calib_pred))\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(0.595399931445676, 0.590084907734808, 0.5886806763778378)"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.mean(ll_01), np.mean(ll_02), np.mean(ll_03)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Training period 3 seasons with current season"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "ll_04 = list()\n",
    "\n",
    "for i in range(1, len(seasons)-3):\n",
    "    \n",
    "    train_df = wl_odds_player_df.loc[wl_odds_player_df[\"season\"].isin(seasons[i:i+4][:-1])].dropna()\n",
    "    test_df = wl_odds_player_df.loc[wl_odds_player_df[\"season\"].isin([seasons[i:i+4][-1]])].dropna()\n",
    "    \n",
    "    test_dates = sorted(test_df[\"date\"].unique())\n",
    "    n = len(test_dates) // 2\n",
    "\n",
    "    train_df = pd.concat([train_df, \n",
    "                          test_df.loc[test_df[\"date\"].isin(test_dates[:n])]])\n",
    "\n",
    "    test_df = test_df.loc[test_df[\"date\"].isin(test_dates[n:])]\n",
    "\n",
    "    train_dates = sorted(train_df[\"date\"].unique())\n",
    "\n",
    "    calib_dates = random.sample(train_dates, int(len(train_dates)*0.25))\n",
    "\n",
    "    calib_df = train_df.loc[train_df[\"date\"].isin(calib_dates)]\n",
    "\n",
    "    train_df = train_df.loc[~train_df[\"date\"].isin(calib_dates)]\n",
    "\n",
    "    feat_col = [c for c in train_df.columns if \"avg\" in c]\n",
    "    feat_col = feat_col + [\"rating_diff\"]\n",
    "\n",
    "\n",
    "    skf = StratifiedKFold(n_splits=6, shuffle=True)\n",
    "\n",
    "    lgb_param_grid = {'lgb__n_estimators': [100, 200, 300,],\n",
    "                     'lgb__num_leaves':  [2, 4, 6, 10, 15, 20],\n",
    "                     'lgb__subsample': [0.4, 0.6, 0.8, 1],\n",
    "                     'lgb__colsample_bytree': [0.4, 0.6, 0.8, 1],\n",
    "                     'lgb__is_unbalance': [False]}\n",
    "\n",
    "    lgb_pipe = Pipeline([(\"lgb\", lightgbm.LGBMClassifier())])\n",
    "\n",
    "    lgb_cv = RandomizedSearchCV(lgb_pipe, \n",
    "                                param_distributions=lgb_param_grid, \n",
    "                                n_jobs=-1,\n",
    "                                cv=skf, \n",
    "                                     n_iter=50,\n",
    "                                     refit=True,\n",
    "                                    )\n",
    "\n",
    "    lgb_cv.fit(train_df.drop([\"result\", \"win\", \"lose\", \"draw\", \"team_id\", \"m_rating\",\n",
    "                              \"cumulative_lose\", \"cumulative_win\",\n",
    "                              \"season\", \"date\", \"stage\", \"match_id\", 'match_num'], axis=1),\n",
    "                    train_df[\"win\"])\n",
    "\n",
    "\n",
    "    lgb_calib = CalibratedClassifierCV(lgb_cv, method='sigmoid', cv=5)\n",
    "\n",
    "    lgb_calib.fit(calib_df.drop([\"result\", \"win\", \"lose\", \"draw\", \"team_id\", \"m_rating\",\n",
    "                                 \"cumulative_lose\", \"cumulative_win\",\n",
    "                                 \"season\", \"date\", \"stage\", \"match_id\", 'match_num'], axis=1),\n",
    "                  calib_df[\"win\"])\n",
    "\n",
    "    lgb_calib_pred = lgb_calib.predict_proba(test_df.drop([\"result\", \"win\", \"lose\", \"draw\", \"team_id\", \"m_rating\",\n",
    "                                                           \"cumulative_lose\", \"cumulative_win\",\n",
    "                                                           \"season\", \"date\", \"stage\", \"match_id\", 'match_num'], axis=1))\n",
    "\n",
    "    ll_04.append(log_loss(test_df[\"win\"], lgb_calib_pred))\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "ll_05 = list()\n",
    "\n",
    "for i in range(1, len(seasons)-3):\n",
    "    \n",
    "    train_df = wl_odds_player_df.loc[wl_odds_player_df[\"season\"].isin(seasons[i:i+4][:-1])].dropna()\n",
    "    test_df = wl_odds_player_df.loc[wl_odds_player_df[\"season\"].isin([seasons[i:i+4][-1]])].dropna()\n",
    "    \n",
    "    test_dates = sorted(test_df[\"date\"].unique())\n",
    "    n = len(test_dates) // 2\n",
    "\n",
    "    #train_df = pd.concat([train_df, \n",
    "    #                      test_df.loc[test_df[\"date\"].isin(test_dates[:n])]])\n",
    "\n",
    "    test_df = test_df.loc[test_df[\"date\"].isin(test_dates[n:])]\n",
    "\n",
    "    train_dates = sorted(train_df[\"date\"].unique())\n",
    "\n",
    "    calib_dates = random.sample(train_dates, int(len(train_dates)*0.25))\n",
    "\n",
    "    calib_df = train_df.loc[train_df[\"date\"].isin(calib_dates)]\n",
    "\n",
    "    train_df = train_df.loc[~train_df[\"date\"].isin(calib_dates)]\n",
    "\n",
    "    feat_col = [c for c in train_df.columns if \"avg\" in c]\n",
    "    feat_col = feat_col + [\"rating_diff\"]\n",
    "\n",
    "\n",
    "    skf = StratifiedKFold(n_splits=6, shuffle=True)\n",
    "\n",
    "    lgb_param_grid = {'lgb__n_estimators': [100, 200, 300,],\n",
    "                     'lgb__num_leaves':  [2, 4, 6, 10, 15, 20],\n",
    "                     'lgb__subsample': [0.4, 0.6, 0.8, 1],\n",
    "                     'lgb__colsample_bytree': [0.4, 0.6, 0.8, 1],\n",
    "                     'lgb__is_unbalance': [False]}\n",
    "\n",
    "    lgb_pipe = Pipeline([(\"lgb\", lightgbm.LGBMClassifier())])\n",
    "\n",
    "    lgb_cv = RandomizedSearchCV(lgb_pipe, \n",
    "                                param_distributions=lgb_param_grid, \n",
    "                                n_jobs=-1,\n",
    "                                cv=skf, \n",
    "                                     n_iter=50,\n",
    "                                     refit=True,\n",
    "                                    )\n",
    "\n",
    "    lgb_cv.fit(train_df.drop([\"result\", \"win\", \"lose\", \"draw\", \"team_id\", \"m_rating\",\n",
    "                              \"cumulative_lose\", \"cumulative_win\",\n",
    "                              \"season\", \"date\", \"stage\", \"match_id\", 'match_num'], axis=1),\n",
    "                    train_df[\"win\"])\n",
    "\n",
    "\n",
    "    lgb_calib = CalibratedClassifierCV(lgb_cv, method='sigmoid', cv=5)\n",
    "\n",
    "    lgb_calib.fit(calib_df.drop([\"result\", \"win\", \"lose\", \"draw\", \"team_id\", \"m_rating\",\n",
    "                                 \"cumulative_lose\", \"cumulative_win\",\n",
    "                                 \"season\", \"date\", \"stage\", \"match_id\", 'match_num'], axis=1),\n",
    "                  calib_df[\"win\"])\n",
    "\n",
    "    lgb_calib_pred = lgb_calib.predict_proba(test_df.drop([\"result\", \"win\", \"lose\", \"draw\", \"team_id\", \"m_rating\",\n",
    "                                                           \"cumulative_lose\", \"cumulative_win\",\n",
    "                                                           \"season\", \"date\", \"stage\", \"match_id\", 'match_num'], axis=1))\n",
    "\n",
    "    ll_05.append(log_loss(test_df[\"win\"], lgb_calib_pred))\n",
    "\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(0.5836123186733118, 0.5829456665238759)"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.mean(ll_04), np.mean(ll_05)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.calibration import calibration_curve"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [],
   "source": [
    "i = 2\n",
    "    \n",
    "train_df = wl_odds_player_df.loc[wl_odds_player_df[\"season\"].isin(seasons[i:i+4][:-1])].dropna()\n",
    "test_df = wl_odds_player_df.loc[wl_odds_player_df[\"season\"].isin([seasons[i:i+4][-1]])].dropna()\n",
    "\n",
    "test_dates = sorted(test_df[\"date\"].unique())\n",
    "n = len(test_dates) // 2\n",
    "\n",
    "train_df = pd.concat([train_df, \n",
    "                      test_df.loc[test_df[\"date\"].isin(test_dates[:n])]])\n",
    "\n",
    "test_df = test_df.loc[test_df[\"date\"].isin(test_dates[n:])]\n",
    "\n",
    "train_dates = sorted(train_df[\"date\"].unique())\n",
    "\n",
    "calib_dates = random.sample(train_dates, int(len(train_dates)*0.25))\n",
    "\n",
    "calib_df = train_df.loc[train_df[\"date\"].isin(calib_dates)]\n",
    "\n",
    "train_df = train_df.loc[~train_df[\"date\"].isin(calib_dates)]\n",
    "\n",
    "feat_col = [c for c in train_df.columns if \"avg\" in c]\n",
    "feat_col = feat_col + [\"rating_diff\"]\n",
    "\n",
    "\n",
    "skf = StratifiedKFold(n_splits=6, shuffle=True)\n",
    "\n",
    "lgb_param_grid = {'lgb__n_estimators': [100, 200, 300,],\n",
    "                 'lgb__num_leaves':  [2, 4, 6, 10, 15, 20],\n",
    "                 'lgb__subsample': [0.4, 0.6, 0.8, 1],\n",
    "                 'lgb__colsample_bytree': [0.4, 0.6, 0.8, 1],\n",
    "                 'lgb__is_unbalance': [False]}\n",
    "\n",
    "lgb_pipe = Pipeline([(\"lgb\", lightgbm.LGBMClassifier())])\n",
    "\n",
    "lgb_cv = RandomizedSearchCV(lgb_pipe, \n",
    "                            param_distributions=lgb_param_grid, \n",
    "                            n_jobs=-1,\n",
    "                            cv=skf, \n",
    "                                 n_iter=50,\n",
    "                                 refit=True,\n",
    "                                )\n",
    "\n",
    "lgb_cv.fit(train_df.drop([\"result\", \"win\", \"lose\", \"draw\", \"team_id\", \"m_rating\",\n",
    "                          \"cumulative_lose\", \"cumulative_win\",\n",
    "                          \"season\", \"date\", \"stage\", \"match_id\", 'match_num'], axis=1),\n",
    "                train_df[\"win\"])\n",
    "\n",
    "\n",
    "lgb_calib = CalibratedClassifierCV(lgb_cv, method='sigmoid', cv=5)\n",
    "\n",
    "lgb_calib.fit(calib_df.drop([\"result\", \"win\", \"lose\", \"draw\", \"team_id\", \"m_rating\",\n",
    "                             \"cumulative_lose\", \"cumulative_win\",\n",
    "                             \"season\", \"date\", \"stage\", \"match_id\", 'match_num'], axis=1),\n",
    "              calib_df[\"win\"])\n",
    "\n",
    "lgb_calib_pred = lgb_calib.predict_proba(test_df.drop([\"result\", \"win\", \"lose\", \"draw\", \"team_id\", \"m_rating\",\n",
    "                                                       \"cumulative_lose\", \"cumulative_win\",\n",
    "                                                       \"season\", \"date\", \"stage\", \"match_id\", 'match_num'], axis=1))\n",
    "\n",
    "\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [],
   "source": [
    "lgb_og_pred = lgb_cv.predict_proba(test_df.drop([\"result\", \"win\", \"lose\", \"draw\", \"team_id\", \"m_rating\",\n",
    "                                                    \"cumulative_lose\", \"cumulative_win\",\n",
    "                                                    \"season\", \"date\", \"stage\", \"match_id\", 'match_num'], axis=1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAD4CAYAAAD8Zh1EAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3dd3hUZfrG8e+TSaOGkiACgQABEVBUAqiAFBtWRFFA7CjLrr1jXV1dV1flZ1mVRRcVUFEpiopgF0QRAkIoUkIPNaETCCnz/v6YqBEjGWCSyczcn+viMjPn5JznXMGbN++c87zmnENEREJfVLALEBGRwFCgi4iECQW6iEiYUKCLiIQJBbqISJiIDtaJExMTXUpKSrBOLyISkubMmZPjnEsqbVvQAj0lJYX09PRgnV5EJCSZ2Zo/26YpFxGRMKFAFxEJEwp0EZEwoUAXEQkTCnQRkTBRZqCb2Ugz22JmC/9ku5nZC2aWaWYZZnZS4MsUEZGy+DNCfwPodZDt5wAtiv8MBl458rJERCrIulkw/Vnff0NcmfehO+emmVnKQXbpDYxyvj68M82slpkd7ZzbGKAaRUTKx8pvYUwfcIAnFq6eBMkdy+10BUVe1m3bS7Ok6uVy/EDMoTcE1pV4nVX83h+Y2WAzSzez9Ozs7ACcWkTkMOXnwqRbwFsErgiK8mH19HI73cL1O+n9nxkMeHUme/MLy+UcgQh0K+W9UlfNcM6NcM6lOefSkpJKfXJVRKT87d8Db10KO9aAJwbM4xuhp3QN+KnyCop4asoSer80gy279/PohW2oGls+D+kH4qhZQHKJ142ADQE4rohI4O3f7QvzdbOg7/8gIdk3Mk/pWi7TLYNHz2Hasmwubd+IB89rTULVmICf4xeBCPRJwE1mNhboBOzU/LmIVEp5u+CtvpCVDpe8Bm0v9r0f4CDfs7+Q6CgjPsbDX7s154auTenaovxnJcoMdDN7B+gOJJpZFvB3IAbAOTccmAycC2QCe4Fry6tYEZHDlrcLxlwCG+ZC35HQ5qJyOc23y7K5f8ICLjqxAXef3YpTmtctl/OUxp+7XAaUsd0BNwasIhGRQMvbWRzmP0Hf16H1hQE/xY69+Tz28c+Mn5tF86Rq9GxVL+DnKEvQ2ueKiFSIfTtgzMWwcT5c+iYce37ATzEjM4dbx85jx958buqRyk09U4mP8QT8PGVRoItI+Nq3HUb3gU0L4bJR0Oq8cjlN3eqxJNepwpvXdaBNg4RyOYc/FOgiEp72bvOF+ZbF0G80HHNOwA7tnGPcnCwWbdjFIxe2oVX9mkz466mYlXYXd8VRoItI+Nm7DUb1huwl0G8MtDw7YIdet20v909cwPTlOXRMqUNeQRHxMZ6ghzko0EUk3OzdBqMuhOxl0P9taHFmQA5b5HWM+mE1/56ylCiDxy5qy8COjYmKCn6Q/0KBLiLhI3erb2SeswwGvA2pZwTs0Nty8xn2+TI6NavDP/scR8NaVQJ27EBRoItIeMjNgTcvhG0rYMA7kHr6ER+yoMjLBz+t55KTGpFUI45Pbu5Kcp0qlWJ6pTQKdBEJfXuyfdMs21bCgLHQvMcRH3JB1k7uHjefJZt2U69mPN1aJtG4btUAFFt+FOgiEtr2bIE3L4Dta+Dy96BZtyM6XF5BEc99sZxXp6+kbrVY/ntle7q1DI1mggp0EQlduzf7wnznOhj4PjQ98m6JN4xKZ/ryHPp3SOa+c48loUr5NdMKNAW6iISm3ZuKwzzLF+YpXQ7/UHkFxHiiiI/xcGOPVIZ0a07n1MQAFlsxtEi0iISeXRvhjfNg53q4YvwRhfnXS7Zw9v9N44UvlwNwcrO6IRnmoBG6iISaJZ/AhzdBwT64ciI0OeWwDrMtN5/HPl7MxJ/W06Jedc5ofVSAC614CnQRCR3LpsLYgYADTxxEHV4DrOnLs7lt7Dx27ivgltNbcGOP5sRFV3wzrUBToItIaCjMh0/u4tcVLr2FvpWGDmNxino14mmaWI3H+7SlVf2aga0ziDSHLiKhYcpQ2Ln2sNYAdc4xdtZaHvpgIQDH1K/B+0NOCaswB43QRSQUpI+E9P9B51uh1fmHtAbo2q17GTohg+9XbOXkZpWrmVagKdBFpHJb8z1MvtvXl+X0v/vmzf0I8iKv4/UZq3jms6VER0XxRJ/j6N8huVI10wo0BbqIVF47s+C9q6BWE7jkf4f0Iei23Hye/3I5nZsn8nifthydUPmaaQWaAl1EKqf8vTD2cijIg2s+gSq1yv6WQl8zrb7tfc20Jt/SlUa1K28zrUBToItI5eMcfHQLbMzwdU5MOqbMb5m/bgf3jMtg6ebd1E+I57SWSSTXqdzNtAJNgS4ilc/3L8CC96HnQ2UuHbcvv4hhny/lf9+tol6NeF67Ko3TQqSZVqAp0EWkcln+BXz+d2h9EXS9s8zdbxiVzneZOQzo2Jj7zm1FzfjQaaYVaAp0Eak8cjJh3HVwVFu46GX4k7nvXXkFxBY307q5Zyp/69GcU5uHZv+VQNKDRSJSOeTtgrEDfHey9H8LYquVutuXP2/mrGHTeL64mVanZnUV5sU0QheR4PN6YcINsHUFXPUh1G7yh1227tnPox8tZtL8DbSqX4NebeoHodDKTYEuIsH39T9h2RQ495lSF6mYtiyb296dx+68Am4/oyV/7d6c2GhNMBxIgS4iwbVwAkx/Bk66CjpcX+ou9RPiSU2qzuN92tLyqBoVXGDo0D9xIhI8mxbAhzdCo46+0Xnxh6Ber+PtH9fywMQFALQ8qgbvDTlFYV4GjdBFJDhyt8I7l0N8Leg3GqLjAFidk8vQCRnMXLmNU5rV/bWZlpRNgS4iFa+oAN6/GvZshus+hRr1KfI6Rn63imc/X0pMVBRPXnwc/TokR8xj+4Hg15SLmfUys6VmlmlmQ0vZnmBmH5nZfDNbZGbXBr5UEQkbU+/3tcC98AVo2B7wNdN68avldElN4vM7utG/Y2OF+SEqc4RuZh7gJeBMIAuYbWaTnHOLS+x2I7DYOXeBmSUBS83sLedcfrlULSKha86bMGsEnHIT+9tcyoRZa+mXluxrpnVrVxrWipxmWoHmz5RLRyDTObcSwMzGAr2BkoHugBrm+ylUB7YBhQGuVURC3dof4ZM7oVkPfjrmNu598TuWbd5Dw1pVOK1lEo1qR1YzrUDzZ8qlIbCuxOus4vdK+g9wLLABWADc6pzzHnggMxtsZulmlp6dnX2YJYtISNq5Ht69Am9CI56ueS8X/3cWu/MKef2aDhHbTCvQ/An00n73cQe8PhuYBzQATgD+Y2Z/WKzPOTfCOZfmnEtLStIPUCRiFOyDdwdCwV7ui72Pl2ZuY2Cnxnx2+2n0aFUv2NWFDX+mXLKA5BKvG+EbiZd0LfCkc84BmWa2CmgFzApIlSISupwj/4Obid3wE/R/m0viTuFi5+jUrG6wKws7/ozQZwMtzKypmcUC/YFJB+yzFjgdwMyOAo4BVgayUBEJTUs/eJLYRe8zrdFgaHUeHZvWUZiXkzJH6M65QjO7CZgKeICRzrlFZjakePtw4DHgDTNbgG+K5l7nXE451i0ilVzOnv2MfecN/pr1FN/FnkKts+8Ldklhz3yzJBUvLS3NpaenB+XcIlK+vlm6hWfHfspo730UVqtPws3fEFPlDx+ryWEwsznOubTStulJUREJuIZVi3jZ8ww1YmPwXD8eFOYVQs25ROSIeb2O0TPXcN+EBeD10uK7O0kuWo+n35tQp2mwy4sYGqGLyBFZmb2HoeMXMGv1Nrq2SKTwqyeIXvoJ9HoKmnUPdnkRRYEuIoelsMjLq9NX8X9fLCM+Ooqn+x5P3ypzsPefhhOugE5/CXaJEUeBLiKHZfveAoZ/u4IexyTxWO+21Nu3Al77GzRMg/OH/ekCz1J+FOgi4rf9hUWMm5PFgA6NSaoRx6e3dqVBrSqwdxu8MwDiakC/Mb/2NpeKpUAXEb/MWbOde8dnkLllD03qVKNLi0RfmBcV+nqb794I134KNY8OdqkRS4EuIgeVu7+QZz5byhvfr6ZBQhXevK4jXVok+jaumwVTH4SsH+GiV6BRqbdHSwVRoIvIQQ0enc6MzK1cfUoT7u7Viupx0bAzy9fXfPqz4IogKhrqpga71IinQBeRP9i5t4C4mCjiYzzcdkZL7j51MycULoCp/4VV02H7qt9/g3O+FYiSOwanYAEU6CJygCkLN/LcB98xJGUjFyWspMPq6bA107cxLgFSOkPHwb7FnT+5A4rywRMLKV2DW7go0EUE2JPNziVfM2/aRzTfMYcpUeshE4itAU1OhfbX+AK7/nEQ5fnt+xJTfSPzlK4anVcCCnSRSJS7FdbM8IXxqumQ/TMJQHsXz9bE9hSdeAOeZqdB/XbgOUhMJHdUkFciCnSRSLBvO6yeAau/84X45oW+92OqQuOTyWnWmxdW1ueqiy8i9ejawa1VDpsCXSQc5e2ENd/7Rt+rp8OmBYCD6HhI7oS3x4NMyW3BjNxk/nlpexKBfwS7ZjliCnSRcLDiG8h4F7z5kJMJmzLAecET55sS6X4fNO0KDduzYnsB947LIH3Ndk5rWUReQRHxMZ4yTyGVnwJdJNRNHwZfPvrb6/rHwWn3QEoXaNQBYuIBKCjyMmLaSp7/cjlVYjw8c2k7LjmpIaaeK2FDgS4Syua9A1+WmCwxD7TpA13v/MOuO/cVMGLaSs44th6PXNiGejXiK7BQqQgKdJFQ9eN/4dN74OgTIHsJFBX84X7wvIIi3k9fx8BOTUisHseU27pydEKVIBYt5UmBLhJqnINpT8PX/4RW50PfkbBx/h/uB5+9ehv3jstgZU4uTROr06VFosI8zCnQRUKJc/DZg/DDf6Dd5XDhi777xEvcD75nfyH/nrKEUT+soVHtKoweVKKZloQ1BbpIqPAWwUe3wk+joeNfoNeTEPXHZYEHj0rnh5VbubZzCneddQzV4vS/eaTQT1okFBTuhwk3wOIPodu9vtsQS9ydsmNvPnHRHqrEerjzrJaA0b6JHhCKNH/8511EKpf8XN9qQIs/hLOfgB73/y7MJy/YyBnDvuW5L5YB0L5JHYV5hNIIXaQy27cD3u4HWbPgwv/ASVf+umnLrjwe+nAhUxdt5riGCfQ+oWEQC5XKQIEuUlntyYYxfWDLEuj7OrS56NdNXy3ZzG1j57G/0MvQc1pxfZemRHv0C3ekU6CLVEY71sHoi2Dnerh8LKSe8bvNjetUpV1yLR69sA3NkqoHqUipbBToIpVNTiaM6g37d8NVH0DjkynyOt78fjVLNu3i333bkVqvBqMHdQp2pVLJKNBFKpONGTDmYt/95td8DEcfz/LNu7l3fAZz1+6gxzFJaqYlf0qBLlJZrJ0Jb10GcTXgqg/Jr9WM/365nBe/yqRanIfn+p1A7xMaqJmW/Cm/PkUxs15mttTMMs1s6J/s093M5pnZIjP7NrBlioS5zC9hdB+ongTXTYHEVHblFfC/Gas4q81RfH5HNy46UZ0R5eDKHKGbmQd4CTgTyAJmm9kk59ziEvvUAl4Gejnn1ppZvfIqWCTsLPoAxl8P9VqR138c7y7O48qTHYnV45h622kcVVNdEcU//ky5dAQynXMrAcxsLNAbWFxin8uBCc65tQDOuS2BLlQkLM0dDR/dAo06kt55OHe/toRVObmk1qtO59REhbkcEn+mXBoC60q8zip+r6SWQG0z+8bM5pjZVaUdyMwGm1m6maVnZ2cfXsUi4eKHl2HSTRSmdOPRWo/R943FFHq9vHV9JzqnqpmWHDp/RuilTdq5Uo7THjgdqAL8YGYznXPLfvdNzo0ARgCkpaUdeAyRyOAcfPMv+PYpaN2ba7dfz3dLshnUpSl3ntWSqrG6V0EOjz9/c7KA5BKvGwEbStknxzmXC+Sa2TSgHbAMEfmN1wtT74Mfh1PYbiDRvV/ktnW7uN3gpMbqvyJHxp8pl9lACzNramaxQH9g0gH7fAh0NbNoM6sKdAJ+DmypIiGuqBD34d/gx+GMsfP5d+xNEOWhfZPaCnMJiDJH6M65QjO7CZgKeICRzrlFZjakePtw59zPZjYFyAC8wGvOuYXlWbhISCncT97Ya4jPnMyzBX2ZVv8anmrfKNhVSZgx54IzlZ2WlubS09ODcm6RCrV/D1tHXkrdzd/zuPcajjrjVq7r0hRPlO4pl0NnZnOcc2mlbdOnLyLlad92eOtS6myZw4g693DF5beRklgt2FVJmFKgi5SDIq9j7Fez6TF7CA0K12GXjWLwsRcEuywJc2qgLBJgyzbvZsh/JnLqtCuok7+e/H5jQWEuFUAjdJEAyS/08so3K5j89Te8GfMv6sQVEHPlJKyx2txKxVCgiwTIrrwCZs74kvHxT1A1Lpaoqz6F+m2DXZZEEE25iByBfflFjPxuFUVeR+LWObwV8zjVqycQNWiqwlwqnEboIofp+xU5DB2/gLXb9tJ927s0m/c0UTWO9rW/TdCCzVLxFOgih2hXXgH/mryEd2atpVUdY/Yxb5M092Pfxtxs2LVegS5BoUAXOUSDR6Uza9VWhrVdQ5/NL2JrNuLrYeegqABWT4fkjsEuUyKQ5tBF/LB1z3725RcB8OCpVZnX/FUuzrwfq5YI578A0fFgHvDEQkrXIFcrkUojdJGDcM4xaf4GHpm0iP4nHcW9NT6j7fRnICoazv4XdBwMnmg46ljfyDylq0bnEjQKdJE/sXHnPh6cuJAvl2zhiqPWcHvmUNixAlpfBL3+BTUb/LZzckcFuQSdAl2kFJ8v3szt786jlnc7X6R8SOqmyVA7BQaOhxZnBLs8kVIp0EVK0bROPHfX/Y4r97xB1JZ9cNo90PUOiKkS7NJE/pQCXQQoLPIycsYqlmzczbCukPrx7aRumwtNT4PzhkFii2CXKFImBbpEvJ837uLe8RmszNrIc/U+wb36IVY1ES5+DY7rC6a+5RIaFOgSsfYXFvHS1yt4+evlXBo/m/dqjSFuVw7WYRD0fAiq1Ap2iSKHRIEuEWtPXiHf/jCTj2uPolXubKjdDs5/Fxq2D3ZpIodFgS4RZW9+IW//uJZrOx1N3fTn+cCGYQWxcM6/ocP1EOUJdokih02BLhFjRmYOQydk0HjHLAbOeocqu1djbS6Gs5+AmkcHuzyRI6ZAl7C3c18BT3zyM1+lZ/Bk9bGcHjsdYprBlROhec9glycSMAp0CXtDRv3IMeveZ3q194lzBdD9Puh8G8TEB7s0kYBSoEtYyt69n2pxHqpmZ/C//KFUjV4ATXrAec9C3ebBLk+kXCjQJaw455j403o+m/QOD1abSNXcRVStfhT0HQltLtY95RLWFOgSNtbv2Mez739JpzUjeCX6W8jFd9dKn+GaK5eIoECX0FdUyE9fjmXXjNd4mnlERTvAt+QEDtjwkwJdIoICXULX9tW4uaOxn8Zw4p5N7Iiuy56TbiWhaXuYMBiK8rXghEQUBbqElsJ8WDoZ75w3sZVf4zCsxZnQ/v+o1eIs32ITADXqa8EJiTgKdAkNW1fA3FEw7y3IzWZrVCJjCi5mc2pfHrnsbOJjDnjCUwtOSARSoEvlVbgffv4I5r4Jq6bhzENmrc48ueNaFsSn8Wj/47n9OD3hKfILBbpUPjnLYc4bMO9t2LcNEhpDzwfZ3vIy+r26jB7t6vHZ+cdSq2pssCsVqVT8CnQz6wU8D3iA15xzT/7Jfh2AmUA/59y4gFUp4a9gHyye5BuNr5nhW4T5mHPJO/4qRm1JYVCXVOpEGZ/fXp+61eOCXa1IpVRmoJuZB3gJOBPIAmab2STn3OJS9nsKmFoehUqY2vIzzHkT5r8DeTugdlM44xE4YSDTNhj3TVjAhp3LaNuoNqc2T1SYixyEPyP0jkCmc24lgJmNBXoDiw/Y72ZgPNAhoBVK+MnfC4sm+kbj636EqBg49gJofw2kdGVHXiGPf/Iz4+Zk0SypGu//5RTSUuoEu2qRSs+fQG8IrCvxOgvoVHIHM2sI9AF6cpBAN7PBwGCAxo0bH2qtEuo2LfCNxjPeg/07oW4qnPU4tBsA1RJ/3W3w6DnMWbOdG3s05+aeLf54B4uIlMqfQC+t+YU74PVzwL3OuSI7SK8M59wIYARAWlragceQcLTiG5j9qu+Dzpyl4ImD1r19o/Emp/7aW2XL7jyqx0VTNTaa+889lhiP0aZBQlBLFwk1/gR6FpBc4nUjYMMB+6QBY4vDPBE418wKnXMfBKRKCU0/joBP7y5+YdDpr9DtHqj62/SJc45xc7J4/JOfubR9Ix48vzUnJGstT5HD4U+gzwZamFlTYD3QH7i85A7Ouaa/fG1mbwAfK8wj3KxX4dN7fnttUVA96Xdhvm7bXu6fuIDpy3PokFKbAZ00DSdyJMoMdOdcoZndhO/uFQ8w0jm3yMyGFG8fXs41SigpzIcp90L6SGjUCTbNh6KCP/RUmbJwE3e8Nw8D/tG7DVd0akJUlFrbihwJv+5Dd85NBiYf8F6pQe6cu+bIy5KQlJsD710Na77zrQh0+sOwfs7veqo45zAzWh5Vnc6pifz9gtY0ql012JWLhAVzLjifTaalpbn09PSgnFvKwaaFMHYA7N4Mvf8Dx1/2u80FRV5GTFvJ0k27eWHAiUEqUiT0mdkc51xaadv06L8cuZ8/ggl/gfiacN2n0LD97zYvXL+Te8ZlsHjjLs47/mj2FxYRF61bEUUCTYEuh885mPY0fP1PX4j3ewtq/tYsK6+giOe/XM6IaSupUy2W/17ZnrPb1A9iwSLhTYEuhyc/Fz74Gyz+AI7vDxc8DzHxv9tlb34R781exyUnNeSBc1uTUDUmSMWKRAYFuhy6Het88+WbFsKZj8GpN//6gNCe/YWMmbmGG7o2o061WD6/oxt1qqkrokhFUKDLoVnzA7x7hW95t8vfg5Zn/brpm6VbeGDiQjbs3Ee7RrU4pXldhblIBVKgi//mjoKP74BajWHAWEhqCcD23Hwe+2QxE+auJ7VedcYNOZX2TWoHuViRyKNAl7IVFcJnD8CPw6FZD7j0dajyW2D/Zcwc5q7Zzi09U7mxZ6ruYBEJEgW6HNzebTDuWlj5DZx8I5z5D/BEs2VXHtXioqkWF80D5x5LjCeK1g1qBrtakYgWFewCpBLbsgRe7QlrvofeL0GvJ3BRHt6bvY7Th33LsM+XAdAuuZbCXKQS0AhdSrd0Coy/HmKqwNUfQ+NOrN3qa6b1XWYOHZvWYaCaaYlUKgp0+T3nYMZz8MWjcPTx0P9tSGjElIUbuf3d+XiijMcvasvlHRurmZZIJaNAl98U7INJN8OC96FNH+j9Mi6mCgYcU78m3Vom8fAFrWlQq0qwKxWRUijQxWfXBhh7OWz4CXo+RP4pt/PfaStZtmUPL/Q/gaaJ1Rh+ZfuyjyMiQaNAF8hKh7EDIX8P9H+bjOqdueelGSzZtJsL2jUgv8irWxFFQoACPdLNHwuTboEa9dk/YBzD5kfz6vQZJNWI49Wr0jiz9VHBrlBE/KRAj1TeIvjiEfj+Bd/iE5eNItdVZ9ycb+nXIZmh5xxLQhU10xIJJQr0SJS3E8YNgszPyT9pECOr38AN8bWpE2V8cUc3aqv/ikhIUqBHmpxMeKc/bF/FkrR/cO2CtmzetZJ2TepxSvO6CnOREKYnRSNJ5pfwWk+8e7fxQqNn6fVdKjXioxn/11M5pXndYFcnIkdII/RI4BzMfMXXYKtea24uuovPVsRx2xmp/K17KrHR+nddJBwo0MPd6u9g6gOwcR6FLc8j+pIRDMku4pboKI6pXyPY1YlIACnQw9mSybh3B2LOS4HzMNou5Lq46hzXKNiFiUh50O/a4WrND3jHXw/OC0CUOfrUWR3cmkSkXCnQw41z8OMIvG+cz+b8OPJdDF48REXHUbt1z2BXJyLlSFMu4aRgH+6jW7GMd9mXcibPRN3MfR2iSMyZ5Xt4KLljsCsUkXKkQA8T+Tmr2fH6ZSTmLoPu91HttHt4NuqXX8C6BrU2EakYmnIJA5kzPyLvpS7E71nHq42eIL/L3RClH61IpNEIPYTt21/ID6Mfotu6V1gVlcyW8/7HXzpqWkUkUinQQ9X+3XjGD6Fn1sdk1OpJ00EjSa1ZO9hViUgQ+fV7uZn1MrOlZpZpZkNL2T7QzDKK/3xvZu0CX6oA7MorYMzHX+BePYPY5ZPZ1/1Rjr9tAjUU5iIRr8wRupl5gJeAM4EsYLaZTXLOLS6x2yqgm3Nuu5mdA4wAOpVHwZHsi8WbmTJhJH8veJ7C+CrEXPkBVZp1C3ZZIlJJ+DPl0hHIdM6tBDCzsUBv4NdAd859X2L/mYCeRQygrXv28+ikhTRf/CLPRE9kb9LxxFzxNtRKDnZpIlKJ+BPoDYF1JV5ncfDR9yDg09I2mNlgYDBA48aN/SxR7hr1LVdv+ifdo+dR1G4gVc8fBjHxwS5LRCoZfwLdSnnPlbqjWQ98gd6ltO3OuRH4pmNIS0sr9Rjis3HnPmrGx1Btx1KG591FbPQGOGcYnrTrwEr7kYhIpPMn0LOAkr/bNwI2HLiTmR0PvAac45zbGpjyIo/X63hn9lr+NXkJjzZbwiVZTxIXVxOunawnPUXkoPwJ9NlACzNrCqwH+gOXl9zBzBoDE4ArnXPLAl5lhFiVk8vQ8Rmkr8rmxcSJnLtqAjQ+BS59E2posWYRObgyA905V2hmNwFTAQ8w0jm3yMyGFG8fDjwM1AVeNt90QKFzLq38yg4/n2Rs5I735lE/ejc/NBhBvW2zoeNgOOufEK1l4USkbH49WOScmwxMPuC94SW+vh64PrClRQbnHGZGmwY1GdRsO3duewzPrm1w0XA4YUCwyxOREKKGH0Gyv7CIYZ8t5ca35+KcI2XtBO5ZfxseTzRcN1VhLiKHTI/+B8Hctdu5d1wGy7fs4dJ2SXg/ug3P3DegWXfo+zpUrRPkCkUkFCnQK9De/EKembqM179fxdE143mrXzKd59wBS2dD59vg9IchyhPsMkUkRCnQK9D+Ai8fZWzgypObMLTNdqpOvATyc+GyUdC6d7DLE5EQpzn0crZzXwEvfLmcwiIvtavF8sXtp/GPo76j6tsXQXxNuOErhbmIBIRG6MkTdHMAAAkuSURBVOVo6qJNPPTBQrbm5tOpaR06sYiEzx6ATRlwzLnQZzjEJwS7TBEJEwr0cpC9ez+PTFrEJws2cuzRNfnf1R04bvsXMH4Q4CAqGk69VWEuIgGlKZdy8Le35vD54s3cdVZLJt3UmeNyJsPEv/BrCxznYO2MoNYoIuFHI/QAWb9jHwlVYqgeF83fL2hDXHQULWoZfPhXyBgL9Y+HnGVQVACeWEjRws0iElgK9CPk9TrG/LiGpz5dQr8OjXn4gta0bZgAGzNgxLWwbSV0vw9OuxvWz4HV031hrkZbIhJgCvQjsCJ7D0PHZzB79Xa6tkjk2s4pvumUWa/CZw9A1bpw9UeQUtxNOLmjglxEyo0C/TB9nLGBO96bT3x0FE/3PZ6+7Rth+7bDuzfAko+hxdlw0StQrW6wSxWRCKFAP0S/NNM6rmECvdrU58Hzj6VejXhYOxPGDYI9m+HsJ+Dkv2khChGpUAp0P+UVFPHiV8tZsSWXV644iSZ1q/HCgBPBWwTTnoav/+Vb43PQZ9DwpGCXKyIRSIHuhzlrtnHPuAxWZOdyyUmNyC/yEhftgd2bYcINsOpbaHsJnP+c7+lPEZEgUKAfRO7+Qp6eupQ3f1hNg4QqvHldR7q1TPJtzPwCJg6B/XvgwhfhxCs1xSIiQaVAP4iCIi+TF2zkqpObcHevVlSPi/bdR/7VYzDjeUg61ncXS71jg12qiIgC/UA79ubz+ozV3NwzlVpVY/nizm7UjI/xbdy+2vfB5/p0aH8NnP0viK0azHJFRH6lQC/h0wUbeejDRWzfm8+pzevSqVnd38J80Qcw6RbA+RahaHtxUGsVETmQAh3YsiuPhz9cxJRFm2jToCZvXteBNg2KG2cV7IOp90P6SGjYHvqOhNopQa1XRKQ0CnTgxrfnMj9rJ/f2asUNXZsS7SnuWZa9FN6/FrYsglNvhp4PQ3RscIsVEfkTERvoWdv3UqtqLNXjonnkwjbEx3honlTdt9E5+GkMfHoPxFSBgeOgxZnBLVhEpAwR1z7X63W8MWMVZ/3fNJ79bCkAbRok/Bbmebt895ZPusk3xTJkhsJcREJCRI3QM7f4mmmlr9lOt5ZJDOrS9LeN62ZBxnu+Pix7NkOPB6DrnVq0WURCRsQE+qT5G7jrvflUjfMw7LJ29DmxIfbLg0DrZsHr54K3ADA471noMCio9YqIHKqwD3Sv1xEVZbRrlMC5x9XngfNak1Qj7vc7rZ4O3kLf1xYFeTsqvlARkSMUtnPoeQVFPPnpEoaMmYNzjiZ1q/Fc/xP/GObgW3AiOh7Mo9WERCRkheUIfdaqbQwdn8HKnFz6pSVTUOSIjT5In5XkjnD1JK0mJCIhLawCfc/+Qp76dAmjZ64huU4VxgzqRJcWif59s1YTEpEQF1aBXljk5bPFm7iuc1PuOrslVWPD6vJERA4q5BNve24+r89YxS2nt6BW1Vi+vLO7ryuiiEiE8etDUTPrZWZLzSzTzIaWst3M7IXi7RlmVu5L9jjn+CRjI2f+37e8/M0K5q713ZmiMBeRSFVm+pmZB3gJOBPIAmab2STn3OISu50DtCj+0wl4pfi/5WLzrjwe+mAhny3ezHENExh1XSdaN9BKQSIS2fwZznYEMp1zKwHMbCzQGygZ6L2BUc45B8w0s1pmdrRzbmPAKwZufGsuC9bv5L5zWjGoS4lmWiIiEcyfQG8IrCvxOos/jr5L26ch8LtAN7PBwGCAxo0bH2qtv/pH77bEx0TR7Jf+KyIi4tccemk3cLvD2Afn3AjnXJpzLi0pKcmf+krVukFNhbmIyAH8CfQsILnE60bAhsPYR0REypE/gT4baGFmTc0sFugPTDpgn0nAVcV3u5wM7Cyv+XMRESldmXPozrlCM7sJmAp4gJHOuUVmNqR4+3BgMnAukAnsBa4tv5JFRKQ0ft207ZybjC+0S743vMTXDrgxsKWJiMih0P1+IiJhQoEuIhImFOgiImFCgS4iEibM93lmEE5slg2sOcxvTwRyAlhOKNA1RwZdc2Q4kmtu4pwr9cnMoAX6kTCzdOdcWrDrqEi65siga44M5XXNmnIREQkTCnQRkTARqoE+ItgFBIGuOTLomiNDuVxzSM6hi4jIH4XqCF1ERA6gQBcRCROVOtAr4+LU5c2Pax5YfK0ZZva9mbULRp2BVNY1l9ivg5kVmVnfiqyvPPhzzWbW3czmmdkiM/u2omsMND/+bieY2UdmNr/4mkO6a6uZjTSzLWa28E+2Bz6/nHOV8g++Vr0rgGZALDAfaH3APucCn+JbMelk4Mdg110B13wqULv463Mi4ZpL7PcVvq6ffYNddwX8nGvhW7e3cfHresGuuwKu+X7gqeKvk4BtQGywaz+Caz4NOAlY+CfbA55flXmE/uvi1M65fOCXxalL+nVxaufcTKCWmR1d0YUGUJnX7Jz73jm3vfjlTHyrQ4Uyf37OADcD44EtFVlcOfHnmi8HJjjn1gI450L9uv25ZgfUMDMDquML9MKKLTNwnHPT8F3Dnwl4flXmQP+zhacPdZ9QcqjXMwjfv/ChrMxrNrOGQB9gOOHBn59zS6C2mX1jZnPM7KoKq658+HPN/wGOxbd85QLgVuect2LKC4qA55dfC1wEScAWpw4hfl+PmfXAF+hdyrWi8ufPNT8H3OucK/IN3kKeP9ccDbQHTgeqAD+Y2Uzn3LLyLq6c+HPNZwPzgJ5Ac+BzM5vunNtV3sUFScDzqzIHeiQuTu3X9ZjZ8cBrwDnOua0VVFt58eea04CxxWGeCJxrZoXOuQ8qpsSA8/fvdo5zLhfINbNpQDsgVAPdn2u+FnjS+SaYM81sFdAKmFUxJVa4gOdXZZ5yicTFqcu8ZjNrDEwArgzh0VpJZV6zc66pcy7FOZcCjAP+FsJhDv793f4Q6Gpm0WZWFegE/FzBdQaSP9e8Ft9vJJjZUcAxwMoKrbJiBTy/Ku0I3UXg4tR+XvPDQF3g5eIRa6EL4U51fl5zWPHnmp1zP5vZFCAD8AKvOedKvf0tFPj5c34MeMPMFuCbjrjXOReybXXN7B2gO5BoZlnA34EYKL/80qP/IiJhojJPuYiIyCFQoIuIhAkFuohImFCgi4iECQW6iEiYUKCLiIQJBbqISJj4f97NJm3gHL+kAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "fop, mpv = calibration_curve(test_df[\"win\"], lgb_og_pred[:,1], n_bins=10)\n",
    "\n",
    "plt.plot([0, 1], [0, 1], linestyle='--')\n",
    "plt.plot(mpv, fop, marker='.')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [],
   "source": [
    "lgb_calib_pred = lgb_calib.predict_proba(test_df.drop([\"result\", \"win\", \"lose\", \"draw\", \"team_id\", \"m_rating\",\n",
    "                                                    \"cumulative_lose\", \"cumulative_win\",\n",
    "                                                    \"season\", \"date\", \"stage\", \"match_id\", 'match_num'], axis=1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAD4CAYAAAD8Zh1EAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3deXyU1dn/8c+VnbAESNgJhCXsKEIAZZFFUaQgdfuJG6go0qqtVVu32tra53mstVatC0XrBioioCIiixvggkAQwg6BsIR9kz3b5Pz+mKgBEjLAJJOZ+b5fL17NzH1m5rpN+uVwct/XMeccIiIS/CICXYCIiPiHAl1EJEQo0EVEQoQCXUQkRCjQRURCRFSgPjgpKcmlpKQE6uNFRIJSenr6HudcnZKOBSzQU1JSWLRoUaA+XkQkKJnZptKOaclFRCREKNBFREKEAl1EJEQo0EVEQoQCXUQkRJQZ6Gb2qpntMrPlpRw3M3vOzDLNLMPMOvu/TBERKYsvM/TXgYGnOH4ZkFr0ZxTw0tmXJSIip6vMQHfOzQX2nWLIUOBN5zUfqGlmDfxVoIhIqMj3FLJh9+Fye39/rKE3ArYUe5xd9NxJzGyUmS0ys0W7d+/2w0eLiASH5VsPMPT5r7nu5fkczSsol8/wR6BbCc+VuGuGc26scy7NOZdWp06Jd66KiISUnHwPf5+xmqEvfM2uQ7n85fL2xMeUz036/njXbCC52OPGwDY/vK+ISNAbNS6duWt3c02XxvzxF+1IiI8ut8/yR6BPBe4yswlAd+CAc267H95XRCQoHc4tICrCiIuO5Fd9WnB772b0Ti3/VYkyA93M3gH6Aklmlg38GYgGcM6NAaYDg4BM4ChwS3kVKyJS2c1Zu5uHpyzjl+c15PeXtuGCFokV9tllBrpz7royjjvgTr9VJCIShH44msfj01YxeXE2LepUpX+buhVeQ8Da54qIhIqvM/fw2wlL+OFoHnf1a8ld/VsSFx1Z4XUo0EVEzlJitRiSa1fhjVu70r5hQsDqUKCLiJwm5xyT0rNZse0gj13enjb1azDlVz0wK+kq7oqjQBcROQ1b9h3l4feXMW/dHrql1CYn30NcdGTAwxwU6CIiPvEUOt78diNPzlhDhMHjv+zADd2aEBER+CD/kQJdRMQH+47k8fTstXRvXpv/uaIjjWpWCXRJJ1Ggi4iUIt9TyAffb+Wqzo2pUz2Wj+/uTXLtKpVieaUkCnQRkRIsyz7A7yctZfWOQ9StEUefVnVokhgf6LJOSYEuIlJMTr6HZz5dx8vzNpBYNYb/3NSFPq2Co5mgAl1EpJjb31zEvHV7GNY1mYcGtSWhSvk10/I3BbqIhL1DOflER0YQFx3Jnf1aMrpPC3q2TAp0WadNm0SLSFj7YvUuLv3XXJ77bB0A5zdPDMowB83QRSRM7TuSx+PTVvL+91tJrVuNi9vVC3RJZ02BLiJhZ9663dwzYQkHjuXzm4tSubNfC2KjKr6Zlr8p0EUk7NStHkezpKr87YoOtKlfI9Dl+I3W0EUk5DnnmLBgM49+sByA1vWr897oC0IqzEEzdBEJcZv3HuXBKRl8s34v5zevXM20/E2BLiIhyVPoeO3rLJ6atYaoiAj+94qODOuaXKmaafmbAl1EKq/N38GmryClNyR3O62X7juSx7OfraNniyT+dkUHGiRUvmZa/qZAF5HKac0MmFC0pXFkLIyYWmao5xV4m2ld3cXbTGv6b3rTuFblbablbwp0Eal8Du2AD+8EV+h97MmDjfNOGehLt/zAHyZlsGbnIeonxHFhqzok167czbT8TYEuIpXLwe3wxmDIPeydmRcWQGSMd9mlBMfyPDw9ew3//SqLutXjeGV4GhcGSTMtf1Ogi0jlcXAbvD4YDu+EER+CRXhn5qdYQ7/9zUV8lbmH67o14aFBbagRFzzNtPxNgS4ilcOBbG+YH9kDN06BJt29z5cQ5Adz8okpaqZ1d/+W/LpfC3q0CM7+K/6kG4tEJPB+2AKv/wKO7oWb3v85zEvw2aqdXPL0XJ4taqbVvXmiwryIZugiElj7N3nXzI8dgJs+gMZdShy293Auf/loJVOXbqNN/eoMbF+/ggut/BToIhI4+zfC60Mg9wAM/wAadS5x2Ny1u7nn3SUcysnndxe34ld9WxATpQWGEynQRSQw9mXBG0Mg9xAMnwoNO5U6tH5CHC3rVONvV3SgVb3qFVhkcNFfcSJS8fau966Z5x323jB0QpgXFjre/m4zj7y/DIBW9aozcfQFCvMyaIYuIhVr73rv1SwFOTDiI6jf8bjDG/cc4cEpGczfsI8Lmif+1ExLyqZAF5GKs2edN8wLC+DmaVCv/U+HPIWOV7/K4p+z1xAdEcETV3bk2q7JYXPbvj/4tORiZgPNbI2ZZZrZgyUcTzCzj8xsqZmtMLNb/F+qiAS13Wu8yyzOc1KYg7eZ1r8/X0evlnWYfW8fhnVrojA/TWXO0M0sEngBGABkAwvNbKpzbmWxYXcCK51zQ8ysDrDGzN5yzuWVS9UiElx2rfb+AhRgxDSo2waA3AIPUxZv5dq0ZG8zrd/2plHN8Gmm5W++LLl0AzKdcxsAzGwCMBQoHugOqG7e70I1YB9Q4OdaRSQY7VzpDfOISG+Y12kFwPeb9/PA5AzW7jxMo5pVuLBVHRrXCq9mWv7mS6A3ArYUe5wNnHgb1/PAVGAbUB241rkf26T9zMxGAaMAmjRpcib1ikgw2bmiKMyjvcssSakczSvgn7PW8urXWdSvEcdrN3cN22Za/uZLoJf0bx93wuNLgSVAf6AFMNvM5jnnDh73IufGAmMB0tLSTnwPEQklO5bBG5dDVJw3zBNbADDqzXS+ytzDjec34YGBbagexs20/M2XQM8Gkos9box3Jl7cLcATzjkHZJpZFtAGWOCXKkUkuGxfCm8Oheh4GPERB+KbEFt0+eFvLkrl7v4t6d48MdBVhhxfrnJZCKSaWTMziwGG4V1eKW4zcBGAmdUDWgMb/FmoiASJbd97Z+Yx1eDmj5m9sxqX/GsOz3zqbabVrVlthXk5KXOG7pwrMLO7gJlAJPCqc26FmY0uOj4GeBx43cyW4V2iecA5t6cc6xaRymhrOoy7AmIT2HfNZP40Yz/TMlbSpn51BnVUM63y5tONRc656cD0E54bU+zrbcAl/i1NRIJK9iIYdyVUSWB+7zcY/epGjuZ6uG9AK0b3bUF0pDqNlDfdKSoiZ2/LQhh/JcTXhhHTqJ1bk9b1cvnbLzuQqv4rFUaBLiJnZ/N3uPFXcSiqJi/W+ycP1kymFfDuHRcEurKwo38DiciZ2/QtheOuYLunBgP2PcCKI9XJyfcEuqqwpRm6iJyRgg3zcOOvYYunFrfbo9x/dS+u7tJYt+0HkAJdRE7PlgXw/Tgil75LVmESL6X8i3eu6kPdGnGBrizsKdBFxGd5a2YROeE6IlwBhpFw+RP8o/OlgS5LiijQRaR0znn7saybyeFl04nflU7Ej50/LILEI+sCW58cR4EuIsfLOwpZc2DtTFg3Gw5mA7CxMIVNUf24jK+IcB6IjIGU3gEuVopToIsI7N8Ia2fBupmQNQ88uRBdFVr04+XIq3l5eyqXXdCJ3w9sQ8SuxbBxnjfMk7sFunIpRoEuEo48+bB5vjfA186CPWu8z9duAV1HcrjJRUQ160FclXg6bdzHC0DXlNreMcndFOSVlAJdJFwc3g2Zs71LKeu/gNwD3j7lKT2hy83Q6lJIbMGM5dt59P0VXNl5Ew9d1vbnIJdKT4EuEqoKC2HH0p+XUrYuBhxUqw/tLvcGePO+EOu9NX/XoRz+PD6dT5bvoF2DGgw5p2Egq5czoEAXCSW5h7yz73VFv9A8vBMwaNQF+j0MqZdA/XMg4vibxL9Ys4t7JizhWL6H31/amlEXNlczrSCkQBcJdiunwvfj4PAu7yWGhfkQmwAt+0PqpdDyYqh26i3eGtesQvuGNfjr0A60rFutggoXf1OgiwSz78fDh3cWPTDoeLV3PTy5O0SWvrVbYaFj3PxNrNp+kCeuOofUetV5+/bzK6RkKT8KdJFgdSAbPnng58cWAXXbQkqvU75s/e7DPDApg0Wb9nNhqzrkFG0NJ8FPgS4SjHIPwdvXgiuEqFjwFJR5o0++p5Cxczfw7GfrqBIdyVPXnMtVnRupmVYIUaCLBBtPAbx3C+xaBTdO8u7d6cONPgeO5TN27gYubluXxy5vT93qaqYVahToIsHEOZjxgPd68iHPQov+3udLCfKcfA/vLdrCDd2bklQtlhn39KZBQpUKLFgqkgJdJJjMfwkWvgI9fuP95ecpLNy4jwcmZbBhzxGaJVWjV2qSwjzEKdBFgsXqj2Hmw9B2CFz8l1KHHc4t4MkZq3nz2000rlWFcSO70Ss1qQILlUBRoIsEg23fw+TboOF5cMXYk24MKm7Um4v4dsNebumZwv2XtKZqrP5vHi70nRap7A5kw9vDID4RrpsAMfEnDfnhaB6xUZFUiYnkvktaAUaXprUqvlYJKN3bK1KZ/Xh5Yv5RuH4iVK930pDpy7Zz8dNzeObTtQB0aVpbYR6mNEMXqayKX554w3tQr91xh3cdzOHRD5czc8VOOjZKYGinRgEqVCoLBbpIZVT88sTBz0DLi447/PnqndwzYQm5BYU8eFkbbuvVjCg10wp7CnSRyqj45Ylpt5x0uEnteM5NrslfLm9P8zpqpiVe+itdpLJZPf2kyxM9hY5Xv8riD5OWAtCybnXGjeyuMJfjaIYuUpls+x4mjzzu8sR1Ow/xwOQMFm/+gX6t1UxLSqdAF6ksTrg8MS8ijv98to5/f55J1dhInrm2E0M7NVQzLSmVT0suZjbQzNaYWaaZPVjKmL5mtsTMVpjZHP+WKRLiSrg88WBOPv/9OotL2tdj9r19+OV56owop1bmDN3MIoEXgAFANrDQzKY651YWG1MTeBEY6JzbbGZ1y6tgkZBT7PLEvGHv8s76eG6q40iqFsvMey6kXg11RRTf+LLk0g3IdM5tADCzCcBQYGWxMdcDU5xzmwGcc7v8XahISHIOZjwImbPZcP7/MPKjaLL2rKBl3Wr0bJmkMJfT4suSSyNgS7HH2UXPFdcKqGVmX5pZupkNL+mNzGyUmS0ys0W7d+8+s4pFQsl3Y2Dhy8yrcx39v2xGQWEhb93WnZ4t1UxLTp8vM/SSFu1cCe/TBbgIqAJ8a2bznXNrj3uRc2OBsQBpaWknvodIeFk9HWY8xIK4nozI/gUjezXjvktaER+jaxXkzPjyk5MNJBd73BjYVsKYPc65I8ARM5sLnAusRUROcmD9QmpMHok1PI+oi15hUnQVOjdR/xU5O74suSwEUs2smZnFAMOAqSeM+RDobWZRZhYPdAdW+bdUkeDnnGPWt+nkjbuGg1YDrptA5xYNFebiF2XO0J1zBWZ2FzATiARedc6tMLPRRcfHOOdWmdkMIAMoBF5xzi0vz8JFgs2OAzk8PmUBd2XdSXxkLjt/OZGEEronipwpcy4wS9lpaWlu0aJFAflskYr22aqd3DchnWd4kgsjMnDXv0dk6kVlv1DkBGaW7pxLK+mYermIVICmteN5qvo79LXvifjFUwpzKRcKdJFy4Cl0vDJvA/dNLGqmlTWeiw9PhR53Q9qtAa5OQpWujxLxs7U7D/GHSRks2fID/dvUJW/FNGJmPARtBsPFfw10eRLCFOgifpJXUMhLX67n+S/WUT0ummeHdeLyuruw126Hhp3gypdPubmzyNlSoIv4ycGcfF7/JotBHRvwp8HtSPTshpd/7J74bombO4v4kwJd5Cwcy/PwzoLNjOiR8lMzrbo14rzdE8ddC3lHYOSsEjd3FvE3BbrIGfpm/R4enLyMzfuO0rp+dXq2TPKGuacAJt1atLnzxJM2dxYpLwp0kdN0MCef/5u+mncWbKZpYjzv3H4+F7RI9B78sXviulkw+F/Q8uLAFithRYEucppGvbmIBVn7uOPC5txzcSuqxBTbDq6oeyIX3KXLE6XCKdBFfLD3cC7xMVFUiYnkDwPbEGnGuck1fx6wZQEsfAUy3vVenjjg8cAVK2FLgS5yCs45pi7dxmNTV3BNWjIPD2p7ciOtLQvg9cHgyQUMut2hyxMlIPRTJ1KK7QeOcdsbi/jthCU0TazK1V0aHz+gIA9WfABTRhWFOWARsHVhxRcrgmboIiWavXInv3t3CZ5Cx6OD23FzjxQiI4r2etm3ARa/Cd+PhyO7oWpdiIjy/kI0MgZSege2eAlbCnSREjRLqkpaSi3+enkHmiTGe2fjq6ZD+muw4UuwSGg1ELrcDC0vgq3psHGeN8yTuwW6fAlTap8rAhR4Cnn16yxWbz/E09d2+vnAibPxhGToPBzOuxFqNAxcwRK2TtU+VzN0CXurth/kgckZZGQfYEC7euTkHCNu/czSZ+MRkWW9pUhAKNAlbOUWeHjhi/W8+EUmNeOjee3yJPoemYb9e5h3Nl6jMfR7RLNxCRoKdAlbh3MKmPDtev7YbC03RH1B9KwvvVeptBoIXW7RbFyCjgJdwsrRvALe/m4zt7SFxCVvMj9uPBFbNRuX0KBAl7DxzdrtfDLpvww4NoPIz5aBRRCh2biEEAW6hLxD29ayaMozdNg9jR52gNzqDaC7ZuMSehToElq2LPBeD97kAji8C9Jfp/qGL7jQGRtq9SLh0ruIbT1As3EJSQp0CR1bFsAbg6EgD4fDAGo0ZlvneznQehhtW7cOdIUi5UqBLsFv/yZYMQXmvwQF3p4qDshIHMS5d46nYUQkWliRcKBAl+B0cJu3MdbyybDVe8dxfu1WGHvBFeKxKBL73KGlFQkrCnQJHod3w6oPYfkU2PQN4KB+R7j4MebF9GL0tL2cw1rub7WLTr2H0Lhp90BXLFKhFOhSuR3bD6umeWfiWXPBeSCpNfR9CDpciUtsiZnRcPdhzm++iscuv43k2vGBrlokIBToUvnkHoI1n3hDPPMzKMyHWs2g1z3Q4Sqo246CQsfL87JYs2MJzww7jxZ1qvHfm7sGunKRgFKgS+WQfwzWzvSG+LpZUJDjvXuz+x3eEG94Hpi3H/nKbQf5w+SlLN96kEvb1yMn30NctNbKRRToEjgFubD+c2+Ir/kE8g57N4voPNwb4o27HbeVW06+h+c/z2TMnPXUjI/hpRs6c1nHBgE8AZHKRYEuFctTAFlzvL/YXP0R5ByAKrW8Ad7hKkjpVeqVKUdyC3h7wWaGdmrEo4PbUjM+poKLF6ncfAp0MxsIPAtEAq84554oZVxXYD5wrXNukt+qlOBW6IHN33pn4is/hKN7IbYGtPmFN8Sb94XI6BJfeiS3gLe+28TIXs1JrBbL7N9dSGK12AotXyRYlBnoZhYJvAAMALKBhWY21Tm3soRxfwdmlkehEoTSi3b62ZfpDfHoeG9r2g5XQcuLITrulC+fu3Y3D01ZxrYDx+jQKIEeLZIU5iKn4MsMvRuQ6ZzbAGBmE4ChwMoTxt0NTAZ0qUG4cw7evwMy3vU+tgjo9zBccBfEVC3z5T8czeNvH69iUno2zetU5b07LiAtpXY5Fy0S/HwJ9EbAlmKPs4Hj7tgws0bAFUB/ThHoZjYKGAXQpEmT061VgoEnH6be/XOYA2AQEeVTmAOMGpdO+qb93NmvBXf3T9UVLCI+8iXQrYTnTtxZ+hngAeecx6yk4UUvcm4sMBa8m0T7WqQEibwjMHEEZM6GziMgYyJ48iAyBlJ6n/Kluw7lUC02iviYKB4e1JboSKN9w4QKKlwkNPgS6NlAcrHHjYFtJ4xJAyYUhXkSMMjMCpxzH/ilSqn8juyBt66B7UtgyHPQZYS33/jGed4wT+5W4succ0xKz+ZvH6/imi6N+ePgdnRKrlnBxYuEBl8CfSGQambNgK3AMOD64gOcc81+/NrMXgemKczDyP6NMO5KOLgVrn0L2gzyPp/crdQgB9iy7ygPv7+Meev20DWlFtd11zKcyNkoM9CdcwVmdhfeq1cigVedcyvMbHTR8THlXKNUZtsz4K2rvTcJDZ8KTXxriDVj+Q7unbgEA/46tD03dm9KRETpy3UiUjafrkN3zk0Hpp/wXIlB7py7+ezLkqCwYQ5MuAHiEmDkR1Cn7A0knHOYGa3qVaNnyyT+PKQdjWupmZaIP0SUPUSkBMsnw/iroGYyjJxVZpjnewp54YtMfjthCQDN61Tj5eFpCnMRP1Kgy+mbPwYmjYTGXeGW6ZDQ6JTDl289wNDnv+YfM9fgcY7cAk8FFSoSXtTLRXznHHz6GHz9DLQZDFe9AtFVSh2ek+/h2c/WMXbuBmpXjeE/N3Xh0vb1K65ekTCjQBff/HjD0NJ3IO1WGPRUmdu7Hc3zMHHhFq7q3IhHBrUjIb7kfi0i4h8KdClb7mF4bwRkfgr9/ggX3v9Tb/ITHc4tYPz8Tdzeuzm1q8Yw+94+1K6qrogiFUGBLqdW0g1DpfhyzS4eeX852w4c49zGNbmgRaLCXKQCKdCldKXdMHTisCN5PP7xSqYs3krLutWYNLoHXZrWqthaRUSBLqXYvtQ7M/fhhqE7xqezeNN+ftO/JXf2b0lslJppiQSCAl1OtuFLmHDjKW8Y2nUwh6qxUVSNjeKRQW2JjoygXcMaFV+riPxE16HL8ZZNgvFXe28Yum32SWHunGPiwi1c9PQcnp69FoBzk2sqzEUqAc3Q5WfzX4IZD0KTHnDd2969PovZvNfbTOurzD10a1abG9RMS6RSUaBL0Q1Df4avn4W2Q+DKV07aHm7G8u387t2lREYYf/tlB67v1kTNtEQqGQV6uDvuhqGRMOgfx90w9GMzrdb1a9CnVR3+NKQdDWuWfneoiASOAj2c5R6GicNh/WfQ/4/Q++cbhvIKCvnPnPWs3XWY54Z1ollSVcbc1CXABYvIqSjQw1XxG4Yu/zd0Hv7ToYzsH/jDpAxW7zjEkHMbkucp1KWIIkFAgR6O9mXB+Cvh4HYY9ja0vgzwNtP61+y1vDxvA3Wqx/Ly8DQGtKsX4GJFxFcK9HCzfan3ssTCfBgx9bgt4o7meZiUns21XZN58LK2JFRRMy2RYKJADyc/3jBUpSbcOA3qtOZQTj7j5m/ijgtbULtqDJ/e24da6r8iEpQU6OFi2SR4fzQkpcKNk6FGQz5fvZNH3l/OzoM5nJdciwtaJCrMRYKYAj3UbVkAc5+CdTOhaU8Y9jZ7PVX464Tv+XDJNlrVq8aLN/TgvCZqpiUS7BTooWzzd/D6ICgsAIuAvg9BlZr8asy3fL9lP/dcnMqv+7YkJkodIERCgQI9lH3xP94wBxxG/sb5xDTrzaOD2xETFUHr+tUDXKCI+JOmZqHqm+chaw7OIikkkpzCSMbv9PZe6dg4QWEuEoI0Qw9Fi16DWY9wpOUQntzfl6o7FnC4wfmMHDA40JWJSDlSoIeaZZNg2u/YVb8PF625FiJieHjoJQzrmoyVsg+oiIQGBXooWT0dN2UUltKLowNfpfdnm3h0cDsaJKiZlkg4UKCHiPy1n2PvDic7NpWmw94mJa4GL96QFOiyRKQC6ZeiIWDtok/xvD2MdZ76/Cf5SfKiqga6JBEJAM3Qg9ixPA9vfTCV/7fi1+yy2uy54l3+77z2gS5LRAJEgR7E8nas4qqVv8ETU53at31Ck3rNAl2SiASQT0suZjbQzNaYWaaZPVjC8RvMLKPozzdmdq7/SxWAgzn5PP/5Ogr2ZJHw3tUkVI2j1uhPqKYwFwl7Zc7QzSwSeAEYAGQDC81sqnNuZbFhWUAf59x+M7sMGAt0L4+Cw9mnK3fyyAfLiDi0ndsW/50oTw4RN0+HxBaBLk1EKgFflly6AZnOuQ0AZjYBGAr8FOjOuW+KjZ8PNPZnkeFu7+FcHvtoJR8t3Ub3uoW8Ef8McUd/gBEfQr12gS5PRCoJX5ZcGgFbij3OLnquNCOBT0o6YGajzGyRmS3avXu371WGuV+NX8yM5dt5sF8D3ol/krjD2XD9u9BIe3yKyM98maGXdHuhK3GgWT+8gd6rpOPOubF4l2NIS0sr8T3Ea/uBY9SIi6ZqbBR/GtKOWJdD6sybYNcquG4CpPQMdIkiUsn4MkPPBpKLPW4MbDtxkJmdA7wCDHXO7fVPeeGnsNDx1nebGPD0XP45ay0AHerGkvr5KMheCFf/F1IvDnCVIlIZ+TJDXwikmlkzYCswDLi++AAzawJMAW5yzq31e5VhImvPER6cnMF3Wfvo2TKRm3ukgCcfJt3q3T7uly9Bu6GBLlNEKqkyA905V2BmdwEzgUjgVefcCjMbXXR8DPAnIBF4sagBVIFzLq38yg49H2ds596JS4iJiuDJq87hmrTGmHPw/ihY8zEMego6XV/2G4lI2PLpxiLn3HRg+gnPjSn29W3Abf4tLTw45zAz2jeswYB29Xh0cDvq1YgD5+Dje2HZe3DRn6Hb7YEuVUQqOfVyCZDcAg9Pz1rDnW8vxjlHSlJVnr++889hPuuPkP4a9L4Pet8b6HJFJAgo0ANg8eb9DH7uK577PJO4qEjyPIXHD5jzJHz7PHS7A/o/GpgiRSToqJdLBTqaV8BTM9fy2jdZNKgRx2u3dKVf67rHD/r2Bfjyf6HTDTDwCdCmFCLiIwV6BcrNL+SjjG3cdH5T/jCwDdViT/jPn/4GzHzYeyXLkOcgQv+AEhHfKdDL2YFj+bzxzUZ+3bcFtarG8Om9fUioEn3ywGWT4KPfQssBcOUrEKlvjYicHqVGOZq5YgePfrCcvUfy6N6sNt2bJ5Yc5ms+gffvgKY94dpxEBVT8cWKSNBToJeD3YdyeWzqCj5etp22DWrw3xFd6dg4oeTBG76EiSOgwblw/QSI1v6fInJmFOjl4NdvpbN0ywHuv6QVd/RpQXRkKWvhi16H6fdDQiO4YRLEVq/QOkUktCjQ/WTrD8dIqBJNtdgo/jykPbFREaTWO0VAr/8Spv3W+/WhHbA3E+K7VUitIhKadBnFWSosdLz57UYueXoOT//YTKtRwqnDHGBbOj/95/fkw8Z55VqniIQ+zdDPwvrdh3lwcgYLN+6nd2oSt/RM8f3FKb0hKhY8eRAZ430sInIWFOhnaFrGNu6duJS4qPZ9HHgAAAcPSURBVAj+cfU5XN2lMXY6NwEld4MRU70z85Te3sciImdBgX6afmym1bFRAgPb1+ePg9tSt3rcmb1ZcjcFuYj4jdbQfZST7+EfM1fzq/HeZlpNE6vy3HXnnXmYi4j4mQLdB+mb9vGL5+bxwhfrqRobdXIzLRGRSkBLLqdwJLeAf8xcwxvfbqRhQhXeuLUbfVrVCXRZIiIlUqCfQr6nkOnLtjP8/Kb8vqRmWiIilYgS6gQ/HM3jta83cnf/ltSMj+HT+/pQI66E/isiIpWMAr2YT5Zt59EPV7D/aB49WiTSvXmiwlxEgoYCHdh1MIc/fbiCGSt20L5hDd64tSvtG5bSTEtEpJJSoAN3vr2YpdkHeGBgG27v3Yyo0pppiYhUYmEb6Nn7j1IzPoZqsVE8dnl74qIjaVGnWqDLEhE5Y2E3FS0sdLz+dRaX/Gsu/5y1BoD2DRMU5iIS9MJqhp65y9tMa9Gm/fRpVYeRvZoFuiQREb8Jm0CfunQb909cSnxsJE//v3O54rxGp9dMS0Skkgv5QC8sdEREGOc2TmBQx/o88ot21KkeG+iyRET8LmTX0HPyPTzxyWpGj0//qZnWM8POU5iLSMgKyUBfkLWPQc/OY8yc9dSKjyHf4wJdkohIuQupJZfDuQX8/ZPVjJu/ieTaVRg/sju9UpMCXZaISIUIqUAv8BQya+UObu3ZjPsvbUV8TEidnojIKQV94u0/ksdrX2fxm4tSqRkfw2f39VVXRBEJSz6toZvZQDNbY2aZZvZgCcfNzJ4rOp5hZp39X+rxnHN8nLGdAf+aw4tfrmfx5h8AFOYiErbKTD8ziwReAAYA2cBCM5vqnFtZbNhlQGrRn+7AS0X/Wy52Hszh0Q+WM2vlTjo2SuDNW7vTrmGN8vo4EZGg4Mt0thuQ6ZzbAGBmE4ChQPFAHwq86ZxzwHwzq2lmDZxz2/1eMXDnW4tZtvUAD13WhpG91ExLRAR8C/RGwJZij7M5efZd0phGwHGBbmajgFEATZo0Od1af/LXoR2Ii46gufqviIj8xJepbUn3x594YbcvY3DOjXXOpTnn0urUOfO9Ods1rKEwFxE5gS+Bng0kF3vcGNh2BmNERKQc+RLoC4FUM2tmZjHAMGDqCWOmAsOLrnY5HzhQXuvnIiJSsjLX0J1zBWZ2FzATiARedc6tMLPRRcfHANOBQUAmcBS4pfxKFhGRkvh00bZzbjre0C7+3JhiXzvgTv+WJiIip0PX+4mIhAgFuohIiFCgi4iECAW6iEiIMO/vMwPwwWa7gU1n+PIkYI8fywkGOufwoHMOD2dzzk2dcyXemRmwQD8bZrbIOZcW6Doqks45POicw0N5nbOWXEREQoQCXUQkRARroI8NdAEBoHMODzrn8FAu5xyUa+giInKyYJ2hi4jICRToIiIholIHemXcnLq8+XDONxSda4aZfWNm5waiTn8q65yLjetqZh4zu7oi6ysPvpyzmfU1syVmtsLM5lR0jf7mw892gpl9ZGZLi845qLu2mtmrZrbLzJaXctz/+eWcq5R/8LbqXQ80B2KApUC7E8YMAj7Bu2PS+cB3ga67As65B1Cr6OvLwuGci437HG/Xz6sDXXcFfJ9r4t23t0nR47qBrrsCzvlh4O9FX9cB9gExga79LM75QqAzsLyU437Pr8o8Q/9pc2rnXB7w4+bUxf20ObVzbj5Q08waVHShflTmOTvnvnHO7S96OB/v7lDBzJfvM8DdwGRgV0UWV058OefrgSnOuc0AzrlgP29fztkB1c3MgGp4A72gYsv0H+fcXLznUBq/51dlDvTSNp4+3THB5HTPZyTev+GDWZnnbGaNgCuAMYQGX77PrYBaZvalmaWb2fAKq658+HLOzwNt8W5fuQz4rXOusGLKCwi/55dPG1wEiN82pw4iPp+PmfXDG+i9yrWi8ufLOT8DPOCc83gnb0HPl3OOAroAFwFVgG/NbL5zbm15F1dOfDnnS4ElQH+gBTDbzOY55w6Wd3EB4vf8qsyBHo6bU/t0PmZ2DvAKcJlzbm8F1VZefDnnNGBCUZgnAYPMrMA590HFlOh3vv5s73HOHQGOmNlc4FwgWAPdl3O+BXjCeReYM80sC2gDLKiYEiuc3/OrMi+5hOPm1GWes5k1AaYANwXxbK24Ms/ZOdfMOZfinEsBJgG/DuIwB99+tj8EeptZlJnFA92BVRVcpz/5cs6b8f6LBDOrB7QGNlRolRXL7/lVaWfoLgw3p/bxnP8EJAIvFs1YC1wQd6rz8ZxDii/n7JxbZWYzgAygEHjFOVfi5W/BwMfv8+PA62a2DO9yxAPOuaBtq2tm7wB9gSQzywb+DERD+eWXbv0XEQkRlXnJRUREToMCXUQkRCjQRURChAJdRCREKNBFREKEAl1EJEQo0EVEQsT/B1hhgH8tAf78AAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "fop, mpv = calibration_curve(test_df[\"win\"], lgb_calib_pred[:,1], n_bins=10)\n",
    "\n",
    "plt.plot([0, 1], [0, 1], linestyle='--')\n",
    "plt.plot(mpv, fop, marker='.')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}