4. Model Validation.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import numpy as np \n",
    "import pandas as pd \n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "from sklearn import preprocessing\n",
    "from sklearn.ensemble import RandomForestRegressor\n",
    "from lightgbm import LGBMRegressor\n",
    "from catboost import CatBoostRegressor\n",
    "from sklearn.ensemble import StackingRegressor\n",
    "from sklearn.linear_model import LinearRegression\n",
    "from xgboost import XGBRegressor\n",
    "from sklearn.pipeline import Pipeline\n",
    "from sklearn.model_selection import KFold\n",
    "from sklearn.svm import SVR\n",
    "from sklearn.model_selection import cross_val_score\n",
    "from sklearn.ensemble import GradientBoostingRegressor\n",
    "from sklearn import model_selection\n",
    "from sklearn.metrics import mean_squared_error\n",
    "import re\n",
    "from sklearn.linear_model import ElasticNet\n",
    "import pickle\n",
    "import warnings\n",
    "warnings.filterwarnings('ignore')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_input = pd.read_csv('data/train_model_input_3.csv')\n",
    "test_input = pd.read_csv('data/test_model_input_3.csv')\n",
    "\n",
    "X_train = train_input.iloc[:,0:6]\n",
    "y_train = train_input.iloc[:,-1]\n",
    "X_test = test_input.iloc[:,0:6]\n",
    "y_test = test_input.iloc[:,-1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "def stack():\n",
    "    # define the base models\n",
    "    level0 = list()\n",
    "    level0.append(('XGB', XGBRegressor()))\n",
    "    level0.append(('SVR', SVR()))\n",
    "    level0.append(('GBR', GradientBoostingRegressor()))\n",
    "    # level0.append(('CATB', CatBoostRegressor())),\n",
    "    level0.append(('ElNET', ElasticNet())),\n",
    "    level0.append(('LGBM', LGBMRegressor()))\n",
    "    # define meta learner model\n",
    "    level1 = LinearRegression()\n",
    "    # define the stacking ensemble\n",
    "    model = StackingRegressor(estimators=level0, final_estimator=level1, cv=10)\n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "def validate_model(X_train, y_train, X_test, y_test) -> pd.DataFrame:\n",
    "    \n",
    "    dfs = []\n",
    "    models = [\n",
    "         ('Stack', stack()),\n",
    "          ('XGB', XGBRegressor(n_estimators = 100, max_depth = 10, learning_rate = 0.05)), \n",
    "          ('LGBM', LGBMRegressor(num_leaves = 300, feature_fraction = 1, bagging_freq = 100, learning = 0.05)),\n",
    "          ('GBR', GradientBoostingRegressor(n_estimators = 200, max_depth = 5,learning_rate=0.05)),\n",
    "          ('SVR', SVR(C = 1000)),\n",
    "          ('ElNet', ElasticNet(alpha=6.428073117284319e-05, l1_ratio=0.8, max_iter=5))\n",
    "#          ('CATB', CatBoostRegressor(n_estimators = 500, max_depth = 4, learning_rate = 0.1)),)\n",
    "          ]\n",
    "    results = []\n",
    "    names = []\n",
    "    scoring = ['neg_mean_squared_error', 'neg_mean_absolute_error', 'r2']\n",
    "\n",
    "    for name, model in models:\n",
    "        kfold = model_selection.RepeatedKFold(n_splits=5, n_repeats = 3, random_state=90210)\n",
    "        cv_results = model_selection.cross_validate(model, X_train, y_train, cv=kfold, scoring=scoring)\n",
    "        clf = model.fit(X_train, y_train)\n",
    "        f = 'models/{}.sav'.format(name)\n",
    "        pickle.dump(model, open(f, 'wb'))\n",
    "        y_pred = clf.predict(X_test)\n",
    "        print(name)\n",
    "        print(mean_squared_error(y_test, y_pred, squared = False))\n",
    "        \n",
    "        results.append(cv_results)\n",
    "        names.append(name)\n",
    "\n",
    "        this_df = pd.DataFrame(cv_results)\n",
    "        this_df['model'] = name\n",
    "        dfs.append(this_df)\n",
    "\n",
    "        final = pd.concat(dfs, ignore_index=True)\n",
    "        \n",
    "\n",
    "    return final_result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%timeit\n",
    "final_result = validate_model(X_train, y_train, X_test, y_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "| **Model** | **RMSE** |\n",
    "|---|---|\n",
    "| Light Gradient Boosting Regressor | 24.0045 |\n",
    "| Gradient Boosting Regressor | 24.0235 |\n",
    "| Support Vector Regressor (Radial) | 24.1036 |\n",
    "| XGBoost Regressor | 23.8852 |\n",
    "| Elastic Net | 26.2607 |\n",
    "| Stacked Ensemble |23.8775  |"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "final_result.to_csv('data/validation_result_4.csv', index = False)  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}