3_compare_models.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "810e5887-ae1a-489c-ba61-7d0ef8ab9868",
   "metadata": {},
   "source": [
    "# Model Comparison\n",
    "\n",
    "Author: Marco Pellegrino<br>\n",
    "Year: 2024\n",
    "\n",
    "This overall project aims to build a simple model to predict the probability of loan default based on loan application data. This information helps assess business risk and improve loan approval decisions.\n",
    "\n",
    "In this notebook, metrics of previously trained models are compared."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0a66ecca-6087-45f0-9157-afd7e4c3fab5",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "import os\n",
    "\n",
    "# Import paths\n",
    "from config import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "982e9670-ef52-4e3b-bfc9-e2ab340acf22",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Check if the directory for the plots exists\n",
    "if not os.path.exists(PATH_PLOTS_COMPARISON):\n",
    "    # If it doesn't exist, create the directory\n",
    "    os.makedirs(PATH_PLOTS_COMPARISON)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0dd29057-6074-4806-897a-c6faccce42ee",
   "metadata": {},
   "source": [
    "# Load data"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d867ce48-1136-4cc4-9482-652bbf5cf11c",
   "metadata": {},
   "source": [
    "All major recorded metrics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cada1f63-9f61-4fc6-a1d8-014388d0cd88",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_XGBoost = pd.read_csv(PATH_RESULTS+'all/evaluation-XGBoost.csv', index_col=False)\n",
    "df_DT = pd.read_csv(PATH_RESULTS+'all/evaluation-DT.csv', index_col=False)\n",
    "df_RF = pd.read_csv(PATH_RESULTS+'all/evaluation-RF.csv', index_col=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "019ffd87-1c60-42a1-a491-adec9f50a28f",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Concatenate them vertically\n",
    "concatenated_df = pd.concat([df_XGBoost, df_DT, df_RF], ignore_index=True)\n",
    "concatenated_df"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "79c0f454-86ad-4628-bcb6-4b5b8997907e",
   "metadata": {},
   "source": [
    "ROC true and false positive rates"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1a82ed94-7014-4854-9bca-856fad35cc56",
   "metadata": {},
   "outputs": [],
   "source": [
    "fpr_XGBoost = pd.read_csv(PATH_RESULTS+'fpr/evaluation_fpr-XGBoost.csv', index_col=False)\n",
    "fpr_DT = pd.read_csv(PATH_RESULTS+'fpr/evaluation_fpr-DT.csv', index_col=False)\n",
    "fpr_RF = pd.read_csv(PATH_RESULTS+'fpr/evaluation_fpr-RF.csv', index_col=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1cf8393c-5828-4138-9c5b-3fafce328329",
   "metadata": {},
   "outputs": [],
   "source": [
    "tpr_XGBoost = pd.read_csv(PATH_RESULTS+'tpr/evaluation_tpr-XGBoost.csv', index_col=False)\n",
    "tpr_DT = pd.read_csv(PATH_RESULTS+'tpr/evaluation_tpr-DT.csv', index_col=False)\n",
    "tpr_RF = pd.read_csv(PATH_RESULTS+'tpr/evaluation_tpr-RF.csv', index_col=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "26831d19-b723-46e8-8663-f13fb63142d0",
   "metadata": {},
   "source": [
    "# F1 Score Comparison"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "95bf62f0-4ae3-4c18-a739-0ec8f93b41c9",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Melt the dataframe\n",
    "melted_df = pd.melt(concatenated_df[['Model', 'F1 Weighted-averaged', 'F1 Default=1', 'F1 Default=0']], id_vars=['Model'], var_name='metric', value_name='value')\n",
    "\n",
    "# Plot\n",
    "plt.figure(figsize=(10, 6))\n",
    "sns.barplot(data=melted_df, x='metric', y='value', hue='Model')\n",
    "plt.xticks(rotation=45)\n",
    "plt.xlabel('Metrics')\n",
    "plt.ylabel('Values')\n",
    "plt.title('Comparison of F1 Score for Different Models')\n",
    "plt.tight_layout()\n",
    "\n",
    "# Save the plot\n",
    "plt.savefig(PATH_PLOTS_COMPARISON+\"F1_scores.png\")\n",
    "\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3fda9550-1f26-4692-80cb-a8ccc6d49677",
   "metadata": {},
   "source": [
    "# Log Loss Comparison"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c329415b-956e-4d5e-828e-9320ff7c3e45",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Melt the dataframes\n",
    "melted_df = pd.melt(concatenated_df[['Model', 'LogLoss']], id_vars=['Model'], var_name='metric', value_name='value')\n",
    "\n",
    "# Plot\n",
    "plt.figure(figsize=(10, 6))\n",
    "sns.barplot(data=melted_df, x='metric', y='value', hue='Model')\n",
    "plt.xticks(rotation=45)\n",
    "plt.xlabel('Metrics')\n",
    "plt.ylabel('Values')\n",
    "plt.title('Comparison of Log Loss for Different Models')\n",
    "plt.tight_layout()\n",
    "\n",
    "# Save the plot\n",
    "plt.savefig(PATH_PLOTS_COMPARISON+\"logloss.png\")\n",
    "\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "50de3173-8193-465c-b27d-b1c411774c05",
   "metadata": {},
   "source": [
    "# AUC-ROC comparison"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "71e766db-77ba-482b-bc5d-2e3c11d74050",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Assuming you have ROC AUC scores, FPR, and TPR for three models stored in lists or arrays\n",
    "model_names = ['XGBoost', 'Decision Tree', 'Random Forest']\n",
    "roc_auc_scores = concatenated_df['ROC-AUC'].values\n",
    "\n",
    "fpr_list = [fpr_XGBoost['XGBoost'].values, fpr_DT['Decision Tree'].values, fpr_RF['Random Forest'].values]  # List of FPR values for each model\n",
    "tpr_list = [tpr_XGBoost['XGBoost'].values, tpr_DT['Decision Tree'].values, tpr_RF['Random Forest'].values]  # List of TPR values for each model\n",
    "\n",
    "plt.figure(figsize=(10, 8))\n",
    "\n",
    "# Plot ROC curve for each model\n",
    "for i in range(len(model_names)):\n",
    "    plt.plot(fpr_list[i], tpr_list[i], label=f'{model_names[i]} (AUC = {roc_auc_scores[i]:.2f})')\n",
    "\n",
    "# Plot ROC curve for random guessing (diagonal dotted line)\n",
    "plt.plot([0, 1], [0, 1], 'r--', label='Random Guessing')\n",
    "\n",
    "plt.xlim([0.0, 1.0])\n",
    "plt.ylim([0.0, 1.05])\n",
    "plt.xlabel('False Positive Rate (FPR)')\n",
    "plt.ylabel('True Positive Rate (TPR)')\n",
    "plt.title('Receiver Operating Characteristic (ROC) Curve')\n",
    "plt.legend(loc='lower right')\n",
    "plt.grid(True)\n",
    "\n",
    "# Save the plot\n",
    "plt.savefig(PATH_PLOTS_COMPARISON+\"roc_auc.png\")\n",
    "\n",
    "plt.show()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python (marcoenv)",
   "language": "python",
   "name": "marcoenv"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}