Flight delays - my best model.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "60a422d0",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "from sklearn.model_selection import train_test_split, RandomizedSearchCV\n",
    "from sklearn.preprocessing import LabelEncoder, OneHotEncoder, StandardScaler\n",
    "from sklearn.impute import SimpleImputer\n",
    "from sklearn.metrics import accuracy_score, classification_report, roc_auc_score\n",
    "from imblearn.over_sampling import RandomOverSampler\n",
    "from xgboost import XGBClassifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "d05f2e23",
   "metadata": {},
   "outputs": [],
   "source": [
    "#load data\n",
    "df = pd.read_csv(\"C:/Users/cj.alonzo/OneDrive - Nice Systems Ltd/Documents/Personal/Data Science/personal projects/delays/merged_data_with_airlines.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "8f968a81",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 20604118 entries, 0 to 20604117\n",
      "Data columns (total 23 columns):\n",
      " #   Column               Dtype         \n",
      "---  ------               -----         \n",
      " 0   YEAR                 int64         \n",
      " 1   MONTH                int64         \n",
      " 2   DAY                  int64         \n",
      " 3   DAY_OF_WEEK          int64         \n",
      " 4   TAIL_NUM             object        \n",
      " 5   OP_CARRIER_FL_NUM    int64         \n",
      " 6   ORIGIN               object        \n",
      " 7   ORIGIN_CITY_NAME     object        \n",
      " 8   DEST                 object        \n",
      " 9   DEST_CITY_NAME       object        \n",
      " 10  DEP_TIME             object        \n",
      " 11  DEP_DELAY            float64       \n",
      " 12  DEP_DEL15            float64       \n",
      " 13  ARR_TIME             object        \n",
      " 14  ARR_DELAY            float64       \n",
      " 15  ARR_DEL15            float64       \n",
      " 16  CARRIER_DELAY        float64       \n",
      " 17  WEATHER_DELAY        float64       \n",
      " 18  NAS_DELAY            float64       \n",
      " 19  SECURITY_DELAY       float64       \n",
      " 20  LATE_AIRCRAFT_DELAY  float64       \n",
      " 21  AIRLINE              object        \n",
      " 22  DATE                 datetime64[ns]\n",
      "dtypes: datetime64[ns](1), float64(9), int64(5), object(8)\n",
      "memory usage: 3.5+ GB\n"
     ]
    }
   ],
   "source": [
    "#made a new column, date\n",
    "df['DATE'] = pd.to_datetime(df[['YEAR', 'MONTH', 'DAY']])\n",
    "df.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "9ba7d9ca",
   "metadata": {},
   "outputs": [],
   "source": [
    "#convert to string and then to datetime\n",
    "df['DEP_TIME'] = pd.to_datetime(df['DEP_TIME'].astype(str), format='%H:%M:%S', errors='coerce')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "5bf95762",
   "metadata": {},
   "outputs": [],
   "source": [
    "df['ARR_TIME'] = pd.to_datetime(df['ARR_TIME'].astype(str), format='%H:%M:%S', errors='coerce')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "692549c2",
   "metadata": {},
   "outputs": [],
   "source": [
    "#save space so my laptop can run the model\n",
    "\n",
    "#convert int64 to int32\n",
    "int_columns = df.select_dtypes(include=['int64']).columns\n",
    "df[int_columns] = df[int_columns].astype('int32')\n",
    "\n",
    "#convert float64 to float32\n",
    "float_columns = df.select_dtypes(include=['float64']).columns\n",
    "df[float_columns] = df[float_columns].astype('float32')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "8f950e46",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 20604118 entries, 0 to 20604117\n",
      "Data columns (total 23 columns):\n",
      " #   Column               Dtype         \n",
      "---  ------               -----         \n",
      " 0   YEAR                 int32         \n",
      " 1   MONTH                int32         \n",
      " 2   DAY                  int32         \n",
      " 3   DAY_OF_WEEK          int32         \n",
      " 4   TAIL_NUM             object        \n",
      " 5   OP_CARRIER_FL_NUM    int32         \n",
      " 6   ORIGIN               object        \n",
      " 7   ORIGIN_CITY_NAME     object        \n",
      " 8   DEST                 object        \n",
      " 9   DEST_CITY_NAME       object        \n",
      " 10  DEP_TIME             datetime64[ns]\n",
      " 11  DEP_DELAY            float32       \n",
      " 12  DEP_DEL15            float32       \n",
      " 13  ARR_TIME             datetime64[ns]\n",
      " 14  ARR_DELAY            float32       \n",
      " 15  ARR_DEL15            float32       \n",
      " 16  CARRIER_DELAY        float32       \n",
      " 17  WEATHER_DELAY        float32       \n",
      " 18  NAS_DELAY            float32       \n",
      " 19  SECURITY_DELAY       float32       \n",
      " 20  LATE_AIRCRAFT_DELAY  float32       \n",
      " 21  AIRLINE              object        \n",
      " 22  DATE                 datetime64[ns]\n",
      "dtypes: datetime64[ns](3), float32(9), int32(5), object(6)\n",
      "memory usage: 2.5+ GB\n"
     ]
    }
   ],
   "source": [
    "df.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "ef22bdb9",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "ARR_DEL15              2.156841\n",
       "ARR_DELAY              2.156841\n",
       "ARR_TIME               1.940379\n",
       "DEP_DELAY              1.865768\n",
       "DEP_DEL15              1.865768\n",
       "DEP_TIME               1.864666\n",
       "TAIL_NUM               0.413311\n",
       "AIRLINE                0.000000\n",
       "LATE_AIRCRAFT_DELAY    0.000000\n",
       "SECURITY_DELAY         0.000000\n",
       "NAS_DELAY              0.000000\n",
       "WEATHER_DELAY          0.000000\n",
       "CARRIER_DELAY          0.000000\n",
       "YEAR                   0.000000\n",
       "MONTH                  0.000000\n",
       "DEST_CITY_NAME         0.000000\n",
       "DEST                   0.000000\n",
       "ORIGIN_CITY_NAME       0.000000\n",
       "ORIGIN                 0.000000\n",
       "OP_CARRIER_FL_NUM      0.000000\n",
       "DAY_OF_WEEK            0.000000\n",
       "DAY                    0.000000\n",
       "DATE                   0.000000\n",
       "dtype: float64"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#drop missing because < 3% of 20mil rows\n",
    "((df.isnull().sum() / len(df)) * 100).sort_values(ascending = False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "7d9d3673",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Original shape: (20604118, 23)\n",
      "Shape after dropping rows with missing values: (20159720, 23)\n"
     ]
    }
   ],
   "source": [
    "#drop rows with missing values in the specified columns\n",
    "columns_with_missing = ['ARR_DEL15', 'ARR_DELAY', 'ARR_TIME', 'DEP_DELAY', 'DEP_DEL15', 'DEP_TIME',\n",
    "                        'TAIL_NUM']\n",
    "df1 = df.dropna(subset=columns_with_missing)\n",
    "\n",
    "# Verify the shape of the cleaned DataFrame\n",
    "print(\"Original shape:\", df.shape)\n",
    "print(\"Shape after dropping rows with missing values:\", df1.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "019dc474",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "YEAR                   0.0\n",
       "DEP_DEL15              0.0\n",
       "AIRLINE                0.0\n",
       "LATE_AIRCRAFT_DELAY    0.0\n",
       "SECURITY_DELAY         0.0\n",
       "NAS_DELAY              0.0\n",
       "WEATHER_DELAY          0.0\n",
       "CARRIER_DELAY          0.0\n",
       "ARR_DEL15              0.0\n",
       "ARR_DELAY              0.0\n",
       "ARR_TIME               0.0\n",
       "DEP_DELAY              0.0\n",
       "MONTH                  0.0\n",
       "DEP_TIME               0.0\n",
       "DEST_CITY_NAME         0.0\n",
       "DEST                   0.0\n",
       "ORIGIN_CITY_NAME       0.0\n",
       "ORIGIN                 0.0\n",
       "OP_CARRIER_FL_NUM      0.0\n",
       "TAIL_NUM               0.0\n",
       "DAY_OF_WEEK            0.0\n",
       "DAY                    0.0\n",
       "DATE                   0.0\n",
       "dtype: float64"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "((df1.isnull().sum() / len(df1)) * 100).sort_values(ascending = False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "269de41b",
   "metadata": {},
   "outputs": [],
   "source": [
    "#define features and target variable\n",
    "features = ['ORIGIN', 'ORIGIN_CITY_NAME', 'DEST', 'DEST_CITY_NAME', 'AIRLINE',\n",
    "            'YEAR', 'DAY_OF_WEEK', 'MONTH', 'TAIL_NUM']\n",
    "X = df1[features].copy()\n",
    "y = df1['DEP_DEL15']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "e0d20b06",
   "metadata": {},
   "outputs": [],
   "source": [
    "#label encoding for high-cardinality features (many possible values)\n",
    "label_encoders = {}\n",
    "for feature in ['ORIGIN', 'ORIGIN_CITY_NAME', 'DEST', 'DEST_CITY_NAME', 'AIRLINE', 'TAIL_NUM']:\n",
    "    le = LabelEncoder()\n",
    "    X[feature] = le.fit_transform(X[feature])\n",
    "    label_encoders[feature] = le\n",
    "    \n",
    "#perform one-hot encoding for low-cardinality features\n",
    "low_cardinality_features = ['YEAR', 'DAY_OF_WEEK', 'MONTH']\n",
    "X = pd.get_dummies(X, columns=low_cardinality_features)\n",
    "\n",
    "#standardize features\n",
    "scaler = StandardScaler()\n",
    "X_scaled = scaler.fit_transform(X)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "00fd111a",
   "metadata": {},
   "outputs": [],
   "source": [
    "#split the data\n",
    "X_train, X_test, y_train, y_test = train_test_split(X_scaled, y, test_size=0.2, random_state=42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "22bca360",
   "metadata": {},
   "outputs": [],
   "source": [
    "#RandomOverSampler\n",
    "oversampler = RandomOverSampler(random_state=42)\n",
    "X_train_resampled, y_train_resampled = oversampler.fit_resample(X_train, y_train.ravel())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "579583fc",
   "metadata": {},
   "outputs": [],
   "source": [
    "#hyperparameter Tuning with RandomizedSearchCV\n",
    "best_params = {'colsample_bytree': 0.8, 'learning_rate': 0.2, 'max_depth': 7, 'subsample': 0.8}\n",
    "\n",
    "param_distributions = {\n",
    "    'learning_rate': [best_params['learning_rate']],\n",
    "    'max_depth': [best_params['max_depth']],\n",
    "    'subsample': [best_params['subsample']],\n",
    "    'colsample_bytree': [best_params['colsample_bytree']],\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "bbbe8b2a",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\cj.alonzo\\AppData\\Local\\anaconda3\\Lib\\site-packages\\sklearn\\model_selection\\_search.py:307: UserWarning: The total space of parameters 1 is smaller than n_iter=5. Running 1 iterations. For exhaustive searches, use GridSearchCV.\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<style>#sk-container-id-1 {color: black;}#sk-container-id-1 pre{padding: 0;}#sk-container-id-1 div.sk-toggleable {background-color: white;}#sk-container-id-1 label.sk-toggleable__label {cursor: pointer;display: block;width: 100%;margin-bottom: 0;padding: 0.3em;box-sizing: border-box;text-align: center;}#sk-container-id-1 label.sk-toggleable__label-arrow:before {content: \"▸\";float: left;margin-right: 0.25em;color: #696969;}#sk-container-id-1 label.sk-toggleable__label-arrow:hover:before {color: black;}#sk-container-id-1 div.sk-estimator:hover label.sk-toggleable__label-arrow:before {color: black;}#sk-container-id-1 div.sk-toggleable__content {max-height: 0;max-width: 0;overflow: hidden;text-align: left;background-color: #f0f8ff;}#sk-container-id-1 div.sk-toggleable__content pre {margin: 0.2em;color: black;border-radius: 0.25em;background-color: #f0f8ff;}#sk-container-id-1 input.sk-toggleable__control:checked~div.sk-toggleable__content {max-height: 200px;max-width: 100%;overflow: auto;}#sk-container-id-1 input.sk-toggleable__control:checked~label.sk-toggleable__label-arrow:before {content: \"▾\";}#sk-container-id-1 div.sk-estimator input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-1 div.sk-label input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-1 input.sk-hidden--visually {border: 0;clip: rect(1px 1px 1px 1px);clip: rect(1px, 1px, 1px, 1px);height: 1px;margin: -1px;overflow: hidden;padding: 0;position: absolute;width: 1px;}#sk-container-id-1 div.sk-estimator {font-family: monospace;background-color: #f0f8ff;border: 1px dotted black;border-radius: 0.25em;box-sizing: border-box;margin-bottom: 0.5em;}#sk-container-id-1 div.sk-estimator:hover {background-color: #d4ebff;}#sk-container-id-1 div.sk-parallel-item::after {content: \"\";width: 100%;border-bottom: 1px solid gray;flex-grow: 1;}#sk-container-id-1 div.sk-label:hover label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-1 div.sk-serial::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: 0;}#sk-container-id-1 div.sk-serial {display: flex;flex-direction: column;align-items: center;background-color: white;padding-right: 0.2em;padding-left: 0.2em;position: relative;}#sk-container-id-1 div.sk-item {position: relative;z-index: 1;}#sk-container-id-1 div.sk-parallel {display: flex;align-items: stretch;justify-content: center;background-color: white;position: relative;}#sk-container-id-1 div.sk-item::before, #sk-container-id-1 div.sk-parallel-item::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: -1;}#sk-container-id-1 div.sk-parallel-item {display: flex;flex-direction: column;z-index: 1;position: relative;background-color: white;}#sk-container-id-1 div.sk-parallel-item:first-child::after {align-self: flex-end;width: 50%;}#sk-container-id-1 div.sk-parallel-item:last-child::after {align-self: flex-start;width: 50%;}#sk-container-id-1 div.sk-parallel-item:only-child::after {width: 0;}#sk-container-id-1 div.sk-dashed-wrapped {border: 1px dashed gray;margin: 0 0.4em 0.5em 0.4em;box-sizing: border-box;padding-bottom: 0.4em;background-color: white;}#sk-container-id-1 div.sk-label label {font-family: monospace;font-weight: bold;display: inline-block;line-height: 1.2em;}#sk-container-id-1 div.sk-label-container {text-align: center;}#sk-container-id-1 div.sk-container {/* jupyter's `normalize.less` sets `[hidden] { display: none; }` but bootstrap.min.css set `[hidden] { display: none !important; }` so we also need the `!important` here to be able to override the default hidden behavior on the sphinx rendered scikit-learn.org. See: https://github.com/scikit-learn/scikit-learn/issues/21755 */display: inline-block !important;position: relative;}#sk-container-id-1 div.sk-text-repr-fallback {display: none;}</style><div id=\"sk-container-id-1\" class=\"sk-top-container\"><div class=\"sk-text-repr-fallback\"><pre>RandomizedSearchCV(cv=3,\n",
       "                   estimator=XGBClassifier(base_score=None, booster=None,\n",
       "                                           callbacks=None,\n",
       "                                           colsample_bylevel=None,\n",
       "                                           colsample_bynode=None,\n",
       "                                           colsample_bytree=None, device=None,\n",
       "                                           early_stopping_rounds=None,\n",
       "                                           enable_categorical=False,\n",
       "                                           eval_metric=&#x27;logloss&#x27;,\n",
       "                                           feature_types=None, gamma=None,\n",
       "                                           grow_policy=None,\n",
       "                                           importance_type=None,\n",
       "                                           interaction_constraints=None,\n",
       "                                           learning...\n",
       "                                           max_delta_step=None, max_depth=None,\n",
       "                                           max_leaves=None,\n",
       "                                           min_child_weight=None, missing=nan,\n",
       "                                           monotone_constraints=None,\n",
       "                                           multi_strategy=None,\n",
       "                                           n_estimators=None, n_jobs=None,\n",
       "                                           num_parallel_tree=None,\n",
       "                                           random_state=None, ...),\n",
       "                   n_iter=5,\n",
       "                   param_distributions={&#x27;colsample_bytree&#x27;: [0.8],\n",
       "                                        &#x27;learning_rate&#x27;: [0.2],\n",
       "                                        &#x27;max_depth&#x27;: [7], &#x27;subsample&#x27;: [0.8]},\n",
       "                   random_state=42, scoring=&#x27;accuracy&#x27;)</pre><b>In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook. <br />On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.</b></div><div class=\"sk-container\" hidden><div class=\"sk-item sk-dashed-wrapped\"><div class=\"sk-label-container\"><div class=\"sk-label sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-1\" type=\"checkbox\" ><label for=\"sk-estimator-id-1\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">RandomizedSearchCV</label><div class=\"sk-toggleable__content\"><pre>RandomizedSearchCV(cv=3,\n",
       "                   estimator=XGBClassifier(base_score=None, booster=None,\n",
       "                                           callbacks=None,\n",
       "                                           colsample_bylevel=None,\n",
       "                                           colsample_bynode=None,\n",
       "                                           colsample_bytree=None, device=None,\n",
       "                                           early_stopping_rounds=None,\n",
       "                                           enable_categorical=False,\n",
       "                                           eval_metric=&#x27;logloss&#x27;,\n",
       "                                           feature_types=None, gamma=None,\n",
       "                                           grow_policy=None,\n",
       "                                           importance_type=None,\n",
       "                                           interaction_constraints=None,\n",
       "                                           learning...\n",
       "                                           max_delta_step=None, max_depth=None,\n",
       "                                           max_leaves=None,\n",
       "                                           min_child_weight=None, missing=nan,\n",
       "                                           monotone_constraints=None,\n",
       "                                           multi_strategy=None,\n",
       "                                           n_estimators=None, n_jobs=None,\n",
       "                                           num_parallel_tree=None,\n",
       "                                           random_state=None, ...),\n",
       "                   n_iter=5,\n",
       "                   param_distributions={&#x27;colsample_bytree&#x27;: [0.8],\n",
       "                                        &#x27;learning_rate&#x27;: [0.2],\n",
       "                                        &#x27;max_depth&#x27;: [7], &#x27;subsample&#x27;: [0.8]},\n",
       "                   random_state=42, scoring=&#x27;accuracy&#x27;)</pre></div></div></div><div class=\"sk-parallel\"><div class=\"sk-parallel-item\"><div class=\"sk-item\"><div class=\"sk-label-container\"><div class=\"sk-label sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-2\" type=\"checkbox\" ><label for=\"sk-estimator-id-2\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">estimator: XGBClassifier</label><div class=\"sk-toggleable__content\"><pre>XGBClassifier(base_score=None, booster=None, callbacks=None,\n",
       "              colsample_bylevel=None, colsample_bynode=None,\n",
       "              colsample_bytree=None, device=None, early_stopping_rounds=None,\n",
       "              enable_categorical=False, eval_metric=&#x27;logloss&#x27;,\n",
       "              feature_types=None, gamma=None, grow_policy=None,\n",
       "              importance_type=None, interaction_constraints=None,\n",
       "              learning_rate=None, max_bin=None, max_cat_threshold=None,\n",
       "              max_cat_to_onehot=None, max_delta_step=None, max_depth=None,\n",
       "              max_leaves=None, min_child_weight=None, missing=nan,\n",
       "              monotone_constraints=None, multi_strategy=None, n_estimators=None,\n",
       "              n_jobs=None, num_parallel_tree=None, random_state=None, ...)</pre></div></div></div><div class=\"sk-serial\"><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-3\" type=\"checkbox\" ><label for=\"sk-estimator-id-3\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">XGBClassifier</label><div class=\"sk-toggleable__content\"><pre>XGBClassifier(base_score=None, booster=None, callbacks=None,\n",
       "              colsample_bylevel=None, colsample_bynode=None,\n",
       "              colsample_bytree=None, device=None, early_stopping_rounds=None,\n",
       "              enable_categorical=False, eval_metric=&#x27;logloss&#x27;,\n",
       "              feature_types=None, gamma=None, grow_policy=None,\n",
       "              importance_type=None, interaction_constraints=None,\n",
       "              learning_rate=None, max_bin=None, max_cat_threshold=None,\n",
       "              max_cat_to_onehot=None, max_delta_step=None, max_depth=None,\n",
       "              max_leaves=None, min_child_weight=None, missing=nan,\n",
       "              monotone_constraints=None, multi_strategy=None, n_estimators=None,\n",
       "              n_jobs=None, num_parallel_tree=None, random_state=None, ...)</pre></div></div></div></div></div></div></div></div></div></div>"
      ],
      "text/plain": [
       "RandomizedSearchCV(cv=3,\n",
       "                   estimator=XGBClassifier(base_score=None, booster=None,\n",
       "                                           callbacks=None,\n",
       "                                           colsample_bylevel=None,\n",
       "                                           colsample_bynode=None,\n",
       "                                           colsample_bytree=None, device=None,\n",
       "                                           early_stopping_rounds=None,\n",
       "                                           enable_categorical=False,\n",
       "                                           eval_metric='logloss',\n",
       "                                           feature_types=None, gamma=None,\n",
       "                                           grow_policy=None,\n",
       "                                           importance_type=None,\n",
       "                                           interaction_constraints=None,\n",
       "                                           learning...\n",
       "                                           max_delta_step=None, max_depth=None,\n",
       "                                           max_leaves=None,\n",
       "                                           min_child_weight=None, missing=nan,\n",
       "                                           monotone_constraints=None,\n",
       "                                           multi_strategy=None,\n",
       "                                           n_estimators=None, n_jobs=None,\n",
       "                                           num_parallel_tree=None,\n",
       "                                           random_state=None, ...),\n",
       "                   n_iter=5,\n",
       "                   param_distributions={'colsample_bytree': [0.8],\n",
       "                                        'learning_rate': [0.2],\n",
       "                                        'max_depth': [7], 'subsample': [0.8]},\n",
       "                   random_state=42, scoring='accuracy')"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "xgb_model = XGBClassifier(eval_metric='logloss')\n",
    "random_search = RandomizedSearchCV(estimator=xgb_model, param_distributions=param_distributions,\n",
    "                                   n_iter=5, cv=3, scoring='accuracy', random_state=42)\n",
    "random_search.fit(X_train_resampled, y_train_resampled)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "95736419",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy Score: 0.6238427418634782\n",
      "ROC-AUC Score: 0.6204257449361099\n"
     ]
    }
   ],
   "source": [
    "#evaluate the best model\n",
    "best_xgb_model = random_search.best_estimator_\n",
    "y_pred = best_xgb_model.predict(X_test)\n",
    "accuracy = accuracy_score(y_test, y_pred)\n",
    "roc_auc = roc_auc_score(y_test, y_pred)\n",
    "print(\"Accuracy Score:\", accuracy)\n",
    "print(\"ROC-AUC Score:\", roc_auc)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "c6b3bf20",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Classification Report:\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "         0.0       0.87      0.63      0.73   3240999\n",
      "         1.0       0.29      0.61      0.39    790945\n",
      "\n",
      "    accuracy                           0.62   4031944\n",
      "   macro avg       0.58      0.62      0.56   4031944\n",
      "weighted avg       0.76      0.62      0.66   4031944\n",
      "\n"
     ]
    }
   ],
   "source": [
    "#generate classification report\n",
    "print(\"Classification Report:\")\n",
    "print(classification_report(y_test, y_pred))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e2ac5782",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}