notebooks/experiment01_baseline_models_accuracy.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "6878099f",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Python 3.10.13\r\n"
     ]
    }
   ],
   "source": [
    "!python --version"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "20286930",
   "metadata": {},
   "outputs": [],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "aa9d9387",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os, time, datetime\n",
    "import json, pickle\n",
    "from pprint import pprint\n",
    "\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "# import tree based models\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.tree import DecisionTreeClassifier\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.ensemble import GradientBoostingClassifier\n",
    "from xgboost import XGBClassifier\n",
    "\n",
    "# training/testing utils\n",
    "from sklearn.utils import resample\n",
    "from sklearn.utils.class_weight import compute_class_weight, compute_sample_weight\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.metrics import accuracy_score, classification_report, roc_auc_score\n",
    "from sklearn.utils.class_weight import compute_sample_weight, compute_class_weight\n",
    "\n",
    "# Symbolic regression\n",
    "from gplearn.genetic import SymbolicClassifier"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e39ba408",
   "metadata": {},
   "source": [
    "## Load and prepare data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "42ef89d7",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(14560, 756) (14560, 749) (14560,) {0: 'healthy', 1: 'IBD', 2: 'CRC', 3: 'adenoma', 4: 'T2D'} {'healthy': 10761, 'IBD': 1736, 'CRC': 701, 'adenoma': 209, 'T2D': 1153}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/var/folders/qp/4w02sqhj6_d43815lhzjb7900000gn/T/ipykernel_73514/129412594.py:1: DtypeWarning: Columns (756) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  data = pd.read_csv('../data/data_diet_filtered.csv', index_col=0)\n"
     ]
    }
   ],
   "source": [
    "data = pd.read_csv('../data/data_diet_filtered.csv', index_col=0)\n",
    "\n",
    "data.disease, enc_values = pd.factorize(data.disease) # to get back original labels, do \"enc_values[data.disease]\"\n",
    "\n",
    "drop_col = ['index', 'disease', 'subject_id','gender', 'country','age_category', 'diet']\n",
    "X = data.drop(labels=drop_col, axis=1)\n",
    "y = data.disease.values\n",
    "\n",
    "class_map = {v: k for v, k in enumerate(list(enc_values.values))}  # coded label to names\n",
    "class_counts = {k: len(y[y==v]) for v, k in enumerate(list(enc_values.values))}\n",
    "assert sum([class_counts[k] for k in class_counts]) == len(X), 'total #samples not matching when summing for each class'\n",
    "\n",
    "print(data.shape, X.shape, y.shape, class_map, class_counts)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "0723e316",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "X.shape: (14156, 749), y.shape: (14156,)\n"
     ]
    }
   ],
   "source": [
    "# only take normalized rows!\n",
    "normalized_idx = (X.sum(1) > 99)\n",
    "X = X[normalized_idx]\n",
    "y = y[X.index]\n",
    "\n",
    "print(f\"X.shape: {X.shape}, y.shape: {y.shape}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "657d015a",
   "metadata": {},
   "outputs": [],
   "source": [
    "from srmb.utils import calculate_metrics"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "eee9fdd2",
   "metadata": {},
   "source": [
    "## Healthy vs. CRC classification\n",
    "For the subsequent analysis, we will only choose the healthy and CRC patients from the \n",
    "dataset."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "36a0acfa",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Error exception: Only one class present in y_true. ROC AUC score is not defined in that case.\n"
     ]
    }
   ],
   "source": [
    "from srmb.fitness_functions import customacc\n",
    "from srmb.special_functions import (\n",
    "    presence, absence, add3, add10, ifelse, ifelseless,\n",
    "    presence2, absence2,\n",
    "    presence3, absence3\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "fc8ef45b",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "doing for class = CRC, X1.shape=(11137, 749), y1.shape=(11137,), #CRC samples = 664\n"
     ]
    }
   ],
   "source": [
    "k = 2\n",
    "idxs = (y == 0) | (y == k)  # get healthy and that class' data\n",
    "\n",
    "X1, y1 = X.iloc[idxs], y[idxs]\n",
    "y1[y1 == k] = 1  # relabel 1 --> CRC, 0 --> healthy\n",
    "print(f'doing for class = {class_map[k]}, {X1.shape=}, {y1.shape=}, #{class_map[k]} samples = {y1.sum()}')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c7736bec",
   "metadata": {},
   "source": [
    "# 1. Experiments with undersampling the healthy class"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "9b865d20",
   "metadata": {},
   "outputs": [],
   "source": [
    "from imblearn.under_sampling import RandomUnderSampler\n",
    "from collections import Counter"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "aab32c13",
   "metadata": {},
   "outputs": [],
   "source": [
    "def store_result(key, acc, f1, auroc):\n",
    "    ACCURACIES[key].append(acc)\n",
    "    F1SCORES[key].append(f1)\n",
    "    AUCROCS[key].append(auroc)\n",
    "    \n",
    "def convert_to_arrays(prec=4):\n",
    "    for key in MODEL_NAMES:\n",
    "        ACCURACIES[key] = np.around(np.asarray(ACCURACIES[key]), prec)\n",
    "        F1SCORES[key] = np.around(np.asarray(F1SCORES[key]), prec)\n",
    "        AUCROCS[key] = np.around(np.asarray(AUCROCS[key]), prec)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "e2421c0a",
   "metadata": {},
   "outputs": [],
   "source": [
    "# seeds for 20 trials\n",
    "RANDOM_SEEDS_FOR_UNDERSAMPLING = [42, 2024, 1234, 2405, 11, 9345,  858, 8590, 4754, 1959,\n",
    "                                  707, 10524, 83946, 63297, 78035, 22664, 49283, 35253, 82273, 90378]\n",
    "MODEL_NAMES = ['LR', 'DT', 'RF', 'XG', 'SR', 'SRf']\n",
    "ACCURACIES = {k: [] for k in MODEL_NAMES}\n",
    "F1SCORES = {k: [] for k in MODEL_NAMES}\n",
    "AUCROCS = {k: [] for k in MODEL_NAMES}\n",
    "\n",
    "SRmodels = []\n",
    "SRfmodels = []"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "2af5eb61",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "seed=42\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/swagatam/miniconda3/envs/hiwi/lib/python3.10/site-packages/sklearn/linear_model/_logistic.py:460: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "LogisticRegression\n",
      "Test accuracy: 0.7265 Test AUROC: 0.7981 Test F1 score: 0.7009\n",
      "DecisionTreeClassifier\n",
      "Test accuracy: 0.7514 Test AUROC: 0.8186 Test F1 score: 0.7078\n",
      "RandomForestClassifier\n",
      "Test accuracy: 0.8011 Test AUROC: 0.9015 Test F1 score: 0.7647\n",
      "XGBClassifier\n",
      "Test accuracy: 0.8260 Test AUROC: 0.9131 Test F1 score: 0.8013\n",
      "Time to fit symbolic classifier: 42.2771680355072 seconds\n",
      "SymbolicClassifier\n",
      "Test accuracy: 0.7431 Test AUROC: 0.7488 Test F1 score: 0.6847\n",
      "Time to fit symbolic classifier: 41.15023899078369 seconds\n",
      "SymbolicClassifier\n",
      "Test accuracy: 0.7514 Test AUROC: 0.7579 Test F1 score: 0.6939\n",
      "seed=2024\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/swagatam/miniconda3/envs/hiwi/lib/python3.10/site-packages/sklearn/linear_model/_logistic.py:460: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "LogisticRegression\n",
      "Test accuracy: 0.7210 Test AUROC: 0.7726 Test F1 score: 0.6967\n",
      "DecisionTreeClassifier\n",
      "Test accuracy: 0.7182 Test AUROC: 0.7402 Test F1 score: 0.6982\n",
      "RandomForestClassifier\n",
      "Test accuracy: 0.8122 Test AUROC: 0.8961 Test F1 score: 0.7792\n",
      "XGBClassifier\n",
      "Test accuracy: 0.8177 Test AUROC: 0.9159 Test F1 score: 0.7911\n",
      "Time to fit symbolic classifier: 41.51435089111328 seconds\n",
      "SymbolicClassifier\n",
      "Test accuracy: 0.7569 Test AUROC: 0.7640 Test F1 score: 0.6966\n",
      "Time to fit symbolic classifier: 41.539920806884766 seconds\n",
      "SymbolicClassifier\n",
      "Test accuracy: 0.7541 Test AUROC: 0.7641 Test F1 score: 0.6962\n",
      "seed=1234\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/swagatam/miniconda3/envs/hiwi/lib/python3.10/site-packages/sklearn/linear_model/_logistic.py:460: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "LogisticRegression\n",
      "Test accuracy: 0.7155 Test AUROC: 0.7885 Test F1 score: 0.6997\n",
      "DecisionTreeClassifier\n",
      "Test accuracy: 0.7845 Test AUROC: 0.8185 Test F1 score: 0.7365\n",
      "RandomForestClassifier\n",
      "Test accuracy: 0.8287 Test AUROC: 0.9015 Test F1 score: 0.7947\n",
      "XGBClassifier\n",
      "Test accuracy: 0.8398 Test AUROC: 0.9181 Test F1 score: 0.8165\n",
      "Time to fit symbolic classifier: 43.203129053115845 seconds\n",
      "SymbolicClassifier\n",
      "Test accuracy: 0.7403 Test AUROC: 0.7617 Test F1 score: 0.7006\n",
      "Time to fit symbolic classifier: 44.14738321304321 seconds\n",
      "SymbolicClassifier\n",
      "Test accuracy: 0.7652 Test AUROC: 0.7692 Test F1 score: 0.7079\n",
      "seed=2405\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/swagatam/miniconda3/envs/hiwi/lib/python3.10/site-packages/sklearn/linear_model/_logistic.py:460: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "LogisticRegression\n",
      "Test accuracy: 0.7818 Test AUROC: 0.8474 Test F1 score: 0.7508\n",
      "DecisionTreeClassifier\n",
      "Test accuracy: 0.7348 Test AUROC: 0.7518 Test F1 score: 0.6667\n",
      "RandomForestClassifier\n",
      "Test accuracy: 0.8122 Test AUROC: 0.8982 Test F1 score: 0.7655\n",
      "XGBClassifier\n",
      "Test accuracy: 0.8177 Test AUROC: 0.9079 Test F1 score: 0.7925\n",
      "Time to fit symbolic classifier: 43.750773906707764 seconds\n",
      "SymbolicClassifier\n",
      "Test accuracy: 0.7459 Test AUROC: 0.7500 Test F1 score: 0.6913\n",
      "Time to fit symbolic classifier: 43.03351879119873 seconds\n",
      "SymbolicClassifier\n",
      "Test accuracy: 0.7486 Test AUROC: 0.7465 Test F1 score: 0.6851\n",
      "seed=11\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/swagatam/miniconda3/envs/hiwi/lib/python3.10/site-packages/sklearn/linear_model/_logistic.py:460: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "LogisticRegression\n",
      "Test accuracy: 0.7238 Test AUROC: 0.7970 Test F1 score: 0.7059\n",
      "DecisionTreeClassifier\n",
      "Test accuracy: 0.7597 Test AUROC: 0.7982 Test F1 score: 0.7307\n",
      "RandomForestClassifier\n",
      "Test accuracy: 0.7707 Test AUROC: 0.8785 Test F1 score: 0.7314\n",
      "XGBClassifier\n",
      "Test accuracy: 0.8232 Test AUROC: 0.9068 Test F1 score: 0.8012\n",
      "Time to fit symbolic classifier: 43.38354301452637 seconds\n",
      "SymbolicClassifier\n",
      "Test accuracy: 0.7597 Test AUROC: 0.7643 Test F1 score: 0.7010\n",
      "Time to fit symbolic classifier: 43.27070212364197 seconds\n",
      "SymbolicClassifier\n",
      "Test accuracy: 0.7155 Test AUROC: 0.7301 Test F1 score: 0.6532\n",
      "seed=9345\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/swagatam/miniconda3/envs/hiwi/lib/python3.10/site-packages/sklearn/linear_model/_logistic.py:460: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "LogisticRegression\n",
      "Test accuracy: 0.6989 Test AUROC: 0.7772 Test F1 score: 0.6625\n",
      "DecisionTreeClassifier\n",
      "Test accuracy: 0.7155 Test AUROC: 0.7750 Test F1 score: 0.6485\n",
      "RandomForestClassifier\n",
      "Test accuracy: 0.8039 Test AUROC: 0.8891 Test F1 score: 0.7641\n",
      "XGBClassifier\n",
      "Test accuracy: 0.8370 Test AUROC: 0.9182 Test F1 score: 0.8103\n",
      "Time to fit symbolic classifier: 43.272748947143555 seconds\n",
      "SymbolicClassifier\n",
      "Test accuracy: 0.7459 Test AUROC: 0.7491 Test F1 score: 0.6892\n",
      "Time to fit symbolic classifier: 42.66357135772705 seconds\n",
      "SymbolicClassifier\n",
      "Test accuracy: 0.7486 Test AUROC: 0.7533 Test F1 score: 0.6915\n",
      "seed=858\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/swagatam/miniconda3/envs/hiwi/lib/python3.10/site-packages/sklearn/linear_model/_logistic.py:460: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "LogisticRegression\n",
      "Test accuracy: 0.7624 Test AUROC: 0.8271 Test F1 score: 0.7346\n",
      "DecisionTreeClassifier\n",
      "Test accuracy: 0.7735 Test AUROC: 0.7956 Test F1 score: 0.7500\n",
      "RandomForestClassifier\n",
      "Test accuracy: 0.8315 Test AUROC: 0.9196 Test F1 score: 0.7973\n",
      "XGBClassifier\n",
      "Test accuracy: 0.8204 Test AUROC: 0.9280 Test F1 score: 0.7855\n",
      "Time to fit symbolic classifier: 43.11080884933472 seconds\n",
      "SymbolicClassifier\n",
      "Test accuracy: 0.7569 Test AUROC: 0.7890 Test F1 score: 0.6986\n",
      "Time to fit symbolic classifier: 43.20073890686035 seconds\n",
      "SymbolicClassifier\n",
      "Test accuracy: 0.7348 Test AUROC: 0.7411 Test F1 score: 0.6643\n",
      "seed=8590\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/swagatam/miniconda3/envs/hiwi/lib/python3.10/site-packages/sklearn/linear_model/_logistic.py:460: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "LogisticRegression\n",
      "Test accuracy: 0.7403 Test AUROC: 0.8040 Test F1 score: 0.7134\n",
      "DecisionTreeClassifier\n",
      "Test accuracy: 0.7514 Test AUROC: 0.7889 Test F1 score: 0.6897\n",
      "RandomForestClassifier\n",
      "Test accuracy: 0.8122 Test AUROC: 0.9011 Test F1 score: 0.7792\n",
      "XGBClassifier\n",
      "Test accuracy: 0.8232 Test AUROC: 0.9098 Test F1 score: 0.7987\n",
      "Time to fit symbolic classifier: 42.94043493270874 seconds\n",
      "SymbolicClassifier\n",
      "Test accuracy: 0.7541 Test AUROC: 0.7544 Test F1 score: 0.6899\n",
      "Time to fit symbolic classifier: 43.01768684387207 seconds\n",
      "SymbolicClassifier\n",
      "Test accuracy: 0.7486 Test AUROC: 0.7573 Test F1 score: 0.6915\n",
      "seed=4754\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/swagatam/miniconda3/envs/hiwi/lib/python3.10/site-packages/sklearn/linear_model/_logistic.py:460: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "LogisticRegression\n",
      "Test accuracy: 0.7762 Test AUROC: 0.8377 Test F1 score: 0.7461\n",
      "DecisionTreeClassifier\n",
      "Test accuracy: 0.7348 Test AUROC: 0.7938 Test F1 score: 0.6620\n",
      "RandomForestClassifier\n",
      "Test accuracy: 0.8260 Test AUROC: 0.9111 Test F1 score: 0.7921\n",
      "XGBClassifier\n",
      "Test accuracy: 0.8315 Test AUROC: 0.9308 Test F1 score: 0.8039\n",
      "Time to fit symbolic classifier: 43.672722816467285 seconds\n",
      "SymbolicClassifier\n",
      "Test accuracy: 0.7762 Test AUROC: 0.7759 Test F1 score: 0.7138\n",
      "Time to fit symbolic classifier: 43.33226490020752 seconds\n",
      "SymbolicClassifier\n",
      "Test accuracy: 0.7597 Test AUROC: 0.7694 Test F1 score: 0.7031\n",
      "seed=1959\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/swagatam/miniconda3/envs/hiwi/lib/python3.10/site-packages/sklearn/linear_model/_logistic.py:460: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "LogisticRegression\n",
      "Test accuracy: 0.7624 Test AUROC: 0.8320 Test F1 score: 0.7312\n",
      "DecisionTreeClassifier\n",
      "Test accuracy: 0.6878 Test AUROC: 0.7568 Test F1 score: 0.6744\n",
      "RandomForestClassifier\n",
      "Test accuracy: 0.8287 Test AUROC: 0.8998 Test F1 score: 0.8050\n",
      "XGBClassifier\n",
      "Test accuracy: 0.7928 Test AUROC: 0.8996 Test F1 score: 0.7734\n",
      "Time to fit symbolic classifier: 42.98330283164978 seconds\n",
      "SymbolicClassifier\n",
      "Test accuracy: 0.7486 Test AUROC: 0.7580 Test F1 score: 0.6915\n",
      "Time to fit symbolic classifier: 44.140511989593506 seconds\n",
      "SymbolicClassifier\n",
      "Test accuracy: 0.7652 Test AUROC: 0.7697 Test F1 score: 0.7157\n",
      "seed=707\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/swagatam/miniconda3/envs/hiwi/lib/python3.10/site-packages/sklearn/linear_model/_logistic.py:460: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "LogisticRegression\n",
      "Test accuracy: 0.7624 Test AUROC: 0.8203 Test F1 score: 0.7394\n",
      "DecisionTreeClassifier\n",
      "Test accuracy: 0.7569 Test AUROC: 0.8137 Test F1 score: 0.6667\n",
      "RandomForestClassifier\n",
      "Test accuracy: 0.8260 Test AUROC: 0.9097 Test F1 score: 0.7921\n",
      "XGBClassifier\n",
      "Test accuracy: 0.8232 Test AUROC: 0.9166 Test F1 score: 0.7949\n",
      "Time to fit symbolic classifier: 43.830246925354004 seconds\n",
      "SymbolicClassifier\n",
      "Test accuracy: 0.7486 Test AUROC: 0.7530 Test F1 score: 0.7217\n",
      "Time to fit symbolic classifier: 43.57986807823181 seconds\n",
      "SymbolicClassifier\n",
      "Test accuracy: 0.7790 Test AUROC: 0.8150 Test F1 score: 0.7143\n",
      "seed=10524\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/swagatam/miniconda3/envs/hiwi/lib/python3.10/site-packages/sklearn/linear_model/_logistic.py:460: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "LogisticRegression\n",
      "Test accuracy: 0.7514 Test AUROC: 0.8078 Test F1 score: 0.7289\n",
      "DecisionTreeClassifier\n",
      "Test accuracy: 0.7265 Test AUROC: 0.7624 Test F1 score: 0.6991\n",
      "RandomForestClassifier\n",
      "Test accuracy: 0.8204 Test AUROC: 0.8905 Test F1 score: 0.7883\n",
      "XGBClassifier\n",
      "Test accuracy: 0.8260 Test AUROC: 0.8991 Test F1 score: 0.8037\n",
      "Time to fit symbolic classifier: 43.50123906135559 seconds\n",
      "SymbolicClassifier\n",
      "Test accuracy: 0.7514 Test AUROC: 0.7622 Test F1 score: 0.7039\n",
      "Time to fit symbolic classifier: 43.24294877052307 seconds\n",
      "SymbolicClassifier\n",
      "Test accuracy: 0.7486 Test AUROC: 0.7613 Test F1 score: 0.6997\n",
      "seed=83946\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/swagatam/miniconda3/envs/hiwi/lib/python3.10/site-packages/sklearn/linear_model/_logistic.py:460: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "LogisticRegression\n",
      "Test accuracy: 0.7403 Test AUROC: 0.7988 Test F1 score: 0.7081\n",
      "DecisionTreeClassifier\n",
      "Test accuracy: 0.7238 Test AUROC: 0.7863 Test F1 score: 0.6552\n",
      "RandomForestClassifier\n",
      "Test accuracy: 0.8204 Test AUROC: 0.8863 Test F1 score: 0.7855\n",
      "XGBClassifier\n",
      "Test accuracy: 0.8287 Test AUROC: 0.9158 Test F1 score: 0.8062\n",
      "Time to fit symbolic classifier: 43.89677596092224 seconds\n",
      "SymbolicClassifier\n",
      "Test accuracy: 0.7459 Test AUROC: 0.7638 Test F1 score: 0.6954\n",
      "Time to fit symbolic classifier: 42.90026021003723 seconds\n",
      "SymbolicClassifier\n",
      "Test accuracy: 0.7348 Test AUROC: 0.7510 Test F1 score: 0.6800\n",
      "seed=63297\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/swagatam/miniconda3/envs/hiwi/lib/python3.10/site-packages/sklearn/linear_model/_logistic.py:460: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "LogisticRegression\n",
      "Test accuracy: 0.7762 Test AUROC: 0.8216 Test F1 score: 0.7508\n",
      "DecisionTreeClassifier\n",
      "Test accuracy: 0.7845 Test AUROC: 0.8127 Test F1 score: 0.7665\n",
      "RandomForestClassifier\n",
      "Test accuracy: 0.8260 Test AUROC: 0.9118 Test F1 score: 0.7948\n",
      "XGBClassifier\n",
      "Test accuracy: 0.8564 Test AUROC: 0.9337 Test F1 score: 0.8354\n",
      "Time to fit symbolic classifier: 43.374119997024536 seconds\n",
      "SymbolicClassifier\n",
      "Test accuracy: 0.7652 Test AUROC: 0.7754 Test F1 score: 0.7195\n",
      "Time to fit symbolic classifier: 43.408135175704956 seconds\n",
      "SymbolicClassifier\n",
      "Test accuracy: 0.7652 Test AUROC: 0.7666 Test F1 score: 0.7099\n",
      "seed=78035\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/swagatam/miniconda3/envs/hiwi/lib/python3.10/site-packages/sklearn/linear_model/_logistic.py:460: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "LogisticRegression\n",
      "Test accuracy: 0.7735 Test AUROC: 0.8348 Test F1 score: 0.7405\n",
      "DecisionTreeClassifier\n",
      "Test accuracy: 0.7403 Test AUROC: 0.7901 Test F1 score: 0.6846\n",
      "RandomForestClassifier\n",
      "Test accuracy: 0.8370 Test AUROC: 0.9148 Test F1 score: 0.8115\n",
      "XGBClassifier\n",
      "Test accuracy: 0.8536 Test AUROC: 0.9395 Test F1 score: 0.8296\n",
      "Time to fit symbolic classifier: 43.04957699775696 seconds\n",
      "SymbolicClassifier\n",
      "Test accuracy: 0.7541 Test AUROC: 0.7593 Test F1 score: 0.7023\n",
      "Time to fit symbolic classifier: 42.84919810295105 seconds\n",
      "SymbolicClassifier\n",
      "Test accuracy: 0.7762 Test AUROC: 0.8103 Test F1 score: 0.7178\n",
      "seed=22664\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/swagatam/miniconda3/envs/hiwi/lib/python3.10/site-packages/sklearn/linear_model/_logistic.py:460: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "LogisticRegression\n",
      "Test accuracy: 0.7790 Test AUROC: 0.8231 Test F1 score: 0.7531\n",
      "DecisionTreeClassifier\n",
      "Test accuracy: 0.7155 Test AUROC: 0.8195 Test F1 score: 0.7099\n",
      "RandomForestClassifier\n",
      "Test accuracy: 0.8287 Test AUROC: 0.9080 Test F1 score: 0.7987\n",
      "XGBClassifier\n",
      "Test accuracy: 0.8453 Test AUROC: 0.9225 Test F1 score: 0.8228\n",
      "Time to fit symbolic classifier: 43.30705690383911 seconds\n",
      "SymbolicClassifier\n",
      "Test accuracy: 0.7514 Test AUROC: 0.7704 Test F1 score: 0.7020\n",
      "Time to fit symbolic classifier: 44.09287762641907 seconds\n",
      "SymbolicClassifier\n",
      "Test accuracy: 0.7431 Test AUROC: 0.7477 Test F1 score: 0.6714\n",
      "seed=49283\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/swagatam/miniconda3/envs/hiwi/lib/python3.10/site-packages/sklearn/linear_model/_logistic.py:460: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "LogisticRegression\n",
      "Test accuracy: 0.7541 Test AUROC: 0.8084 Test F1 score: 0.7405\n",
      "DecisionTreeClassifier\n",
      "Test accuracy: 0.7293 Test AUROC: 0.7896 Test F1 score: 0.7168\n",
      "RandomForestClassifier\n",
      "Test accuracy: 0.8287 Test AUROC: 0.9166 Test F1 score: 0.7947\n",
      "XGBClassifier\n",
      "Test accuracy: 0.8398 Test AUROC: 0.9318 Test F1 score: 0.8129\n",
      "Time to fit symbolic classifier: 42.77116394042969 seconds\n",
      "SymbolicClassifier\n",
      "Test accuracy: 0.7569 Test AUROC: 0.7585 Test F1 score: 0.7007\n",
      "Time to fit symbolic classifier: 45.94197463989258 seconds\n",
      "SymbolicClassifier\n",
      "Test accuracy: 0.7597 Test AUROC: 0.7513 Test F1 score: 0.7010\n",
      "seed=35253\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/swagatam/miniconda3/envs/hiwi/lib/python3.10/site-packages/sklearn/linear_model/_logistic.py:460: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "LogisticRegression\n",
      "Test accuracy: 0.7569 Test AUROC: 0.8184 Test F1 score: 0.7317\n",
      "DecisionTreeClassifier\n",
      "Test accuracy: 0.7514 Test AUROC: 0.8107 Test F1 score: 0.7059\n",
      "RandomForestClassifier\n",
      "Test accuracy: 0.8039 Test AUROC: 0.8988 Test F1 score: 0.7717\n",
      "XGBClassifier\n",
      "Test accuracy: 0.8094 Test AUROC: 0.9091 Test F1 score: 0.7903\n",
      "Time to fit symbolic classifier: 44.62361788749695 seconds\n",
      "SymbolicClassifier\n",
      "Test accuracy: 0.7210 Test AUROC: 0.7384 Test F1 score: 0.6731\n",
      "Time to fit symbolic classifier: 44.03053903579712 seconds\n",
      "SymbolicClassifier\n",
      "Test accuracy: 0.7403 Test AUROC: 0.7603 Test F1 score: 0.6928\n",
      "seed=82273\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/swagatam/miniconda3/envs/hiwi/lib/python3.10/site-packages/sklearn/linear_model/_logistic.py:460: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "LogisticRegression\n",
      "Test accuracy: 0.7293 Test AUROC: 0.7904 Test F1 score: 0.7048\n",
      "DecisionTreeClassifier\n",
      "Test accuracy: 0.7403 Test AUROC: 0.7878 Test F1 score: 0.6908\n",
      "RandomForestClassifier\n",
      "Test accuracy: 0.8039 Test AUROC: 0.8927 Test F1 score: 0.7609\n",
      "XGBClassifier\n",
      "Test accuracy: 0.8425 Test AUROC: 0.9112 Test F1 score: 0.8190\n",
      "Time to fit symbolic classifier: 45.13826107978821 seconds\n",
      "SymbolicClassifier\n",
      "Test accuracy: 0.7486 Test AUROC: 0.7551 Test F1 score: 0.6936\n",
      "Time to fit symbolic classifier: 44.299291133880615 seconds\n",
      "SymbolicClassifier\n",
      "Test accuracy: 0.7486 Test AUROC: 0.7536 Test F1 score: 0.6915\n",
      "seed=90378\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/swagatam/miniconda3/envs/hiwi/lib/python3.10/site-packages/sklearn/linear_model/_logistic.py:460: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "LogisticRegression\n",
      "Test accuracy: 0.7541 Test AUROC: 0.8200 Test F1 score: 0.7343\n",
      "DecisionTreeClassifier\n",
      "Test accuracy: 0.7541 Test AUROC: 0.8313 Test F1 score: 0.6877\n",
      "RandomForestClassifier\n",
      "Test accuracy: 0.8011 Test AUROC: 0.9094 Test F1 score: 0.7647\n",
      "XGBClassifier\n",
      "Test accuracy: 0.8287 Test AUROC: 0.9243 Test F1 score: 0.8025\n",
      "Time to fit symbolic classifier: 44.41036295890808 seconds\n",
      "SymbolicClassifier\n",
      "Test accuracy: 0.7597 Test AUROC: 0.7663 Test F1 score: 0.7031\n",
      "Time to fit symbolic classifier: 46.336766958236694 seconds\n",
      "SymbolicClassifier\n",
      "Test accuracy: 0.7597 Test AUROC: 0.7671 Test F1 score: 0.7031\n"
     ]
    }
   ],
   "source": [
    "USE_BALANCED_SUBSAMPLE=True # perform undersampling of healthy classes\n",
    "\n",
    "for random_state in RANDOM_SEEDS_FOR_UNDERSAMPLING:\n",
    "    rus = RandomUnderSampler(sampling_strategy=0.85, # this is another hyperparameter\n",
    "                             random_state=random_state)\n",
    "    X1b, y1b = rus.fit_resample(X1, y1)\n",
    "    print(f'seed={random_state}')\n",
    "    # print(sorted(Counter(y1b).items()))\n",
    "    \n",
    "    if not USE_BALANCED_SUBSAMPLE:\n",
    "        X_train, X_test, y_train, y_test = train_test_split(X1, y1,\n",
    "                                                        test_size=0.25,\n",
    "                                                        # train_size=0.5, # if slow use this\n",
    "                                                        random_state=random_state, stratify=y1)\n",
    "        class_weight = compute_class_weight(class_weight='balanced', classes=np.unique(y1), y=y1)\n",
    "    else:\n",
    "        # print('using a balanced subsample of the data ...')\n",
    "        X_train, X_test, y_train, y_test = train_test_split(X1b, y1b,\n",
    "                                                            test_size=0.25,\n",
    "                                                            # train_size=0.5, # if slow use this\n",
    "                                                            random_state=42, stratify=y1b)\n",
    "        class_weight = compute_class_weight(class_weight='balanced', classes=np.unique(y1b), y=y1b)\n",
    "        \n",
    "    \n",
    "    sample_weights = compute_sample_weight(class_weight='balanced', y=y_train)\n",
    "    \n",
    "    # logistic regression classifier\n",
    "    model_lr = LogisticRegression(max_iter=500, class_weight=dict(enumerate(class_weight)))\n",
    "    model_lr.fit(X_train, y_train)\n",
    "    store_result('LR', *calculate_metrics(model_lr, X_train, y_train, X_test, y_test))\n",
    "    \n",
    "    # Decision tree classifier\n",
    "    model_dt = DecisionTreeClassifier(max_depth=5, class_weight=dict(enumerate(class_weight)))\n",
    "    model_dt.fit(X_train, y_train)\n",
    "    store_result('DT', *calculate_metrics(model_dt, X_train, y_train, X_test, y_test))\n",
    "    \n",
    "    \n",
    "    # Random forest classifier\n",
    "    model_rf = RandomForestClassifier(n_estimators=50, class_weight=dict(enumerate(class_weight)))\n",
    "    model_rf.fit(X_train, y_train)\n",
    "    store_result('RF', *calculate_metrics(model_rf, X_train, y_train, X_test, y_test))\n",
    "    \n",
    "    \n",
    "    # Create an XGBoost classifier for multiclass classification\n",
    "    model_xg = XGBClassifier(n_estimators=50, max_depth=5, learning_rate=0.1, objective='binary:logistic')\n",
    "    model_xg.fit(X_train, y_train, sample_weight=sample_weights)\n",
    "    store_result('XG', *calculate_metrics(model_xg, X_train, y_train, X_test, y_test))\n",
    "    \n",
    "    \n",
    "    # Do it for vanilla symbolic regression\n",
    "    function_set = ['add', 'sub', 'mul', 'div', 'neg', 'max', 'min', 'sqrt', 'log']\n",
    "\n",
    "    est = SymbolicClassifier(population_size=6000,\n",
    "                             generations=20,\n",
    "                             tournament_size=25,\n",
    "\n",
    "                             init_depth=(2, 6),\n",
    "                             const_range=(0., 100.),\n",
    "                             # init_method=\"full\",\n",
    "                             parsimony_coefficient=0.001,\n",
    "                             function_set=function_set,\n",
    "\n",
    "                             stopping_criteria=1.0, metric=customacc, #use custom acc as fitness\n",
    "                             \n",
    "                             feature_names=X1.columns.to_list(),\n",
    "                             # verbose=True,\n",
    "                             random_state=42)\n",
    "\n",
    "    t0 = time.time()\n",
    "    est.fit(X_train, y_train)\n",
    "    print('Time to fit symbolic classifier:', time.time() - t0, 'seconds')\n",
    "    store_result('SR', *calculate_metrics(est, X_train, y_train, X_test, y_test))\n",
    "    SRmodels.append(est)\n",
    "    \n",
    "    \n",
    "    # SR with special functions\n",
    "    special_functions = [presence, absence, presence2, absence2, ifelse]#, add3, add10]\n",
    "    function_set = ['add', 'sub', 'mul', 'div', 'neg', 'max', 'min'] + special_functions\n",
    "\n",
    "    est = SymbolicClassifier(population_size=6000,\n",
    "                             generations=20,\n",
    "                             tournament_size=25,\n",
    "\n",
    "                             init_depth=(2, 6),\n",
    "                             const_range=(0., 100.),\n",
    "                             # init_method=\"full\",\n",
    "                             parsimony_coefficient=0.001,\n",
    "                             function_set=function_set,\n",
    "\n",
    "                             stopping_criteria=1.0, metric=customacc, #use custom acc as fitness\n",
    "                             \n",
    "                             feature_names=X1.columns.to_list(),\n",
    "                             # verbose=True,\n",
    "                             random_state=42)\n",
    "\n",
    "    t0 = time.time()\n",
    "    est.fit(X_train, y_train)\n",
    "    print('Time to fit symbolic classifier:', time.time() - t0, 'seconds')\n",
    "    store_result('SRf', *calculate_metrics(est, X_train, y_train, X_test, y_test))\n",
    "    SRfmodels.append(est)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f90118ba",
   "metadata": {},
   "source": [
    "## Process results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "6e3b09ed",
   "metadata": {},
   "outputs": [],
   "source": [
    "LENGTHS = {'SR': [], 'SRf': []}\n",
    "for est in SRmodels:\n",
    "    LENGTHS['SR'].append(est._program.length_)\n",
    "for est in SRfmodels:\n",
    "    LENGTHS['SRf'].append(est._program.length_)\n",
    "    \n",
    "LENGTHS['SR'] = np.asarray(LENGTHS['SR'])\n",
    "LENGTHS['SRf'] = np.asarray(LENGTHS['SRf'])\n",
    "\n",
    "convert_to_arrays()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "b240af30",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "LR acc: 0.75 ± 0.0232\n",
      "DT acc: 0.74 ± 0.0238\n",
      "RF acc: 0.82 ± 0.0152\n",
      "XG acc: 0.83 ± 0.0146\n",
      "SR acc: 0.75 ± 0.0107\n",
      "SRf acc: 0.75 ± 0.0145\n",
      "--------\n",
      "LR F1: 0.72 ± 0.0230\n",
      "DT F1: 0.70 ± 0.0309\n",
      "RF F1: 0.78 ± 0.0187\n",
      "XG F1: 0.80 ± 0.0148\n",
      "SR F1: 0.70 ± 0.0110\n",
      "SRf F1: 0.69 ± 0.0166\n",
      "--------\n"
     ]
    }
   ],
   "source": [
    "plusminus = pm = u\"\\u00B1\"\n",
    "for model_name in MODEL_NAMES:\n",
    "    print(f\"{model_name} acc: {ACCURACIES[model_name].mean():.2f} {pm} {ACCURACIES[model_name].std():.4f}\")\n",
    "print('--------')\n",
    "for model_name in MODEL_NAMES:\n",
    "    print(f\"{model_name} F1: {F1SCORES[model_name].mean():.2f} {pm} {F1SCORES[model_name].std():.4f}\")\n",
    "\n",
    "print('--------')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2da8a5aa",
   "metadata": {},
   "source": [
    "## Compare average length of SR models with and without special functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "6423e4b2",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SR mean length: 19.55\n",
      "SRf mean length: 14.55\n"
     ]
    }
   ],
   "source": [
    "print(f\"SR mean length: {LENGTHS['SR'].mean()}\")\n",
    "print(f\"SRf mean length: {LENGTHS['SRf'].mean()}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0c68c351",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "6206b7ab",
   "metadata": {},
   "source": [
    "## Save symbolic regression models for later use"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "c7f1affa",
   "metadata": {},
   "outputs": [],
   "source": [
    "from srmb.utils import save_sr_models, load_sr_models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "7b9e71cb",
   "metadata": {},
   "outputs": [],
   "source": [
    "save_sr_models(SRmodels, key='SR', save_dir='../results_srmb/sr_vanilla_models/')\n",
    "save_sr_models(SRfmodels, key='SRf', save_dir='../results_srmb/sr_special_models/')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e8fc5337",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "e6e022ca",
   "metadata": {},
   "source": [
    "## Load the models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "e2e28880",
   "metadata": {},
   "outputs": [],
   "source": [
    "SRMODELS = load_sr_models('SR', save_dir='../results_srmb/sr_vanilla_models/')\n",
    "SRFMODELS = load_sr_models('SRf', save_dir='../results_srmb/sr_special_models/')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "524998fe",
   "metadata": {},
   "outputs": [],
   "source": [
    "# SRMODELS"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "469804bd",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8d93a60f",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "21e4adc3",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "74adf013",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "648358de",
   "metadata": {},
   "source": [
    "# Ablation: what if we use the entire imbalanced dataset?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "dd64eb4e",
   "metadata": {},
   "outputs": [],
   "source": [
    "# seeds for 20 trials\n",
    "RANDOM_SEEDS_FOR_UNDERSAMPLING = [42, 2024, 1234, 2405, 11, 9345,  858, 8590, 4754, 1959,\n",
    "                                  707, 10524, 83946, 63297, 78035, 22664, 49283, 35253, 82273, 90378]\n",
    "MODEL_NAMES = ['LR', 'DT', 'RF', 'XG', 'SR', 'SRf']\n",
    "ACCURACIES = {k: [] for k in MODEL_NAMES}\n",
    "F1SCORES = {k: [] for k in MODEL_NAMES}\n",
    "AUCROCS = {k: [] for k in MODEL_NAMES}\n",
    "\n",
    "SRmodels = []\n",
    "SRfmodels = []"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "5917643e",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "seed=42\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/swagatam/miniconda3/envs/hiwi/lib/python3.10/site-packages/sklearn/linear_model/_logistic.py:460: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "LogisticRegression\n",
      "Test accuracy: 0.8467 Test AUROC: 0.8193 Test F1 score: 0.3380\n",
      "DecisionTreeClassifier\n",
      "Test accuracy: 0.8700 Test AUROC: 0.8443 Test F1 score: 0.3926\n",
      "RandomForestClassifier\n",
      "Test accuracy: 0.9490 Test AUROC: 0.9315 Test F1 score: 0.2526\n",
      "XGBClassifier\n",
      "Test accuracy: 0.9346 Test AUROC: 0.9426 Test F1 score: 0.5806\n",
      "Time to fit symbolic classifier: 67.33232116699219 seconds\n",
      "SymbolicClassifier\n",
      "Test accuracy: 0.9411 Test AUROC: 0.6109 Test F1 score: 0.3223\n",
      "Time to fit symbolic classifier: 65.69597911834717 seconds\n",
      "SymbolicClassifier\n",
      "Test accuracy: 0.9447 Test AUROC: 0.5897 Test F1 score: 0.2870\n",
      "seed=2024\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/swagatam/miniconda3/envs/hiwi/lib/python3.10/site-packages/sklearn/linear_model/_logistic.py:460: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "LogisticRegression\n",
      "Test accuracy: 0.8492 Test AUROC: 0.8519 Test F1 score: 0.3558\n",
      "DecisionTreeClassifier\n",
      "Test accuracy: 0.7745 Test AUROC: 0.8317 Test F1 score: 0.2991\n",
      "RandomForestClassifier\n",
      "Test accuracy: 0.9504 Test AUROC: 0.9401 Test F1 score: 0.3030\n",
      "XGBClassifier\n",
      "Test accuracy: 0.9318 Test AUROC: 0.9497 Test F1 score: 0.5662\n",
      "Time to fit symbolic classifier: 66.04439783096313 seconds\n",
      "SymbolicClassifier\n",
      "Test accuracy: 0.9479 Test AUROC: 0.6489 Test F1 score: 0.4130\n",
      "Time to fit symbolic classifier: 67.61290097236633 seconds\n",
      "SymbolicClassifier\n",
      "Test accuracy: 0.9483 Test AUROC: 0.6368 Test F1 score: 0.3950\n",
      "seed=1234\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/swagatam/miniconda3/envs/hiwi/lib/python3.10/site-packages/sklearn/linear_model/_logistic.py:460: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "LogisticRegression\n",
      "Test accuracy: 0.8492 Test AUROC: 0.8427 Test F1 score: 0.3458\n",
      "DecisionTreeClassifier\n",
      "Test accuracy: 0.8205 Test AUROC: 0.8374 Test F1 score: 0.3225\n",
      "RandomForestClassifier\n",
      "Test accuracy: 0.9508 Test AUROC: 0.9278 Test F1 score: 0.3184\n",
      "XGBClassifier\n",
      "Test accuracy: 0.9307 Test AUROC: 0.9266 Test F1 score: 0.5522\n",
      "Time to fit symbolic classifier: 64.78528308868408 seconds\n",
      "SymbolicClassifier\n",
      "Test accuracy: 0.9436 Test AUROC: 0.6266 Test F1 score: 0.3592\n",
      "Time to fit symbolic classifier: 63.20682096481323 seconds\n",
      "SymbolicClassifier\n",
      "Test accuracy: 0.9436 Test AUROC: 0.6266 Test F1 score: 0.3592\n",
      "seed=2405\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/swagatam/miniconda3/envs/hiwi/lib/python3.10/site-packages/sklearn/linear_model/_logistic.py:460: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "LogisticRegression\n",
      "Test accuracy: 0.8478 Test AUROC: 0.8632 Test F1 score: 0.3653\n",
      "DecisionTreeClassifier\n",
      "Test accuracy: 0.8014 Test AUROC: 0.8131 Test F1 score: 0.3061\n",
      "RandomForestClassifier\n",
      "Test accuracy: 0.9501 Test AUROC: 0.9250 Test F1 score: 0.2798\n",
      "XGBClassifier\n",
      "Test accuracy: 0.9278 Test AUROC: 0.9367 Test F1 score: 0.5483\n",
      "Time to fit symbolic classifier: 67.03891324996948 seconds\n",
      "SymbolicClassifier\n",
      "Test accuracy: 0.9479 Test AUROC: 0.6488 Test F1 score: 0.4130\n",
      "Time to fit symbolic classifier: 64.79723119735718 seconds\n",
      "SymbolicClassifier\n",
      "Test accuracy: 0.9479 Test AUROC: 0.6488 Test F1 score: 0.4130\n",
      "seed=11\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/swagatam/miniconda3/envs/hiwi/lib/python3.10/site-packages/sklearn/linear_model/_logistic.py:460: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "LogisticRegression\n",
      "Test accuracy: 0.8442 Test AUROC: 0.8298 Test F1 score: 0.3344\n",
      "DecisionTreeClassifier\n",
      "Test accuracy: 0.7874 Test AUROC: 0.8179 Test F1 score: 0.2936\n",
      "RandomForestClassifier\n",
      "Test accuracy: 0.9519 Test AUROC: 0.9456 Test F1 score: 0.3431\n",
      "XGBClassifier\n",
      "Test accuracy: 0.9314 Test AUROC: 0.9412 Test F1 score: 0.5708\n",
      "Time to fit symbolic classifier: 68.0005111694336 seconds\n",
      "SymbolicClassifier\n",
      "Test accuracy: 0.9458 Test AUROC: 0.5620 Test F1 score: 0.2176\n",
      "Time to fit symbolic classifier: 65.93679213523865 seconds\n",
      "SymbolicClassifier\n",
      "Test accuracy: 0.9504 Test AUROC: 0.6351 Test F1 score: 0.4000\n",
      "seed=9345\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/swagatam/miniconda3/envs/hiwi/lib/python3.10/site-packages/sklearn/linear_model/_logistic.py:460: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "LogisticRegression\n",
      "Test accuracy: 0.8560 Test AUROC: 0.8329 Test F1 score: 0.3543\n",
      "DecisionTreeClassifier\n",
      "Test accuracy: 0.8039 Test AUROC: 0.8499 Test F1 score: 0.3175\n",
      "RandomForestClassifier\n",
      "Test accuracy: 0.9476 Test AUROC: 0.9272 Test F1 score: 0.2316\n",
      "XGBClassifier\n",
      "Test accuracy: 0.9332 Test AUROC: 0.9536 Test F1 score: 0.5830\n",
      "Time to fit symbolic classifier: 63.743890047073364 seconds\n",
      "SymbolicClassifier\n",
      "Test accuracy: 0.9469 Test AUROC: 0.5684 Test F1 score: 0.2371\n",
      "Time to fit symbolic classifier: 67.9835159778595 seconds\n",
      "SymbolicClassifier\n",
      "Test accuracy: 0.9472 Test AUROC: 0.6024 Test F1 score: 0.3226\n",
      "seed=858\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/swagatam/miniconda3/envs/hiwi/lib/python3.10/site-packages/sklearn/linear_model/_logistic.py:460: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "LogisticRegression\n",
      "Test accuracy: 0.8474 Test AUROC: 0.8279 Test F1 score: 0.3472\n",
      "DecisionTreeClassifier\n",
      "Test accuracy: 0.8126 Test AUROC: 0.7509 Test F1 score: 0.2965\n",
      "RandomForestClassifier\n",
      "Test accuracy: 0.9508 Test AUROC: 0.9173 Test F1 score: 0.3046\n",
      "XGBClassifier\n",
      "Test accuracy: 0.9364 Test AUROC: 0.9368 Test F1 score: 0.5755\n",
      "Time to fit symbolic classifier: 65.58412313461304 seconds\n",
      "SymbolicClassifier\n",
      "Test accuracy: 0.9422 Test AUROC: 0.6175 Test F1 score: 0.3374\n",
      "Time to fit symbolic classifier: 67.61177802085876 seconds\n",
      "SymbolicClassifier\n",
      "Test accuracy: 0.9422 Test AUROC: 0.6175 Test F1 score: 0.3374\n",
      "seed=8590\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/swagatam/miniconda3/envs/hiwi/lib/python3.10/site-packages/sklearn/linear_model/_logistic.py:460: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "LogisticRegression\n",
      "Test accuracy: 0.8445 Test AUROC: 0.8175 Test F1 score: 0.3224\n",
      "DecisionTreeClassifier\n",
      "Test accuracy: 0.7318 Test AUROC: 0.8034 Test F1 score: 0.2537\n",
      "RandomForestClassifier\n",
      "Test accuracy: 0.9479 Test AUROC: 0.9159 Test F1 score: 0.2408\n",
      "XGBClassifier\n",
      "Test accuracy: 0.9264 Test AUROC: 0.9206 Test F1 score: 0.5060\n",
      "Time to fit symbolic classifier: 69.19223213195801 seconds\n",
      "SymbolicClassifier\n",
      "Test accuracy: 0.9429 Test AUROC: 0.5949 Test F1 score: 0.2933\n",
      "Time to fit symbolic classifier: 73.34092903137207 seconds\n",
      "SymbolicClassifier\n",
      "Test accuracy: 0.9451 Test AUROC: 0.5956 Test F1 score: 0.3014\n",
      "seed=4754\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/swagatam/miniconda3/envs/hiwi/lib/python3.10/site-packages/sklearn/linear_model/_logistic.py:460: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "LogisticRegression\n",
      "Test accuracy: 0.8571 Test AUROC: 0.8440 Test F1 score: 0.3539\n",
      "DecisionTreeClassifier\n",
      "Test accuracy: 0.8226 Test AUROC: 0.8046 Test F1 score: 0.3270\n",
      "RandomForestClassifier\n",
      "Test accuracy: 0.9483 Test AUROC: 0.9139 Test F1 score: 0.2500\n",
      "XGBClassifier\n",
      "Test accuracy: 0.9329 Test AUROC: 0.9465 Test F1 score: 0.5600\n",
      "Time to fit symbolic classifier: 68.17389011383057 seconds\n",
      "SymbolicClassifier\n",
      "Test accuracy: 0.9454 Test AUROC: 0.6299 Test F1 score: 0.3719\n",
      "Time to fit symbolic classifier: 68.23464798927307 seconds\n",
      "SymbolicClassifier\n",
      "Test accuracy: 0.9476 Test AUROC: 0.5997 Test F1 score: 0.3178\n",
      "seed=1959\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/swagatam/miniconda3/envs/hiwi/lib/python3.10/site-packages/sklearn/linear_model/_logistic.py:460: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "LogisticRegression\n",
      "Test accuracy: 0.8524 Test AUROC: 0.8342 Test F1 score: 0.3568\n",
      "DecisionTreeClassifier\n",
      "Test accuracy: 0.8779 Test AUROC: 0.8453 Test F1 score: 0.4178\n",
      "RandomForestClassifier\n",
      "Test accuracy: 0.9494 Test AUROC: 0.9401 Test F1 score: 0.2694\n",
      "XGBClassifier\n",
      "Test accuracy: 0.9411 Test AUROC: 0.9409 Test F1 score: 0.6186\n",
      "Time to fit symbolic classifier: 73.17940306663513 seconds\n",
      "SymbolicClassifier\n",
      "Test accuracy: 0.9483 Test AUROC: 0.6459 Test F1 score: 0.4098\n",
      "Time to fit symbolic classifier: 68.82048106193542 seconds\n",
      "SymbolicClassifier\n",
      "Test accuracy: 0.9490 Test AUROC: 0.6202 Test F1 score: 0.3661\n",
      "seed=707\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/swagatam/miniconda3/envs/hiwi/lib/python3.10/site-packages/sklearn/linear_model/_logistic.py:460: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "LogisticRegression\n",
      "Test accuracy: 0.8607 Test AUROC: 0.8348 Test F1 score: 0.3762\n",
      "DecisionTreeClassifier\n",
      "Test accuracy: 0.7415 Test AUROC: 0.8041 Test F1 score: 0.2608\n",
      "RandomForestClassifier\n",
      "Test accuracy: 0.9504 Test AUROC: 0.9217 Test F1 score: 0.3100\n",
      "XGBClassifier\n",
      "Test accuracy: 0.9278 Test AUROC: 0.9294 Test F1 score: 0.5421\n",
      "Time to fit symbolic classifier: 69.46331691741943 seconds\n",
      "SymbolicClassifier\n",
      "Test accuracy: 0.9465 Test AUROC: 0.6451 Test F1 score: 0.4016\n",
      "Time to fit symbolic classifier: 69.6081268787384 seconds\n",
      "SymbolicClassifier\n",
      "Test accuracy: 0.9497 Test AUROC: 0.6460 Test F1 score: 0.4167\n",
      "seed=10524\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/swagatam/miniconda3/envs/hiwi/lib/python3.10/site-packages/sklearn/linear_model/_logistic.py:460: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "LogisticRegression\n",
      "Test accuracy: 0.8467 Test AUROC: 0.8328 Test F1 score: 0.3421\n",
      "DecisionTreeClassifier\n",
      "Test accuracy: 0.7910 Test AUROC: 0.8272 Test F1 score: 0.3038\n",
      "RandomForestClassifier\n",
      "Test accuracy: 0.9487 Test AUROC: 0.9240 Test F1 score: 0.2741\n",
      "XGBClassifier\n",
      "Test accuracy: 0.9278 Test AUROC: 0.9417 Test F1 score: 0.5543\n",
      "Time to fit symbolic classifier: 69.89840602874756 seconds\n",
      "SymbolicClassifier\n",
      "Test accuracy: 0.9476 Test AUROC: 0.6452 Test F1 score: 0.4065\n",
      "Time to fit symbolic classifier: 74.91070580482483 seconds\n",
      "SymbolicClassifier\n",
      "Test accuracy: 0.9483 Test AUROC: 0.6312 Test F1 score: 0.3846\n",
      "seed=83946\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/swagatam/miniconda3/envs/hiwi/lib/python3.10/site-packages/sklearn/linear_model/_logistic.py:460: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "LogisticRegression\n",
      "Test accuracy: 0.8560 Test AUROC: 0.8386 Test F1 score: 0.3665\n",
      "DecisionTreeClassifier\n",
      "Test accuracy: 0.8086 Test AUROC: 0.8499 Test F1 score: 0.3227\n",
      "RandomForestClassifier\n",
      "Test accuracy: 0.9540 Test AUROC: 0.9249 Test F1 score: 0.3786\n",
      "XGBClassifier\n",
      "Test accuracy: 0.9203 Test AUROC: 0.9442 Test F1 score: 0.5216\n",
      "Time to fit symbolic classifier: 69.23418998718262 seconds\n",
      "SymbolicClassifier\n",
      "Test accuracy: 0.9447 Test AUROC: 0.6325 Test F1 score: 0.3740\n",
      "Time to fit symbolic classifier: 68.76166296005249 seconds\n",
      "SymbolicClassifier\n",
      "Test accuracy: 0.9447 Test AUROC: 0.6325 Test F1 score: 0.3740\n",
      "seed=63297\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/swagatam/miniconda3/envs/hiwi/lib/python3.10/site-packages/sklearn/linear_model/_logistic.py:460: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "LogisticRegression\n",
      "Test accuracy: 0.8481 Test AUROC: 0.8317 Test F1 score: 0.3542\n",
      "DecisionTreeClassifier\n",
      "Test accuracy: 0.8061 Test AUROC: 0.8330 Test F1 score: 0.3095\n",
      "RandomForestClassifier\n",
      "Test accuracy: 0.9490 Test AUROC: 0.9349 Test F1 score: 0.2680\n",
      "XGBClassifier\n",
      "Test accuracy: 0.9336 Test AUROC: 0.9520 Test F1 score: 0.5708\n",
      "Time to fit symbolic classifier: 68.85216999053955 seconds\n",
      "SymbolicClassifier\n",
      "Test accuracy: 0.9454 Test AUROC: 0.6216 Test F1 score: 0.3559\n",
      "Time to fit symbolic classifier: 68.83684873580933 seconds\n",
      "SymbolicClassifier\n",
      "Test accuracy: 0.9479 Test AUROC: 0.6112 Test F1 score: 0.3439\n",
      "seed=78035\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/swagatam/miniconda3/envs/hiwi/lib/python3.10/site-packages/sklearn/linear_model/_logistic.py:460: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "LogisticRegression\n",
      "Test accuracy: 0.8370 Test AUROC: 0.8402 Test F1 score: 0.3477\n",
      "DecisionTreeClassifier\n",
      "Test accuracy: 0.8456 Test AUROC: 0.8096 Test F1 score: 0.3302\n",
      "RandomForestClassifier\n",
      "Test accuracy: 0.9522 Test AUROC: 0.9357 Test F1 score: 0.3448\n",
      "XGBClassifier\n",
      "Test accuracy: 0.9268 Test AUROC: 0.9450 Test F1 score: 0.5565\n",
      "Time to fit symbolic classifier: 75.26121234893799 seconds\n",
      "SymbolicClassifier\n",
      "Test accuracy: 0.9479 Test AUROC: 0.6688 Test F1 score: 0.4444\n",
      "Time to fit symbolic classifier: 69.09115386009216 seconds\n",
      "SymbolicClassifier\n",
      "Test accuracy: 0.9501 Test AUROC: 0.6265 Test F1 score: 0.3822\n",
      "seed=22664\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/swagatam/miniconda3/envs/hiwi/lib/python3.10/site-packages/sklearn/linear_model/_logistic.py:460: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "LogisticRegression\n",
      "Test accuracy: 0.8445 Test AUROC: 0.8247 Test F1 score: 0.3389\n",
      "DecisionTreeClassifier\n",
      "Test accuracy: 0.7372 Test AUROC: 0.8404 Test F1 score: 0.2591\n",
      "RandomForestClassifier\n",
      "Test accuracy: 0.9497 Test AUROC: 0.9347 Test F1 score: 0.2857\n",
      "XGBClassifier\n",
      "Test accuracy: 0.9314 Test AUROC: 0.9377 Test F1 score: 0.5649\n",
      "Time to fit symbolic classifier: 69.37272310256958 seconds\n",
      "SymbolicClassifier\n",
      "Test accuracy: 0.9422 Test AUROC: 0.6203 Test F1 score: 0.3429\n",
      "Time to fit symbolic classifier: 69.08472180366516 seconds\n",
      "SymbolicClassifier\n",
      "Test accuracy: 0.9422 Test AUROC: 0.6203 Test F1 score: 0.3429\n",
      "seed=49283\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/swagatam/miniconda3/envs/hiwi/lib/python3.10/site-packages/sklearn/linear_model/_logistic.py:460: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "LogisticRegression\n",
      "Test accuracy: 0.8718 Test AUROC: 0.8807 Test F1 score: 0.4138\n",
      "DecisionTreeClassifier\n",
      "Test accuracy: 0.7540 Test AUROC: 0.8027 Test F1 score: 0.2736\n",
      "RandomForestClassifier\n",
      "Test accuracy: 0.9522 Test AUROC: 0.9192 Test F1 score: 0.3383\n",
      "XGBClassifier\n",
      "Test accuracy: 0.9239 Test AUROC: 0.9314 Test F1 score: 0.5330\n",
      "Time to fit symbolic classifier: 69.41814804077148 seconds\n",
      "SymbolicClassifier\n",
      "Test accuracy: 0.9472 Test AUROC: 0.6570 Test F1 score: 0.4235\n",
      "Time to fit symbolic classifier: 68.55552625656128 seconds\n",
      "SymbolicClassifier\n",
      "Test accuracy: 0.9479 Test AUROC: 0.6225 Test F1 score: 0.3668\n",
      "seed=35253\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/swagatam/miniconda3/envs/hiwi/lib/python3.10/site-packages/sklearn/linear_model/_logistic.py:460: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "LogisticRegression\n",
      "Test accuracy: 0.8348 Test AUROC: 0.8110 Test F1 score: 0.3195\n",
      "DecisionTreeClassifier\n",
      "Test accuracy: 0.7907 Test AUROC: 0.8045 Test F1 score: 0.2899\n",
      "RandomForestClassifier\n",
      "Test accuracy: 0.9483 Test AUROC: 0.9151 Test F1 score: 0.2500\n",
      "XGBClassifier\n",
      "Test accuracy: 0.9293 Test AUROC: 0.9249 Test F1 score: 0.5343\n",
      "Time to fit symbolic classifier: 75.61543703079224 seconds\n",
      "SymbolicClassifier\n",
      "Test accuracy: 0.9451 Test AUROC: 0.5476 Test F1 score: 0.1730\n",
      "Time to fit symbolic classifier: 68.48590612411499 seconds\n",
      "SymbolicClassifier\n",
      "Test accuracy: 0.9461 Test AUROC: 0.5877 Test F1 score: 0.2857\n",
      "seed=82273\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/swagatam/miniconda3/envs/hiwi/lib/python3.10/site-packages/sklearn/linear_model/_logistic.py:460: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "LogisticRegression\n",
      "Test accuracy: 0.8531 Test AUROC: 0.8229 Test F1 score: 0.3599\n",
      "DecisionTreeClassifier\n",
      "Test accuracy: 0.8456 Test AUROC: 0.8416 Test F1 score: 0.3364\n",
      "RandomForestClassifier\n",
      "Test accuracy: 0.9476 Test AUROC: 0.9164 Test F1 score: 0.2551\n",
      "XGBClassifier\n",
      "Test accuracy: 0.9268 Test AUROC: 0.9287 Test F1 score: 0.5256\n",
      "Time to fit symbolic classifier: 68.64090394973755 seconds\n",
      "SymbolicClassifier\n",
      "Test accuracy: 0.9422 Test AUROC: 0.6260 Test F1 score: 0.3534\n",
      "Time to fit symbolic classifier: 68.7185320854187 seconds\n",
      "SymbolicClassifier\n",
      "Test accuracy: 0.9436 Test AUROC: 0.6005 Test F1 score: 0.3084\n",
      "seed=90378\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/swagatam/miniconda3/envs/hiwi/lib/python3.10/site-packages/sklearn/linear_model/_logistic.py:460: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "LogisticRegression\n",
      "Test accuracy: 0.8657 Test AUROC: 0.8305 Test F1 score: 0.3746\n",
      "DecisionTreeClassifier\n",
      "Test accuracy: 0.7899 Test AUROC: 0.7964 Test F1 score: 0.2875\n",
      "RandomForestClassifier\n",
      "Test accuracy: 0.9490 Test AUROC: 0.9247 Test F1 score: 0.2680\n",
      "XGBClassifier\n",
      "Test accuracy: 0.9311 Test AUROC: 0.9397 Test F1 score: 0.5450\n",
      "Time to fit symbolic classifier: 68.76493072509766 seconds\n",
      "SymbolicClassifier\n",
      "Test accuracy: 0.9465 Test AUROC: 0.6394 Test F1 score: 0.3918\n",
      "Time to fit symbolic classifier: 67.91902613639832 seconds\n",
      "SymbolicClassifier\n",
      "Test accuracy: 0.9487 Test AUROC: 0.6257 Test F1 score: 0.3755\n"
     ]
    }
   ],
   "source": [
    "USE_BALANCED_SUBSAMPLE=False # perform undersampling of healthy classes\n",
    "\n",
    "for random_state in RANDOM_SEEDS_FOR_UNDERSAMPLING:\n",
    "    rus = RandomUnderSampler(sampling_strategy=0.85, # this is another hyperparameter\n",
    "                             random_state=random_state)\n",
    "    X1b, y1b = rus.fit_resample(X1, y1)\n",
    "    print(f'seed={random_state}')\n",
    "    # print(sorted(Counter(y1b).items()))\n",
    "    \n",
    "    if not USE_BALANCED_SUBSAMPLE:\n",
    "        X_train, X_test, y_train, y_test = train_test_split(X1, y1,\n",
    "                                                        test_size=0.25,\n",
    "                                                        # train_size=0.5, # if slow use this\n",
    "                                                        random_state=random_state, stratify=y1)\n",
    "        class_weight = compute_class_weight(class_weight='balanced', classes=np.unique(y1), y=y1)\n",
    "    else:\n",
    "        # print('using a balanced subsample of the data ...')\n",
    "        X_train, X_test, y_train, y_test = train_test_split(X1b, y1b,\n",
    "                                                            test_size=0.25,\n",
    "                                                            # train_size=0.5, # if slow use this\n",
    "                                                            random_state=42, stratify=y1b)\n",
    "        class_weight = compute_class_weight(class_weight='balanced', classes=np.unique(y1b), y=y1b)\n",
    "        \n",
    "    \n",
    "    sample_weights = compute_sample_weight(class_weight='balanced', y=y_train)\n",
    "    \n",
    "    # logistic regression classifier\n",
    "    model_lr = LogisticRegression(max_iter=500, class_weight=dict(enumerate(class_weight)))\n",
    "    model_lr.fit(X_train, y_train)\n",
    "    store_result('LR', *calculate_metrics(model_lr, X_train, y_train, X_test, y_test))\n",
    "    \n",
    "    # Decision tree classifier\n",
    "    model_dt = DecisionTreeClassifier(max_depth=5, class_weight=dict(enumerate(class_weight)))\n",
    "    model_dt.fit(X_train, y_train)\n",
    "    store_result('DT', *calculate_metrics(model_dt, X_train, y_train, X_test, y_test))\n",
    "    \n",
    "    \n",
    "    # Random forest classifier\n",
    "    model_rf = RandomForestClassifier(n_estimators=50, class_weight=dict(enumerate(class_weight)))\n",
    "    model_rf.fit(X_train, y_train)\n",
    "    store_result('RF', *calculate_metrics(model_rf, X_train, y_train, X_test, y_test))\n",
    "    \n",
    "    \n",
    "    # Create an XGBoost classifier for multiclass classification\n",
    "    model_xg = XGBClassifier(n_estimators=50, max_depth=5, learning_rate=0.1, objective='binary:logistic')\n",
    "    model_xg.fit(X_train, y_train, sample_weight=sample_weights)\n",
    "    store_result('XG', *calculate_metrics(model_xg, X_train, y_train, X_test, y_test))\n",
    "    \n",
    "    \n",
    "    # Do it for vanilla symbolic regression\n",
    "    function_set = ['add', 'sub', 'mul', 'div', 'neg', 'max', 'min', 'sqrt', 'log']\n",
    "\n",
    "    est = SymbolicClassifier(population_size=6000,\n",
    "                             generations=20,\n",
    "                             tournament_size=25,\n",
    "\n",
    "                             init_depth=(2, 6),\n",
    "                             const_range=(0., 100.),\n",
    "                             # init_method=\"full\",\n",
    "                             parsimony_coefficient=0.001,\n",
    "                             function_set=function_set,\n",
    "\n",
    "                             stopping_criteria=1.0, metric=customacc, #use custom acc as fitness\n",
    "                             \n",
    "                             feature_names=X1.columns.to_list(),\n",
    "                             # verbose=True,\n",
    "                             random_state=42)\n",
    "\n",
    "    t0 = time.time()\n",
    "    est.fit(X_train, y_train)\n",
    "    print('Time to fit symbolic classifier:', time.time() - t0, 'seconds')\n",
    "    store_result('SR', *calculate_metrics(est, X_train, y_train, X_test, y_test))\n",
    "    SRmodels.append(est)\n",
    "    \n",
    "    \n",
    "    # SR with special functions\n",
    "    special_functions = [presence, absence, presence2, absence2, ifelse]#, add3, add10]\n",
    "    function_set = ['add', 'sub', 'mul', 'div', 'neg', 'max', 'min'] + special_functions\n",
    "\n",
    "    est = SymbolicClassifier(population_size=6000,\n",
    "                             generations=20,\n",
    "                             tournament_size=25,\n",
    "\n",
    "                             init_depth=(2, 6),\n",
    "                             const_range=(0., 100.),\n",
    "                             # init_method=\"full\",\n",
    "                             parsimony_coefficient=0.001,\n",
    "                             function_set=function_set,\n",
    "\n",
    "                             stopping_criteria=1.0, metric=customacc, #use custom acc as fitness\n",
    "                             \n",
    "                             feature_names=X1.columns.to_list(),\n",
    "                             # verbose=True,\n",
    "                             random_state=42)\n",
    "\n",
    "    t0 = time.time()\n",
    "    est.fit(X_train, y_train)\n",
    "    print('Time to fit symbolic classifier:', time.time() - t0, 'seconds')\n",
    "    store_result('SRf', *calculate_metrics(est, X_train, y_train, X_test, y_test))\n",
    "    SRfmodels.append(est)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "id": "8ade794c",
   "metadata": {},
   "outputs": [],
   "source": [
    "LENGTHS = {'SR': [], 'SRf': []}\n",
    "for est in SRmodels:\n",
    "    LENGTHS['SR'].append(est._program.length_)\n",
    "for est in SRfmodels:\n",
    "    LENGTHS['SRf'].append(est._program.length_)\n",
    "    \n",
    "LENGTHS['SR'] = np.asarray(LENGTHS['SR'])\n",
    "LENGTHS['SRf'] = np.asarray(LENGTHS['SRf'])\n",
    "\n",
    "convert_to_arrays()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "id": "b42dbf68",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "LR acc: 0.85 ± 0.0087\n",
      "DT acc: 0.80 ± 0.0397\n",
      "RF acc: 0.95 ± 0.0017\n",
      "XG acc: 0.93 ± 0.0045\n",
      "SR acc: 0.95 ± 0.0022\n",
      "SRf acc: 0.95 ± 0.0025\n",
      "--------\n",
      "LR F1: 0.35 ± 0.0202\n",
      "DT F1: 0.31 ± 0.0395\n",
      "RF F1: 0.29 ± 0.0393\n",
      "XG F1: 0.56 ± 0.0248\n",
      "SR F1: 0.35 ± 0.0708\n",
      "SRf F1: 0.35 ± 0.0393\n",
      "--------\n"
     ]
    }
   ],
   "source": [
    "plusminus = pm = u\"\\u00B1\"\n",
    "for model_name in MODEL_NAMES:\n",
    "    print(f\"{model_name} acc: {ACCURACIES[model_name].mean():.2f} {pm} {ACCURACIES[model_name].std():.4f}\")\n",
    "print('--------')\n",
    "for model_name in MODEL_NAMES:\n",
    "    print(f\"{model_name} F1: {F1SCORES[model_name].mean():.2f} {pm} {F1SCORES[model_name].std():.4f}\")\n",
    "\n",
    "print('--------')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "id": "240606ff",
   "metadata": {},
   "outputs": [],
   "source": [
    "# LENGTHS['SR'].mean(), LENGTHS['SRf'].mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "id": "5eb2863a",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SR mean length: 1.6\n",
      "SRf mean length: 4.0\n"
     ]
    }
   ],
   "source": [
    "print(f\"SR mean length: {LENGTHS['SR'].mean()}\")\n",
    "print(f\"SRf mean length: {LENGTHS['SRf'].mean()}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b37436f9",
   "metadata": {},
   "source": [
    "**Conclusion**: From the results above we see that symbolic regression models have F1 \n",
    "score below 0.5, i.e., they have very poor classification performance. Hence for a \n",
    "fairer comparison among the classifiers, we do not use imbalanced data in our \n",
    "experiments."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3391730a",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f443f957",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "75c8a20b",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}