open-world/multi/open_multi_down_feature8_30.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 1. Data Load"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loading data...\n"
     ]
    }
   ],
   "source": [
    "import pickle\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from sklearn.utils import resample\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.metrics import accuracy_score, confusion_matrix, classification_report\n",
    "\n",
    "# load data\n",
    "print(\"Loading data...\")\n",
    "with open(\"mon_standard.pkl\", 'rb') as fi:\n",
    "    mon_data = pickle.load(fi)\n",
    "\n",
    "with open(\"unmon_standard10_3000.pkl\", 'rb') as fi:\n",
    "    unmon_data = pickle.load(fi)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 2. Data Preprocess"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "# process Monitored data\n",
    "def process_monitored(data, total_urls=950, use_sublabel=False, url_per_site=10):\n",
    "    X1, X2, y = [], [], []\n",
    "    for i in range(total_urls):\n",
    "        label = i if use_sublabel else i // url_per_site\n",
    "        for sample in data[i]:\n",
    "            size_seq = []\n",
    "            time_seq = []\n",
    "            for c in sample:\n",
    "                dr = 1 if c > 0 else -1\n",
    "                time_seq.append(abs(c))\n",
    "                size_seq.append(dr * 512)\n",
    "            X1.append(time_seq)\n",
    "            X2.append(size_seq)\n",
    "            y.append(label)\n",
    "    return X1, X2, y\n",
    "\n",
    "# process Unmonitored data\n",
    "def process_unmonitored(data, total_urls=10000):\n",
    "    X1, X2 = [], []\n",
    "    for i in range(total_urls):\n",
    "        size_seq = []\n",
    "        time_seq = []\n",
    "        for c in data[i]:\n",
    "            dr = 1 if c > 0 else -1\n",
    "            time_seq.append(abs(c))\n",
    "            size_seq.append(dr * 512)\n",
    "        X1.append(time_seq)\n",
    "        X2.append(size_seq)\n",
    "    y = [-1] * len(X1)  # label Unmonitored data as -1\n",
    "    return X1, X2, y\n",
    "\n",
    "# process Monitored data\n",
    "X1_mon, X2_mon, y_mon = process_monitored(\n",
    "    mon_data, total_urls=950, use_sublabel=False, url_per_site=10\n",
    ")\n",
    "\n",
    "# process Unmonitored data\n",
    "X1_unmon, X2_unmon, y_unmon = process_unmonitored(\n",
    "    unmon_data, total_urls=3000\n",
    ")\n",
    "\n",
    "# concatenate data\n",
    "X1 = X1_mon + X1_unmon\n",
    "X2 = X2_mon + X2_unmon\n",
    "y = y_mon + y_unmon"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 3. Feature Extraction"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "# create feautures\n",
    "def create_features(X1, X2):\n",
    "    X = []\n",
    "    for i in range(len(X1)):\n",
    "        packet_size_direction = sum(X2[i])\n",
    "        cumulative_packet_size = np.sum([abs(c) for c in X2[i]])\n",
    "        burst_lengths = len([c for c in X2[i] if c != 0])\n",
    "\n",
    "        num_incoming_packets = len([c for c in X2[i] if c > 0])\n",
    "        ratio_incoming_packets = num_incoming_packets / len(X2[i]) if len(X2[i]) > 0 else 0\n",
    "        num_outgoing_packets = len([c for c in X2[i] if c < 0])\n",
    "        total_packet_count = len(X2[i])\n",
    "\n",
    "        feature_vector = [\n",
    "            packet_size_direction,\n",
    "            np.mean(X1[i]) if X1[i] else 0,\n",
    "            cumulative_packet_size,\n",
    "            burst_lengths,\n",
    "            num_incoming_packets,\n",
    "            ratio_incoming_packets,\n",
    "            num_outgoing_packets,\n",
    "            total_packet_count\n",
    "        ]\n",
    "        X.append(feature_vector)\n",
    "    return np.array(X)\n",
    "\n",
    "X = create_features(X1, X2)\n",
    "y = np.array(y)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 4. Data split and downsampling"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "# split data\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42, stratify=y)\n",
    "\n",
    "# take Monitored and Unmonitored data from Train data\n",
    "train_data = pd.DataFrame({\n",
    "    'features': list(X_train),\n",
    "    'label': y_train\n",
    "})\n",
    "train_monitored = train_data[train_data['label'] != -1]\n",
    "train_unmonitored = train_data[train_data['label'] == -1]\n",
    "\n",
    "# downsample Monitored data\n",
    "train_monitored_downsampled = resample(\n",
    "    train_monitored,\n",
    "    replace=True,\n",
    "    n_samples=len(train_unmonitored),\n",
    "    random_state=42\n",
    ")\n",
    "\n",
    "# re-concatenate Train data\n",
    "balanced_train_data = pd.concat([train_monitored_downsampled, train_unmonitored])\n",
    "X_train_balanced = np.array(balanced_train_data['features'].tolist())\n",
    "y_train_balanced = balanced_train_data['label'].values\n",
    "\n",
    "# scale data \n",
    "scaler = StandardScaler()\n",
    "X_train_balanced = scaler.fit_transform(X_train_balanced)\n",
    "X_test = scaler.transform(X_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 5. Train the Random Forest model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "==== Random Forest ====\n",
      "Accuracy: 0.3104545454545454\n",
      "F1 Score: 0.28640803161082506\n",
      "Confusion Matrix:\n",
      " [[522   1   0 ...   0   0   0]\n",
      " [ 31   2   0 ...   0   0   0]\n",
      " [ 19   0   4 ...   0   0   0]\n",
      " ...\n",
      " [ 30   0   0 ...   0   0   0]\n",
      " [ 21   0   0 ...   0   8   0]\n",
      " [ 36   0   0 ...   0   1   0]]\n",
      "Classification Report:\n",
      "               precision    recall  f1-score   support\n",
      "\n",
      "          -1       0.18      0.87      0.30       600\n",
      "           0       0.25      0.05      0.08        40\n",
      "           1       0.33      0.10      0.15        40\n",
      "           2       0.62      0.33      0.43        40\n",
      "           3       0.73      0.28      0.40        40\n",
      "           4       0.35      0.15      0.21        40\n",
      "           5       0.36      0.12      0.19        40\n",
      "           6       0.64      0.23      0.33        40\n",
      "           7       0.28      0.17      0.22        40\n",
      "           8       0.69      0.28      0.39        40\n",
      "           9       0.43      0.07      0.13        40\n",
      "          10       0.78      0.17      0.29        40\n",
      "          11       0.55      0.15      0.24        40\n",
      "          12       0.61      0.57      0.59        40\n",
      "          13       0.50      0.15      0.23        40\n",
      "          14       0.40      0.10      0.16        40\n",
      "          15       0.44      0.20      0.28        40\n",
      "          16       0.69      0.50      0.58        40\n",
      "          17       0.50      0.03      0.05        40\n",
      "          18       0.46      0.28      0.34        40\n",
      "          19       0.73      0.20      0.31        40\n",
      "          20       0.82      0.78      0.79        40\n",
      "          21       0.00      0.00      0.00        40\n",
      "          22       0.14      0.03      0.04        40\n",
      "          23       0.67      0.20      0.31        40\n",
      "          24       0.25      0.03      0.05        40\n",
      "          25       0.54      0.17      0.26        40\n",
      "          26       0.48      0.30      0.37        40\n",
      "          27       0.00      0.00      0.00        40\n",
      "          28       0.62      0.40      0.48        40\n",
      "          29       0.42      0.20      0.27        40\n",
      "          30       0.59      0.33      0.42        40\n",
      "          31       0.56      0.12      0.20        40\n",
      "          32       0.25      0.07      0.12        40\n",
      "          33       0.83      0.25      0.38        40\n",
      "          34       0.00      0.00      0.00        40\n",
      "          35       0.33      0.12      0.18        40\n",
      "          36       0.53      0.25      0.34        40\n",
      "          37       0.00      0.00      0.00        40\n",
      "          38       0.25      0.12      0.17        40\n",
      "          39       0.58      0.17      0.27        40\n",
      "          40       0.38      0.12      0.19        40\n",
      "          41       0.69      0.28      0.39        40\n",
      "          42       0.27      0.07      0.12        40\n",
      "          43       0.87      0.50      0.63        40\n",
      "          44       0.90      0.88      0.89        40\n",
      "          45       0.38      0.23      0.28        40\n",
      "          46       0.64      0.35      0.45        40\n",
      "          47       0.20      0.05      0.08        40\n",
      "          48       0.29      0.05      0.09        40\n",
      "          49       0.60      0.53      0.56        40\n",
      "          50       0.41      0.23      0.29        40\n",
      "          51       0.29      0.05      0.09        40\n",
      "          52       0.50      0.15      0.23        40\n",
      "          53       0.62      0.12      0.21        40\n",
      "          54       0.53      0.20      0.29        40\n",
      "          55       0.22      0.05      0.08        40\n",
      "          56       0.70      0.57      0.63        40\n",
      "          57       0.14      0.03      0.04        40\n",
      "          58       0.73      0.60      0.66        40\n",
      "          59       0.47      0.17      0.25        40\n",
      "          60       0.38      0.12      0.19        40\n",
      "          61       0.12      0.03      0.04        40\n",
      "          62       0.44      0.10      0.16        40\n",
      "          63       0.22      0.05      0.08        40\n",
      "          64       0.64      0.17      0.27        40\n",
      "          65       0.52      0.28      0.36        40\n",
      "          66       0.59      0.42      0.49        40\n",
      "          67       0.70      0.17      0.28        40\n",
      "          68       0.00      0.00      0.00        40\n",
      "          69       0.58      0.17      0.27        40\n",
      "          70       0.90      0.68      0.77        40\n",
      "          71       0.38      0.07      0.12        40\n",
      "          72       0.35      0.20      0.25        40\n",
      "          73       0.54      0.35      0.42        40\n",
      "          74       0.20      0.07      0.11        40\n",
      "          75       0.90      0.70      0.79        40\n",
      "          76       0.95      0.90      0.92        40\n",
      "          77       0.09      0.03      0.04        40\n",
      "          78       0.17      0.03      0.04        40\n",
      "          79       0.00      0.00      0.00        40\n",
      "          80       0.73      0.40      0.52        40\n",
      "          81       0.29      0.15      0.20        40\n",
      "          82       1.00      0.40      0.57        40\n",
      "          83       0.55      0.15      0.24        40\n",
      "          84       0.39      0.23      0.29        40\n",
      "          85       0.42      0.42      0.42        40\n",
      "          86       0.84      0.68      0.75        40\n",
      "          87       0.40      0.15      0.22        40\n",
      "          88       0.50      0.30      0.37        40\n",
      "          89       0.25      0.05      0.08        40\n",
      "          90       0.42      0.25      0.31        40\n",
      "          91       0.52      0.28      0.36        40\n",
      "          92       0.00      0.00      0.00        40\n",
      "          93       0.47      0.20      0.28        40\n",
      "          94       0.00      0.00      0.00        40\n",
      "\n",
      "    accuracy                           0.31      4400\n",
      "   macro avg       0.45      0.23      0.28      4400\n",
      "weighted avg       0.42      0.31      0.29      4400\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/sojeonglee/anaconda3/lib/python3.11/site-packages/sklearn/metrics/_classification.py:1344: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n",
      "/Users/sojeonglee/anaconda3/lib/python3.11/site-packages/sklearn/metrics/_classification.py:1344: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n",
      "/Users/sojeonglee/anaconda3/lib/python3.11/site-packages/sklearn/metrics/_classification.py:1344: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n"
     ]
    }
   ],
   "source": [
    "from sklearn.metrics import f1_score\n",
    "# train Random Forest model\n",
    "rf_model = RandomForestClassifier(\n",
    "    n_estimators=300, max_depth=30, random_state=42, n_jobs=-1\n",
    ")\n",
    "rf_model.fit(X_train_balanced, y_train_balanced)\n",
    "\n",
    "# prediction and evaluate\n",
    "y_pred = rf_model.predict(X_test)\n",
    "\n",
    "y_pred = rf_model.predict(X_test)\n",
    "print(\"==== Random Forest ====\")\n",
    "print(\"Accuracy:\", accuracy_score(y_test, y_pred))\n",
    "print(\"F1 Score:\", f1_score(y_test, y_pred, average='weighted'))\n",
    "print(\"Confusion Matrix:\\n\", confusion_matrix(y_test, y_pred))\n",
    "print(\"Classification Report:\\n\", classification_report(y_test, y_pred))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Original Training Data Distribution:\n",
      "Label -1 count: 2400\n",
      "Other labels count: 15200\n",
      "\n",
      "Balanced Training Data Distribution:\n",
      "Label -1 count: 2400\n",
      "Other labels count: 2400\n",
      "\n",
      "Test Data Distribution:\n",
      "Label -1 count: 600\n",
      "Other labels count: 3800\n"
     ]
    }
   ],
   "source": [
    "# evaluate data distribution\n",
    "print(\"\\nOriginal Training Data Distribution:\")\n",
    "print(f\"Label -1 count: {sum(y_train == -1)}\")\n",
    "print(f\"Other labels count: {sum(y_train != -1)}\")\n",
    "\n",
    "print(\"\\nBalanced Training Data Distribution:\")\n",
    "print(f\"Label -1 count: {sum(y_train_balanced == -1)}\")\n",
    "print(f\"Other labels count: {sum(y_train_balanced != -1)}\")\n",
    "\n",
    "print(\"\\nTest Data Distribution:\")\n",
    "print(f\"Label -1 count: {sum(y_test == -1)}\")\n",
    "print(f\"Other labels count: {sum(y_test != -1)}\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 3 folds for each of 81 candidates, totalling 243 fits\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/sojeonglee/anaconda3/lib/python3.11/site-packages/joblib/externals/loky/process_executor.py:700: UserWarning: A worker stopped while some jobs were given to the executor. This can be caused by a too short worker timeout or by a memory leak.\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Best Random Forest Parameters: {'max_depth': 20, 'min_samples_leaf': 1, 'min_samples_split': 2, 'n_estimators': 300}\n",
      "\n",
      "Random Forest Performance:\n",
      "Accuracy: 0.30477272727272725\n",
      "F1 Score: 0.2788512756528181\n",
      "Classification Report:\n",
      "               precision    recall  f1-score   support\n",
      "\n",
      "          -1       0.18      0.88      0.30       600\n",
      "           0       0.40      0.05      0.09        40\n",
      "           1       0.27      0.07      0.12        40\n",
      "           2       0.65      0.33      0.43        40\n",
      "           3       0.73      0.28      0.40        40\n",
      "           4       0.27      0.10      0.15        40\n",
      "           5       0.31      0.10      0.15        40\n",
      "           6       0.56      0.12      0.20        40\n",
      "           7       0.30      0.17      0.22        40\n",
      "           8       0.62      0.25      0.36        40\n",
      "           9       0.50      0.07      0.13        40\n",
      "          10       0.67      0.15      0.24        40\n",
      "          11       0.62      0.12      0.21        40\n",
      "          12       0.61      0.57      0.59        40\n",
      "          13       0.46      0.15      0.23        40\n",
      "          14       0.44      0.10      0.16        40\n",
      "          15       0.45      0.23      0.30        40\n",
      "          16       0.74      0.50      0.60        40\n",
      "          17       0.00      0.00      0.00        40\n",
      "          18       0.57      0.30      0.39        40\n",
      "          19       0.75      0.15      0.25        40\n",
      "          20       0.84      0.78      0.81        40\n",
      "          21       0.00      0.00      0.00        40\n",
      "          22       0.20      0.03      0.04        40\n",
      "          23       0.75      0.23      0.35        40\n",
      "          24       0.25      0.03      0.05        40\n",
      "          25       0.50      0.15      0.23        40\n",
      "          26       0.42      0.25      0.31        40\n",
      "          27       0.00      0.00      0.00        40\n",
      "          28       0.68      0.42      0.52        40\n",
      "          29       0.47      0.20      0.28        40\n",
      "          30       0.62      0.33      0.43        40\n",
      "          31       0.56      0.12      0.20        40\n",
      "          32       0.23      0.07      0.11        40\n",
      "          33       0.83      0.25      0.38        40\n",
      "          34       0.00      0.00      0.00        40\n",
      "          35       0.31      0.10      0.15        40\n",
      "          36       0.67      0.15      0.24        40\n",
      "          37       0.00      0.00      0.00        40\n",
      "          38       0.20      0.10      0.13        40\n",
      "          39       0.50      0.15      0.23        40\n",
      "          40       0.40      0.05      0.09        40\n",
      "          41       0.73      0.28      0.40        40\n",
      "          42       0.38      0.07      0.12        40\n",
      "          43       0.91      0.53      0.67        40\n",
      "          44       0.89      0.85      0.87        40\n",
      "          45       0.35      0.20      0.25        40\n",
      "          46       0.67      0.30      0.41        40\n",
      "          47       0.25      0.05      0.08        40\n",
      "          48       0.50      0.07      0.13        40\n",
      "          49       0.58      0.55      0.56        40\n",
      "          50       0.50      0.20      0.29        40\n",
      "          51       0.40      0.05      0.09        40\n",
      "          52       0.44      0.10      0.16        40\n",
      "          53       0.71      0.12      0.21        40\n",
      "          54       0.55      0.15      0.24        40\n",
      "          55       0.29      0.05      0.09        40\n",
      "          56       0.72      0.57      0.64        40\n",
      "          57       0.20      0.03      0.04        40\n",
      "          58       0.72      0.53      0.61        40\n",
      "          59       0.46      0.15      0.23        40\n",
      "          60       0.33      0.10      0.15        40\n",
      "          61       0.25      0.05      0.08        40\n",
      "          62       0.44      0.10      0.16        40\n",
      "          63       0.25      0.05      0.08        40\n",
      "          64       0.47      0.17      0.25        40\n",
      "          65       0.73      0.28      0.40        40\n",
      "          66       0.55      0.40      0.46        40\n",
      "          67       0.54      0.17      0.26        40\n",
      "          68       0.00      0.00      0.00        40\n",
      "          69       0.58      0.17      0.27        40\n",
      "          70       0.90      0.70      0.79        40\n",
      "          71       0.50      0.05      0.09        40\n",
      "          72       0.38      0.20      0.26        40\n",
      "          73       0.54      0.35      0.42        40\n",
      "          74       0.24      0.10      0.14        40\n",
      "          75       0.91      0.72      0.81        40\n",
      "          76       0.95      0.90      0.92        40\n",
      "          77       0.00      0.00      0.00        40\n",
      "          78       0.17      0.03      0.04        40\n",
      "          79       0.00      0.00      0.00        40\n",
      "          80       0.70      0.40      0.51        40\n",
      "          81       0.32      0.15      0.20        40\n",
      "          82       1.00      0.40      0.57        40\n",
      "          83       0.67      0.15      0.24        40\n",
      "          84       0.38      0.23      0.28        40\n",
      "          85       0.43      0.45      0.44        40\n",
      "          86       0.85      0.70      0.77        40\n",
      "          87       0.38      0.15      0.21        40\n",
      "          88       0.55      0.30      0.39        40\n",
      "          89       0.20      0.05      0.08        40\n",
      "          90       0.53      0.25      0.34        40\n",
      "          91       0.62      0.25      0.36        40\n",
      "          92       0.00      0.00      0.00        40\n",
      "          93       0.50      0.23      0.31        40\n",
      "          94       0.00      0.00      0.00        40\n",
      "\n",
      "    accuracy                           0.30      4400\n",
      "   macro avg       0.46      0.22      0.28      4400\n",
      "weighted avg       0.43      0.30      0.28      4400\n",
      "\n",
      "\n",
      "Training Gradient Boosting...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/sojeonglee/anaconda3/lib/python3.11/site-packages/sklearn/metrics/_classification.py:1344: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n",
      "/Users/sojeonglee/anaconda3/lib/python3.11/site-packages/sklearn/metrics/_classification.py:1344: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n",
      "/Users/sojeonglee/anaconda3/lib/python3.11/site-packages/sklearn/metrics/_classification.py:1344: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Gradient Boosting Performance:\n",
      "Accuracy: 0.21954545454545454\n",
      "F1 Score: 0.1881001285458769\n",
      "Classification Report:\n",
      "               precision    recall  f1-score   support\n",
      "\n",
      "          -1       0.16      0.74      0.26       600\n",
      "           0       0.17      0.07      0.10        40\n",
      "           1       0.10      0.03      0.04        40\n",
      "           2       0.50      0.17      0.26        40\n",
      "           3       0.55      0.28      0.37        40\n",
      "           4       0.31      0.10      0.15        40\n",
      "           5       0.00      0.00      0.00        40\n",
      "           6       0.33      0.10      0.15        40\n",
      "           7       0.06      0.05      0.05        40\n",
      "           8       0.22      0.05      0.08        40\n",
      "           9       0.10      0.03      0.04        40\n",
      "          10       0.33      0.10      0.15        40\n",
      "          11       0.50      0.12      0.20        40\n",
      "          12       0.38      0.38      0.38        40\n",
      "          13       0.00      0.00      0.00        40\n",
      "          14       0.13      0.07      0.10        40\n",
      "          15       0.32      0.15      0.20        40\n",
      "          16       0.36      0.40      0.38        40\n",
      "          17       0.00      0.00      0.00        40\n",
      "          18       0.44      0.28      0.34        40\n",
      "          19       0.38      0.15      0.21        40\n",
      "          20       0.79      0.28      0.41        40\n",
      "          21       0.00      0.00      0.00        40\n",
      "          22       0.08      0.03      0.04        40\n",
      "          23       0.30      0.15      0.20        40\n",
      "          24       0.07      0.03      0.04        40\n",
      "          25       0.38      0.20      0.26        40\n",
      "          26       0.39      0.17      0.24        40\n",
      "          27       0.00      0.00      0.00        40\n",
      "          28       0.36      0.23      0.28        40\n",
      "          29       0.20      0.05      0.08        40\n",
      "          30       0.62      0.33      0.43        40\n",
      "          31       0.00      0.00      0.00        40\n",
      "          32       0.19      0.07      0.11        40\n",
      "          33       0.47      0.17      0.25        40\n",
      "          34       0.05      0.03      0.03        40\n",
      "          35       0.50      0.12      0.20        40\n",
      "          36       0.00      0.00      0.00        40\n",
      "          37       0.00      0.00      0.00        40\n",
      "          38       0.67      0.10      0.17        40\n",
      "          39       0.00      0.00      0.00        40\n",
      "          40       0.00      0.00      0.00        40\n",
      "          41       0.62      0.20      0.30        40\n",
      "          42       0.16      0.07      0.10        40\n",
      "          43       0.53      0.23      0.32        40\n",
      "          44       0.86      0.47      0.61        40\n",
      "          45       0.12      0.03      0.04        40\n",
      "          46       0.42      0.12      0.19        40\n",
      "          47       0.07      0.03      0.04        40\n",
      "          48       0.23      0.07      0.11        40\n",
      "          49       0.52      0.40      0.45        40\n",
      "          50       0.22      0.10      0.14        40\n",
      "          51       0.10      0.03      0.04        40\n",
      "          52       0.36      0.10      0.16        40\n",
      "          53       0.10      0.03      0.04        40\n",
      "          54       0.33      0.12      0.18        40\n",
      "          55       0.17      0.05      0.08        40\n",
      "          56       0.59      0.50      0.54        40\n",
      "          57       0.17      0.07      0.10        40\n",
      "          58       0.61      0.28      0.38        40\n",
      "          59       0.25      0.10      0.14        40\n",
      "          60       0.14      0.07      0.10        40\n",
      "          61       0.00      0.00      0.00        40\n",
      "          62       0.23      0.07      0.11        40\n",
      "          63       0.00      0.00      0.00        40\n",
      "          64       0.00      0.00      0.00        40\n",
      "          65       0.10      0.03      0.04        40\n",
      "          66       0.48      0.25      0.33        40\n",
      "          67       0.44      0.10      0.16        40\n",
      "          68       0.00      0.00      0.00        40\n",
      "          69       0.25      0.05      0.08        40\n",
      "          70       0.84      0.65      0.73        40\n",
      "          71       0.00      0.00      0.00        40\n",
      "          72       0.19      0.12      0.15        40\n",
      "          73       0.50      0.33      0.39        40\n",
      "          74       0.11      0.05      0.07        40\n",
      "          75       0.87      0.68      0.76        40\n",
      "          76       0.79      0.65      0.71        40\n",
      "          77       0.00      0.00      0.00        40\n",
      "          78       0.00      0.00      0.00        40\n",
      "          79       0.06      0.03      0.04        40\n",
      "          80       0.00      0.00      0.00        40\n",
      "          81       0.11      0.05      0.07        40\n",
      "          82       0.94      0.38      0.54        40\n",
      "          83       0.50      0.20      0.29        40\n",
      "          84       0.08      0.03      0.04        40\n",
      "          85       0.51      0.45      0.48        40\n",
      "          86       0.89      0.60      0.72        40\n",
      "          87       0.44      0.10      0.16        40\n",
      "          88       0.36      0.20      0.26        40\n",
      "          89       0.33      0.05      0.09        40\n",
      "          90       0.16      0.07      0.10        40\n",
      "          91       0.47      0.17      0.25        40\n",
      "          92       0.00      0.00      0.00        40\n",
      "          93       0.43      0.15      0.22        40\n",
      "          94       0.00      0.00      0.00        40\n",
      "\n",
      "    accuracy                           0.22      4400\n",
      "   macro avg       0.28      0.14      0.18      4400\n",
      "weighted avg       0.27      0.22      0.19      4400\n",
      "\n",
      "\n",
      "Training SVM...\n",
      "SVM Performance:\n",
      "Accuracy: 0.13636363636363635\n",
      "F1 Score: 0.03272727272727273\n",
      "Classification Report:\n",
      "               precision    recall  f1-score   support\n",
      "\n",
      "          -1       0.14      1.00      0.24       600\n",
      "           0       0.00      0.00      0.00        40\n",
      "           1       0.00      0.00      0.00        40\n",
      "           2       0.00      0.00      0.00        40\n",
      "           3       0.00      0.00      0.00        40\n",
      "           4       0.00      0.00      0.00        40\n",
      "           5       0.00      0.00      0.00        40\n",
      "           6       0.00      0.00      0.00        40\n",
      "           7       0.00      0.00      0.00        40\n",
      "           8       0.00      0.00      0.00        40\n",
      "           9       0.00      0.00      0.00        40\n",
      "          10       0.00      0.00      0.00        40\n",
      "          11       0.00      0.00      0.00        40\n",
      "          12       0.00      0.00      0.00        40\n",
      "          13       0.00      0.00      0.00        40\n",
      "          14       0.00      0.00      0.00        40\n",
      "          15       0.00      0.00      0.00        40\n",
      "          16       0.00      0.00      0.00        40\n",
      "          17       0.00      0.00      0.00        40\n",
      "          18       0.00      0.00      0.00        40\n",
      "          19       0.00      0.00      0.00        40\n",
      "          20       0.00      0.00      0.00        40\n",
      "          21       0.00      0.00      0.00        40\n",
      "          22       0.00      0.00      0.00        40\n",
      "          23       0.00      0.00      0.00        40\n",
      "          24       0.00      0.00      0.00        40\n",
      "          25       0.00      0.00      0.00        40\n",
      "          26       0.00      0.00      0.00        40\n",
      "          27       0.00      0.00      0.00        40\n",
      "          28       0.00      0.00      0.00        40\n",
      "          29       0.00      0.00      0.00        40\n",
      "          30       0.00      0.00      0.00        40\n",
      "          31       0.00      0.00      0.00        40\n",
      "          32       0.00      0.00      0.00        40\n",
      "          33       0.00      0.00      0.00        40\n",
      "          34       0.00      0.00      0.00        40\n",
      "          35       0.00      0.00      0.00        40\n",
      "          36       0.00      0.00      0.00        40\n",
      "          37       0.00      0.00      0.00        40\n",
      "          38       0.00      0.00      0.00        40\n",
      "          39       0.00      0.00      0.00        40\n",
      "          40       0.00      0.00      0.00        40\n",
      "          41       0.00      0.00      0.00        40\n",
      "          42       0.00      0.00      0.00        40\n",
      "          43       0.00      0.00      0.00        40\n",
      "          44       0.00      0.00      0.00        40\n",
      "          45       0.00      0.00      0.00        40\n",
      "          46       0.00      0.00      0.00        40\n",
      "          47       0.00      0.00      0.00        40\n",
      "          48       0.00      0.00      0.00        40\n",
      "          49       0.00      0.00      0.00        40\n",
      "          50       0.00      0.00      0.00        40\n",
      "          51       0.00      0.00      0.00        40\n",
      "          52       0.00      0.00      0.00        40\n",
      "          53       0.00      0.00      0.00        40\n",
      "          54       0.00      0.00      0.00        40\n",
      "          55       0.00      0.00      0.00        40\n",
      "          56       0.00      0.00      0.00        40\n",
      "          57       0.00      0.00      0.00        40\n",
      "          58       0.00      0.00      0.00        40\n",
      "          59       0.00      0.00      0.00        40\n",
      "          60       0.00      0.00      0.00        40\n",
      "          61       0.00      0.00      0.00        40\n",
      "          62       0.00      0.00      0.00        40\n",
      "          63       0.00      0.00      0.00        40\n",
      "          64       0.00      0.00      0.00        40\n",
      "          65       0.00      0.00      0.00        40\n",
      "          66       0.00      0.00      0.00        40\n",
      "          67       0.00      0.00      0.00        40\n",
      "          68       0.00      0.00      0.00        40\n",
      "          69       0.00      0.00      0.00        40\n",
      "          70       0.00      0.00      0.00        40\n",
      "          71       0.00      0.00      0.00        40\n",
      "          72       0.00      0.00      0.00        40\n",
      "          73       0.00      0.00      0.00        40\n",
      "          74       0.00      0.00      0.00        40\n",
      "          75       0.00      0.00      0.00        40\n",
      "          76       0.00      0.00      0.00        40\n",
      "          77       0.00      0.00      0.00        40\n",
      "          78       0.00      0.00      0.00        40\n",
      "          79       0.00      0.00      0.00        40\n",
      "          80       0.00      0.00      0.00        40\n",
      "          81       0.00      0.00      0.00        40\n",
      "          82       0.00      0.00      0.00        40\n",
      "          83       0.00      0.00      0.00        40\n",
      "          84       0.00      0.00      0.00        40\n",
      "          85       0.00      0.00      0.00        40\n",
      "          86       0.00      0.00      0.00        40\n",
      "          87       0.00      0.00      0.00        40\n",
      "          88       0.00      0.00      0.00        40\n",
      "          89       0.00      0.00      0.00        40\n",
      "          90       0.00      0.00      0.00        40\n",
      "          91       0.00      0.00      0.00        40\n",
      "          92       0.00      0.00      0.00        40\n",
      "          93       0.00      0.00      0.00        40\n",
      "          94       0.00      0.00      0.00        40\n",
      "\n",
      "    accuracy                           0.14      4400\n",
      "   macro avg       0.00      0.01      0.00      4400\n",
      "weighted avg       0.02      0.14      0.03      4400\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/sojeonglee/anaconda3/lib/python3.11/site-packages/sklearn/metrics/_classification.py:1344: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n",
      "/Users/sojeonglee/anaconda3/lib/python3.11/site-packages/sklearn/metrics/_classification.py:1344: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n",
      "/Users/sojeonglee/anaconda3/lib/python3.11/site-packages/sklearn/metrics/_classification.py:1344: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n"
     ]
    }
   ],
   "source": [
    "from sklearn.ensemble import RandomForestClassifier, GradientBoostingClassifier\n",
    "from sklearn.svm import SVC\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "from sklearn.metrics import accuracy_score, classification_report\n",
    "import xgboost as xgb\n",
    "\n",
    "# hyper-parameter tuning for random forest model\n",
    "rf = RandomForestClassifier(random_state=42)\n",
    "param_grid_rf = {\n",
    "    'n_estimators': [100, 300, 500],\n",
    "    'max_depth': [10, 20, 30],\n",
    "    'min_samples_split': [2, 5, 10],\n",
    "    'min_samples_leaf': [1, 2, 4],\n",
    "}\n",
    "\n",
    "grid_search_rf = GridSearchCV(\n",
    "    rf, param_grid_rf, cv=3, scoring='accuracy', n_jobs=-1, verbose=1\n",
    ")\n",
    "grid_search_rf.fit(X_train_balanced, y_train_balanced)\n",
    "\n",
    "# Best parameters and performance\n",
    "best_rf = grid_search_rf.best_estimator_\n",
    "print(\"\\nBest Random Forest Parameters:\", grid_search_rf.best_params_)\n",
    "y_pred_rf = best_rf.predict(X_test)\n",
    "print(\"\\nRandom Forest Performance:\")\n",
    "print(\"Accuracy:\", accuracy_score(y_test, y_pred_rf))\n",
    "print(\"F1 Score:\", f1_score(y_test, y_pred_rf, average='weighted'))\n",
    "print(\"Classification Report:\\n\", classification_report(y_test, y_pred_rf))\n",
    "\n",
    "\n",
    "# Other models' performance\n",
    "models = {\n",
    "    \"Gradient Boosting\": GradientBoostingClassifier(random_state=42),\n",
    "    #\"XGBoost\": xgb.XGBClassifier(use_label_encoder=False, eval_metric='mlogloss', random_state=42),\n",
    "    \"SVM\": SVC(probability=True, random_state=42)\n",
    "}\n",
    "\n",
    "for model_name, model in models.items():\n",
    "    print(f\"\\nTraining {model_name}...\")\n",
    "    model.fit(X_train_balanced, y_train_balanced)\n",
    "    y_pred = model.predict(X_test)\n",
    "    \n",
    "    print(f\"{model_name} Performance:\")\n",
    "    print(\"Accuracy:\", accuracy_score(y_test, y_pred))\n",
    "    print(\"F1 Score:\", f1_score(y_test, y_pred, average='weighted'))\n",
    "    print(\"Classification Report:\\n\", classification_report(y_test, y_pred))"
   ]
  }
 ],
 "metadata": {
  "colab": {
   "provenance": []
  },
  "gpuClass": "standard",
  "kernelspec": {
   "display_name": "myenv",
   "language": "python",
   "name": "myenv"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}