Part16_ClassificationAfterSampling_DL.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "2aef2ab6",
   "metadata": {},
   "source": [
    "# Classification after Sampling Part 1 (LSTM, RNN, GRU, 1D-CNN) on Concatenated Multivariate Time Series Data (2D)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ea62f006",
   "metadata": {},
   "source": [
    "### Evaluation Metrics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "78559b58",
   "metadata": {},
   "outputs": [],
   "source": [
    "# float(\"{:.2f}\".format(13.949999999999999))\n",
    "\n",
    "def TSS(TP,TN,FP,FN):\n",
    "    TSS_value = (TP / (TP + FN)) - (FP / (FP + TN))\n",
    "    return TSS_value\n",
    "\n",
    "def HSS1(TP,TN,FP,FN):\n",
    "    HSS1_value = (2 * (TP * TN - FP * FN)) / ((TP + FN) * (FN + TN) + (TP + FP) * (FP + TN))\n",
    "    return HSS1_value\n",
    "    \n",
    "def HSS2(TP,TN,FP,FN):\n",
    "    HSS2_value = (2 * (TP * TN - FP * FN)) / ((TP + FP) * (FN + TN) + (TP + FN) * (FP + TN))\n",
    "    return HSS2_value\n",
    "\n",
    "def GSS(TP,TN,FP,FN):\n",
    "    GSS_value = (TP - (TP + FP) * (TP + FN) / (TP + FP + FN + TN))\n",
    "    return GSS_value\n",
    "\n",
    "def Recall(TP,TN,FP,FN):\n",
    "    Recall_value = (TP) / (TP + FN)\n",
    "    return Recall_value\n",
    "\n",
    "def FPR(TP,TN,FP,FN):\n",
    "    fpr_value = (FP) / (FP + TN)\n",
    "    return fpr_value\n",
    "\n",
    "def Accuracy(TP,TN,FP,FN):\n",
    "    accuracy_value = (TP + TN) / (TP + TN + FP + FN)\n",
    "    return accuracy_value\n",
    "\n",
    "def Precision(TP,TN,FP,FN):\n",
    "    precision_value = (FP) / (TP + FP)\n",
    "    return precision_value"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "70d7efdb",
   "metadata": {},
   "source": [
    "### Reading the Data built by the Previous Notebooks (LSBZM Normalization and FPCKNN Imputation and Without C Class and RUS-TomekLinks-SMOTE)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "777f08e8",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "P1 Nan-Value: False\n",
      "P2 Nan-Value: False\n",
      "P3 Nan-Value: False\n",
      "P4 Nan-Value: False\n",
      "P5 Nan-Value: False\n"
     ]
    }
   ],
   "source": [
    "import pickle\n",
    "import numpy as np\n",
    "\n",
    "data_dir = \"/Users/samskanderi/Documents/Research_Project/SWANSF/code/8_3_FinalData_OUSampling_Concatenation_LSBZM_KnnImputation/\"\n",
    "X_train_concat_ZM_RTS = []\n",
    "Y_train_concat_ZM_RTS = []\n",
    "\n",
    "\n",
    "num_partitions = 5\n",
    "\n",
    "for i in range(0,num_partitions):\n",
    "    with open(data_dir + \"RUS_Tomek_Smote_\" +\"Partition\" + str(i+1) + \"_WithoutC_Concatenation_LSBZM_KnnImputation\" +\".pkl\", 'rb') as f:\n",
    "        X_train_concat_ZM_RTS.append(pickle.load(f))\n",
    "    with open(data_dir + \"RUS_Tomek_Smote_\" + \"Partition\" + str(i+1) + \"_Labels_WithoutC_Concatenation_LSBZM_KnnImputation\" +\".pkl\", 'rb') as f:\n",
    "        Y_train_concat_ZM_RTS.append(pickle.load(f))\n",
    "    print(\"P\"+str(i+1)+\" Nan-Value: \"+ str(np.isnan(X_train_concat_ZM_RTS[i]).any() or np.isinf(X_train_concat_ZM_RTS[i]).any()))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "17bfcf0e",
   "metadata": {},
   "source": [
    "### Reading the Data built by the Previous Notebooks (LSBZM Normalization and FPCKNN Imputation and Without C Class and RUS-TomekLinks-ADASYN)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "013c9a76",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "P1 Nan-Value: False\n",
      "P2 Nan-Value: False\n",
      "P3 Nan-Value: False\n",
      "P4 Nan-Value: False\n",
      "P5 Nan-Value: False\n"
     ]
    }
   ],
   "source": [
    "import pickle\n",
    "import numpy as np\n",
    "\n",
    "data_dir = \"/Users/samskanderi/Documents/Research_Project/SWANSF/code/8_3_FinalData_OUSampling_Concatenation_LSBZM_KnnImputation/\"\n",
    "X_train_concat_ZM_RTA = []\n",
    "Y_train_concat_ZM_RTA = []\n",
    "\n",
    "\n",
    "num_partitions = 5\n",
    "\n",
    "for i in range(0,num_partitions):\n",
    "    with open(data_dir + \"RUS_Tomek_Adasyn_\" +\"Partition\" + str(i+1) + \"_WithoutC_Concatenation_LSBZM_KnnImputation\" +\".pkl\", 'rb') as f:\n",
    "        X_train_concat_ZM_RTA.append(pickle.load(f))\n",
    "    with open(data_dir + \"RUS_Tomek_Adasyn_\" + \"Partition\" + str(i+1) + \"_Labels_WithoutC_Concatenation_LSBZM_KnnImputation\" +\".pkl\", 'rb') as f:\n",
    "        Y_train_concat_ZM_RTA.append(pickle.load(f))\n",
    "    print(\"P\"+str(i+1)+\" Nan-Value: \"+ str(np.isnan(X_train_concat_ZM_RTA[i]).any() or np.isinf(X_train_concat_ZM_RTA[i]).any()))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "cd16e3c3",
   "metadata": {},
   "source": [
    "### Reading the Data built by the Previous Notebooks (LSBZM Normalization and FPCKNN Imputation and Without C Class and SMOTE)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "d53204d0",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "P1 Nan-Value: False\n",
      "P2 Nan-Value: False\n",
      "P3 Nan-Value: False\n",
      "P4 Nan-Value: False\n",
      "P5 Nan-Value: False\n"
     ]
    }
   ],
   "source": [
    "import pickle\n",
    "import numpy as np\n",
    "\n",
    "data_dir = \"/Users/samskanderi/Documents/Research_Project/SWANSF/code/8_4_FinalData_OSampling_Concatenation_LSBZM_KnnImputation/\"\n",
    "X_train_concat_ZM_S = []\n",
    "Y_train_concat_ZM_S = []\n",
    "\n",
    "\n",
    "num_partitions = 5\n",
    "\n",
    "for i in range(0,num_partitions):\n",
    "    with open(data_dir + \"Smote_\" +\"Partition\" + str(i+1) + \"_OSampling_WithoutC_Concatenation_LSBZM_KnnImputation\" +\".pkl\", 'rb') as f:\n",
    "        X_train_concat_ZM_S.append(pickle.load(f))\n",
    "    with open(data_dir + \"Smote_\" + \"Partition\" + str(i+1) + \"_Labels_OSampling_WithoutC_Concatenation_LSBZM_KnnImputation\" +\".pkl\", 'rb') as f:\n",
    "        Y_train_concat_ZM_S.append(pickle.load(f))\n",
    "    print(\"P\"+str(i+1)+\" Nan-Value: \"+ str(np.isnan(X_train_concat_ZM_S[i]).any() or np.isinf(X_train_concat_ZM_S[i]).any()))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c2fce791",
   "metadata": {},
   "source": [
    "### Reading the Data built by the Previous Notebooks (LSBZM Normalization and FPCKNN Imputation and Without C Class and ADASYN)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "c08af0de",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "P1 Nan-Value: False\n",
      "P2 Nan-Value: False\n",
      "P3 Nan-Value: False\n",
      "P4 Nan-Value: False\n",
      "P5 Nan-Value: False\n"
     ]
    }
   ],
   "source": [
    "import pickle\n",
    "import numpy as np\n",
    "\n",
    "data_dir = \"/Users/samskanderi/Documents/Research_Project/SWANSF/code/8_4_FinalData_OSampling_Concatenation_LSBZM_KnnImputation/\"\n",
    "X_train_concat_ZM_A = []\n",
    "Y_train_concat_ZM_A = []\n",
    "\n",
    "\n",
    "num_partitions = 5\n",
    "\n",
    "for i in range(0,num_partitions):\n",
    "    with open(data_dir + \"Adasyn_\" +\"Partition\" + str(i+1) + \"_OSampling_WithoutC_Concatenation_LSBZM_KnnImputation\" +\".pkl\", 'rb') as f:\n",
    "        X_train_concat_ZM_A.append(pickle.load(f))\n",
    "    with open(data_dir + \"Adasyn_\" + \"Partition\" + str(i+1) + \"_Labels_OSampling_WithoutC_Concatenation_LSBZM_KnnImputation\" +\".pkl\", 'rb') as f:\n",
    "        Y_train_concat_ZM_A.append(pickle.load(f))\n",
    "    print(\"P\"+str(i+1)+\" Nan-Value: \"+ str(np.isnan(X_train_concat_ZM_A[i]).any() or np.isinf(X_train_concat_ZM_A[i]).any()))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "fad7a417",
   "metadata": {},
   "source": [
    "### Test Data- Reading the Data built by the Previous Notebooks (LSBZM Normalization and FPCKNN Imputation)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "f80dc488",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "P1 Nan-Value: False\n",
      "P2 Nan-Value: False\n",
      "P3 Nan-Value: False\n",
      "P4 Nan-Value: False\n",
      "P5 Nan-Value: False\n"
     ]
    }
   ],
   "source": [
    "import pickle\n",
    "import numpy as np\n",
    "\n",
    "data_dir = \"/Users/samskanderi/Documents/Research_Project/SWANSF/code/5_4_FinalData_Concatenation_LSBZM_KnnImputation/\"\n",
    "X_test_concat_ZM = []\n",
    "Y_test_concat_ZM = []\n",
    "\n",
    "\n",
    "num_partitions = 5\n",
    "\n",
    "for i in range(0,num_partitions):\n",
    "    with open(data_dir + \"Partition\" + str(i+1) + \"_Concatenation_LSBZM_KnnImputation\" + \".pkl\", 'rb') as f:\n",
    "        X_test_concat_ZM.append(pickle.load(f))\n",
    "    print(\"P\"+str(i+1)+\" Nan-Value: \"+ str(np.isnan(X_test_concat_ZM[i]).any() or np.isinf(X_test_concat_ZM[i]).any()))\n",
    "    \n",
    "    with open(data_dir + \"Partition\" + str(i+1) + \"_Labels_Concatenation_LSBZM_KnnImputation\" + \".pkl\", 'rb') as f:\n",
    "        Y_test_concat_ZM.append(pickle.load(f))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e879937b",
   "metadata": {},
   "source": [
    "### Function to run the classification on different Train-Test Splits"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "69a74f1d",
   "metadata": {},
   "outputs": [],
   "source": [
    "def kfold_training(name, X_train, Y_train, X_test, Y_test, training_func, num):\n",
    "    kfold = np.array([[1,2],[2,3],[3,4],[4,5]])\n",
    "\n",
    "    metrics = []\n",
    "    metrics_values = np.array([])\n",
    "    \n",
    "    for i in range(0, num):\n",
    "        train_index = kfold[i,0]\n",
    "        test_index = kfold[i,1]\n",
    "        metrics_values = training_func(X_train[train_index-1], Y_train[train_index-1], X_test[test_index-1], Y_test[test_index-1])\n",
    "        while (metrics_values[4] < 0.001):\n",
    "            metrics_values = training_func(X_train[train_index-1], Y_train[train_index-1], X_test[test_index-1], Y_test[test_index-1])\n",
    "        metrics.append(np.append(np.append(train_index, test_index), metrics_values))\n",
    "    return metrics"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "56a024ab",
   "metadata": {},
   "source": [
    "## Code of the Classification techniques"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b13583cb",
   "metadata": {},
   "source": [
    "### Rocket (RidgeClassifierCV) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "8a7ff942",
   "metadata": {},
   "outputs": [],
   "source": [
    "# ROCKET with RidgeClassifierCV\n",
    "\n",
    "import numpy as np\n",
    "from sklearn.linear_model import RidgeClassifierCV\n",
    "from sktime.transformations.panel.rocket import Rocket\n",
    "from sklearn.metrics import confusion_matrix\n",
    "import joblib\n",
    "from sktime.datatypes._panel._convert import from_2d_array_to_nested\n",
    "\n",
    "def rocket_model(X_train, Y_train, X_test, Y_test, rocket_kernels=1500):\n",
    "\n",
    "    data_dir = \"/Users/samskanderi/Documents/Research_Project/SWANSF/code/models/rocket/\"\n",
    "    \n",
    "    X_train = from_2d_array_to_nested(X_train)\n",
    "    X_test = from_2d_array_to_nested(X_test)\n",
    "\n",
    "    rocket = Rocket(num_kernels=rocket_kernels)\n",
    "    rocket.fit(X_train)\n",
    "    X_train_transform = rocket.transform(X_train)\n",
    "        \n",
    "    classifier = RidgeClassifierCV()\n",
    "    classifier.fit(X_train_transform, Y_train)\n",
    "    \n",
    "    X_test_transform = rocket.transform(X_test)\n",
    "    \n",
    "    y_pred = classifier.predict(X_test_transform)\n",
    "    \n",
    "    print(str(X_train.shape)+': Rocket Classifier is Done! \\n')\n",
    "\n",
    "    confusion = confusion_matrix(Y_test, y_pred)\n",
    "    tn, fp, fn, tp = confusion.ravel()\n",
    "\n",
    "    tss = TSS(tp,tn,fp,fn)\n",
    "    hss1 = HSS1(tp,tn,fp,fn)\n",
    "    hss2 = HSS2(tp,tn,fp,fn)\n",
    "    gss = GSS(tp,tn,fp,fn)\n",
    "    recall = Recall(tp,tn,fp,fn)\n",
    "    precision = Precision(tp,tn,fp,fn)\n",
    "    \n",
    "    output_values = np.array([tp, fn, fp, tn, tss, hss1, hss2, gss, recall, precision])\n",
    "\n",
    "\n",
    "    #joblib.dump(classifier, data_dir + \"rocket_model.pkl\")\n",
    "\n",
    "    #loaded_rocket_model = joblib.load(data_dir + \"rocket_model_sgd.pkl\")\n",
    "    \n",
    "    return output_values"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8840de35",
   "metadata": {},
   "source": [
    "### TimeSeriesForest"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "06c0fef8",
   "metadata": {},
   "outputs": [],
   "source": [
    "# TimeSeriesForest\n",
    "\n",
    "from sktime.classification.interval_based import TimeSeriesForestClassifier\n",
    "import numpy as np\n",
    "from sklearn.metrics import confusion_matrix\n",
    "import joblib\n",
    "\n",
    "def tsf_model(X_train, Y_train, X_test, Y_test, tsf_estimator=60):\n",
    "\n",
    "    data_dir = \"/Users/samskanderi/Documents/Research_Project/SWANSF/code/models/TSF/\"\n",
    "    \n",
    "    \n",
    "    tsf_classifier = TimeSeriesForestClassifier(n_estimators=tsf_estimator)\n",
    "    tsf_classifier.fit(X_train, Y_train)\n",
    "    y_pred = tsf_classifier.predict(X_test)\n",
    "    \n",
    "    print(str(X_train.shape)+': TSF Classifier is Done! \\n')\n",
    "    \n",
    "\n",
    "    confusion = confusion_matrix(Y_test, y_pred)\n",
    "    tn, fp, fn, tp = confusion.ravel()\n",
    "\n",
    "    tss = TSS(tp,tn,fp,fn)\n",
    "    hss1 = HSS1(tp,tn,fp,fn)\n",
    "    hss2 = HSS2(tp,tn,fp,fn)\n",
    "    gss = GSS(tp,tn,fp,fn)\n",
    "    recall = Recall(tp,tn,fp,fn)\n",
    "    precision = Precision(tp,tn,fp,fn)\n",
    "    \n",
    "    output_values = np.array([tp, fn, fp, tn, tss, hss1, hss2, gss, recall, precision])\n",
    "\n",
    "\n",
    "    #joblib.dump(classifier, data_dir + \"tsf_model.pkl\")\n",
    "\n",
    "    #loaded_rocket_model = joblib.load(data_dir + \"tsf_model.pkl\")\n",
    "    \n",
    "    return output_values"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "18eb4d25",
   "metadata": {},
   "source": [
    "### LSTM"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "92a02e8f",
   "metadata": {},
   "outputs": [],
   "source": [
    "import warnings\n",
    "warnings.filterwarnings(\"ignore\")\n",
    "\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import keras\n",
    "from keras.models import Sequential\n",
    "from keras.layers import LSTM, Dense, Dropout\n",
    "from sklearn.metrics import confusion_matrix\n",
    "\n",
    "def lstm_model(X_train, Y_train, X_test, Y_test):\n",
    "    \n",
    "    data_dir = \"/Users/samskanderi/Documents/Research_Project/SWANSF/code/models/\"\n",
    "    \n",
    "    verbose, epochs, batch_size = 0, 15, 32\n",
    "    n_timesteps, n_features = 60, 24\n",
    "\n",
    "    model = Sequential()\n",
    "    model.add(LSTM(120, input_shape=(n_timesteps,n_features)))\n",
    "    model.add(Dropout(0.3))\n",
    "    model.add(Dense(120, activation='relu'))\n",
    "    model.add(Dense(2, activation='relu'))\n",
    "    model.add(Dense(1, activation='sigmoid'))\n",
    "    model.compile(loss='binary_crossentropy', optimizer='adam', metrics=keras.metrics.SpecificityAtSensitivity(sensitivity=0.95))\n",
    "    \n",
    "    model.fit(X_train, Y_train, epochs=epochs, batch_size=batch_size, verbose=verbose)\n",
    "    \n",
    "    best_threshold = 0.0\n",
    "    best_tss = 0.0\n",
    "    y_pred = model.predict(X_test)\n",
    "    # evaluate model\n",
    "    for i in range(1, 1000):\n",
    "\n",
    "        threshold = i / 1000 # Adjust the threshold as needed\n",
    "        y_pred_binary = (y_pred > threshold).astype(int)\n",
    "        confusion = confusion_matrix(Y_test, y_pred_binary)\n",
    "        tn, fp, fn, tp = confusion.ravel()\n",
    "        tss = TSS(tp,tn,fp,fn)\n",
    "        if tss > best_tss:\n",
    "            best_tss = tss\n",
    "            best_threshold = i / 1000\n",
    "        \n",
    "    \n",
    "    print(str(X_train.shape)+': LSTM Classifier is Done! \\n')\n",
    "\n",
    "\n",
    "    threshold = best_threshold # Adjust the threshold as needed\n",
    "    y_pred_binary = (y_pred > threshold).astype(int)\n",
    "    confusion = confusion_matrix(Y_test, y_pred_binary)\n",
    "    tn, fp, fn, tp = confusion.ravel()\n",
    "\n",
    "    tss = TSS(tp,tn,fp,fn)\n",
    "    hss1 = HSS1(tp,tn,fp,fn)\n",
    "    hss2 = HSS2(tp,tn,fp,fn)\n",
    "    gss = GSS(tp,tn,fp,fn)\n",
    "    recall = Recall(tp,tn,fp,fn)\n",
    "    precision = Precision(tp,tn,fp,fn)\n",
    "    \n",
    "    output_values = np.array([tp, fn, fp, tn, tss, hss1, hss2, gss, recall, precision])\n",
    "\n",
    "\n",
    "    #joblib.dump(classifier, data_dir + \"mlp_model.pkl\")\n",
    "\n",
    "    #loaded_mlp_model = joblib.load(data_dir + \"mlp_model.pkl\")\n",
    "    \n",
    "    return output_values"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "694e693a",
   "metadata": {},
   "source": [
    "### 1D-CNN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "265014e8",
   "metadata": {},
   "outputs": [],
   "source": [
    "import warnings\n",
    "warnings.filterwarnings(\"ignore\")\n",
    "\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import keras\n",
    "from keras.models import Sequential\n",
    "from keras.layers import Conv1D, MaxPooling1D, Flatten, Dense, Dropout\n",
    "from sklearn.metrics import confusion_matrix\n",
    "\n",
    "def cnn_model(X_train, Y_train, X_test, Y_test):\n",
    "    \n",
    "    data_dir = \"/Users/samskanderi/Documents/Research_Project/SWANSF/code/models/\"\n",
    "    \n",
    "    verbose, epochs, batch_size = 0, 15, 32\n",
    "    n_timesteps, n_features = 60, 24\n",
    "    \n",
    "    model = Sequential()\n",
    "    model.add(Conv1D(filters=64, kernel_size=5, activation='relu', input_shape=(n_timesteps,n_features)))\n",
    "    model.add(Dropout(0.2))\n",
    "    model.add(MaxPooling1D(pool_size=2))\n",
    "    model.add(Conv1D(filters=128, kernel_size=5, activation='relu'))\n",
    "    model.add(Dropout(0.3))\n",
    "    model.add(MaxPooling1D(pool_size=2))\n",
    "    model.add(Flatten())\n",
    "    model.add(Dense(357, activation='relu'))\n",
    "    model.add(Dense(128, activation='relu'))\n",
    "    model.add(Dense(2, activation='relu'))\n",
    "    model.add(Dense(1, activation='sigmoid'))\n",
    "    model.compile(loss='binary_crossentropy', optimizer='adam', metrics=keras.metrics.SpecificityAtSensitivity(sensitivity=0.95))\n",
    "    \n",
    "    model.fit(X_train, Y_train, epochs=epochs, batch_size=batch_size, verbose=verbose)\n",
    "    \n",
    "    best_threshold = 0.0\n",
    "    best_tss = 0.0\n",
    "    y_pred = model.predict(X_test)\n",
    "    # evaluate model\n",
    "    for i in range(1, 1000):\n",
    "\n",
    "        threshold = i / 1000 # Adjust the threshold as needed\n",
    "        y_pred_binary = (y_pred > threshold).astype(int)\n",
    "        confusion = confusion_matrix(Y_test, y_pred_binary)\n",
    "        tn, fp, fn, tp = confusion.ravel()\n",
    "        tss = TSS(tp,tn,fp,fn)\n",
    "        if tss > best_tss:\n",
    "            best_tss = tss\n",
    "            best_threshold = i / 1000\n",
    "        \n",
    "    \n",
    "    print(str(X_train.shape)+': CNN Classifier is Done! \\n')\n",
    "\n",
    "\n",
    "    threshold = best_threshold # Adjust the threshold as needed\n",
    "    y_pred_binary = (y_pred > threshold).astype(int)\n",
    "    confusion = confusion_matrix(Y_test, y_pred_binary)\n",
    "    tn, fp, fn, tp = confusion.ravel()\n",
    "\n",
    "    tss = TSS(tp,tn,fp,fn)\n",
    "    hss1 = HSS1(tp,tn,fp,fn)\n",
    "    hss2 = HSS2(tp,tn,fp,fn)\n",
    "    gss = GSS(tp,tn,fp,fn)\n",
    "    recall = Recall(tp,tn,fp,fn)\n",
    "    precision = Precision(tp,tn,fp,fn)\n",
    "    \n",
    "    output_values = np.array([tp, fn, fp, tn, tss, hss1, hss2, gss, recall, precision])\n",
    "\n",
    "\n",
    "    #joblib.dump(classifier, data_dir + \"mlp_model.pkl\")\n",
    "\n",
    "    #loaded_mlp_model = joblib.load(data_dir + \"mlp_model.pkl\")\n",
    "    \n",
    "    return output_values"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4a74348c",
   "metadata": {},
   "source": [
    "### RNN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "445052a3",
   "metadata": {},
   "outputs": [],
   "source": [
    "import warnings\n",
    "warnings.filterwarnings(\"ignore\")\n",
    "\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import keras\n",
    "from keras.models import Sequential\n",
    "from keras.layers import SimpleRNN, Dense, Dropout\n",
    "from sklearn.metrics import confusion_matrix\n",
    "\n",
    "def rnn_model(X_train, Y_train, X_test, Y_test):\n",
    "    \n",
    "    data_dir = \"/Users/samskanderi/Documents/Research_Project/SWANSF/code/models/\"\n",
    "    \n",
    "    verbose, epochs, batch_size = 0, 15, 32\n",
    "    n_timesteps, n_features = 60, 24\n",
    "    \n",
    "    model = Sequential()\n",
    "    model.add(SimpleRNN(units=120, activation='relu', input_shape=(n_timesteps,n_features)))\n",
    "    model.add(Dropout(0.3))\n",
    "    model.add(Dense(120, activation='relu'))\n",
    "    model.add(Dense(2, activation='relu'))\n",
    "    model.add(Dense(1, activation='sigmoid'))\n",
    "    model.compile(loss='binary_crossentropy', optimizer='adam', metrics=keras.metrics.SpecificityAtSensitivity(sensitivity=0.95))\n",
    "\n",
    "    best_threshold = 0.0\n",
    "    best_tss = 0.0\n",
    "    y_pred = model.predict(X_test)\n",
    "    # evaluate model\n",
    "    for i in range(1, 1000):\n",
    "\n",
    "        threshold = i / 1000 # Adjust the threshold as needed\n",
    "        y_pred_binary = (y_pred > threshold).astype(int)\n",
    "        confusion = confusion_matrix(Y_test, y_pred_binary)\n",
    "        tn, fp, fn, tp = confusion.ravel()\n",
    "        tss = TSS(tp,tn,fp,fn)\n",
    "        if tss > best_tss:\n",
    "            best_tss = tss\n",
    "            best_threshold = i / 1000\n",
    "        \n",
    "    \n",
    "    print(str(X_train.shape)+': RNN Classifier is Done! \\n')\n",
    "\n",
    "\n",
    "    threshold = best_threshold # Adjust the threshold as needed\n",
    "    y_pred_binary = (y_pred > threshold).astype(int)\n",
    "    confusion = confusion_matrix(Y_test, y_pred_binary)\n",
    "    tn, fp, fn, tp = confusion.ravel()\n",
    "\n",
    "    tss = TSS(tp,tn,fp,fn)\n",
    "    hss1 = HSS1(tp,tn,fp,fn)\n",
    "    hss2 = HSS2(tp,tn,fp,fn)\n",
    "    gss = GSS(tp,tn,fp,fn)\n",
    "    recall = Recall(tp,tn,fp,fn)\n",
    "    precision = Precision(tp,tn,fp,fn)\n",
    "    \n",
    "    output_values = np.array([tp, fn, fp, tn, tss, hss1, hss2, gss, recall, precision])\n",
    "\n",
    "\n",
    "    #joblib.dump(classifier, data_dir + \"mlp_model.pkl\")\n",
    "\n",
    "    #loaded_mlp_model = joblib.load(data_dir + \"mlp_model.pkl\")\n",
    "    \n",
    "    return output_values"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5e51f840",
   "metadata": {},
   "source": [
    "### GRU "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "12073183",
   "metadata": {},
   "outputs": [],
   "source": [
    "import warnings\n",
    "warnings.filterwarnings(\"ignore\")\n",
    "\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import keras\n",
    "from keras.models import Sequential\n",
    "from keras.layers import GRU, Dense, Dropout\n",
    "from sklearn.metrics import confusion_matrix\n",
    "\n",
    "def gru_model(X_train, Y_train, X_test, Y_test):\n",
    "    \n",
    "    data_dir = \"/Users/samskanderi/Documents/Research_Project/SWANSF/code/models/\"\n",
    "    \n",
    "    verbose, epochs, batch_size = 0, 15, 32\n",
    "    n_timesteps, n_features = 60, 24\n",
    "    \n",
    "    model = Sequential()\n",
    "    model.add(GRU(units=120, activation='relu', input_shape=(n_timesteps,n_features)))\n",
    "    model.add(Dropout(0.3))\n",
    "    model.add(Dense(120, activation='relu'))\n",
    "    model.add(Dense(2, activation='relu'))\n",
    "    model.add(Dense(1, activation='sigmoid'))\n",
    "    model.compile(loss='binary_crossentropy', optimizer='adam', metrics=keras.metrics.SpecificityAtSensitivity(sensitivity=0.95))\n",
    "\n",
    "    model.fit(X_train, Y_train, epochs=epochs, batch_size=batch_size, verbose=verbose)\n",
    "    \n",
    "    best_threshold = 0.0\n",
    "    best_tss = 0.0\n",
    "    y_pred = model.predict(X_test)\n",
    "    # evaluate model\n",
    "    for i in range(1, 1000):\n",
    "\n",
    "        threshold = i / 1000 # Adjust the threshold as needed\n",
    "        y_pred_binary = (y_pred > threshold).astype(int)\n",
    "        confusion = confusion_matrix(Y_test, y_pred_binary)\n",
    "        tn, fp, fn, tp = confusion.ravel()\n",
    "        tss = TSS(tp,tn,fp,fn)\n",
    "        if tss > best_tss:\n",
    "            best_tss = tss\n",
    "            best_threshold = i / 1000\n",
    "        \n",
    "    \n",
    "    print(str(X_train.shape)+': GRU Classifier is Done! \\n')\n",
    "\n",
    "\n",
    "    threshold = best_threshold # Adjust the threshold as needed\n",
    "    y_pred_binary = (y_pred > threshold).astype(int)\n",
    "    confusion = confusion_matrix(Y_test, y_pred_binary)\n",
    "    tn, fp, fn, tp = confusion.ravel()\n",
    "\n",
    "    tss = TSS(tp,tn,fp,fn)\n",
    "    hss1 = HSS1(tp,tn,fp,fn)\n",
    "    hss2 = HSS2(tp,tn,fp,fn)\n",
    "    gss = GSS(tp,tn,fp,fn)\n",
    "    recall = Recall(tp,tn,fp,fn)\n",
    "    precision = Precision(tp,tn,fp,fn)\n",
    "    \n",
    "    output_values = np.array([tp, fn, fp, tn, tss, hss1, hss2, gss, recall, precision])\n",
    "\n",
    "\n",
    "    #joblib.dump(classifier, data_dir + \"mlp_model.pkl\")\n",
    "\n",
    "    #loaded_mlp_model = joblib.load(data_dir + \"mlp_model.pkl\")\n",
    "    \n",
    "    return output_values"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a09b2613",
   "metadata": {},
   "source": [
    "### Saving the Result of Classification on File"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "2fae2c6d",
   "metadata": {},
   "outputs": [],
   "source": [
    "def save_results(reslut, name):\n",
    "    data_dir = \"/Users/samskanderi/Documents/Research_Project/SWANSF/code/results/\"\n",
    "\n",
    "    with open(data_dir + name + \".pkl\", 'wb') as f:\n",
    "        pickle.dump(reslut, f)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "52ccb512",
   "metadata": {},
   "source": [
    "## Running Experiments "
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f90d5e67",
   "metadata": {},
   "source": [
    "### 2D to 3D"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "ba8c56e9",
   "metadata": {},
   "outputs": [],
   "source": [
    "def the_2dto3d(X_train):\n",
    "    num_attributes = 24\n",
    "    num_partitions = 5\n",
    "    num_timestamps = 60\n",
    "    X_train_concat_ZM_3D = []\n",
    "    for i in range(0, num_partitions):\n",
    "        new_3D = np.zeros((X_train[i].shape[0], num_timestamps, num_attributes))\n",
    "\n",
    "        for j in range(0, X_train[i].shape[0]):\n",
    "            for m in range(0, num_attributes):\n",
    "                new_3D[j,:,m] = X_train[i][j,m*num_timestamps:(m+1)*num_timestamps]\n",
    "        X_train_concat_ZM_3D.append(new_3D)\n",
    "    return X_train_concat_ZM_3D"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "25643957",
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train_concat_ZM_RTS_3D = the_2dto3d(X_train_concat_ZM_RTS)\n",
    "X_train_concat_ZM_RTA_3D = the_2dto3d(X_train_concat_ZM_RTA)\n",
    "del X_train_concat_ZM_RTS\n",
    "del X_train_concat_ZM_RTA"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "cfd4349e",
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train_concat_ZM_S_3D = the_2dto3d(X_train_concat_ZM_S)\n",
    "X_train_concat_ZM_A_3D = the_2dto3d(X_train_concat_ZM_A)\n",
    "del X_train_concat_ZM_S\n",
    "del X_train_concat_ZM_A"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "570ba9f1",
   "metadata": {},
   "outputs": [],
   "source": [
    "X_test_concat_ZM_3D = the_2dto3d(X_test_concat_ZM)\n",
    "del X_test_concat_ZM"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4e814653",
   "metadata": {},
   "source": [
    "### LSTM"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "cfbf7110",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2768/2768 [==============================] - 16s 6ms/step\n",
      "(19996, 60, 24): LSTM Classifier is Done! \n",
      "\n",
      "1329/1329 [==============================] - 178s 134ms/step\n",
      "(19998, 60, 24): LSTM Classifier is Done! \n",
      "\n",
      "1602/1602 [==============================] - 10s 6ms/step\n",
      "(19994, 60, 24): LSTM Classifier is Done! \n",
      "\n",
      "1602/1602 [==============================] - 9s 6ms/step\n",
      "(19994, 60, 24): LSTM Classifier is Done! \n",
      "\n",
      "2356/2356 [==============================] - 14s 6ms/step\n",
      "(19996, 60, 24): LSTM Classifier is Done! \n",
      "\n",
      "2356/2356 [==============================] - 14s 6ms/step\n",
      "(19996, 60, 24): LSTM Classifier is Done! \n",
      "\n",
      "2356/2356 [==============================] - 14s 6ms/step\n",
      "(19996, 60, 24): LSTM Classifier is Done! \n",
      "\n"
     ]
    }
   ],
   "source": [
    "# LSTM Concatenation\n",
    "lstm_concat_RTS = kfold_training('LSTM', X_train_concat_ZM_RTS_3D, Y_train_concat_ZM_RTS, X_test_concat_ZM_3D, Y_test_concat_ZM, lstm_model, 4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "556a91a4",
   "metadata": {},
   "outputs": [],
   "source": [
    "save_results(lstm_concat_RTS, \"LSTM_RusTomekSmote_WithoutC_Concatenation_Results\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "825c966e",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2768/2768 [==============================] - 16s 6ms/step\n",
      "(20061, 60, 24): LSTM Classifier is Done! \n",
      "\n",
      "1329/1329 [==============================] - 9s 7ms/step\n",
      "(20043, 60, 24): LSTM Classifier is Done! \n",
      "\n",
      "1329/1329 [==============================] - 10s 7ms/step\n",
      "(20043, 60, 24): LSTM Classifier is Done! \n",
      "\n",
      "1329/1329 [==============================] - 8s 6ms/step\n",
      "(20043, 60, 24): LSTM Classifier is Done! \n",
      "\n",
      "1602/1602 [==============================] - 12s 7ms/step\n",
      "(19955, 60, 24): LSTM Classifier is Done! \n",
      "\n",
      "1602/1602 [==============================] - 12s 8ms/step\n",
      "(19955, 60, 24): LSTM Classifier is Done! \n",
      "\n",
      "2356/2356 [==============================] - 17s 7ms/step\n",
      "(19981, 60, 24): LSTM Classifier is Done! \n",
      "\n"
     ]
    }
   ],
   "source": [
    "lstm_concat_RTA = kfold_training('LSTM', X_train_concat_ZM_RTA_3D, Y_train_concat_ZM_RTA, X_test_concat_ZM_3D, Y_test_concat_ZM, lstm_model, 4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "95caa983",
   "metadata": {},
   "outputs": [],
   "source": [
    "save_results(lstm_concat_RTA, \"LSTM_RusTomekAdasyn_WithoutC_Concatenation_Results\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "54737735",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2768/2768 [==============================] - 22s 8ms/step\n",
      "(131644, 60, 24): LSTM Classifier is Done! \n",
      "\n",
      "1329/1329 [==============================] - 11s 8ms/step\n",
      "(156692, 60, 24): LSTM Classifier is Done! \n",
      "\n",
      "1602/1602 [==============================] - 14s 9ms/step\n",
      "(70894, 60, 24): LSTM Classifier is Done! \n",
      "\n",
      "2356/2356 [==============================] - 18s 8ms/step\n",
      "(88280, 60, 24): LSTM Classifier is Done! \n",
      "\n",
      "2356/2356 [==============================] - 17s 7ms/step\n",
      "(88280, 60, 24): LSTM Classifier is Done! \n",
      "\n"
     ]
    }
   ],
   "source": [
    "lstm_concat_S = kfold_training('LSTM', X_train_concat_ZM_S_3D, Y_train_concat_ZM_S, X_test_concat_ZM_3D, Y_test_concat_ZM, lstm_model, 4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "da6e53b1",
   "metadata": {},
   "outputs": [],
   "source": [
    "save_results(lstm_concat_S, \"LSTM_Smote_WithoutC_Concatenation_Results\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "44495efd",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2768/2768 [==============================] - 20s 7ms/step\n",
      "(131540, 60, 24): LSTM Classifier is Done! \n",
      "\n",
      "1329/1329 [==============================] - 9s 7ms/step\n",
      "(156622, 60, 24): LSTM Classifier is Done! \n",
      "\n",
      "1602/1602 [==============================] - 10s 6ms/step\n",
      "(70907, 60, 24): LSTM Classifier is Done! \n",
      "\n",
      "2356/2356 [==============================] - 12s 5ms/step\n",
      "(88334, 60, 24): LSTM Classifier is Done! \n",
      "\n"
     ]
    }
   ],
   "source": [
    "lstm_concat_A = kfold_training('LSTM', X_train_concat_ZM_A_3D, Y_train_concat_ZM_A, X_test_concat_ZM_3D, Y_test_concat_ZM, lstm_model, 4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "fb3431e7",
   "metadata": {},
   "outputs": [],
   "source": [
    "save_results(lstm_concat_A, \"LSTM_Adasyn_WithoutC_Concatenation_Results\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "cba990f3",
   "metadata": {},
   "source": [
    "### 1D-CNN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "95bd7e1a",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2768/2768 [==============================] - 3s 1ms/step\n",
      "(19996, 60, 24): CNN Classifier is Done! \n",
      "\n",
      "2768/2768 [==============================] - 3s 1ms/step\n",
      "(19996, 60, 24): CNN Classifier is Done! \n",
      "\n",
      "1329/1329 [==============================] - 1s 1ms/step\n",
      "(19998, 60, 24): CNN Classifier is Done! \n",
      "\n",
      "1602/1602 [==============================] - 2s 1ms/step\n",
      "(19994, 60, 24): CNN Classifier is Done! \n",
      "\n",
      "1602/1602 [==============================] - 2s 1ms/step\n",
      "(19994, 60, 24): CNN Classifier is Done! \n",
      "\n",
      "1602/1602 [==============================] - 2s 1ms/step\n",
      "(19994, 60, 24): CNN Classifier is Done! \n",
      "\n",
      "2356/2356 [==============================] - 3s 1ms/step\n",
      "(19996, 60, 24): CNN Classifier is Done! \n",
      "\n"
     ]
    }
   ],
   "source": [
    "cnn_concat_RTS = kfold_training('1DCNN', X_train_concat_ZM_RTS_3D, Y_train_concat_ZM_RTS, X_test_concat_ZM_3D, Y_test_concat_ZM, cnn_model, 4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "81b80bad",
   "metadata": {},
   "outputs": [],
   "source": [
    "save_results(cnn_concat_RTS, \"1DCNN_RusTomekSmote_WithoutC_Concatenation_Results\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "a7e6ae32",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2768/2768 [==============================] - 3s 1ms/step\n",
      "(20061, 60, 24): CNN Classifier is Done! \n",
      "\n",
      "2768/2768 [==============================] - 3s 989us/step\n",
      "(20061, 60, 24): CNN Classifier is Done! \n",
      "\n",
      "2768/2768 [==============================] - 3s 1ms/step\n",
      "(20061, 60, 24): CNN Classifier is Done! \n",
      "\n",
      "2768/2768 [==============================] - 3s 1ms/step\n",
      "(20061, 60, 24): CNN Classifier is Done! \n",
      "\n",
      "1329/1329 [==============================] - 2s 1ms/step\n",
      "(20043, 60, 24): CNN Classifier is Done! \n",
      "\n",
      "1329/1329 [==============================] - 1s 1ms/step\n",
      "(20043, 60, 24): CNN Classifier is Done! \n",
      "\n",
      "1329/1329 [==============================] - 1s 1ms/step\n",
      "(20043, 60, 24): CNN Classifier is Done! \n",
      "\n",
      "1329/1329 [==============================] - 2s 1ms/step\n",
      "(20043, 60, 24): CNN Classifier is Done! \n",
      "\n",
      "1329/1329 [==============================] - 2s 1ms/step\n",
      "(20043, 60, 24): CNN Classifier is Done! \n",
      "\n",
      "1602/1602 [==============================] - 2s 1ms/step\n",
      "(19955, 60, 24): CNN Classifier is Done! \n",
      "\n",
      "1602/1602 [==============================] - 2s 1ms/step\n",
      "(19955, 60, 24): CNN Classifier is Done! \n",
      "\n",
      "2356/2356 [==============================] - 3s 1ms/step\n",
      "(19981, 60, 24): CNN Classifier is Done! \n",
      "\n"
     ]
    }
   ],
   "source": [
    "cnn_concat_RTA = kfold_training('1DCNN', X_train_concat_ZM_RTA_3D, Y_train_concat_ZM_RTA, X_test_concat_ZM_3D, Y_test_concat_ZM, cnn_model, 4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "6fe620ea",
   "metadata": {},
   "outputs": [],
   "source": [
    "save_results(cnn_concat_RTA, \"1DCNN_RusTomekAdasyn_WithoutC_Concatenation_Results\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "f2f25369",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2768/2768 [==============================] - 3s 1ms/step\n",
      "(131644, 60, 24): CNN Classifier is Done! \n",
      "\n",
      "2768/2768 [==============================] - 3s 1ms/step\n",
      "(131644, 60, 24): CNN Classifier is Done! \n",
      "\n",
      "1329/1329 [==============================] - 1s 1ms/step\n",
      "(156692, 60, 24): CNN Classifier is Done! \n",
      "\n",
      "1602/1602 [==============================] - 2s 981us/step\n",
      "(70894, 60, 24): CNN Classifier is Done! \n",
      "\n",
      "2356/2356 [==============================] - 2s 958us/step\n",
      "(88280, 60, 24): CNN Classifier is Done! \n",
      "\n"
     ]
    }
   ],
   "source": [
    "cnn_concat_S = kfold_training('1DCNN', X_train_concat_ZM_S_3D, Y_train_concat_ZM_S, X_test_concat_ZM_3D, Y_test_concat_ZM, cnn_model, 4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "136a17eb",
   "metadata": {},
   "outputs": [],
   "source": [
    "save_results(cnn_concat_S, \"1DCNN_Smote_WithoutC_Concatenation_Results\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "4776beaa",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2768/2768 [==============================] - 3s 939us/step\n",
      "(131540, 60, 24): CNN Classifier is Done! \n",
      "\n",
      "2768/2768 [==============================] - 3s 979us/step\n",
      "(131540, 60, 24): CNN Classifier is Done! \n",
      "\n",
      "1329/1329 [==============================] - 1s 1ms/step\n",
      "(156622, 60, 24): CNN Classifier is Done! \n",
      "\n",
      "1602/1602 [==============================] - 2s 968us/step\n",
      "(70907, 60, 24): CNN Classifier is Done! \n",
      "\n",
      "1602/1602 [==============================] - 2s 1ms/step\n",
      "(70907, 60, 24): CNN Classifier is Done! \n",
      "\n",
      "2356/2356 [==============================] - 2s 1ms/step\n",
      "(88334, 60, 24): CNN Classifier is Done! \n",
      "\n"
     ]
    }
   ],
   "source": [
    "cnn_concat_A = kfold_training('1DCNN', X_train_concat_ZM_A_3D, Y_train_concat_ZM_A, X_test_concat_ZM_3D, Y_test_concat_ZM, cnn_model, 4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "098bc9e9",
   "metadata": {},
   "outputs": [],
   "source": [
    "save_results(cnn_concat_A, \"1DCNN_Adasyn_WithoutC_Concatenation_Results\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2bd7cde9",
   "metadata": {},
   "source": [
    "### RNN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "a8922bad",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2768/2768 [==============================] - 4s 2ms/step\n",
      "(19996, 60, 24): RNN Classifier is Done! \n",
      "\n",
      "2768/2768 [==============================] - 4s 2ms/step\n",
      "(19996, 60, 24): RNN Classifier is Done! \n",
      "\n",
      "2768/2768 [==============================] - 4s 1ms/step\n",
      "(19996, 60, 24): RNN Classifier is Done! \n",
      "\n",
      "2768/2768 [==============================] - 4s 1ms/step\n",
      "(19996, 60, 24): RNN Classifier is Done! \n",
      "\n",
      "2768/2768 [==============================] - 4s 1ms/step\n",
      "(19996, 60, 24): RNN Classifier is Done! \n",
      "\n",
      "2768/2768 [==============================] - 4s 1ms/step\n",
      "(19996, 60, 24): RNN Classifier is Done! \n",
      "\n",
      "2768/2768 [==============================] - 4s 2ms/step\n",
      "(19996, 60, 24): RNN Classifier is Done! \n",
      "\n",
      "2768/2768 [==============================] - 5s 2ms/step\n",
      "(19996, 60, 24): RNN Classifier is Done! \n",
      "\n",
      "2768/2768 [==============================] - 4s 1ms/step\n",
      "(19996, 60, 24): RNN Classifier is Done! \n",
      "\n",
      "2768/2768 [==============================] - 4s 1ms/step\n",
      "(19996, 60, 24): RNN Classifier is Done! \n",
      "\n",
      "2768/2768 [==============================] - 4s 1ms/step\n",
      "(19996, 60, 24): RNN Classifier is Done! \n",
      "\n",
      "2768/2768 [==============================] - 4s 1ms/step\n",
      "(19996, 60, 24): RNN Classifier is Done! \n",
      "\n",
      "1329/1329 [==============================] - 2s 2ms/step\n",
      "(19998, 60, 24): RNN Classifier is Done! \n",
      "\n",
      "1602/1602 [==============================] - 2s 1ms/step\n",
      "(19994, 60, 24): RNN Classifier is Done! \n",
      "\n",
      "2356/2356 [==============================] - 4s 2ms/step\n",
      "(19996, 60, 24): RNN Classifier is Done! \n",
      "\n",
      "2356/2356 [==============================] - 4s 1ms/step\n",
      "(19996, 60, 24): RNN Classifier is Done! \n",
      "\n",
      "2356/2356 [==============================] - 4s 1ms/step\n",
      "(19996, 60, 24): RNN Classifier is Done! \n",
      "\n",
      "2356/2356 [==============================] - 4s 1ms/step\n",
      "(19996, 60, 24): RNN Classifier is Done! \n",
      "\n"
     ]
    }
   ],
   "source": [
    "rnn_concat_RTS = kfold_training('RNN', X_train_concat_ZM_RTS_3D, Y_train_concat_ZM_RTS, X_test_concat_ZM_3D, Y_test_concat_ZM, rnn_model, 4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "cd386aba",
   "metadata": {},
   "outputs": [],
   "source": [
    "save_results(rnn_concat_RTS, \"RNN_RusTomekSmote_WithoutC_Concatenation_Results\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "5ca92d3a",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2768/2768 [==============================] - 4s 2ms/step\n",
      "(20061, 60, 24): RNN Classifier is Done! \n",
      "\n",
      "2768/2768 [==============================] - 4s 2ms/step\n",
      "(20061, 60, 24): RNN Classifier is Done! \n",
      "\n",
      "2768/2768 [==============================] - 4s 1ms/step\n",
      "(20061, 60, 24): RNN Classifier is Done! \n",
      "\n",
      "2768/2768 [==============================] - 4s 2ms/step\n",
      "(20061, 60, 24): RNN Classifier is Done! \n",
      "\n",
      "2768/2768 [==============================] - 4s 1ms/step\n",
      "(20061, 60, 24): RNN Classifier is Done! \n",
      "\n",
      "1329/1329 [==============================] - 2s 1ms/step\n",
      "(20043, 60, 24): RNN Classifier is Done! \n",
      "\n",
      "1602/1602 [==============================] - 2s 2ms/step\n",
      "(19955, 60, 24): RNN Classifier is Done! \n",
      "\n",
      "2356/2356 [==============================] - 4s 1ms/step\n",
      "(19981, 60, 24): RNN Classifier is Done! \n",
      "\n",
      "2356/2356 [==============================] - 4s 2ms/step\n",
      "(19981, 60, 24): RNN Classifier is Done! \n",
      "\n"
     ]
    }
   ],
   "source": [
    "rnn_concat_RTA = kfold_training('RNN', X_train_concat_ZM_RTA_3D, Y_train_concat_ZM_RTA, X_test_concat_ZM_3D, Y_test_concat_ZM, rnn_model, 4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "2979fd5b",
   "metadata": {},
   "outputs": [],
   "source": [
    "save_results(rnn_concat_RTA, \"RNN_RusTomekAdasyn_WithoutC_Concatenation_Results\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "5a199a68",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2768/2768 [==============================] - 4s 1ms/step\n",
      "(131644, 60, 24): RNN Classifier is Done! \n",
      "\n",
      "1329/1329 [==============================] - 2s 1ms/step\n",
      "(156692, 60, 24): RNN Classifier is Done! \n",
      "\n",
      "1329/1329 [==============================] - 2s 1ms/step\n",
      "(156692, 60, 24): RNN Classifier is Done! \n",
      "\n",
      "1602/1602 [==============================] - 2s 1ms/step\n",
      "(70894, 60, 24): RNN Classifier is Done! \n",
      "\n",
      "1602/1602 [==============================] - 2s 1ms/step\n",
      "(70894, 60, 24): RNN Classifier is Done! \n",
      "\n",
      "1602/1602 [==============================] - 2s 1ms/step\n",
      "(70894, 60, 24): RNN Classifier is Done! \n",
      "\n",
      "1602/1602 [==============================] - 4s 2ms/step\n",
      "(70894, 60, 24): RNN Classifier is Done! \n",
      "\n",
      "1602/1602 [==============================] - 2s 1ms/step\n",
      "(70894, 60, 24): RNN Classifier is Done! \n",
      "\n",
      "1602/1602 [==============================] - 2s 1ms/step\n",
      "(70894, 60, 24): RNN Classifier is Done! \n",
      "\n",
      "1602/1602 [==============================] - 2s 1ms/step\n",
      "(70894, 60, 24): RNN Classifier is Done! \n",
      "\n",
      "1602/1602 [==============================] - 2s 1ms/step\n",
      "(70894, 60, 24): RNN Classifier is Done! \n",
      "\n",
      "1602/1602 [==============================] - 2s 1ms/step\n",
      "(70894, 60, 24): RNN Classifier is Done! \n",
      "\n",
      "1602/1602 [==============================] - 2s 1ms/step\n",
      "(70894, 60, 24): RNN Classifier is Done! \n",
      "\n",
      "2356/2356 [==============================] - 3s 1ms/step\n",
      "(88280, 60, 24): RNN Classifier is Done! \n",
      "\n",
      "2356/2356 [==============================] - 4s 1ms/step\n",
      "(88280, 60, 24): RNN Classifier is Done! \n",
      "\n"
     ]
    }
   ],
   "source": [
    "rnn_concat_S = kfold_training('RNN', X_train_concat_ZM_S_3D, Y_train_concat_ZM_S, X_test_concat_ZM_3D, Y_test_concat_ZM, rnn_model, 4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "25a5c8a8",
   "metadata": {},
   "outputs": [],
   "source": [
    "save_results(rnn_concat_S, \"RNN_Smote_WithoutC_Concatenation_Results\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "181676a7",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2768/2768 [==============================] - 4s 1ms/step\n",
      "(131540, 60, 24): RNN Classifier is Done! \n",
      "\n",
      "1329/1329 [==============================] - 2s 1ms/step\n",
      "(156622, 60, 24): RNN Classifier is Done! \n",
      "\n",
      "1329/1329 [==============================] - 2s 1ms/step\n",
      "(156622, 60, 24): RNN Classifier is Done! \n",
      "\n",
      "1329/1329 [==============================] - 2s 1ms/step\n",
      "(156622, 60, 24): RNN Classifier is Done! \n",
      "\n",
      "1329/1329 [==============================] - 2s 1ms/step\n",
      "(156622, 60, 24): RNN Classifier is Done! \n",
      "\n",
      "1329/1329 [==============================] - 2s 1ms/step\n",
      "(156622, 60, 24): RNN Classifier is Done! \n",
      "\n",
      "1329/1329 [==============================] - 2s 2ms/step\n",
      "(156622, 60, 24): RNN Classifier is Done! \n",
      "\n",
      "1329/1329 [==============================] - 2s 1ms/step\n",
      "(156622, 60, 24): RNN Classifier is Done! \n",
      "\n",
      "1329/1329 [==============================] - 2s 1ms/step\n",
      "(156622, 60, 24): RNN Classifier is Done! \n",
      "\n",
      "1602/1602 [==============================] - 2s 1ms/step\n",
      "(70907, 60, 24): RNN Classifier is Done! \n",
      "\n",
      "1602/1602 [==============================] - 2s 1ms/step\n",
      "(70907, 60, 24): RNN Classifier is Done! \n",
      "\n",
      "1602/1602 [==============================] - 2s 1ms/step\n",
      "(70907, 60, 24): RNN Classifier is Done! \n",
      "\n",
      "2356/2356 [==============================] - 3s 1ms/step\n",
      "(88334, 60, 24): RNN Classifier is Done! \n",
      "\n",
      "2356/2356 [==============================] - 4s 1ms/step\n",
      "(88334, 60, 24): RNN Classifier is Done! \n",
      "\n",
      "2356/2356 [==============================] - 4s 1ms/step\n",
      "(88334, 60, 24): RNN Classifier is Done! \n",
      "\n",
      "2356/2356 [==============================] - 3s 1ms/step\n",
      "(88334, 60, 24): RNN Classifier is Done! \n",
      "\n",
      "2356/2356 [==============================] - 3s 1ms/step\n",
      "(88334, 60, 24): RNN Classifier is Done! \n",
      "\n"
     ]
    }
   ],
   "source": [
    "rnn_concat_A = kfold_training('RNN', X_train_concat_ZM_A_3D, Y_train_concat_ZM_A, X_test_concat_ZM_3D, Y_test_concat_ZM, rnn_model, 4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "1718475d",
   "metadata": {},
   "outputs": [],
   "source": [
    "save_results(rnn_concat_A, \"RNN_Adasyn_WithoutC_Concatenation_Results\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b00618b5",
   "metadata": {},
   "source": [
    "### GRU"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "98bba36a",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2768/2768 [==============================] - 13s 5ms/step\n",
      "(19996, 60, 24): GRU Classifier is Done! \n",
      "\n",
      "1329/1329 [==============================] - 6s 5ms/step\n",
      "(19998, 60, 24): GRU Classifier is Done! \n",
      "\n",
      "1602/1602 [==============================] - 7s 5ms/step\n",
      "(19994, 60, 24): GRU Classifier is Done! \n",
      "\n",
      "1602/1602 [==============================] - 8s 5ms/step\n",
      "(19994, 60, 24): GRU Classifier is Done! \n",
      "\n",
      "2356/2356 [==============================] - 11s 5ms/step\n",
      "(19996, 60, 24): GRU Classifier is Done! \n",
      "\n"
     ]
    }
   ],
   "source": [
    "gru_concat_RTS = kfold_training('GRU', X_train_concat_ZM_RTS_3D, Y_train_concat_ZM_RTS, X_test_concat_ZM_3D, Y_test_concat_ZM, gru_model, 4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "e6bc4f27",
   "metadata": {},
   "outputs": [],
   "source": [
    "save_results(gru_concat_RTS, \"GRU_RusTomekSmote_WithoutC_Concatenation_Results\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "f033da76",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2768/2768 [==============================] - 13s 5ms/step\n",
      "(20061, 60, 24): GRU Classifier is Done! \n",
      "\n",
      "2768/2768 [==============================] - 13s 5ms/step\n",
      "(20061, 60, 24): GRU Classifier is Done! \n",
      "\n",
      "1329/1329 [==============================] - 6s 5ms/step\n",
      "(20043, 60, 24): GRU Classifier is Done! \n",
      "\n",
      "1602/1602 [==============================] - 8s 5ms/step\n",
      "(19955, 60, 24): GRU Classifier is Done! \n",
      "\n",
      "1602/1602 [==============================] - 8s 5ms/step\n",
      "(19955, 60, 24): GRU Classifier is Done! \n",
      "\n",
      "2356/2356 [==============================] - 12s 5ms/step\n",
      "(19981, 60, 24): GRU Classifier is Done! \n",
      "\n",
      "2356/2356 [==============================] - 11s 5ms/step\n",
      "(19981, 60, 24): GRU Classifier is Done! \n",
      "\n"
     ]
    }
   ],
   "source": [
    "gru_concat_RTA = kfold_training('GRU', X_train_concat_ZM_RTA_3D, Y_train_concat_ZM_RTA, X_test_concat_ZM_3D, Y_test_concat_ZM, gru_model, 4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "ab46dc19",
   "metadata": {},
   "outputs": [],
   "source": [
    "save_results(gru_concat_RTA, \"GRU_RusTomekAdasyn_WithoutC_Concatenation_Results\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "23441ac6",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2768/2768 [==============================] - 13s 5ms/step\n",
      "(131644, 60, 24): GRU Classifier is Done! \n",
      "\n",
      "1329/1329 [==============================] - 6s 5ms/step\n",
      "(156692, 60, 24): GRU Classifier is Done! \n",
      "\n",
      "1602/1602 [==============================] - 7s 5ms/step\n",
      "(70894, 60, 24): GRU Classifier is Done! \n",
      "\n",
      "1602/1602 [==============================] - 8s 5ms/step\n",
      "(70894, 60, 24): GRU Classifier is Done! \n",
      "\n",
      "2356/2356 [==============================] - 11s 5ms/step\n",
      "(88280, 60, 24): GRU Classifier is Done! \n",
      "\n",
      "2356/2356 [==============================] - 11s 5ms/step\n",
      "(88280, 60, 24): GRU Classifier is Done! \n",
      "\n"
     ]
    }
   ],
   "source": [
    "gru_concat_S = kfold_training('GRU', X_train_concat_ZM_S_3D, Y_train_concat_ZM_S, X_test_concat_ZM_3D, Y_test_concat_ZM, gru_model, 4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "5120e055",
   "metadata": {},
   "outputs": [],
   "source": [
    "save_results(gru_concat_S, \"GRU_Smote_WithoutC_Concatenation_Results\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "bcde897f",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2768/2768 [==============================] - 13s 5ms/step\n",
      "(131540, 60, 24): GRU Classifier is Done! \n",
      "\n",
      "1329/1329 [==============================] - 6s 5ms/step\n",
      "(156622, 60, 24): GRU Classifier is Done! \n",
      "\n",
      "1602/1602 [==============================] - 7s 5ms/step\n",
      "(70907, 60, 24): GRU Classifier is Done! \n",
      "\n",
      "1602/1602 [==============================] - 7s 5ms/step\n",
      "(70907, 60, 24): GRU Classifier is Done! \n",
      "\n",
      "2356/2356 [==============================] - 11s 5ms/step\n",
      "(88334, 60, 24): GRU Classifier is Done! \n",
      "\n",
      "2356/2356 [==============================] - 11s 5ms/step\n",
      "(88334, 60, 24): GRU Classifier is Done! \n",
      "\n",
      "2356/2356 [==============================] - 11s 5ms/step\n",
      "(88334, 60, 24): GRU Classifier is Done! \n",
      "\n"
     ]
    }
   ],
   "source": [
    "gru_concat_A = kfold_training('GRU', X_train_concat_ZM_A_3D, Y_train_concat_ZM_A, X_test_concat_ZM_3D, Y_test_concat_ZM, gru_model, 4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "aa4a3b95",
   "metadata": {},
   "outputs": [],
   "source": [
    "save_results(gru_concat_A, \"GRU_Adasyn_WithoutC_Concatenation_Results\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c600f8e3",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}