DataProcessing.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "from dataloading import *\n",
    "from preprocessing import *\n",
    "from utils import *\n",
    "from modelling import *\n",
    "import os\n",
    "from sklearn.model_selection import train_test_split, KFold\n",
    "from sklearn.metrics import roc_curve, roc_auc_score\n",
    "import tensorflow as tf\n",
    "import numpy as np\n",
    "from datetime import datetime\n",
    "import matplotlib.pyplot as plt\n",
    "import json\n",
    "\n",
    "db_dir = \"data/physionet\"\n",
    "pkl_path = db_dir + \"normalisedrecords_fslist_labels.pkl\"\n",
    "cutoff = 60 # Hz\n",
    "resample_fs = 120 # Hz\n",
    "crop_length = 30 # s\n",
    "afib_dup_factor = 2\n",
    "\n",
    "if not os.path.exists(pkl_path):\n",
    "    # Read filter, and normalise\n",
    "    record_list, fs_list, labels = read_challenge17_data(db_dir)\n",
    "    resampled_records = lowpass_filter_and_resample_record_list(record_list, fs_list, 512, cutoff, resample_fs)\n",
    "    normalised_records = normalise_record_list(resampled_records)\n",
    "    # Save it out \n",
    "    save_challenge17_pkl(pkl_path, (normalised_records, fs_list, labels))\n",
    "else:\n",
    "    # Read in the pkl file\n",
    "    normalised_records, fs_list, labels = load_challenge17_pkl(pkl_path)\n",
    "\n",
    "normalised_records, labels = drop_other_class_records_and_labels(normalised_records, labels)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(len(normalised_records))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Percentage of Afib cases: 0.3054578810579422\n",
      "Percentage of Afib cases: 0.3043611323641928\n",
      "Percentage of Afib cases: 0.3058230169646951\n",
      "Percentage of Afib cases: 0.308736087818265\n",
      "Percentage of Afib cases: 0.2992162286768096\n",
      "Percentage of Afib cases: 0.305092521792323\n",
      "Percentage of Afib cases: 0.305092521792323\n",
      "Percentage of Afib cases: 0.304726938962827\n",
      "Percentage of Afib cases: 0.3065058032987172\n",
      "Percentage of Afib cases: 0.30284926470588236\n"
     ]
    }
   ],
   "source": [
    "# dup_records, dup_labels = duplicate_afib_records_in_list(normalised_records, labels, afib_dup_factor)\n",
    "# cropped_records = crop_and_pad_record_list(dup_records, resample_fs, crop_length)\n",
    "\n",
    "BATCH_SIZE = 64\n",
    "EPOCHS = 400\n",
    "K_FOLDS = 10\n",
    "STREAM2_SIZE = 9\n",
    "\n",
    "kf = KFold(n_splits=K_FOLDS, shuffle=True)\n",
    "\n",
    "test_scores = []\n",
    "fold_id = 0\n",
    "\n",
    "for train_index, test_index in kf.split(normalised_records):\n",
    "\n",
    "    train_samples = [normalised_records[index] for index in train_index]\n",
    "    test_samples = [normalised_records[index] for index in test_index]\n",
    "    train_labels = labels.iloc[train_index].reset_index(drop=True)\n",
    "    test_labels = labels.iloc[test_index].reset_index(drop=True)\n",
    "\n",
    "    train_samples, train_labels = duplicate_afib_records_in_list(train_samples, train_labels, afib_dup_factor)\n",
    "\n",
    "    a_cases = len(train_labels[train_labels['A']])\n",
    "    print(f\"Percentage of Afib cases: {a_cases/len(train_labels)}\")\n",
    "\n",
    "    train_samples = crop_and_pad_record_list(train_samples, resample_fs, crop_length)\n",
    "    test_samples = crop_and_pad_record_list(test_samples, resample_fs, crop_length)\n",
    "\n",
    "    X_train, X_test = np.array(train_samples), np.array(test_samples)\n",
    "    X_train, X_test = np.expand_dims(X_train, -1), np.expand_dims(X_test, -1)\n",
    "    y_train, y_test = train_labels['A'].to_numpy().astype(int), test_labels['A'].to_numpy().astype(int)\n",
    "\n",
    "    X_train, X_valid, y_train, y_valid = train_test_split(X_train, y_train, test_size=0.1)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "\n",
    "    model = create_dual_stream_cnn_model((X_train.shape[1], 1), stream2_size = STREAM2_SIZE)\n",
    "    print_gpu_availability()\n",
    "\n",
    "    tensorboard_callback = tf.keras.callbacks.TensorBoard(log_dir=logs_dir, histogram_freq=0)\n",
    "    lr_scheduler = tf.keras.callbacks.LearningRateScheduler(exponential_decay_fn)\n",
    "    early_stopping_callback = tf.keras.callbacks.EarlyStopping(monitor='val_loss', min_delta=0.001, patience=20, mode='min', restore_best_weights=True, verbose=1)\n",
    "    model.fit(train_dataset, validation_data = validation_dataset,\n",
    "            epochs=EPOCHS, verbose=1,\n",
    "            callbacks=[lr_scheduler, tensorboard_callback, early_stopping_callback])\n",
    "    \n",
    "    models_path = models_dir + f'/fold_{fold_id}_model_weights.h5'\n",
    "    model.save_weights(models_path)\n",
    "\n",
    "    test_loss, test_accuracy, test_precision, test_recall = model.evaluate(test_dataset)\n",
    "    \n",
    "    y_scores = model.predict(X_test, verbose=0)\n",
    "    y_scores = y_scores.flatten()\n",
    "    test_fpr, test_tpr, _ = roc_curve(y_test, y_scores)\n",
    "    test_auc = roc_auc_score(y_test, y_scores)\n",
    "\n",
    "    test_scores.append({'loss':test_loss, \n",
    "                        'acc': test_accuracy, \n",
    "                        'prec':test_precision, \n",
    "                        'rec':test_recall, \n",
    "                        'auc':test_auc, \n",
    "                        'fpr':test_fpr.tolist(), \n",
    "                        'tpr':test_tpr.tolist()})\n",
    "\n",
    "    fold_id += 1\n",
    "\n",
    "test_scores_path = models_dir + \"/test_scores.json\"\n",
    "with open(test_scores_path, \"w\") as file:\n",
    "    json.dump(test_scores, file, indent=4)\n",
    "\n",
    "kfold_accuracy = np.array([elem['acc'] for elem in test_scores])\n",
    "kfold_precision = np.array([elem['prec'] for elem in test_scores])\n",
    "kfold_recall = np.array([elem['rec'] for elem in test_scores])\n",
    "print(f\"Accuracy:\\t{100*kfold_accuracy.mean():.1f}% \\nPrecision:\\t{100*kfold_precision.mean():.1f}% \\nRecall:\\t\\t{100*kfold_recall.mean():.1f}%\")\n",
    "\n",
    "tpr = test_scores[0]['tpr']\n",
    "fpr = test_scores[0]['fpr']\n",
    "auc = test_scores[0]['auc']\n",
    "\n",
    "plt.plot(fpr, tpr)\n",
    "plt.plot([0, 1], [0, 1], '--', color='black')\n",
    "plt.xlabel('False positive rate')\n",
    "plt.ylabel('True positive rate')\n",
    "plt.title(f\"AUC: {auc:.4f}\")\n",
    "plt.savefig(models_dir+'/roc_curve.png')  # Save the plot as a PNG file"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "X = [1, 2, 3, 4, 5, 6, 7, 8, 9]\n",
    "kf = KFold(n_splits=3, shuffle=True)\n",
    "\n",
    "for train_index, test_index in kf.split(X):\n",
    "    print(labels.iloc[train_index])\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(5788, 2)\n",
      "(5788,)\n",
      "(5788,)\n",
      "True\n"
     ]
    }
   ],
   "source": [
    "y = np.array(labels)\n",
    "print(y.shape)\n",
    "y = y[:,0].astype(int) # Atrial Fibrillation column\n",
    "print(y.shape)\n",
    "\n",
    "y2 = labels['A'].to_numpy().astype(int)\n",
    "print(y2.shape)\n",
    "\n",
    "print(np.array_equal(y, y2))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "venv",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}