main-csa.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "aa2427b3-ec3c-4699-bb80-c9b2aa439fd1",
   "metadata": {},
   "outputs": [],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "8aefeae0-a243-4263-97f9-123b155a786b",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Class Weights:\n",
      "[ 0.86694915  0.465       1.51331361 28.41666667]\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "from sklearn.utils.class_weight import compute_class_weight\n",
    "from data.aware_new import AwareCSA\n",
    "from data.cats_and_dogs import Dataset\n",
    "\n",
    "classes = [\n",
    "    'Control / healthy / no pulmonary disease',\n",
    "    'Asthma',\n",
    "    'CF',\n",
    "    'COPD'\n",
    "]\n",
    "dataset = AwareCSA(\n",
    "    csv_data = 'data/exhale_data_v8_ave.csv',\n",
    "    csv_outcome = 'data/exhale_outcome_v8_ave.csv',\n",
    "    csv_info = 'data/exhale_verbose_v8_ave.csv',\n",
    "    redcap_csv_file = \"data/AWARE_DATA_LABELS_2023-12-08_1611.csv\",\n",
    "    id_map_file = \"data/id_map.csv\",\n",
    "    target_classes=classes, \n",
    "    age_balanced=False, \n",
    "    output_demogr=True, \n",
    "    output_spiro_raw=True, \n",
    "    output_spiro_pred=False, \n",
    "    output_oscil_raw=False, \n",
    "    output_oscil_zscore=False, \n",
    "    output_disease_label=False\n",
    ")\n",
    "\n",
    "# dataset.save_to_pickle('data/aware_spectrogram.pkl')\n",
    "\n",
    "# dataset = Dataset(\"data/cats_and_dogs\", video=False, dim_order=\"BTCHW\")\n",
    "\n",
    "print(\"Class Weights:\")\n",
    "print(dataset.class_weights)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "4f9c5729-e514-49b0-aa4d-c6ca827c05ce",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(84,)\n",
      "float32\n",
      "[ 30.7    1.   160.02  80.  ]\n",
      "[2.94 3.34 0.88 3.97]\n"
     ]
    }
   ],
   "source": [
    "inputs, demogr, labels = dataset[0]\n",
    "print(inputs.shape)\n",
    "print(inputs.dtype)\n",
    "print(demogr)\n",
    "print(labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "b5f09561-69f1-47be-a49c-4c26dba03e2f",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Current device: cuda\n",
      "CNN1D(\n",
      "  (encoder): Sequential(\n",
      "    (0): Conv1d(1, 8, kernel_size=(3,), stride=(1,), padding=(1,))\n",
      "    (1): BatchNorm1d(8, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "    (2): ReLU()\n",
      "    (3): MaxPool1d(kernel_size=2, stride=2, padding=0, dilation=1, ceil_mode=False)\n",
      "    (4): Conv1d(8, 16, kernel_size=(3,), stride=(1,), padding=(1,))\n",
      "    (5): BatchNorm1d(16, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "    (6): ReLU()\n",
      "    (7): MaxPool1d(kernel_size=2, stride=2, padding=0, dilation=1, ceil_mode=False)\n",
      "    (8): Conv1d(16, 16, kernel_size=(3,), stride=(1,), padding=(1,))\n",
      "    (9): BatchNorm1d(16, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "    (10): ReLU()\n",
      "    (11): AvgPool1d(kernel_size=(3,), stride=(3,), padding=(0,))\n",
      "    (12): Flatten(start_dim=1, end_dim=-1)\n",
      "    (13): Linear(in_features=112, out_features=32, bias=True)\n",
      "  )\n",
      "  (output): Sequential(\n",
      "    (0): Linear(in_features=36, out_features=64, bias=True)\n",
      "    (1): ReLU()\n",
      "    (2): Linear(in_features=64, out_features=64, bias=True)\n",
      "    (3): ReLU()\n",
      "    (4): Linear(in_features=64, out_features=4, bias=True)\n",
      "  )\n",
      ")\n",
      "torch.Size([16, 84])\n",
      "torch.Size([16, 4])\n",
      "tensor([[2.1500, 2.9800, 0.7210, 1.4500],\n",
      "        [1.7900, 2.7100, 0.6610, 1.0000],\n",
      "        [3.1000, 4.0100, 0.7730, 2.8300],\n",
      "        [2.2200, 2.2900, 0.9690, 1.8500],\n",
      "        [1.8600, 2.8200, 0.6600, 1.1600],\n",
      "        [2.5200, 2.6600, 0.9470, 3.4100],\n",
      "        [2.2800, 2.4100, 0.9460, 3.2500],\n",
      "        [1.5200, 1.6700, 0.9100, 1.9900],\n",
      "        [2.6400, 4.0200, 0.6570, 1.7800],\n",
      "        [3.1800, 3.7100, 0.8570, 3.5600],\n",
      "        [3.3700, 4.0700, 0.8280, 3.3400],\n",
      "        [3.8000, 4.5100, 0.8430, 5.0100],\n",
      "        [2.0900, 2.6700, 0.7830, 1.8000],\n",
      "        [2.8600, 3.7900, 0.7550, 2.4000],\n",
      "        [3.3900, 4.2100, 0.8050, 3.4400],\n",
      "        [2.4700, 2.7200, 0.9080, 2.9000]])\n",
      "tensor([[-1.8601,  0.6265,  1.5095, -6.7770],\n",
      "        [-1.7194,  0.5517,  1.2947, -6.2864],\n",
      "        [-2.0484,  1.1091,  1.7816, -7.1200],\n",
      "        [-1.9168,  0.4478,  1.5006, -6.7453],\n",
      "        [-2.2885,  0.2874,  2.0642, -7.1015],\n",
      "        [-1.9192,  0.5378,  1.5115, -6.7266],\n",
      "        [-1.9485,  0.6990,  1.5488, -7.1400],\n",
      "        [-1.5891,  0.4257,  1.2104, -5.7957],\n",
      "        [-2.4185,  0.3023,  2.1879, -7.5580],\n",
      "        [-2.0662,  0.5428,  1.6353, -7.2135],\n",
      "        [-2.5150,  0.4517,  2.2683, -7.8167],\n",
      "        [-3.8916,  0.2031,  3.9937, -9.1985],\n",
      "        [-2.0150,  0.9752,  1.7349, -6.8342],\n",
      "        [-1.9603,  0.6443,  1.6062, -7.0629],\n",
      "        [-2.4954,  0.4691,  2.1747, -7.9880],\n",
      "        [-1.8086,  0.6057,  1.3262, -6.6723]], device='cuda:0')\n"
     ]
    }
   ],
   "source": [
    "import torch\n",
    "from torchview import draw_graph\n",
    "import math\n",
    "from models.utils import select_model\n",
    "import matplotlib.pyplot as plt\n",
    "from matplotlib import animation\n",
    "from utils.explainability import GradCAM\n",
    "from peft import get_peft_model, LoraConfig, TaskType\n",
    "\n",
    "RANDOM_SEED = [4399,114514,1234,1024,304,1,2,3,4,5]\n",
    "# RANDOM_SEED = [1]\n",
    "BATCH_SIZE = 16\n",
    "LEARN_RATE = 1e-4\n",
    "MAX_NUM_EPOCH = 50\n",
    "MODEL_NAME = \"cnn1d_reg\"\n",
    "VISUALIZE_MODEL = False\n",
    "\n",
    "device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n",
    "print(\"Current device:\", device)\n",
    "\n",
    "model = select_model(MODEL_NAME)\n",
    "model.to(device)\n",
    "\n",
    "if VISUALIZE_MODEL:\n",
    "    model_graph = draw_graph(\n",
    "        model,\n",
    "        input_size=(BATCH_SIZE, 3, 32, 224, 224),\n",
    "        expand_nested=True,\n",
    "        save_graph=True,\n",
    "        filename=MODEL_NAME\n",
    "    )\n",
    "    # display(model_graph.visual_graph)\n",
    "    # print(model)\n",
    "\n",
    "# target_layers = [model.layer4[-1]]\n",
    "# target_layers = [model.vivit.layernorm]\n",
    "# cam = GradCAM(model=model, target_layers=target_layers)\n",
    "\n",
    "def print_trainable_parameters(model):\n",
    "    \"\"\"\n",
    "    Prints the number of trainable parameters in the model.\n",
    "    \"\"\"\n",
    "    trainable_params = 0\n",
    "    all_param = 0\n",
    "    for _, param in model.named_parameters():\n",
    "        all_param += param.numel()\n",
    "        if param.requires_grad:\n",
    "            trainable_params += param.numel()\n",
    "    print(\n",
    "        f\"trainable params: {trainable_params} || all params: {all_param} || trainable%: {100 * trainable_params / all_param:.2f}\"\n",
    "    )\n",
    "\n",
    "peft_config = LoraConfig(\n",
    "    r=16,\n",
    "    lora_alpha=16,\n",
    "    target_modules=[\"query\", \"value\"],\n",
    "    lora_dropout=0.1,\n",
    "    bias=\"none\",\n",
    "    modules_to_save=[\"classifier\"],\n",
    ")\n",
    "print(model)\n",
    "# model = get_peft_model(model, peft_config)\n",
    "# model.vit.gradient_checkpointing_enable()\n",
    "# print_trainable_parameters(model)\n",
    "\n",
    "def show_image(inputs, musk=None, flipud=False):\n",
    "    inputs = inputs/2+0.5\n",
    "    if flipud:\n",
    "        inputs = inputs.flip(dims=(2,))\n",
    "    d = math.isqrt(inputs.shape[0]-1)+1\n",
    "    fig, axs = plt.subplots(d, d, figsize=(8,8))\n",
    "    im = []\n",
    "    if inputs.shape[0]==1:\n",
    "        im += [axs.imshow(inputs[0,:,:,:].permute(1,2,0))]\n",
    "        if musk is not None:\n",
    "            axs.imshow(musk[0,:,:], cmap='jet', alpha=0.4)\n",
    "    else:\n",
    "        for i in range(inputs.shape[0]):\n",
    "            # im += [axs[i//d, i%d].pcolormesh(inputs[i,0,0,:,:], shading='gouraud', cmap='gray')]\n",
    "            im += [axs[i//d, i%d].imshow(inputs[i,:,:,:].permute(1,2,0))]\n",
    "            if musk is not None:\n",
    "                axs[i//d, i%d].imshow(musk[i,:,:], cmap='jet', alpha=0.4)\n",
    "    plt.show()\n",
    "    \n",
    "def show_video(inputs, musk=None, animate=False, flipud=False):\n",
    "    inputs = inputs/2+0.5\n",
    "    if flipud:\n",
    "        inputs = inputs.flip(dims=(3,))\n",
    "    plt.rcParams[\"animation.html\"] = \"jshtml\"\n",
    "    d = math.isqrt(inputs.shape[0]-1)+1\n",
    "    fig, axs = plt.subplots(d, d, figsize=(8,8))\n",
    "    im = []\n",
    "    msk = []\n",
    "    if inputs.shape[0]==1:\n",
    "        im += [axs.imshow(inputs[0,:,0,:,:].permute(1,2,0))]\n",
    "        if musk is not None:\n",
    "            msk += [axs.imshow(musk[0,:,:], cmap='jet', alpha=0.4)]\n",
    "    else:\n",
    "        for i in range(inputs.shape[0]):\n",
    "            # im += [axs[i//d, i%d].pcolormesh(inputs[i,0,0,:,:], shading='gouraud', cmap='gray')]\n",
    "            im += [axs[i//d, i%d].imshow(inputs[i,:,0,:,:].permute(1,2,0))]\n",
    "            if musk is not None:\n",
    "                 msk += [axs[i//d, i%d].imshow(musk[i,:,:], cmap='jet', alpha=0.4)]\n",
    "    \n",
    "    def update(frame):\n",
    "        for i in range(inputs.shape[0]):\n",
    "            im[i].set_array(inputs[i,:,frame,:,:].permute(1,2,0))\n",
    "    \n",
    "    if animate:\n",
    "        ani = animation.FuncAnimation(fig=fig, func=update, frames=inputs.shape[2], interval=100)\n",
    "        display(ani)\n",
    "        ani.save('animation.gif', writer='imagemagick', fps=10)\n",
    "        plt.close(fig)\n",
    "    else:\n",
    "        plt.show()\n",
    "\n",
    "data_loader = torch.utils.data.DataLoader(dataset, batch_size=BATCH_SIZE, shuffle=True)\n",
    "inputs, demogr, labels = next(iter(data_loader))\n",
    "print(inputs.size())\n",
    "\n",
    "# inputs = image_processor(list(inputs.view(-1, *inputs.shape[2:])), return_tensors='pt')\n",
    "# inputs['pixel_values'] = inputs['pixel_values'].view(BATCH_SIZE, -1, *inputs['pixel_values'].shape[2:])\n",
    "# print(inputs['pixel_values'].size())\n",
    "# plot(inputs['pixel_values'])\n",
    "\n",
    "inputs = inputs.to(device)\n",
    "demogr = demogr.to(device)\n",
    "with torch.no_grad():\n",
    "    # outputs = model(pixel_values=input)\n",
    "    outputs = model(inputs, demogr)\n",
    "predicted_label = outputs.argmax(-1)\n",
    "print(outputs.size())\n",
    "print(labels)\n",
    "print(outputs)\n",
    "\n",
    "# gradcam = cam(input_tensor=inputs[0:1,:,:,:,:], targets=None)\n",
    "# print(gradcam.shape)\n",
    "\n",
    "# show_video(inputs[0:4,:,:,:,:].permute(0,2,1,3,4).cpu(), animate=False, flipud=True)\n",
    "# show_video(inputs[0:1,:,:,:,:].permute(0,2,1,3,4).cpu(), musk=gradcam[0:1,:,:], animate=False, flipud=True)\n",
    "# show_image(inputs[0:4,:,:,:].cpu(), musk=gradcam[0:4,:,:])\n",
    "# show_image(inputs[0:4,:,:,:].cpu(), flipud=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "67c483a3-833c-42fb-96a5-af0eb3990382",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Seed 4399 | Fold #0 | Training...\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "593f67fdaa4a435b9dcd4bb982dce2fc",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0.00/50.0 [00:00<?, ?epoch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validataion:\n",
      "Best testing loss: 0.58\n",
      "On which epoch reach the highest accuracy: 25\n",
      "Test:\n",
      "Best testing loss: 0.63\n",
      "On which epoch reach the highest accuracy: 25\n"
     ]
    },
    {
     "ename": "AttributeError",
     "evalue": "'Trainer' object has no attribute 'info'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mAttributeError\u001b[0m                            Traceback (most recent call last)",
      "Cell \u001b[0;32mIn[5], line 60\u001b[0m\n\u001b[1;32m     58\u001b[0m trainer\u001b[38;5;241m.\u001b[39mtest(test_loader, no_print\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mFalse\u001b[39;00m, calculate_ig\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mFalse\u001b[39;00m)\n\u001b[1;32m     59\u001b[0m metrics_test\u001b[38;5;241m.\u001b[39mappend_from(trainer)\n\u001b[0;32m---> 60\u001b[0m \u001b[43moutputs_test\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mappend_from\u001b[49m\u001b[43m(\u001b[49m\u001b[43mtrainer\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m     61\u001b[0m \u001b[38;5;66;03m# ig_test = torch.concat((ig_test, trainer.attr_ig), dim=0)\u001b[39;00m\n\u001b[1;32m     62\u001b[0m \u001b[38;5;28mprint\u001b[39m()\n",
      "File \u001b[0;32m/mnt/d/Xiangyu/AWARE/utils/others.py:129\u001b[0m, in \u001b[0;36mBasicOutputs.append_from\u001b[0;34m(self, trainer)\u001b[0m\n\u001b[1;32m    125\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mappend_from\u001b[39m(\u001b[38;5;28mself\u001b[39m, trainer):\n\u001b[1;32m    126\u001b[0m     df \u001b[38;5;241m=\u001b[39m pd\u001b[38;5;241m.\u001b[39mDataFrame({\n\u001b[1;32m    127\u001b[0m         \u001b[38;5;124m'\u001b[39m\u001b[38;5;124mLabels\u001b[39m\u001b[38;5;124m'\u001b[39m:trainer\u001b[38;5;241m.\u001b[39mlabels\u001b[38;5;241m.\u001b[39mtolist(), \n\u001b[1;32m    128\u001b[0m         \u001b[38;5;124m'\u001b[39m\u001b[38;5;124mOutputs\u001b[39m\u001b[38;5;124m'\u001b[39m:trainer\u001b[38;5;241m.\u001b[39moutputs\u001b[38;5;241m.\u001b[39mtolist(),\n\u001b[0;32m--> 129\u001b[0m         \u001b[38;5;124m'\u001b[39m\u001b[38;5;124mInfo\u001b[39m\u001b[38;5;124m'\u001b[39m:\u001b[43mtrainer\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43minfo\u001b[49m\u001b[38;5;241m.\u001b[39mtolist()\n\u001b[1;32m    130\u001b[0m     })\n\u001b[1;32m    131\u001b[0m     \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39moutputs \u001b[38;5;241m=\u001b[39m pd\u001b[38;5;241m.\u001b[39mconcat((\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39moutputs, df))\u001b[38;5;241m.\u001b[39mreset_index(drop\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mTrue\u001b[39;00m)\n",
      "\u001b[0;31mAttributeError\u001b[0m: 'Trainer' object has no attribute 'info'"
     ]
    }
   ],
   "source": [
    "from tqdm.notebook import tqdm\n",
    "from torch.utils.tensorboard import SummaryWriter\n",
    "from IPython.display import clear_output\n",
    "import time\n",
    "import jupyter_beeper\n",
    "import warnings\n",
    "\n",
    "from data.aware_raw import AwareSplitter\n",
    "from data.cats_and_dogs import Splitter\n",
    "from trainer.spectrogram_reg import Trainer\n",
    "from utils.others import weight_reset, BasicMetrics, BasicOutputs, RegressionMetrics\n",
    "from utils.clustering import evaluate\n",
    "from utils.outlier import novelty_detection\n",
    "\n",
    "# metrics_cluster = np.zeros((5,3,3))\n",
    "metrics_val = RegressionMetrics()\n",
    "metrics_test = RegressionMetrics()\n",
    "outputs_test = BasicOutputs()\n",
    "ig_test = torch.Tensor([])\n",
    "\n",
    "beeper = jupyter_beeper.Beeper()\n",
    "warnings.filterwarnings(\"ignore\")  ## ignore warnings\n",
    "\n",
    "for rand_seed in RANDOM_SEED:\n",
    "    splitter = AwareSplitter(dataset, BATCH_SIZE, random_seed=rand_seed)\n",
    "\n",
    "    timestr = time.strftime(\"%Y%m%d-%H%M%S\")\n",
    "    for split_idx, (train_loader, val_loader, test_loader) in enumerate(splitter):\n",
    "        writer = SummaryWriter(\"runs/\" + timestr + \"-fold\" + str(split_idx))\n",
    "        model = select_model(MODEL_NAME)\n",
    "        model.to(device)\n",
    "        # model = get_peft_model(model, peft_config)\n",
    "        # model.vit.gradient_checkpointing_enable()\n",
    "        # print_trainable_parameters(model)\n",
    "\n",
    "        # target_layers = [model.layer4[-1]]\n",
    "        # cam = GradCAM(model=model, target_layers=target_layers)\n",
    "        \n",
    "#         clear_output(wait=True)\n",
    "        print(\"Seed \" + str(rand_seed) + \" | Fold #\" + str(split_idx) + \" | Training...\")\n",
    "\n",
    "        trainer = Trainer(\n",
    "            model,\n",
    "            lr = LEARN_RATE,\n",
    "            T_max = MAX_NUM_EPOCH,\n",
    "            device = device,\n",
    "            summarywriter = writer,\n",
    "            class_weights = torch.Tensor(dataset.class_weights)\n",
    "        )\n",
    "        \n",
    "        for epoch in tqdm(range(MAX_NUM_EPOCH), unit_scale=True, unit=\"epoch\"):\n",
    "            trainer.train(epoch, train_loader)\n",
    "            trainer.validate(epoch, val_loader)\n",
    "        print(\"Validataion:\")\n",
    "        trainer.test(val_loader, no_print=False)\n",
    "        metrics_val.append_from(trainer)\n",
    "        print(\"Test:\")\n",
    "        trainer.test(test_loader, no_print=False, calculate_ig=False)\n",
    "        metrics_test.append_from(trainer)\n",
    "        outputs_test.append_from(trainer)\n",
    "        # ig_test = torch.concat((ig_test, trainer.attr_ig), dim=0)\n",
    "        print()\n",
    "        \n",
    "        # inputs, labels = next(iter(test_loader))\n",
    "        # inputs = inputs.to(device)\n",
    "        # with torch.no_grad():\n",
    "        #     # outputs = model(pixel_values=input)\n",
    "        #     outputs = model(inputs)\n",
    "        # predicted_label = outputs.argmax(-1)\n",
    "        # print(labels)\n",
    "        # print(predicted_label)\n",
    "\n",
    "        # gradcam = cam(input_tensor=inputs[0:1,:,:,:,:], targets=None)\n",
    "        # show_video(inputs[0:4,:,:,:,:].cpu(), animate=True, flipud=True)\n",
    "        # show_video(inputs[0:1,:,:,:,:].cpu(), musk=gradcam[0:1,:,:], animate=False, flipud=True)\n",
    "        break\n",
    "\n",
    "#     beeper.beep(frequency=600, secs=0.5)\n",
    "#     novelty_detection(model, train_loader, val_loader, test_loader)\n",
    "#     metrics_cluster[split_idx,:,:] = evaluate(model, train_loader, val_loader, test_loader)    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5920582c-adab-4b8d-9d19-d110dce054c7",
   "metadata": {},
   "outputs": [],
   "source": [
    "import sklearn.metrics as M\n",
    "# print(ig_test.size())\n",
    "print(\"Final Validation Results\")\n",
    "display(metrics_val)\n",
    "print()\n",
    "print(\"Final Test Results\")\n",
    "display(metrics_test)\n",
    "print()\n",
    "# beeper.beep(frequency=600, secs=0.5)\n",
    "display(outputs_test)\n",
    "outputs_test.outputs.to_excel(\"outputs.xlsx\")\n",
    "labels = np.array(outputs_test.outputs['Labels'].to_list())\n",
    "outputs = np.array(outputs_test.outputs['Outputs'].to_list())\n",
    "display(np.sqrt(np.mean(np.square(outputs-labels), axis=0))) # RMSE\n",
    "display(np.mean(np.abs(outputs-labels)/labels, axis=0)) # MAPE\n",
    "display(\n",
    "    M.r2_score(labels[:,0], outputs[:,0]),\n",
    "    M.r2_score(labels[:,1], outputs[:,1]),\n",
    "    M.r2_score(labels[:,2], outputs[:,2]),\n",
    "    M.r2_score(labels[:,3], outputs[:,3])\n",
    ")\n",
    "fig, axs = plt.subplots(2, 2, figsize=(8,6))\n",
    "axs[0,0].scatter(labels[:,0], outputs[:,0], marker='.')\n",
    "axs[0,1].scatter(labels[:,1], outputs[:,1], marker='.')\n",
    "axs[1,0].scatter(labels[:,2], outputs[:,2], marker='.')\n",
    "axs[1,1].scatter(labels[:,3], outputs[:,3], marker='.')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fe14e959-c7cf-4eb7-b083-e342f6da0bb3",
   "metadata": {},
   "outputs": [],
   "source": [
    "# import ast\n",
    "# import matplotlib.pyplot as plt\n",
    "# import pandas as pd\n",
    "\n",
    "# labels = outputs_test.outputs['Labels'].to_list()\n",
    "# labels = pd.DataFrame(labels)\n",
    "# labels.columns = ['Diagnosis']\n",
    "\n",
    "# info = outputs_test.outputs['Info'].to_list()\n",
    "# info = pd.DataFrame(info)\n",
    "# info.columns = ['Age', 'Sex', 'Height', 'Weight']\n",
    "#                # 'FEV1', 'FVC', 'FEV1/FVC', 'FEF2575']\n",
    "\n",
    "# meta = pd.concat([labels, info], axis=1)\n",
    "\n",
    "# outputs_cls = outputs_test.outputs['Outputs'].to_list()\n",
    "# outputs_cls = pd.DataFrame(outputs_cls)\n",
    "\n",
    "# meta.insert(1, 'Prediction', np.exp(outputs_cls[1])/np.sum(np.exp(outputs_cls),axis=1)) # Softmax\n",
    "\n",
    "# idx_tp = (meta['Diagnosis']==1) & (meta['Prediction']>=0.5)\n",
    "# tp = idx_tp.sum()\n",
    "# idx_fp = (meta['Diagnosis']==0) & (meta['Prediction']>=0.5)\n",
    "# fp = idx_fp.sum()\n",
    "# idx_tn = (meta['Diagnosis']==0) & (meta['Prediction']<0.5)\n",
    "# tn = idx_tn.sum()\n",
    "# idx_fn = (meta['Diagnosis']==1) & (meta['Prediction']<0.5)\n",
    "# fn = idx_fn.sum()\n",
    "# print(tn, fp)\n",
    "# print(fn, tp)\n",
    "\n",
    "# plt.figure()\n",
    "# plt.hist(meta['Prediction'][meta['Diagnosis']==0], bins=50, range=(0,1), alpha = 0.3, color='b', edgecolor='k', linewidth=1)\n",
    "# plt.hist(meta['Prediction'][meta['Diagnosis']==1], bins=50, range=(0,1), alpha = 0.3, color='r', edgecolor='k', linewidth=1)\n",
    "# plt.legend(['Healthy', 'Asthma'])\n",
    "# plt.show()\n",
    "\n",
    "# # cross_entropy = -(y_true*np.log10(y_pred[1]) + (1-y_true)*np.log10(y_pred[0]))\n",
    "\n",
    "# # plt.figure()\n",
    "# # # plt.hist(cross_entropy, bins=np.logspace(np.log10(0.1),np.log10(10.0), 50), edgecolor='k', linewidth=1)\n",
    "# # plt.hist(cross_entropy, bins=50, edgecolor='k', linewidth=1)\n",
    "# # plt.legend(['Healthy', 'Asthma'])\n",
    "# # # plt.gca().set_xscale(\"log\")\n",
    "# # plt.vlines(-np.log10(0.5), 0, 500, color='r')\n",
    "# # plt.show()\n",
    "\n",
    "# # y_pred = y_pred.idxmax(axis=1)\n",
    "\n",
    "# sens = tp/(tp+fn)\n",
    "# spec = tn/(fp+tn)\n",
    "# print('Sens:', sens)\n",
    "# print('Spec:', spec)\n",
    "# print('BalAcc:', (sens+spec)/2)\n",
    "\n",
    "# plt.figure(figsize=(20,4))\n",
    "# plt.subplot(2,4,1)\n",
    "# plt.hist(pd.concat([meta['Age'][idx_tp], meta['Age'][idx_fn]], axis=1), range=(meta['Age'].min(),meta['Age'].max()), alpha = 0.3, stacked=True, edgecolor='k', linewidth=1)\n",
    "# plt.legend(['True Asthma', 'False Healthy'])\n",
    "# plt.title('Age')\n",
    "# plt.subplot(2,4,5)\n",
    "# plt.hist(pd.concat([meta['Age'][idx_tn], meta['Age'][idx_fp]], axis=1), range=(meta['Age'].min(),meta['Age'].max()), alpha = 0.3, stacked=True, edgecolor='k', linewidth=1)\n",
    "# plt.legend(['True Healthy', 'False Asthma'])\n",
    "# plt.subplot(2,4,2)\n",
    "# plt.hist(pd.concat([meta['Sex'][idx_tp], meta['Sex'][idx_fn]], axis=1), range=(meta['Sex'].min(),meta['Sex'].max()), alpha = 0.3, stacked=True, edgecolor='k', linewidth=1)\n",
    "# plt.legend(['True Asthma', 'False Healthy'])\n",
    "# plt.title('Sex')\n",
    "# plt.subplot(2,4,6)\n",
    "# plt.hist(pd.concat([meta['Sex'][idx_tn], meta['Sex'][idx_fp]], axis=1), range=(meta['Sex'].min(),meta['Sex'].max()), alpha = 0.3, stacked=True, edgecolor='k', linewidth=1)\n",
    "# plt.legend(['True Healthy', 'False Asthma'])\n",
    "# plt.subplot(2,4,3)\n",
    "# plt.hist(pd.concat([meta['Height'][idx_tp], meta['Height'][idx_fn]], axis=1), range=(meta['Height'].min(),meta['Height'].max()), alpha = 0.3, stacked=True, edgecolor='k', linewidth=1)\n",
    "# plt.legend(['True Asthma', 'False Healthy'])\n",
    "# plt.title('Height')\n",
    "# plt.subplot(2,4,7)\n",
    "# plt.hist(pd.concat([meta['Height'][idx_tn], meta['Height'][idx_fp]], axis=1), range=(meta['Height'].min(),meta['Height'].max()), alpha = 0.3, stacked=True, edgecolor='k', linewidth=1)\n",
    "# plt.legend(['True Healthy', 'False Asthma'])\n",
    "# plt.subplot(2,4,4)\n",
    "# plt.hist(pd.concat([meta['Weight'][idx_tp], meta['Weight'][idx_fn]], axis=1), range=(meta['Weight'].min(),meta['Weight'].max()), alpha = 0.3, stacked=True, edgecolor='k', linewidth=1)\n",
    "# plt.legend(['True Asthma', 'False Healthy'])\n",
    "# plt.title('Weight')\n",
    "# plt.subplot(2,4,8)\n",
    "# plt.hist(pd.concat([meta['Weight'][idx_tn], meta['Weight'][idx_fp]], axis=1), range=(meta['Weight'].min(),meta['Weight'].max()), alpha = 0.3, stacked=True, edgecolor='k', linewidth=1)\n",
    "# plt.legend(['True Healthy', 'False Asthma'])\n",
    "# plt.show()\n",
    "\n",
    "# plt.figure(figsize=(6,4))\n",
    "# plt.subplot(2,1,1)\n",
    "# plt.hist(pd.concat([meta['Age'][idx_tp], meta['Age'][idx_fn]], axis=1), bins=list(range(0, 78, 6)), alpha = 0.3, stacked=True, edgecolor='k', linewidth=1)\n",
    "# plt.legend(['True Asthma', 'False Healthy'])\n",
    "# plt.subplot(2,1,2)\n",
    "# plt.hist(pd.concat([meta['Age'][idx_tn], meta['Age'][idx_fp]], axis=1), bins=list(range(0, 78, 6)), alpha = 0.3, stacked=True, edgecolor='k', linewidth=1)\n",
    "# plt.legend(['True Healthy', 'False Asthma'])\n",
    "# plt.show()\n",
    "\n",
    "# thre = np.zeros(1001)\n",
    "# sens = np.zeros(1001)\n",
    "# spec = np.zeros(1001)\n",
    "# for i in range(0,1001):\n",
    "#     idx_tp = (meta['Diagnosis']==1) & (meta['Prediction']>=i/1000)\n",
    "#     tp = idx_tp.sum()\n",
    "#     idx_fp = (meta['Diagnosis']==0) & (meta['Prediction']>=i/1000)\n",
    "#     fp = idx_fp.sum()\n",
    "#     idx_tn = (meta['Diagnosis']==0) & (meta['Prediction']<i/1000)\n",
    "#     tn = idx_tn.sum()\n",
    "#     idx_fn = (meta['Diagnosis']==1) & (meta['Prediction']<i/1000)\n",
    "#     fn = idx_fn.sum()\n",
    "#     thre[i] = i/1000\n",
    "#     sens[i] = tp/(tp+fn)\n",
    "#     spec[i] = tn/(fp+tn)\n",
    "    \n",
    "# plt.figure()\n",
    "# plt.plot(thre, sens)\n",
    "# plt.plot(thre, spec)\n",
    "# plt.title('Sensitivity and Specificity vs. Threshold')\n",
    "# plt.legend(['Sensitivity', 'Specificity'])\n",
    "# plt.xlabel('Threshold')\n",
    "# plt.show()\n",
    "\n",
    "# plt.figure()\n",
    "# # skplt.metrics.plot_roc_curve(meta['Diagnosis'], outputs_cls)\n",
    "# plt.plot(1-spec, sens)\n",
    "# plt.plot([0,1], [0,1], '--k')\n",
    "# plt.title('ROC Curve')\n",
    "# plt.xlabel('False Positive Rate\\n(1-Specificity)')\n",
    "# plt.ylabel('True Positive Rate\\n(Sensitivity)')\n",
    "# plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7339b96d-5ba5-4cc4-a224-95b13a3c1bd9",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}