main-spectrogram.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "aa2427b3-ec3c-4699-bb80-c9b2aa439fd1",
   "metadata": {},
   "outputs": [],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "8aefeae0-a243-4263-97f9-123b155a786b",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "df7339596b014137b8e2e1dd00916055",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Class Distribution:\n",
      "[1062. 1944.]\n",
      "Class Weights:\n",
      "[1, 1]\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "from sklearn.utils.class_weight import compute_class_weight\n",
    "from data.aware_raw import AwareRaw, AwareSpectrogram\n",
    "from data.cats_and_dogs import Dataset\n",
    "\n",
    "classes = [\n",
    "    'Control / healthy / no pulmonary disease',\n",
    "    'Asthma',\n",
    "    # 'CF',\n",
    "    # 'COPD'\n",
    "]\n",
    "dataset_raw = AwareRaw(\"data/AWARE_DATA_LABELS_2023-12-08_1611.csv\", \"data/id_map.csv\", \"data/aware_full_1704385505.db\", pickle_file=\"data/aware_segmented.pkl\")\n",
    "dataset = AwareSpectrogram(\n",
    "    dataset_raw, \n",
    "    target_classes=classes, \n",
    "    age_balanced=False, \n",
    "    output_demogr=False, \n",
    "    output_spiro_raw=False, \n",
    "    output_spiro_pred=False, \n",
    "    output_oscil_raw=False, \n",
    "    output_oscil_zscore=False, \n",
    "    output_disease_label=False,\n",
    "    output_inhale_exhale=True,\n",
    "    relative_change=False, \n",
    "    calibration=True, \n",
    "    averaged=False, \n",
    "    num_channels=3,\n",
    "    dim_order='BTCHW',\n",
    "    modality='ir'\n",
    ")\n",
    "imflip = True\n",
    "\n",
    "# dataset.save_to_pickle('data/aware_spectrogram.pkl')\n",
    "\n",
    "# dataset = Dataset(\"data/cats_and_dogs\", video=False, dim_order=\"BTCHW\")\n",
    "# imflip = False\n",
    "\n",
    "print(\"Class Distribution:\")\n",
    "print(dataset.class_distribution)\n",
    "print(\"Class Weights:\")\n",
    "print(dataset.class_weights)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "4f9c5729-e514-49b0-aa4d-c6ca827c05ce",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([25, 3, 224, 224])\n",
      "torch.float32\n",
      "0\n"
     ]
    }
   ],
   "source": [
    "inputs, labels = dataset[0]\n",
    "# inputs, demogr, labels = dataset[0]\n",
    "print(inputs.shape)\n",
    "print(inputs.dtype)\n",
    "# print(demogr)\n",
    "print(labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "b5f09561-69f1-47be-a49c-4c26dba03e2f",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Some weights of VivitModel were not initialized from the model checkpoint at google/vivit-b-16x2-kinetics400 and are newly initialized: ['vivit.pooler.dense.bias', 'vivit.pooler.dense.weight']\n",
      "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Current device: cuda\n",
      "ViViT(\n",
      "  (vivit): VivitModel(\n",
      "    (embeddings): VivitEmbeddings(\n",
      "      (patch_embeddings): VivitTubeletEmbeddings(\n",
      "        (projection): Conv3d(3, 768, kernel_size=(2, 16, 16), stride=(2, 16, 16))\n",
      "      )\n",
      "      (dropout): Dropout(p=0.0, inplace=False)\n",
      "    )\n",
      "    (encoder): VivitEncoder(\n",
      "      (layer): ModuleList(\n",
      "        (0-11): 12 x VivitLayer(\n",
      "          (attention): VivitAttention(\n",
      "            (attention): VivitSelfAttention(\n",
      "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
      "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
      "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
      "              (dropout): Dropout(p=0.0, inplace=False)\n",
      "            )\n",
      "            (output): VivitSelfOutput(\n",
      "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
      "              (dropout): Dropout(p=0.0, inplace=False)\n",
      "            )\n",
      "          )\n",
      "          (intermediate): VivitIntermediate(\n",
      "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
      "            (dropout): Dropout(p=0.0, inplace=False)\n",
      "            (intermediate_act_fn): FastGELUActivation()\n",
      "          )\n",
      "          (output): VivitOutput(\n",
      "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
      "            (dropout): Dropout(p=0.0, inplace=False)\n",
      "          )\n",
      "          (layernorm_before): LayerNorm((768,), eps=1e-06, elementwise_affine=True)\n",
      "          (layernorm_after): LayerNorm((768,), eps=1e-06, elementwise_affine=True)\n",
      "        )\n",
      "      )\n",
      "    )\n",
      "    (layernorm): LayerNorm((768,), eps=1e-06, elementwise_affine=True)\n",
      "    (pooler): VivitPooler(\n",
      "      (dense): Linear(in_features=768, out_features=768, bias=True)\n",
      "      (activation): Tanh()\n",
      "    )\n",
      "  )\n",
      "  (classifier): Linear(in_features=768, out_features=2, bias=True)\n",
      ")\n",
      "trainable params: 591362 || all params: 89829892 || trainable%: 0.66\n",
      "Input Size: torch.Size([16, 25, 3, 224, 224])\n"
     ]
    },
    {
     "ename": "RuntimeError",
     "evalue": "The size of tensor a (2353) must match the size of tensor b (3137) at non-singleton dimension 1",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mRuntimeError\u001b[0m                              Traceback (most recent call last)",
      "\u001b[0;32m/tmp/ipykernel_2078158/3553871536.py\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m    138\u001b[0m \u001b[0;32mwith\u001b[0m \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mno_grad\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    139\u001b[0m \u001b[0;31m#     outputs = model(pixel_values=inputs)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 140\u001b[0;31m     \u001b[0moutputs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmodel\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minputs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    141\u001b[0m     \u001b[0;31m# outputs = model(inputs, demogr)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    142\u001b[0m \u001b[0mpredicted_label\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0moutputs\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0margmax\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m-\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.local/lib/python3.8/site-packages/torch/nn/modules/module.py\u001b[0m in \u001b[0;36m_wrapped_call_impl\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m   1551\u001b[0m             \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_compiled_call_impl\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m  \u001b[0;31m# type: ignore[misc]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1552\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1553\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_call_impl\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1554\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1555\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m_call_impl\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.local/lib/python3.8/site-packages/torch/nn/modules/module.py\u001b[0m in \u001b[0;36m_call_impl\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m   1560\u001b[0m                 \u001b[0;32mor\u001b[0m \u001b[0m_global_backward_pre_hooks\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0m_global_backward_hooks\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1561\u001b[0m                 or _global_forward_hooks or _global_forward_pre_hooks):\n\u001b[0;32m-> 1562\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0mforward_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1563\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1564\u001b[0m         \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.local/lib/python3.8/site-packages/peft/peft_model.py\u001b[0m in \u001b[0;36mforward\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m    760\u001b[0m         \u001b[0;32mwith\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_enable_peft_forward_hooks\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    761\u001b[0m             \u001b[0mkwargs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m{\u001b[0m\u001b[0mk\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mv\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mk\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mv\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mkwargs\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mitems\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mk\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mspecial_peft_forward_args\u001b[0m\u001b[0;34m}\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 762\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_base_model\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    763\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    764\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mgenerate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.local/lib/python3.8/site-packages/torch/nn/modules/module.py\u001b[0m in \u001b[0;36m_wrapped_call_impl\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m   1551\u001b[0m             \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_compiled_call_impl\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m  \u001b[0;31m# type: ignore[misc]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1552\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1553\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_call_impl\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1554\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1555\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m_call_impl\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.local/lib/python3.8/site-packages/torch/nn/modules/module.py\u001b[0m in \u001b[0;36m_call_impl\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m   1560\u001b[0m                 \u001b[0;32mor\u001b[0m \u001b[0m_global_backward_pre_hooks\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0m_global_backward_hooks\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1561\u001b[0m                 or _global_forward_hooks or _global_forward_pre_hooks):\n\u001b[0;32m-> 1562\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0mforward_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1563\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1564\u001b[0m         \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/AWARE/models/vivit.py\u001b[0m in \u001b[0;36mforward\u001b[0;34m(self, x)\u001b[0m\n\u001b[1;32m     21\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     22\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mforward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mx\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 23\u001b[0;31m         \u001b[0moutputs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvivit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpixel_values\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     24\u001b[0m         \u001b[0msequence_output\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0moutputs\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     25\u001b[0m         \u001b[0mlogits\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mclassifier\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msequence_output\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m:\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.local/lib/python3.8/site-packages/torch/nn/modules/module.py\u001b[0m in \u001b[0;36m_wrapped_call_impl\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m   1551\u001b[0m             \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_compiled_call_impl\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m  \u001b[0;31m# type: ignore[misc]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1552\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1553\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_call_impl\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1554\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1555\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m_call_impl\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.local/lib/python3.8/site-packages/torch/nn/modules/module.py\u001b[0m in \u001b[0;36m_call_impl\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m   1560\u001b[0m                 \u001b[0;32mor\u001b[0m \u001b[0m_global_backward_pre_hooks\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0m_global_backward_hooks\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1561\u001b[0m                 or _global_forward_hooks or _global_forward_pre_hooks):\n\u001b[0;32m-> 1562\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0mforward_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1563\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1564\u001b[0m         \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.local/lib/python3.8/site-packages/transformers/models/vivit/modeling_vivit.py\u001b[0m in \u001b[0;36mforward\u001b[0;34m(self, pixel_values, head_mask, output_attentions, output_hidden_states, interpolate_pos_encoding, return_dict)\u001b[0m\n\u001b[1;32m    607\u001b[0m         \u001b[0mhead_mask\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_head_mask\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mhead_mask\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mconfig\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnum_hidden_layers\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    608\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 609\u001b[0;31m         \u001b[0membedding_output\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0membeddings\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpixel_values\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minterpolate_pos_encoding\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0minterpolate_pos_encoding\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    610\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    611\u001b[0m         encoder_outputs = self.encoder(\n",
      "\u001b[0;32m~/.local/lib/python3.8/site-packages/torch/nn/modules/module.py\u001b[0m in \u001b[0;36m_wrapped_call_impl\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m   1551\u001b[0m             \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_compiled_call_impl\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m  \u001b[0;31m# type: ignore[misc]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1552\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1553\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_call_impl\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1554\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1555\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m_call_impl\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.local/lib/python3.8/site-packages/torch/nn/modules/module.py\u001b[0m in \u001b[0;36m_call_impl\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m   1560\u001b[0m                 \u001b[0;32mor\u001b[0m \u001b[0m_global_backward_pre_hooks\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0m_global_backward_hooks\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1561\u001b[0m                 or _global_forward_hooks or _global_forward_pre_hooks):\n\u001b[0;32m-> 1562\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0mforward_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1563\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1564\u001b[0m         \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.local/lib/python3.8/site-packages/transformers/models/vivit/modeling_vivit.py\u001b[0m in \u001b[0;36mforward\u001b[0;34m(self, pixel_values, interpolate_pos_encoding)\u001b[0m\n\u001b[1;32m    144\u001b[0m             \u001b[0membeddings\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0membeddings\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0minterpolate_pos_encoding\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0membeddings\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mheight\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mwidth\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    145\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 146\u001b[0;31m             \u001b[0membeddings\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0membeddings\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mposition_embeddings\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    147\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    148\u001b[0m         \u001b[0membeddings\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdropout\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0membeddings\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mRuntimeError\u001b[0m: The size of tensor a (2353) must match the size of tensor b (3137) at non-singleton dimension 1"
     ]
    }
   ],
   "source": [
    "import torch\n",
    "from torchview import draw_graph\n",
    "import cv2\n",
    "import math\n",
    "from models.utils import select_model\n",
    "import matplotlib.pyplot as plt\n",
    "from matplotlib import animation\n",
    "from sklearn.manifold import TSNE\n",
    "from sklearn.metrics import pairwise_distances\n",
    "import seaborn as sns\n",
    "# from utils.explainability import GradCAM\n",
    "from pytorch_grad_cam import GradCAM\n",
    "from peft import get_peft_model, LoraConfig, TaskType\n",
    "\n",
    "RANDOM_SEED = [4399,114514,1234,1024,304,1,2,3,4,5]\n",
    "# RANDOM_SEED = [1]\n",
    "BATCH_SIZE = 16\n",
    "LEARN_RATE = 1e-3\n",
    "MAX_NUM_EPOCH = 10\n",
    "MODEL_NAME = \"vit_binary\"\n",
    "VISUALIZE_MODEL = False\n",
    "\n",
    "device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n",
    "print(\"Current device:\", device)\n",
    "\n",
    "model = select_model(MODEL_NAME)\n",
    "model.to(device)\n",
    "\n",
    "if VISUALIZE_MODEL:\n",
    "    model_graph = draw_graph(\n",
    "        model,\n",
    "        input_size=(BATCH_SIZE, 3, 224, 224),\n",
    "        expand_nested=True,\n",
    "        save_graph=True,\n",
    "        filename=MODEL_NAME\n",
    "    )\n",
    "    # display(model_graph.visual_graph)\n",
    "    # print(model)\n",
    "\n",
    "print(model)\n",
    "\n",
    "# target_layers = [model.layer4[-1]]\n",
    "# target_layers = [model.vit.vit.layernorm]\n",
    "# target_layers = [model.vit.layernorm]\n",
    "# cam = GradCAM(model=model, target_layers=target_layers)\n",
    "\n",
    "def print_trainable_parameters(model):\n",
    "    \"\"\"\n",
    "    Prints the number of trainable parameters in the model.\n",
    "    \"\"\"\n",
    "    trainable_params = 0\n",
    "    all_param = 0\n",
    "    for _, param in model.named_parameters():\n",
    "        all_param += param.numel()\n",
    "        if param.requires_grad:\n",
    "            trainable_params += param.numel()\n",
    "    print(\n",
    "        f\"trainable params: {trainable_params} || all params: {all_param} || trainable%: {100 * trainable_params / all_param:.2f}\"\n",
    "    )\n",
    "\n",
    "peft_config = LoraConfig(\n",
    "    r=16,\n",
    "    lora_alpha=16,\n",
    "    target_modules=[\"query\", \"value\"],\n",
    "    lora_dropout=0.1,\n",
    "    bias=\"none\",\n",
    "    modules_to_save=[\"classifier\"],\n",
    ")\n",
    "model = get_peft_model(model, peft_config)\n",
    "model.vivit.gradient_checkpointing_enable()\n",
    "print_trainable_parameters(model)\n",
    "\n",
    "def show_image(inputs, mask=None, flipud=False):\n",
    "    inputs = inputs/2+0.5\n",
    "    if flipud:\n",
    "        inputs = inputs.flip(dims=(2,))\n",
    "        if mask is not None:\n",
    "            mask = mask.flip(dims=(1,))\n",
    "    d = math.isqrt(inputs.shape[0]-1)+1\n",
    "    fig, axs = plt.subplots(d, d, figsize=(8,8))\n",
    "    im = []\n",
    "    if inputs.shape[0]==1:\n",
    "        im += [axs.imshow(inputs[0,:,:,:].permute(1,2,0))]\n",
    "        if mask is not None:\n",
    "            axs.imshow(mask[0,:,:], cmap='jet', alpha=0.4)\n",
    "    else:\n",
    "        for i in range(inputs.shape[0]):\n",
    "            # im += [axs[i//d, i%d].pcolormesh(inputs[i,0,0,:,:], shading='gouraud', cmap='gray')]\n",
    "            im += [axs[i//d, i%d].imshow(inputs[i,:,:,:].permute(1,2,0))]\n",
    "            if mask is not None:\n",
    "                axs[i//d, i%d].imshow(mask[i,:,:], cmap='jet', alpha=0.4)\n",
    "    plt.show()\n",
    "    \n",
    "def show_video(inputs, mask=None, animate=False, flipud=False):\n",
    "    inputs = inputs/2+0.5\n",
    "    if flipud:\n",
    "        inputs = inputs.flip(dims=(3,))\n",
    "    plt.rcParams[\"animation.html\"] = \"jshtml\"\n",
    "    d = math.isqrt(inputs.shape[0]-1)+1\n",
    "    fig, axs = plt.subplots(d, d, figsize=(8,8))\n",
    "    im = []\n",
    "    msk = []\n",
    "    if inputs.shape[0]==1:\n",
    "        im += [axs.imshow(inputs[0,:,0,:,:].permute(1,2,0))]\n",
    "        if mask is not None:\n",
    "            msk += [axs.imshow(mask[0,:,:], cmap='jet', alpha=0.4)]\n",
    "    else:\n",
    "        for i in range(inputs.shape[0]):\n",
    "            # im += [axs[i//d, i%d].pcolormesh(inputs[i,0,0,:,:], shading='gouraud', cmap='gray')]\n",
    "            im += [axs[i//d, i%d].imshow(inputs[i,:,0,:,:].permute(1,2,0))]\n",
    "            if mask is not None:\n",
    "                 msk += [axs[i//d, i%d].imshow(mask[i,:,:], cmap='jet', alpha=0.4)]\n",
    "    \n",
    "    def update(frame):\n",
    "        for i in range(inputs.shape[0]):\n",
    "            im[i].set_array(inputs[i,:,frame,:,:].permute(1,2,0))\n",
    "    \n",
    "    if animate:\n",
    "        ani = animation.FuncAnimation(fig=fig, func=update, frames=inputs.shape[2], interval=100)\n",
    "        display(ani)\n",
    "        ani.save('animation.gif', writer='imagemagick', fps=10)\n",
    "        plt.close(fig)\n",
    "    else:\n",
    "        plt.show()\n",
    "\n",
    "data_loader = torch.utils.data.DataLoader(dataset, batch_size=BATCH_SIZE, shuffle=True)\n",
    "inputs, labels = next(iter(data_loader))\n",
    "# inputs, demogr, labels = next(iter(data_loader))\n",
    "print('Input Size:', inputs.size())\n",
    "\n",
    "# inputs = image_processor(list(inputs.view(-1, *inputs.shape[2:])), return_tensors='pt')\n",
    "# inputs['pixel_values'] = inputs['pixel_values'].view(BATCH_SIZE, -1, *inputs['pixel_values'].shape[2:])\n",
    "# print(inputs['pixel_values'].size())\n",
    "# plot(inputs['pixel_values'])\n",
    "\n",
    "inputs = inputs.to(device)\n",
    "# demogr = demogr.to(device)\n",
    "with torch.no_grad():\n",
    "#     outputs = model(pixel_values=inputs)\n",
    "    outputs = model(inputs)\n",
    "    # outputs = model(inputs, demogr)\n",
    "predicted_label = outputs.argmax(-1)\n",
    "print('Output Size:', outputs.size())\n",
    "print('Labels:', labels)\n",
    "print('Predicted:', predicted_label)\n",
    "\n",
    "def get_attention_map(inputs, logits, att_mat):\n",
    "    att_mat = torch.stack(att_mat, dim=1)\n",
    "    print(att_mat.size())\n",
    "\n",
    "    # Average the attention weights across all heads.\n",
    "    att_mat = torch.mean(att_mat, dim=2)\n",
    "    print(att_mat.size())\n",
    "\n",
    "    # To account for residual connections, we add an identity matrix to the\n",
    "    # attention matrix and re-normalize the weights.\n",
    "    residual_att = torch.eye(att_mat.size(-1)).to(device)\n",
    "    aug_att_mat = att_mat + residual_att\n",
    "    aug_att_mat = aug_att_mat / aug_att_mat.sum(dim=-1).unsqueeze(-1)\n",
    "    print(aug_att_mat.size())\n",
    "\n",
    "    # Recursively multiply the weight matrices\n",
    "    joint_attentions = torch.zeros(aug_att_mat.size()).to(device)\n",
    "    joint_attentions[:,0] = aug_att_mat[:,0]\n",
    "\n",
    "    for n in range(1, aug_att_mat.size(1)):\n",
    "        joint_attentions[:,n] = torch.matmul(aug_att_mat[:,n], joint_attentions[:,n-1])\n",
    "\n",
    "    v = joint_attentions[:,-1]\n",
    "    grid_size = int(np.sqrt(aug_att_mat.size(-1)))\n",
    "    mask = v[:, 0, 1:].reshape(v.size(0), grid_size, grid_size).detach().cpu().numpy()\n",
    "    print(mask.shape)\n",
    "    result = []\n",
    "    for i in range(mask.shape[0]):\n",
    "        mask[i,...] = (mask[i,...]-mask[i,...].min()) / (mask[i,...].max()-mask[i,...].min())\n",
    "        result += [cv2.resize(mask[i,...], inputs[i,0,...].size())]\n",
    "    result = torch.Tensor(np.array(result))\n",
    "\n",
    "    return result\n",
    "\n",
    "# m = get_attention_map(inputs, outputs, model.attentions)\n",
    "\n",
    "# for i in range(12):\n",
    "#     print(model.attentions[i].shape)\n",
    "#     show_image(model.attentions[i][0:1,:,:,:].permute(1,0,2,3).cpu(), flipud=True)\n",
    "\n",
    "# gradcam = cam(input_tensor=inputs[0:4,:,:,:], targets=None)\n",
    "# print(gradcam.shape)\n",
    "\n",
    "# show_video(inputs[0:4,:,:,:,:].permute(0,2,1,3,4).cpu(), animate=False, flipud=True)\n",
    "# show_video(inputs[0:1,:,:,:,:].permute(0,2,1,3,4).cpu(), mask=gradcam[0:1,:,:], animate=False, flipud=True)\n",
    "# show_image(inputs[0:4,:,:,:].cpu(), mask=gradcam[0:4,:,:])\n",
    "\n",
    "# show_image(inputs[0:4,...].cpu(), flipud=imflip)\n",
    "# show_image(inputs[0:4,...].cpu(), mask=m, flipud=imflip)\n",
    "\n",
    "# t-SNE visualization\n",
    "def tsne_plot(X, y):\n",
    "    print(X.shape)\n",
    "    tsne = TSNE(n_components=2, verbose=0, perplexity=10, n_iter=300)\n",
    "    tsne_results = tsne.fit_transform(X)\n",
    "    \n",
    "    results = {'tsne-2d-one': tsne_results[:,0],\n",
    "               'tsne-2d-two': tsne_results[:,1],\n",
    "               'y': y}\n",
    "    \n",
    "    plt.figure(figsize=(8,6))\n",
    "    sns.scatterplot(\n",
    "        x=\"tsne-2d-one\", y=\"tsne-2d-two\",\n",
    "        hue=\"y\",\n",
    "        data=results,\n",
    "        palette=sns.color_palette(\"Set2\"),\n",
    "        legend=\"full\",\n",
    "    )\n",
    "    plt.legend(classes)\n",
    "    plt.title(\"t-SNE Scatter Plot\")\n",
    "    plt.show()\n",
    "\n",
    "tsne_plot(outputs.cpu(), labels.cpu())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "67c483a3-833c-42fb-96a5-af0eb3990382",
   "metadata": {},
   "outputs": [],
   "source": [
    "from tqdm.notebook import tqdm\n",
    "from torch.utils.tensorboard import SummaryWriter\n",
    "from IPython.display import clear_output\n",
    "import time\n",
    "import jupyter_beeper\n",
    "import warnings\n",
    "\n",
    "from data.aware_raw import AwareSplitter\n",
    "from data.cats_and_dogs import Splitter\n",
    "from trainer.spectrogram import Trainer\n",
    "from utils.others import weight_reset, BasicMetrics, BasicOutputs, RegressionMetrics\n",
    "from utils.clustering import evaluate\n",
    "from utils.outlier import novelty_detection\n",
    "\n",
    "# metrics_cluster = np.zeros((5,3,3))\n",
    "metrics_val = RegressionMetrics()\n",
    "metrics_test = RegressionMetrics()\n",
    "outputs_test = BasicOutputs()\n",
    "ig_test = torch.Tensor([])\n",
    "\n",
    "beeper = jupyter_beeper.Beeper()\n",
    "warnings.filterwarnings(\"ignore\")  ## ignore warnings\n",
    "\n",
    "for rand_seed in RANDOM_SEED:\n",
    "    splitter = AwareSplitter(dataset, BATCH_SIZE, random_seed=rand_seed)\n",
    "\n",
    "    timestr = time.strftime(\"%Y%m%d-%H%M%S\")\n",
    "    for split_idx, (train_loader, val_loader, test_loader) in enumerate(splitter):\n",
    "        writer = SummaryWriter(\"runs/\" + timestr + \"-fold\" + str(split_idx))\n",
    "        model = select_model(MODEL_NAME)\n",
    "        model.to(device)\n",
    "        # model = get_peft_model(model, peft_config)\n",
    "        # model.vit.gradient_checkpointing_enable()\n",
    "        # print_trainable_parameters(model)\n",
    "\n",
    "        # target_layers = [model.layer4[-1]]\n",
    "        # cam = GradCAM(model=model, target_layers=target_layers)\n",
    "        \n",
    "#         clear_output(wait=True)\n",
    "        print(\"Seed \" + str(rand_seed) + \" | Fold #\" + str(split_idx) + \" | Training...\")\n",
    "\n",
    "        trainer = Trainer(\n",
    "            model,\n",
    "            lr = LEARN_RATE,\n",
    "            T_max = MAX_NUM_EPOCH,\n",
    "            device = device,\n",
    "            summarywriter = writer,\n",
    "            class_weights = torch.Tensor(dataset.class_weights)\n",
    "        )\n",
    "        \n",
    "        for epoch in tqdm(range(MAX_NUM_EPOCH), unit_scale=True, unit=\"epoch\"):\n",
    "            trainer.train(epoch, train_loader)\n",
    "            trainer.validate(epoch, val_loader)\n",
    "        print(\"Train:\")\n",
    "        trainer.test(train_loader, no_print=False)\n",
    "        # tsne_plot(trainer.outputs.cpu(), trainer.labels.cpu())\n",
    "        print(\"Validataion:\")\n",
    "        trainer.test(val_loader, no_print=False)\n",
    "        metrics_val.append_from(trainer)\n",
    "        # tsne_plot(trainer.outputs.cpu(), trainer.labels.cpu())\n",
    "        print(\"Test:\")\n",
    "        trainer.test(test_loader, no_print=False, calculate_ig=False)\n",
    "        metrics_test.append_from(trainer)\n",
    "        outputs_test.append_from(trainer)\n",
    "        # tsne_plot(trainer.outputs.cpu(), trainer.labels.cpu())\n",
    "        # ig_test = torch.concat((ig_test, trainer.attr_ig), dim=0)\n",
    "        print()\n",
    "\n",
    "        inputs, labels = next(iter(test_loader))\n",
    "        # inputs, demogr, labels = next(iter(test_loader))\n",
    "        inputs = inputs.to(device)\n",
    "        # demogr = demogr.to(device)\n",
    "        with torch.no_grad():\n",
    "            outputs = model(inputs)\n",
    "            # outputs = model(inputs, demogr)\n",
    "        predicted_label = outputs.argmax(-1)\n",
    "        print(labels)\n",
    "        print(predicted_label)\n",
    "\n",
    "#         m = get_attention_map(inputs, outputs, model.attentions)\n",
    "#         show_image(inputs[0:4,:,:,:].cpu(), flipud=imflip)\n",
    "#         show_image(inputs[0:4,:,:,:].cpu(), mask=m, flipud=imflip)\n",
    "        # gradcam = cam(input_tensor=inputs[0:1,:,:,:,:], targets=None)\n",
    "        # show_video(inputs[0:4,:,:,:,:].cpu(), animate=True, flipud=True)\n",
    "        # show_video(inputs[0:1,:,:,:,:].cpu(), mask=gradcam[0:1,:,:], animate=False, flipud=True)\n",
    "        break\n",
    "\n",
    "#     beeper.beep(frequency=600, secs=0.5)\n",
    "#     novelty_detection(model, train_loader, val_loader, test_loader)\n",
    "#     metrics_cluster[split_idx,:,:] = evaluate(model, train_loader, val_loader, test_loader)    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5920582c-adab-4b8d-9d19-d110dce054c7",
   "metadata": {},
   "outputs": [],
   "source": [
    "import sklearn.metrics as M\n",
    "# print(ig_test.size())\n",
    "print(\"Final Validation Results\")\n",
    "display(metrics_val)\n",
    "print()\n",
    "print(\"Final Test Results\")\n",
    "display(metrics_test)\n",
    "print()\n",
    "# beeper.beep(frequency=600, secs=0.5)\n",
    "display(outputs_test)\n",
    "labels = np.array(outputs_test.outputs['Labels'].to_list())\n",
    "outputs = np.array(outputs_test.outputs['Outputs'].to_list())\n",
    "display(np.sqrt(np.mean(np.square(outputs-labels), axis=0))) # RMSE\n",
    "display(np.mean(np.abs(outputs-labels)/labels, axis=0)) # MAPE\n",
    "display(\n",
    "    M.r2_score(labels[:,0], outputs[:,0]),\n",
    "    M.r2_score(labels[:,1], outputs[:,1]),\n",
    "    M.r2_score(labels[:,2], outputs[:,2]),\n",
    "    M.r2_score(labels[:,3], outputs[:,3]),\n",
    "    # M.r2_score(labels[:,4], outputs[:,4])\n",
    ")\n",
    "fig, axs = plt.subplots(2, 2, figsize=(8,6))\n",
    "axs[0,0].scatter(labels[:,0], outputs[:,0], marker='.')\n",
    "axs[0,1].scatter(labels[:,1], outputs[:,1], marker='.')\n",
    "axs[1,0].scatter(labels[:,2], outputs[:,2], marker='.')\n",
    "axs[1,1].scatter(labels[:,3], outputs[:,3], marker='.')\n",
    "# axs[1,1].scatter(labels[:,4], outputs[:,4], marker='.')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fe14e959-c7cf-4eb7-b083-e342f6da0bb3",
   "metadata": {},
   "outputs": [],
   "source": [
    "# import ast\n",
    "# import matplotlib.pyplot as plt\n",
    "# import pandas as pd\n",
    "\n",
    "# labels = outputs_test.outputs['Labels'].to_list()\n",
    "# labels = pd.DataFrame(labels)\n",
    "# labels.columns = ['Diagnosis']\n",
    "\n",
    "# info = outputs_test.outputs['Info'].to_list()\n",
    "# info = pd.DataFrame(info)\n",
    "# info.columns = ['Age', 'Sex', 'Height', 'Weight']\n",
    "#                # 'FEV1', 'FVC', 'FEV1/FVC', 'FEF2575']\n",
    "\n",
    "# meta = pd.concat([labels, info], axis=1)\n",
    "\n",
    "# outputs_cls = outputs_test.outputs['Outputs'].to_list()\n",
    "# outputs_cls = pd.DataFrame(outputs_cls)\n",
    "\n",
    "# meta.insert(1, 'Prediction', np.exp(outputs_cls[1])/np.sum(np.exp(outputs_cls),axis=1)) # Softmax\n",
    "\n",
    "# idx_tp = (meta['Diagnosis']==1) & (meta['Prediction']>=0.5)\n",
    "# tp = idx_tp.sum()\n",
    "# idx_fp = (meta['Diagnosis']==0) & (meta['Prediction']>=0.5)\n",
    "# fp = idx_fp.sum()\n",
    "# idx_tn = (meta['Diagnosis']==0) & (meta['Prediction']<0.5)\n",
    "# tn = idx_tn.sum()\n",
    "# idx_fn = (meta['Diagnosis']==1) & (meta['Prediction']<0.5)\n",
    "# fn = idx_fn.sum()\n",
    "# print(tn, fp)\n",
    "# print(fn, tp)\n",
    "\n",
    "# plt.figure()\n",
    "# plt.hist(meta['Prediction'][meta['Diagnosis']==0], bins=50, range=(0,1), alpha = 0.3, color='b', edgecolor='k', linewidth=1)\n",
    "# plt.hist(meta['Prediction'][meta['Diagnosis']==1], bins=50, range=(0,1), alpha = 0.3, color='r', edgecolor='k', linewidth=1)\n",
    "# plt.legend(['Healthy', 'Asthma'])\n",
    "# plt.show()\n",
    "\n",
    "# # cross_entropy = -(y_true*np.log10(y_pred[1]) + (1-y_true)*np.log10(y_pred[0]))\n",
    "\n",
    "# # plt.figure()\n",
    "# # # plt.hist(cross_entropy, bins=np.logspace(np.log10(0.1),np.log10(10.0), 50), edgecolor='k', linewidth=1)\n",
    "# # plt.hist(cross_entropy, bins=50, edgecolor='k', linewidth=1)\n",
    "# # plt.legend(['Healthy', 'Asthma'])\n",
    "# # # plt.gca().set_xscale(\"log\")\n",
    "# # plt.vlines(-np.log10(0.5), 0, 500, color='r')\n",
    "# # plt.show()\n",
    "\n",
    "# # y_pred = y_pred.idxmax(axis=1)\n",
    "\n",
    "# sens = tp/(tp+fn)\n",
    "# spec = tn/(fp+tn)\n",
    "# print('Sens:', sens)\n",
    "# print('Spec:', spec)\n",
    "# print('BalAcc:', (sens+spec)/2)\n",
    "\n",
    "# plt.figure(figsize=(20,4))\n",
    "# plt.subplot(2,4,1)\n",
    "# plt.hist(pd.concat([meta['Age'][idx_tp], meta['Age'][idx_fn]], axis=1), range=(meta['Age'].min(),meta['Age'].max()), alpha = 0.3, stacked=True, edgecolor='k', linewidth=1)\n",
    "# plt.legend(['True Asthma', 'False Healthy'])\n",
    "# plt.title('Age')\n",
    "# plt.subplot(2,4,5)\n",
    "# plt.hist(pd.concat([meta['Age'][idx_tn], meta['Age'][idx_fp]], axis=1), range=(meta['Age'].min(),meta['Age'].max()), alpha = 0.3, stacked=True, edgecolor='k', linewidth=1)\n",
    "# plt.legend(['True Healthy', 'False Asthma'])\n",
    "# plt.subplot(2,4,2)\n",
    "# plt.hist(pd.concat([meta['Sex'][idx_tp], meta['Sex'][idx_fn]], axis=1), range=(meta['Sex'].min(),meta['Sex'].max()), alpha = 0.3, stacked=True, edgecolor='k', linewidth=1)\n",
    "# plt.legend(['True Asthma', 'False Healthy'])\n",
    "# plt.title('Sex')\n",
    "# plt.subplot(2,4,6)\n",
    "# plt.hist(pd.concat([meta['Sex'][idx_tn], meta['Sex'][idx_fp]], axis=1), range=(meta['Sex'].min(),meta['Sex'].max()), alpha = 0.3, stacked=True, edgecolor='k', linewidth=1)\n",
    "# plt.legend(['True Healthy', 'False Asthma'])\n",
    "# plt.subplot(2,4,3)\n",
    "# plt.hist(pd.concat([meta['Height'][idx_tp], meta['Height'][idx_fn]], axis=1), range=(meta['Height'].min(),meta['Height'].max()), alpha = 0.3, stacked=True, edgecolor='k', linewidth=1)\n",
    "# plt.legend(['True Asthma', 'False Healthy'])\n",
    "# plt.title('Height')\n",
    "# plt.subplot(2,4,7)\n",
    "# plt.hist(pd.concat([meta['Height'][idx_tn], meta['Height'][idx_fp]], axis=1), range=(meta['Height'].min(),meta['Height'].max()), alpha = 0.3, stacked=True, edgecolor='k', linewidth=1)\n",
    "# plt.legend(['True Healthy', 'False Asthma'])\n",
    "# plt.subplot(2,4,4)\n",
    "# plt.hist(pd.concat([meta['Weight'][idx_tp], meta['Weight'][idx_fn]], axis=1), range=(meta['Weight'].min(),meta['Weight'].max()), alpha = 0.3, stacked=True, edgecolor='k', linewidth=1)\n",
    "# plt.legend(['True Asthma', 'False Healthy'])\n",
    "# plt.title('Weight')\n",
    "# plt.subplot(2,4,8)\n",
    "# plt.hist(pd.concat([meta['Weight'][idx_tn], meta['Weight'][idx_fp]], axis=1), range=(meta['Weight'].min(),meta['Weight'].max()), alpha = 0.3, stacked=True, edgecolor='k', linewidth=1)\n",
    "# plt.legend(['True Healthy', 'False Asthma'])\n",
    "# plt.show()\n",
    "\n",
    "# plt.figure(figsize=(6,4))\n",
    "# plt.subplot(2,1,1)\n",
    "# plt.hist(pd.concat([meta['Age'][idx_tp], meta['Age'][idx_fn]], axis=1), bins=list(range(0, 78, 6)), alpha = 0.3, stacked=True, edgecolor='k', linewidth=1)\n",
    "# plt.legend(['True Asthma', 'False Healthy'])\n",
    "# plt.subplot(2,1,2)\n",
    "# plt.hist(pd.concat([meta['Age'][idx_tn], meta['Age'][idx_fp]], axis=1), bins=list(range(0, 78, 6)), alpha = 0.3, stacked=True, edgecolor='k', linewidth=1)\n",
    "# plt.legend(['True Healthy', 'False Asthma'])\n",
    "# plt.show()\n",
    "\n",
    "# thre = np.zeros(1001)\n",
    "# sens = np.zeros(1001)\n",
    "# spec = np.zeros(1001)\n",
    "# for i in range(0,1001):\n",
    "#     idx_tp = (meta['Diagnosis']==1) & (meta['Prediction']>=i/1000)\n",
    "#     tp = idx_tp.sum()\n",
    "#     idx_fp = (meta['Diagnosis']==0) & (meta['Prediction']>=i/1000)\n",
    "#     fp = idx_fp.sum()\n",
    "#     idx_tn = (meta['Diagnosis']==0) & (meta['Prediction']<i/1000)\n",
    "#     tn = idx_tn.sum()\n",
    "#     idx_fn = (meta['Diagnosis']==1) & (meta['Prediction']<i/1000)\n",
    "#     fn = idx_fn.sum()\n",
    "#     thre[i] = i/1000\n",
    "#     sens[i] = tp/(tp+fn)\n",
    "#     spec[i] = tn/(fp+tn)\n",
    "    \n",
    "# plt.figure()\n",
    "# plt.plot(thre, sens)\n",
    "# plt.plot(thre, spec)\n",
    "# plt.title('Sensitivity and Specificity vs. Threshold')\n",
    "# plt.legend(['Sensitivity', 'Specificity'])\n",
    "# plt.xlabel('Threshold')\n",
    "# plt.show()\n",
    "\n",
    "# plt.figure()\n",
    "# # skplt.metrics.plot_roc_curve(meta['Diagnosis'], outputs_cls)\n",
    "# plt.plot(1-spec, sens)\n",
    "# plt.plot([0,1], [0,1], '--k')\n",
    "# plt.title('ROC Curve')\n",
    "# plt.xlabel('False Positive Rate\\n(1-Specificity)')\n",
    "# plt.ylabel('True Positive Rate\\n(Sensitivity)')\n",
    "# plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7339b96d-5ba5-4cc4-a224-95b13a3c1bd9",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}