sk_mobilenet.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from __future__ import division,print_function\n",
    "\n",
    "%matplotlib inline\n",
    "%load_ext autoreload\n",
    "%autoreload 2\n",
    "\n",
    "import sys\n",
    "sys.path.append('../')\n",
    "from tqdm.notebook import tqdm\n",
    "import numpy as np\n",
    "import os\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "from torchvision import transforms, models, datasets\n",
    "import utils.calculate_log as callog\n",
    "from my_models import mobilenet\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Setting the model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/patcha/.local/lib/python3.6/site-packages/torch/serialization.py:593: SourceChangeWarning: source code of class 'torch.nn.modules.loss.CrossEntropyLoss' has changed. you can retrieve the original source code by accessing the object's source attribute or set `torch.nn.Module.dump_patches = True` and use the patch tool to revert the changes.\n",
      "  warnings.warn(msg, SourceChangeWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Done!\n"
     ]
    }
   ],
   "source": [
    "torch_model = mobilenet.Net(models.mobilenet_v2(pretrained=False), 8)\n",
    "ckpt = torch.load(\"checkpoints/mobilenet_checkpoint.pth\")\n",
    "torch_model.load_state_dict(ckpt['model_state_dict'])\n",
    "torch_model.eval()\n",
    "torch_model.cuda()\n",
    "print(\"Done!\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Setting the hook register"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total number of registered hooked layers: 35\n"
     ]
    }
   ],
   "source": [
    "feat_maps = list()\n",
    "def _hook_fn(self, input, output):\n",
    "    feat_maps.append(output)\n",
    "    \n",
    "\n",
    "def hook_layers(model):\n",
    "    hooked_layers = list()\n",
    "    for layer in torch_model.modules():\n",
    "        if isinstance(layer, models.mobilenet.ConvBNReLU):        \n",
    "#         if isinstance(layer, models.mobilenet.ConvBNReLU) or isinstance(layer, nn.Conv2d):\n",
    "            hooked_layers.append(layer)\n",
    "    return hooked_layers\n",
    "\n",
    "\n",
    "def register_layers(layers):\n",
    "    regs_layers = list()\n",
    "    for lay in layers:\n",
    "        regs_layers.append(lay.register_forward_hook(_hook_fn))\n",
    "    return regs_layers\n",
    "\n",
    "\n",
    "def unregister_layers(reg_layers):\n",
    "    for lay in reg_layers:\n",
    "        lay.remove()\n",
    "                    \n",
    "\n",
    "def get_feat_maps(model, batch_img):\n",
    "    batch_img = batch_img.cuda()\n",
    "    with torch.no_grad():\n",
    "        preds = model(batch_img)\n",
    "\n",
    "    preds = F.softmax(preds, dim=1)\n",
    "    maps = feat_maps.copy()\n",
    "    feat_maps.clear()\n",
    "    return preds, maps\n",
    "\n",
    "## Setting the hook\n",
    "hl = hook_layers (torch_model)\n",
    "rgl = register_layers (hl)\n",
    "print (\"Total number of registered hooked layers:\", len(rgl))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Loading the data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## In distributions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "batch_size = 15\n",
    "trans = transforms.Compose([\n",
    "#             transforms.Resize((224,224)),\n",
    "            transforms.ToTensor(),\n",
    "            transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225])\n",
    "        ])\n",
    "\n",
    "\n",
    "sk_train = torch.utils.data.DataLoader(\n",
    "                datasets.ImageFolder(\"data/skin_cancer/train/\",transform=trans),\n",
    "                batch_size=batch_size,\n",
    "                shuffle=False)\n",
    "\n",
    "sk_test = torch.utils.data.DataLoader(\n",
    "                datasets.ImageFolder(\"data/skin_cancer/test/\",transform=trans),\n",
    "                batch_size=batch_size,\n",
    "                shuffle=False)\n",
    "\n",
    "sk_val = torch.utils.data.DataLoader(\n",
    "                datasets.ImageFolder(\"data/skin_cancer/val/\",transform=trans),\n",
    "                batch_size=batch_size,\n",
    "                shuffle=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Out-of-distributions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "skin_cli = torch.utils.data.DataLoader(\n",
    "                datasets.ImageFolder(\"data/skins/clinical/\",transform=trans),\n",
    "                batch_size=batch_size,\n",
    "                shuffle=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "skin_derm = torch.utils.data.DataLoader(\n",
    "                datasets.ImageFolder(\"data/skins/dermoscopy/\",transform=trans),\n",
    "                batch_size=batch_size,\n",
    "                shuffle=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "imgnet = torch.utils.data.DataLoader(\n",
    "                datasets.ImageFolder(\"data/imagenet/\",transform=trans),\n",
    "                batch_size=batch_size,\n",
    "                shuffle=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "corrupted = torch.utils.data.DataLoader(\n",
    "                datasets.ImageFolder(\"data/corrupted/bbox/\",transform=trans),\n",
    "                batch_size=batch_size,\n",
    "                shuffle=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "corrupted_70 = torch.utils.data.DataLoader(\n",
    "                datasets.ImageFolder(\"data/corrupted/bbox_70/\",transform=trans),\n",
    "                batch_size=batch_size,\n",
    "                shuffle=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "nct = torch.utils.data.DataLoader(\n",
    "                datasets.ImageFolder(\"data/nct/\",transform=trans),     \n",
    "                batch_size=batch_size,\n",
    "                shuffle=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Gram-Matrix operations"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Gram matrix operator"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "def norm_min_max(x):\n",
    "    ma = torch.max(x,dim=1)[0].unsqueeze(1)\n",
    "    mi = torch.min(x,dim=1)[0].unsqueeze(1)\n",
    "    x = (x-mi)/(ma-mi)\n",
    "    return x\n",
    "\n",
    "def get_sims_gram_matrix (maps, power): \n",
    "    maps = maps ** power    \n",
    "    maps = maps.reshape(maps.shape[0],maps.shape[1],-1)    \n",
    "    gram = ((torch.matmul(maps,maps.transpose(dim0=2,dim1=1)))).sum(2)\n",
    "    gram = (gram.sign()*torch.abs(gram)**(1/power)).reshape(gram.shape[0],-1)  \n",
    "    gram = norm_min_max(gram)\n",
    "    return gram\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Considering samples per label"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "def _get_sim_per_labels(data_loader, power, use_preds=True):\n",
    "    \n",
    "    sims_per_label = None\n",
    "    if not isinstance(power, list) and not isinstance(power, range):\n",
    "        power = [power]\n",
    "    \n",
    "    for data in tqdm(data_loader):\n",
    "        img_batch, labels = data \n",
    "        preds, maps_list = get_feat_maps(torch_model, img_batch)\n",
    "      \n",
    "        if use_preds:\n",
    "            labels = preds.argmax(dim=1)  \n",
    "                \n",
    "        if sims_per_label is None:\n",
    "            sims_per_label = [[[] for _ in range(len(maps_list))] for _ in range(preds.shape[1])]  \n",
    "           \n",
    "        for layer, maps in enumerate(maps_list): \n",
    "            for p in power:\n",
    "                sims = get_sims_gram_matrix (maps, p)\n",
    "\n",
    "                for sim, lab in zip(sims, labels):              \n",
    "                    sims_per_label[lab.item()][layer].append(sim.cpu()) \n",
    "                \n",
    "    return sims_per_label\n",
    "\n",
    "\n",
    "def get_min_max_per_label(data_loader, power):\n",
    "    \n",
    "    sims_per_label = _get_sim_per_labels(data_loader, power)\n",
    "    sims_per_label_min = [[[] for _ in range(len(sims_per_label[0]))] for _ in range(len(sims_per_label))] \n",
    "    sims_per_label_max = [[[] for _ in range(len(sims_per_label[0]))] for _ in range(len(sims_per_label))] \n",
    "    \n",
    "    \n",
    "    print (\"-- Computing the values...\")\n",
    "    for lab_idx in range(len(sims_per_label)):\n",
    "        for layer_idx in range(len(sims_per_label[lab_idx])):\n",
    "            temp = torch.stack(sims_per_label[lab_idx][layer_idx])\n",
    "            sims_per_label_min[lab_idx][layer_idx] = temp.min(dim=0)[0] \n",
    "            sims_per_label_max[lab_idx][layer_idx] = temp.max(dim=0)[0]\n",
    "    \n",
    "    del sims_per_label\n",
    "    \n",
    "    return sims_per_label_min, sims_per_label_max\n",
    "\n",
    "\n",
    "def get_layer_gaps(mins, maxs):  \n",
    "    num_lab, num_lay = len(mins), len(mins[0])    \n",
    "    gaps = torch.zeros(num_lab, num_lay)\n",
    "    gaps = gaps.cuda()\n",
    "    \n",
    "    for lab in range(num_lab):      \n",
    "        for layer in range(num_lay):\n",
    "            gaps[lab][layer] = (maxs[lab][layer]-mins[lab][layer]).sum()\n",
    "            \n",
    "    return gaps.cpu().numpy()\n",
    "\n",
    "\n",
    "def get_dev_scores_per_label(data_loader, power, sims_min, sims_max, ep=10e-6):\n",
    "    \n",
    "    if not isinstance(power, list) and not isinstance(power, range):\n",
    "        power = [power]\n",
    "    \n",
    "    dev_scores = list()    \n",
    "    for data in tqdm(data_loader):\n",
    "        img_batch, _ = data \n",
    "        preds_batch, maps_list = get_feat_maps(torch_model, img_batch)                \n",
    "        labels = preds_batch.argmax(dim=1)\n",
    "        batch_scores = list()\n",
    "       \n",
    "        for layer, maps in enumerate(maps_list):\n",
    "                \n",
    "            score_layer = 0\n",
    "            for p in power:\n",
    "                sims = get_sims_gram_matrix (maps, p)  \n",
    "                _sim_min = torch.zeros(sims.shape[0], sims.shape[1]).cuda()\n",
    "                _sim_max = torch.zeros(sims.shape[0], sims.shape[1]).cuda()\n",
    "            \n",
    "                for k, lab in enumerate(labels):\n",
    "                    _sim_min[k] = sims_min[lab.item()][layer]\n",
    "                    _sim_max[k] = sims_max[lab.item()][layer]            \n",
    "            \n",
    "                score_layer += (F.relu(_sim_min-sims)/torch.abs(_sim_min+ep)).sum(dim=1, keepdim=True)\n",
    "                score_layer += (F.relu(sims-_sim_max)/torch.abs(_sim_max+ep)).sum(dim=1, keepdim=True)\n",
    "           \n",
    "            batch_scores.append(score_layer)            \n",
    "            \n",
    "        dev_scores.append(torch.cat(batch_scores, dim=1)) \n",
    "\n",
    "    return torch.cat(dev_scores).cpu().numpy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "def detect_mean(all_test_std, all_ood_std, gaps=None): \n",
    "    \n",
    "    avg_results = dict()\n",
    "    indices = list(range(len(all_test_std)))\n",
    "    split = int(np.floor(0.1 * len(all_test_std))) \n",
    "    for i in range(1,11):\n",
    "        np.random.seed(i)\n",
    "        np.random.shuffle(indices)\n",
    "        \n",
    "        val_std = all_test_std[indices[:split]]\n",
    "        test_std = all_test_std[indices[split:]]\n",
    "        \n",
    "        if gaps is not None:\n",
    "            t95 = (val_std.sum(axis=0) + gaps.mean(0))\n",
    "        else:\n",
    "            t95 = val_std.mean(axis=0) + 10**-7\n",
    "        \n",
    "        test_std = ((test_std)/t95[np.newaxis,:]).sum(axis=1)\n",
    "        ood_std = ((all_ood_std)/t95[np.newaxis,:]).sum(axis=1)\n",
    "\n",
    "        results = callog.compute_metric(-test_std,-ood_std)  \n",
    "\n",
    "        for m in results:\n",
    "            avg_results[m] = avg_results.get(m,0)+results[m]\n",
    "    \n",
    "    for m in avg_results:\n",
    "        avg_results[m] /= i\n",
    "        \n",
    "        \n",
    "    callog.print_results(avg_results)\n",
    "    \n",
    "    return avg_results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "def detect(all_test_std, all_ood_std):     \n",
    "    \n",
    "    indices = list(range(len(all_test_std)))\n",
    "    split = int(np.floor(0.1 * len(all_test_std))) \n",
    "    np.random.seed(10)\n",
    "    np.random.shuffle(indices)\n",
    "        \n",
    "    val_std = all_test_std[indices[:split]]\n",
    "    test_std = all_test_std[indices[split:]]\n",
    "        \n",
    "    t95 = val_std.mean(axis=0) + 10**-7\n",
    "        \n",
    "    test_std = ((test_std)/t95[np.newaxis,:]).sum(axis=1)\n",
    "    ood_std = ((all_ood_std)/t95[np.newaxis,:]).sum(axis=1)\n",
    "\n",
    "    results = callog.compute_metric(-test_std,-ood_std)  \n",
    "\n",
    "    callog.print_results(results)\n",
    "    \n",
    "    return results"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# OOD detection per label"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "- Getting mins/maxs\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ede95c740b704f8d9b19c3d89e4d5350",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=1351.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "-- Computing the values...\n",
      "- Getting the gaps\n",
      "- Getting test stdevs\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "9fea1ab2bf8d468ba625fa20eb02ce16",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=169.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "- Getting test stdevs\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a49de18bf7e440f69b040cd7f7b9f043",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=169.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "power = 1\n",
    "\n",
    "print (\"- Getting mins/maxs\")\n",
    "mins, maxs = get_min_max_per_label(sk_train, power)\n",
    "\n",
    "print(\"- Getting the gaps\")\n",
    "gaps = get_layer_gaps(mins, maxs) \n",
    "\n",
    "print (\"- Getting test stdevs\")\n",
    "sk_test_stdev = get_dev_scores_per_label(sk_test, power, mins, maxs)\n",
    "\n",
    "print (\"- Getting test stdevs\")\n",
    "sk_val_stdev = get_dev_scores_per_label(sk_val, power, mins, maxs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Releasing the GPU cache memory\n",
    "torch.cuda.empty_cache()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Testing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Skins dermoscopy\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6b18eb0ef3b8437bb2b08389f7064b07",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=105.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      " TNR    AUROC  DTACC  AUIN   AUOUT \n",
      " 72.773 94.040 87.863 93.462 91.418\n"
     ]
    }
   ],
   "source": [
    "print(\"Skins dermoscopy\")\n",
    "skin_derm_stdev = get_dev_scores_per_label(skin_derm, power, mins, maxs)\n",
    "skin_derm_results = detect_mean(sk_test_stdev, skin_derm_stdev)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Skins clinical\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "35c0f2f7a962492b8d39e65c43bd41f9",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=49.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      " TNR    AUROC  DTACC  AUIN   AUOUT \n",
      " 83.817 96.352 90.997 98.483 88.027\n"
     ]
    }
   ],
   "source": [
    "print(\"Skins clinical\")\n",
    "skin_cli_stdev = get_dev_scores_per_label(skin_cli, power, mins, maxs)\n",
    "skin_cli_results = detect_mean(sk_test_stdev, skin_cli_stdev)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ImageNet\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "90a28e3869cd4e5eb0fe4fa8f3b26f3f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=200.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      " TNR    AUROC  DTACC  AUIN   AUOUT \n",
      " 92.420 98.458 94.362 98.387 98.426\n"
     ]
    }
   ],
   "source": [
    "print(\"ImageNet\")\n",
    "imgnet_stdev = get_dev_scores_per_label(imgnet, power, mins, maxs)\n",
    "imgent_results = detect_mean(sk_test_stdev, imgnet_stdev)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Corrupted images bbox\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "505467bc7a024cd3b344dd2cfeea4cfb",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=139.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      " TNR    AUROC  DTACC  AUIN   AUOUT \n",
      " 98.742 98.755 97.052 99.192 97.093\n"
     ]
    }
   ],
   "source": [
    "print(\"Corrupted images bbox\")\n",
    "corrupted_stdev = get_dev_scores_per_label(corrupted, power, mins, maxs)\n",
    "corrupted_results = detect_mean(sk_test_stdev, corrupted_stdev)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Corrupted images bbox 70\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "61947d48a5b7476a883170ec7fcd8ddc",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=164.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      " TNR    AUROC  DTACC  AUIN   AUOUT \n",
      " 100.000 99.886 99.483 99.909 99.682\n"
     ]
    }
   ],
   "source": [
    "print(\"Corrupted images bbox 70\")\n",
    "corrupted_70_stdev = get_dev_scores_per_label(corrupted_70, power, mins, maxs)\n",
    "corrupted_70_results = detect_mean(sk_test_stdev, corrupted_70_stdev)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "NCT\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ac6208b1a6cf4607995023458bb2562c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=90.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      " TNR    AUROC  DTACC  AUIN   AUOUT \n",
      " 100.000 99.739 98.898 99.854 99.241\n"
     ]
    }
   ],
   "source": [
    "print(\"NCT\")\n",
    "nct_stdev = get_dev_scores_per_label(nct, power, mins, maxs)\n",
    "nct_results = detect_mean(sk_test_stdev, nct_stdev)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Summary"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "72.773\n",
      "83.817\n",
      "92.42\n",
      "98.742\n",
      "100.0\n",
      "100.0\n",
      "10.41\n"
     ]
    }
   ],
   "source": [
    "print(round(skin_derm_results['TNR']*100,3))\n",
    "print(round(skin_cli_results['TNR']*100,3))\n",
    "print(round(imgent_results['TNR']*100,3))\n",
    "print(round(corrupted_results['TNR']*100,3))\n",
    "print(round(corrupted_70_results['TNR']*100,3))\n",
    "print(round(nct_results['TNR']*100,3))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}