WSOL_Localization_Inference.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "Kz3ensRZr23X"
      },
      "outputs": [],
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/drive')"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!unzip '/content/drive/MyDrive/Kaggle/data.zip' -d '/content/'"
      ],
      "metadata": {
        "id": "7HxM0gARr9BG"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "import numpy as np # linear algebra\n",
        "import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n",
        "import os\n",
        "from cv2 import imread, createCLAHE # read and equalize images\n",
        "from glob import glob\n",
        "import matplotlib.pyplot as plt\n",
        "%matplotlib inline\n",
        "from pandas.core.common import flatten\n",
        "\n",
        "import os\n",
        "import torch\n",
        "import pandas as pd\n",
        "from skimage import io, transform\n",
        "import numpy as np\n",
        "import matplotlib.pyplot as plt\n",
        "from torch.utils.data import Dataset, DataLoader\n",
        "from torchvision import transforms, utils\n",
        "\n",
        "# Ignore warnings\n",
        "import warnings\n",
        "warnings.filterwarnings(\"ignore\")\n",
        "\n",
        "plt.ion()   # interactive mode"
      ],
      "metadata": {
        "id": "pEWB_H_UuSlr"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "from __future__ import print_function, division\n",
        "\n",
        "# pytorch imports\n",
        "import torch\n",
        "import torch.nn as nn\n",
        "import torch.optim as optim\n",
        "from torch.optim import lr_scheduler\n",
        "from torch.autograd import Variable\n",
        "import torchvision\n",
        "from torchvision import datasets, models, transforms\n",
        "from torch.utils.data import Dataset, DataLoader\n",
        "from torchvision import transforms, utils\n",
        "\n",
        "# image imports\n",
        "from skimage import io, transform\n",
        "from PIL import Image\n",
        "\n",
        "# general imports\n",
        "import os\n",
        "import time\n",
        "from shutil import copyfile\n",
        "from shutil import rmtree\n",
        "\n",
        "# data science imports\n",
        "import pandas as pd\n",
        "import numpy as np\n",
        "import csv"
      ],
      "metadata": {
        "id": "4zpoygRsuZzg"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "%matplotlib inline\n",
        "# All imports\n",
        "import matplotlib.pyplot as plt\n",
        "import torch\n",
        "from torch.nn.functional import softmax, interpolate\n",
        "from torchvision.io.image import read_image\n",
        "from torchvision.models import resnet18, densenet121, resnet50\n",
        "from torchvision.transforms.functional import normalize, resize, to_pil_image"
      ],
      "metadata": {
        "id": "oeRSvH1467n5"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "from itertools import chain\n",
        "import random\n",
        "np.random.seed(25)\n",
        "\n",
        "df = pd.read_csv('/content/Data_Entry_2017.csv')\n",
        "df.drop(['OriginalImage[Width', 'Height]', 'OriginalImagePixelSpacing[x', 'y]', 'Unnamed: 11'], axis=1, inplace=True)\n",
        "num_obs = len(df)\n",
        "# print('Number of observations:',num_obs)\n",
        "\n",
        "my_glob = glob('/content/images*/images/*.png')\n",
        "# print('Number of Observations: ', len(my_glob))\n",
        "\n",
        "full_img_paths = {os.path.basename(x): x for x in my_glob}\n",
        "df['full_path'] = df['Image Index'].map(full_img_paths.get)\n",
        "\n",
        "train_val_list = pd.read_csv('/content/train_val_list.txt', header=None, names = ['image_list'])\n",
        "test_list = pd.read_csv('/content/test_list.txt', header=None, names = ['image_list'])\n",
        "\n",
        "train = df[df['Image Index'].isin(train_val_list['image_list'].values)].reset_index(drop=True)\n",
        "test = df[df['Image Index'].isin(test_list['image_list'].values)].reset_index(drop=True)\n",
        "\n",
        "labels_discard = ['Consolidation', 'Edema', 'Emphysema', 'Fibrosis', 'Hernia', 'Pleural_Thickening']\n",
        "for i in labels_discard:\n",
        "    train = train[~train['Finding Labels'].str.contains(i)]\n",
        "    test = test[~test['Finding Labels'].str.contains(i)]\n",
        "\n",
        "# train = pd.concat([train[~train['Finding Labels'].str.contains('No Finding')],\n",
        "#                   train[train['Finding Labels'].str.contains('No Finding')].drop_duplicates(subset=['Finding Labels', 'Patient ID', 'View Position'], keep='first')]).sort_values(by='Image Index')\n",
        "\n",
        "def one_hot_enc(df):\n",
        "    df['Finding Labels'] = df['Finding Labels'].map(lambda x: x.replace('No Finding', ''))\n",
        "    all_labels = np.unique(list(chain(*df['Finding Labels'].map(lambda x: x.split('|')).tolist())))\n",
        "    for c_label in all_labels:\n",
        "        if len(c_label)>1: # leave out empty labels\n",
        "            df[c_label] = df['Finding Labels'].map(lambda finding: 1 if c_label in finding else 0)\n",
        "    return df\n",
        "\n",
        "train = one_hot_enc(train)\n",
        "test = one_hot_enc(test)\n",
        "\n",
        "train_image_paths = list(flatten(train['full_path'].values))\n",
        "test_image_paths = list(flatten(test['full_path'].values))\n",
        "\n",
        "l = np.unique(train['Patient ID'].values)\n",
        "np.random.shuffle(l)\n",
        "cut = int(np.round(((90/100)*len(l)), decimals=0))\n",
        "train_values = l[:cut]\n",
        "val_values = l[cut:]\n",
        "train_dict = dict.fromkeys(train_values, 'train')\n",
        "train_dict.update(dict.fromkeys(val_values, 'val'))\n",
        "train['fold'] = train['Patient ID'].map(train_dict.get)\n",
        "\n",
        "train = train[['full_path', 'Image Index', 'fold', 'Atelectasis', 'Cardiomegaly', 'Effusion', 'Infiltration',\n",
        "               'Mass', 'Nodule', 'Pneumonia', 'Pneumothorax']]\n",
        "test['fold'] = 'test'\n",
        "test = test[['full_path', 'Image Index', 'fold', 'Atelectasis', 'Cardiomegaly', 'Effusion', 'Infiltration',\n",
        "             'Mass', 'Nodule', 'Pneumonia', 'Pneumothorax']]\n",
        "train.reset_index(drop=True, inplace=True)\n",
        "test.reset_index(drop=True, inplace=True)"
      ],
      "metadata": {
        "id": "pPyQqpOgWWIy"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "len(df_box)"
      ],
      "metadata": {
        "id": "otMp6UxyWXEH"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "import pandas as pd\n",
        "import numpy as np\n",
        "from glob import glob\n",
        "import os\n",
        "\n",
        "df_box = pd.read_csv('/content/BBox_List_2017.csv')\n",
        "df_box['xmin'] = df_box['Bbox [x']\n",
        "df_box['ymin'] = df_box['y']\n",
        "# df_box['xmax'] = df_box['xmin'] + df_box['w']\n",
        "# df_box['ymax'] = df_box['ymin'] + df_box['h]']\n",
        "df_box['w'] = df_box['w']\n",
        "df_box['h'] = df_box['h]']\n",
        "\n",
        "my_glob = glob('/content/images*/images/*.png')\n",
        "# print('Number of Observations: ', len(my_glob))\n",
        "\n",
        "full_img_paths = {os.path.basename(x): x for x in my_glob}\n",
        "df_box['full_path'] = df_box['Image Index'].map(full_img_paths.get)\n",
        "\n",
        "df_box = df_box[['full_path', 'Image Index', 'Finding Label', 'xmin', 'ymin', 'w', 'h']]\n",
        "# df_box['xmin'] = df_box['xmin'].round()\n",
        "# df_box['ymin'] = df_box['ymin'].round()\n",
        "# df_box['xmax'] = df_box['xmax'].round()\n",
        "# df_box['ymax'] = df_box['ymax'].round()\n",
        "df_box.head()"
      ],
      "metadata": {
        "id": "sODaj9d9ucij"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def large_to_small(x1, y1, w1, h1, cropped = True): # convert 1024x1024 to 224x224 (which is center-cropped from 256x256)\n",
        "    x2 = x1 / 4\n",
        "    y2 = y1 / 4\n",
        "    w2 = w1 / 4\n",
        "    h2 = h1 / 4\n",
        "    if cropped:\n",
        "        if x2 < 16:\n",
        "            x2 = 0\n",
        "            w2 = w2 - 16\n",
        "        else:\n",
        "            x2 = x2 - 16\n",
        "        if x2 + w2 > 224:\n",
        "            w2 = 224 - x2\n",
        "        if y2 < 16:\n",
        "            y2 = 0\n",
        "            h2 = h2 - 16\n",
        "        else:\n",
        "            y2 = y2 - 16\n",
        "        if y2 + h2 > 224:\n",
        "            h2 = 224 - y2\n",
        "    return int(x2), int(y2), int(w2), int(h2)\n",
        "\n",
        "def intersect(box_a, box_b):\n",
        "    A = box_a.size(0)\n",
        "    B = box_b.size(0)\n",
        "    max_xy = torch.min(box_a[:, 2:].unsqueeze(1).expand(A, B, 2),\n",
        "                       box_b[:, 2:].unsqueeze(0).expand(A, B, 2))\n",
        "    min_xy = torch.max(box_a[:, :2].unsqueeze(1).expand(A, B, 2),\n",
        "                       box_b[:, :2].unsqueeze(0).expand(A, B, 2))\n",
        "    inter = torch.clamp((max_xy - min_xy), min=0)\n",
        "    return inter[:, :, 0] * inter[:, :, 1]\n",
        "\n",
        "def jaccard(box_a, box_b):\n",
        "    inter = intersect(box_a, box_b)\n",
        "    area_a = ((box_a[:, 2]-box_a[:, 0]) *\n",
        "              (box_a[:, 3]-box_a[:, 1])).unsqueeze(1).expand_as(inter)  # [A,B]\n",
        "    area_b = ((box_b[:, 2]-box_b[:, 0]) *\n",
        "              (box_b[:, 3]-box_b[:, 1])).unsqueeze(0).expand_as(inter)  # [A,B]\n",
        "    union = area_a + area_b - inter\n",
        "    return inter / union  # [A,B]\n",
        "\n",
        "def iobb(box_a, box_b):\n",
        "    inter = intersect(box_a, box_b)\n",
        "    area_a = ((box_a[:, 2]-box_a[:, 0]) *\n",
        "              (box_a[:, 3]-box_a[:, 1])).unsqueeze(1).expand_as(inter)  # [A,B]\n",
        "    area_b = ((box_b[:, 2]-box_b[:, 0]) *\n",
        "              (box_b[:, 3]-box_b[:, 1])).unsqueeze(0).expand_as(inter)  # [A,B]\n",
        "#     union = area_a + area_b - inter\n",
        "    return inter / area_a  # [A,B]\n",
        "\n",
        "def contains(xywh1, xywh2): # returns True if xywh2 is completely inside xywh1\n",
        "    x1, x2, x3, x4 = xywh1\n",
        "    y1, y2, y3, y4 = xywh2\n",
        "    if y1 < x1:\n",
        "        return False\n",
        "    if y2 < x2:\n",
        "        return False\n",
        "    if y3 > x3:\n",
        "        return False\n",
        "    if y4 > x4:\n",
        "        return False\n",
        "    return True"
      ],
      "metadata": {
        "id": "Cn72jQNBu5Ws"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "import torchvision.transforms as T\n",
        "\n",
        "preprocess = T.Compose([\n",
        "   T.Resize(224),\n",
        "   T.CenterCrop(224),\n",
        "   T.ToTensor(),\n",
        "   T.Normalize(\n",
        "       mean=[0.485, 0.456, 0.406],\n",
        "       std=[0.229, 0.224, 0.225]\n",
        "   )\n",
        "])\n",
        "\n",
        "convert_tensor = T.Compose([\n",
        "   T.ToTensor(),\n",
        "])"
      ],
      "metadata": {
        "id": "EpDTlWQ2u5Zj"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "from PIL import Image\n",
        "\n",
        "class CreateDataset(Dataset):\n",
        "\n",
        "    def __init__(self, train, transform=None):\n",
        "        self.df = train\n",
        "        self.transform = transform\n",
        "\n",
        "        self.df = self.df.set_index(\"Image Index\")\n",
        "\n",
        "        RESULT_PATH = \"results/\"\n",
        "\n",
        "    def __len__(self):\n",
        "        return len(self.df)\n",
        "\n",
        "    def __getitem__(self, idx):\n",
        "        if torch.is_tensor(idx):\n",
        "            idx = idx.tolist()\n",
        "\n",
        "        image = Image.open(self.df.iloc[idx, 0])\n",
        "        image = image.convert('RGB')\n",
        "\n",
        "        if self.transform:\n",
        "            image = self.transform(image)\n",
        "\n",
        "        return image"
      ],
      "metadata": {
        "id": "3uNqv-GYu5eW"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "mean = [0.485, 0.456, 0.406]\n",
        "std = [0.229, 0.224, 0.225]\n",
        "N_LABELS = 8  # we are predicting 8 labels\n",
        "data_transforms = {\n",
        "    'train': transforms.Compose([\n",
        "#         transforms.RandomHorizontalFlip(),\n",
        "        transforms.Resize(224),\n",
        "        transforms.CenterCrop(224),\n",
        "        transforms.ToTensor(),\n",
        "        transforms.Normalize(mean, std)\n",
        "    ]),\n",
        "}\n",
        "\n",
        "transformed_datasets = {}\n",
        "transformed_datasets = CreateDataset(df_box, transform=data_transforms['train'])\n",
        "\n",
        "dataloaders = {}\n",
        "dataloaders = torch.utils.data.DataLoader(\n",
        "    transformed_datasets,\n",
        "    batch_size=32,\n",
        "    shuffle=True,\n",
        "    num_workers=8,\n",
        "    pin_memory=True)"
      ],
      "metadata": {
        "id": "qjMLxEK3vN3d"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "lung = np.load('/content/drive/MyDrive/56_lung_mask.npy', allow_pickle=True)\n",
        "\n",
        "lung = lung[0]\n",
        "lung = lung.to(torch.float32)\n",
        "\n",
        "# l = []\n",
        "# for i in range(256):\n",
        "#     l.append(lung)\n",
        "# lung = torch.stack(l)\n",
        "# lung = lung.to(torch.float32)"
      ],
      "metadata": {
        "id": "tDpPaoBPK2np"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "centre_lung = np.load('/content/drive/MyDrive/56_center_lung_mask.npy', allow_pickle=True)\n",
        "\n",
        "centre_lung = centre_lung[0]\n",
        "centre_lung = centre_lung.to(torch.float32)\n",
        "\n",
        "# l = []\n",
        "# for i in range(256):\n",
        "#     l.append(centre_lung)\n",
        "# centre_lung = torch.stack(l)\n",
        "# centre_lung = centre_lung.to(torch.float32)"
      ],
      "metadata": {
        "id": "UMvnPDDBK3CE"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "masks = np.load('/content/drive/MyDrive/thesis_models/final_fixed_aligned_layercam_normalized_masks.npy', allow_pickle=True)\n",
        "for i in range(1, 8):\n",
        "    masks[i] = masks[i].squeeze(0)\n",
        "for i in range(len(masks)):\n",
        "    masks[i] = masks[i].to(torch.float32)\n",
        "\n",
        "# def ptp(t, axis):\n",
        "#   return t.max(axis).values - t.min(axis).values\n",
        "\n",
        "# for i in range(len(masks)):\n",
        "#   mean, std, var = torch.mean(masks[i]), torch.std(masks[i]), torch.var(masks[i])\n",
        "#   min = torch.min\n",
        "#   masks[i]  = (masks[i]-mean)/std\n",
        "# print('-------')\n",
        "# for i in masks:\n",
        "#   mean, std, var = torch.mean(i), torch.std(i), torch.var(i)\n",
        "#   print(mean, std, var)\n",
        "\n",
        "# temp = []\n",
        "# for i in range(7):\n",
        "#     temp1 = [0., 0., 0., 0., 0., 0., 0.]\n",
        "#     temp.append(temp1)\n",
        "# masks[6] = torch.Tensor(temp)\n",
        "\n",
        "l = []\n",
        "for i in masks:\n",
        "    l.append(i)\n",
        "masks = torch.stack(l)\n",
        "\n",
        "# batch_masks = []\n",
        "# for i in range(32):\n",
        "#     batch_masks.append(masks)\n",
        "\n",
        "# batch_masks = torch.stack(batch_masks).cuda()\n",
        "# batch_masks = batch_masks.to(torch.float32)\n",
        "\n",
        "batch_masks = masks.to(torch.float32)\n",
        "\n",
        "batch_masks[0]"
      ],
      "metadata": {
        "id": "SGO1uaVNTLC4"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "batch_masks[0]"
      ],
      "metadata": {
        "id": "fzJGUOWyXdhJ"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "from collections import namedtuple\n",
        "import os\n",
        "import torch\n",
        "from torchvision import models\n",
        "import torch.nn.functional as F\n",
        "torch.manual_seed(98)\n",
        "\n",
        "class AlignmentNetwork(torch.nn.Module):\n",
        "    def __init__(self):\n",
        "        super(AlignmentNetwork, self).__init__()\n",
        "\n",
        "        resnet18 = models.resnet18()\n",
        "        num_ftrs = resnet18.fc.in_features\n",
        "        resnet18.fc = nn.Linear(num_ftrs, 6)\n",
        "\n",
        "        self.model  = resnet18\n",
        "\n",
        "\n",
        "    def forward(self, x):\n",
        "        theta = self.model(x)\n",
        "        theta = theta.view(-1, 2, 3)\n",
        "\n",
        "        self.grid = F.affine_grid(theta, x.size())\n",
        "        x = F.grid_sample(x, self.grid)\n",
        "#         m = nn.AvgPool2d(16, stride=16)\n",
        "#         x = m(x)\n",
        "#         x = F.interpolate(x, (224,224), mode='bilinear')\n",
        "        return x"
      ],
      "metadata": {
        "id": "DcExV_-tUkF1"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# the vgg16 is borrowed from the pytorch tutorial\n",
        "# the resent is used to reimplemented and the comparision experiments\n",
        "from collections import namedtuple\n",
        "import os\n",
        "import torch\n",
        "from torchvision import models\n",
        "\n",
        "class ResNet50(torch.nn.Module):\n",
        "    def __init__(self):\n",
        "        super(ResNet50, self).__init__()\n",
        "\n",
        "        resnet50_pretrained = models.resnet50()\n",
        "        # num_ftrs = resnet50_pretrained.fc.in_features\n",
        "        # resnet50_pretrained.fc = nn.Sequential(\n",
        "        #     nn.Linear(num_ftrs, 8), nn.Sigmoid())\n",
        "\n",
        "        # resnet50_pretrained.load_state_dict(torch.load('/kaggle/input/models-for-localisation-testing/resnet50_5_chestxray8.pth'))\n",
        "\n",
        "        self.model  = list(resnet50_pretrained.children())\n",
        "\n",
        "        self.noslice1 = self.model[0]\n",
        "        self.noslice2 = self.model[1]\n",
        "        self.noslice3 = self.model[2]\n",
        "        self.noslice4 = self.model[3]\n",
        "\n",
        "        self.slice1 = self.model[4]\n",
        "        self.slice2 = self.model[5]\n",
        "        self.slice3 = self.model[6]\n",
        "        self.slice4 = self.model[7]\n",
        "\n",
        "#         self.fc = nn.Sequential(nn.AdaptiveAvgPool2d((1,1)),\n",
        "        self.conv = nn.Sequential(nn.Conv2d(in_channels=2048, out_channels=8, kernel_size=1))\n",
        "                                # nn.ReLU(inplace=True))\n",
        "        self.anothaone = nn.Sequential(nn.Conv2d(in_channels=8, out_channels=8, kernel_size=1))\n",
        "                                # nn.ReLU(inplace=True))\n",
        "        self.fc = nn.Sequential(nn.AdaptiveAvgPool2d((1,1)), nn.Sigmoid())\n",
        "        # self.fc = nn.Sequential(nn.AdaptiveAvgPool2d((1,1)),\n",
        "        #                         nn.Flatten(),\n",
        "        #                         nn.Linear(2048, 8),\n",
        "        #                         nn.Sigmoid())\n",
        "\n",
        "    def forward(self, x, alignment_network, batch_masks, lung_mask):\n",
        "#         x = x.to(device)\n",
        "        o = x\n",
        "#         for i in range(4):\n",
        "#             o = self.model[i](o)\n",
        "\n",
        "        # alignment_network.eval()\n",
        "        # o = alignment_network(o)\n",
        "\n",
        "        o = self.noslice1(o)\n",
        "        o = self.noslice2(o)\n",
        "        o = self.noslice3(o)\n",
        "        o = self.noslice4(o)\n",
        "\n",
        "#         anchor = cv.resize()\n",
        "#         o = o*anchor\n",
        "        o = self.slice1(o)\n",
        "        feature1 = o\n",
        "        # o = (o * lung_mask) + o\n",
        "        o = self.slice2(o)\n",
        "        feature2 = o\n",
        "        o = self.slice3(o)\n",
        "        feature3 = o\n",
        "        o = self.slice4(o)\n",
        "        feature4 = o\n",
        "        o = self.conv(o)\n",
        "        final = o\n",
        "\n",
        "        o = (o * batch_masks) + o\n",
        "\n",
        "        # if(o.shape[0] == 32):\n",
        "        #   o = (o * batch_masks) + o\n",
        "        # else:\n",
        "        #   temp = o.shape[0]\n",
        "        #   o = (o * batch_masks[:temp]) + o\n",
        "\n",
        "        o = self.anothaone(o)\n",
        "        final2 = o\n",
        "        o = self.fc(o)\n",
        "        final3 = o\n",
        "\n",
        "        return final2, final3"
      ],
      "metadata": {
        "id": "hi4RvbLovN60"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "device = \"cuda\" if torch.cuda.is_available() else \"cpu\"\n",
        "\n",
        "model = ResNet50()\n",
        "model = model.to(device)\n",
        "model.load_state_dict(torch.load('/content/drive/MyDrive/thesis_models/layerdmask_noalign_10.pth', map_location=torch.device('cpu')))\n",
        "model.eval()\n",
        "print('Model loaded')\n",
        "# model = model.type(torch.cuda.FloatTensor)"
      ],
      "metadata": {
        "id": "nMzF6wD7RHXb"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# from skimage.filters import threshold_otsu\n",
        "# from scipy import ndimage\n",
        "# from sklearn.mixture import GaussianMixture\n",
        "\n",
        "# img_path = df_box['full_path'].iloc[0]\n",
        "# cam_extractor = GradCAMpp(model)\n",
        "# img = Image.open(img_path)\n",
        "# img = img.convert('RGB')\n",
        "# img = preprocess(img)\n",
        "# # with torch.no_grad():\n",
        "# out = model(img.unsqueeze(0))\n",
        "# #         out = out[0][:8].unsqueeze(0)\n",
        "# cams = cam_extractor(out.squeeze(0).argmax().item(), out)\n",
        "\n",
        "# cam_extractor.remove_hooks()"
      ],
      "metadata": {
        "id": "l6wZKTStVUlX"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "img_path = df_box['full_path'].iloc[214]\n",
        "img = Image.open(img_path)\n",
        "img = img.convert('RGB')\n",
        "img = preprocess(img)\n",
        "# # img = img.type(torch.cuda.FloatTensor)\n",
        "out = model(img.unsqueeze(0), alignment_network, batch_masks, lung)"
      ],
      "metadata": {
        "id": "viwy_yduFpD8"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "plt.axis('off')\n",
        "plt.imshow(cv2.resize(out[0][0][1].detach().numpy(), (224, 224)))"
      ],
      "metadata": {
        "id": "XnwToyfTGU4y"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "plt.axis('off')\n",
        "plt.imshow(cv2.resize(out[0][0][0].detach().numpy(), (224, 224)))"
      ],
      "metadata": {
        "id": "EcjnCoJQGDCs"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "import io\n",
        "import requests\n",
        "import torchvision.transforms as T\n",
        "import numpy as np\n",
        "from PIL import Image\n",
        "\n",
        "from torchvision import transforms\n",
        "from torchvision.utils import draw_segmentation_masks\n",
        "from torchvision.ops import masks_to_boxes\n",
        "from torchvision.utils import draw_bounding_boxes\n",
        "from matplotlib.patches import Rectangle\n",
        "\n",
        "from skimage.filters import threshold_otsu\n",
        "import cv2\n",
        "\n",
        "# cam_list = [GradCAM, GradCAMpp, LayerCAM]\n",
        "# cam_list = [GradCAMpp]\n",
        "\n",
        "# iou_gcam, iou_gcampp, iou_lcam = [], [], []\n",
        "# contain_gcam, contain_gcampp, contain_lcam = [], [], []\n",
        "# all_cams = []\n",
        "\n",
        "pred_class, iou_score, temp_prob_val = [], [], []\n",
        "p_box_list = []\n",
        "\n",
        "alignment_network = AlignmentNetwork()\n",
        "alignment_network.load_state_dict(torch.load('/content/drive/MyDrive/2feat_no_crop_fixed_pt_alignment_network_5.pth', map_location=torch.device('cpu')))\n",
        "# alignment_network.load_state_dict(torch.load('/content/drive/MyDrive/fixed_pt_alignment_network_5.pth', map_location=torch.device('cpu')))\n",
        "alignment_network = alignment_network.to(device)\n",
        "alignment_network.eval()\n",
        "\n",
        "for i in range(len(df_box)):\n",
        "    img_path = df_box['full_path'].iloc[i]\n",
        "    img = Image.open(img_path)\n",
        "    img = img.convert('RGB')\n",
        "    img = preprocess(img)\n",
        "    # # img = img.type(torch.cuda.FloatTensor)\n",
        "    out = model(img.unsqueeze(0), alignment_network, batch_masks, lung)\n",
        "\n",
        "    # out = alignment_network(img.unsqueeze(0))\n",
        "    # out = model(out, alignment_network, batch_masks)\n",
        "\n",
        "    temp_prob_val.append(out[1].max().item())\n",
        "\n",
        "    temp = out[1].argmax().item()\n",
        "    pred_class.append(temp)\n",
        "    # all_cams.append(out[0][0][temp])\n",
        "\n",
        "    img2 = Image.open(img_path)\n",
        "    img2 = img2.convert('RGB')\n",
        "    preprocess_temp = T.Compose([\n",
        "        T.Resize(224),\n",
        "        T.CenterCrop(224),\n",
        "        T.ToTensor(),\n",
        "    ])\n",
        "    img2 = preprocess_temp(img2)\n",
        "\n",
        "    a = out[0][0][temp].cpu().detach().numpy()\n",
        "    a = (a - np.min(a))/np.ptp(a)\n",
        "    a = cv2.resize(a, (224, 224))\n",
        "\n",
        "    if (np.isnan(a).all() == False):\n",
        "      otsu_thresh = threshold_otsu(a) * 1.25\n",
        "      mean = np.mean(a) * 2.05\n",
        "      a[a < otsu_thresh] = np.nan\n",
        "\n",
        "      if (np.isnan(a).all() == False):\n",
        "        xmax, ymax = np.max(np.where(~np.isnan(a)), 1)\n",
        "        xmin, ymin = np.min(np.where(~np.isnan(a)), 1)\n",
        "        pred_boxes = torch.Tensor([xmin, ymin, xmax, ymax])\n",
        "        p_box_list.append(pred_boxes)\n",
        "\n",
        "        gt_xmin, gt_ymin, gt_w, gt_h = df_box['xmin'].iloc[i], df_box['ymin'].iloc[i], df_box['w'].iloc[i], df_box['h'].iloc[i]\n",
        "        gt_xmin, gt_ymin, gt_w, gt_h = large_to_small(gt_xmin, gt_ymin, gt_w, gt_h)\n",
        "        gt_xmax, gt_ymax = gt_xmin + gt_w, gt_ymin + gt_h\n",
        "        gt_boxes = torch.Tensor([gt_xmin, gt_ymin, gt_xmax, gt_ymax])\n",
        "\n",
        "        iou_score.append(jaccard(pred_boxes.reshape(-1,4), gt_boxes.reshape(-1,4)).squeeze().tolist())\n",
        "\n",
        "      else:\n",
        "        iou_score.append(0.0)\n",
        "\n",
        "    else:\n",
        "      iou_score.append(0.0)\n",
        "\n",
        "print(np.round(np.mean(iou_score), 4))\n",
        "\n",
        "c = 0\n",
        "for i in iou_score:\n",
        "    if i >= 0.1:\n",
        "        c+=1\n",
        "print('0.1: ', np.round((c / len(iou_score)), 3))\n",
        "\n",
        "c = 0\n",
        "for i in iou_score:\n",
        "    if i >= 0.3:\n",
        "        c+=1\n",
        "\n",
        "print('0.3: ', np.round((c / len(iou_score)), 3))\n",
        "\n",
        "c = 0\n",
        "for i in iou_score:\n",
        "    if i >= 0.5:\n",
        "        c+=1\n",
        "\n",
        "print('0.5: ', np.round((c / len(iou_score)), 3))\n",
        "\n",
        "c = 0\n",
        "for i in iou_score:\n",
        "    if i >= 0.7:\n",
        "        c+=1\n",
        "\n",
        "print('0.7: ', np.round((c / len(iou_score)), 3))"
      ],
      "metadata": {
        "id": "re2JYuISvOBC"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "class_names = df_box['Finding Label'].unique()\n",
        "\n",
        "ious_temp = np.array([iou_score]).squeeze()\n",
        "\n",
        "for c in class_names:\n",
        "    iou_acc = 0\n",
        "    class_indices = df_box.loc[df_box['Finding Label'] == c].index.tolist()\n",
        "#     class_iou = ious_temp[class_indices].mean()\n",
        "    class_iou = ious_temp[class_indices]\n",
        "    for i in class_iou:\n",
        "        if i >= 0.1:\n",
        "            iou_acc+=1\n",
        "#     contain = contain_list_temp[class_indices].sum()\n",
        "    print(c + \" total        \" + str(len(class_indices)))\n",
        "#     print(c + \" iobb          \" + str(class_iou.sum()))\n",
        "    print(c + \" iobb > 0.5 acc          \" + str(np.round(iou_acc/len(class_indices), 3)))\n",
        "#     print(c + \" contains     \" + str(contain / len(class_indices)))"
      ],
      "metadata": {
        "id": "t6jrd2mAWNDF"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "class_names = df_box['Finding Label'].unique()\n",
        "\n",
        "ious_temp = np.array([iou_score]).squeeze()\n",
        "\n",
        "for c in class_names:\n",
        "    iou_acc = 0\n",
        "    class_indices = df_box.loc[df_box['Finding Label'] == c].index.tolist()\n",
        "#     class_iou = ious_temp[class_indices].mean()\n",
        "    class_iou = ious_temp[class_indices]\n",
        "    for i in class_iou:\n",
        "        if i >= 0.3:\n",
        "            iou_acc+=1\n",
        "#     contain = contain_list_temp[class_indices].sum()\n",
        "    print(c + \" total        \" + str(len(class_indices)))\n",
        "#     print(c + \" iobb          \" + str(class_iou.sum()))\n",
        "    print(c + \" iobb > 0.5 acc          \" + str(np.round(iou_acc/len(class_indices), 3)))\n",
        "#     print(c + \" contains     \" + str(contain / len(class_indices)))"
      ],
      "metadata": {
        "id": "ceCBZLbzWLYe"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "iou_score[676]"
      ],
      "metadata": {
        "id": "4OKsYea7jw2m"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "from matplotlib.patches import Rectangle\n",
        "\n",
        "temp = 676\n",
        "\n",
        "img_path = df_box['full_path'].iloc[temp]\n",
        "img = Image.open(img_path)\n",
        "img = img.resize((224, 224))\n",
        "img = img.convert('RGB')\n",
        "# img = preprocess(img)\n",
        "\n",
        "gt_xmin, gt_ymin, gt_w, gt_h = df_box['xmin'].iloc[temp], df_box['ymin'].iloc[temp], df_box['w'].iloc[temp], df_box['h'].iloc[temp]\n",
        "gt_xmin, gt_ymin, gt_w, gt_h = large_to_small(gt_xmin, gt_ymin, gt_w, gt_h)\n",
        "gt_xmax, gt_ymax = gt_xmin + gt_w, gt_ymin + gt_h\n",
        "gt_boxes = torch.Tensor([gt_xmin, gt_ymin, gt_xmax, gt_ymax])\n",
        "gt_rect = Rectangle((gt_boxes[0], gt_boxes[1]), gt_boxes[2]-gt_boxes[0], gt_boxes[3]-gt_boxes[1], fill=False, color='b')\n",
        "\n",
        "pred_boxes = p_box_list[temp]\n",
        "pred_rect = Rectangle((pred_boxes[0], pred_boxes[1]), pred_boxes[2]-pred_boxes[0], pred_boxes[3]-pred_boxes[1], fill=False, color='r')\n",
        "\n",
        "\n",
        "fig, ax = plt.subplots()\n",
        "ax.imshow(img)\n",
        "ax.add_patch(gt_rect)\n",
        "ax.add_patch(pred_rect)\n",
        "plt.axis('off')\n",
        "plt.title(df_box['Finding Label'].iloc[temp])\n",
        "plt.imshow(img)\n",
        "plt.show()"
      ],
      "metadata": {
        "id": "obczBA_Qblwu"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "iou_score[214]"
      ],
      "metadata": {
        "id": "z5rNPTJk5GQE"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "from matplotlib.patches import Rectangle\n",
        "\n",
        "temp = 214\n",
        "\n",
        "img_path = df_box['full_path'].iloc[temp]\n",
        "img = Image.open(img_path)\n",
        "img = img.resize((224, 224))\n",
        "img = img.convert('RGB')\n",
        "plt.axis('off')\n",
        "plt.imshow(img)"
      ],
      "metadata": {
        "id": "lyHvty5iHVOd"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "from matplotlib.patches import Rectangle\n",
        "\n",
        "temp = 214\n",
        "\n",
        "img_path = df_box['full_path'].iloc[temp]\n",
        "img = Image.open(img_path)\n",
        "img = img.resize((224, 224))\n",
        "img = img.convert('RGB')\n",
        "# img = preprocess(img)\n",
        "\n",
        "gt_xmin, gt_ymin, gt_w, gt_h = df_box['xmin'].iloc[temp], df_box['ymin'].iloc[temp], df_box['w'].iloc[temp], df_box['h'].iloc[temp]\n",
        "gt_xmin, gt_ymin, gt_w, gt_h = large_to_small(gt_xmin, gt_ymin, gt_w, gt_h)\n",
        "gt_xmax, gt_ymax = gt_xmin + gt_w, gt_ymin + gt_h\n",
        "gt_boxes = torch.Tensor([gt_xmin, gt_ymin, gt_xmax, gt_ymax])\n",
        "gt_rect = Rectangle((gt_boxes[0], gt_boxes[1]), gt_boxes[2]-gt_boxes[0], gt_boxes[3]-gt_boxes[1], fill=False, color='b')\n",
        "\n",
        "pred_boxes = p_box_list[temp]\n",
        "pred_rect = Rectangle((pred_boxes[0], pred_boxes[1]), pred_boxes[2]-pred_boxes[0], pred_boxes[3]-pred_boxes[1], fill=False, color='r')\n",
        "\n",
        "\n",
        "fig, ax = plt.subplots()\n",
        "ax.imshow(img)\n",
        "ax.add_patch(gt_rect)\n",
        "ax.add_patch(pred_rect)\n",
        "# plt.axis('off')\n",
        "plt.title(df_box['Finding Label'].iloc[temp])\n",
        "plt.imshow(img)\n",
        "plt.show()"
      ],
      "metadata": {
        "id": "uxuKWvq_nJ7d"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "iou_score[959]"
      ],
      "metadata": {
        "id": "XBdVwga95ovO"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "from matplotlib.patches import Rectangle\n",
        "\n",
        "temp = 959\n",
        "\n",
        "img_path = df_box['full_path'].iloc[temp]\n",
        "img = Image.open(img_path)\n",
        "img = img.resize((224, 224))\n",
        "img = img.convert('RGB')\n",
        "# img = preprocess(img)\n",
        "\n",
        "gt_xmin, gt_ymin, gt_w, gt_h = df_box['xmin'].iloc[temp], df_box['ymin'].iloc[temp], df_box['w'].iloc[temp], df_box['h'].iloc[temp]\n",
        "gt_xmin, gt_ymin, gt_w, gt_h = large_to_small(gt_xmin, gt_ymin, gt_w, gt_h)\n",
        "gt_xmax, gt_ymax = gt_xmin + gt_w, gt_ymin + gt_h\n",
        "gt_boxes = torch.Tensor([gt_xmin, gt_ymin, gt_xmax, gt_ymax])\n",
        "gt_rect = Rectangle((gt_boxes[0], gt_boxes[1]), gt_boxes[2]-gt_boxes[0], gt_boxes[3]-gt_boxes[1], fill=False, color='b')\n",
        "\n",
        "pred_boxes = p_box_list[temp]\n",
        "pred_rect = Rectangle((pred_boxes[0], pred_boxes[1]), pred_boxes[2]-pred_boxes[0], pred_boxes[3]-pred_boxes[1], fill=False, color='r')\n",
        "\n",
        "\n",
        "fig, ax = plt.subplots()\n",
        "ax.imshow(img)\n",
        "ax.add_patch(gt_rect)\n",
        "ax.add_patch(pred_rect)\n",
        "plt.axis('off')\n",
        "plt.title(df_box['Finding Label'].iloc[temp])\n",
        "plt.imshow(img)\n",
        "plt.show()"
      ],
      "metadata": {
        "id": "v3gfAl1F5loT"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "Hth8apao5lrj"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "masks = np.load('/content/drive/MyDrive/thesis_models/final_fixed_aligned_layercam_normalized_masks.npy', allow_pickle=True)\n",
        "for i in range(1, 8):\n",
        "    masks[i] = masks[i].squeeze(0)\n",
        "for i in range(len(masks)):\n",
        "    masks[i] = masks[i].to(torch.float32)"
      ],
      "metadata": {
        "id": "TYBEQzs9nJ_-"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "temp = []\n",
        "class_names = df_box['Finding Label'].unique()\n",
        "for i in range(8):\n",
        "  norm_image = cv2.normalize(masks[i].numpy(), None, alpha = 0, beta = 255, norm_type = cv2.NORM_MINMAX, dtype = cv2.CV_32F)\n",
        "  norm_image = norm_image.astype(np.uint8)\n",
        "  norm_image = cv2.resize(norm_image, (28, 28))\n",
        "  temp.append(norm_image)\n",
        "  # plt.imshow(norm_image, cmap='gray', vmin=0, vmax=255)"
      ],
      "metadata": {
        "id": "DYxZs0gdnRrY"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "f, axarr = plt.subplots(2,4)\n",
        "f.tight_layout()\n",
        "\n",
        "for ax in axarr.ravel():\n",
        "    ax.set_axis_off()\n",
        "\n",
        "axarr[0,0].set_title(class_names[0], fontsize=10)\n",
        "axarr[0,0].imshow(temp[0], cmap='gray', vmin=0, vmax=255)\n",
        "axarr[0,1].set_title(class_names[1], fontsize=10)\n",
        "axarr[0,1].imshow(temp[1], cmap='gray', vmin=0, vmax=255)\n",
        "axarr[0,2].set_title(class_names[2], fontsize=10)\n",
        "axarr[0,2].imshow(temp[2], cmap='gray', vmin=0, vmax=255)\n",
        "axarr[0,3].set_title(class_names[3], fontsize=10)\n",
        "axarr[0,3].imshow(temp[3], cmap='gray', vmin=0, vmax=255)\n",
        "axarr[1,0].set_title(class_names[4], fontsize=10)\n",
        "axarr[1,0].imshow(temp[4], cmap='gray', vmin=0, vmax=255)\n",
        "axarr[1,1].set_title(class_names[5], fontsize=10)\n",
        "axarr[1,1].imshow(temp[5], cmap='gray', vmin=0, vmax=255)\n",
        "axarr[1,2].set_title(class_names[6], fontsize=10)\n",
        "axarr[1,2].imshow(temp[6], cmap='gray', vmin=0, vmax=255)\n",
        "axarr[1,3].set_title(class_names[7], fontsize=10)\n",
        "axarr[1,3].imshow(temp[7], cmap='gray', vmin=0, vmax=255)\n",
        "plt.subplots_adjust(wspace=0, hspace=0)\n",
        "plt.show()"
      ],
      "metadata": {
        "id": "mvkih6canKXc"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "actual_class[-10:]"
      ],
      "metadata": {
        "id": "q_hvE_BBN-43"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "actual_class[0:10]"
      ],
      "metadata": {
        "id": "sJH2Ca-2OEW6"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "temp_pred = pred_class"
      ],
      "metadata": {
        "id": "rKPNaBF2KrGk"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "class_names = df_box['Finding Label'].unique()\n",
        "actual_class = df_box['Finding Label'].replace(class_names, [0, 1, 2, 3, 4, 5, 6, 7]).to_list()"
      ],
      "metadata": {
        "id": "9grqHSyHeSs-"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "from sklearn.metrics import accuracy_score\n",
        "accuracy_score(actual_class, temp_pred)"
      ],
      "metadata": {
        "id": "YsI3vIW6L7Bu"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "from sklearn.metrics import accuracy_score\n",
        "accuracy_score(actual_class, pred_class)"
      ],
      "metadata": {
        "id": "UZ95x0S8LEjE"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "c = 0\n",
        "for i in iou_score:\n",
        "    if i >= 0.1:\n",
        "        c+=1\n",
        "print('0.1: ', np.round((c / len(iou_score)), 3))\n",
        "\n",
        "c = 0\n",
        "for i in iou_score:\n",
        "    if i >= 0.3:\n",
        "        c+=1\n",
        "\n",
        "print('0.3: ', np.round((c / len(iou_score)), 3))\n",
        "\n",
        "c = 0\n",
        "for i in iou_score:\n",
        "    if i >= 0.5:\n",
        "        c+=1\n",
        "\n",
        "print('0.5: ', np.round((c / len(iou_score)), 3))\n",
        "\n",
        "c = 0\n",
        "for i in iou_score:\n",
        "    if i >= 0.7:\n",
        "        c+=1\n",
        "\n",
        "print('0.7: ', np.round((c / len(iou_score)), 3))"
      ],
      "metadata": {
        "id": "dRjjJcTDH1Xt"
      },
      "execution_count": null,
      "outputs": []
    }
  ]
}