Adapters.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from __future__ import print_function, division\n",
    "import os\n",
    "import torch\n",
    "import pandas as pd\n",
    "from skimage import io, transform\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "from torch.utils.data import Dataset, DataLoader\n",
    "from torchvision import transforms, utils\n",
    "import PIL\n",
    "# Ignore warnings\n",
    "import warnings\n",
    "warnings.filterwarnings(\"ignore\")\n",
    "\n",
    "plt.ion()   # interactive mode\n",
    "from random import randint\n",
    "from sklearn.model_selection import train_test_split\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "root_dir = os.path.abspath('')\n",
    "data_dir = os.path.join(root_dir, 'Images')\n",
    "# check for existence\n",
    "os.path.exists(root_dir)\n",
    "os.path.exists(data_dir)\n",
    "workers_test = pd.read_csv(os.path.join(data_dir,  'test_workers_dataset_normalized.csv'), sep=' ')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from PIL import Image\n",
    "class ImageRatingsDataset(Dataset):\n",
    "    \"\"\"Images dataset.\"\"\"\n",
    "\n",
    "    def __init__(self, csv_file, root_dir, transform=None):\n",
    "        \"\"\"\n",
    "        Args:\n",
    "            csv_file (string): Path to the csv file with annotations.\n",
    "            root_dir (string): Directory with all the images.\n",
    "            transform (callable, optional): Optional transform to be applied\n",
    "                on a sample.\n",
    "        \"\"\"\n",
    "        self.images_frame = pd.read_csv(csv_file, sep=' ')\n",
    "        self.root_dir = root_dir\n",
    "        self.transform = transform\n",
    "\n",
    "    def __len__(self):\n",
    "        return len(self.images_frame)\n",
    "\n",
    "    def __getitem__(self, idx):\n",
    "        try:\n",
    "            img_name = str(os.path.join(self.root_dir,str(self.images_frame.iloc[idx, 0])))\n",
    "            im = Image.open(img_name).convert('RGB')\n",
    "            if im.mode == 'P':\n",
    "                im = im.convert('RGB')\n",
    "            image = np.asarray(im)\n",
    "            #image = io.imread(img_name+'.jpg', mode='RGB').convert('RGB')\n",
    "            rating = self.images_frame.iloc[idx, 1]\n",
    "            sample = {'image': image, 'rating': rating}\n",
    "\n",
    "            if self.transform:\n",
    "                sample = self.transform(sample)\n",
    "            return sample            \n",
    "        except Exception as e:\n",
    "            pass\n",
    "        \n",
    "\n",
    "            \n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import random\n",
    "class Rescale(object):\n",
    "    \"\"\"Rescale the image in a sample to a given size.\n",
    "\n",
    "    Args:\n",
    "        output_size (tuple or int): Desired output size. If tuple, output is\n",
    "            matched to output_size. If int, smaller of image edges is matched\n",
    "            to output_size keeping aspect ratio the same.\n",
    "    \"\"\"\n",
    "\n",
    "    def __init__(self, output_size):\n",
    "        assert isinstance(output_size, (int, tuple))\n",
    "        self.output_size = output_size\n",
    "\n",
    "    def __call__(self, sample):\n",
    "        image, rating = sample['image'], sample['rating']\n",
    "        h, w = image.shape[:2]\n",
    "        if isinstance(self.output_size, int):\n",
    "            if h > w:\n",
    "                new_h, new_w = self.output_size * h / w, self.output_size\n",
    "            else:\n",
    "                new_h, new_w = self.output_size, self.output_size * w / h\n",
    "        else:\n",
    "            new_h, new_w = self.output_size\n",
    "\n",
    "        new_h, new_w = int(new_h), int(new_w)\n",
    "\n",
    "        image = transform.resize(image, (new_h, new_w))\n",
    "\n",
    "        return {'image': image, 'rating': rating}\n",
    "\n",
    "        \n",
    "    \n",
    "\n",
    "class RandomCrop(object):\n",
    "    \"\"\"Crop randomly the image in a sample.\n",
    "\n",
    "    Args:\n",
    "        output_size (tuple or int): Desired output size. If int, square crop\n",
    "            is made.\n",
    "    \"\"\"\n",
    "\n",
    "    def __init__(self, output_size):\n",
    "        assert isinstance(output_size, (int, tuple))\n",
    "        if isinstance(output_size, int):\n",
    "            self.output_size = (output_size, output_size)\n",
    "        else:\n",
    "            assert len(output_size) == 2\n",
    "            self.output_size = output_size\n",
    "\n",
    "    def __call__(self, sample):\n",
    "        image, rating = sample['image'], sample['rating']\n",
    "        h, w = image.shape[:2]\n",
    "        new_h, new_w = self.output_size\n",
    "\n",
    "        top = np.random.randint(0, h - new_h)\n",
    "        left = np.random.randint(0, w - new_w)\n",
    "\n",
    "        image = image[top: top + new_h,\n",
    "                      left: left + new_w]\n",
    "\n",
    "\n",
    "        return {'image': image, 'rating': rating}\n",
    "\n",
    "class RandomHorizontalFlip(object):\n",
    "    def __init__(self, p):\n",
    "        self.p = p\n",
    "    \n",
    "    def __call__(self, sample):\n",
    "        \n",
    "        image, rating = sample['image'], sample['rating']\n",
    "        if random.random() < self.p:\n",
    "            image = np.flip(image, 1)\n",
    "            #image = io.imread(img_name+'.jpg', mode='RGB').convert('RGB')\n",
    "        return {'image': image, 'rating': rating}\n",
    "\n",
    "class Normalize(object):\n",
    "    def __init__(self ):\n",
    "        self.means=np.array([0.485, 0.456, 0.406]) \n",
    "        self.stds=np.array([0.229, 0.224, 0.225])\n",
    "        \n",
    "    def __call__(self, sample):\n",
    "        image, rating = sample['image'], sample['rating']\n",
    "        im=image/255\n",
    "        im[:,:,0]=(image[:,:,0] - 0.485)/ 0.229 \n",
    "        im[:,:,1]=(image[:,:,1]  - self.means[1])/ self.stds[1]\n",
    "        im[:,:,2]=(image[:,:,2]  - self.means[2])/ self.stds[2]\n",
    "        image=im\n",
    "        return {'image': image, 'rating': rating}\n",
    "\n",
    "    \n",
    "    \n",
    "class ToTensor(object):\n",
    "    \"\"\"Convert ndarrays in sample to Tensors.\"\"\"\n",
    "\n",
    "    def __call__(self, sample):\n",
    "        image, rating = sample['image'], sample['rating']\n",
    "        \n",
    "        # swap color axis because\n",
    "        # numpy image: H x W x C\n",
    "        # torch image: C X H X W\n",
    "            \n",
    "        image = image.transpose((2, 0, 1))\n",
    "        return {'image': torch.from_numpy(image).double(),\n",
    "                'rating': torch.from_numpy(np.float64([rating])).double()}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "worker=workers_test['worker'].unique()[0]\n",
    "num_images = workers_test[workers_test['worker'].isin([worker])].shape[0]\n",
    "percent=100/num_images\n",
    "images=workers_test[workers_test['worker'].isin([worker])][[' imagePair', ' score']]\n",
    "train_dataframe, valid_dataframe = train_test_split(images, train_size=percent)\n",
    "train_path=\"train_means_normalized\" + worker +\".csv\"\n",
    "test_path=\"test_means_normalized\" + worker +\".csv\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_dataframe.to_csv(train_path, sep=' ', index_label = False)\n",
    "valid_dataframe.to_csv(test_path, sep=' ', index_label = False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "output_size=(224,224)\n",
    "transformed_dataset_train = ImageRatingsDataset(csv_file=train_path,root_dir='Images/',\n",
    "                                           transform=transforms.Compose([Rescale(output_size=(256,256)),\n",
    "                                                                         RandomHorizontalFlip(0.5),\n",
    "                                                                         RandomCrop(output_size=output_size),\n",
    "                                                                         Normalize(),\n",
    "                                                                        ToTensor(),\n",
    "                                           ]))\n",
    "transformed_dataset_valid = ImageRatingsDataset(csv_file=test_path,root_dir='Images/',\n",
    "                                           transform=transforms.Compose([Rescale(output_size=(224,224)),\n",
    "                                                                         Normalize(),\n",
    "                                                                        ToTensor(),\n",
    "                                           ]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from torch.utils.data.dataloader import default_collate\n",
    "bsize=12\n",
    "def my_collate(batch):\n",
    "    batch = list(filter (lambda x:x is not None, batch))\n",
    "    return default_collate(batch)\n",
    "\n",
    "dataloader_train = DataLoader(transformed_dataset_train, batch_size=bsize,\n",
    "                        shuffle=True, num_workers=0,collate_fn=my_collate)\n",
    "dataloader_valid = DataLoader(transformed_dataset_valid, batch_size=8,\n",
    "                        shuffle=True, num_workers=0,collate_fn=my_collate)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from __future__ import print_function, division\n",
    "import os\n",
    "import torch\n",
    "import pandas as pd\n",
    "from skimage import io, transform\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "from torch.utils.data import Dataset, DataLoader\n",
    "from torchvision import transforms, utils\n",
    "import copy\n",
    "from torch import nn\n",
    "# Ignore warnings\n",
    "import warnings\n",
    "warnings.filterwarnings(\"ignore\")\n",
    "\n",
    "plt.ion()   # interactive mode\n",
    "from random import randint"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from torchvision import models\n",
    "model_ft = models.resnet18(pretrained=True)\n",
    "num_ftrs = model_ft.fc.in_features\n",
    "model_ft.fc = nn.Sequential(nn.Dropout(0.5),nn.Linear(num_ftrs,1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "class BaselineModel(nn.Module):\n",
    "    def __init__(self, num_classes, keep_probability, inputsize):\n",
    "    \n",
    "        super(BaselineModel, self).__init__()\n",
    "        self.fc1 = nn.Linear(inputsize, 256)\n",
    "        self.drop_prob = (1 - keep_probability)\n",
    "        self.relu1= nn.PReLU()\n",
    "        self.drop1 = nn.Dropout(self.drop_prob)\n",
    "        self.bn1=nn.BatchNorm1d(256)\n",
    "        self.fc2=nn.Linear(256,256)\n",
    "        self.relu2=nn.PReLU()\n",
    "        self.drop2 = nn.Dropout(p=self.drop_prob)\n",
    "        self.bn2=nn.BatchNorm1d(256)\n",
    "        self.fc3 = nn.Linear(256, num_classes)\n",
    "\n",
    "        for m in self.modules():\n",
    "            if isinstance(m, nn.Conv2d):\n",
    "                # Weight initialization reference: https://arxiv.org/abs/1502.01852\n",
    "                n = m.kernel_size[0] * m.kernel_size[1] * m.out_channels\n",
    "                m.weight.data.normal_(0, math.sqrt(2. / n))\n",
    "            elif isinstance(m, nn.BatchNorm2d):\n",
    "                m.weight.data.fill_(1)\n",
    "                m.bias.data.zero_()\n",
    "\n",
    "    def forward(self, x):\n",
    "        \"\"\"\n",
    "        Feed-forward pass.\n",
    "        :param x: Input tensor\n",
    "        : return: Output tensor\n",
    "        \"\"\"\n",
    "        out=self.fc1(x)\n",
    "        out =self.relu1(out)\n",
    "        out=self.drop1(out)\n",
    "        out=self.bn1(out)\n",
    "        out=self.fc2(out)\n",
    "        out=self.relu2(out)\n",
    "        out=self.drop2(out)\n",
    "        out=self.bn2(out)\n",
    "        out=self.fc3(out)\n",
    "        return out"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "class convNet(nn.Module):\n",
    "    #constructor\n",
    "    def __init__(self,resnet,mynet):\n",
    "        super(convNet, self).__init__()\n",
    "        #defining layers in convnet\n",
    "        self.resnet=resnet\n",
    "        self.myNet=mynet\n",
    "    def forward(self, x):\n",
    "        x=self.resnet(x)\n",
    "        x=self.myNet(x)\n",
    "        return x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model_ft = models.resnet18(pretrained=True)\n",
    "num_ftrs = model_ft.fc.out_features\n",
    "net1 = BaselineModel(1, 0.5,num_ftrs)\n",
    "net2=convNet(resnet=model_ft, mynet=net1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "net2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch.nn as nn\n",
    "import torch.optim as optim\n",
    "from torch.autograd import Variable\n",
    "use_gpu = True\n",
    "if use_gpu:\n",
    "    cuda = torch.device('cuda:0')     # Default CUDA device\n",
    "count=0\n",
    "torch.cuda.set_device(cuda.index)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def exp_lr_scheduler(optimizer, epoch, init_lr=0.001, lr_decay_epoch=5):\n",
    "    \"\"\"Decay learning rate by a factor of DECAY_WEIGHT every lr_decay_epoch epochs.\"\"\"\n",
    "\n",
    "    if epoch % lr_decay_epoch == 0:\n",
    "        #print('LR is set to {}'.format(lr))\n",
    "        pass\n",
    "    lr = init_lr * (0.9**(epoch // lr_decay_epoch))\n",
    "\n",
    "    for param_group in optimizer.param_groups:\n",
    "        param_group['lr'] = lr\n",
    "\n",
    "    return optimizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "net_2 = (torch.load('fine_tuned_flickerAES_normalized_dropout_resnet18_customnetworkadamnormalized.pt'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "class rsn18 (nn.Module):\n",
    "    def __init__(self):\n",
    "        super(rsn18, self).__init__()\n",
    "        self.conv1 = models.resnet18().conv1\n",
    "        self.bn1=models.resnet18().bn1\n",
    "        self.relu=models.resnet18().relu\n",
    "        self.maxpool=models.resnet18().maxpool\n",
    "        self.adapter1_0=nn.Conv2d(64,64, kernel_size=1, padding=0)\n",
    "        self.adapter1_1=nn.Conv2d(64,64, kernel_size=1, padding=0)\n",
    "        self.adapter1_2=nn.Conv2d(64,64, kernel_size=1, padding=0)\n",
    "        self.adapter1_3=nn.Conv2d(64,64, kernel_size=1, padding=0)\n",
    "        \n",
    "        \n",
    "        self.adapter2_0=nn.Conv2d(64,128, kernel_size=1, padding=0, stride=2)\n",
    "        self.adapter2_1=nn.Conv2d(128,128, kernel_size=1, padding=0)\n",
    "        self.adapter2_2=nn.Conv2d(128,128, kernel_size=1, padding=0)\n",
    "        self.adapter2_3=nn.Conv2d(128,128, kernel_size=1, padding=0)\n",
    "        \n",
    "        self.adapter3_0=nn.Conv2d(128,256, kernel_size=1, padding=0, stride=2)\n",
    "        self.adapter3_1=nn.Conv2d(256,256, kernel_size=1, padding=0)\n",
    "        self.adapter3_2=nn.Conv2d(256,256, kernel_size=1, padding=0)\n",
    "        self.adapter3_3=nn.Conv2d(256,256, kernel_size=1, padding=0)\n",
    "        \n",
    "        self.adapter4_0=nn.Conv2d(256,512, kernel_size=1, padding=0, stride=2)\n",
    "        self.adapter4_1=nn.Conv2d(512,512, kernel_size=1, padding=0)\n",
    "        self.adapter4_2=nn.Conv2d(512,512, kernel_size=1, padding=0)\n",
    "        self.adapter4_3=nn.Conv2d(512,512, kernel_size=1, padding=0)\n",
    "\n",
    "        self.layer1=models.resnet18().layer1\n",
    "        self.layer2=models.resnet18().layer2\n",
    "        self.layer3=models.resnet18().layer3\n",
    "        self.layer4=models.resnet18().layer4\n",
    "        self.avgpool=models.resnet18().avgpool\n",
    "        #nn.init.xavier_uniform_(self.adapter1.weight, gain=0.01)\n",
    "        #nn.init.xavier_uniform_(self.adapter2.weight, gain=0.01)\n",
    "        #nn.init.xavier_uniform_(self.adapter3.weight, gain=0.01)\n",
    "        #nn.init.xavier_uniform_(self.adapter4.weight, gain=0.01)\n",
    "\n",
    "        self.fc=models.resnet18().fc\n",
    "    def forward(self, x):\n",
    "        x = self.conv1(x)\n",
    "        x = self.bn1(x)\n",
    "        x = self.relu(x)\n",
    "        x = self.maxpool(x)\n",
    "        \n",
    "        residual=x\n",
    "        x = self.layer1[0].conv1(x) + self.adapter1_0(x)\n",
    "        x = self.layer1[0].bn1(x)\n",
    "        x = self.layer1[0].relu(x)\n",
    "        x = self.layer1[0].conv2(x) + self.adapter1_1(x)\n",
    "        x = self.layer1[0].bn2(x)\n",
    "        x = x + residual\n",
    "        x = self.layer1[0].relu(x)\n",
    "        \n",
    "        residual=x\n",
    "        x = self.layer1[1].conv1(x) + self.adapter1_2(x)\n",
    "        x = self.layer1[1].bn1(x)\n",
    "        x = self.layer1[1].relu(x)\n",
    "        x = self.layer1[1].conv2(x) + self.adapter1_3(x)\n",
    "        x = self.layer1[1].bn2(x)\n",
    "        x = x + residual\n",
    "        x = self.layer1[1].relu(x)\n",
    "        \n",
    "        \n",
    "        residual=x\n",
    "        x = self.layer2[0].conv1(x) + self.adapter2_0(x)\n",
    "        x = self.layer2[0].bn1(x)\n",
    "        x = self.layer2[0].relu(x)\n",
    "        x = self.layer2[0].conv2(x) + self.adapter2_1(x)\n",
    "        x = self.layer2[0].bn2(x)\n",
    "        residual = self.layer2[0].downsample(residual)\n",
    "        x = x + residual\n",
    "        x = self.layer2[0].relu(x)\n",
    "        \n",
    "        residual=x\n",
    "        x = self.layer2[1].conv1(x) + self.adapter2_2(x)\n",
    "        x = self.layer2[1].bn1(x)\n",
    "        x = self.layer2[1].relu(x)\n",
    "        x = self.layer2[1].conv2(x) + self.adapter2_3(x)\n",
    "        x = self.layer2[1].bn2(x)\n",
    "        x = x + residual\n",
    "        x = self.layer2[1].relu(x)\n",
    "        \n",
    "        \n",
    "        residual=x\n",
    "        x = self.layer3[0].conv1(x) + self.adapter3_0(x)\n",
    "        x = self.layer3[0].bn1(x)\n",
    "        x = self.layer3[0].relu(x)\n",
    "        x = self.layer3[0].conv2(x) + self.adapter3_1(x)\n",
    "        x = self.layer3[0].bn2(x)\n",
    "        residual = self.layer3[0].downsample(residual)\n",
    "        x = x + residual\n",
    "        x = self.layer3[0].relu(x)\n",
    "        \n",
    "        residual=x\n",
    "        x = self.layer3[1].conv1(x) + self.adapter3_2(x)\n",
    "        x = self.layer3[1].bn1(x)\n",
    "        x = self.layer3[1].relu(x)\n",
    "        x = self.layer3[1].conv2(x) + self.adapter3_3(x)\n",
    "        x = self.layer3[1].bn2(x)\n",
    "        x = x + residual\n",
    "        x = self.layer3[1].relu(x)\n",
    "        \n",
    "        \n",
    "        residual=x\n",
    "        x = self.layer4[0].conv1(x) + self.adapter4_0(x)\n",
    "        x = self.layer4[0].bn1(x)\n",
    "        x = self.layer4[0].relu(x)\n",
    "        x = self.layer4[0].conv2(x) + self.adapter4_1(x)\n",
    "        x = self.layer4[0].bn2(x)\n",
    "        residual = self.layer4[0].downsample(residual)\n",
    "        x = x + residual\n",
    "        x = self.layer4[0].relu(x)\n",
    "        \n",
    "        residual=x\n",
    "        x = self.layer4[1].conv1(x) + self.adapter4_2(x)\n",
    "        x = self.layer4[1].bn1(x)\n",
    "        x = self.layer4[1].relu(x)\n",
    "        x = self.layer4[1].conv2(x) + self.adapter4_3(x)\n",
    "        x = self.layer4[1].bn2(x)\n",
    "        x = x + residual\n",
    "        x = self.layer4[1].relu(x)\n",
    "        x = self.avgpool(x)\n",
    "        x = x.view(x.size(0), -1)\n",
    "        x = self.fc(x)\n",
    "        return x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "rs=rsn18()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "rs.conv1.weight.data = net_2.resnet.conv1.weight.data\n",
    "rs.bn1.weight.data = net_2.resnet.bn1.weight.data\n",
    "\n",
    "rs.layer1[0].conv1.weight.data = net_2.resnet.layer1[0].conv1.weight.data\n",
    "rs.layer1[0].bn1.weight.data = net_2.resnet.layer1[0].bn1.weight.data\n",
    "rs.layer1[0].conv2.weight.data = net_2.resnet.layer1[0].conv2.weight.data\n",
    "rs.layer1[0].bn2.weight.data = net_2.resnet.layer1[0].bn2.weight.data\n",
    "\n",
    "rs.layer1[1].conv1.weight.data = net_2.resnet.layer1[1].conv1.weight.data\n",
    "rs.layer1[1].bn1.weight.data = net_2.resnet.layer1[1].bn1.weight.data\n",
    "rs.layer1[1].conv2.weight.data = net_2.resnet.layer1[1].conv2.weight.data\n",
    "rs.layer1[1].bn2.weight.data = net_2.resnet.layer1[1].bn2.weight.data\n",
    "\n",
    "\n",
    "rs.layer2[0].conv1.weight.data = net_2.resnet.layer2[0].conv1.weight.data\n",
    "rs.layer2[0].bn1.weight.data = net_2.resnet.layer2[0].bn1.weight.data\n",
    "rs.layer2[0].conv2.weight.data = net_2.resnet.layer2[0].conv2.weight.data\n",
    "rs.layer2[0].bn2.weight.data = net_2.resnet.layer2[0].bn2.weight.data\n",
    "\n",
    "rs.layer2[1].conv1.weight.data = net_2.resnet.layer2[1].conv1.weight.data\n",
    "rs.layer2[1].bn1.weight.data = net_2.resnet.layer2[1].bn1.weight.data\n",
    "rs.layer2[1].conv2.weight.data = net_2.resnet.layer2[1].conv2.weight.data\n",
    "rs.layer2[1].bn2.weight.data = net_2.resnet.layer2[1].bn2.weight.data\n",
    "rs.layer2[0].downsample[0].weight.data=net_2.resnet.layer2[0].downsample[0].weight.data\n",
    "rs.layer2[0].downsample[1].weight.data=net_2.resnet.layer2[0].downsample[1].weight.data\n",
    "\n",
    "rs.layer3[0].conv1.weight.data = net_2.resnet.layer3[0].conv1.weight.data\n",
    "rs.layer3[0].bn1.weight.data = net_2.resnet.layer3[0].bn1.weight.data\n",
    "rs.layer3[0].conv2.weight.data = net_2.resnet.layer3[0].conv2.weight.data\n",
    "rs.layer3[0].bn2.weight.data = net_2.resnet.layer3[0].bn2.weight.data\n",
    "\n",
    "rs.layer3[1].conv1.weight.data = net_2.resnet.layer3[1].conv1.weight.data\n",
    "rs.layer3[1].bn1.weight.data = net_2.resnet.layer3[1].bn1.weight.data\n",
    "rs.layer3[1].conv2.weight.data = net_2.resnet.layer3[1].conv2.weight.data\n",
    "rs.layer3[1].bn2.weight.data = net_2.resnet.layer3[1].bn2.weight.data\n",
    "rs.layer3[0].downsample[0].weight.data=net_2.resnet.layer3[0].downsample[0].weight.data\n",
    "rs.layer3[0].downsample[1].weight.data=net_2.resnet.layer3[0].downsample[1].weight.data\n",
    "\n",
    "rs.layer4[0].conv1.weight.data = net_2.resnet.layer4[0].conv1.weight.data\n",
    "rs.layer4[0].bn1.weight.data = net_2.resnet.layer4[0].bn1.weight.data\n",
    "rs.layer4[0].conv2.weight.data = net_2.resnet.layer4[0].conv2.weight.data\n",
    "rs.layer4[0].bn2.weight.data = net_2.resnet.layer4[0].bn2.weight.data\n",
    "\n",
    "rs.layer4[1].conv1.weight.data = net_2.resnet.layer4[1].conv1.weight.data\n",
    "rs.layer4[1].bn1.weight.data = net_2.resnet.layer4[1].bn1.weight.data\n",
    "rs.layer4[1].conv2.weight.data = net_2.resnet.layer4[1].conv2.weight.data\n",
    "rs.layer4[1].bn2.weight.data = net_2.resnet.layer4[1].bn2.weight.data\n",
    "rs.layer4[0].downsample[0].weight.data=net_2.resnet.layer4[0].downsample[0].weight.data\n",
    "rs.layer4[0].downsample[1].weight.data=net_2.resnet.layer4[0].downsample[1].weight.data\n",
    "\n",
    "rs.fc.weight.data=net_2.resnet.fc.weight.data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "class convNet2(nn.Module):\n",
    "    #constructor\n",
    "    def __init__(self,resnet,mynet):\n",
    "        super(convNet2, self).__init__()\n",
    "        #defining layers in convnet\n",
    "        self.rsn=resnet\n",
    "        self.myNet=mynet\n",
    "    def forward(self, x):\n",
    "        x=self.rsn(x)\n",
    "        x=self.myNet(x)\n",
    "        return x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "cnvnet2=convNet2(resnet=rs, mynet=net1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "cnvnet2.myNet.bn1.weight.data=net_2.myNet.bn1.weight.data\n",
    "cnvnet2.myNet.fc2.weight.data=net_2.myNet.fc2.weight.data\n",
    "cnvnet2.myNet.bn2.weight.data=net_2.myNet.bn2.weight.data\n",
    "cnvnet2.myNet.fc3.weight.data=net_2.myNet.fc3.weight.data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "torch.save(cnvnet2, 'adaptersnetwork1.pt')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "cnvnet2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from scipy.stats import spearmanr\n",
    "def computeSpearman(dataloader_valid, model):\n",
    "    ratings = []\n",
    "    predictions=[]\n",
    "    #device = cuda\n",
    "    #criterion = nn.MSELoss()\n",
    "    #criterion.cuda()\n",
    "    with torch.no_grad():\n",
    "        cum_loss=0\n",
    "        for batch_idx, data in enumerate(dataloader_valid):\n",
    "            inputs = data['image']\n",
    "            labels=data['rating']\n",
    "            if use_gpu:\n",
    "                try:\n",
    "                    inputs, labels = Variable(inputs.float().cuda()), Variable(labels.float().cuda())\n",
    "                except:\n",
    "                    print(inputs,labels)\n",
    "            else:\n",
    "                inputs, labels = Variable(inputs), Variable(labels)\n",
    "            outputs = model(inputs)\n",
    "            ratings.append(labels.float())\n",
    "            predictions.append(outputs.float())\n",
    "    \n",
    "    ratings_i=[(list(np.float_([j for j in i]))) for i in ratings]\n",
    "    predictions_i=[(list(np.float_([j for j in i]))) for i in predictions]\n",
    "    ratings_i=np.vstack(ratings)\n",
    "    predictions_i=np.vstack(predictions)\n",
    "    from scipy.stats import spearmanr\n",
    "    return spearmanr(ratings_i, predictions_i)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import time\n",
    "from PIL import ImageFile\n",
    "ImageFile.LOAD_TRUNCATED_IMAGES = True\n",
    "def train_model(model, criterion, optimizer1, optimizer2, lr_scheduler,dataloader_train,dataloader_valid,  num_epochs=100):\n",
    "    since = time.time()\n",
    "    train_loss_average=[]\n",
    "    test_loss=[]\n",
    "    spearman_test=[]\n",
    "    best_model = model\n",
    "    best_loss = 100\n",
    "    best_spearman=0\n",
    "    use_gpu=True\n",
    "    model_1=model\n",
    "    criterion.cuda()\n",
    "    for epoch in range(num_epochs):\n",
    "        #print('Epoch {}/{}'.format(epoch, num_epochs - 1))\n",
    "        #print('-' * 10)\n",
    "\n",
    "        # Each epoch has a training and validation phase\n",
    "        for phase in ['val', 'train']:\n",
    "            if phase == 'train':\n",
    "                model=model_1\n",
    "                mode='train'\n",
    "                optimizer1 = lr_scheduler(optimizer1, epoch)\n",
    "                optimizer2 = lr_scheduler(optimizer2, epoch)\n",
    "                model.train()  # Set model to training mode\n",
    "                dataloader=dataloader_train\n",
    "            elif phase=='val' and epoch==0:\n",
    "                net_2 = (torch.load('fine_tuned_flickerAES_normalized_dropout_resnet18_customnetworkadamnormalized.pt'))\n",
    "                model=net_2\n",
    "                model.eval()\n",
    "                mode='val'\n",
    "                dataloader=dataloader_valid\n",
    "            else:\n",
    "                model=model_1\n",
    "                model.eval()\n",
    "                mode='val'\n",
    "                dataloader=dataloader_valid\n",
    "\n",
    "            running_loss = 0.0\n",
    "            model.cuda()\n",
    "\n",
    "            counter=0\n",
    "            # Iterate over data.\n",
    "            for batch_idx, data in enumerate(dataloader):\n",
    "                inputs = data['image']\n",
    "                labels=data['rating']\n",
    "                if use_gpu:\n",
    "                    try:\n",
    "                        inputs, labels = Variable(inputs.float().cuda()), Variable(labels.float().cuda())\n",
    "                    except:\n",
    "                        print(inputs,labels)\n",
    "                else:\n",
    "                    inputs, labels = Variable(inputs), Variable(labels)\n",
    "                # wrap them in Variable\n",
    "\n",
    "                # Set gradient to zero to delete history of computations in previous epoch. Track operations so that differentiation can be done automatically.\n",
    "                optimizer1.zero_grad()\n",
    "                optimizer2.zero_grad()\n",
    "                outputs = model(inputs)\n",
    "                \n",
    "                loss = criterion(outputs, labels)\n",
    "                #print('loss done')                \n",
    "                # Just so that you can keep track that something's happening and don't feel like the program isn't running.\n",
    "                #if counter%200==0:\n",
    "                    #print(\"Reached iteration \",counter)\n",
    "                counter+=1\n",
    "\n",
    "                # backward + optimize only if in training phase\n",
    "                if phase == 'train':\n",
    "                    #print('loss backward')\n",
    "                    loss.backward()\n",
    "                    #print('done loss backward')\n",
    "                    optimizer1.step()\n",
    "                    optimizer2.step()\n",
    "\n",
    "                    #print('done optim')\n",
    "                # print evaluation statistics\n",
    "                try:\n",
    "                    running_loss += loss.data[0]\n",
    "                except:\n",
    "                    print('unexpected error, could not calculate loss or do a sum.')\n",
    "            #print('trying epoch loss')\n",
    "            epoch_loss = running_loss / len(dataloader)\n",
    "            #print('{} Loss: {:.4f} '.format(\n",
    "            #    phase, epoch_loss))\n",
    "            if phase == 'train':\n",
    "                train_loss_average.append(epoch_loss)\n",
    "\n",
    "\n",
    "            # deep copy the model\n",
    "            if phase == 'val':\n",
    "                test_loss.append(epoch_loss)\n",
    "                sp=computeSpearman(dataloader, model)[0]\n",
    "                spearman_test.append(sp)\n",
    "\n",
    "                if epoch_loss < best_loss:\n",
    "                    best_loss=epoch_loss\n",
    "                    #print('new best loss = ',epoch_loss)\n",
    "                \n",
    "                if sp>best_spearman:\n",
    "                    best_spearman=sp\n",
    "                    best_model = copy.deepcopy(model)\n",
    "\n",
    "    time_elapsed = time.time() - since\n",
    "    print('Training complete in {:.0f}m {:.0f}s'.format(\n",
    "        time_elapsed // 60, time_elapsed % 60))\n",
    "    print('Best val loss: {:4f}'.format(best_loss))\n",
    "    print('returning and looping back')\n",
    "    return best_model.cuda(), train_loss_average, np.asarray(test_loss), np.asarray(spearman_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from torchvision import models\n",
    "from torch import optim\n",
    "import torch.nn.functional as F\n",
    "\n",
    "model_ft=cnvnet2\n",
    "\n",
    "\n",
    "#device = cuda\n",
    "criterion = nn.MSELoss()\n",
    "criterion.cuda()\n",
    "model_ft.cuda()\n",
    "for name, m in model_ft.named_modules():\n",
    "        if isinstance(m, nn.Conv2d) and (m.kernel_size[0]==3 or m.kernel_size[0]==7):\n",
    "            m.weight.requires_grad = False\n",
    "model_ft.rsn.fc.weight.requires_grad=False\n",
    "model_ft.rsn.layer2[0].downsample[0].weight.requires_grad=False\n",
    "model_ft.rsn.layer3[0].downsample[0].weight.requires_grad=False\n",
    "model_ft.rsn.layer4[0].downsample[0].weight.requires_grad=False\n",
    "\n",
    "\n",
    "\n",
    "optimizer = optim.Adam(filter(lambda p: p.requires_grad, model_ft.parameters()), lr=0.001, weight_decay=0.001)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def ComputeIncreaseSP(sp):\n",
    "    return max(sp) - sp[0]\n",
    "\n",
    "def ComputeDecreaseMSE(MSEerrors):\n",
    "    return min(MSEerrors)-MSEerrors[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "epochs=100\n",
    "stats=[]\n",
    "for worker_idx in range(0,37):\n",
    "    resu= []\n",
    "    for i in range(0,3):\n",
    "        worker=workers_test['worker'].unique()[worker_idx]\n",
    "        print(worker, i)\n",
    "        num_images = workers_test[workers_test['worker'].isin([worker])].shape[0]\n",
    "        percent=100/num_images\n",
    "        images=workers_test[workers_test['worker'].isin([worker])][[' imagePair', ' score']]\n",
    "        train_dataframe, valid_dataframe = train_test_split(images, train_size=percent)\n",
    "        train_path=\"train_means_normalized\" + worker +\".csv\"\n",
    "        test_path=\"test_means_normalized\" + worker +\".csv\"\n",
    "        train_dataframe.to_csv(train_path, sep=' ', index_label = False)\n",
    "        valid_dataframe.to_csv(test_path, sep=' ', index_label = False)\n",
    "\n",
    "        output_size=(224,224)\n",
    "        transformed_dataset_train = ImageRatingsDataset(csv_file=train_path,root_dir='Images/',\n",
    "                                               transform=transforms.Compose([Rescale(output_size=(256,256)),\n",
    "                                                                             RandomHorizontalFlip(0.5),\n",
    "                                                                             RandomCrop(output_size),\n",
    "                                                                             Normalize(),\n",
    "                                                                            ToTensor(),\n",
    "                                               ]))\n",
    "        transformed_dataset_valid = ImageRatingsDataset(csv_file=test_path,root_dir='Images/',\n",
    "                                               transform=transforms.Compose([Rescale(output_size=(224,224)),\n",
    "                                                                             Normalize(),\n",
    "                                                                            ToTensor(),\n",
    "                                               ]))\n",
    "\n",
    "        from torch.utils.data.dataloader import default_collate\n",
    "        bsize=30\n",
    "        def my_collate(batch):\n",
    "            batch = list(filter (lambda x:x is not None, batch))\n",
    "            return default_collate(batch)\n",
    "\n",
    "        dataloader_train = DataLoader(transformed_dataset_train, batch_size=bsize,\n",
    "                                shuffle=True, num_workers=0,collate_fn=my_collate)\n",
    "        dataloader_valid = DataLoader(transformed_dataset_valid, batch_size=20,\n",
    "                                shuffle=True, num_workers=0,collate_fn=my_collate)\n",
    "\n",
    "\n",
    "        cnvnet2 = (torch.load('adaptersnetwork1.pt'))\n",
    "        model_ft=cnvnet2\n",
    "        for name, m in model_ft.named_modules():\n",
    "            if isinstance(m, nn.Conv2d) and (m.kernel_size[0]==3 or m.kernel_size[0]==7):\n",
    "                m.weight.requires_grad = False\n",
    "        model_ft.rsn.fc.weight.requires_grad=False\n",
    "        model_ft.rsn.layer2[0].downsample[0].weight.requires_grad=False\n",
    "        model_ft.rsn.layer3[0].downsample[0].weight.requires_grad=False\n",
    "        model_ft.rsn.layer4[0].downsample[0].weight.requires_grad=False\n",
    "\n",
    "        #device = cuda\n",
    "        criterion = nn.MSELoss()\n",
    "        criterion.cuda()\n",
    "        model_ft.cuda()\n",
    "        optimizer1 = optim.Adam(filter(lambda p: p.requires_grad, model_ft.rsn.parameters()),  lr=0.001, weight_decay=0.01)\n",
    "        optimizer2 = optim.Adam(filter(lambda p: p.requires_grad, model_ft.myNet.parameters()),  lr=0.001, weight_decay=0)\n",
    "\n",
    "        iteration=0\n",
    "        network_path= worker + 'network_0 K100 adapters it' + str(i) + '.pt'\n",
    "        # Run the functions and save the best model in the function model_ft.\n",
    "        model_ft, train_loss, test_loss, spearman = train_model(model_ft, criterion, optimizer1, optimizer2, exp_lr_scheduler,dataloader_train,dataloader_valid,\n",
    "                               num_epochs=epochs)\n",
    " \n",
    "\n",
    "        #torch.save(model_ft, network_path)\n",
    "        results=worker, ComputeDecreaseMSE(test_loss), ComputeIncreaseSP(spearman), np.argmin(test_loss), np.argmax(spearman)\n",
    "        resu.append(results)\n",
    "        \n",
    "        \n",
    "    resupd=pd.DataFrame(resu)\n",
    "    resupd.columns=['Worker', 'DecreaseMSE', 'IncreaseSpearman', 'ItMSE', 'ItSpearman']\n",
    "    path=worker+'k100 adapters.csv'\n",
    "    resupd.to_csv(path, sep=' ', index_label = False)\n",
    "    number=images[' imagePair'].unique().shape[0]\n",
    "    stat=resupd['Worker'][0],number, np.mean(resupd['DecreaseMSE']), np.std(resupd['DecreaseMSE']), np.mean(resupd['IncreaseSpearman']), np.std(resupd['IncreaseSpearman']), np.mean(resupd['ItMSE']), np.std(resupd['ItMSE']),np.mean(resupd['ItSpearman']), np.std(resupd['ItSpearman']),\n",
    "    stats.append(stat)\n",
    "stats=pd.DataFrame(stats)\n",
    "stats.to_csv('100 epochs k100 adaptersfinetuning01 WD01.csv', sep=' ', index_label=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def ComputeIncreaseSP(sp):\n",
    "    return max(sp) - sp[0]\n",
    "\n",
    "def ComputeDecreaseMSE(MSEerrors):\n",
    "    return min(MSEerrors)-MSEerrors[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for name, m in model_ft.named_modules():\n",
    "    if (isinstance(m, nn.Conv2d) or isinstance(m, nn.Linear) or isinstance(m, nn.BatchNorm1d) or isinstance(m, nn.BatchNorm2d)):\n",
    "        print(name, str(m.weight.requires_grad))\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "from graphviz import Digraph\n",
    "import re\n",
    "import torch\n",
    "import torch.nn.functional as F\n",
    "from torch.autograd import Variable\n",
    "from torch.autograd import Variable\n",
    "import torchvision.models as models\n",
    "\n",
    "\n",
    "def make_dot(var, params):\n",
    "    \"\"\" Produces Graphviz representation of PyTorch autograd graph\n",
    "    \n",
    "    Blue nodes are the Variables that require grad, orange are Tensors\n",
    "    saved for backward in torch.autograd.Function\n",
    "    \n",
    "    Args:\n",
    "        var: output Variable\n",
    "        params: dict of (name, Variable) to add names to node that\n",
    "            require grad (TODO: make optional)\n",
    "    \"\"\"\n",
    "    param_map = {id(v): k for k, v in params.items()}\n",
    "    print(param_map)\n",
    "    \n",
    "    node_attr = dict(style='filled',\n",
    "                     shape='box',\n",
    "                     align='left',\n",
    "                     fontsize='12',\n",
    "                     ranksep='0.1',\n",
    "                     height='0.2')\n",
    "    dot = Digraph(node_attr=node_attr, graph_attr=dict(size=\"12,12\"))\n",
    "    seen = set()\n",
    "    \n",
    "    def size_to_str(size):\n",
    "        return '('+(', ').join(['%d'% v for v in size])+')'\n",
    "\n",
    "    def add_nodes(var):\n",
    "        if var not in seen:\n",
    "            if torch.is_tensor(var):\n",
    "                dot.node(str(id(var)), size_to_str(var.size()), fillcolor='orange')\n",
    "            elif hasattr(var, 'variable'):\n",
    "                u = var.variable\n",
    "                node_name = '%s\\n %s' % (param_map.get(id(u)), size_to_str(u.size()))\n",
    "                dot.node(str(id(var)), node_name, fillcolor='lightblue')\n",
    "            else:\n",
    "                dot.node(str(id(var)), str(type(var).__name__))\n",
    "            seen.add(var)\n",
    "            if hasattr(var, 'next_functions'):\n",
    "                for u in var.next_functions:\n",
    "                    if u[0] is not None:\n",
    "                        dot.edge(str(id(u[0])), str(id(var)))\n",
    "                        add_nodes(u[0])\n",
    "            if hasattr(var, 'saved_tensors'):\n",
    "                for t in var.saved_tensors:\n",
    "                    dot.edge(str(id(t)), str(id(var)))\n",
    "                    add_nodes(t)\n",
    "    add_nodes(var.grad_fn)\n",
    "    return dot\n",
    "\n",
    "inputs = torch.randn(2,3,224,224)\n",
    "y = model_ft(Variable(inputs.cuda()))\n",
    "print(y)\n",
    "\n",
    "g = make_dot(y, model_ft.state_dict())\n",
    "g"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "g.render()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}