lecture44.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Lecture 44: Transfer Learning GoogLeNet"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "import copy\n",
    "import time\n",
    "import tqdm\n",
    "import torch\n",
    "import numpy as np\n",
    "import torchvision\n",
    "import torch.nn as nn\n",
    "import torch.optim as optim\n",
    "import torch.nn.functional as F\n",
    "import matplotlib.pyplot as plt\n",
    "from torchvision import transforms,datasets, models\n",
    "\n",
    "print(torch.__version__) # This code has been updated for PyTorch 1.0.0"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Load Data:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# inception_v3 in pytorch requries input to be of size(3x299x299)\n",
    "apply_transform = transforms.Compose([transforms.Resize(299),transforms.ToTensor()])\n",
    "BatchSize = 2 # Batchsize > 1\n",
    "\n",
    "trainset = datasets.CIFAR10(root='./CIFAR10', train=True, download=True, transform=apply_transform)\n",
    "trainLoader = torch.utils.data.DataLoader(trainset, batch_size=BatchSize,\n",
    "                                          shuffle=True, num_workers=4) # Creating dataloader\n",
    "\n",
    "testset = datasets.CIFAR10(root='./CIFAR10', train=False, download=True, transform=apply_transform)\n",
    "testLoader = torch.utils.data.DataLoader(testset, batch_size=BatchSize,\n",
    "                                         shuffle=False, num_workers=4) # Creating dataloader"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Size of train and test datasets\n",
    "print('No. of samples in train set: '+str(len(trainLoader.dataset)))\n",
    "print('No. of samples in test set: '+str(len(testLoader.dataset)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Define network architecture"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "net1 = models.inception_v3() # Training from scratch\n",
    "net2 = models.inception_v3(pretrained=True) # End-to-end fine-tuning\n",
    "net3 = models.inception_v3(pretrained=True) # Training only the last layer\n",
    "print(net1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Counting number of trainable parameters\n",
    "totalParams = 0\n",
    "for name,params in net1.named_parameters():\n",
    "    print(name,'-->',params.size())\n",
    "    totalParams += np.sum(np.prod(params.size()))\n",
    "print('Total number of parameters: '+str(totalParams))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Modifying the last fully-connected layers(including aux network) for 10 classes\n",
    "net1.AuxLogits.fc = nn.Linear(768,10)\n",
    "net1.fc = nn.Linear(2048,10)\n",
    "net2.AuxLogits.fc = nn.Linear(768,10)\n",
    "net2.fc = nn.Linear(2048,10)\n",
    "net3.AuxLogits.fc = nn.Linear(768,10)\n",
    "net3.fc = nn.Linear(2048,10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Copying initial weights for visualization\n",
    "# Model 1\n",
    "init_weightConv1_1 = copy.deepcopy(net1.Conv2d_1a_3x3.conv.weight.data) # 1st conv layer\n",
    "init_weightConv2_1 = copy.deepcopy(net1.Conv2d_2a_3x3.conv.weight.data) # 2nd conv layer\n",
    "Model 2\n",
    "init_weightConv1_2 = copy.deepcopy(net2.Conv2d_1a_3x3.conv.weight.data) # 1st conv layer\n",
    "init_weightConv2_2 = copy.deepcopy(net2.Conv2d_2a_3x3.conv.weight.data) # 2nd conv layer\n",
    "# Model 3\n",
    "init_weightConv1_3 = copy.deepcopy(net3.Conv2d_1a_3x3.conv.weight.data) # 1st conv layer\n",
    "init_weightConv2_3 = copy.deepcopy(net3.Conv2d_2a_3x3.conv.weight.data) # 2nd conv layer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Check availability of GPU\n",
    "\n",
    "use_gpu = torch.cuda.is_available()\n",
    "# use_gpu = False # Uncomment in case of GPU memory error\n",
    "if use_gpu:\n",
    "    print('GPU is available!')\n",
    "    device = \"cuda\"\n",
    "else:\n",
    "    print('GPU is not available!')\n",
    "    device = \"cpu\"\n",
    "    \n",
    "net1 = net1.to(device)\n",
    "net2 = net2.to(device)\n",
    "net3 = net3.to(device)\n",
    "# Freezing all parameters to save memory and computation\n",
    "for parameter in net3.parameters(): \n",
    "    parameter.requires_grad = False\n",
    "for parameter in net3.fc.parameters(): # Unfreezing fully-conencted layer\n",
    "    parameter.requires_grad = True                           "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Define loss function and optimizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "criterion = nn.NLLLoss() # Negative Log-likelihood\n",
    "optimizer1 = optim.Adam(net1.parameters(), lr=1e-4) # Adam; passing all params\n",
    "optimizer2 = optim.Adam(net2.parameters(), lr=1e-4) # Adam; passing all params\n",
    "optimizer3 = optim.Adam(net3.fc.parameters(), lr=1e-4) # Adam; passing params of only the last fc layer"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Train the network"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "iterations = 10\n",
    "# Model 1\n",
    "trainLoss1 = [] # List for saving main loss per epoch\n",
    "trainAuxLoss1 = [] # List for saving auxillary loss per epoch\n",
    "trainTotalLoss1 = [] # List for saving total loss per epoch\n",
    "trainAcc1 = [] # List for saving training accuracy per epoch\n",
    "testLoss1 = [] # List for saving testing loss per epoc\n",
    "testAcc1 = [] # List for saving testing accuracy per epoch\n",
    "# Model 2\n",
    "trainLoss2 = [] # List for saving main loss per epoch\n",
    "trainAuxLoss2 = [] # List for saving auxillary loss per epoch\n",
    "trainTotalLoss2 = [] # List for saving total loss per epoch\n",
    "trainAcc2 = [] # List for saving training accuracy per epoch\n",
    "testLoss2 = [] # List for saving testing loss per epoc\n",
    "testAcc2 = [] # List for saving testing accuracy per epoch\n",
    "# Model 3\n",
    "trainTotalLoss3 = [] # List for saving total loss per epoch\n",
    "trainAcc3 = [] # List for saving training accuracy per epoch\n",
    "testLoss3 = [] # List for saving testing loss per epoc\n",
    "testAcc3 = [] # List for saving testing accuracy per epoch\n",
    "\n",
    "\n",
    "start = time.time()\n",
    "for epoch in range(iterations):\n",
    "    epochStart = time.time()\n",
    "    # Model 1\n",
    "    runningLoss1 = 0.0\n",
    "    runningAuxLoss1 = 0.0 \n",
    "    runningTotalLoss1 = 0.0\n",
    "    avgAuxLoss1 = 0.0\n",
    "    avgTotalLoss1 = 0.0\n",
    "    running_correct1 = 0\n",
    "    # Model 2\n",
    "    runningLoss2 = 0.0\n",
    "    runningAuxLoss2 = 0.0 \n",
    "    runningTotalLoss2 = 0.0\n",
    "    avgAuxLoss2 = 0.0\n",
    "    avgTotalLoss2 = 0.0\n",
    "    running_correct2 = 0\n",
    "    # Model 3\n",
    "    runningLoss3 = 0.0\n",
    "    runningAuxLoss3 = 0.0 \n",
    "    runningTotalLoss3 = 0.0\n",
    "    avgAuxLoss3 = 0.0\n",
    "    avgTotalLoss3 = 0.0\n",
    "    running_correct3 = 0\n",
    "    \n",
    "    net1.train() # For training\n",
    "    net2.train()\n",
    "    net3.train()\n",
    "\n",
    "    for data in tqdm.tqdm_notebook(trainLoader):\n",
    "        inputs,labels = data\n",
    "       \n",
    "        inputs, labels = inputs.to(device), labels.to(device)\n",
    "        # Feed-forward input data through model 1     \n",
    "        outputs1,aux_outputs1 = net1(inputs)        \n",
    "        _, predicted1 = torch.max(outputs1.data, 1)\n",
    "        running_correct1 += (predicted1 == labels.data).sum()\n",
    "        # Feed-forward input data through model 2     \n",
    "        outputs2,aux_outputs2 = net2(inputs)        \n",
    "        _, predicted2 = torch.max(outputs2.data, 1)\n",
    "        running_correct2 += (predicted2 == labels.data).sum()\n",
    "        # Feed-forward input data through model 3     \n",
    "        outputs3,_ = net3(inputs)   # Training only the last fc layer  \n",
    "        _, predicted3 = torch.max(outputs3.data, 1)\n",
    "        running_correct3 += (predicted3 == labels.data).sum()\n",
    "       \n",
    "        # Initialize gradients to zero\n",
    "        optimizer1.zero_grad()\n",
    "        optimizer2.zero_grad()\n",
    "        optimizer3.zero_grad()\n",
    "        \n",
    "        # Compute loss/error\n",
    "        # Model 1\n",
    "        loss1 = criterion(F.log_softmax(outputs1,dim=1), labels)\n",
    "        aux_loss1 = criterion(F.log_softmax(aux_outputs1,dim=1), labels)\n",
    "        total_loss1 = loss1+aux_loss1\n",
    "        # Model 2\n",
    "        loss2 = criterion(F.log_softmax(outputs2,dim=1), labels)\n",
    "        aux_loss2 = criterion(F.log_softmax(aux_outputs2,dim=1), labels)\n",
    "        total_loss2 = loss2+aux_loss2\n",
    "        # Model 3\n",
    "        loss3 = criterion(F.log_softmax(outputs3,dim=1), labels)        \n",
    "        total_loss3 = loss3\n",
    "        \n",
    "        # Backpropagate loss and compute gradients\n",
    "        total_loss1.backward()\n",
    "        total_loss2.backward()\n",
    "        total_loss3.backward()\n",
    "        # Update the network parameters\n",
    "        optimizer1.step()\n",
    "        optimizer2.step()\n",
    "        optimizer3.step()\n",
    "        # Accumulate loss per batch\n",
    "        runningLoss1 += loss1.item()    \n",
    "        runningAuxLoss1 += aux_loss1.item()    \n",
    "        runningTotalLoss1 += total_loss1.item()  \n",
    "        #--------------------------------------\n",
    "        runningLoss2 += loss2.item()    \n",
    "        runningAuxLoss2 += aux_loss2.item()    \n",
    "        runningTotalLoss2 += total_loss2.item()  \n",
    "        #--------------------------------------\n",
    "        runningLoss3 += loss3.item()        \n",
    "        runningTotalLoss3 += total_loss3.item()  \n",
    "    \n",
    "        \n",
    "    avgTrainAcc1 = 100*float(running_correct1)/50000.0\n",
    "    avgTrainLoss1 = runningLoss1/(50000.0/BatchSize)\n",
    "    avgAuxLoss1 = runningAuxLoss1/(50000.0/BatchSize)\n",
    "    avgTotalLoss1 = runningTotalLoss1/(50000.0/BatchSize)\n",
    "    trainAcc1.append(avgTrainAcc1)\n",
    "    trainLoss1.append(avgTrainLoss1)\n",
    "    trainAuxLoss1.append(avgAuxLoss1)\n",
    "    trainTotalLoss1.append(avgTotalLoss1)\n",
    "    #------------------------------------\n",
    "    avgTrainAcc2 = 100*float(running_correct2)/50000.0\n",
    "    avgTrainLoss2 = runningLoss2/(50000.0/BatchSize)\n",
    "    avgAuxLoss2 = runningAuxLoss2/(50000.0/BatchSize)\n",
    "    avgTotalLoss2 = runningTotalLoss2/(50000.0/BatchSize)\n",
    "    trainAcc2.append(avgTrainAcc2)\n",
    "    trainLoss2.append(avgTrainLoss2)\n",
    "    trainAuxLoss2.append(avgAuxLoss2)\n",
    "    trainTotalLoss2.append(avgTotalLoss2)\n",
    "    #------------------------------------\n",
    "    avgTrainAcc3 = 100*float(running_correct3)/50000.0\n",
    "    avgTotalLoss3 = runningTotalLoss3/(50000.0/BatchSize)\n",
    "    trainAcc3.append(avgTrainAcc3)    \n",
    "    trainTotalLoss3.append(avgTotalLoss3)\n",
    "    \n",
    "    # Evaluating performance on test set for each epoch\n",
    "    net1.eval() # For testing [Affects batch-norm and dropout layers (if any)]\n",
    "    net2.eval()\n",
    "    net3.eval()\n",
    "    # Model 1\n",
    "    runningLoss1 = 0.0\n",
    "    running_correct1 = 0\n",
    "    # Model 2\n",
    "    runningLoss2 = 0.0\n",
    "    running_correct2 = 0\n",
    "    # Model 3\n",
    "    runningLoss3 = 0.0\n",
    "    running_correct3 = 0\n",
    "    \n",
    "    with torch.no_grad():\n",
    "        for data in tqdm.tqdm_notebook(testLoader):\n",
    "            inputs,labels = data\n",
    "            \n",
    "            inputs, labels = inputs.to(device), labels.to(device)\n",
    "            # Model 1\n",
    "            outputs1= net1(inputs)\n",
    "            _, predicted1 = torch.max(outputs1.data, 1)\n",
    "            running_correct1 += (predicted1 == labels.data).sum()\n",
    "            Model 2\n",
    "            outputs2 = net2(inputs)\n",
    "            _, predicted2 = torch.max(outputs2.data, 1)\n",
    "            running_correct2 += (predicted2 == labels.data).sum()\n",
    "            # Model 3\n",
    "            outputs3 = net3(inputs)\n",
    "            _, predicted3 = torch.max(outputs3.data, 1)\n",
    "            running_correct3 += (predicted3 == labels.data).sum()\n",
    "\n",
    "            loss1 = criterion(F.log_softmax(outputs1,dim=1), labels)\n",
    "            runningLoss1 += loss1.item() \n",
    "            #-----------------------------\n",
    "            loss2 = criterion(F.log_softmax(outputs2,dim=1), labels)\n",
    "            runningLoss2 += loss2.item() \n",
    "            #-----------------------------\n",
    "            loss3 = criterion(F.log_softmax(outputs3,dim=1), labels)\n",
    "            runningLoss3 += loss3.item() \n",
    "            \n",
    "        \n",
    "    avgTestLoss1 = runningLoss1/(10000.0/BatchSize)\n",
    "    avgTestAcc1 = 100*float(running_correct1)/10000.0\n",
    "    testLoss1.append(avgTestLoss1)\n",
    "    testAcc1.append(avgTestAcc1)\n",
    "    #---------------------------------------\n",
    "    avgTestLoss2 = runningLoss2/(10000.0/BatchSize)\n",
    "    avgTestAcc2 = 100*float(running_correct2)/10000.0\n",
    "    testLoss2.append(avgTestLoss2)\n",
    "    testAcc2.append(avgTestAcc2)\n",
    "    #---------------------------------------\n",
    "    avgTestLoss3 = runningLoss3/(10000.0/BatchSize)\n",
    "    avgTestAcc3 = 100*float(running_correct3)/10000.0\n",
    "    testLoss3.append(avgTestLoss3)\n",
    "    testAcc3.append(avgTestAcc3)\n",
    "        \n",
    "    # Plotting training loss vs aux_loss\n",
    "    fig1 = plt.figure(1)            \n",
    "    plt.plot(range(epoch+1),trainAuxLoss1,'r-',label='Model 1') \n",
    "    plt.plot(range(epoch+1),trainAuxLoss2,'g-',label='Model 2')     \n",
    "    if epoch==0:\n",
    "        plt.legend(loc='upper left')\n",
    "        plt.xlabel('Epochs')\n",
    "        plt.ylabel('Auxilalry loss')  \n",
    "    \n",
    "    # Plotting training loss vs Epochs: Model 1\n",
    "    fig2 = plt.figure(2)        \n",
    "    plt.plot(range(epoch+1),trainTotalLoss1,'r-',label='train')  \n",
    "    plt.plot(range(epoch+1),testLoss1,'g-',label='test') \n",
    "    if epoch==0:\n",
    "        plt.legend(loc='upper left')\n",
    "        plt.xlabel('Epochs')\n",
    "        plt.ylabel('Loss')   \n",
    "    # Plotting testing accuracy vs Epochs\n",
    "    fig3 = plt.figure(3)        \n",
    "    plt.plot(range(epoch+1),trainAcc1,'r-',label='train')    \n",
    "    plt.plot(range(epoch+1),testAcc1,'g-',label='test')        \n",
    "    if epoch==0:\n",
    "        plt.legend(loc='upper left')\n",
    "        plt.xlabel('Epochs')\n",
    "        plt.ylabel('Accuracy')    \n",
    "        \n",
    "    # Plotting training loss vs Epochs: Model 2\n",
    "    fig4 = plt.figure(4)        \n",
    "    plt.plot(range(epoch+1),trainTotalLoss2,'r-',label='train')  \n",
    "    plt.plot(range(epoch+1),testLoss2,'g-',label='test') \n",
    "    if epoch==0:\n",
    "        plt.legend(loc='upper left')\n",
    "        plt.xlabel('Epochs')\n",
    "        plt.ylabel('Loss')   \n",
    "    # Plotting testing accuracy vs Epochs\n",
    "    fig5 = plt.figure(5)        \n",
    "    plt.plot(range(epoch+1),trainAcc2,'r-',label='train')    \n",
    "    plt.plot(range(epoch+1),testAcc2,'g-',label='test')        \n",
    "    if epoch==0:\n",
    "        plt.legend(loc='upper left')\n",
    "        plt.xlabel('Epochs')\n",
    "        plt.ylabel('Accuracy')  \n",
    "        \n",
    "     # Plotting training loss vs Epochs: Model 3\n",
    "    fig6 = plt.figure(6)        \n",
    "    plt.plot(range(epoch+1),trainTotalLoss3,'r-',label='train')  \n",
    "    plt.plot(range(epoch+1),testLoss3,'g-',label='test') \n",
    "    if epoch==0:\n",
    "        plt.legend(loc='upper left')\n",
    "        plt.xlabel('Epochs')\n",
    "        plt.ylabel('Loss')   \n",
    "    # Plotting testing accuracy vs Epochs\n",
    "    fig7 = plt.figure(7)        \n",
    "    plt.plot(range(epoch+1),trainAcc3,'r-',label='train')    \n",
    "    plt.plot(range(epoch+1),testAcc3,'g-',label='test')        \n",
    "    if epoch==0:\n",
    "        plt.legend(loc='upper left')\n",
    "        plt.xlabel('Epochs')\n",
    "        plt.ylabel('Accuracy')  \n",
    "    \n",
    "    epochEnd = time.time()-epochStart\n",
    "    print('Iteration: {:.0f} /{:.0f} Model 1  ;  Training Loss: {:.6f} ; Testing Acc: {:.3f}'\\\n",
    "          .format(epoch + 1,iterations,avgTrainLoss1,avgTestAcc1))\n",
    "    print('Iteration: {:.0f} /{:.0f} Model 2  ;  Training Loss: {:.6f} ; Testing Acc: {:.3f} '\\\n",
    "          .format(epoch + 1,iterations,avgTrainLoss2,avgTestAcc2))\n",
    "    print('Iteration: {:.0f} /{:.0f} Model 3  ;  Training Loss: {:.6f} ; Testing Acc: {:.3f} '\\\n",
    "      .format(epoch + 1,iterations,avgTotalLoss3,avgTestAcc3))\n",
    "    print('Time consumed: {:.0f}m {:.0f}s'.format(epochEnd//60,epochEnd%60))\n",
    "end = time.time()-start\n",
    "print('Training completed in {:.0f}m {:.0f}s'.format(end//60,end%60))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# Plotting training loss vs Epochs\n",
    "fig8 = plt.figure(8)        \n",
    "plt.plot(range(epoch+1),trainLoss1,'r-',label='model1')  \n",
    "plt.plot(range(epoch+1),trainLoss2,'g-',label='model2') \n",
    "plt.plot(range(epoch+1),trainTotalLoss3,'b-',label='model3') \n",
    "plt.legend(loc='upper left')\n",
    "plt.xlabel('Epochs')\n",
    "plt.ylabel('Train Loss')  \n",
    "\n",
    "fig9 = plt.figure(9)        \n",
    "plt.plot(range(epoch+1),testLoss1,'r-',label='model1')  \n",
    "plt.plot(range(epoch+1),testLoss2,'g-',label='model2') \n",
    "plt.plot(range(epoch+1),testLoss3,'b-',label='model3') \n",
    "plt.legend(loc='upper left')\n",
    "plt.xlabel('Epochs')\n",
    "plt.ylabel('Test Loss') \n",
    "\n",
    "fig10 = plt.figure(10)        \n",
    "plt.plot(range(epoch+1),testAcc1,'r-',label='model1')  \n",
    "plt.plot(range(epoch+1),testAcc2,'g-',label='model2') \n",
    "plt.plot(range(epoch+1),testAcc3,'b-',label='model3') \n",
    "plt.legend(loc='upper left')\n",
    "plt.xlabel('Epochs')\n",
    "plt.ylabel('Test Accuracy') "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Copying trained weights for visualization\n",
    "trained_weightConv1_1 = copy.deepcopy(net1.Conv2d_1a_3x3.conv.weight.data)\n",
    "trained_weightConv2_1 = copy.deepcopy(net1.Conv2d_2a_3x3.conv.weight.data)\n",
    "\n",
    "trained_weightConv1_2 = copy.deepcopy(net2.Conv2d_1a_3x3.conv.weight.data)\n",
    "trained_weightConv2_2 = copy.deepcopy(net2.Conv2d_2a_3x3.conv.weight.data)\n",
    "\n",
    "trained_weightConv1_3 = copy.deepcopy(net3.Conv2d_1a_3x3.conv.weight.data)\n",
    "trained_weightConv2_3 = copy.deepcopy(net3.Conv2d_2a_3x3.conv.weight.data)\n",
    "if use_gpu:\n",
    "    trained_weightConv1_1 = trained_weightConv1_1.cpu()\n",
    "    trained_weightConv2_1 = trained_weightConv2_1.cpu()\n",
    "    \n",
    "    trained_weightConv1_2 = trained_weightConv1_2.cpu()\n",
    "    trained_weightConv2_2 = trained_weightConv2_2.cpu()\n",
    "    \n",
    "    trained_weightConv1_3 = trained_weightConv1_3.cpu()\n",
    "    trained_weightConv2_3 = trained_weightConv2_3.cpu()    "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "## Visualization of weights"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# functions to show an image\n",
    "def imshow(img, strlabel):\n",
    "    npimg = img.numpy()\n",
    "    npimg = np.abs(npimg)\n",
    "    fig_size = plt.rcParams[\"figure.figsize\"]\n",
    "    fig_size[0] = 10\n",
    "    fig_size[1] = 10\n",
    "    plt.rcParams[\"figure.figsize\"] = fig_size\n",
    "    plt.figure()\n",
    "    plt.title(strlabel)\n",
    "    plt.imshow(np.transpose(npimg, (1, 2, 0)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Visualizing weights of 1st convolutional layer of Model 1\n",
    "imshow(torchvision.utils.make_grid(init_weightConv1_1,nrow=8,normalize=True),'Initial weights: conv1')\n",
    "imshow(torchvision.utils.make_grid(trained_weightConv1_1,nrow=8,normalize=True),'Trained weights: conv1')\n",
    "imshow(torchvision.utils.make_grid(init_weightConv1_1-trained_weightConv1_1,nrow=8,normalize=True),'Difference of weights: conv1')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Visualizing weights of 2nd convolutional layer of Model 1\n",
    "imshow(torchvision.utils.make_grid(init_weightConv2_1[0].unsqueeze(1),nrow=8,normalize=True),'Initial weights: conv2')\n",
    "imshow(torchvision.utils.make_grid(trained_weightConv2_1[0].unsqueeze(1),nrow=8,normalize=True),'Trained weights: conv2')\n",
    "imshow(torchvision.utils.make_grid(init_weightConv2_1[0].unsqueeze(1)-trained_weightConv2_1[0].unsqueeze(1),nrow=8,normalize=True),'Difference of weights: conv2')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Visualizing weights of 1st convolutional layer of Model 2\n",
    "imshow(torchvision.utils.make_grid(init_weightConv1_2,nrow=8,normalize=True),'Initial weights: conv1')\n",
    "imshow(torchvision.utils.make_grid(trained_weightConv1_2,nrow=8,normalize=True),'Trained weights: conv1')\n",
    "imshow(torchvision.utils.make_grid(init_weightConv1_2-trained_weightConv1_2,nrow=8,normalize=True),'Difference of weights: conv1')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Visualizing weights of 2nd convolutional layer of Model 2\n",
    "imshow(torchvision.utils.make_grid(init_weightConv2_2[0].unsqueeze(1),nrow=8,normalize=True),'Initial weights: conv2')\n",
    "imshow(torchvision.utils.make_grid(trained_weightConv2_2[0].unsqueeze(1),nrow=8,normalize=True),'Trained weights: conv2')\n",
    "imshow(torchvision.utils.make_grid(init_weightConv2_2[0].unsqueeze(1)-trained_weightConv2_2[0].unsqueeze(1),nrow=8,normalize=True),'Difference of weights: conv2')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Visualizing weights of 1st convolutional layer of Model 3\n",
    "imshow(torchvision.utils.make_grid(init_weightConv1_3,nrow=8,normalize=True),'Initial weights: conv1')\n",
    "imshow(torchvision.utils.make_grid(trained_weightConv1_3,nrow=8,normalize=True),'Trained weights: conv1')\n",
    "imshow(torchvision.utils.make_grid(init_weightConv1_3-trained_weightConv1_3,nrow=8,normalize=True),'Difference of weights: conv1')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Visualizing weights of 2nd convolutional layer of Model 3\n",
    "imshow(torchvision.utils.make_grid(init_weightConv2_3[0].unsqueeze(1),nrow=8,normalize=True),'Initial weights: conv2')\n",
    "imshow(torchvision.utils.make_grid(trained_weightConv2_3[0].unsqueeze(1),nrow=8,normalize=True),'Trained weights: conv2')\n",
    "imshow(torchvision.utils.make_grid(init_weightConv2_3[0].unsqueeze(1)-trained_weightConv2_3[0].unsqueeze(1),nrow=8,normalize=True),'Difference of weights: conv2')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}