lecture33.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Lecture 33: AlexNet"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "import tqdm\n",
    "import copy\n",
    "import time\n",
    "import torch\n",
    "import numpy as np\n",
    "import torch.nn as nn\n",
    "import torch.optim as optim\n",
    "import torch.nn.functional as F\n",
    "import matplotlib.pyplot as plt\n",
    "import torchvision\n",
    "from torchvision import transforms,datasets, models\n",
    "\n",
    "print(torch.__version__) # This code has been updated for PyTorch 1.0.0"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Load Data:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "apply_transform = transforms.Compose([transforms.Resize(224),transforms.ToTensor()])\n",
    "BatchSize = 4\n",
    "\n",
    "trainset = datasets.CIFAR10(root='./CIFAR10', train=True, download=True, transform=apply_transform)\n",
    "trainLoader = torch.utils.data.DataLoader(trainset, batch_size=BatchSize,\n",
    "                                          shuffle=True, num_workers=4) # Creating dataloader\n",
    "\n",
    "testset = datasets.CIFAR10(root='./CIFAR10', train=False, download=True, transform=apply_transform)\n",
    "testLoader = torch.utils.data.DataLoader(testset, batch_size=BatchSize,\n",
    "                                         shuffle=False, num_workers=4) # Creating dataloader"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Size of train and test datasets\n",
    "print('No. of samples in train set: '+str(len(trainLoader.dataset)))\n",
    "print('No. of samples in test set: '+str(len(testLoader.dataset)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Define network architecture"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "net = models.AlexNet()\n",
    "print(net)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Counting number of trainable parameters\n",
    "totalParams = 0\n",
    "for name,params in net.named_parameters():\n",
    "    print(name,'-->',params.size())\n",
    "    totalParams += np.sum(np.prod(params.size()))\n",
    "print('Total number of parameters: '+str(totalParams))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Copying initial weights for visualization\n",
    "init_weightConv1 = copy.deepcopy(net.features[0].weight.data) # 1st conv layer\n",
    "init_weightConv2 = copy.deepcopy(net.features[3].weight.data) # 2nd conv layer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Check availability of GPU\n",
    "\n",
    "use_gpu = torch.cuda.is_available()\n",
    "if use_gpu:\n",
    "    print('GPU is available!')\n",
    "    device = \"cuda\"\n",
    "else:\n",
    "    print('GPU is not available!')\n",
    "    device = \"cpu\"\n",
    "\n",
    "net = net.to(device)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Define loss function and optimizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "criterion = nn.NLLLoss() # Negative Log-likelihood\n",
    "optimizer = optim.Adam(net.parameters(), lr=1e-4) # Adam"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Train the network"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "iterations = 5\n",
    "trainLoss = []\n",
    "testAcc = []\n",
    "start = time.time()\n",
    "for epoch in range(iterations):\n",
    "    epochStart = time.time()\n",
    "    runningLoss = 0    \n",
    "    net.train() # For training\n",
    "    for data in tqdm.tqdm_notebook(trainLoader):\n",
    "        inputs,labels = data\n",
    "        inputs, labels = inputs.to(device), labels.to(device)  \n",
    "       \n",
    "        # Initialize gradients to zero\n",
    "        optimizer.zero_grad()\n",
    "        # Feed-forward input data through the network        \n",
    "        outputs = net(inputs)        \n",
    "        # Compute loss/error\n",
    "        loss = criterion(F.log_softmax(outputs,dim=1), labels)\n",
    "        # Backpropagate loss and compute gradients\n",
    "        loss.backward()\n",
    "        # Update the network parameters\n",
    "        optimizer.step()\n",
    "        # Accumulate loss per batch\n",
    "        runningLoss += loss.item()  \n",
    "    avgTrainLoss = runningLoss/(len(trainset)/BatchSize)\n",
    "    trainLoss.append(avgTrainLoss)\n",
    "    \n",
    "    # Evaluating performance on test set for each epoch\n",
    "    net.eval() # For testing [Affects batch-norm and dropout layers (if any)]\n",
    "    running_correct = 0\n",
    "    with torch.no_grad():\n",
    "        for data in testLoader:\n",
    "            inputs,labels = data\n",
    "            inputs = inputs.to(device)\n",
    "            outputs = net(inputs)\n",
    "            _, predicted = torch.max(outputs.data, 1)\n",
    "            if use_gpu:                \n",
    "                predicted = predicted.cpu()            \n",
    "            running_correct += (predicted == labels).sum()\n",
    "    avgTestAcc = float(running_correct)*100/10000.0\n",
    "    testAcc.append(avgTestAcc)\n",
    "        \n",
    "    # Plotting training loss vs Epochs\n",
    "    fig1 = plt.figure(1)        \n",
    "    plt.plot(range(epoch+1),trainLoss,'r-',label='train')        \n",
    "    if epoch==0:\n",
    "        plt.legend(loc='upper left')\n",
    "        plt.xlabel('Epochs')\n",
    "        plt.ylabel('Training loss')   \n",
    "    # Plotting testing accuracy vs Epochs\n",
    "    fig2 = plt.figure(2)        \n",
    "    plt.plot(range(epoch+1),testAcc,'g-',label='test')        \n",
    "    if epoch==0:\n",
    "        plt.legend(loc='upper left')\n",
    "        plt.xlabel('Epochs')\n",
    "        plt.ylabel('Testing accuracy')    \n",
    "    epochEnd = time.time()-epochStart\n",
    "    print('Iteration: {:.0f} /{:.0f}  ;  Training Loss: {:.6f} ; Testing Acc: {:.3f} ; Time consumed: {:.0f}m {:.0f}s '\\\n",
    "          .format(epoch + 1,iterations,avgTrainLoss,avgTestAcc,epochEnd//60,epochEnd%60))\n",
    "end = time.time()-start\n",
    "print('Training completed in {:.0f}m {:.0f}s'.format(end//60,end%60))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Copying trained weights for visualization\n",
    "trained_weightConv1 = copy.deepcopy(net.features[0].weight.data)\n",
    "trained_weightConv2 = copy.deepcopy(net.features[3].weight.data)\n",
    "if use_gpu:\n",
    "    trained_weightConv1 = trained_weightConv1.cpu()\n",
    "    trained_weightConv2 = trained_weightConv2.cpu()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "## Visualization of weights"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# functions to show an image\n",
    "def imshow(img, strlabel):\n",
    "    npimg = img.numpy()\n",
    "    npimg = np.abs(npimg)\n",
    "    fig_size = plt.rcParams[\"figure.figsize\"]\n",
    "    fig_size[0] = 10\n",
    "    fig_size[1] = 10\n",
    "    plt.rcParams[\"figure.figsize\"] = fig_size\n",
    "    plt.figure()\n",
    "    plt.title(strlabel)\n",
    "    plt.imshow(np.transpose(npimg, (1, 2, 0)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "imshow(torchvision.utils.make_grid(init_weightConv1,nrow=8,normalize=True),'Initial weights: conv1')\n",
    "imshow(torchvision.utils.make_grid(trained_weightConv1,nrow=8,normalize=True),'Trained weights: conv1')\n",
    "imshow(torchvision.utils.make_grid(init_weightConv1-trained_weightConv1,nrow=8,normalize=True),'Difference of weights: conv1')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "imshow(torchvision.utils.make_grid(init_weightConv2[0].unsqueeze(1),nrow=8,normalize=True),'Initial weights: conv2')\n",
    "imshow(torchvision.utils.make_grid(trained_weightConv2[0].unsqueeze(1),nrow=8,normalize=True),'Trained weights: conv2')\n",
    "imshow(torchvision.utils.make_grid(init_weightConv2[0].unsqueeze(1)-trained_weightConv2[0].unsqueeze(1),nrow=8,normalize=True),'Difference of weights: conv2')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}