PNAS_module.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Sun Jul  5 13:13:58 2020       \r\n",
      "+-----------------------------------------------------------------------------+\r\n",
      "| NVIDIA-SMI 440.33.01    Driver Version: 440.33.01    CUDA Version: 10.2     |\r\n",
      "|-------------------------------+----------------------+----------------------+\r\n",
      "| GPU  Name        Persistence-M| Bus-Id        Disp.A | Volatile Uncorr. ECC |\r\n",
      "| Fan  Temp  Perf  Pwr:Usage/Cap|         Memory-Usage | GPU-Util  Compute M. |\r\n",
      "|===============================+======================+======================|\r\n",
      "|   0  GeForce RTX 208...  Off  | 00000000:3B:00.0 Off |                  N/A |\r\n",
      "| 31%   47C    P0    64W / 250W |      0MiB / 11019MiB |      1%      Default |\r\n",
      "+-------------------------------+----------------------+----------------------+\r\n",
      "|   1  GeForce RTX 208...  Off  | 00000000:5E:00.0 Off |                  N/A |\r\n",
      "| 33%   41C    P0    64W / 250W |      0MiB / 11019MiB |      1%      Default |\r\n",
      "+-------------------------------+----------------------+----------------------+\r\n",
      "|   2  GeForce RTX 208...  Off  | 00000000:86:00.0 Off |                  N/A |\r\n",
      "| 30%   39C    P0    47W / 250W |      0MiB / 11019MiB |      0%      Default |\r\n",
      "+-------------------------------+----------------------+----------------------+\r\n",
      "|   3  GeForce RTX 208...  Off  | 00000000:AF:00.0 Off |                  N/A |\r\n",
      "| 24%   44C    P0    34W / 250W |      0MiB / 11016MiB |      0%      Default |\r\n",
      "+-------------------------------+----------------------+----------------------+\r\n",
      "                                                                               \r\n",
      "+-----------------------------------------------------------------------------+\r\n",
      "| Processes:                                                       GPU Memory |\r\n",
      "|  GPU       PID   Type   Process name                             Usage      |\r\n",
      "|=============================================================================|\r\n",
      "|  No running processes found                                                 |\r\n",
      "+-----------------------------------------------------------------------------+\r\n"
     ]
    }
   ],
   "source": [
    "!nvidia-smi"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "import torchvision\n",
    "import torchvision.transforms as transforms"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Files already downloaded and verified\n",
      "Files already downloaded and verified\n"
     ]
    }
   ],
   "source": [
    "transform = transforms.Compose(\n",
    "    [transforms.ToTensor(),\n",
    "     transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))])\n",
    "\n",
    "trainset = torchvision.datasets.CIFAR10(root='./data', train=True,\n",
    "                                        download=True, transform=transform)\n",
    "trainloader = torch.utils.data.DataLoader(trainset, batch_size=128,\n",
    "                                          shuffle=True, num_workers=2)\n",
    "\n",
    "testset = torchvision.datasets.CIFAR10(root='./data', train=False,\n",
    "                                       download=True, transform=transform)\n",
    "testloader = torch.utils.data.DataLoader(testset, batch_size=128,\n",
    "                                         shuffle=False, num_workers=2)\n",
    "\n",
    "classes = ('plane', 'car', 'bird', 'cat',\n",
    "           'deer', 'dog', 'frog', 'horse', 'ship', 'truck')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "\n",
    "class MBConv(nn.Module):\n",
    "    def __init__(self,in_channels,out_channels,kernel_size=3,stride=1,expand=3):\n",
    "        super(MBConv, self).__init__()\n",
    "        \n",
    "        self.in_channels = in_channels\n",
    "        self.out_channels = out_channels\n",
    "        self.kernel_size = kernel_size\n",
    "        self.stride = stride\n",
    "        self.expand = expand\n",
    "        \n",
    "        wide_channel = round(self.in_channels * self.expand)\n",
    "        \n",
    "        self.inv_conv = nn.Conv2d(self.in_channels,wide_channel,1,1,0,bias=False)\n",
    "        self.inv_bn = nn.BatchNorm2d(wide_channel)\n",
    "        self.inv_relu6 = nn.ReLU6()\n",
    "        \n",
    "        padding = kernel_size//2\n",
    "        \n",
    "        self.wide_conv = nn.Conv2d(wide_channel,wide_channel,kernel_size,stride,padding,groups=wide_channel,bias=False)\n",
    "        self.wide_bn = nn.BatchNorm2d(wide_channel)\n",
    "        self.wide_relu6 = nn.ReLU6()\n",
    "        \n",
    "        self.out_conv = nn.Conv2d(wide_channel,self.out_channels,1,1,0,bias=False)\n",
    "        self.out_bn = nn.BatchNorm2d(self.out_channels)\n",
    "        \n",
    "    def forward(self,x):\n",
    "        x = self.inv_conv(x)\n",
    "        x = self.inv_bn(x)\n",
    "        x = self.inv_relu6(x)\n",
    "        \n",
    "        x = self.wide_conv(x)\n",
    "        x = self.wide_bn(x)\n",
    "        x = self.wide_relu6(x)\n",
    "        \n",
    "        x = self.out_conv(x)\n",
    "        x = self.out_bn(x)\n",
    "        \n",
    "        return x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "class zero_layer(nn.Module):\n",
    "    def __init__(self,in_channels,out_channels,stride):\n",
    "        super(zero_layer,self).__init__()\n",
    "        \n",
    "        self.in_channels = in_channels\n",
    "        self.out_channels = out_channels\n",
    "        self.stride = stride\n",
    "        \n",
    "    def forward(self,x):\n",
    "        size = [d for d in x.size()]\n",
    "        size[1] = self.out_channels\n",
    "        if self.stride>1:\n",
    "            size[2] = size[2]//self.stride\n",
    "            size[3] = size[3]//self.stride\n",
    "\n",
    "        new_x = torch.zeros(size)\n",
    "        return new_x.to(x.device)\n",
    "\n",
    "# zero = zero_layer(3,6,2)\n",
    "# temp = torch.ones([4,3,224,224]).to(device)\n",
    "# zero(temp).size()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([4.2039e-45, 2.6709e-42, 6.8374e-40, 1.8206e-34, 2.5353e+30, 1.1210e-44])"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "torch.Tensor(6)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch.nn.functional as F\n",
    "import numpy as np\n",
    "\n",
    "class random_durum(nn.Module):\n",
    "    def __init__(self,in_channels,out_channels,stride,include_zero = False):\n",
    "        super(random_durum,self).__init__()\n",
    "        self.all_edges = nn.ModuleList()\n",
    "        self.num_edges = 6\n",
    "        \n",
    "        #(3,3),(3,5),(3,7),(6,3),(6,5),(6,7)\n",
    "        for expand in [3,6]:\n",
    "            for kernel_size in [3,5,7]:\n",
    "                self.all_edges.append(MBConv(in_channels,out_channels,kernel_size,stride,expand))\n",
    "        \n",
    "        if include_zero:\n",
    "            self.all_edges.append(zero_layer(in_channels,out_channels,stride))\n",
    "            self.num_edges += 1\n",
    "            \n",
    "        self.prob_edges = torch.Tensor(self.num_edges)\n",
    "            \n",
    "        self.active = [0]\n",
    "        self.inactive = [1]\n",
    "        \n",
    "    def select_active(self):\n",
    "#         probs = F.softmax(self.prob_edges)\n",
    "        edge = np.random.choice(self.num_edges,1)[0]\n",
    "        self.active[0] = edge\n",
    "    \n",
    "    def forward(self,x):\n",
    "        return self.all_edges[self.active[0]](x)\n",
    "\n",
    "# temp = torch.ones([4,3,224,224])\n",
    "# random_durum(3,6,1,False)(temp)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "\n",
    "\n",
    "class Net(nn.Module):\n",
    "    def __init__(self):\n",
    "        super(Net, self).__init__()\n",
    "        self.conv1 = random_durum(3, 6,1)\n",
    "        self.pool = nn.MaxPool2d(2, 2)\n",
    "        self.conv2 = random_durum(6, 16,1)\n",
    "        self.pool2 = nn.MaxPool2d(2, 2)\n",
    "        self.fc1 = nn.Linear(16 * 8 * 8, 120)\n",
    "        self.fc2 = nn.Linear(120, 84)\n",
    "        self.fc3 = nn.Linear(84, 10)\n",
    "        \n",
    "        self.edges = []\n",
    "        for m in self.modules():\n",
    "            if isinstance(m,random_durum):\n",
    "                self.edges.append(m)\n",
    "    \n",
    "    def select_active(self):\n",
    "#         active_list = []\n",
    "        for m in self.edges:\n",
    "            m.select_active()\n",
    "#             active_list.append(m.active[0])\n",
    "#         print(active_list)\n",
    "\n",
    "    def forward(self, x):\n",
    "        x = self.pool(self.conv1(x))\n",
    "        x = self.pool2(self.conv2(x))\n",
    "        x = x.view(-1, 16 * 8 * 8)\n",
    "        x = F.relu(self.fc1(x))\n",
    "        x = F.relu(self.fc2(x))\n",
    "        x = self.fc3(x)\n",
    "        return x\n",
    "\n",
    "\n",
    "net = Net()\n",
    "# temp = torch.ones([4,3,224,224])\n",
    "# net.select_active()\n",
    "# net.select_active()\n",
    "# net.select_active()\n",
    "# net.select_active()\n",
    "# net.select_active()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "cuda:0\n"
     ]
    }
   ],
   "source": [
    "device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\n",
    "\n",
    "# CUDA 기기가 존재한다면, 아래 코드가 CUDA 장치를 출력합니다:\n",
    "\n",
    "print(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Net(\n",
       "  (conv1): random_durum(\n",
       "    (all_edges): ModuleList(\n",
       "      (0): MBConv(\n",
       "        (inv_conv): Conv2d(3, 9, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "        (inv_bn): BatchNorm2d(9, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (inv_relu6): ReLU6()\n",
       "        (wide_conv): Conv2d(9, 9, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=9, bias=False)\n",
       "        (wide_bn): BatchNorm2d(9, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (wide_relu6): ReLU6()\n",
       "        (out_conv): Conv2d(9, 6, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "        (out_bn): BatchNorm2d(6, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      )\n",
       "      (1): MBConv(\n",
       "        (inv_conv): Conv2d(3, 9, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "        (inv_bn): BatchNorm2d(9, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (inv_relu6): ReLU6()\n",
       "        (wide_conv): Conv2d(9, 9, kernel_size=(5, 5), stride=(1, 1), padding=(2, 2), groups=9, bias=False)\n",
       "        (wide_bn): BatchNorm2d(9, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (wide_relu6): ReLU6()\n",
       "        (out_conv): Conv2d(9, 6, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "        (out_bn): BatchNorm2d(6, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      )\n",
       "      (2): MBConv(\n",
       "        (inv_conv): Conv2d(3, 9, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "        (inv_bn): BatchNorm2d(9, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (inv_relu6): ReLU6()\n",
       "        (wide_conv): Conv2d(9, 9, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=9, bias=False)\n",
       "        (wide_bn): BatchNorm2d(9, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (wide_relu6): ReLU6()\n",
       "        (out_conv): Conv2d(9, 6, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "        (out_bn): BatchNorm2d(6, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      )\n",
       "      (3): MBConv(\n",
       "        (inv_conv): Conv2d(3, 18, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "        (inv_bn): BatchNorm2d(18, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (inv_relu6): ReLU6()\n",
       "        (wide_conv): Conv2d(18, 18, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=18, bias=False)\n",
       "        (wide_bn): BatchNorm2d(18, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (wide_relu6): ReLU6()\n",
       "        (out_conv): Conv2d(18, 6, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "        (out_bn): BatchNorm2d(6, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      )\n",
       "      (4): MBConv(\n",
       "        (inv_conv): Conv2d(3, 18, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "        (inv_bn): BatchNorm2d(18, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (inv_relu6): ReLU6()\n",
       "        (wide_conv): Conv2d(18, 18, kernel_size=(5, 5), stride=(1, 1), padding=(2, 2), groups=18, bias=False)\n",
       "        (wide_bn): BatchNorm2d(18, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (wide_relu6): ReLU6()\n",
       "        (out_conv): Conv2d(18, 6, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "        (out_bn): BatchNorm2d(6, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      )\n",
       "      (5): MBConv(\n",
       "        (inv_conv): Conv2d(3, 18, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "        (inv_bn): BatchNorm2d(18, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (inv_relu6): ReLU6()\n",
       "        (wide_conv): Conv2d(18, 18, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=18, bias=False)\n",
       "        (wide_bn): BatchNorm2d(18, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (wide_relu6): ReLU6()\n",
       "        (out_conv): Conv2d(18, 6, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "        (out_bn): BatchNorm2d(6, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      )\n",
       "    )\n",
       "  )\n",
       "  (pool): MaxPool2d(kernel_size=2, stride=2, padding=0, dilation=1, ceil_mode=False)\n",
       "  (conv2): random_durum(\n",
       "    (all_edges): ModuleList(\n",
       "      (0): MBConv(\n",
       "        (inv_conv): Conv2d(6, 18, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "        (inv_bn): BatchNorm2d(18, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (inv_relu6): ReLU6()\n",
       "        (wide_conv): Conv2d(18, 18, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=18, bias=False)\n",
       "        (wide_bn): BatchNorm2d(18, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (wide_relu6): ReLU6()\n",
       "        (out_conv): Conv2d(18, 16, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "        (out_bn): BatchNorm2d(16, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      )\n",
       "      (1): MBConv(\n",
       "        (inv_conv): Conv2d(6, 18, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "        (inv_bn): BatchNorm2d(18, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (inv_relu6): ReLU6()\n",
       "        (wide_conv): Conv2d(18, 18, kernel_size=(5, 5), stride=(1, 1), padding=(2, 2), groups=18, bias=False)\n",
       "        (wide_bn): BatchNorm2d(18, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (wide_relu6): ReLU6()\n",
       "        (out_conv): Conv2d(18, 16, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "        (out_bn): BatchNorm2d(16, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      )\n",
       "      (2): MBConv(\n",
       "        (inv_conv): Conv2d(6, 18, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "        (inv_bn): BatchNorm2d(18, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (inv_relu6): ReLU6()\n",
       "        (wide_conv): Conv2d(18, 18, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=18, bias=False)\n",
       "        (wide_bn): BatchNorm2d(18, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (wide_relu6): ReLU6()\n",
       "        (out_conv): Conv2d(18, 16, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "        (out_bn): BatchNorm2d(16, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      )\n",
       "      (3): MBConv(\n",
       "        (inv_conv): Conv2d(6, 36, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "        (inv_bn): BatchNorm2d(36, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (inv_relu6): ReLU6()\n",
       "        (wide_conv): Conv2d(36, 36, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=36, bias=False)\n",
       "        (wide_bn): BatchNorm2d(36, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (wide_relu6): ReLU6()\n",
       "        (out_conv): Conv2d(36, 16, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "        (out_bn): BatchNorm2d(16, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      )\n",
       "      (4): MBConv(\n",
       "        (inv_conv): Conv2d(6, 36, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "        (inv_bn): BatchNorm2d(36, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (inv_relu6): ReLU6()\n",
       "        (wide_conv): Conv2d(36, 36, kernel_size=(5, 5), stride=(1, 1), padding=(2, 2), groups=36, bias=False)\n",
       "        (wide_bn): BatchNorm2d(36, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (wide_relu6): ReLU6()\n",
       "        (out_conv): Conv2d(36, 16, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "        (out_bn): BatchNorm2d(16, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      )\n",
       "      (5): MBConv(\n",
       "        (inv_conv): Conv2d(6, 36, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "        (inv_bn): BatchNorm2d(36, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (inv_relu6): ReLU6()\n",
       "        (wide_conv): Conv2d(36, 36, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=36, bias=False)\n",
       "        (wide_bn): BatchNorm2d(36, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (wide_relu6): ReLU6()\n",
       "        (out_conv): Conv2d(36, 16, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "        (out_bn): BatchNorm2d(16, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      )\n",
       "    )\n",
       "  )\n",
       "  (pool2): MaxPool2d(kernel_size=2, stride=2, padding=0, dilation=1, ceil_mode=False)\n",
       "  (fc1): Linear(in_features=1024, out_features=120, bias=True)\n",
       "  (fc2): Linear(in_features=120, out_features=84, bias=True)\n",
       "  (fc3): Linear(in_features=84, out_features=10, bias=True)\n",
       ")"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "net.to(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch.optim as optim\n",
    "\n",
    "criterion = nn.CrossEntropyLoss()\n",
    "optimizer = optim.SGD(net.parameters(), lr=0.01, momentum=0.9)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 391/391 [00:04<00:00, 81.67it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch: 0\n",
      "Accuracy of     0 : 42 %\n",
      "Accuracy of     1 : 76 %\n",
      "Accuracy of     2 : 30 %\n",
      "Accuracy of     3 : 15 %\n",
      "Accuracy of     4 : 15 %\n",
      "Accuracy of     5 : 33 %\n",
      "Accuracy of     6 : 41 %\n",
      "Accuracy of     7 : 43 %\n",
      "Accuracy of     8 : 54 %\n",
      "Accuracy of     9 : 27 %\n",
      "epoch: 1/10 | step: 10/391 | trn loss: 1.9824 | val loss: 1.7208\n",
      "Accuracy of the network on the test images: 37 %\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 391/391 [00:05<00:00, 73.12it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch: 1\n",
      "Accuracy of     0 : 59 %\n",
      "Accuracy of     1 : 50 %\n",
      "Accuracy of     2 : 42 %\n",
      "Accuracy of     3 : 24 %\n",
      "Accuracy of     4 : 19 %\n",
      "Accuracy of     5 : 38 %\n",
      "Accuracy of     6 : 45 %\n",
      "Accuracy of     7 : 38 %\n",
      "Accuracy of     8 : 65 %\n",
      "Accuracy of     9 : 48 %\n",
      "epoch: 2/10 | step: 10/391 | trn loss: 1.5933 | val loss: 1.5726\n",
      "Accuracy of the network on the test images: 43 %\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 391/391 [00:05<00:00, 75.03it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch: 2\n",
      "Accuracy of     0 : 52 %\n",
      "Accuracy of     1 : 59 %\n",
      "Accuracy of     2 : 32 %\n",
      "Accuracy of     3 : 13 %\n",
      "Accuracy of     4 : 51 %\n",
      "Accuracy of     5 : 44 %\n",
      "Accuracy of     6 : 59 %\n",
      "Accuracy of     7 : 55 %\n",
      "Accuracy of     8 : 66 %\n",
      "Accuracy of     9 : 57 %\n",
      "epoch: 3/10 | step: 10/391 | trn loss: 1.4621 | val loss: 1.3981\n",
      "Accuracy of the network on the test images: 49 %\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 391/391 [00:05<00:00, 75.68it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch: 3\n",
      "Accuracy of     0 : 47 %\n",
      "Accuracy of     1 : 34 %\n",
      "Accuracy of     2 : 43 %\n",
      "Accuracy of     3 : 41 %\n",
      "Accuracy of     4 : 34 %\n",
      "Accuracy of     5 : 45 %\n",
      "Accuracy of     6 : 61 %\n",
      "Accuracy of     7 : 63 %\n",
      "Accuracy of     8 : 72 %\n",
      "Accuracy of     9 : 63 %\n",
      "epoch: 4/10 | step: 10/391 | trn loss: 1.3670 | val loss: 1.3581\n",
      "Accuracy of the network on the test images: 50 %\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 391/391 [00:05<00:00, 67.48it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch: 4\n",
      "Accuracy of     0 : 60 %\n",
      "Accuracy of     1 : 68 %\n",
      "Accuracy of     2 : 31 %\n",
      "Accuracy of     3 : 27 %\n",
      "Accuracy of     4 : 34 %\n",
      "Accuracy of     5 : 59 %\n",
      "Accuracy of     6 : 82 %\n",
      "Accuracy of     7 : 51 %\n",
      "Accuracy of     8 : 69 %\n",
      "Accuracy of     9 : 51 %\n",
      "epoch: 5/10 | step: 10/391 | trn loss: 1.2889 | val loss: 1.3090\n",
      "Accuracy of the network on the test images: 53 %\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 391/391 [00:04<00:00, 83.22it/s] \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch: 5\n",
      "Accuracy of     0 : 72 %\n",
      "Accuracy of     1 : 44 %\n",
      "Accuracy of     2 : 41 %\n",
      "Accuracy of     3 : 31 %\n",
      "Accuracy of     4 : 34 %\n",
      "Accuracy of     5 : 41 %\n",
      "Accuracy of     6 : 70 %\n",
      "Accuracy of     7 : 56 %\n",
      "Accuracy of     8 : 81 %\n",
      "Accuracy of     9 : 59 %\n",
      "epoch: 6/10 | step: 10/391 | trn loss: 1.2332 | val loss: 1.3175\n",
      "Accuracy of the network on the test images: 53 %\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 391/391 [00:05<00:00, 70.58it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch: 6\n",
      "Accuracy of     0 : 66 %\n",
      "Accuracy of     1 : 67 %\n",
      "Accuracy of     2 : 31 %\n",
      "Accuracy of     3 : 43 %\n",
      "Accuracy of     4 : 43 %\n",
      "Accuracy of     5 : 52 %\n",
      "Accuracy of     6 : 64 %\n",
      "Accuracy of     7 : 72 %\n",
      "Accuracy of     8 : 63 %\n",
      "Accuracy of     9 : 72 %\n",
      "epoch: 7/10 | step: 10/391 | trn loss: 1.1754 | val loss: 1.1787\n",
      "Accuracy of the network on the test images: 57 %\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 391/391 [00:05<00:00, 70.93it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch: 7\n",
      "Accuracy of     0 : 58 %\n",
      "Accuracy of     1 : 69 %\n",
      "Accuracy of     2 : 38 %\n",
      "Accuracy of     3 : 41 %\n",
      "Accuracy of     4 : 55 %\n",
      "Accuracy of     5 : 39 %\n",
      "Accuracy of     6 : 68 %\n",
      "Accuracy of     7 : 69 %\n",
      "Accuracy of     8 : 76 %\n",
      "Accuracy of     9 : 67 %\n",
      "epoch: 8/10 | step: 10/391 | trn loss: 1.1322 | val loss: 1.1676\n",
      "Accuracy of the network on the test images: 58 %\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 391/391 [00:05<00:00, 77.68it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch: 8\n",
      "Accuracy of     0 : 66 %\n",
      "Accuracy of     1 : 51 %\n",
      "Accuracy of     2 : 38 %\n",
      "Accuracy of     3 : 51 %\n",
      "Accuracy of     4 : 54 %\n",
      "Accuracy of     5 : 46 %\n",
      "Accuracy of     6 : 66 %\n",
      "Accuracy of     7 : 71 %\n",
      "Accuracy of     8 : 73 %\n",
      "Accuracy of     9 : 63 %\n",
      "epoch: 9/10 | step: 10/391 | trn loss: 1.0893 | val loss: 1.1683\n",
      "Accuracy of the network on the test images: 58 %\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 391/391 [00:04<00:00, 81.08it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch: 9\n",
      "Accuracy of     0 : 60 %\n",
      "Accuracy of     1 : 79 %\n",
      "Accuracy of     2 : 51 %\n",
      "Accuracy of     3 : 39 %\n",
      "Accuracy of     4 : 53 %\n",
      "Accuracy of     5 : 32 %\n",
      "Accuracy of     6 : 75 %\n",
      "Accuracy of     7 : 57 %\n",
      "Accuracy of     8 : 77 %\n",
      "Accuracy of     9 : 72 %\n",
      "epoch: 10/10 | step: 10/391 | trn loss: 1.0527 | val loss: 1.1415\n",
      "Accuracy of the network on the test images: 59 %\n",
      "Finished Training\n"
     ]
    }
   ],
   "source": [
    "from tqdm import tqdm\n",
    "\n",
    "num_classes = 10\n",
    "trn_loss_list = []\n",
    "val_loss_list = []\n",
    "acc_list = []\n",
    "acc_list_aug = []\n",
    "num_batches=len(trainloader)\n",
    "total_epoch = 10\n",
    "\n",
    "for epoch in range(total_epoch):   # 데이터셋을 수차례 반복합니다.\n",
    "    net.train()\n",
    "    running_loss = 0.0\n",
    "    for i, data in tqdm(enumerate(trainloader, 0),total=len(trainloader)):\n",
    "        net.select_active()\n",
    "        # [inputs, labels]의 목록인 data로부터 입력을 받은 후;\n",
    "        inputs, labels = data[0].to(device), data[1].to(device)\n",
    "\n",
    "        # 변화도(Gradient) 매개변수를 0으로 만들고\n",
    "        optimizer.zero_grad()\n",
    "\n",
    "        # 순전파 + 역전파 + 최적화를 한 후\n",
    "        outputs = net(inputs)\n",
    "        loss = criterion(outputs, labels)\n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "        running_loss += loss.item()\n",
    "        \n",
    "    net.eval()\n",
    "    with torch.no_grad(): # very very very very important!!!\n",
    "        val_loss = 0.0\n",
    "        class_correct = list(0. for i in range(num_classes))\n",
    "        class_total = list(0. for i in range(num_classes))\n",
    "        correct = 0\n",
    "        total = 0\n",
    "        for j,val in enumerate(testloader):\n",
    "            v1 = val[0].to(device)\n",
    "            val_labels = val[1].to(device)\n",
    "            val_output = net(v1)\n",
    "            v_loss = criterion(val_output, val_labels)\n",
    "            val_loss += v_loss\n",
    "            _, predicted = torch.max(val_output, 1)\n",
    "            c = (predicted == val_labels).squeeze()\n",
    "            total += val_labels.size(0)\n",
    "            correct += (predicted == val_labels).sum().item()\n",
    "            for i in range(len(val_labels)):\n",
    "                val_label = val_labels[i]\n",
    "                class_correct[val_label] += c[i].item()\n",
    "                class_total[val_label] += 1\n",
    "\n",
    "        print(\"epoch:\",str(epoch))\n",
    "        temp_acc = []\n",
    "        for i in range(num_classes):\n",
    "            if class_total[i]==0:\n",
    "                print('class_total = 0',class_correct,class_total)\n",
    "            else:\n",
    "                print('Accuracy of %5s : %2d %%' % (i, 100 * class_correct[i] / class_total[i]))\n",
    "                temp_acc.append(100 * class_correct[i] / class_total[i])\n",
    "        acc_list.append(temp_acc)\n",
    "\n",
    "    print(\"epoch: {}/{} | step: {}/{} | trn loss: {:.4f} | val loss: {:.4f}\".format(\n",
    "        epoch+1, total_epoch, i+1, num_batches, running_loss / len(trainloader), val_loss / len(testloader)\n",
    "    ))        \n",
    "    print('Accuracy of the network on the test images: %d %%' % (100 * correct / total))\n",
    "\n",
    "    trn_loss_list.append(running_loss/1875)\n",
    "    val_loss_list.append(val_loss/len(testloader))\n",
    "    running_loss = 0.0\n",
    "\n",
    "print('Finished Training')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}