Code_Controller_(1) (1).ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.7.3"
    },
    "colab": {
      "name": "Code_Controller_(1).ipynb",
      "provenance": [],
      "collapsed_sections": []
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "code",
      "metadata": {
        "id": "YFHHFQzv_qJ0",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from __future__ import print_function\n",
        "import torch\n",
        "import torch.nn as nn\n",
        "import torch.nn.functional as F\n",
        "import torch.optim as optim\n",
        "from torchvision import datasets, transforms\n",
        "from tqdm import tqdm\n",
        "from torch.optim.lr_scheduler import StepLR\n",
        "from torch.optim.lr_scheduler import OneCycleLR\n",
        "import matplotlib.pyplot as plt\n",
        "import numpy as np\n",
        "import torchvision\n",
        "import torchsummary\n",
        "from torchsummary import summary"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "N-fkSGTa_qJ9",
        "colab_type": "text"
      },
      "source": [
        "## Importing all the files created "
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8jjHeSdO_qJ_",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from Getting_the_Data import get_and_transform_the_data\n",
        "from Displaying_the_images import display_my_images\n",
        "from Model_and_summary import ResNet18, disp_summary\n",
        "from train_test import train , test"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "M-CMa8kc_qKF",
        "colab_type": "text"
      },
      "source": [
        "## First step is to download and transform the data which here is CIFAR-10"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "FN0DK5Cx_qKH",
        "colab_type": "code",
        "outputId": "551773f2-da7b-46d5-eb56-82bc58f33852",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 67
        }
      },
      "source": [
        "train_set,test_set,train_loader,test_loader,classes = get_and_transform_the_data()"
      ],
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "CUDA Available? True\n",
            "Files already downloaded and verified\n",
            "Files already downloaded and verified\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "8Qd5l1Lm_qKW",
        "colab_type": "text"
      },
      "source": [
        "## Moving the model for GPU usage"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "iNZbU4Jn_qKY",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "use_cuda= torch.cuda.is_available()\n",
        "device=torch.device('cuda' if use_cuda else 'cpu')\n",
        "model=ResNet18().to(device)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "tgdDWmiN_qKd",
        "colab_type": "code",
        "outputId": "1c322df6-1694-427c-f833-3adce9b96d68",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        }
      },
      "source": [
        "disp_summary(model)"
      ],
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "----------------------------------------------------------------\n",
            "        Layer (type)               Output Shape         Param #\n",
            "================================================================\n",
            "            Conv2d-1           [-1, 64, 32, 32]           1,728\n",
            "       BatchNorm2d-2           [-1, 64, 32, 32]             128\n",
            "            Conv2d-3           [-1, 64, 32, 32]          36,864\n",
            "       BatchNorm2d-4           [-1, 64, 32, 32]             128\n",
            "            Conv2d-5           [-1, 64, 32, 32]          36,864\n",
            "       BatchNorm2d-6           [-1, 64, 32, 32]             128\n",
            "        BasicBlock-7           [-1, 64, 32, 32]               0\n",
            "            Conv2d-8           [-1, 64, 32, 32]          36,864\n",
            "       BatchNorm2d-9           [-1, 64, 32, 32]             128\n",
            "           Conv2d-10           [-1, 64, 32, 32]          36,864\n",
            "      BatchNorm2d-11           [-1, 64, 32, 32]             128\n",
            "       BasicBlock-12           [-1, 64, 32, 32]               0\n",
            "           Conv2d-13          [-1, 128, 16, 16]          73,728\n",
            "      BatchNorm2d-14          [-1, 128, 16, 16]             256\n",
            "           Conv2d-15          [-1, 128, 16, 16]         147,456\n",
            "      BatchNorm2d-16          [-1, 128, 16, 16]             256\n",
            "           Conv2d-17          [-1, 128, 16, 16]           8,192\n",
            "      BatchNorm2d-18          [-1, 128, 16, 16]             256\n",
            "       BasicBlock-19          [-1, 128, 16, 16]               0\n",
            "           Conv2d-20          [-1, 128, 16, 16]         147,456\n",
            "      BatchNorm2d-21          [-1, 128, 16, 16]             256\n",
            "           Conv2d-22          [-1, 128, 16, 16]         147,456\n",
            "      BatchNorm2d-23          [-1, 128, 16, 16]             256\n",
            "       BasicBlock-24          [-1, 128, 16, 16]               0\n",
            "           Conv2d-25            [-1, 256, 8, 8]         294,912\n",
            "      BatchNorm2d-26            [-1, 256, 8, 8]             512\n",
            "           Conv2d-27            [-1, 256, 8, 8]         589,824\n",
            "      BatchNorm2d-28            [-1, 256, 8, 8]             512\n",
            "           Conv2d-29            [-1, 256, 8, 8]          32,768\n",
            "      BatchNorm2d-30            [-1, 256, 8, 8]             512\n",
            "       BasicBlock-31            [-1, 256, 8, 8]               0\n",
            "           Conv2d-32            [-1, 256, 8, 8]         589,824\n",
            "      BatchNorm2d-33            [-1, 256, 8, 8]             512\n",
            "           Conv2d-34            [-1, 256, 8, 8]         589,824\n",
            "      BatchNorm2d-35            [-1, 256, 8, 8]             512\n",
            "       BasicBlock-36            [-1, 256, 8, 8]               0\n",
            "           Conv2d-37            [-1, 512, 4, 4]       1,179,648\n",
            "      BatchNorm2d-38            [-1, 512, 4, 4]           1,024\n",
            "           Conv2d-39            [-1, 512, 4, 4]       2,359,296\n",
            "      BatchNorm2d-40            [-1, 512, 4, 4]           1,024\n",
            "           Conv2d-41            [-1, 512, 4, 4]         131,072\n",
            "      BatchNorm2d-42            [-1, 512, 4, 4]           1,024\n",
            "       BasicBlock-43            [-1, 512, 4, 4]               0\n",
            "           Conv2d-44            [-1, 512, 4, 4]       2,359,296\n",
            "      BatchNorm2d-45            [-1, 512, 4, 4]           1,024\n",
            "           Conv2d-46            [-1, 512, 4, 4]       2,359,296\n",
            "      BatchNorm2d-47            [-1, 512, 4, 4]           1,024\n",
            "       BasicBlock-48            [-1, 512, 4, 4]               0\n",
            "           Linear-49                   [-1, 10]           5,130\n",
            "================================================================\n",
            "Total params: 11,173,962\n",
            "Trainable params: 11,173,962\n",
            "Non-trainable params: 0\n",
            "----------------------------------------------------------------\n",
            "Input size (MB): 0.01\n",
            "Forward/backward pass size (MB): 11.25\n",
            "Params size (MB): 42.63\n",
            "Estimated Total Size (MB): 53.89\n",
            "----------------------------------------------------------------\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "eYmRk12R_qKj",
        "colab_type": "code",
        "outputId": "e2385419-c9ec-4a60-e9dd-96da0d1b750c",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        }
      },
      "source": [
        "device= 'cuda' if torch.cuda.is_available() else 'cpu'\n",
        "\n",
        "optimizer = optim.RMSprop(model.parameters(), lr=0.01)\n",
        "scheduler = OneCycleLR(optimizer,max_lr=0.1,total_steps=40)\n",
        "\n",
        "for epoch in range(1, 41):\n",
        "    print(f'Epoch: {epoch} Learning_Rate {scheduler.get_lr()}')\n",
        "    train(model, device, train_loader, optimizer, epoch)\n",
        "    test(model, device, test_loader)\n",
        "    scheduler.step()"
      ],
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "\r  0%|          | 0/391 [00:00<?, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Epoch: 1 Learning_Rate [0.0040000000000000036]\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Loss=1.761474370956421 Batch_id=390 Accuracy=26.69: 100%|██████████| 391/391 [01:02<00:00,  6.96it/s]\n",
            "  0%|          | 0/391 [00:00<?, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Test set: Average loss: 0.0137, Accuracy: 3484/10000 (34.84%)\n",
            "\n",
            "Epoch: 2 Learning_Rate [0.005944337266504118]\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Loss=1.4846844673156738 Batch_id=390 Accuracy=38.04: 100%|██████████| 391/391 [01:01<00:00,  7.11it/s]\n",
            "  0%|          | 0/391 [00:00<?, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Test set: Average loss: 0.0129, Accuracy: 4017/10000 (40.17%)\n",
            "\n",
            "Epoch: 3 Learning_Rate [0.011619830424103306]\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Loss=1.5220177173614502 Batch_id=390 Accuracy=44.88: 100%|██████████| 391/391 [01:01<00:00,  7.28it/s]\n",
            "  0%|          | 0/391 [00:00<?, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Test set: Average loss: 0.0115, Accuracy: 4891/10000 (48.91%)\n",
            "\n",
            "Epoch: 4 Learning_Rate [0.020566684770626315]\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Loss=1.357172966003418 Batch_id=390 Accuracy=50.48: 100%|██████████| 391/391 [01:00<00:00,  6.51it/s]\n",
            "  0%|          | 0/391 [00:00<?, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Test set: Average loss: 0.0123, Accuracy: 4694/10000 (46.94%)\n",
            "\n",
            "Epoch: 5 Learning_Rate [0.03206007937590945]\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Loss=1.0350217819213867 Batch_id=390 Accuracy=54.42: 100%|██████████| 391/391 [00:59<00:00,  6.56it/s]\n",
            "  0%|          | 0/391 [00:00<?, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Test set: Average loss: 0.0170, Accuracy: 4515/10000 (45.15%)\n",
            "\n",
            "Epoch: 6 Learning_Rate [0.04516888776288231]\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Loss=0.9740197062492371 Batch_id=390 Accuracy=59.03: 100%|██████████| 391/391 [00:59<00:00,  6.58it/s]\n",
            "  0%|          | 0/391 [00:00<?, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Test set: Average loss: 0.0111, Accuracy: 5689/10000 (56.89%)\n",
            "\n",
            "Epoch: 7 Learning_Rate [0.058831112237117685]\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Loss=0.9719946980476379 Batch_id=390 Accuracy=64.70: 100%|██████████| 391/391 [00:59<00:00,  7.34it/s]\n",
            "  0%|          | 0/391 [00:00<?, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Test set: Average loss: 0.0078, Accuracy: 6582/10000 (65.82%)\n",
            "\n",
            "Epoch: 8 Learning_Rate [0.07193992062409055]\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Loss=1.978492021560669 Batch_id=390 Accuracy=36.47: 100%|██████████| 391/391 [00:59<00:00,  6.59it/s]\n",
            "  0%|          | 0/391 [00:00<?, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Test set: Average loss: 0.0157, Accuracy: 1916/10000 (19.16%)\n",
            "\n",
            "Epoch: 9 Learning_Rate [0.08343331522937368]\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Loss=1.709959626197815 Batch_id=390 Accuracy=22.48: 100%|██████████| 391/391 [00:59<00:00,  6.59it/s]\n",
            "  0%|          | 0/391 [00:00<?, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Test set: Average loss: 0.0133, Accuracy: 3543/10000 (35.43%)\n",
            "\n",
            "Epoch: 10 Learning_Rate [0.0923801695758967]\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Loss=1.245566725730896 Batch_id=390 Accuracy=39.42: 100%|██████████| 391/391 [00:59<00:00,  6.61it/s]\n",
            "  0%|          | 0/391 [00:00<?, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Test set: Average loss: 0.0115, Accuracy: 4859/10000 (48.59%)\n",
            "\n",
            "Epoch: 11 Learning_Rate [0.09805566273349588]\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Loss=0.9965742826461792 Batch_id=390 Accuracy=57.24: 100%|██████████| 391/391 [00:59<00:00,  6.60it/s]\n",
            "  0%|          | 0/391 [00:00<?, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Test set: Average loss: 0.0097, Accuracy: 5741/10000 (57.41%)\n",
            "\n",
            "Epoch: 12 Learning_Rate [0.1]\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Loss=1.6655919551849365 Batch_id=390 Accuracy=62.31: 100%|██████████| 391/391 [00:59<00:00,  6.62it/s]\n",
            "  0%|          | 0/391 [00:00<?, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Test set: Average loss: 0.0115, Accuracy: 4894/10000 (48.94%)\n",
            "\n",
            "Epoch: 13 Learning_Rate [0.09968561175222017]\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Loss=0.9053387641906738 Batch_id=390 Accuracy=68.07: 100%|██████████| 391/391 [00:59<00:00,  7.35it/s]\n",
            "  0%|          | 0/391 [00:00<?, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Test set: Average loss: 0.0090, Accuracy: 6188/10000 (61.88%)\n",
            "\n",
            "Epoch: 14 Learning_Rate [0.09874640062350874]\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Loss=1.1221002340316772 Batch_id=390 Accuracy=68.60: 100%|██████████| 391/391 [00:58<00:00,  7.43it/s]\n",
            "  0%|          | 0/391 [00:00<?, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Test set: Average loss: 0.0072, Accuracy: 6878/10000 (68.78%)\n",
            "\n",
            "Epoch: 15 Learning_Rate [0.09719417773875232]\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Loss=0.5830186605453491 Batch_id=390 Accuracy=72.51: 100%|██████████| 391/391 [00:59<00:00,  6.60it/s]\n",
            "  0%|          | 0/391 [00:00<?, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Test set: Average loss: 0.0066, Accuracy: 7231/10000 (72.31%)\n",
            "\n",
            "Epoch: 16 Learning_Rate [0.09504846320134738]\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Loss=0.8894365429878235 Batch_id=390 Accuracy=74.35: 100%|██████████| 391/391 [00:59<00:00,  6.61it/s]\n",
            "  0%|          | 0/391 [00:00<?, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Test set: Average loss: 0.0065, Accuracy: 7348/10000 (73.48%)\n",
            "\n",
            "Epoch: 17 Learning_Rate [0.09233624061657436]\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Loss=0.7495899200439453 Batch_id=390 Accuracy=76.26: 100%|██████████| 391/391 [00:59<00:00,  6.62it/s]\n",
            "  0%|          | 0/391 [00:00<?, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Test set: Average loss: 0.0061, Accuracy: 7478/10000 (74.78%)\n",
            "\n",
            "Epoch: 18 Learning_Rate [0.089091617757105]\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Loss=0.5970357060432434 Batch_id=390 Accuracy=77.40: 100%|██████████| 391/391 [00:58<00:00,  6.63it/s]\n",
            "  0%|          | 0/391 [00:00<?, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Test set: Average loss: 0.0061, Accuracy: 7539/10000 (75.39%)\n",
            "\n",
            "Epoch: 19 Learning_Rate [0.08535539763797113]\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Loss=0.6495188474655151 Batch_id=390 Accuracy=77.70: 100%|██████████| 391/391 [00:59<00:00,  6.61it/s]\n",
            "  0%|          | 0/391 [00:00<?, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Test set: Average loss: 0.0060, Accuracy: 7472/10000 (74.72%)\n",
            "\n",
            "Epoch: 20 Learning_Rate [0.0811745653949763]\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Loss=0.7715296149253845 Batch_id=390 Accuracy=79.69: 100%|██████████| 391/391 [00:58<00:00,  6.64it/s]\n",
            "  0%|          | 0/391 [00:00<?, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Test set: Average loss: 0.0065, Accuracy: 7434/10000 (74.34%)\n",
            "\n",
            "Epoch: 21 Learning_Rate [0.07660169741935154]\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Loss=0.5066594481468201 Batch_id=390 Accuracy=80.13: 100%|██████████| 391/391 [00:59<00:00,  6.61it/s]\n",
            "  0%|          | 0/391 [00:00<?, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Test set: Average loss: 0.0057, Accuracy: 7795/10000 (77.95%)\n",
            "\n",
            "Epoch: 22 Learning_Rate [0.07169430017913009]\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Loss=0.8729343414306641 Batch_id=390 Accuracy=80.95: 100%|██████████| 391/391 [00:59<00:00,  6.62it/s]\n",
            "  0%|          | 0/391 [00:00<?, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Test set: Average loss: 0.0054, Accuracy: 7852/10000 (78.52%)\n",
            "\n",
            "Epoch: 23 Learning_Rate [0.06651408704194597]\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Loss=0.48248404264450073 Batch_id=390 Accuracy=81.97: 100%|██████████| 391/391 [00:59<00:00,  6.61it/s]\n",
            "  0%|          | 0/391 [00:00<?, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Test set: Average loss: 0.0051, Accuracy: 8006/10000 (80.06%)\n",
            "\n",
            "Epoch: 24 Learning_Rate [0.06112620219362893]\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Loss=0.6306929588317871 Batch_id=390 Accuracy=82.80: 100%|██████████| 391/391 [00:58<00:00,  7.40it/s]\n",
            "  0%|          | 0/391 [00:00<?, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Test set: Average loss: 0.0054, Accuracy: 7931/10000 (79.31%)\n",
            "\n",
            "Epoch: 25 Learning_Rate [0.055598401412270175]\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Loss=0.5138142108917236 Batch_id=390 Accuracy=83.26: 100%|██████████| 391/391 [00:59<00:00,  6.56it/s]\n",
            "  0%|          | 0/391 [00:00<?, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Test set: Average loss: 0.0050, Accuracy: 8057/10000 (80.57%)\n",
            "\n",
            "Epoch: 26 Learning_Rate [0.0500002]\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Loss=0.5265241861343384 Batch_id=390 Accuracy=83.90: 100%|██████████| 391/391 [00:59<00:00,  6.60it/s]\n",
            "  0%|          | 0/391 [00:00<?, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Test set: Average loss: 0.0048, Accuracy: 8088/10000 (80.88%)\n",
            "\n",
            "Epoch: 27 Learning_Rate [0.04440199858772983]\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Loss=0.4352378845214844 Batch_id=390 Accuracy=84.43: 100%|██████████| 391/391 [00:59<00:00,  6.57it/s]\n",
            "  0%|          | 0/391 [00:00<?, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Test set: Average loss: 0.0044, Accuracy: 8261/10000 (82.61%)\n",
            "\n",
            "Epoch: 28 Learning_Rate [0.03887419780637107]\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Loss=0.7814135551452637 Batch_id=390 Accuracy=82.02: 100%|██████████| 391/391 [00:59<00:00,  6.57it/s]\n",
            "  0%|          | 0/391 [00:00<?, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Test set: Average loss: 0.0062, Accuracy: 7570/10000 (75.70%)\n",
            "\n",
            "Epoch: 29 Learning_Rate [0.03348631295805405]\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Loss=0.5283379554748535 Batch_id=390 Accuracy=85.21: 100%|██████████| 391/391 [00:59<00:00,  6.62it/s]\n",
            "  0%|          | 0/391 [00:00<?, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Test set: Average loss: 0.0044, Accuracy: 8264/10000 (82.64%)\n",
            "\n",
            "Epoch: 30 Learning_Rate [0.028306099820869922]\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Loss=0.3929148018360138 Batch_id=390 Accuracy=86.40: 100%|██████████| 391/391 [00:58<00:00,  6.65it/s]\n",
            "  0%|          | 0/391 [00:00<?, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Test set: Average loss: 0.0042, Accuracy: 8389/10000 (83.89%)\n",
            "\n",
            "Epoch: 31 Learning_Rate [0.023398702580648485]\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Loss=0.6126952171325684 Batch_id=390 Accuracy=86.89: 100%|██████████| 391/391 [00:58<00:00,  6.64it/s]\n",
            "  0%|          | 0/391 [00:00<?, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Test set: Average loss: 0.0046, Accuracy: 8246/10000 (82.46%)\n",
            "\n",
            "Epoch: 32 Learning_Rate [0.0188258346050237]\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Loss=0.15967121720314026 Batch_id=390 Accuracy=87.85: 100%|██████████| 391/391 [00:58<00:00,  6.66it/s]\n",
            "  0%|          | 0/391 [00:00<?, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Test set: Average loss: 0.0040, Accuracy: 8456/10000 (84.56%)\n",
            "\n",
            "Epoch: 33 Learning_Rate [0.014645002362028864]\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Loss=0.3884727358818054 Batch_id=390 Accuracy=88.39: 100%|██████████| 391/391 [00:58<00:00,  6.65it/s]\n",
            "  0%|          | 0/391 [00:00<?, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Test set: Average loss: 0.0039, Accuracy: 8449/10000 (84.49%)\n",
            "\n",
            "Epoch: 34 Learning_Rate [0.010908782242895003]\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Loss=0.2804277837276459 Batch_id=390 Accuracy=89.33: 100%|██████████| 391/391 [00:59<00:00,  6.63it/s]\n",
            "  0%|          | 0/391 [00:00<?, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Test set: Average loss: 0.0038, Accuracy: 8528/10000 (85.28%)\n",
            "\n",
            "Epoch: 35 Learning_Rate [0.0076641593834256404]\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Loss=0.3510071635246277 Batch_id=390 Accuracy=89.56: 100%|██████████| 391/391 [00:58<00:00,  6.65it/s]\n",
            "  0%|          | 0/391 [00:00<?, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Test set: Average loss: 0.0037, Accuracy: 8523/10000 (85.23%)\n",
            "\n",
            "Epoch: 36 Learning_Rate [0.004951936798652629]\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Loss=0.2674649655818939 Batch_id=390 Accuracy=90.27: 100%|██████████| 391/391 [00:59<00:00,  6.58it/s]\n",
            "  0%|          | 0/391 [00:00<?, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Test set: Average loss: 0.0036, Accuracy: 8614/10000 (86.14%)\n",
            "\n",
            "Epoch: 37 Learning_Rate [0.002806222261247683]\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Loss=0.19625303149223328 Batch_id=390 Accuracy=90.96: 100%|██████████| 391/391 [00:59<00:00,  6.60it/s]\n",
            "  0%|          | 0/391 [00:00<?, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Test set: Average loss: 0.0035, Accuracy: 8626/10000 (86.26%)\n",
            "\n",
            "Epoch: 38 Learning_Rate [0.0012539993764912555]\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Loss=0.31562408804893494 Batch_id=390 Accuracy=91.21: 100%|██████████| 391/391 [00:59<00:00,  6.57it/s]\n",
            "  0%|          | 0/391 [00:00<?, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Test set: Average loss: 0.0035, Accuracy: 8621/10000 (86.21%)\n",
            "\n",
            "Epoch: 39 Learning_Rate [0.0003147882477798485]\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Loss=0.15003962814807892 Batch_id=390 Accuracy=91.34: 100%|██████████| 391/391 [00:59<00:00,  6.58it/s]\n",
            "  0%|          | 0/391 [00:00<?, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Test set: Average loss: 0.0034, Accuracy: 8643/10000 (86.43%)\n",
            "\n",
            "Epoch: 40 Learning_Rate [4e-07]\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Loss=0.18976888060569763 Batch_id=390 Accuracy=91.46: 100%|██████████| 391/391 [00:59<00:00,  6.60it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Test set: Average loss: 0.0035, Accuracy: 8631/10000 (86.31%)\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "qQiieTU8NnLT",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        ""
      ],
      "execution_count": 0,
      "outputs": []
    }
  ]
}