train_efficientdet.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# setup dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# import stuff\n",
    "import os\n",
    "import numpy as np\n",
    "import time\n",
    "import pandas as pd\n",
    "\n",
    "import torch\n",
    "import torch.utils.data as data\n",
    "from itertools import product as product\n",
    "\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.init as init\n",
    "import torch.nn.functional as F\n",
    "from torch.autograd import Function\n",
    "from utils.to_fp16 import network_to_half"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# import dataset\n",
    "from utils.dataset import VOCDataset, DatasetTransform, make_datapath_list, Anno_xml2list, od_collate_fn"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "## meta settings\n",
    "\n",
    "# select from efficientnet backbone or resnet backbone\n",
    "backbone = \"efficientnet-b2\"\n",
    "scale = 1\n",
    "# scale==1: resolution 300\n",
    "# scale==2: resolution 600\n",
    "useBiFPN = True\n",
    "HALF = False # enable FP16\n",
    "DATASET = \"VOC\"\n",
    "retina = False # for trying retinanets"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## make data.Dataset for training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "trainlist:  16551\n",
      "vallist:  4952\n"
     ]
    }
   ],
   "source": [
    "if not DATASET == \"COCO\":\n",
    "    # load files\n",
    "    # set your VOCdevkit path here.\n",
    "    vocpath = \"../VOCdevkit/VOC2007\"\n",
    "    train_img_list, train_anno_list, val_img_list, val_anno_list = make_datapath_list(vocpath)\n",
    "\n",
    "    vocpath = \"../VOCdevkit/VOC2012\"\n",
    "    train_img_list2, train_anno_list2, _, _ = make_datapath_list(vocpath)\n",
    "\n",
    "    train_img_list.extend(train_img_list2)\n",
    "    train_anno_list.extend(train_anno_list2)\n",
    "\n",
    "    print(\"trainlist: \", len(train_img_list))\n",
    "    print(\"vallist: \", len(val_img_list))\n",
    "\n",
    "    # make Dataset\n",
    "    voc_classes = ['aeroplane', 'bicycle', 'bird', 'boat',\n",
    "                   'bottle', 'bus', 'car', 'cat', 'chair',\n",
    "                   'cow', 'diningtable', 'dog', 'horse',\n",
    "                   'motorbike', 'person', 'pottedplant',\n",
    "                   'sheep', 'sofa', 'train', 'tvmonitor']\n",
    "\n",
    "    color_mean = (104, 117, 123)  # (BGR)の色の平均値\n",
    "    if scale == 1:\n",
    "        input_size = 300  # 画像のinputサイズを300×300にする\n",
    "    else:\n",
    "        input_size = 512\n",
    "\n",
    "    ## DatasetTransformを適応\n",
    "    transform = DatasetTransform(input_size, color_mean)\n",
    "    transform_anno = Anno_xml2list(voc_classes)\n",
    "\n",
    "    # Dataloaderに入れるデータセットファイル。\n",
    "    # ゲットで叩くと画像とGTを前処理して出力してくれる。\n",
    "    train_dataset = VOCDataset(train_img_list, train_anno_list, phase = \"train\", transform=transform, transform_anno = transform_anno)\n",
    "    val_dataset = VOCDataset(val_img_list, val_anno_list, phase=\"val\", transform=DatasetTransform(\n",
    "        input_size, color_mean), transform_anno=Anno_xml2list(voc_classes))\n",
    "\n",
    "else:\n",
    "    from dataset.coco import COCODetection\n",
    "    import torch.utils.data as data\n",
    "    from utils.dataset import VOCDataset, COCODatasetTransform, make_datapath_list, Anno_xml2list, od_collate_fn\n",
    "\n",
    "    color_mean = (104, 117, 123)  # (BGR)の色の平均値\n",
    "    if scale == 1:\n",
    "        input_size = 300  # 画像のinputサイズを300×300にする\n",
    "    else:\n",
    "        input_size = 512\n",
    "\n",
    "    ## DatasetTransformを適応\n",
    "    transform = COCODatasetTransform(input_size, color_mean)\n",
    "    train_dataset = COCODetection(\"../data/coco/\", image_set=\"train2014\", phase=\"train\", transform=transform)\n",
    "    val_dataset = COCODetection(\"../data/coco/\", image_set=\"val2014\", phase=\"val\", transform=transform)\n",
    "\n",
    "batch_size = int(32/scale)\n",
    "\n",
    "train_dataloader = data.DataLoader(\n",
    "    train_dataset, batch_size=batch_size, shuffle=True, collate_fn=od_collate_fn, num_workers=8)\n",
    "\n",
    "val_dataloader = data.DataLoader(\n",
    "    val_dataset, batch_size=batch_size, shuffle=False, collate_fn=od_collate_fn, num_workers=8)\n",
    "\n",
    "# 辞書型変数にまとめる\n",
    "dataloaders_dict = {\"train\": train_dataloader, \"val\": val_dataloader}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([32, 3, 300, 300])\n",
      "32\n",
      "torch.Size([1, 5])\n"
     ]
    }
   ],
   "source": [
    "# 動作の確認\n",
    "batch_iterator = iter(dataloaders_dict[\"val\"])  # イタレータに変換\n",
    "images, targets = next(batch_iterator)  # 1番目の要素を取り出す\n",
    "print(images.size())  # torch.Size([4, 3, 300, 300])\n",
    "print(len(targets))\n",
    "print(targets[1].shape)  # ミニバッチのサイズのリスト、各要素は[n, 5]、nは物体数"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# define EfficientDet model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "from utils.efficientdet import EfficientDet"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loaded pretrained weights for efficientnet-b2\n",
      "208\n",
      "use BiFPN\n"
     ]
    }
   ],
   "source": [
    "if not DATASET == \"COCO\":\n",
    "    num_class = 21\n",
    "else:\n",
    "    num_class = 81\n",
    "\n",
    "if scale==1:\n",
    "    ssd_cfg = {\n",
    "        'num_classes': num_class,  # 背景クラスを含めた合計クラス数\n",
    "        'input_size': 300*scale,  # 画像の入力サイズ\n",
    "        'bbox_aspect_num': [4, 6, 6, 6, 4, 4],  # 出力するDBoxのアスペクト比の種類\n",
    "        'feature_maps': [37, 18, 9, 5, 3, 1],  # 各sourceの画像サイズ\n",
    "        'steps': [8, 16, 32, 64, 100, 300],  # DBOXの大きさを決める\n",
    "        'min_sizes': [30, 60, 111, 162, 213, 264],  # DBOXの大きさを決める\n",
    "        'max_sizes': [60, 111, 162, 213, 264, 315],  # DBOXの大きさを決める\n",
    "        'aspect_ratios': [[2], [2, 3], [2, 3], [2, 3], [2], [2]],\n",
    "    }\n",
    "elif scale==2:\n",
    "    ssd_cfg = {\n",
    "        'num_classes': num_class,  # 背景クラスを含めた合計クラス数\n",
    "        'input_size': 512,  # 画像の入力サイズ\n",
    "        'bbox_aspect_num': [4, 6, 6, 6, 4, 4],  # 出力するDBoxのアスペクト比の種類\n",
    "        'feature_maps': [64, 32, 16, 8, 4, 2],  # 各sourceの画像サイズ\n",
    "        'steps': [8, 16, 32, 64, 100, 300],  # DBOXの大きさを決める\n",
    "        'min_sizes': [30, 60, 111, 162, 213, 264]*scale,  # DBOXの大きさを決める\n",
    "        'max_sizes': [60, 111, 162, 213, 264, 315]*scale,  # DBOXの大きさを決める\n",
    "        'aspect_ratios': [[2], [2, 3], [2, 3], [2, 3], [2], [2]],\n",
    "    }\n",
    "\n",
    "# test if net works\n",
    "net = EfficientDet(phase=\"train\", cfg=ssd_cfg, verbose=True, backbone=backbone, useBiFPN=useBiFPN)\n",
    "#out = net(torch.rand([1,3,input_size,input_size]))\n",
    "#print(out[0].size())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loaded pretrained weights for efficientnet-b2\n",
      "208\n",
      "use BiFPN\n",
      "using: cuda:0\n",
      "set weights!\n"
     ]
    }
   ],
   "source": [
    "net = EfficientDet(phase=\"train\", cfg=ssd_cfg, verbose=False, backbone=backbone, useBiFPN=useBiFPN)\n",
    "\n",
    "# call retinanet for test purpose\n",
    "if retina:\n",
    "    from utils.retinanet import RetinaFPN\n",
    "    ssd_cfg = {\n",
    "        'num_classes': num_class,  # 背景クラスを含めた合計クラス数\n",
    "        'input_size': 300*scale,  # 画像の入力サイズ\n",
    "        'bbox_aspect_num': [4, 6, 6, 6, 4, 4],  # 出力するDBoxのアスペクト比の種類\n",
    "        'feature_maps': [38, 19, 10, 5, 3, 1],  # 各sourceの画像サイズ\n",
    "        'steps': [8, 16, 32, 64, 100, 300],  # DBOXの大きさを決める\n",
    "        'min_sizes': [30, 60, 111, 162, 213, 264],  # DBOXの大きさを決める\n",
    "        'max_sizes': [60, 111, 162, 213, 264, 315],  # DBOXの大きさを決める\n",
    "        'aspect_ratios': [[2], [2, 3], [2, 3], [2, 3], [2], [2]],\n",
    "    }\n",
    "    net = RetinaFPN(\"train\", ssd_cfg)\n",
    "\n",
    "# GPUが使えるか確認\n",
    "device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\n",
    "print(\"using:\", device)\n",
    "\n",
    "print(\"set weights!\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "# FP16..\n",
    "if HALF:\n",
    "    net = network_to_half(net)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Freeze backbone layers\n",
    "#for param in net.layer0.parameters():\n",
    "#    param.requires_grad = False\n",
    "#for param in net.layer2.parameters():\n",
    "#    param.requires_grad = False\n",
    "#for param in net.layer3.parameters():\n",
    "#    param.requires_grad = False\n",
    "#for param in net.layer4.parameters():\n",
    "#    param.requires_grad = False\n",
    "#for param in net.layer5.parameters():\n",
    "#    param.requires_grad = False"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "from utils.ssd_model import MultiBoxLoss\n",
    "# define loss\n",
    "criterion = MultiBoxLoss(jaccard_thresh=0.5,neg_pos=3, device=device, half=HALF)\n",
    "# optim\n",
    "import torch.optim as optim\n",
    "optimizer = optim.SGD(filter(lambda p: p.requires_grad, net.parameters()), lr=1e-3, momentum=0.9, weight_decay=5e-4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "# while the original efficientdet uses cosine annealining lr scheduling, we utilize epoch-based lr decreasing for simplicity.\n",
    "def get_current_lr(epoch): \n",
    "    if DATASET == \"COCO\":\n",
    "        reduce = [120, 180]\n",
    "        lr = 1e-3\n",
    "    else:\n",
    "        reduce = [120,180]\n",
    "        lr = 1e-3\n",
    "        \n",
    "    for i,lr_decay_epoch in enumerate(reduce):\n",
    "        if epoch >= lr_decay_epoch:\n",
    "            lr *= 0.1\n",
    "    return lr\n",
    "\n",
    "def adjust_learning_rate(optimizer, epoch):\n",
    "    lr = get_current_lr(epoch)\n",
    "    print(\"lr is:\", lr)\n",
    "    for param_group in optimizer.param_groups:\n",
    "        param_group['lr'] = lr"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "# train script. nothing special..\n",
    "def train_model(net, dataloaders_dict, criterion, optimizer, num_epochs):\n",
    "\n",
    "    # GPUが使えるかを確認\n",
    "    device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\n",
    "    print(\"used device:\", device)\n",
    "\n",
    "    # ネットワークをGPUへ\n",
    "    net.to(device)\n",
    "\n",
    "    # ネットワークがある程度固定であれば、高速化させる\n",
    "    torch.backends.cudnn.benchmark = True\n",
    "\n",
    "    # イテレーションカウンタをセット\n",
    "    iteration = 1\n",
    "    epoch_train_loss = 0.0  # epochの損失和\n",
    "    epoch_val_loss = 0.0  # epochの損失和\n",
    "    logs = []\n",
    "\n",
    "    # epochのループ\n",
    "    for epoch in range(num_epochs+1):\n",
    "        \n",
    "        adjust_learning_rate(optimizer, epoch)\n",
    "        \n",
    "        # 開始時刻を保存\n",
    "        t_epoch_start = time.time()\n",
    "        t_iter_start = time.time()\n",
    "\n",
    "        print('-------------')\n",
    "        print('Epoch {}/{}'.format(epoch+1, num_epochs))\n",
    "        print('-------------')\n",
    "\n",
    "        # epochごとの訓練と検証のループ\n",
    "        for phase in ['train', 'val']:\n",
    "            if phase == 'train':\n",
    "                net.train()  # モデルを訓練モードに\n",
    "                print('(train)')\n",
    "            else:\n",
    "                if((epoch+1) % 10 == 0):\n",
    "                    net.eval()   # モデルを検証モードに\n",
    "                    print('-------------')\n",
    "                    print('(val)')\n",
    "                else:\n",
    "                    # 検証は10回に1回だけ行う\n",
    "                    continue\n",
    "\n",
    "            # データローダーからminibatchずつ取り出すループ\n",
    "            for images, targets in dataloaders_dict[phase]:\n",
    "\n",
    "                # GPUが使えるならGPUにデータを送る\n",
    "                images = images.to(device)\n",
    "                targets = [ann.to(device)\n",
    "                           for ann in targets]  # リストの各要素のテンソルをGPUへ\n",
    "                if HALF:\n",
    "                    images = images.half()\n",
    "                    targets = [ann.half() for ann in targets]\n",
    "                # optimizerを初期化\n",
    "                optimizer.zero_grad()\n",
    "\n",
    "                # 順伝搬（forward）計算\n",
    "                with torch.set_grad_enabled(phase == 'train'):\n",
    "                    # 順伝搬（forward）計算\n",
    "                    outputs = net(images)\n",
    "                    #print(outputs[0].type())\n",
    "                    # 損失の計算\n",
    "                    loss_l, loss_c = criterion(outputs, targets)\n",
    "                    loss = loss_l + loss_c\n",
    "\n",
    "                    # 訓練時はバックプロパゲーション\n",
    "                    if phase == 'train':\n",
    "                        loss.backward()  # 勾配の計算\n",
    "\n",
    "                        # 勾配が大きくなりすぎると計算が不安定になるので、clipで最大でも勾配2.0に留める\n",
    "                        nn.utils.clip_grad_value_(\n",
    "                            net.parameters(), clip_value=2.0)\n",
    "\n",
    "                        optimizer.step()  # パラメータ更新\n",
    "\n",
    "                        if (iteration % 10 == 0):  # 10iterに1度、lossを表示\n",
    "                            t_iter_finish = time.time()\n",
    "                            duration = t_iter_finish - t_iter_start\n",
    "                            print('Iter {} || Loss: {:.4f} || 10iter: {:.4f} sec.'.format(\n",
    "                                iteration, loss.item(), duration))\n",
    "                            t_iter_start = time.time()\n",
    "                        # filter inf..\n",
    "                        if not loss.item() == float(\"inf\"):\n",
    "                            epoch_train_loss += loss.item()\n",
    "                        iteration += 1\n",
    "\n",
    "                    # 検証時\n",
    "                    else:\n",
    "                        if not loss.item() == float(\"inf\"):\n",
    "                            epoch_val_loss += loss.item()\n",
    "\n",
    "        # epochのphaseごとのlossと正解率\n",
    "        t_epoch_finish = time.time()\n",
    "        print('-------------')\n",
    "        print('epoch {} || Epoch_TRAIN_Loss:{:.4f} ||Epoch_VAL_Loss:{:.4f}'.format(\n",
    "            epoch+1, epoch_train_loss, epoch_val_loss))\n",
    "        print('timer:  {:.4f} sec.'.format(t_epoch_finish - t_epoch_start))\n",
    "        t_epoch_start = time.time()\n",
    "\n",
    "        # ログを保存\n",
    "        log_epoch = {'epoch': epoch+1,\n",
    "                     'train_loss': epoch_train_loss, 'val_loss': epoch_val_loss}\n",
    "        logs.append(log_epoch)\n",
    "        df = pd.DataFrame(logs)\n",
    "        df.to_csv(\"log/\"+DATASET+\"_\"+backbone+\"_\" + str(300*scale) +\"log_output.csv\")\n",
    "\n",
    "        epoch_train_loss = 0.0  # epochの損失和\n",
    "        epoch_val_loss = 0.0  # epochの損失和\n",
    "\n",
    "        # ネットワークを保存する\n",
    "        if ((epoch+1) % 5 == 0):\n",
    "            if useBiFPN:\n",
    "                word=\"BiFPN\"\n",
    "            else:\n",
    "                word=\"FPN\"\n",
    "            torch.save(net.state_dict(), 'weights/'+DATASET+\"_\"+backbone+\"_\" + str(300*scale) + \"_\" + word + \"_\" + \n",
    "                       str(epoch+1) + '.pth')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "used device: cuda:0\n",
      "lr is: 0.001\n",
      "-------------\n",
      "Epoch 1/200\n",
      "-------------\n",
      "(train)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/ubuntu/anaconda3/envs/pytorch_p36/lib/python3.6/site-packages/torch/nn/functional.py:2390: UserWarning: nn.functional.upsample is deprecated. Use nn.functional.interpolate instead.\n",
      "  warnings.warn(\"nn.functional.upsample is deprecated. Use nn.functional.interpolate instead.\")\n",
      "/home/ubuntu/anaconda3/envs/pytorch_p36/lib/python3.6/site-packages/torch/nn/functional.py:2479: UserWarning: Default upsampling behavior when mode=bilinear is changed to align_corners=False since 0.4.0. Please specify align_corners=True if the old behavior is desired. See the documentation of nn.Upsample for details.\n",
      "  \"See the documentation of nn.Upsample for details.\".format(mode))\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Iter 10 || Loss: 14.6762 || 10iter: 7.9883 sec.\n",
      "Iter 20 || Loss: 12.6147 || 10iter: 3.3072 sec.\n",
      "Iter 30 || Loss: 10.7831 || 10iter: 3.3764 sec.\n",
      "Iter 40 || Loss: 9.5539 || 10iter: 3.4383 sec.\n",
      "Iter 50 || Loss: 8.7597 || 10iter: 3.3863 sec.\n",
      "Iter 60 || Loss: 8.3781 || 10iter: 3.4459 sec.\n",
      "Iter 70 || Loss: 8.5997 || 10iter: 3.4078 sec.\n",
      "Iter 80 || Loss: 8.3461 || 10iter: 3.4021 sec.\n",
      "Iter 90 || Loss: 8.3254 || 10iter: 3.3984 sec.\n",
      "Iter 100 || Loss: 8.1916 || 10iter: 3.4102 sec.\n",
      "Iter 110 || Loss: 8.5010 || 10iter: 3.3978 sec.\n",
      "Iter 120 || Loss: 8.0312 || 10iter: 3.3895 sec.\n",
      "Iter 130 || Loss: 7.5776 || 10iter: 3.4147 sec.\n",
      "Iter 140 || Loss: 7.4582 || 10iter: 3.3953 sec.\n",
      "Iter 150 || Loss: 7.8578 || 10iter: 3.4497 sec.\n",
      "Iter 160 || Loss: 7.9628 || 10iter: 3.3826 sec.\n",
      "Iter 170 || Loss: 7.7113 || 10iter: 3.4399 sec.\n",
      "Iter 180 || Loss: 7.7354 || 10iter: 3.3790 sec.\n",
      "Iter 190 || Loss: 7.6620 || 10iter: 3.3862 sec.\n",
      "Iter 200 || Loss: 7.6940 || 10iter: 3.3755 sec.\n",
      "Iter 210 || Loss: 7.4468 || 10iter: 3.4130 sec.\n",
      "Iter 220 || Loss: 7.0901 || 10iter: 3.3777 sec.\n",
      "Iter 230 || Loss: 7.6807 || 10iter: 3.4167 sec.\n",
      "Iter 240 || Loss: 7.3931 || 10iter: 3.3779 sec.\n",
      "Iter 250 || Loss: 7.0309 || 10iter: 3.3901 sec.\n",
      "Iter 260 || Loss: 7.7965 || 10iter: 3.3897 sec.\n",
      "Iter 270 || Loss: 7.2047 || 10iter: 3.4298 sec.\n",
      "Iter 280 || Loss: 7.4838 || 10iter: 3.3914 sec.\n",
      "Iter 290 || Loss: 7.2852 || 10iter: 3.3917 sec.\n",
      "Iter 300 || Loss: 7.8419 || 10iter: 3.5271 sec.\n",
      "Iter 310 || Loss: 7.4730 || 10iter: 3.4624 sec.\n",
      "Iter 320 || Loss: 6.9173 || 10iter: 3.4627 sec.\n",
      "Iter 330 || Loss: 6.9936 || 10iter: 3.4804 sec.\n",
      "Iter 340 || Loss: 7.0920 || 10iter: 3.4729 sec.\n",
      "Iter 350 || Loss: 7.2672 || 10iter: 3.3754 sec.\n",
      "Iter 360 || Loss: 6.7265 || 10iter: 3.4447 sec.\n",
      "Iter 370 || Loss: 7.0137 || 10iter: 3.3979 sec.\n",
      "Iter 380 || Loss: 6.7315 || 10iter: 3.4203 sec.\n",
      "Iter 390 || Loss: 6.9814 || 10iter: 3.4017 sec.\n",
      "Iter 400 || Loss: 7.2596 || 10iter: 3.3769 sec.\n",
      "Iter 410 || Loss: 7.5059 || 10iter: 3.4078 sec.\n",
      "Iter 420 || Loss: 6.9578 || 10iter: 3.4079 sec.\n",
      "Iter 430 || Loss: 6.8931 || 10iter: 3.3843 sec.\n",
      "Iter 440 || Loss: 7.0741 || 10iter: 3.3803 sec.\n",
      "Iter 450 || Loss: 6.9327 || 10iter: 3.3740 sec.\n",
      "Iter 460 || Loss: 6.7873 || 10iter: 3.4166 sec.\n",
      "Iter 470 || Loss: 7.0147 || 10iter: 3.3570 sec.\n",
      "Iter 480 || Loss: 6.9948 || 10iter: 3.3959 sec.\n",
      "Iter 490 || Loss: 6.7472 || 10iter: 3.3642 sec.\n",
      "Iter 500 || Loss: 6.3939 || 10iter: 3.3633 sec.\n",
      "Iter 510 || Loss: 6.7486 || 10iter: 3.3512 sec.\n",
      "-------------\n",
      "epoch 1 || Epoch_TRAIN_Loss:4080.5465 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  182.0846 sec.\n",
      "lr is: 0.001\n",
      "-------------\n",
      "Epoch 2/200\n",
      "-------------\n",
      "(train)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/ubuntu/anaconda3/envs/pytorch_p36/lib/python3.6/site-packages/torch/nn/functional.py:2390: UserWarning: nn.functional.upsample is deprecated. Use nn.functional.interpolate instead.\n",
      "  warnings.warn(\"nn.functional.upsample is deprecated. Use nn.functional.interpolate instead.\")\n",
      "/home/ubuntu/anaconda3/envs/pytorch_p36/lib/python3.6/site-packages/torch/nn/functional.py:2479: UserWarning: Default upsampling behavior when mode=bilinear is changed to align_corners=False since 0.4.0. Please specify align_corners=True if the old behavior is desired. See the documentation of nn.Upsample for details.\n",
      "  \"See the documentation of nn.Upsample for details.\".format(mode))\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Iter 520 || Loss: 6.9472 || 10iter: 2.3994 sec.\n",
      "Iter 530 || Loss: 6.9510 || 10iter: 3.4278 sec.\n",
      "Iter 540 || Loss: 6.6578 || 10iter: 3.3295 sec.\n",
      "Iter 550 || Loss: 6.7569 || 10iter: 3.3849 sec.\n",
      "Iter 560 || Loss: 6.6651 || 10iter: 3.4692 sec.\n",
      "Iter 570 || Loss: 6.7154 || 10iter: 3.3967 sec.\n",
      "Iter 580 || Loss: 6.8194 || 10iter: 3.3336 sec.\n",
      "Iter 590 || Loss: 6.6416 || 10iter: 3.4128 sec.\n",
      "Iter 600 || Loss: 7.0260 || 10iter: 3.4065 sec.\n",
      "Iter 610 || Loss: 6.6993 || 10iter: 3.4378 sec.\n",
      "Iter 620 || Loss: 6.7130 || 10iter: 3.3429 sec.\n",
      "Iter 630 || Loss: 6.6879 || 10iter: 3.3698 sec.\n",
      "Iter 640 || Loss: 6.7656 || 10iter: 3.4128 sec.\n",
      "Iter 650 || Loss: 6.6890 || 10iter: 3.4234 sec.\n",
      "Iter 660 || Loss: 6.7733 || 10iter: 3.4324 sec.\n",
      "Iter 670 || Loss: 6.4825 || 10iter: 3.5050 sec.\n",
      "Iter 680 || Loss: 6.7527 || 10iter: 3.3603 sec.\n",
      "Iter 690 || Loss: 6.6672 || 10iter: 3.3518 sec.\n",
      "Iter 700 || Loss: 6.3197 || 10iter: 3.3612 sec.\n",
      "Iter 710 || Loss: 6.6545 || 10iter: 3.3912 sec.\n",
      "Iter 720 || Loss: 6.8526 || 10iter: 3.3532 sec.\n",
      "Iter 730 || Loss: 6.1515 || 10iter: 3.3754 sec.\n",
      "Iter 740 || Loss: 6.2790 || 10iter: 3.3865 sec.\n",
      "Iter 750 || Loss: 6.4217 || 10iter: 3.3455 sec.\n",
      "Iter 760 || Loss: 6.1095 || 10iter: 3.4578 sec.\n",
      "Iter 770 || Loss: 6.9900 || 10iter: 3.3986 sec.\n",
      "Iter 780 || Loss: 6.5425 || 10iter: 3.3769 sec.\n",
      "Iter 790 || Loss: 6.2541 || 10iter: 3.3947 sec.\n",
      "Iter 800 || Loss: 6.2351 || 10iter: 3.5440 sec.\n",
      "Iter 810 || Loss: 6.3850 || 10iter: 3.3746 sec.\n",
      "Iter 820 || Loss: 6.3424 || 10iter: 3.3778 sec.\n",
      "Iter 830 || Loss: 6.3329 || 10iter: 3.3422 sec.\n",
      "Iter 840 || Loss: 6.1427 || 10iter: 3.3788 sec.\n",
      "Iter 850 || Loss: 6.3155 || 10iter: 3.3990 sec.\n",
      "Iter 860 || Loss: 6.5385 || 10iter: 3.4287 sec.\n",
      "Iter 870 || Loss: 6.7060 || 10iter: 3.3748 sec.\n",
      "Iter 880 || Loss: 6.6168 || 10iter: 3.4113 sec.\n",
      "Iter 890 || Loss: 6.3317 || 10iter: 3.4622 sec.\n",
      "Iter 900 || Loss: 6.1234 || 10iter: 3.3928 sec.\n",
      "Iter 910 || Loss: 6.3658 || 10iter: 3.3789 sec.\n",
      "Iter 920 || Loss: 6.1676 || 10iter: 3.3857 sec.\n",
      "Iter 930 || Loss: 6.5839 || 10iter: 3.3807 sec.\n",
      "Iter 940 || Loss: 6.2183 || 10iter: 3.3744 sec.\n",
      "Iter 950 || Loss: 6.0194 || 10iter: 3.3649 sec.\n",
      "Iter 960 || Loss: 6.1450 || 10iter: 3.3765 sec.\n",
      "Iter 970 || Loss: 6.0713 || 10iter: 3.4035 sec.\n",
      "Iter 980 || Loss: 5.8554 || 10iter: 3.4733 sec.\n",
      "Iter 990 || Loss: 6.5032 || 10iter: 3.3750 sec.\n",
      "Iter 1000 || Loss: 5.9616 || 10iter: 3.4450 sec.\n",
      "Iter 1010 || Loss: 5.9796 || 10iter: 3.3341 sec.\n",
      "Iter 1020 || Loss: 6.4863 || 10iter: 3.4197 sec.\n",
      "Iter 1030 || Loss: 5.8776 || 10iter: 3.3085 sec.\n",
      "-------------\n",
      "epoch 2 || Epoch_TRAIN_Loss:3357.0348 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  177.4286 sec.\n",
      "lr is: 0.001\n",
      "-------------\n",
      "Epoch 3/200\n",
      "-------------\n",
      "(train)\n",
      "Iter 1040 || Loss: 5.5674 || 10iter: 3.3006 sec.\n",
      "Iter 1050 || Loss: 6.1985 || 10iter: 3.3766 sec.\n",
      "Iter 1060 || Loss: 6.1799 || 10iter: 3.4175 sec.\n",
      "Iter 1070 || Loss: 5.8693 || 10iter: 3.3989 sec.\n",
      "Iter 1080 || Loss: 6.6458 || 10iter: 3.3712 sec.\n",
      "Iter 1090 || Loss: 6.1802 || 10iter: 3.4018 sec.\n",
      "Iter 1100 || Loss: 6.1411 || 10iter: 3.3660 sec.\n",
      "Iter 1110 || Loss: 6.4173 || 10iter: 3.4167 sec.\n",
      "Iter 1120 || Loss: 6.0469 || 10iter: 3.3572 sec.\n",
      "Iter 1130 || Loss: 5.9920 || 10iter: 3.3654 sec.\n",
      "Iter 1140 || Loss: 6.1453 || 10iter: 3.4093 sec.\n",
      "Iter 1150 || Loss: 6.3970 || 10iter: 3.3702 sec.\n",
      "Iter 1160 || Loss: 6.5036 || 10iter: 3.3677 sec.\n",
      "Iter 1170 || Loss: 5.8871 || 10iter: 3.3874 sec.\n",
      "Iter 1180 || Loss: 5.7497 || 10iter: 3.3774 sec.\n",
      "Iter 1190 || Loss: 6.4121 || 10iter: 3.3427 sec.\n",
      "Iter 1200 || Loss: 6.2581 || 10iter: 3.4103 sec.\n",
      "Iter 1210 || Loss: 6.0120 || 10iter: 3.3796 sec.\n",
      "Iter 1220 || Loss: 5.9606 || 10iter: 3.3459 sec.\n",
      "Iter 1230 || Loss: 5.8456 || 10iter: 3.4624 sec.\n",
      "Iter 1240 || Loss: 5.5922 || 10iter: 3.3915 sec.\n",
      "Iter 1250 || Loss: 5.9974 || 10iter: 3.3961 sec.\n",
      "Iter 1260 || Loss: 6.2425 || 10iter: 3.3770 sec.\n",
      "Iter 1270 || Loss: 5.6455 || 10iter: 3.4496 sec.\n",
      "Iter 1280 || Loss: 5.9387 || 10iter: 3.5558 sec.\n",
      "Iter 1290 || Loss: 5.9458 || 10iter: 3.3486 sec.\n",
      "Iter 1300 || Loss: 5.7390 || 10iter: 3.3495 sec.\n",
      "Iter 1310 || Loss: 5.9368 || 10iter: 3.5734 sec.\n",
      "Iter 1320 || Loss: 5.9990 || 10iter: 3.4123 sec.\n",
      "Iter 1330 || Loss: 6.4312 || 10iter: 3.3843 sec.\n",
      "Iter 1340 || Loss: 6.2937 || 10iter: 3.3758 sec.\n",
      "Iter 1350 || Loss: 5.7269 || 10iter: 3.4153 sec.\n",
      "Iter 1360 || Loss: 5.9134 || 10iter: 3.3875 sec.\n",
      "Iter 1370 || Loss: 5.8195 || 10iter: 3.3964 sec.\n",
      "Iter 1380 || Loss: 5.9071 || 10iter: 3.3968 sec.\n",
      "Iter 1390 || Loss: 5.7844 || 10iter: 3.4501 sec.\n",
      "Iter 1400 || Loss: 5.3931 || 10iter: 3.3701 sec.\n",
      "Iter 1410 || Loss: 6.1411 || 10iter: 3.3563 sec.\n",
      "Iter 1420 || Loss: 5.5713 || 10iter: 3.3507 sec.\n",
      "Iter 1430 || Loss: 5.6741 || 10iter: 3.4549 sec.\n",
      "Iter 1440 || Loss: 6.0635 || 10iter: 3.3693 sec.\n",
      "Iter 1450 || Loss: 5.9443 || 10iter: 3.3515 sec.\n",
      "Iter 1460 || Loss: 5.5470 || 10iter: 3.3982 sec.\n",
      "Iter 1470 || Loss: 5.9251 || 10iter: 3.3495 sec.\n",
      "Iter 1480 || Loss: 5.8048 || 10iter: 3.3586 sec.\n",
      "Iter 1490 || Loss: 4.9968 || 10iter: 3.4164 sec.\n",
      "Iter 1500 || Loss: 6.0812 || 10iter: 3.4000 sec.\n",
      "Iter 1510 || Loss: 5.6916 || 10iter: 3.4305 sec.\n",
      "Iter 1520 || Loss: 6.1723 || 10iter: 3.3464 sec.\n",
      "Iter 1530 || Loss: 5.8203 || 10iter: 3.4091 sec.\n",
      "Iter 1540 || Loss: 5.8156 || 10iter: 3.3567 sec.\n",
      "Iter 1550 || Loss: 5.9790 || 10iter: 3.3206 sec.\n",
      "-------------\n",
      "epoch 3 || Epoch_TRAIN_Loss:3100.4523 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  177.5168 sec.\n",
      "lr is: 0.001\n",
      "-------------\n",
      "Epoch 4/200\n",
      "-------------\n",
      "(train)\n",
      "Iter 1560 || Loss: 5.9799 || 10iter: 3.8830 sec.\n",
      "Iter 1570 || Loss: 5.5508 || 10iter: 3.3975 sec.\n",
      "Iter 1580 || Loss: 5.6072 || 10iter: 3.3891 sec.\n",
      "Iter 1590 || Loss: 5.9854 || 10iter: 3.3590 sec.\n",
      "Iter 1600 || Loss: 5.4864 || 10iter: 3.4496 sec.\n",
      "Iter 1610 || Loss: 5.7796 || 10iter: 3.3447 sec.\n",
      "Iter 1620 || Loss: 5.1998 || 10iter: 3.3758 sec.\n",
      "Iter 1630 || Loss: 5.4988 || 10iter: 3.4389 sec.\n",
      "Iter 1640 || Loss: 6.5233 || 10iter: 3.3597 sec.\n",
      "Iter 1650 || Loss: 6.1757 || 10iter: 3.4362 sec.\n",
      "Iter 1660 || Loss: 5.7565 || 10iter: 3.3796 sec.\n",
      "Iter 1670 || Loss: 6.0810 || 10iter: 3.3692 sec.\n",
      "Iter 1680 || Loss: 5.6137 || 10iter: 3.3973 sec.\n",
      "Iter 1690 || Loss: 5.7863 || 10iter: 3.3457 sec.\n",
      "Iter 1700 || Loss: 5.3208 || 10iter: 3.3608 sec.\n",
      "Iter 1710 || Loss: 5.8368 || 10iter: 3.3854 sec.\n",
      "Iter 1720 || Loss: 6.3569 || 10iter: 3.3897 sec.\n",
      "Iter 1730 || Loss: 5.6071 || 10iter: 3.3572 sec.\n",
      "Iter 1740 || Loss: 6.0559 || 10iter: 3.4194 sec.\n",
      "Iter 1750 || Loss: 5.5833 || 10iter: 3.3956 sec.\n",
      "Iter 1760 || Loss: 5.6293 || 10iter: 3.4716 sec.\n",
      "Iter 1770 || Loss: 5.1258 || 10iter: 3.4242 sec.\n",
      "Iter 1780 || Loss: 5.8077 || 10iter: 3.3856 sec.\n",
      "Iter 1790 || Loss: 6.0024 || 10iter: 3.3742 sec.\n",
      "Iter 1800 || Loss: 5.8675 || 10iter: 3.3650 sec.\n",
      "Iter 1810 || Loss: 5.4309 || 10iter: 3.3867 sec.\n",
      "Iter 1820 || Loss: 5.7111 || 10iter: 3.4044 sec.\n",
      "Iter 1830 || Loss: 5.5614 || 10iter: 3.4254 sec.\n",
      "Iter 1840 || Loss: 5.4821 || 10iter: 3.3395 sec.\n",
      "Iter 1850 || Loss: 5.6724 || 10iter: 3.4383 sec.\n",
      "Iter 1860 || Loss: 5.2310 || 10iter: 3.4124 sec.\n",
      "Iter 1870 || Loss: 5.7805 || 10iter: 3.3480 sec.\n",
      "Iter 1880 || Loss: 5.4647 || 10iter: 3.3888 sec.\n",
      "Iter 1890 || Loss: 5.6374 || 10iter: 3.3505 sec.\n",
      "Iter 1900 || Loss: 6.1289 || 10iter: 3.3597 sec.\n",
      "Iter 1910 || Loss: 5.3480 || 10iter: 3.3675 sec.\n",
      "Iter 1920 || Loss: 5.4142 || 10iter: 3.3739 sec.\n",
      "Iter 1930 || Loss: 5.4423 || 10iter: 3.4005 sec.\n",
      "Iter 1940 || Loss: 5.3832 || 10iter: 3.3777 sec.\n",
      "Iter 1950 || Loss: 5.6668 || 10iter: 3.4947 sec.\n",
      "Iter 1960 || Loss: 5.7623 || 10iter: 3.4734 sec.\n",
      "Iter 1970 || Loss: 5.6145 || 10iter: 3.3711 sec.\n",
      "Iter 1980 || Loss: 5.3514 || 10iter: 3.4602 sec.\n",
      "Iter 1990 || Loss: 5.5273 || 10iter: 3.4239 sec.\n",
      "Iter 2000 || Loss: 5.3952 || 10iter: 3.4728 sec.\n",
      "Iter 2010 || Loss: 5.7252 || 10iter: 3.4553 sec.\n",
      "Iter 2020 || Loss: 5.2708 || 10iter: 3.4196 sec.\n",
      "Iter 2030 || Loss: 5.3306 || 10iter: 3.3798 sec.\n",
      "Iter 2040 || Loss: 5.5061 || 10iter: 3.3608 sec.\n",
      "Iter 2050 || Loss: 5.5394 || 10iter: 3.3605 sec.\n",
      "Iter 2060 || Loss: 5.4732 || 10iter: 3.3161 sec.\n",
      "Iter 2070 || Loss: 5.5331 || 10iter: 3.2867 sec.\n",
      "-------------\n",
      "epoch 4 || Epoch_TRAIN_Loss:2915.8172 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  177.4296 sec.\n",
      "lr is: 0.001\n",
      "-------------\n",
      "Epoch 5/200\n",
      "-------------\n",
      "(train)\n",
      "Iter 2080 || Loss: 5.3294 || 10iter: 4.3892 sec.\n",
      "Iter 2090 || Loss: 5.0832 || 10iter: 3.4082 sec.\n",
      "Iter 2100 || Loss: 5.9817 || 10iter: 3.3547 sec.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Iter 2110 || Loss: 5.3264 || 10iter: 3.3978 sec.\n",
      "Iter 2120 || Loss: 5.3107 || 10iter: 3.3645 sec.\n",
      "Iter 2130 || Loss: 5.4049 || 10iter: 3.3747 sec.\n",
      "Iter 2140 || Loss: 5.6159 || 10iter: 3.4130 sec.\n",
      "Iter 2150 || Loss: 5.3737 || 10iter: 3.3554 sec.\n",
      "Iter 2160 || Loss: 5.6128 || 10iter: 3.3987 sec.\n",
      "Iter 2170 || Loss: 5.6840 || 10iter: 3.4465 sec.\n",
      "Iter 2180 || Loss: 5.4251 || 10iter: 3.4079 sec.\n",
      "Iter 2190 || Loss: 5.4585 || 10iter: 3.4962 sec.\n",
      "Iter 2200 || Loss: 4.8977 || 10iter: 3.3691 sec.\n",
      "Iter 2210 || Loss: 5.7564 || 10iter: 3.3954 sec.\n",
      "Iter 2220 || Loss: 5.2696 || 10iter: 3.3692 sec.\n",
      "Iter 2230 || Loss: 5.3867 || 10iter: 3.3577 sec.\n",
      "Iter 2240 || Loss: 4.7236 || 10iter: 3.3490 sec.\n",
      "Iter 2250 || Loss: 5.1422 || 10iter: 3.3428 sec.\n",
      "Iter 2260 || Loss: 5.4216 || 10iter: 3.3415 sec.\n",
      "Iter 2270 || Loss: 5.6682 || 10iter: 3.3664 sec.\n",
      "Iter 2280 || Loss: 5.5349 || 10iter: 3.3840 sec.\n",
      "Iter 2290 || Loss: 5.5123 || 10iter: 3.4147 sec.\n",
      "Iter 2300 || Loss: 5.8075 || 10iter: 3.3575 sec.\n",
      "Iter 2310 || Loss: 5.2453 || 10iter: 3.3589 sec.\n",
      "Iter 2320 || Loss: 5.5742 || 10iter: 3.3421 sec.\n",
      "Iter 2330 || Loss: 5.2661 || 10iter: 3.3936 sec.\n",
      "Iter 2340 || Loss: 5.1020 || 10iter: 3.4413 sec.\n",
      "Iter 2350 || Loss: 5.2872 || 10iter: 3.3606 sec.\n",
      "Iter 2360 || Loss: 5.3270 || 10iter: 3.3486 sec.\n",
      "Iter 2370 || Loss: 5.8302 || 10iter: 3.3676 sec.\n",
      "Iter 2380 || Loss: 5.1423 || 10iter: 3.4300 sec.\n",
      "Iter 2390 || Loss: 5.2949 || 10iter: 3.3819 sec.\n",
      "Iter 2400 || Loss: 5.2698 || 10iter: 3.4141 sec.\n",
      "Iter 2410 || Loss: 5.1737 || 10iter: 3.4935 sec.\n",
      "Iter 2420 || Loss: 5.1211 || 10iter: 3.4262 sec.\n",
      "Iter 2430 || Loss: 5.3296 || 10iter: 3.4220 sec.\n",
      "Iter 2440 || Loss: 5.1766 || 10iter: 3.4188 sec.\n",
      "Iter 2450 || Loss: 5.8961 || 10iter: 3.4120 sec.\n",
      "Iter 2460 || Loss: 5.4441 || 10iter: 3.4251 sec.\n",
      "Iter 2470 || Loss: 5.8294 || 10iter: 3.3646 sec.\n",
      "Iter 2480 || Loss: 5.7525 || 10iter: 3.3862 sec.\n",
      "Iter 2490 || Loss: 4.7155 || 10iter: 3.4026 sec.\n",
      "Iter 2500 || Loss: 5.6746 || 10iter: 3.4795 sec.\n",
      "Iter 2510 || Loss: 5.2167 || 10iter: 3.3580 sec.\n",
      "Iter 2520 || Loss: 5.5177 || 10iter: 3.3717 sec.\n",
      "Iter 2530 || Loss: 5.8399 || 10iter: 3.3669 sec.\n",
      "Iter 2540 || Loss: 5.4010 || 10iter: 3.3840 sec.\n",
      "Iter 2550 || Loss: 5.4305 || 10iter: 3.3418 sec.\n",
      "Iter 2560 || Loss: 5.3445 || 10iter: 3.4084 sec.\n",
      "Iter 2570 || Loss: 5.0642 || 10iter: 3.3754 sec.\n",
      "Iter 2580 || Loss: 5.4277 || 10iter: 3.3155 sec.\n",
      "Iter 2590 || Loss: 5.7585 || 10iter: 3.0804 sec.\n",
      "-------------\n",
      "epoch 5 || Epoch_TRAIN_Loss:2787.7734 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  176.9936 sec.\n",
      "lr is: 0.001\n",
      "-------------\n",
      "Epoch 6/200\n",
      "-------------\n",
      "(train)\n",
      "Iter 2600 || Loss: 5.1288 || 10iter: 5.2803 sec.\n",
      "Iter 2610 || Loss: 5.0692 || 10iter: 3.4138 sec.\n",
      "Iter 2620 || Loss: 4.8678 || 10iter: 3.4073 sec.\n",
      "Iter 2630 || Loss: 5.7774 || 10iter: 3.4430 sec.\n",
      "Iter 2640 || Loss: 4.8985 || 10iter: 3.3773 sec.\n",
      "Iter 2650 || Loss: 5.5676 || 10iter: 3.4293 sec.\n",
      "Iter 2660 || Loss: 5.2061 || 10iter: 3.4343 sec.\n",
      "Iter 2670 || Loss: 5.5605 || 10iter: 3.4016 sec.\n",
      "Iter 2680 || Loss: 5.9544 || 10iter: 3.3901 sec.\n",
      "Iter 2690 || Loss: 5.0868 || 10iter: 3.3688 sec.\n",
      "Iter 2700 || Loss: 5.2073 || 10iter: 3.3698 sec.\n",
      "Iter 2710 || Loss: 5.1464 || 10iter: 3.3585 sec.\n",
      "Iter 2720 || Loss: 5.2350 || 10iter: 3.3562 sec.\n",
      "Iter 2730 || Loss: 4.8155 || 10iter: 3.3503 sec.\n",
      "Iter 2740 || Loss: 5.3389 || 10iter: 3.4014 sec.\n",
      "Iter 2750 || Loss: 5.5505 || 10iter: 3.4290 sec.\n",
      "Iter 2760 || Loss: 4.8003 || 10iter: 3.3623 sec.\n",
      "Iter 2770 || Loss: 4.9201 || 10iter: 3.3576 sec.\n",
      "Iter 2780 || Loss: 5.3539 || 10iter: 3.3448 sec.\n",
      "Iter 2790 || Loss: 5.2739 || 10iter: 3.4221 sec.\n",
      "Iter 2800 || Loss: 5.2493 || 10iter: 3.3922 sec.\n",
      "Iter 2810 || Loss: 5.0645 || 10iter: 3.3891 sec.\n",
      "Iter 2820 || Loss: 5.4998 || 10iter: 3.3893 sec.\n",
      "Iter 2830 || Loss: 4.9218 || 10iter: 3.3521 sec.\n",
      "Iter 2840 || Loss: 5.4572 || 10iter: 3.3764 sec.\n",
      "Iter 2850 || Loss: 5.2022 || 10iter: 3.3731 sec.\n",
      "Iter 2860 || Loss: 5.0886 || 10iter: 3.3638 sec.\n",
      "Iter 2870 || Loss: 5.3664 || 10iter: 3.3613 sec.\n",
      "Iter 2880 || Loss: 5.6510 || 10iter: 3.3509 sec.\n",
      "Iter 2890 || Loss: 4.9461 || 10iter: 3.3517 sec.\n",
      "Iter 2900 || Loss: 5.2534 || 10iter: 3.3601 sec.\n",
      "Iter 2910 || Loss: 5.4036 || 10iter: 3.3772 sec.\n",
      "Iter 2920 || Loss: 5.1080 || 10iter: 3.4180 sec.\n",
      "Iter 2930 || Loss: 5.0302 || 10iter: 3.3692 sec.\n",
      "Iter 2940 || Loss: 5.7639 || 10iter: 3.4611 sec.\n",
      "Iter 2950 || Loss: 5.0709 || 10iter: 3.3403 sec.\n",
      "Iter 2960 || Loss: 5.1821 || 10iter: 3.4102 sec.\n",
      "Iter 2970 || Loss: 5.1731 || 10iter: 3.4250 sec.\n",
      "Iter 2980 || Loss: 4.6264 || 10iter: 3.4121 sec.\n",
      "Iter 2990 || Loss: 5.0477 || 10iter: 3.3859 sec.\n",
      "Iter 3000 || Loss: 4.9625 || 10iter: 3.3554 sec.\n",
      "Iter 3010 || Loss: 5.1828 || 10iter: 3.3571 sec.\n",
      "Iter 3020 || Loss: 5.3519 || 10iter: 3.3411 sec.\n",
      "Iter 3030 || Loss: 5.2634 || 10iter: 3.3441 sec.\n",
      "Iter 3040 || Loss: 5.1069 || 10iter: 3.3460 sec.\n",
      "Iter 3050 || Loss: 5.0603 || 10iter: 3.3598 sec.\n",
      "Iter 3060 || Loss: 5.0679 || 10iter: 3.3526 sec.\n",
      "Iter 3070 || Loss: 4.7272 || 10iter: 3.3563 sec.\n",
      "Iter 3080 || Loss: 5.0513 || 10iter: 3.3924 sec.\n",
      "Iter 3090 || Loss: 4.8833 || 10iter: 3.4150 sec.\n",
      "Iter 3100 || Loss: 4.5966 || 10iter: 3.3033 sec.\n",
      "-------------\n",
      "epoch 6 || Epoch_TRAIN_Loss:2710.0843 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  176.7840 sec.\n",
      "lr is: 0.001\n",
      "-------------\n",
      "Epoch 7/200\n",
      "-------------\n",
      "(train)\n",
      "Iter 3110 || Loss: 4.7860 || 10iter: 2.5910 sec.\n",
      "Iter 3120 || Loss: 5.5041 || 10iter: 3.3748 sec.\n",
      "Iter 3130 || Loss: 5.3920 || 10iter: 3.3158 sec.\n",
      "Iter 3140 || Loss: 5.3774 || 10iter: 3.4849 sec.\n",
      "Iter 3150 || Loss: 5.1896 || 10iter: 3.3832 sec.\n",
      "Iter 3160 || Loss: 4.9054 || 10iter: 3.3577 sec.\n",
      "Iter 3170 || Loss: 4.7008 || 10iter: 3.3493 sec.\n",
      "Iter 3180 || Loss: 5.2452 || 10iter: 3.3764 sec.\n",
      "Iter 3190 || Loss: 5.2865 || 10iter: 3.4220 sec.\n",
      "Iter 3200 || Loss: 5.0380 || 10iter: 3.3539 sec.\n",
      "Iter 3210 || Loss: 5.0281 || 10iter: 3.4014 sec.\n",
      "Iter 3220 || Loss: 5.6139 || 10iter: 3.3525 sec.\n",
      "Iter 3230 || Loss: 4.9781 || 10iter: 3.3728 sec.\n",
      "Iter 3240 || Loss: 5.3442 || 10iter: 3.4350 sec.\n",
      "Iter 3250 || Loss: 5.5887 || 10iter: 3.3930 sec.\n",
      "Iter 3260 || Loss: 5.2225 || 10iter: 3.3867 sec.\n",
      "Iter 3270 || Loss: 5.1588 || 10iter: 3.4158 sec.\n",
      "Iter 3280 || Loss: 5.1048 || 10iter: 3.3779 sec.\n",
      "Iter 3290 || Loss: 5.3939 || 10iter: 3.3705 sec.\n",
      "Iter 3300 || Loss: 5.2658 || 10iter: 3.3673 sec.\n",
      "Iter 3310 || Loss: 5.2850 || 10iter: 3.3665 sec.\n",
      "Iter 3320 || Loss: 4.6721 || 10iter: 3.4389 sec.\n",
      "Iter 3330 || Loss: 5.1712 || 10iter: 3.3641 sec.\n",
      "Iter 3340 || Loss: 4.8982 || 10iter: 3.4887 sec.\n",
      "Iter 3350 || Loss: 4.7286 || 10iter: 3.3784 sec.\n",
      "Iter 3360 || Loss: 5.0570 || 10iter: 3.3560 sec.\n",
      "Iter 3370 || Loss: 5.1286 || 10iter: 3.3831 sec.\n",
      "Iter 3380 || Loss: 4.9128 || 10iter: 3.3658 sec.\n",
      "Iter 3390 || Loss: 5.0205 || 10iter: 3.3868 sec.\n",
      "Iter 3400 || Loss: 5.1961 || 10iter: 3.3624 sec.\n",
      "Iter 3410 || Loss: 5.1354 || 10iter: 3.3665 sec.\n",
      "Iter 3420 || Loss: 4.7186 || 10iter: 3.3691 sec.\n",
      "Iter 3430 || Loss: 5.0709 || 10iter: 3.3734 sec.\n",
      "Iter 3440 || Loss: 5.2659 || 10iter: 3.4313 sec.\n",
      "Iter 3450 || Loss: 4.7667 || 10iter: 3.3649 sec.\n",
      "Iter 3460 || Loss: 5.1038 || 10iter: 3.4342 sec.\n",
      "Iter 3470 || Loss: 4.8508 || 10iter: 3.3645 sec.\n",
      "Iter 3480 || Loss: 5.0479 || 10iter: 3.3485 sec.\n",
      "Iter 3490 || Loss: 4.9161 || 10iter: 3.3587 sec.\n",
      "Iter 3500 || Loss: 4.8759 || 10iter: 3.3661 sec.\n",
      "Iter 3510 || Loss: 4.8505 || 10iter: 3.3777 sec.\n",
      "Iter 3520 || Loss: 5.3492 || 10iter: 3.3906 sec.\n",
      "Iter 3530 || Loss: 5.1807 || 10iter: 3.4205 sec.\n",
      "Iter 3540 || Loss: 4.8861 || 10iter: 3.3694 sec.\n",
      "Iter 3550 || Loss: 4.8535 || 10iter: 3.4204 sec.\n",
      "Iter 3560 || Loss: 5.2251 || 10iter: 3.3683 sec.\n",
      "Iter 3570 || Loss: 5.7424 || 10iter: 3.3468 sec.\n",
      "Iter 3580 || Loss: 5.1441 || 10iter: 3.4248 sec.\n",
      "Iter 3590 || Loss: 5.1995 || 10iter: 3.3633 sec.\n",
      "Iter 3600 || Loss: 5.0320 || 10iter: 3.3571 sec.\n",
      "Iter 3610 || Loss: 4.6265 || 10iter: 3.3839 sec.\n",
      "Iter 3620 || Loss: 5.3315 || 10iter: 3.2952 sec.\n",
      "-------------\n",
      "epoch 7 || Epoch_TRAIN_Loss:2621.9839 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  176.9218 sec.\n",
      "lr is: 0.001\n",
      "-------------\n",
      "Epoch 8/200\n",
      "-------------\n",
      "(train)\n",
      "Iter 3630 || Loss: 4.8520 || 10iter: 3.1964 sec.\n",
      "Iter 3640 || Loss: 5.0182 || 10iter: 3.3516 sec.\n",
      "Iter 3650 || Loss: 5.1061 || 10iter: 3.3968 sec.\n",
      "Iter 3660 || Loss: 5.0838 || 10iter: 3.4812 sec.\n",
      "Iter 3670 || Loss: 4.5199 || 10iter: 3.3855 sec.\n",
      "Iter 3680 || Loss: 5.0928 || 10iter: 3.4722 sec.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Iter 3690 || Loss: 4.8675 || 10iter: 3.3999 sec.\n",
      "Iter 3700 || Loss: 4.7384 || 10iter: 3.4340 sec.\n",
      "Iter 3710 || Loss: 4.6953 || 10iter: 3.3850 sec.\n",
      "Iter 3720 || Loss: 5.2084 || 10iter: 3.3636 sec.\n",
      "Iter 3730 || Loss: 4.3088 || 10iter: 3.4519 sec.\n",
      "Iter 3740 || Loss: 4.9228 || 10iter: 3.4815 sec.\n",
      "Iter 3750 || Loss: 4.7602 || 10iter: 3.4038 sec.\n",
      "Iter 3760 || Loss: 4.9588 || 10iter: 3.4107 sec.\n",
      "Iter 3770 || Loss: 4.8657 || 10iter: 3.4762 sec.\n",
      "Iter 3780 || Loss: 5.0428 || 10iter: 3.5091 sec.\n",
      "Iter 3790 || Loss: 5.7787 || 10iter: 3.4346 sec.\n",
      "Iter 3800 || Loss: 4.7304 || 10iter: 3.3827 sec.\n",
      "Iter 3810 || Loss: 4.7113 || 10iter: 3.3482 sec.\n",
      "Iter 3820 || Loss: 4.7078 || 10iter: 3.3526 sec.\n",
      "Iter 3830 || Loss: 4.8722 || 10iter: 3.4191 sec.\n",
      "Iter 3840 || Loss: 5.0261 || 10iter: 3.3982 sec.\n",
      "Iter 3850 || Loss: 5.0775 || 10iter: 3.3621 sec.\n",
      "Iter 3860 || Loss: 5.1993 || 10iter: 3.3771 sec.\n",
      "Iter 3870 || Loss: 4.8228 || 10iter: 3.3633 sec.\n",
      "Iter 3880 || Loss: 5.2044 || 10iter: 3.4146 sec.\n",
      "Iter 3890 || Loss: 4.4160 || 10iter: 3.3567 sec.\n",
      "Iter 3900 || Loss: 5.4897 || 10iter: 3.3647 sec.\n",
      "Iter 3910 || Loss: 4.9229 || 10iter: 3.3357 sec.\n",
      "Iter 3920 || Loss: 5.0649 || 10iter: 3.3550 sec.\n",
      "Iter 3930 || Loss: 4.7691 || 10iter: 3.3685 sec.\n",
      "Iter 3940 || Loss: 4.6749 || 10iter: 3.3633 sec.\n",
      "Iter 3950 || Loss: 4.9752 || 10iter: 3.4146 sec.\n",
      "Iter 3960 || Loss: 4.4758 || 10iter: 3.4739 sec.\n",
      "Iter 3970 || Loss: 4.6764 || 10iter: 3.3567 sec.\n",
      "Iter 3980 || Loss: 4.8096 || 10iter: 3.3532 sec.\n",
      "Iter 3990 || Loss: 4.2373 || 10iter: 3.3972 sec.\n",
      "Iter 4000 || Loss: 5.0002 || 10iter: 3.4392 sec.\n",
      "Iter 4010 || Loss: 4.8334 || 10iter: 3.3937 sec.\n",
      "Iter 4020 || Loss: 4.8937 || 10iter: 3.4781 sec.\n",
      "Iter 4030 || Loss: 5.1886 || 10iter: 3.4792 sec.\n",
      "Iter 4040 || Loss: 4.8998 || 10iter: 3.5177 sec.\n",
      "Iter 4050 || Loss: 4.9248 || 10iter: 3.3835 sec.\n",
      "Iter 4060 || Loss: 4.8018 || 10iter: 3.3820 sec.\n",
      "Iter 4070 || Loss: 4.8926 || 10iter: 3.3714 sec.\n",
      "Iter 4080 || Loss: 4.6250 || 10iter: 3.3960 sec.\n",
      "Iter 4090 || Loss: 4.6695 || 10iter: 3.4329 sec.\n",
      "Iter 4100 || Loss: 4.7882 || 10iter: 3.4402 sec.\n",
      "Iter 4110 || Loss: 5.0476 || 10iter: 3.3635 sec.\n",
      "Iter 4120 || Loss: 5.2872 || 10iter: 3.4194 sec.\n",
      "Iter 4130 || Loss: 5.1095 || 10iter: 3.3922 sec.\n",
      "Iter 4140 || Loss: 4.7579 || 10iter: 3.3162 sec.\n",
      "-------------\n",
      "epoch 8 || Epoch_TRAIN_Loss:2555.2803 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  177.9926 sec.\n",
      "lr is: 0.001\n",
      "-------------\n",
      "Epoch 9/200\n",
      "-------------\n",
      "(train)\n",
      "Iter 4150 || Loss: 5.3729 || 10iter: 3.8096 sec.\n",
      "Iter 4160 || Loss: 4.6862 || 10iter: 3.4390 sec.\n",
      "Iter 4170 || Loss: 5.0589 || 10iter: 3.3859 sec.\n",
      "Iter 4180 || Loss: 5.2166 || 10iter: 3.3430 sec.\n",
      "Iter 4190 || Loss: 4.7616 || 10iter: 3.4508 sec.\n",
      "Iter 4200 || Loss: 5.0148 || 10iter: 3.4321 sec.\n",
      "Iter 4210 || Loss: 5.1099 || 10iter: 3.3958 sec.\n",
      "Iter 4220 || Loss: 5.6618 || 10iter: 3.4176 sec.\n",
      "Iter 4230 || Loss: 4.9446 || 10iter: 3.4774 sec.\n",
      "Iter 4240 || Loss: 4.3385 || 10iter: 3.5152 sec.\n",
      "Iter 4250 || Loss: 4.5135 || 10iter: 3.4295 sec.\n",
      "Iter 4260 || Loss: 4.9414 || 10iter: 3.3857 sec.\n",
      "Iter 4270 || Loss: 4.5580 || 10iter: 3.3841 sec.\n",
      "Iter 4280 || Loss: 4.9216 || 10iter: 3.3791 sec.\n",
      "Iter 4290 || Loss: 5.3744 || 10iter: 3.3603 sec.\n",
      "Iter 4300 || Loss: 5.1480 || 10iter: 3.3985 sec.\n",
      "Iter 4310 || Loss: 4.5511 || 10iter: 3.3961 sec.\n",
      "Iter 4320 || Loss: 4.4567 || 10iter: 3.3377 sec.\n",
      "Iter 4330 || Loss: 4.4962 || 10iter: 3.3639 sec.\n",
      "Iter 4340 || Loss: 4.5858 || 10iter: 3.4235 sec.\n",
      "Iter 4350 || Loss: 4.7921 || 10iter: 3.4280 sec.\n",
      "Iter 4360 || Loss: 5.0798 || 10iter: 3.4183 sec.\n",
      "Iter 4370 || Loss: 4.7491 || 10iter: 3.3676 sec.\n",
      "Iter 4380 || Loss: 4.3596 || 10iter: 3.3812 sec.\n",
      "Iter 4390 || Loss: 4.9253 || 10iter: 3.4043 sec.\n",
      "Iter 4400 || Loss: 4.6998 || 10iter: 3.4040 sec.\n",
      "Iter 4410 || Loss: 4.8110 || 10iter: 3.4000 sec.\n",
      "Iter 4420 || Loss: 4.7903 || 10iter: 3.3720 sec.\n",
      "Iter 4430 || Loss: 5.0314 || 10iter: 3.3699 sec.\n",
      "Iter 4440 || Loss: 4.5039 || 10iter: 3.3740 sec.\n",
      "Iter 4450 || Loss: 4.9875 || 10iter: 3.4682 sec.\n",
      "Iter 4460 || Loss: 4.7288 || 10iter: 3.5049 sec.\n",
      "Iter 4470 || Loss: 4.3252 || 10iter: 3.3860 sec.\n",
      "Iter 4480 || Loss: 4.1238 || 10iter: 3.3769 sec.\n",
      "Iter 4490 || Loss: 4.5631 || 10iter: 3.4242 sec.\n",
      "Iter 4500 || Loss: 4.6952 || 10iter: 3.4846 sec.\n",
      "Iter 4510 || Loss: 4.5824 || 10iter: 3.5772 sec.\n",
      "Iter 4520 || Loss: 5.2866 || 10iter: 3.5147 sec.\n",
      "Iter 4530 || Loss: 5.3828 || 10iter: 3.3726 sec.\n",
      "Iter 4540 || Loss: 4.5647 || 10iter: 3.4417 sec.\n",
      "Iter 4550 || Loss: 4.7673 || 10iter: 3.3703 sec.\n",
      "Iter 4560 || Loss: 5.1230 || 10iter: 3.3770 sec.\n",
      "Iter 4570 || Loss: 4.2401 || 10iter: 3.4479 sec.\n",
      "Iter 4580 || Loss: 4.5470 || 10iter: 3.3797 sec.\n",
      "Iter 4590 || Loss: 5.0054 || 10iter: 3.3656 sec.\n",
      "Iter 4600 || Loss: 4.7521 || 10iter: 3.3538 sec.\n",
      "Iter 4610 || Loss: 5.0759 || 10iter: 3.3792 sec.\n",
      "Iter 4620 || Loss: 5.0964 || 10iter: 3.4048 sec.\n",
      "Iter 4630 || Loss: 4.9847 || 10iter: 3.3483 sec.\n",
      "Iter 4640 || Loss: 4.5138 || 10iter: 3.3515 sec.\n",
      "Iter 4650 || Loss: 4.9079 || 10iter: 3.3414 sec.\n",
      "Iter 4660 || Loss: 4.3615 || 10iter: 3.3065 sec.\n",
      "-------------\n",
      "epoch 9 || Epoch_TRAIN_Loss:2498.0552 ||Epoch_VAL_Loss:0.0000\n",
      "timer:  177.9608 sec.\n",
      "lr is: 0.001\n",
      "-------------\n",
      "Epoch 10/200\n",
      "-------------\n",
      "(train)\n",
      "Iter 4670 || Loss: 4.9244 || 10iter: 4.6045 sec.\n",
      "Iter 4680 || Loss: 4.8562 || 10iter: 3.5092 sec.\n",
      "Iter 4690 || Loss: 5.1253 || 10iter: 3.4212 sec.\n",
      "Iter 4700 || Loss: 5.1765 || 10iter: 3.3747 sec.\n",
      "Iter 4710 || Loss: 4.3773 || 10iter: 3.3938 sec.\n",
      "Iter 4720 || Loss: 4.5292 || 10iter: 3.4678 sec.\n",
      "Iter 4730 || Loss: 4.7601 || 10iter: 3.3888 sec.\n",
      "Iter 4740 || Loss: 4.6672 || 10iter: 3.3640 sec.\n",
      "Iter 4750 || Loss: 4.8667 || 10iter: 3.4133 sec.\n",
      "Iter 4760 || Loss: 4.7417 || 10iter: 3.3557 sec.\n",
      "Iter 4770 || Loss: 4.9857 || 10iter: 3.3633 sec.\n",
      "Iter 4780 || Loss: 4.7186 || 10iter: 3.3536 sec.\n",
      "Iter 4790 || Loss: 4.6322 || 10iter: 3.3493 sec.\n",
      "Iter 4800 || Loss: 5.1384 || 10iter: 3.3689 sec.\n",
      "Iter 4810 || Loss: 4.6204 || 10iter: 3.3918 sec.\n",
      "Iter 4820 || Loss: 4.2610 || 10iter: 3.3528 sec.\n",
      "Iter 4830 || Loss: 4.2542 || 10iter: 3.3506 sec.\n",
      "Iter 4840 || Loss: 4.6505 || 10iter: 3.3628 sec.\n",
      "Iter 4850 || Loss: 4.8179 || 10iter: 3.3586 sec.\n",
      "Iter 4860 || Loss: 5.1188 || 10iter: 3.3525 sec.\n",
      "Iter 4870 || Loss: 4.8651 || 10iter: 3.3730 sec.\n",
      "Iter 4880 || Loss: 4.5491 || 10iter: 3.4543 sec.\n",
      "Iter 4890 || Loss: 4.9177 || 10iter: 3.3561 sec.\n",
      "Iter 4900 || Loss: 4.7518 || 10iter: 3.3450 sec.\n",
      "Iter 4910 || Loss: 4.7989 || 10iter: 3.3465 sec.\n",
      "Iter 4920 || Loss: 4.4850 || 10iter: 3.3628 sec.\n",
      "Iter 4930 || Loss: 4.9113 || 10iter: 3.3797 sec.\n",
      "Iter 4940 || Loss: 5.1727 || 10iter: 3.3825 sec.\n",
      "Iter 4950 || Loss: 4.1367 || 10iter: 3.3871 sec.\n",
      "Iter 4960 || Loss: 4.7088 || 10iter: 3.4134 sec.\n",
      "Iter 4970 || Loss: 4.6547 || 10iter: 3.3420 sec.\n",
      "Iter 4980 || Loss: 5.4347 || 10iter: 3.3918 sec.\n",
      "Iter 4990 || Loss: 5.0373 || 10iter: 3.3766 sec.\n",
      "Iter 5000 || Loss: 4.9320 || 10iter: 3.4926 sec.\n",
      "Iter 5010 || Loss: 4.8331 || 10iter: 3.4114 sec.\n",
      "Iter 5020 || Loss: 4.8247 || 10iter: 3.3540 sec.\n",
      "Iter 5030 || Loss: 4.6116 || 10iter: 3.3679 sec.\n",
      "Iter 5040 || Loss: 4.7980 || 10iter: 3.3538 sec.\n",
      "Iter 5050 || Loss: 4.8942 || 10iter: 3.3990 sec.\n",
      "Iter 5060 || Loss: 5.2979 || 10iter: 3.3871 sec.\n",
      "Iter 5070 || Loss: 4.9388 || 10iter: 3.3548 sec.\n",
      "Iter 5080 || Loss: 4.8695 || 10iter: 3.4696 sec.\n",
      "Iter 5090 || Loss: 4.5920 || 10iter: 3.3879 sec.\n",
      "Iter 5100 || Loss: 5.0763 || 10iter: 3.3522 sec.\n",
      "Iter 5110 || Loss: 4.5290 || 10iter: 3.3437 sec.\n",
      "Iter 5120 || Loss: 4.7130 || 10iter: 3.3729 sec.\n",
      "Iter 5130 || Loss: 4.8790 || 10iter: 3.4069 sec.\n",
      "Iter 5140 || Loss: 5.0792 || 10iter: 3.3834 sec.\n",
      "Iter 5150 || Loss: 5.0664 || 10iter: 3.3826 sec.\n",
      "Iter 5160 || Loss: 5.2099 || 10iter: 3.3813 sec.\n",
      "Iter 5170 || Loss: 4.9749 || 10iter: 3.3163 sec.\n",
      "Iter 5180 || Loss: 6.1031 || 10iter: 3.0821 sec.\n",
      "-------------\n",
      "(val)\n",
      "-------------\n",
      "epoch 10 || Epoch_TRAIN_Loss:2481.6381 ||Epoch_VAL_Loss:721.7942\n",
      "timer:  205.4946 sec.\n",
      "lr is: 0.001\n",
      "-------------\n",
      "Epoch 11/200\n",
      "-------------\n",
      "(train)\n",
      "Iter 5190 || Loss: 4.4229 || 10iter: 5.4113 sec.\n",
      "Iter 5200 || Loss: 5.0464 || 10iter: 3.3802 sec.\n",
      "Iter 5210 || Loss: 4.7616 || 10iter: 3.3827 sec.\n",
      "Iter 5220 || Loss: 4.9551 || 10iter: 3.3709 sec.\n",
      "Iter 5230 || Loss: 5.0204 || 10iter: 3.3988 sec.\n",
      "Iter 5240 || Loss: 4.4892 || 10iter: 3.3844 sec.\n",
      "Iter 5250 || Loss: 5.0293 || 10iter: 3.4037 sec.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Iter 5260 || Loss: 4.4406 || 10iter: 3.4452 sec.\n",
      "Iter 5270 || Loss: 4.4373 || 10iter: 3.3466 sec.\n",
      "Iter 5280 || Loss: 4.5616 || 10iter: 3.4416 sec.\n",
      "Iter 5290 || Loss: 4.5757 || 10iter: 3.3746 sec.\n",
      "Iter 5300 || Loss: 4.7119 || 10iter: 3.3633 sec.\n",
      "Iter 5310 || Loss: 4.4746 || 10iter: 3.4697 sec.\n",
      "Iter 5320 || Loss: 4.2691 || 10iter: 3.4707 sec.\n",
      "Iter 5330 || Loss: 4.8476 || 10iter: 3.4119 sec.\n",
      "Iter 5340 || Loss: 5.0027 || 10iter: 3.3624 sec.\n",
      "Iter 5350 || Loss: 4.7545 || 10iter: 3.3863 sec.\n",
      "Iter 5360 || Loss: 4.8939 || 10iter: 3.4075 sec.\n",
      "Iter 5370 || Loss: 4.5954 || 10iter: 3.3554 sec.\n",
      "Iter 5380 || Loss: 4.3597 || 10iter: 3.3572 sec.\n"
     ]
    }
   ],
   "source": [
    "if DATASET == \"COCO\":\n",
    "    num_epochs = 200\n",
    "else:\n",
    "    num_epochs = 200\n",
    "    \n",
    "train_model(net, dataloaders_dict, criterion, optimizer, num_epochs=num_epochs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}