GAT-sample-tunable-products_gpu7.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using backend: pytorch\n"
     ]
    }
   ],
   "source": [
    "import dgl\n",
    "import time\n",
    "import torch as th\n",
    "import numpy as np\n",
    "import torch.nn.functional as F\n",
    "\n",
    "from load_data import load_ogb\n",
    "from model import SAGE, GAT, get_eigen\n",
    "from train_eval import train_ft_embed"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This notebook shows the performance of GAT on the OGBN-products graph with trainable embeddings on the nodes. We use normal distribution to initialize the node embeddings."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:root:The OGB package is out of date. Your version is 1.3.1, while the latest version is 1.3.2.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "load ogbn-products\n",
      "finish loading ogbn-products\n",
      "finish constructing ogbn-products\n",
      "embedding is in  cuda:7\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.6/dist-packages/numpy/core/fromnumeric.py:3373: RuntimeWarning: Mean of empty slice.\n",
      "  out=out, **kwargs)\n",
      "/usr/local/lib/python3.6/dist-packages/numpy/core/_methods.py:170: RuntimeWarning: invalid value encountered in double_scalars\n",
      "  ret = ret.dtype.type(ret / rcount)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 00000 | Step 00000 | Loss 3.7220 | Train Acc 0.1406 | Speed (samples/sec) nan | GPU 4507.8 MB\n",
      "Epoch 00000 | Step 00020 | Loss 2.4933 | Train Acc 0.3086 | Speed (samples/sec) 253.3789 | GPU 6381.6 MB\n",
      "Epoch 00000 | Step 00040 | Loss 2.3375 | Train Acc 0.3086 | Speed (samples/sec) 249.5190 | GPU 6381.6 MB\n",
      "Epoch 00000 | Step 00060 | Loss 1.7505 | Train Acc 0.4668 | Speed (samples/sec) 248.1305 | GPU 6381.6 MB\n",
      "Epoch 00000 | Step 00080 | Loss 1.4110 | Train Acc 0.6367 | Speed (samples/sec) 246.4428 | GPU 6387.7 MB\n",
      "Epoch 00000 | Step 00100 | Loss 0.9202 | Train Acc 0.7617 | Speed (samples/sec) 246.9268 | GPU 6387.7 MB\n",
      "Epoch 00000 | Step 00120 | Loss 0.8583 | Train Acc 0.7773 | Speed (samples/sec) 250.3664 | GPU 6387.7 MB\n",
      "Epoch 00000 | Step 00140 | Loss 0.7290 | Train Acc 0.8184 | Speed (samples/sec) 252.3024 | GPU 6387.7 MB\n",
      "Epoch 00000 | Step 00160 | Loss 0.7815 | Train Acc 0.7910 | Speed (samples/sec) 253.0274 | GPU 6387.7 MB\n",
      "Epoch 00000 | Step 00180 | Loss 0.6294 | Train Acc 0.8359 | Speed (samples/sec) 252.2527 | GPU 6396.6 MB\n",
      "Epoch 00000 | Step 00200 | Loss 0.4897 | Train Acc 0.8906 | Speed (samples/sec) 252.2519 | GPU 6396.6 MB\n",
      "Epoch 00000 | Step 00220 | Loss 0.6239 | Train Acc 0.8477 | Speed (samples/sec) 252.5422 | GPU 6396.6 MB\n",
      "Epoch 00000 | Step 00240 | Loss 0.6610 | Train Acc 0.8379 | Speed (samples/sec) 252.1680 | GPU 6396.6 MB\n",
      "Epoch 00000 | Step 00260 | Loss 0.5198 | Train Acc 0.8809 | Speed (samples/sec) 252.4857 | GPU 6396.6 MB\n",
      "Epoch 00000 | Step 00280 | Loss 0.4283 | Train Acc 0.9023 | Speed (samples/sec) 254.2285 | GPU 6396.6 MB\n",
      "Epoch 00000 | Step 00300 | Loss 0.5048 | Train Acc 0.8711 | Speed (samples/sec) 254.3488 | GPU 6396.6 MB\n",
      "Epoch 00000 | Step 00320 | Loss 0.4608 | Train Acc 0.8926 | Speed (samples/sec) 253.8415 | GPU 6396.6 MB\n",
      "Epoch 00000 | Step 00340 | Loss 0.5865 | Train Acc 0.8770 | Speed (samples/sec) 254.1349 | GPU 6396.6 MB\n",
      "Epoch 00000 | Step 00360 | Loss 0.4755 | Train Acc 0.8750 | Speed (samples/sec) 253.9891 | GPU 6397.5 MB\n",
      "Epoch 00000 | Step 00380 | Loss 0.4548 | Train Acc 0.8848 | Speed (samples/sec) 253.9761 | GPU 6397.5 MB\n",
      "Epoch Time(s): 780.5879\n",
      "Epoch 00001 | Step 00000 | Loss 0.5089 | Train Acc 0.8828 | Speed (samples/sec) 253.4461 | GPU 6397.5 MB\n",
      "Epoch 00001 | Step 00020 | Loss 0.4278 | Train Acc 0.8828 | Speed (samples/sec) 253.6759 | GPU 6397.5 MB\n",
      "Epoch 00001 | Step 00040 | Loss 0.4778 | Train Acc 0.8789 | Speed (samples/sec) 253.7120 | GPU 6397.5 MB\n",
      "Epoch 00001 | Step 00060 | Loss 0.4616 | Train Acc 0.8789 | Speed (samples/sec) 253.2426 | GPU 6397.5 MB\n",
      "Epoch 00001 | Step 00080 | Loss 0.4142 | Train Acc 0.8965 | Speed (samples/sec) 252.8650 | GPU 6397.5 MB\n",
      "Epoch 00001 | Step 00100 | Loss 0.4124 | Train Acc 0.8809 | Speed (samples/sec) 252.4918 | GPU 6397.5 MB\n",
      "Epoch 00001 | Step 00120 | Loss 0.4611 | Train Acc 0.8672 | Speed (samples/sec) 252.2657 | GPU 6397.5 MB\n",
      "Epoch 00001 | Step 00140 | Loss 0.3417 | Train Acc 0.9199 | Speed (samples/sec) 252.5703 | GPU 6397.5 MB\n",
      "Epoch 00001 | Step 00160 | Loss 0.3748 | Train Acc 0.8926 | Speed (samples/sec) 253.3551 | GPU 6397.5 MB\n",
      "Epoch 00001 | Step 00180 | Loss 0.4099 | Train Acc 0.8789 | Speed (samples/sec) 253.5729 | GPU 6397.5 MB\n",
      "Epoch 00001 | Step 00200 | Loss 0.4253 | Train Acc 0.8887 | Speed (samples/sec) 253.1875 | GPU 6397.5 MB\n",
      "Epoch 00001 | Step 00220 | Loss 0.4596 | Train Acc 0.8730 | Speed (samples/sec) 252.9889 | GPU 6397.5 MB\n",
      "Epoch 00001 | Step 00240 | Loss 0.3484 | Train Acc 0.8887 | Speed (samples/sec) 253.4790 | GPU 6397.5 MB\n",
      "Epoch 00001 | Step 00260 | Loss 0.3665 | Train Acc 0.9102 | Speed (samples/sec) 254.2829 | GPU 6397.5 MB\n",
      "Epoch 00001 | Step 00280 | Loss 0.4574 | Train Acc 0.8770 | Speed (samples/sec) 255.4118 | GPU 6397.5 MB\n",
      "Epoch 00001 | Step 00300 | Loss 0.4980 | Train Acc 0.8730 | Speed (samples/sec) 256.0385 | GPU 6397.5 MB\n",
      "Epoch 00001 | Step 00320 | Loss 0.4978 | Train Acc 0.8809 | Speed (samples/sec) 256.2364 | GPU 6397.5 MB\n",
      "Epoch 00001 | Step 00340 | Loss 0.3776 | Train Acc 0.8945 | Speed (samples/sec) 256.7429 | GPU 6397.5 MB\n",
      "Epoch 00001 | Step 00360 | Loss 0.4049 | Train Acc 0.8906 | Speed (samples/sec) 257.1268 | GPU 6397.5 MB\n",
      "Epoch 00001 | Step 00380 | Loss 0.4495 | Train Acc 0.8828 | Speed (samples/sec) 257.6539 | GPU 6397.5 MB\n",
      "Epoch Time(s): 755.8064\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 245/245 [00:49<00:00,  4.91it/s]\n",
      "100%|██████████| 245/245 [02:06<00:00,  1.94it/s]\n",
      "100%|██████████| 245/245 [01:56<00:00,  2.10it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Val acc 0.8969, Test Acc: 0.7135\n",
      "Best val acc 0.8969, best test acc: 0.7135\n",
      "Epoch 00002 | Step 00000 | Loss 0.4232 | Train Acc 0.8965 | Speed (samples/sec) 257.4669 | GPU 6397.5 MB\n",
      "Epoch 00002 | Step 00020 | Loss 0.3260 | Train Acc 0.9102 | Speed (samples/sec) 258.0625 | GPU 6397.5 MB\n",
      "Epoch 00002 | Step 00040 | Loss 0.3407 | Train Acc 0.9102 | Speed (samples/sec) 258.6595 | GPU 6397.5 MB\n",
      "Epoch 00002 | Step 00060 | Loss 0.4655 | Train Acc 0.8691 | Speed (samples/sec) 258.7823 | GPU 6397.5 MB\n",
      "Epoch 00002 | Step 00080 | Loss 0.5067 | Train Acc 0.8770 | Speed (samples/sec) 258.8602 | GPU 6397.5 MB\n",
      "Epoch 00002 | Step 00100 | Loss 0.4710 | Train Acc 0.8594 | Speed (samples/sec) 258.9365 | GPU 6397.5 MB\n",
      "Epoch 00002 | Step 00120 | Loss 0.3879 | Train Acc 0.9023 | Speed (samples/sec) 259.3660 | GPU 6397.5 MB\n",
      "Epoch 00002 | Step 00140 | Loss 0.4770 | Train Acc 0.8574 | Speed (samples/sec) 259.7692 | GPU 6397.5 MB\n",
      "Epoch 00002 | Step 00160 | Loss 0.3761 | Train Acc 0.8848 | Speed (samples/sec) 260.1503 | GPU 6397.5 MB\n",
      "Epoch 00002 | Step 00180 | Loss 0.4122 | Train Acc 0.9004 | Speed (samples/sec) 260.7015 | GPU 6397.5 MB\n",
      "Epoch 00002 | Step 00200 | Loss 0.5011 | Train Acc 0.8672 | Speed (samples/sec) 260.7502 | GPU 6397.5 MB\n",
      "Epoch 00002 | Step 00220 | Loss 0.3770 | Train Acc 0.9004 | Speed (samples/sec) 260.8072 | GPU 6397.5 MB\n",
      "Epoch 00002 | Step 00240 | Loss 0.4047 | Train Acc 0.8887 | Speed (samples/sec) 260.8626 | GPU 6397.5 MB\n",
      "Epoch 00002 | Step 00260 | Loss 0.3232 | Train Acc 0.9121 | Speed (samples/sec) 261.2066 | GPU 6397.5 MB\n",
      "Epoch 00002 | Step 00280 | Loss 0.3842 | Train Acc 0.8965 | Speed (samples/sec) 261.5886 | GPU 6397.5 MB\n",
      "Epoch 00002 | Step 00300 | Loss 0.3927 | Train Acc 0.9062 | Speed (samples/sec) 262.0454 | GPU 6397.5 MB\n",
      "Epoch 00002 | Step 00320 | Loss 0.3066 | Train Acc 0.9199 | Speed (samples/sec) 262.0984 | GPU 6397.5 MB\n",
      "Epoch 00002 | Step 00340 | Loss 0.3715 | Train Acc 0.8887 | Speed (samples/sec) 262.3513 | GPU 6397.5 MB\n",
      "Epoch 00002 | Step 00360 | Loss 0.3272 | Train Acc 0.9258 | Speed (samples/sec) 262.5601 | GPU 6397.5 MB\n",
      "Epoch 00002 | Step 00380 | Loss 0.3496 | Train Acc 0.9004 | Speed (samples/sec) 262.7921 | GPU 6397.5 MB\n",
      "Epoch Time(s): 722.1125\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 245/245 [00:50<00:00,  4.84it/s]\n",
      "100%|██████████| 245/245 [02:09<00:00,  1.89it/s]\n",
      "100%|██████████| 245/245 [02:00<00:00,  2.03it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Val acc 0.8947, Test Acc: 0.7382\n",
      "Epoch 00003 | Step 00000 | Loss 0.4675 | Train Acc 0.8691 | Speed (samples/sec) 262.7564 | GPU 6397.5 MB\n",
      "Epoch 00003 | Step 00020 | Loss 0.3568 | Train Acc 0.8887 | Speed (samples/sec) 262.9540 | GPU 6397.5 MB\n",
      "Epoch 00003 | Step 00040 | Loss 0.4195 | Train Acc 0.8770 | Speed (samples/sec) 263.3997 | GPU 6397.5 MB\n",
      "Epoch 00003 | Step 00060 | Loss 0.3408 | Train Acc 0.9062 | Speed (samples/sec) 263.7645 | GPU 6397.5 MB\n",
      "Epoch 00003 | Step 00080 | Loss 0.2944 | Train Acc 0.9102 | Speed (samples/sec) 264.1420 | GPU 6402.6 MB\n",
      "Epoch 00003 | Step 00100 | Loss 0.4137 | Train Acc 0.8945 | Speed (samples/sec) 264.1047 | GPU 6402.6 MB\n",
      "Epoch 00003 | Step 00120 | Loss 0.3667 | Train Acc 0.8926 | Speed (samples/sec) 264.2923 | GPU 6402.6 MB\n",
      "Epoch 00003 | Step 00140 | Loss 0.3623 | Train Acc 0.9004 | Speed (samples/sec) 264.4167 | GPU 6402.6 MB\n",
      "Epoch 00003 | Step 00160 | Loss 0.4541 | Train Acc 0.8887 | Speed (samples/sec) 264.5469 | GPU 6402.6 MB\n",
      "Epoch 00003 | Step 00180 | Loss 0.4007 | Train Acc 0.8848 | Speed (samples/sec) 264.9811 | GPU 6402.6 MB\n",
      "Epoch 00003 | Step 00200 | Loss 0.3435 | Train Acc 0.9043 | Speed (samples/sec) 265.3656 | GPU 6402.6 MB\n",
      "Epoch 00003 | Step 00220 | Loss 0.3282 | Train Acc 0.9023 | Speed (samples/sec) 265.5526 | GPU 6402.6 MB\n",
      "Epoch 00003 | Step 00240 | Loss 0.3636 | Train Acc 0.9062 | Speed (samples/sec) 265.8226 | GPU 6402.6 MB\n",
      "Epoch 00003 | Step 00260 | Loss 0.4260 | Train Acc 0.8867 | Speed (samples/sec) 266.0581 | GPU 6402.6 MB\n",
      "Epoch 00003 | Step 00280 | Loss 0.3643 | Train Acc 0.9102 | Speed (samples/sec) 266.2231 | GPU 6402.6 MB\n",
      "Epoch 00003 | Step 00300 | Loss 0.3683 | Train Acc 0.9023 | Speed (samples/sec) 266.3277 | GPU 6402.6 MB\n",
      "Epoch 00003 | Step 00320 | Loss 0.3222 | Train Acc 0.9004 | Speed (samples/sec) 266.6739 | GPU 6402.6 MB\n",
      "Epoch 00003 | Step 00340 | Loss 0.3652 | Train Acc 0.8965 | Speed (samples/sec) 266.9779 | GPU 6402.6 MB\n",
      "Epoch 00003 | Step 00360 | Loss 0.3024 | Train Acc 0.9160 | Speed (samples/sec) 267.2272 | GPU 6402.6 MB\n",
      "Epoch 00003 | Step 00380 | Loss 0.3832 | Train Acc 0.8945 | Speed (samples/sec) 267.3597 | GPU 6402.6 MB\n",
      "Epoch Time(s): 703.2283\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 245/245 [00:51<00:00,  4.73it/s]\n",
      "100%|██████████| 245/245 [02:01<00:00,  2.02it/s]\n",
      "100%|██████████| 245/245 [01:52<00:00,  2.18it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Val acc 0.9077, Test Acc: 0.7508\n",
      "Best val acc 0.9077, best test acc: 0.7508\n",
      "Epoch 00004 | Step 00000 | Loss 0.3737 | Train Acc 0.8926 | Speed (samples/sec) 267.2578 | GPU 6402.6 MB\n",
      "Epoch 00004 | Step 00020 | Loss 0.3937 | Train Acc 0.8965 | Speed (samples/sec) 267.3898 | GPU 6402.6 MB\n",
      "Epoch 00004 | Step 00040 | Loss 0.3262 | Train Acc 0.9062 | Speed (samples/sec) 267.5643 | GPU 6402.6 MB\n",
      "Epoch 00004 | Step 00060 | Loss 0.3714 | Train Acc 0.9004 | Speed (samples/sec) 267.8053 | GPU 6402.6 MB\n",
      "Epoch 00004 | Step 00080 | Loss 0.3108 | Train Acc 0.9160 | Speed (samples/sec) 268.2032 | GPU 6402.6 MB\n",
      "Epoch 00004 | Step 00100 | Loss 0.3242 | Train Acc 0.9180 | Speed (samples/sec) 268.5817 | GPU 6402.6 MB\n",
      "Epoch 00004 | Step 00120 | Loss 0.3009 | Train Acc 0.9219 | Speed (samples/sec) 268.6528 | GPU 6402.6 MB\n",
      "Epoch 00004 | Step 00140 | Loss 0.3538 | Train Acc 0.9062 | Speed (samples/sec) 268.7366 | GPU 6402.6 MB\n",
      "Epoch 00004 | Step 00160 | Loss 0.2976 | Train Acc 0.9160 | Speed (samples/sec) 268.9149 | GPU 6402.6 MB\n",
      "Epoch 00004 | Step 00180 | Loss 0.2964 | Train Acc 0.9121 | Speed (samples/sec) 269.0775 | GPU 6402.6 MB\n",
      "Epoch 00004 | Step 00200 | Loss 0.2735 | Train Acc 0.9277 | Speed (samples/sec) 269.5891 | GPU 6402.6 MB\n",
      "Epoch 00004 | Step 00220 | Loss 0.3517 | Train Acc 0.9160 | Speed (samples/sec) 270.0517 | GPU 6402.6 MB\n",
      "Epoch 00004 | Step 00240 | Loss 0.3719 | Train Acc 0.9062 | Speed (samples/sec) 270.3440 | GPU 6414.4 MB\n",
      "Epoch 00004 | Step 00260 | Loss 0.3089 | Train Acc 0.9141 | Speed (samples/sec) 270.6344 | GPU 6414.4 MB\n",
      "Epoch 00004 | Step 00280 | Loss 0.3945 | Train Acc 0.8789 | Speed (samples/sec) 270.8956 | GPU 6414.4 MB\n",
      "Epoch 00004 | Step 00300 | Loss 0.2878 | Train Acc 0.9180 | Speed (samples/sec) 270.9254 | GPU 6414.4 MB\n",
      "Epoch 00004 | Step 00320 | Loss 0.3326 | Train Acc 0.9160 | Speed (samples/sec) 270.9975 | GPU 6414.4 MB\n",
      "Epoch 00004 | Step 00340 | Loss 0.2952 | Train Acc 0.9141 | Speed (samples/sec) 271.2231 | GPU 6414.4 MB\n",
      "Epoch 00004 | Step 00360 | Loss 0.2887 | Train Acc 0.9023 | Speed (samples/sec) 271.4486 | GPU 6414.4 MB\n",
      "Epoch 00004 | Step 00380 | Loss 0.3346 | Train Acc 0.9062 | Speed (samples/sec) 271.6695 | GPU 6414.4 MB\n",
      "Epoch Time(s): 684.5114\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 245/245 [00:49<00:00,  5.00it/s]\n",
      "100%|██████████| 245/245 [02:02<00:00,  2.00it/s]\n",
      "100%|██████████| 245/245 [01:47<00:00,  2.27it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Val acc 0.9102, Test Acc: 0.7645\n",
      "Best val acc 0.9102, best test acc: 0.7645\n",
      "Epoch 00005 | Step 00000 | Loss 0.3041 | Train Acc 0.9023 | Speed (samples/sec) 271.5600 | GPU 6414.4 MB\n",
      "Epoch 00005 | Step 00020 | Loss 0.2962 | Train Acc 0.9102 | Speed (samples/sec) 271.6187 | GPU 6414.4 MB\n",
      "Epoch 00005 | Step 00040 | Loss 0.2875 | Train Acc 0.9180 | Speed (samples/sec) 271.6212 | GPU 6414.4 MB\n",
      "Epoch 00005 | Step 00060 | Loss 0.3074 | Train Acc 0.9180 | Speed (samples/sec) 271.7785 | GPU 6414.4 MB\n",
      "Epoch 00005 | Step 00080 | Loss 0.2864 | Train Acc 0.9199 | Speed (samples/sec) 272.1591 | GPU 6414.4 MB\n",
      "Epoch 00005 | Step 00100 | Loss 0.3468 | Train Acc 0.8984 | Speed (samples/sec) 272.4950 | GPU 6414.4 MB\n",
      "Epoch 00005 | Step 00120 | Loss 0.2241 | Train Acc 0.9199 | Speed (samples/sec) 272.7156 | GPU 6414.4 MB\n",
      "Epoch 00005 | Step 00140 | Loss 0.2473 | Train Acc 0.9180 | Speed (samples/sec) 272.8685 | GPU 6414.4 MB\n",
      "Epoch 00005 | Step 00160 | Loss 0.3000 | Train Acc 0.9180 | Speed (samples/sec) 273.0355 | GPU 6414.4 MB\n",
      "Epoch 00005 | Step 00180 | Loss 0.3176 | Train Acc 0.9121 | Speed (samples/sec) 273.2381 | GPU 6414.4 MB\n",
      "Epoch 00005 | Step 00200 | Loss 0.3615 | Train Acc 0.9023 | Speed (samples/sec) 273.3268 | GPU 6414.4 MB\n",
      "Epoch 00005 | Step 00220 | Loss 0.3201 | Train Acc 0.8984 | Speed (samples/sec) 273.5594 | GPU 6414.4 MB\n",
      "Epoch 00005 | Step 00240 | Loss 0.3918 | Train Acc 0.8770 | Speed (samples/sec) 273.8704 | GPU 6414.4 MB\n",
      "Epoch 00005 | Step 00260 | Loss 0.3960 | Train Acc 0.8770 | Speed (samples/sec) 274.0848 | GPU 6414.4 MB\n",
      "Epoch 00005 | Step 00280 | Loss 0.3682 | Train Acc 0.9102 | Speed (samples/sec) 274.1437 | GPU 6414.4 MB\n",
      "Epoch 00005 | Step 00300 | Loss 0.4151 | Train Acc 0.8809 | Speed (samples/sec) 274.2197 | GPU 6414.4 MB\n",
      "Epoch 00005 | Step 00320 | Loss 0.3943 | Train Acc 0.9062 | Speed (samples/sec) 274.2298 | GPU 6414.4 MB\n",
      "Epoch 00005 | Step 00340 | Loss 0.3427 | Train Acc 0.9043 | Speed (samples/sec) 274.3079 | GPU 6414.4 MB\n",
      "Epoch 00005 | Step 00360 | Loss 0.3162 | Train Acc 0.8984 | Speed (samples/sec) 274.5368 | GPU 6414.4 MB\n",
      "Epoch 00005 | Step 00380 | Loss 0.2980 | Train Acc 0.9102 | Speed (samples/sec) 274.7981 | GPU 6414.4 MB\n",
      "Epoch Time(s): 679.8879\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 245/245 [00:47<00:00,  5.15it/s]\n",
      "100%|██████████| 245/245 [02:09<00:00,  1.90it/s]\n",
      "100%|██████████| 245/245 [01:47<00:00,  2.27it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Val acc 0.9108, Test Acc: 0.7637\n",
      "Best val acc 0.9108, best test acc: 0.7637\n",
      "Epoch 00006 | Step 00000 | Loss 0.3293 | Train Acc 0.8984 | Speed (samples/sec) 274.7432 | GPU 6414.4 MB\n",
      "Epoch 00006 | Step 00020 | Loss 0.3243 | Train Acc 0.9082 | Speed (samples/sec) 274.7052 | GPU 6414.4 MB\n",
      "Epoch 00006 | Step 00040 | Loss 0.3129 | Train Acc 0.9102 | Speed (samples/sec) 274.6568 | GPU 6414.4 MB\n",
      "Epoch 00006 | Step 00060 | Loss 0.2977 | Train Acc 0.9102 | Speed (samples/sec) 274.5863 | GPU 6414.4 MB\n",
      "Epoch 00006 | Step 00080 | Loss 0.3245 | Train Acc 0.9121 | Speed (samples/sec) 274.6271 | GPU 6414.4 MB\n",
      "Epoch 00006 | Step 00100 | Loss 0.3605 | Train Acc 0.8926 | Speed (samples/sec) 274.8595 | GPU 6414.4 MB\n",
      "Epoch 00006 | Step 00120 | Loss 0.3675 | Train Acc 0.8867 | Speed (samples/sec) 275.0772 | GPU 6414.4 MB\n",
      "Epoch 00006 | Step 00140 | Loss 0.2882 | Train Acc 0.9238 | Speed (samples/sec) 275.1975 | GPU 6414.4 MB\n",
      "Epoch 00006 | Step 00160 | Loss 0.3044 | Train Acc 0.9062 | Speed (samples/sec) 275.2277 | GPU 6414.4 MB\n",
      "Epoch 00006 | Step 00180 | Loss 0.2842 | Train Acc 0.9023 | Speed (samples/sec) 275.2934 | GPU 6414.4 MB\n",
      "Epoch 00006 | Step 00200 | Loss 0.3869 | Train Acc 0.8789 | Speed (samples/sec) 275.3696 | GPU 6414.4 MB\n",
      "Epoch 00006 | Step 00220 | Loss 0.3919 | Train Acc 0.8984 | Speed (samples/sec) 275.4757 | GPU 6414.4 MB\n",
      "Epoch 00006 | Step 00240 | Loss 0.4089 | Train Acc 0.8926 | Speed (samples/sec) 275.6534 | GPU 6414.4 MB\n",
      "Epoch 00006 | Step 00260 | Loss 0.2766 | Train Acc 0.9062 | Speed (samples/sec) 275.8368 | GPU 6414.4 MB\n",
      "Epoch 00006 | Step 00280 | Loss 0.2531 | Train Acc 0.9297 | Speed (samples/sec) 275.9315 | GPU 6414.4 MB\n",
      "Epoch 00006 | Step 00300 | Loss 0.3688 | Train Acc 0.8906 | Speed (samples/sec) 275.9470 | GPU 6414.4 MB\n",
      "Epoch 00006 | Step 00320 | Loss 0.3188 | Train Acc 0.9062 | Speed (samples/sec) 275.9551 | GPU 6414.4 MB\n",
      "Epoch 00006 | Step 00340 | Loss 0.3951 | Train Acc 0.8945 | Speed (samples/sec) 275.9100 | GPU 6414.4 MB\n",
      "Epoch 00006 | Step 00360 | Loss 0.3330 | Train Acc 0.9004 | Speed (samples/sec) 275.9905 | GPU 6414.4 MB\n",
      "Epoch 00006 | Step 00380 | Loss 0.2932 | Train Acc 0.9316 | Speed (samples/sec) 276.2104 | GPU 6414.4 MB\n",
      "Epoch Time(s): 694.2125\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 245/245 [00:45<00:00,  5.40it/s]\n",
      "100%|██████████| 245/245 [02:05<00:00,  1.95it/s]\n",
      "100%|██████████| 245/245 [01:49<00:00,  2.24it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Val acc 0.9126, Test Acc: 0.7633\n",
      "Best val acc 0.9126, best test acc: 0.7633\n",
      "Epoch 00007 | Step 00000 | Loss 0.2298 | Train Acc 0.9277 | Speed (samples/sec) 276.1750 | GPU 6414.4 MB\n",
      "Epoch 00007 | Step 00020 | Loss 0.2639 | Train Acc 0.9121 | Speed (samples/sec) 276.3327 | GPU 6414.4 MB\n",
      "Epoch 00007 | Step 00040 | Loss 0.3588 | Train Acc 0.8867 | Speed (samples/sec) 276.3802 | GPU 6414.4 MB\n",
      "Epoch 00007 | Step 00060 | Loss 0.2810 | Train Acc 0.9199 | Speed (samples/sec) 276.8304 | GPU 6414.4 MB\n",
      "Epoch 00007 | Step 00080 | Loss 0.2961 | Train Acc 0.9141 | Speed (samples/sec) 277.1763 | GPU 6414.4 MB\n",
      "Epoch 00007 | Step 00100 | Loss 0.3274 | Train Acc 0.9062 | Speed (samples/sec) 277.4207 | GPU 6414.4 MB\n",
      "Epoch 00007 | Step 00120 | Loss 0.3346 | Train Acc 0.8945 | Speed (samples/sec) 277.7155 | GPU 6414.4 MB\n",
      "Epoch 00007 | Step 00140 | Loss 0.3272 | Train Acc 0.8945 | Speed (samples/sec) 277.9732 | GPU 6414.4 MB\n",
      "Epoch 00007 | Step 00160 | Loss 0.2390 | Train Acc 0.9199 | Speed (samples/sec) 278.2446 | GPU 6414.4 MB\n",
      "Epoch 00007 | Step 00180 | Loss 0.2767 | Train Acc 0.9180 | Speed (samples/sec) 278.5476 | GPU 6414.4 MB\n",
      "Epoch 00007 | Step 00200 | Loss 0.3712 | Train Acc 0.8984 | Speed (samples/sec) 278.7814 | GPU 6414.4 MB\n",
      "Epoch 00007 | Step 00220 | Loss 0.3564 | Train Acc 0.8945 | Speed (samples/sec) 279.0353 | GPU 6414.4 MB\n",
      "Epoch 00007 | Step 00240 | Loss 0.4013 | Train Acc 0.8867 | Speed (samples/sec) 279.2313 | GPU 6414.4 MB\n",
      "Epoch 00007 | Step 00260 | Loss 0.2922 | Train Acc 0.9121 | Speed (samples/sec) 279.5797 | GPU 6414.4 MB\n",
      "Epoch 00007 | Step 00280 | Loss 0.2581 | Train Acc 0.9199 | Speed (samples/sec) 279.8321 | GPU 6414.4 MB\n",
      "Epoch 00007 | Step 00300 | Loss 0.2853 | Train Acc 0.9180 | Speed (samples/sec) 280.0733 | GPU 6414.4 MB\n",
      "Epoch 00007 | Step 00320 | Loss 0.3304 | Train Acc 0.9023 | Speed (samples/sec) 280.3582 | GPU 6414.4 MB\n",
      "Epoch 00007 | Step 00340 | Loss 0.2166 | Train Acc 0.9395 | Speed (samples/sec) 280.6272 | GPU 6414.4 MB\n",
      "Epoch 00007 | Step 00360 | Loss 0.3348 | Train Acc 0.9121 | Speed (samples/sec) 280.8154 | GPU 6414.4 MB\n",
      "Epoch 00007 | Step 00380 | Loss 0.2837 | Train Acc 0.9219 | Speed (samples/sec) 281.0471 | GPU 6414.4 MB\n",
      "Epoch Time(s): 626.8697\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 245/245 [00:42<00:00,  5.77it/s]\n",
      "100%|██████████| 245/245 [01:57<00:00,  2.08it/s]\n",
      "100%|██████████| 245/245 [01:47<00:00,  2.27it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Val acc 0.9145, Test Acc: 0.7612\n",
      "Best val acc 0.9145, best test acc: 0.7612\n",
      "Epoch 00008 | Step 00000 | Loss 0.2930 | Train Acc 0.9141 | Speed (samples/sec) 281.0209 | GPU 6414.4 MB\n",
      "Epoch 00008 | Step 00020 | Loss 0.3873 | Train Acc 0.8984 | Speed (samples/sec) 281.2662 | GPU 6414.4 MB\n",
      "Epoch 00008 | Step 00040 | Loss 0.2773 | Train Acc 0.9238 | Speed (samples/sec) 281.4072 | GPU 6414.4 MB\n",
      "Epoch 00008 | Step 00060 | Loss 0.1909 | Train Acc 0.9512 | Speed (samples/sec) 281.5652 | GPU 6414.4 MB\n",
      "Epoch 00008 | Step 00080 | Loss 0.3186 | Train Acc 0.9199 | Speed (samples/sec) 281.7728 | GPU 6414.4 MB\n",
      "Epoch 00008 | Step 00100 | Loss 0.3234 | Train Acc 0.9004 | Speed (samples/sec) 281.9556 | GPU 6414.4 MB\n",
      "Epoch 00008 | Step 00120 | Loss 0.3347 | Train Acc 0.9023 | Speed (samples/sec) 282.1099 | GPU 6414.4 MB\n",
      "Epoch 00008 | Step 00140 | Loss 0.2801 | Train Acc 0.9297 | Speed (samples/sec) 282.3114 | GPU 6414.4 MB\n",
      "Epoch 00008 | Step 00160 | Loss 0.2985 | Train Acc 0.8965 | Speed (samples/sec) 282.4608 | GPU 6414.4 MB\n",
      "Epoch 00008 | Step 00180 | Loss 0.3016 | Train Acc 0.9082 | Speed (samples/sec) 282.6297 | GPU 6414.4 MB\n",
      "Epoch 00008 | Step 00200 | Loss 0.3177 | Train Acc 0.9219 | Speed (samples/sec) 282.7690 | GPU 6414.4 MB\n",
      "Epoch 00008 | Step 00220 | Loss 0.3183 | Train Acc 0.9160 | Speed (samples/sec) 282.9078 | GPU 6414.4 MB\n",
      "Epoch 00008 | Step 00240 | Loss 0.2651 | Train Acc 0.9277 | Speed (samples/sec) 283.0436 | GPU 6414.4 MB\n",
      "Epoch 00008 | Step 00260 | Loss 0.2733 | Train Acc 0.9141 | Speed (samples/sec) 283.1794 | GPU 6414.4 MB\n",
      "Epoch 00008 | Step 00280 | Loss 0.3002 | Train Acc 0.9043 | Speed (samples/sec) 283.2678 | GPU 6414.4 MB\n",
      "Epoch 00008 | Step 00300 | Loss 0.3194 | Train Acc 0.9043 | Speed (samples/sec) 283.4474 | GPU 6414.4 MB\n",
      "Epoch 00008 | Step 00320 | Loss 0.3499 | Train Acc 0.8984 | Speed (samples/sec) 283.5641 | GPU 6414.4 MB\n",
      "Epoch 00008 | Step 00340 | Loss 0.3388 | Train Acc 0.8984 | Speed (samples/sec) 283.7642 | GPU 6414.4 MB\n",
      "Epoch 00008 | Step 00360 | Loss 0.3272 | Train Acc 0.8965 | Speed (samples/sec) 283.8620 | GPU 6414.4 MB\n",
      "Epoch 00008 | Step 00380 | Loss 0.2986 | Train Acc 0.9219 | Speed (samples/sec) 284.0007 | GPU 6414.4 MB\n",
      "Epoch Time(s): 641.0139\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 245/245 [00:45<00:00,  5.33it/s]\n",
      "100%|██████████| 245/245 [01:55<00:00,  2.11it/s]\n",
      "100%|██████████| 245/245 [01:45<00:00,  2.33it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Val acc 0.9139, Test Acc: 0.7617\n",
      "Epoch 00009 | Step 00000 | Loss 0.2744 | Train Acc 0.9199 | Speed (samples/sec) 283.9583 | GPU 6414.4 MB\n",
      "Epoch 00009 | Step 00020 | Loss 0.2872 | Train Acc 0.9043 | Speed (samples/sec) 284.0752 | GPU 6414.4 MB\n",
      "Epoch 00009 | Step 00040 | Loss 0.2837 | Train Acc 0.9023 | Speed (samples/sec) 284.1913 | GPU 6414.4 MB\n",
      "Epoch 00009 | Step 00060 | Loss 0.3593 | Train Acc 0.8984 | Speed (samples/sec) 284.3048 | GPU 6414.4 MB\n",
      "Epoch 00009 | Step 00080 | Loss 0.2533 | Train Acc 0.9219 | Speed (samples/sec) 284.4589 | GPU 6414.4 MB\n",
      "Epoch 00009 | Step 00100 | Loss 0.2083 | Train Acc 0.9375 | Speed (samples/sec) 284.5544 | GPU 6414.4 MB\n",
      "Epoch 00009 | Step 00120 | Loss 0.2439 | Train Acc 0.9238 | Speed (samples/sec) 284.6455 | GPU 6414.4 MB\n",
      "Epoch 00009 | Step 00140 | Loss 0.2380 | Train Acc 0.9219 | Speed (samples/sec) 284.7674 | GPU 6414.4 MB\n",
      "Epoch 00009 | Step 00160 | Loss 0.2836 | Train Acc 0.9102 | Speed (samples/sec) 284.8751 | GPU 6414.4 MB\n",
      "Epoch 00009 | Step 00180 | Loss 0.2973 | Train Acc 0.9141 | Speed (samples/sec) 285.0045 | GPU 6414.4 MB\n",
      "Epoch 00009 | Step 00200 | Loss 0.3349 | Train Acc 0.9141 | Speed (samples/sec) 285.1604 | GPU 6414.4 MB\n",
      "Epoch 00009 | Step 00220 | Loss 0.3143 | Train Acc 0.9180 | Speed (samples/sec) 285.2615 | GPU 6414.4 MB\n",
      "Epoch 00009 | Step 00240 | Loss 0.2095 | Train Acc 0.9297 | Speed (samples/sec) 285.3486 | GPU 6414.4 MB\n",
      "Epoch 00009 | Step 00260 | Loss 0.2861 | Train Acc 0.9238 | Speed (samples/sec) 285.4987 | GPU 6414.4 MB\n",
      "Epoch 00009 | Step 00280 | Loss 0.2621 | Train Acc 0.9141 | Speed (samples/sec) 285.6256 | GPU 6414.4 MB\n",
      "Epoch 00009 | Step 00300 | Loss 0.3072 | Train Acc 0.9043 | Speed (samples/sec) 285.7198 | GPU 6414.4 MB\n",
      "Epoch 00009 | Step 00320 | Loss 0.3089 | Train Acc 0.9199 | Speed (samples/sec) 285.8703 | GPU 6414.4 MB\n",
      "Epoch 00009 | Step 00340 | Loss 0.3890 | Train Acc 0.8848 | Speed (samples/sec) 286.0627 | GPU 6414.4 MB\n",
      "Epoch 00009 | Step 00360 | Loss 0.3533 | Train Acc 0.9023 | Speed (samples/sec) 286.2195 | GPU 6414.4 MB\n",
      "Epoch 00009 | Step 00380 | Loss 0.2402 | Train Acc 0.9277 | Speed (samples/sec) 286.3136 | GPU 6414.4 MB\n",
      "Epoch Time(s): 641.2424\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 245/245 [00:45<00:00,  5.35it/s]\n",
      "100%|██████████| 245/245 [02:01<00:00,  2.02it/s]\n",
      "100%|██████████| 245/245 [01:51<00:00,  2.19it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Val acc 0.9155, Test Acc: 0.7604\n",
      "Best val acc 0.9155, best test acc: 0.7604\n",
      "Epoch 00010 | Step 00000 | Loss 0.3489 | Train Acc 0.9082 | Speed (samples/sec) 286.2992 | GPU 6414.4 MB\n",
      "Epoch 00010 | Step 00020 | Loss 0.2515 | Train Acc 0.9336 | Speed (samples/sec) 286.4106 | GPU 6414.4 MB\n",
      "Epoch 00010 | Step 00040 | Loss 0.2608 | Train Acc 0.9277 | Speed (samples/sec) 286.5522 | GPU 6414.4 MB\n",
      "Epoch 00010 | Step 00060 | Loss 0.2477 | Train Acc 0.9160 | Speed (samples/sec) 286.6756 | GPU 6414.4 MB\n",
      "Epoch 00010 | Step 00080 | Loss 0.3547 | Train Acc 0.8984 | Speed (samples/sec) 286.7840 | GPU 6414.4 MB\n",
      "Epoch 00010 | Step 00100 | Loss 0.3534 | Train Acc 0.9004 | Speed (samples/sec) 286.8844 | GPU 6414.4 MB\n",
      "Epoch 00010 | Step 00120 | Loss 0.3294 | Train Acc 0.8945 | Speed (samples/sec) 286.9581 | GPU 6414.4 MB\n",
      "Epoch 00010 | Step 00140 | Loss 0.3044 | Train Acc 0.9062 | Speed (samples/sec) 287.0396 | GPU 6414.4 MB\n",
      "Epoch 00010 | Step 00160 | Loss 0.2865 | Train Acc 0.9160 | Speed (samples/sec) 287.1407 | GPU 6414.4 MB\n",
      "Epoch 00010 | Step 00180 | Loss 0.2590 | Train Acc 0.9238 | Speed (samples/sec) 287.2599 | GPU 6414.4 MB\n",
      "Epoch 00010 | Step 00200 | Loss 0.2592 | Train Acc 0.9258 | Speed (samples/sec) 287.4136 | GPU 6414.4 MB\n",
      "Epoch 00010 | Step 00220 | Loss 0.3054 | Train Acc 0.9102 | Speed (samples/sec) 287.5293 | GPU 6414.4 MB\n",
      "Epoch 00010 | Step 00240 | Loss 0.2688 | Train Acc 0.9219 | Speed (samples/sec) 287.6571 | GPU 6414.4 MB\n",
      "Epoch 00010 | Step 00260 | Loss 0.2731 | Train Acc 0.9297 | Speed (samples/sec) 287.7791 | GPU 6414.4 MB\n",
      "Epoch 00010 | Step 00280 | Loss 0.3816 | Train Acc 0.9004 | Speed (samples/sec) 287.9013 | GPU 6414.4 MB\n",
      "Epoch 00010 | Step 00300 | Loss 0.2755 | Train Acc 0.9199 | Speed (samples/sec) 288.0960 | GPU 6414.4 MB\n",
      "Epoch 00010 | Step 00320 | Loss 0.3689 | Train Acc 0.9043 | Speed (samples/sec) 288.3165 | GPU 6414.4 MB\n",
      "Epoch 00010 | Step 00340 | Loss 0.3916 | Train Acc 0.8984 | Speed (samples/sec) 288.5049 | GPU 6414.4 MB\n",
      "Epoch 00010 | Step 00360 | Loss 0.2623 | Train Acc 0.9180 | Speed (samples/sec) 288.6938 | GPU 6414.4 MB\n",
      "Epoch 00010 | Step 00380 | Loss 0.3025 | Train Acc 0.9160 | Speed (samples/sec) 288.9012 | GPU 6414.4 MB\n",
      "Epoch Time(s): 626.5675\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 245/245 [00:44<00:00,  5.55it/s]\n",
      "100%|██████████| 245/245 [01:53<00:00,  2.15it/s]\n",
      "100%|██████████| 245/245 [01:36<00:00,  2.55it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Val acc 0.9108, Test Acc: 0.7521\n",
      "Epoch 00011 | Step 00000 | Loss 0.2543 | Train Acc 0.9297 | Speed (samples/sec) 288.8537 | GPU 6414.4 MB\n",
      "Epoch 00011 | Step 00020 | Loss 0.3319 | Train Acc 0.9023 | Speed (samples/sec) 289.0657 | GPU 6414.4 MB\n",
      "Epoch 00011 | Step 00040 | Loss 0.2785 | Train Acc 0.9102 | Speed (samples/sec) 289.2763 | GPU 6414.4 MB\n",
      "Epoch 00011 | Step 00060 | Loss 0.2581 | Train Acc 0.9277 | Speed (samples/sec) 289.5102 | GPU 6414.4 MB\n",
      "Epoch 00011 | Step 00080 | Loss 0.2869 | Train Acc 0.9180 | Speed (samples/sec) 289.7250 | GPU 6414.4 MB\n",
      "Epoch 00011 | Step 00100 | Loss 0.3231 | Train Acc 0.8887 | Speed (samples/sec) 289.9350 | GPU 6414.4 MB\n",
      "Epoch 00011 | Step 00120 | Loss 0.3352 | Train Acc 0.9121 | Speed (samples/sec) 290.1479 | GPU 6414.4 MB\n",
      "Epoch 00011 | Step 00140 | Loss 0.2931 | Train Acc 0.9102 | Speed (samples/sec) 290.4041 | GPU 6414.4 MB\n",
      "Epoch 00011 | Step 00160 | Loss 0.2954 | Train Acc 0.9082 | Speed (samples/sec) 290.6399 | GPU 6414.4 MB\n",
      "Epoch 00011 | Step 00180 | Loss 0.3809 | Train Acc 0.8926 | Speed (samples/sec) 290.8395 | GPU 6414.4 MB\n",
      "Epoch 00011 | Step 00200 | Loss 0.3601 | Train Acc 0.8887 | Speed (samples/sec) 291.0665 | GPU 6414.4 MB\n",
      "Epoch 00011 | Step 00220 | Loss 0.3673 | Train Acc 0.9180 | Speed (samples/sec) 291.2870 | GPU 6414.4 MB\n",
      "Epoch 00011 | Step 00240 | Loss 0.2979 | Train Acc 0.9180 | Speed (samples/sec) 291.5028 | GPU 6414.4 MB\n",
      "Epoch 00011 | Step 00260 | Loss 0.2867 | Train Acc 0.9180 | Speed (samples/sec) 291.6836 | GPU 6414.4 MB\n",
      "Epoch 00011 | Step 00280 | Loss 0.2438 | Train Acc 0.9336 | Speed (samples/sec) 291.8858 | GPU 6414.4 MB\n",
      "Epoch 00011 | Step 00300 | Loss 0.2860 | Train Acc 0.9316 | Speed (samples/sec) 292.1263 | GPU 6414.4 MB\n",
      "Epoch 00011 | Step 00320 | Loss 0.2638 | Train Acc 0.9219 | Speed (samples/sec) 292.3390 | GPU 6414.4 MB\n",
      "Epoch 00011 | Step 00340 | Loss 0.3329 | Train Acc 0.9121 | Speed (samples/sec) 292.5395 | GPU 6414.4 MB\n",
      "Epoch 00011 | Step 00360 | Loss 0.2842 | Train Acc 0.9238 | Speed (samples/sec) 292.7411 | GPU 6414.4 MB\n",
      "Epoch 00011 | Step 00380 | Loss 0.3137 | Train Acc 0.9023 | Speed (samples/sec) 292.9521 | GPU 6414.4 MB\n",
      "Epoch Time(s): 583.2341\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 245/245 [00:41<00:00,  5.91it/s]\n",
      "100%|██████████| 245/245 [01:41<00:00,  2.41it/s]\n",
      "100%|██████████| 245/245 [01:34<00:00,  2.58it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Val acc 0.9131, Test Acc: 0.7610\n",
      "Epoch 00012 | Step 00000 | Loss 0.3508 | Train Acc 0.9121 | Speed (samples/sec) 292.9503 | GPU 6414.4 MB\n",
      "Epoch 00012 | Step 00020 | Loss 0.3688 | Train Acc 0.9004 | Speed (samples/sec) 293.2123 | GPU 6414.4 MB\n",
      "Epoch 00012 | Step 00040 | Loss 0.3073 | Train Acc 0.9082 | Speed (samples/sec) 293.4504 | GPU 6414.4 MB\n",
      "Epoch 00012 | Step 00060 | Loss 0.3674 | Train Acc 0.9043 | Speed (samples/sec) 293.7010 | GPU 6414.4 MB\n",
      "Epoch 00012 | Step 00080 | Loss 0.2310 | Train Acc 0.9453 | Speed (samples/sec) 293.9539 | GPU 6414.4 MB\n",
      "Epoch 00012 | Step 00100 | Loss 0.3500 | Train Acc 0.9023 | Speed (samples/sec) 294.1877 | GPU 6414.4 MB\n",
      "Epoch 00012 | Step 00120 | Loss 0.2571 | Train Acc 0.9238 | Speed (samples/sec) 294.4670 | GPU 6414.4 MB\n",
      "Epoch 00012 | Step 00140 | Loss 0.2618 | Train Acc 0.9258 | Speed (samples/sec) 294.8232 | GPU 6414.4 MB\n",
      "Epoch 00012 | Step 00160 | Loss 0.2933 | Train Acc 0.9141 | Speed (samples/sec) 295.1798 | GPU 6414.4 MB\n",
      "Epoch 00012 | Step 00180 | Loss 0.2975 | Train Acc 0.9062 | Speed (samples/sec) 295.5200 | GPU 6414.4 MB\n",
      "Epoch 00012 | Step 00200 | Loss 0.2626 | Train Acc 0.9316 | Speed (samples/sec) 295.8684 | GPU 6414.4 MB\n",
      "Epoch 00012 | Step 00220 | Loss 0.3594 | Train Acc 0.9102 | Speed (samples/sec) 296.1869 | GPU 6414.4 MB\n",
      "Epoch 00012 | Step 00240 | Loss 0.3184 | Train Acc 0.9004 | Speed (samples/sec) 296.4788 | GPU 6414.4 MB\n",
      "Epoch 00012 | Step 00260 | Loss 0.2952 | Train Acc 0.9180 | Speed (samples/sec) 296.7914 | GPU 6414.4 MB\n",
      "Epoch 00012 | Step 00280 | Loss 0.3605 | Train Acc 0.9102 | Speed (samples/sec) 297.0966 | GPU 6414.4 MB\n",
      "Epoch 00012 | Step 00300 | Loss 0.2629 | Train Acc 0.9355 | Speed (samples/sec) 297.4020 | GPU 6414.4 MB\n",
      "Epoch 00012 | Step 00320 | Loss 0.3893 | Train Acc 0.8926 | Speed (samples/sec) 297.6714 | GPU 6414.4 MB\n",
      "Epoch 00012 | Step 00340 | Loss 0.2935 | Train Acc 0.9121 | Speed (samples/sec) 297.9513 | GPU 6414.4 MB\n",
      "Epoch 00012 | Step 00360 | Loss 0.3822 | Train Acc 0.8984 | Speed (samples/sec) 298.2716 | GPU 6414.4 MB\n",
      "Epoch 00012 | Step 00380 | Loss 0.2752 | Train Acc 0.9043 | Speed (samples/sec) 298.5702 | GPU 6414.4 MB\n",
      "Epoch Time(s): 538.2580\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 245/245 [00:38<00:00,  6.32it/s]\n",
      "100%|██████████| 245/245 [01:38<00:00,  2.50it/s]\n",
      "100%|██████████| 245/245 [01:27<00:00,  2.81it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Val acc 0.9162, Test Acc: 0.7687\n",
      "Best val acc 0.9162, best test acc: 0.7687\n",
      "Epoch 00013 | Step 00000 | Loss 0.2582 | Train Acc 0.9082 | Speed (samples/sec) 298.5730 | GPU 6414.4 MB\n",
      "Epoch 00013 | Step 00020 | Loss 0.2585 | Train Acc 0.9355 | Speed (samples/sec) 298.8984 | GPU 6414.4 MB\n",
      "Epoch 00013 | Step 00040 | Loss 0.2916 | Train Acc 0.9180 | Speed (samples/sec) 299.2810 | GPU 6414.4 MB\n",
      "Epoch 00013 | Step 00060 | Loss 0.3050 | Train Acc 0.9043 | Speed (samples/sec) 299.5850 | GPU 6414.4 MB\n",
      "Epoch 00013 | Step 00080 | Loss 0.2369 | Train Acc 0.9297 | Speed (samples/sec) 299.9037 | GPU 6414.4 MB\n",
      "Epoch 00013 | Step 00100 | Loss 0.2368 | Train Acc 0.9297 | Speed (samples/sec) 300.2321 | GPU 6414.4 MB\n",
      "Epoch 00013 | Step 00120 | Loss 0.2720 | Train Acc 0.9141 | Speed (samples/sec) 300.5294 | GPU 6414.4 MB\n",
      "Epoch 00013 | Step 00140 | Loss 0.3032 | Train Acc 0.9160 | Speed (samples/sec) 300.7681 | GPU 6414.4 MB\n",
      "Epoch 00013 | Step 00160 | Loss 0.2650 | Train Acc 0.9141 | Speed (samples/sec) 301.0714 | GPU 6414.4 MB\n",
      "Epoch 00013 | Step 00180 | Loss 0.2464 | Train Acc 0.9258 | Speed (samples/sec) 301.3411 | GPU 6414.4 MB\n",
      "Epoch 00013 | Step 00200 | Loss 0.3199 | Train Acc 0.9121 | Speed (samples/sec) 301.5986 | GPU 6414.4 MB\n",
      "Epoch 00013 | Step 00220 | Loss 0.2646 | Train Acc 0.9336 | Speed (samples/sec) 301.8830 | GPU 6414.4 MB\n",
      "Epoch 00013 | Step 00240 | Loss 0.2993 | Train Acc 0.9121 | Speed (samples/sec) 302.1581 | GPU 6414.4 MB\n",
      "Epoch 00013 | Step 00260 | Loss 0.2971 | Train Acc 0.9277 | Speed (samples/sec) 302.4306 | GPU 6414.4 MB\n",
      "Epoch 00013 | Step 00280 | Loss 0.3585 | Train Acc 0.9141 | Speed (samples/sec) 302.6867 | GPU 6414.4 MB\n",
      "Epoch 00013 | Step 00300 | Loss 0.2983 | Train Acc 0.9102 | Speed (samples/sec) 302.9070 | GPU 6414.4 MB\n",
      "Epoch 00013 | Step 00320 | Loss 0.3179 | Train Acc 0.9160 | Speed (samples/sec) 303.1851 | GPU 6414.4 MB\n",
      "Epoch 00013 | Step 00340 | Loss 0.3579 | Train Acc 0.8945 | Speed (samples/sec) 303.4322 | GPU 6414.4 MB\n",
      "Epoch 00013 | Step 00360 | Loss 0.4646 | Train Acc 0.8691 | Speed (samples/sec) 303.6833 | GPU 6414.4 MB\n",
      "Epoch 00013 | Step 00380 | Loss 0.3523 | Train Acc 0.9102 | Speed (samples/sec) 303.9203 | GPU 6414.4 MB\n",
      "Epoch Time(s): 527.4781\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 245/245 [00:37<00:00,  6.47it/s]\n",
      "100%|██████████| 245/245 [01:39<00:00,  2.47it/s]\n",
      "100%|██████████| 245/245 [01:26<00:00,  2.83it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Val acc 0.9162, Test Acc: 0.7612\n",
      "Best val acc 0.9162, best test acc: 0.7612\n",
      "Epoch 00014 | Step 00000 | Loss 0.3165 | Train Acc 0.8945 | Speed (samples/sec) 303.9204 | GPU 6414.4 MB\n",
      "Epoch 00014 | Step 00020 | Loss 0.2255 | Train Acc 0.9316 | Speed (samples/sec) 304.1788 | GPU 6414.4 MB\n",
      "Epoch 00014 | Step 00040 | Loss 0.3221 | Train Acc 0.9023 | Speed (samples/sec) 304.4439 | GPU 6414.4 MB\n",
      "Epoch 00014 | Step 00060 | Loss 0.2543 | Train Acc 0.9219 | Speed (samples/sec) 304.7070 | GPU 6414.4 MB\n",
      "Epoch 00014 | Step 00080 | Loss 0.3079 | Train Acc 0.9082 | Speed (samples/sec) 304.9440 | GPU 6414.4 MB\n",
      "Epoch 00014 | Step 00100 | Loss 0.2606 | Train Acc 0.9219 | Speed (samples/sec) 305.1703 | GPU 6414.4 MB\n",
      "Epoch 00014 | Step 00120 | Loss 0.2845 | Train Acc 0.9238 | Speed (samples/sec) 305.4386 | GPU 6414.4 MB\n",
      "Epoch 00014 | Step 00140 | Loss 0.3108 | Train Acc 0.9102 | Speed (samples/sec) 305.6953 | GPU 6414.4 MB\n",
      "Epoch 00014 | Step 00160 | Loss 0.2400 | Train Acc 0.9297 | Speed (samples/sec) 305.9357 | GPU 6414.4 MB\n",
      "Epoch 00014 | Step 00180 | Loss 0.2607 | Train Acc 0.9160 | Speed (samples/sec) 306.1727 | GPU 6414.4 MB\n",
      "Epoch 00014 | Step 00200 | Loss 0.3129 | Train Acc 0.9160 | Speed (samples/sec) 306.4140 | GPU 6414.4 MB\n",
      "Epoch 00014 | Step 00220 | Loss 0.2450 | Train Acc 0.9160 | Speed (samples/sec) 306.6501 | GPU 6414.4 MB\n",
      "Epoch 00014 | Step 00240 | Loss 0.2601 | Train Acc 0.9238 | Speed (samples/sec) 306.8494 | GPU 6414.4 MB\n",
      "Epoch 00014 | Step 00260 | Loss 0.2997 | Train Acc 0.9141 | Speed (samples/sec) 307.0810 | GPU 6414.4 MB\n",
      "Epoch 00014 | Step 00280 | Loss 0.2768 | Train Acc 0.9121 | Speed (samples/sec) 307.3111 | GPU 6414.4 MB\n",
      "Epoch 00014 | Step 00300 | Loss 0.2827 | Train Acc 0.9102 | Speed (samples/sec) 307.5213 | GPU 6414.4 MB\n",
      "Epoch 00014 | Step 00320 | Loss 0.2770 | Train Acc 0.9277 | Speed (samples/sec) 307.7524 | GPU 6414.4 MB\n",
      "Epoch 00014 | Step 00340 | Loss 0.2950 | Train Acc 0.9238 | Speed (samples/sec) 308.0161 | GPU 6414.4 MB\n",
      "Epoch 00014 | Step 00360 | Loss 0.2539 | Train Acc 0.9336 | Speed (samples/sec) 308.2209 | GPU 6414.4 MB\n",
      "Epoch 00014 | Step 00380 | Loss 0.2794 | Train Acc 0.9316 | Speed (samples/sec) 308.4423 | GPU 6414.4 MB\n",
      "Epoch Time(s): 529.7909\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 245/245 [00:35<00:00,  6.85it/s]\n",
      "100%|██████████| 245/245 [01:32<00:00,  2.64it/s]\n",
      "100%|██████████| 245/245 [01:21<00:00,  3.01it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Val acc 0.9205, Test Acc: 0.7622\n",
      "Best val acc 0.9205, best test acc: 0.7622\n",
      "Epoch 00015 | Step 00000 | Loss 0.2438 | Train Acc 0.9316 | Speed (samples/sec) 308.4569 | GPU 6414.4 MB\n",
      "Epoch 00015 | Step 00020 | Loss 0.2394 | Train Acc 0.9316 | Speed (samples/sec) 308.8352 | GPU 6414.4 MB\n",
      "Epoch 00015 | Step 00040 | Loss 0.2107 | Train Acc 0.9297 | Speed (samples/sec) 309.2021 | GPU 6414.4 MB\n",
      "Epoch 00015 | Step 00060 | Loss 0.3760 | Train Acc 0.9023 | Speed (samples/sec) 309.5378 | GPU 6414.4 MB\n",
      "Epoch 00015 | Step 00080 | Loss 0.2836 | Train Acc 0.9199 | Speed (samples/sec) 309.8936 | GPU 6414.4 MB\n",
      "Epoch 00015 | Step 00100 | Loss 0.2408 | Train Acc 0.9316 | Speed (samples/sec) 310.2320 | GPU 6414.4 MB\n",
      "Epoch 00015 | Step 00120 | Loss 0.3025 | Train Acc 0.9004 | Speed (samples/sec) 310.7286 | GPU 6414.4 MB\n",
      "Epoch 00015 | Step 00140 | Loss 0.2616 | Train Acc 0.9160 | Speed (samples/sec) 311.1933 | GPU 6414.4 MB\n",
      "Epoch 00015 | Step 00160 | Loss 0.2967 | Train Acc 0.9102 | Speed (samples/sec) 311.6215 | GPU 6414.4 MB\n",
      "Epoch 00015 | Step 00180 | Loss 0.2870 | Train Acc 0.9277 | Speed (samples/sec) 312.0896 | GPU 6414.4 MB\n",
      "Epoch 00015 | Step 00200 | Loss 0.2448 | Train Acc 0.9316 | Speed (samples/sec) 312.5532 | GPU 6414.4 MB\n",
      "Epoch 00015 | Step 00220 | Loss 0.2818 | Train Acc 0.9121 | Speed (samples/sec) 313.0149 | GPU 6414.4 MB\n",
      "Epoch 00015 | Step 00240 | Loss 0.2806 | Train Acc 0.9141 | Speed (samples/sec) 313.4831 | GPU 6414.4 MB\n",
      "Epoch 00015 | Step 00260 | Loss 0.2604 | Train Acc 0.9238 | Speed (samples/sec) 313.9226 | GPU 6414.4 MB\n",
      "Epoch 00015 | Step 00280 | Loss 0.3134 | Train Acc 0.9023 | Speed (samples/sec) 314.3713 | GPU 6414.4 MB\n",
      "Epoch 00015 | Step 00300 | Loss 0.3510 | Train Acc 0.9043 | Speed (samples/sec) 314.8318 | GPU 6414.4 MB\n",
      "Epoch 00015 | Step 00320 | Loss 0.2798 | Train Acc 0.9141 | Speed (samples/sec) 315.2998 | GPU 6414.4 MB\n",
      "Epoch 00015 | Step 00340 | Loss 0.2408 | Train Acc 0.9199 | Speed (samples/sec) 315.7764 | GPU 6414.4 MB\n",
      "Epoch 00015 | Step 00360 | Loss 0.2634 | Train Acc 0.9238 | Speed (samples/sec) 316.2449 | GPU 6414.4 MB\n",
      "Epoch 00015 | Step 00380 | Loss 0.2446 | Train Acc 0.9277 | Speed (samples/sec) 316.6760 | GPU 6414.4 MB\n",
      "Epoch Time(s): 446.7036\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 245/245 [00:29<00:00,  8.21it/s]\n",
      "100%|██████████| 245/245 [01:24<00:00,  2.91it/s]\n",
      "100%|██████████| 245/245 [01:16<00:00,  3.18it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Val acc 0.9136, Test Acc: 0.7629\n",
      "Epoch 00016 | Step 00000 | Loss 0.2760 | Train Acc 0.9180 | Speed (samples/sec) 316.7360 | GPU 6414.4 MB\n",
      "Epoch 00016 | Step 00020 | Loss 0.2436 | Train Acc 0.9180 | Speed (samples/sec) 317.1736 | GPU 6414.4 MB\n",
      "Epoch 00016 | Step 00040 | Loss 0.2722 | Train Acc 0.9180 | Speed (samples/sec) 317.6223 | GPU 6414.4 MB\n",
      "Epoch 00016 | Step 00060 | Loss 0.2948 | Train Acc 0.9082 | Speed (samples/sec) 318.0404 | GPU 6414.4 MB\n",
      "Epoch 00016 | Step 00080 | Loss 0.2248 | Train Acc 0.9277 | Speed (samples/sec) 318.4775 | GPU 6414.4 MB\n",
      "Epoch 00016 | Step 00100 | Loss 0.3281 | Train Acc 0.9102 | Speed (samples/sec) 318.9245 | GPU 6414.4 MB\n",
      "Epoch 00016 | Step 00120 | Loss 0.2143 | Train Acc 0.9297 | Speed (samples/sec) 319.3427 | GPU 6414.4 MB\n",
      "Epoch 00016 | Step 00140 | Loss 0.2366 | Train Acc 0.9199 | Speed (samples/sec) 319.7465 | GPU 6414.4 MB\n",
      "Epoch 00016 | Step 00160 | Loss 0.3119 | Train Acc 0.9023 | Speed (samples/sec) 320.1971 | GPU 6414.4 MB\n",
      "Epoch 00016 | Step 00180 | Loss 0.2480 | Train Acc 0.9141 | Speed (samples/sec) 320.6077 | GPU 6414.4 MB\n",
      "Epoch 00016 | Step 00200 | Loss 0.3011 | Train Acc 0.9238 | Speed (samples/sec) 321.0375 | GPU 6414.4 MB\n",
      "Epoch 00016 | Step 00220 | Loss 0.2256 | Train Acc 0.9199 | Speed (samples/sec) 321.4666 | GPU 6414.4 MB\n",
      "Epoch 00016 | Step 00240 | Loss 0.3044 | Train Acc 0.9180 | Speed (samples/sec) 321.8710 | GPU 6414.4 MB\n",
      "Epoch 00016 | Step 00260 | Loss 0.2244 | Train Acc 0.9336 | Speed (samples/sec) 322.2984 | GPU 6414.4 MB\n",
      "Epoch 00016 | Step 00280 | Loss 0.2684 | Train Acc 0.9043 | Speed (samples/sec) 322.6928 | GPU 6414.4 MB\n",
      "Epoch 00016 | Step 00300 | Loss 0.4063 | Train Acc 0.8945 | Speed (samples/sec) 323.0795 | GPU 6414.4 MB\n",
      "Epoch 00016 | Step 00320 | Loss 0.2374 | Train Acc 0.9219 | Speed (samples/sec) 323.4901 | GPU 6414.4 MB\n",
      "Epoch 00016 | Step 00340 | Loss 0.2926 | Train Acc 0.9199 | Speed (samples/sec) 323.8951 | GPU 6414.4 MB\n",
      "Epoch 00016 | Step 00360 | Loss 0.2855 | Train Acc 0.9199 | Speed (samples/sec) 324.3015 | GPU 6414.4 MB\n",
      "Epoch 00016 | Step 00380 | Loss 0.2978 | Train Acc 0.9004 | Speed (samples/sec) 324.7299 | GPU 6414.4 MB\n",
      "Epoch Time(s): 433.3939\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 245/245 [00:30<00:00,  8.08it/s]\n",
      "100%|██████████| 245/245 [01:23<00:00,  2.93it/s]\n",
      "100%|██████████| 245/245 [01:16<00:00,  3.19it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Val acc 0.9169, Test Acc: 0.7751\n",
      "Epoch 00017 | Step 00000 | Loss 0.1882 | Train Acc 0.9414 | Speed (samples/sec) 324.7825 | GPU 6414.4 MB\n",
      "Epoch 00017 | Step 00020 | Loss 0.2594 | Train Acc 0.9336 | Speed (samples/sec) 325.1891 | GPU 6414.4 MB\n",
      "Epoch 00017 | Step 00040 | Loss 0.2885 | Train Acc 0.9102 | Speed (samples/sec) 325.5601 | GPU 6414.4 MB\n",
      "Epoch 00017 | Step 00060 | Loss 0.2772 | Train Acc 0.9180 | Speed (samples/sec) 325.9568 | GPU 6414.4 MB\n",
      "Epoch 00017 | Step 00080 | Loss 0.2677 | Train Acc 0.9121 | Speed (samples/sec) 326.3476 | GPU 6414.4 MB\n",
      "Epoch 00017 | Step 00100 | Loss 0.3000 | Train Acc 0.9180 | Speed (samples/sec) 326.7435 | GPU 6414.4 MB\n",
      "Epoch 00017 | Step 00120 | Loss 0.2493 | Train Acc 0.9219 | Speed (samples/sec) 327.1067 | GPU 6414.4 MB\n",
      "Epoch 00017 | Step 00140 | Loss 0.2688 | Train Acc 0.9355 | Speed (samples/sec) 327.5028 | GPU 6414.4 MB\n",
      "Epoch 00017 | Step 00160 | Loss 0.3166 | Train Acc 0.9180 | Speed (samples/sec) 327.8583 | GPU 6414.4 MB\n",
      "Epoch 00017 | Step 00180 | Loss 0.3338 | Train Acc 0.9141 | Speed (samples/sec) 328.2305 | GPU 6414.4 MB\n",
      "Epoch 00017 | Step 00200 | Loss 0.2699 | Train Acc 0.9199 | Speed (samples/sec) 328.5990 | GPU 6414.4 MB\n",
      "Epoch 00017 | Step 00220 | Loss 0.2746 | Train Acc 0.9102 | Speed (samples/sec) 328.9701 | GPU 6414.4 MB\n",
      "Epoch 00017 | Step 00240 | Loss 0.3284 | Train Acc 0.9004 | Speed (samples/sec) 329.3442 | GPU 6414.4 MB\n",
      "Epoch 00017 | Step 00260 | Loss 0.3598 | Train Acc 0.9023 | Speed (samples/sec) 329.7232 | GPU 6414.4 MB\n",
      "Epoch 00017 | Step 00280 | Loss 0.2656 | Train Acc 0.9121 | Speed (samples/sec) 330.0940 | GPU 6414.4 MB\n",
      "Epoch 00017 | Step 00300 | Loss 0.3427 | Train Acc 0.9043 | Speed (samples/sec) 330.4401 | GPU 6414.4 MB\n",
      "Epoch 00017 | Step 00320 | Loss 0.2534 | Train Acc 0.9219 | Speed (samples/sec) 330.7996 | GPU 6414.4 MB\n",
      "Epoch 00017 | Step 00340 | Loss 0.2688 | Train Acc 0.9199 | Speed (samples/sec) 331.1500 | GPU 6414.4 MB\n",
      "Epoch 00017 | Step 00360 | Loss 0.2746 | Train Acc 0.9316 | Speed (samples/sec) 331.5171 | GPU 6414.4 MB\n",
      "Epoch 00017 | Step 00380 | Loss 0.2445 | Train Acc 0.9258 | Speed (samples/sec) 331.8750 | GPU 6414.4 MB\n",
      "Epoch Time(s): 433.5702\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 245/245 [00:29<00:00,  8.34it/s]\n",
      "100%|██████████| 245/245 [01:25<00:00,  2.87it/s]\n",
      "100%|██████████| 245/245 [01:17<00:00,  3.17it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Val acc 0.9155, Test Acc: 0.7548\n",
      "Epoch 00018 | Step 00000 | Loss 0.2373 | Train Acc 0.9238 | Speed (samples/sec) 331.9173 | GPU 6414.4 MB\n",
      "Epoch 00018 | Step 00020 | Loss 0.2661 | Train Acc 0.9102 | Speed (samples/sec) 332.2772 | GPU 6414.4 MB\n",
      "Epoch 00018 | Step 00040 | Loss 0.3517 | Train Acc 0.8945 | Speed (samples/sec) 332.6245 | GPU 6414.4 MB\n",
      "Epoch 00018 | Step 00060 | Loss 0.2692 | Train Acc 0.9102 | Speed (samples/sec) 332.9836 | GPU 6414.4 MB\n",
      "Epoch 00018 | Step 00080 | Loss 0.2250 | Train Acc 0.9316 | Speed (samples/sec) 333.3375 | GPU 6414.4 MB\n",
      "Epoch 00018 | Step 00100 | Loss 0.3573 | Train Acc 0.8945 | Speed (samples/sec) 333.6805 | GPU 6414.4 MB\n",
      "Epoch 00018 | Step 00120 | Loss 0.2479 | Train Acc 0.9297 | Speed (samples/sec) 334.0101 | GPU 6414.4 MB\n",
      "Epoch 00018 | Step 00140 | Loss 0.2190 | Train Acc 0.9297 | Speed (samples/sec) 334.3606 | GPU 6414.4 MB\n",
      "Epoch 00018 | Step 00160 | Loss 0.3108 | Train Acc 0.9023 | Speed (samples/sec) 334.6973 | GPU 6414.4 MB\n",
      "Epoch 00018 | Step 00180 | Loss 0.3588 | Train Acc 0.8828 | Speed (samples/sec) 335.0450 | GPU 6414.4 MB\n",
      "Epoch 00018 | Step 00200 | Loss 0.2589 | Train Acc 0.9238 | Speed (samples/sec) 335.3721 | GPU 6414.4 MB\n",
      "Epoch 00018 | Step 00220 | Loss 0.2447 | Train Acc 0.9199 | Speed (samples/sec) 335.7113 | GPU 6414.4 MB\n",
      "Epoch 00018 | Step 00240 | Loss 0.3122 | Train Acc 0.9121 | Speed (samples/sec) 336.0416 | GPU 6414.4 MB\n",
      "Epoch 00018 | Step 00260 | Loss 0.3497 | Train Acc 0.9004 | Speed (samples/sec) 336.3644 | GPU 6414.4 MB\n",
      "Epoch 00018 | Step 00280 | Loss 0.2676 | Train Acc 0.9336 | Speed (samples/sec) 336.6775 | GPU 6414.4 MB\n",
      "Epoch 00018 | Step 00300 | Loss 0.2832 | Train Acc 0.9297 | Speed (samples/sec) 337.0036 | GPU 6414.4 MB\n",
      "Epoch 00018 | Step 00320 | Loss 0.3381 | Train Acc 0.8965 | Speed (samples/sec) 337.3381 | GPU 6414.4 MB\n",
      "Epoch 00018 | Step 00340 | Loss 0.2611 | Train Acc 0.9238 | Speed (samples/sec) 337.6655 | GPU 6414.4 MB\n",
      "Epoch 00018 | Step 00360 | Loss 0.2284 | Train Acc 0.9355 | Speed (samples/sec) 337.9996 | GPU 6414.4 MB\n",
      "Epoch 00018 | Step 00380 | Loss 0.2713 | Train Acc 0.9043 | Speed (samples/sec) 338.3136 | GPU 6414.4 MB\n",
      "Epoch Time(s): 432.6652\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 245/245 [00:29<00:00,  8.35it/s]\n",
      "100%|██████████| 245/245 [01:24<00:00,  2.90it/s]\n",
      "100%|██████████| 245/245 [01:16<00:00,  3.22it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Val acc 0.9175, Test Acc: 0.7589\n",
      "Epoch 00019 | Step 00000 | Loss 0.2943 | Train Acc 0.9219 | Speed (samples/sec) 338.3491 | GPU 6414.4 MB\n",
      "Epoch 00019 | Step 00020 | Loss 0.3524 | Train Acc 0.9082 | Speed (samples/sec) 338.6599 | GPU 6414.4 MB\n",
      "Epoch 00019 | Step 00040 | Loss 0.2893 | Train Acc 0.9199 | Speed (samples/sec) 338.9719 | GPU 6414.4 MB\n",
      "Epoch 00019 | Step 00060 | Loss 0.3217 | Train Acc 0.9062 | Speed (samples/sec) 339.2961 | GPU 6414.4 MB\n",
      "Epoch 00019 | Step 00080 | Loss 0.3012 | Train Acc 0.9199 | Speed (samples/sec) 339.6104 | GPU 6414.4 MB\n",
      "Epoch 00019 | Step 00100 | Loss 0.3118 | Train Acc 0.9160 | Speed (samples/sec) 339.9095 | GPU 6414.4 MB\n",
      "Epoch 00019 | Step 00120 | Loss 0.3048 | Train Acc 0.9062 | Speed (samples/sec) 340.2194 | GPU 6414.4 MB\n",
      "Epoch 00019 | Step 00140 | Loss 0.2134 | Train Acc 0.9355 | Speed (samples/sec) 340.5157 | GPU 6414.4 MB\n",
      "Epoch 00019 | Step 00160 | Loss 0.2561 | Train Acc 0.9180 | Speed (samples/sec) 340.8060 | GPU 6414.4 MB\n",
      "Epoch 00019 | Step 00180 | Loss 0.3002 | Train Acc 0.9199 | Speed (samples/sec) 341.1000 | GPU 6414.4 MB\n",
      "Epoch 00019 | Step 00200 | Loss 0.2788 | Train Acc 0.9219 | Speed (samples/sec) 341.3823 | GPU 6414.4 MB\n",
      "Epoch 00019 | Step 00220 | Loss 0.3202 | Train Acc 0.9004 | Speed (samples/sec) 341.6721 | GPU 6414.4 MB\n",
      "Epoch 00019 | Step 00240 | Loss 0.2633 | Train Acc 0.9141 | Speed (samples/sec) 341.9751 | GPU 6414.4 MB\n",
      "Epoch 00019 | Step 00260 | Loss 0.4010 | Train Acc 0.8984 | Speed (samples/sec) 342.2814 | GPU 6414.4 MB\n",
      "Epoch 00019 | Step 00280 | Loss 0.3049 | Train Acc 0.9160 | Speed (samples/sec) 342.5788 | GPU 6414.4 MB\n",
      "Epoch 00019 | Step 00300 | Loss 0.2487 | Train Acc 0.9141 | Speed (samples/sec) 342.8589 | GPU 6414.4 MB\n",
      "Epoch 00019 | Step 00320 | Loss 0.3177 | Train Acc 0.9043 | Speed (samples/sec) 343.1218 | GPU 6414.4 MB\n",
      "Epoch 00019 | Step 00340 | Loss 0.2768 | Train Acc 0.9277 | Speed (samples/sec) 343.3072 | GPU 6414.4 MB\n",
      "Epoch 00019 | Step 00360 | Loss 0.2803 | Train Acc 0.9121 | Speed (samples/sec) 343.4859 | GPU 6414.4 MB\n",
      "Epoch 00019 | Step 00380 | Loss 0.2667 | Train Acc 0.9238 | Speed (samples/sec) 343.6560 | GPU 6414.4 MB\n",
      "Epoch Time(s): 442.3673\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 245/245 [00:31<00:00,  7.77it/s]\n",
      "100%|██████████| 245/245 [01:37<00:00,  2.52it/s]\n",
      "100%|██████████| 245/245 [01:34<00:00,  2.59it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Val acc 0.9180, Test Acc: 0.7606\n",
      "Avg epoch time: 742.8609257857005\n",
      "Best val acc 0.9205, best test acc: 0.7622\n"
     ]
    }
   ],
   "source": [
    "g, num_labels = load_ogb('ogbn-products')\n",
    "g.ndata.pop('features')\n",
    "\n",
    "in_feats = 128\n",
    "nfeat = th.normal(0, 1, size=(g.number_of_nodes(), in_feats), dtype=th.float32)\n",
    "labels = g.ndata.pop('labels')\n",
    "\n",
    "hyperparams = {\n",
    "    'batch_size': 512,\n",
    "    'num_workers': 0,\n",
    "    'num_hidden': 256,\n",
    "    'num_layers': 3,\n",
    "    'num_heads': 2,\n",
    "    'dropout': 0.5,\n",
    "    'lr': 0.003,\n",
    "    'sparse_lr': 0.003,\n",
    "    'num_epochs': 20,\n",
    "    'fanouts': [15,10,5],\n",
    "    'eval_batch_size': 10000\n",
    "}\n",
    "device = th.device('cuda:7')\n",
    "\n",
    "# Pack data\n",
    "data = g, nfeat, labels\n",
    "# Define model\n",
    "model = GAT(in_feats, hyperparams['num_hidden'], num_labels,\n",
    "             hyperparams['num_layers'], hyperparams['num_heads'], F.relu, hyperparams['dropout'])\n",
    "train_ft_embed(model, data, hyperparams, device, 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}