GAT-sample-eigen-ft-products_gpu6.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using backend: pytorch\n"
     ]
    }
   ],
   "source": [
    "import dgl\n",
    "import time\n",
    "import torch as th\n",
    "import numpy as np\n",
    "import torch.nn.functional as F\n",
    "\n",
    "from load_data import load_ogb\n",
    "from model import SAGE, GAT, get_eigen\n",
    "from train_eval import train_ft_embed"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This notebook shows the performance of GAT on the OGBN-products graph with trainable embeddings on the nodes. We use scaled eigenvectors to initialize the node embeddings. The logic here is that if eigenvectors can serve as positional node embeddings, fine-tuning the node embeddings should give us even better performance. In this experiment, we tune the node embeddings with a much smaller learning rate than the one for the GAT model."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "load ogbn-products\n",
      "finish loading ogbn-products\n",
      "finish constructing ogbn-products\n",
      "embedding is in  cuda:6\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.6/dist-packages/numpy/core/fromnumeric.py:3373: RuntimeWarning: Mean of empty slice.\n",
      "  out=out, **kwargs)\n",
      "/usr/local/lib/python3.6/dist-packages/numpy/core/_methods.py:170: RuntimeWarning: invalid value encountered in double_scalars\n",
      "  ret = ret.dtype.type(ret / rcount)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 00000 | Step 00000 | Loss 3.8490 | Train Acc 0.0137 | Speed (samples/sec) nan | GPU 5588.5 MB\n",
      "Epoch 00000 | Step 00020 | Loss 1.8869 | Train Acc 0.4873 | Speed (samples/sec) 315.0262 | GPU 8123.7 MB\n",
      "Epoch 00000 | Step 00040 | Loss 1.2861 | Train Acc 0.6924 | Speed (samples/sec) 313.2386 | GPU 8138.2 MB\n",
      "Epoch 00000 | Step 00060 | Loss 0.8674 | Train Acc 0.7920 | Speed (samples/sec) 314.4934 | GPU 8138.2 MB\n",
      "Epoch 00000 | Step 00080 | Loss 0.6539 | Train Acc 0.8496 | Speed (samples/sec) 319.1254 | GPU 8138.2 MB\n",
      "Epoch 00000 | Step 00100 | Loss 0.5180 | Train Acc 0.8672 | Speed (samples/sec) 318.0370 | GPU 8138.2 MB\n",
      "Epoch 00000 | Step 00120 | Loss 0.4248 | Train Acc 0.8926 | Speed (samples/sec) 315.9198 | GPU 8138.2 MB\n",
      "Epoch 00000 | Step 00140 | Loss 0.4437 | Train Acc 0.8789 | Speed (samples/sec) 318.3943 | GPU 8138.2 MB\n",
      "Epoch 00000 | Step 00160 | Loss 0.4237 | Train Acc 0.8887 | Speed (samples/sec) 323.3136 | GPU 8138.2 MB\n",
      "Epoch 00000 | Step 00180 | Loss 0.3965 | Train Acc 0.9004 | Speed (samples/sec) 325.2781 | GPU 8138.2 MB\n",
      "Epoch Time(s): 608.9578\n",
      "Epoch 00001 | Step 00000 | Loss 0.3637 | Train Acc 0.8984 | Speed (samples/sec) 324.3338 | GPU 8138.2 MB\n",
      "Epoch 00001 | Step 00020 | Loss 0.4053 | Train Acc 0.8926 | Speed (samples/sec) 325.5296 | GPU 8138.2 MB\n",
      "Epoch 00001 | Step 00040 | Loss 0.3403 | Train Acc 0.9141 | Speed (samples/sec) 328.5803 | GPU 8138.2 MB\n",
      "Epoch 00001 | Step 00060 | Loss 0.3610 | Train Acc 0.9082 | Speed (samples/sec) 330.9533 | GPU 8138.2 MB\n",
      "Epoch 00001 | Step 00080 | Loss 0.4334 | Train Acc 0.8916 | Speed (samples/sec) 334.0681 | GPU 8138.2 MB\n",
      "Epoch 00001 | Step 00100 | Loss 0.3299 | Train Acc 0.9141 | Speed (samples/sec) 338.5337 | GPU 8138.2 MB\n",
      "Epoch 00001 | Step 00120 | Loss 0.3448 | Train Acc 0.9102 | Speed (samples/sec) 338.7892 | GPU 8149.1 MB\n",
      "Epoch 00001 | Step 00140 | Loss 0.3226 | Train Acc 0.9189 | Speed (samples/sec) 338.6678 | GPU 8149.1 MB\n",
      "Epoch 00001 | Step 00160 | Loss 0.3434 | Train Acc 0.9014 | Speed (samples/sec) 337.8047 | GPU 8149.1 MB\n",
      "Epoch 00001 | Step 00180 | Loss 0.4288 | Train Acc 0.8760 | Speed (samples/sec) 337.8155 | GPU 8149.1 MB\n",
      "Epoch Time(s): 562.8343\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 245/245 [00:50<00:00,  4.82it/s]\n",
      "100%|██████████| 245/245 [02:01<00:00,  2.01it/s]\n",
      "100%|██████████| 245/245 [01:48<00:00,  2.26it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Val acc 0.9116, Test Acc: 0.7817\n",
      "Best val acc 0.9116, best test acc: 0.7817\n",
      "Epoch 00002 | Step 00000 | Loss 0.3680 | Train Acc 0.8975 | Speed (samples/sec) 337.7296 | GPU 8149.1 MB\n",
      "Epoch 00002 | Step 00020 | Loss 0.3212 | Train Acc 0.9160 | Speed (samples/sec) 339.2276 | GPU 8149.1 MB\n",
      "Epoch 00002 | Step 00040 | Loss 0.3693 | Train Acc 0.9072 | Speed (samples/sec) 339.4118 | GPU 8149.1 MB\n",
      "Epoch 00002 | Step 00060 | Loss 0.3558 | Train Acc 0.9043 | Speed (samples/sec) 339.8712 | GPU 8149.1 MB\n",
      "Epoch 00002 | Step 00080 | Loss 0.3283 | Train Acc 0.8975 | Speed (samples/sec) 341.1143 | GPU 8149.1 MB\n",
      "Epoch 00002 | Step 00100 | Loss 0.4010 | Train Acc 0.8994 | Speed (samples/sec) 341.1471 | GPU 8159.8 MB\n",
      "Epoch 00002 | Step 00120 | Loss 0.3119 | Train Acc 0.9092 | Speed (samples/sec) 341.1300 | GPU 8159.8 MB\n",
      "Epoch 00002 | Step 00140 | Loss 0.2889 | Train Acc 0.9072 | Speed (samples/sec) 341.4259 | GPU 8159.8 MB\n",
      "Epoch 00002 | Step 00160 | Loss 0.3774 | Train Acc 0.8896 | Speed (samples/sec) 342.3280 | GPU 8159.8 MB\n",
      "Epoch 00002 | Step 00180 | Loss 0.2882 | Train Acc 0.9131 | Speed (samples/sec) 342.5807 | GPU 8159.8 MB\n",
      "Epoch Time(s): 561.7618\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 245/245 [00:50<00:00,  4.86it/s]\n",
      "100%|██████████| 245/245 [01:58<00:00,  2.07it/s]\n",
      "100%|██████████| 245/245 [01:52<00:00,  2.17it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Val acc 0.9158, Test Acc: 0.8001\n",
      "Best val acc 0.9158, best test acc: 0.8001\n",
      "Epoch 00003 | Step 00000 | Loss 0.3432 | Train Acc 0.9053 | Speed (samples/sec) 341.8140 | GPU 8159.8 MB\n",
      "Epoch 00003 | Step 00020 | Loss 0.3226 | Train Acc 0.9102 | Speed (samples/sec) 341.8376 | GPU 8159.8 MB\n",
      "Epoch 00003 | Step 00040 | Loss 0.2999 | Train Acc 0.9170 | Speed (samples/sec) 341.6139 | GPU 8159.8 MB\n",
      "Epoch 00003 | Step 00060 | Loss 0.3133 | Train Acc 0.9072 | Speed (samples/sec) 340.4387 | GPU 8159.8 MB\n",
      "Epoch 00003 | Step 00080 | Loss 0.3764 | Train Acc 0.8965 | Speed (samples/sec) 339.1488 | GPU 8159.8 MB\n",
      "Epoch 00003 | Step 00100 | Loss 0.3136 | Train Acc 0.9111 | Speed (samples/sec) 338.3695 | GPU 8159.8 MB\n",
      "Epoch 00003 | Step 00120 | Loss 0.3239 | Train Acc 0.9053 | Speed (samples/sec) 337.8151 | GPU 8159.8 MB\n",
      "Epoch 00003 | Step 00140 | Loss 0.2772 | Train Acc 0.9219 | Speed (samples/sec) 337.0579 | GPU 8159.8 MB\n",
      "Epoch 00003 | Step 00160 | Loss 0.2837 | Train Acc 0.9209 | Speed (samples/sec) 336.2824 | GPU 8159.8 MB\n",
      "Epoch 00003 | Step 00180 | Loss 0.2626 | Train Acc 0.9189 | Speed (samples/sec) 335.6344 | GPU 8159.8 MB\n",
      "Epoch Time(s): 624.3250\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 245/245 [00:51<00:00,  4.75it/s]\n",
      "100%|██████████| 245/245 [02:11<00:00,  1.87it/s]\n",
      "100%|██████████| 245/245 [02:01<00:00,  2.01it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Val acc 0.9152, Test Acc: 0.7921\n",
      "Epoch 00004 | Step 00000 | Loss 0.2698 | Train Acc 0.9180 | Speed (samples/sec) 335.0581 | GPU 8159.8 MB\n",
      "Epoch 00004 | Step 00020 | Loss 0.2865 | Train Acc 0.9180 | Speed (samples/sec) 334.2661 | GPU 8159.8 MB\n",
      "Epoch 00004 | Step 00040 | Loss 0.3053 | Train Acc 0.9072 | Speed (samples/sec) 333.5110 | GPU 8159.8 MB\n",
      "Epoch 00004 | Step 00060 | Loss 0.2365 | Train Acc 0.9258 | Speed (samples/sec) 332.8274 | GPU 8159.8 MB\n",
      "Epoch 00004 | Step 00080 | Loss 0.2663 | Train Acc 0.9238 | Speed (samples/sec) 332.5398 | GPU 8159.8 MB\n",
      "Epoch 00004 | Step 00100 | Loss 0.3146 | Train Acc 0.9092 | Speed (samples/sec) 331.7109 | GPU 8159.8 MB\n",
      "Epoch 00004 | Step 00120 | Loss 0.2798 | Train Acc 0.9121 | Speed (samples/sec) 331.1364 | GPU 8159.8 MB\n",
      "Epoch 00004 | Step 00140 | Loss 0.2991 | Train Acc 0.9170 | Speed (samples/sec) 331.0223 | GPU 8159.8 MB\n",
      "Epoch 00004 | Step 00160 | Loss 0.2580 | Train Acc 0.9150 | Speed (samples/sec) 331.3182 | GPU 8159.8 MB\n",
      "Epoch 00004 | Step 00180 | Loss 0.3180 | Train Acc 0.9102 | Speed (samples/sec) 331.2637 | GPU 8159.8 MB\n",
      "Epoch Time(s): 626.9495\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 245/245 [00:52<00:00,  4.70it/s]\n",
      "100%|██████████| 245/245 [02:04<00:00,  1.97it/s]\n",
      "100%|██████████| 245/245 [02:05<00:00,  1.95it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Val acc 0.9145, Test Acc: 0.7967\n",
      "Epoch 00005 | Step 00000 | Loss 0.3170 | Train Acc 0.8994 | Speed (samples/sec) 330.8348 | GPU 8159.8 MB\n",
      "Epoch 00005 | Step 00020 | Loss 0.3160 | Train Acc 0.9131 | Speed (samples/sec) 330.7067 | GPU 8159.8 MB\n",
      "Epoch 00005 | Step 00040 | Loss 0.2798 | Train Acc 0.9111 | Speed (samples/sec) 330.7291 | GPU 8159.8 MB\n",
      "Epoch 00005 | Step 00060 | Loss 0.2492 | Train Acc 0.9189 | Speed (samples/sec) 330.3017 | GPU 8159.8 MB\n",
      "Epoch 00005 | Step 00080 | Loss 0.2433 | Train Acc 0.9219 | Speed (samples/sec) 330.1199 | GPU 8159.8 MB\n",
      "Epoch 00005 | Step 00100 | Loss 0.2708 | Train Acc 0.9297 | Speed (samples/sec) 330.3168 | GPU 8159.8 MB\n",
      "Epoch 00005 | Step 00120 | Loss 0.3101 | Train Acc 0.9014 | Speed (samples/sec) 330.4711 | GPU 8159.8 MB\n",
      "Epoch 00005 | Step 00140 | Loss 0.2752 | Train Acc 0.9199 | Speed (samples/sec) 330.3273 | GPU 8159.8 MB\n",
      "Epoch 00005 | Step 00160 | Loss 0.2990 | Train Acc 0.9033 | Speed (samples/sec) 330.2013 | GPU 8159.8 MB\n",
      "Epoch 00005 | Step 00180 | Loss 0.3007 | Train Acc 0.9072 | Speed (samples/sec) 330.1181 | GPU 8159.8 MB\n",
      "Epoch Time(s): 604.5095\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 245/245 [00:51<00:00,  4.74it/s]\n",
      "100%|██████████| 245/245 [02:12<00:00,  1.85it/s]\n",
      "100%|██████████| 245/245 [01:46<00:00,  2.29it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Val acc 0.9190, Test Acc: 0.7936\n",
      "Best val acc 0.9190, best test acc: 0.7936\n",
      "Epoch 00006 | Step 00000 | Loss 0.2659 | Train Acc 0.9150 | Speed (samples/sec) 329.8696 | GPU 8159.8 MB\n",
      "Epoch 00006 | Step 00020 | Loss 0.2978 | Train Acc 0.9160 | Speed (samples/sec) 329.9467 | GPU 8159.8 MB\n",
      "Epoch 00006 | Step 00040 | Loss 0.2587 | Train Acc 0.9268 | Speed (samples/sec) 329.9762 | GPU 8159.8 MB\n",
      "Epoch 00006 | Step 00060 | Loss 0.3294 | Train Acc 0.9072 | Speed (samples/sec) 330.0468 | GPU 8159.8 MB\n",
      "Epoch 00006 | Step 00080 | Loss 0.2275 | Train Acc 0.9229 | Speed (samples/sec) 330.0835 | GPU 8159.8 MB\n",
      "Epoch 00006 | Step 00100 | Loss 0.2821 | Train Acc 0.9150 | Speed (samples/sec) 330.1801 | GPU 8159.8 MB\n",
      "Epoch 00006 | Step 00120 | Loss 0.2529 | Train Acc 0.9346 | Speed (samples/sec) 330.0567 | GPU 8159.8 MB\n",
      "Epoch 00006 | Step 00140 | Loss 0.2551 | Train Acc 0.9258 | Speed (samples/sec) 329.9319 | GPU 8159.8 MB\n",
      "Epoch 00006 | Step 00160 | Loss 0.2555 | Train Acc 0.9180 | Speed (samples/sec) 329.8391 | GPU 8159.8 MB\n",
      "Epoch 00006 | Step 00180 | Loss 0.2819 | Train Acc 0.9141 | Speed (samples/sec) 329.9049 | GPU 8159.8 MB\n",
      "Epoch Time(s): 597.6022\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 245/245 [00:51<00:00,  4.72it/s]\n",
      "100%|██████████| 245/245 [02:05<00:00,  1.96it/s]\n",
      "100%|██████████| 245/245 [01:50<00:00,  2.21it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Val acc 0.9207, Test Acc: 0.7964\n",
      "Best val acc 0.9207, best test acc: 0.7964\n",
      "Epoch 00007 | Step 00000 | Loss 0.2714 | Train Acc 0.9316 | Speed (samples/sec) 329.7566 | GPU 8159.8 MB\n",
      "Epoch 00007 | Step 00020 | Loss 0.2833 | Train Acc 0.9111 | Speed (samples/sec) 329.8308 | GPU 8159.8 MB\n",
      "Epoch 00007 | Step 00040 | Loss 0.2993 | Train Acc 0.9043 | Speed (samples/sec) 329.9013 | GPU 8159.8 MB\n",
      "Epoch 00007 | Step 00060 | Loss 0.3184 | Train Acc 0.9111 | Speed (samples/sec) 330.1595 | GPU 8159.8 MB\n",
      "Epoch 00007 | Step 00080 | Loss 0.2493 | Train Acc 0.9258 | Speed (samples/sec) 329.8819 | GPU 8159.8 MB\n",
      "Epoch 00007 | Step 00100 | Loss 0.3003 | Train Acc 0.9043 | Speed (samples/sec) 329.6428 | GPU 8159.8 MB\n",
      "Epoch 00007 | Step 00120 | Loss 0.2893 | Train Acc 0.9102 | Speed (samples/sec) 329.6471 | GPU 8159.8 MB\n",
      "Epoch 00007 | Step 00140 | Loss 0.3014 | Train Acc 0.9170 | Speed (samples/sec) 329.8491 | GPU 8159.8 MB\n",
      "Epoch 00007 | Step 00160 | Loss 0.3200 | Train Acc 0.9170 | Speed (samples/sec) 329.7175 | GPU 8159.8 MB\n",
      "Epoch 00007 | Step 00180 | Loss 0.2957 | Train Acc 0.9170 | Speed (samples/sec) 329.5161 | GPU 8159.8 MB\n",
      "Epoch Time(s): 604.7985\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 245/245 [00:49<00:00,  4.95it/s]\n",
      "100%|██████████| 245/245 [02:02<00:00,  2.00it/s]\n",
      "100%|██████████| 245/245 [01:55<00:00,  2.12it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Val acc 0.9198, Test Acc: 0.7952\n",
      "Epoch 00008 | Step 00000 | Loss 0.2948 | Train Acc 0.9189 | Speed (samples/sec) 329.2125 | GPU 8159.8 MB\n",
      "Epoch 00008 | Step 00020 | Loss 0.2432 | Train Acc 0.9180 | Speed (samples/sec) 329.4098 | GPU 8159.8 MB\n",
      "Epoch 00008 | Step 00040 | Loss 0.2743 | Train Acc 0.9209 | Speed (samples/sec) 329.3480 | GPU 8159.8 MB\n",
      "Epoch 00008 | Step 00060 | Loss 0.2687 | Train Acc 0.9160 | Speed (samples/sec) 329.1992 | GPU 8159.8 MB\n",
      "Epoch 00008 | Step 00080 | Loss 0.2917 | Train Acc 0.9082 | Speed (samples/sec) 329.3530 | GPU 8159.8 MB\n",
      "Epoch 00008 | Step 00100 | Loss 0.2947 | Train Acc 0.9131 | Speed (samples/sec) 329.7897 | GPU 8159.8 MB\n",
      "Epoch 00008 | Step 00120 | Loss 0.2540 | Train Acc 0.9268 | Speed (samples/sec) 329.6129 | GPU 8159.8 MB\n",
      "Epoch 00008 | Step 00140 | Loss 0.2904 | Train Acc 0.9102 | Speed (samples/sec) 329.3688 | GPU 8159.8 MB\n",
      "Epoch 00008 | Step 00160 | Loss 0.3046 | Train Acc 0.9121 | Speed (samples/sec) 329.3704 | GPU 8159.8 MB\n",
      "Epoch 00008 | Step 00180 | Loss 0.2380 | Train Acc 0.9229 | Speed (samples/sec) 329.4790 | GPU 8159.8 MB\n",
      "Epoch Time(s): 596.4294\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 245/245 [00:51<00:00,  4.77it/s]\n",
      "100%|██████████| 245/245 [02:06<00:00,  1.93it/s]\n",
      "100%|██████████| 245/245 [02:01<00:00,  2.01it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Val acc 0.9217, Test Acc: 0.7992\n",
      "Best val acc 0.9217, best test acc: 0.7992\n",
      "Epoch 00009 | Step 00000 | Loss 0.2617 | Train Acc 0.9199 | Speed (samples/sec) 329.3084 | GPU 8159.8 MB\n",
      "Epoch 00009 | Step 00020 | Loss 0.2640 | Train Acc 0.9258 | Speed (samples/sec) 329.1104 | GPU 8159.8 MB\n",
      "Epoch 00009 | Step 00040 | Loss 0.2765 | Train Acc 0.9053 | Speed (samples/sec) 329.3928 | GPU 8159.8 MB\n",
      "Epoch 00009 | Step 00060 | Loss 0.2532 | Train Acc 0.9248 | Speed (samples/sec) 329.6174 | GPU 8159.8 MB\n",
      "Epoch 00009 | Step 00080 | Loss 0.2913 | Train Acc 0.9072 | Speed (samples/sec) 329.4970 | GPU 8159.8 MB\n",
      "Epoch 00009 | Step 00100 | Loss 0.2868 | Train Acc 0.9141 | Speed (samples/sec) 329.3670 | GPU 8159.8 MB\n",
      "Epoch 00009 | Step 00120 | Loss 0.2477 | Train Acc 0.9229 | Speed (samples/sec) 329.7210 | GPU 8159.8 MB\n",
      "Epoch 00009 | Step 00140 | Loss 0.2628 | Train Acc 0.9297 | Speed (samples/sec) 330.1233 | GPU 8159.8 MB\n",
      "Epoch 00009 | Step 00160 | Loss 0.2713 | Train Acc 0.9180 | Speed (samples/sec) 330.0710 | GPU 8159.8 MB\n",
      "Epoch 00009 | Step 00180 | Loss 0.2920 | Train Acc 0.9180 | Speed (samples/sec) 330.0014 | GPU 8159.8 MB\n",
      "Epoch Time(s): 588.5062\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 245/245 [00:48<00:00,  5.08it/s]\n",
      "100%|██████████| 245/245 [02:10<00:00,  1.88it/s]\n",
      "100%|██████████| 245/245 [01:50<00:00,  2.22it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Val acc 0.9119, Test Acc: 0.7861\n",
      "Epoch 00010 | Step 00000 | Loss 0.2867 | Train Acc 0.9150 | Speed (samples/sec) 329.8825 | GPU 8159.8 MB\n",
      "Epoch 00010 | Step 00020 | Loss 0.2287 | Train Acc 0.9307 | Speed (samples/sec) 330.0202 | GPU 8159.8 MB\n",
      "Epoch 00010 | Step 00040 | Loss 0.3067 | Train Acc 0.9033 | Speed (samples/sec) 329.9224 | GPU 8159.8 MB\n",
      "Epoch 00010 | Step 00060 | Loss 0.2560 | Train Acc 0.9238 | Speed (samples/sec) 329.8007 | GPU 8159.8 MB\n",
      "Epoch 00010 | Step 00080 | Loss 0.2263 | Train Acc 0.9238 | Speed (samples/sec) 330.0904 | GPU 8159.8 MB\n",
      "Epoch 00010 | Step 00100 | Loss 0.2593 | Train Acc 0.9180 | Speed (samples/sec) 330.4321 | GPU 8159.8 MB\n",
      "Epoch 00010 | Step 00120 | Loss 0.2327 | Train Acc 0.9287 | Speed (samples/sec) 330.3518 | GPU 8159.8 MB\n",
      "Epoch 00010 | Step 00140 | Loss 0.2682 | Train Acc 0.9150 | Speed (samples/sec) 330.3435 | GPU 8159.8 MB\n",
      "Epoch 00010 | Step 00160 | Loss 0.2736 | Train Acc 0.9170 | Speed (samples/sec) 330.4151 | GPU 8159.8 MB\n",
      "Epoch 00010 | Step 00180 | Loss 0.2493 | Train Acc 0.9238 | Speed (samples/sec) 330.6800 | GPU 8159.8 MB\n",
      "Epoch Time(s): 586.5079\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 245/245 [00:41<00:00,  5.86it/s]\n",
      "100%|██████████| 245/245 [01:45<00:00,  2.33it/s]\n",
      "100%|██████████| 245/245 [01:36<00:00,  2.54it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Val acc 0.9223, Test Acc: 0.7998\n",
      "Best val acc 0.9223, best test acc: 0.7998\n",
      "Epoch 00011 | Step 00000 | Loss 0.2363 | Train Acc 0.9307 | Speed (samples/sec) 330.4479 | GPU 8159.8 MB\n",
      "Epoch 00011 | Step 00020 | Loss 0.3013 | Train Acc 0.9102 | Speed (samples/sec) 330.9281 | GPU 8159.8 MB\n",
      "Epoch 00011 | Step 00040 | Loss 0.2489 | Train Acc 0.9287 | Speed (samples/sec) 331.5316 | GPU 8159.8 MB\n",
      "Epoch 00011 | Step 00060 | Loss 0.2648 | Train Acc 0.9141 | Speed (samples/sec) 332.1338 | GPU 8159.8 MB\n",
      "Epoch 00011 | Step 00080 | Loss 0.2255 | Train Acc 0.9346 | Speed (samples/sec) 332.6635 | GPU 8159.8 MB\n",
      "Epoch 00011 | Step 00100 | Loss 0.2459 | Train Acc 0.9287 | Speed (samples/sec) 333.1839 | GPU 8159.8 MB\n",
      "Epoch 00011 | Step 00120 | Loss 0.2672 | Train Acc 0.9268 | Speed (samples/sec) 333.6524 | GPU 8159.8 MB\n",
      "Epoch 00011 | Step 00140 | Loss 0.2584 | Train Acc 0.9150 | Speed (samples/sec) 334.2462 | GPU 8159.8 MB\n",
      "Epoch 00011 | Step 00160 | Loss 0.3060 | Train Acc 0.9111 | Speed (samples/sec) 334.7444 | GPU 8159.8 MB\n",
      "Epoch 00011 | Step 00180 | Loss 0.2716 | Train Acc 0.9092 | Speed (samples/sec) 335.1587 | GPU 8159.8 MB\n",
      "Epoch Time(s): 505.0245\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 245/245 [00:43<00:00,  5.67it/s]\n",
      "100%|██████████| 245/245 [01:45<00:00,  2.31it/s]\n",
      "100%|██████████| 245/245 [01:39<00:00,  2.47it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Val acc 0.9186, Test Acc: 0.7919\n",
      "Epoch 00012 | Step 00000 | Loss 0.2668 | Train Acc 0.9102 | Speed (samples/sec) 335.2801 | GPU 8159.8 MB\n",
      "Epoch 00012 | Step 00020 | Loss 0.2671 | Train Acc 0.9189 | Speed (samples/sec) 335.7258 | GPU 8159.8 MB\n",
      "Epoch 00012 | Step 00040 | Loss 0.2264 | Train Acc 0.9346 | Speed (samples/sec) 336.1434 | GPU 8159.8 MB\n",
      "Epoch 00012 | Step 00060 | Loss 0.2336 | Train Acc 0.9268 | Speed (samples/sec) 336.5737 | GPU 8159.8 MB\n",
      "Epoch 00012 | Step 00080 | Loss 0.2672 | Train Acc 0.9219 | Speed (samples/sec) 336.9658 | GPU 8159.8 MB\n",
      "Epoch 00012 | Step 00100 | Loss 0.2311 | Train Acc 0.9238 | Speed (samples/sec) 337.3361 | GPU 8159.8 MB\n",
      "Epoch 00012 | Step 00120 | Loss 0.2967 | Train Acc 0.9160 | Speed (samples/sec) 337.6949 | GPU 8159.8 MB\n",
      "Epoch 00012 | Step 00140 | Loss 0.2641 | Train Acc 0.9229 | Speed (samples/sec) 338.1053 | GPU 8159.8 MB\n",
      "Epoch 00012 | Step 00160 | Loss 0.2671 | Train Acc 0.9248 | Speed (samples/sec) 338.5427 | GPU 8159.8 MB\n",
      "Epoch 00012 | Step 00180 | Loss 0.2623 | Train Acc 0.9150 | Speed (samples/sec) 338.9400 | GPU 8159.8 MB\n",
      "Epoch Time(s): 509.7330\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 245/245 [00:41<00:00,  5.97it/s]\n",
      "100%|██████████| 245/245 [01:46<00:00,  2.30it/s]\n",
      "100%|██████████| 245/245 [01:34<00:00,  2.60it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Val acc 0.9220, Test Acc: 0.7910\n",
      "Epoch 00013 | Step 00000 | Loss 0.2426 | Train Acc 0.9248 | Speed (samples/sec) 339.0953 | GPU 8159.8 MB\n",
      "Epoch 00013 | Step 00020 | Loss 0.2327 | Train Acc 0.9316 | Speed (samples/sec) 339.4732 | GPU 8159.8 MB\n",
      "Epoch 00013 | Step 00040 | Loss 0.2385 | Train Acc 0.9248 | Speed (samples/sec) 339.8573 | GPU 8159.8 MB\n",
      "Epoch 00013 | Step 00060 | Loss 0.2567 | Train Acc 0.9180 | Speed (samples/sec) 340.1831 | GPU 8159.8 MB\n",
      "Epoch 00013 | Step 00080 | Loss 0.2469 | Train Acc 0.9160 | Speed (samples/sec) 340.5328 | GPU 8159.8 MB\n",
      "Epoch 00013 | Step 00100 | Loss 0.2950 | Train Acc 0.9180 | Speed (samples/sec) 340.8879 | GPU 8159.8 MB\n",
      "Epoch 00013 | Step 00120 | Loss 0.2276 | Train Acc 0.9268 | Speed (samples/sec) 341.2673 | GPU 8159.8 MB\n",
      "Epoch 00013 | Step 00140 | Loss 0.2288 | Train Acc 0.9316 | Speed (samples/sec) 341.6012 | GPU 8159.8 MB\n",
      "Epoch 00013 | Step 00160 | Loss 0.2684 | Train Acc 0.9180 | Speed (samples/sec) 341.9540 | GPU 8159.8 MB\n",
      "Epoch 00013 | Step 00180 | Loss 0.2832 | Train Acc 0.9131 | Speed (samples/sec) 342.2487 | GPU 8159.8 MB\n",
      "Epoch Time(s): 509.4561\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 245/245 [00:41<00:00,  5.94it/s]\n",
      "100%|██████████| 245/245 [01:43<00:00,  2.36it/s]\n",
      "100%|██████████| 245/245 [01:35<00:00,  2.56it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Val acc 0.9213, Test Acc: 0.7943\n",
      "Epoch 00014 | Step 00000 | Loss 0.2372 | Train Acc 0.9258 | Speed (samples/sec) 342.3701 | GPU 8159.8 MB\n",
      "Epoch 00014 | Step 00020 | Loss 0.2295 | Train Acc 0.9307 | Speed (samples/sec) 342.6969 | GPU 8159.8 MB\n",
      "Epoch 00014 | Step 00040 | Loss 0.2454 | Train Acc 0.9268 | Speed (samples/sec) 343.0883 | GPU 8159.8 MB\n",
      "Epoch 00014 | Step 00060 | Loss 0.2024 | Train Acc 0.9365 | Speed (samples/sec) 343.4358 | GPU 8159.8 MB\n",
      "Epoch 00014 | Step 00080 | Loss 0.2289 | Train Acc 0.9248 | Speed (samples/sec) 343.7556 | GPU 8159.8 MB\n",
      "Epoch 00014 | Step 00100 | Loss 0.2265 | Train Acc 0.9316 | Speed (samples/sec) 344.0977 | GPU 8159.8 MB\n",
      "Epoch 00014 | Step 00120 | Loss 0.2499 | Train Acc 0.9062 | Speed (samples/sec) 344.4124 | GPU 8159.8 MB\n",
      "Epoch 00014 | Step 00140 | Loss 0.2705 | Train Acc 0.9111 | Speed (samples/sec) 344.6665 | GPU 8159.8 MB\n",
      "Epoch 00014 | Step 00160 | Loss 0.2078 | Train Acc 0.9355 | Speed (samples/sec) 344.9477 | GPU 8159.8 MB\n",
      "Epoch 00014 | Step 00180 | Loss 0.2257 | Train Acc 0.9268 | Speed (samples/sec) 345.2105 | GPU 8159.8 MB\n",
      "Epoch Time(s): 507.4373\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 245/245 [00:41<00:00,  5.93it/s]\n",
      "100%|██████████| 245/245 [01:44<00:00,  2.34it/s]\n",
      "100%|██████████| 245/245 [01:32<00:00,  2.65it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Val acc 0.9209, Test Acc: 0.7928\n",
      "Epoch 00015 | Step 00000 | Loss 0.2696 | Train Acc 0.9229 | Speed (samples/sec) 345.3170 | GPU 8159.8 MB\n",
      "Epoch 00015 | Step 00020 | Loss 0.2527 | Train Acc 0.9297 | Speed (samples/sec) 345.8061 | GPU 8159.8 MB\n",
      "Epoch 00015 | Step 00040 | Loss 0.2336 | Train Acc 0.9277 | Speed (samples/sec) 346.2562 | GPU 8159.8 MB\n",
      "Epoch 00015 | Step 00060 | Loss 0.2806 | Train Acc 0.9043 | Speed (samples/sec) 346.7401 | GPU 8159.8 MB\n",
      "Epoch 00015 | Step 00080 | Loss 0.2462 | Train Acc 0.9346 | Speed (samples/sec) 347.2609 | GPU 8159.8 MB\n",
      "Epoch 00015 | Step 00100 | Loss 0.2456 | Train Acc 0.9189 | Speed (samples/sec) 347.7634 | GPU 8159.8 MB\n",
      "Epoch 00015 | Step 00120 | Loss 0.2426 | Train Acc 0.9268 | Speed (samples/sec) 348.2091 | GPU 8159.8 MB\n",
      "Epoch 00015 | Step 00140 | Loss 0.2718 | Train Acc 0.9121 | Speed (samples/sec) 348.6156 | GPU 8159.8 MB\n",
      "Epoch 00015 | Step 00160 | Loss 0.2294 | Train Acc 0.9209 | Speed (samples/sec) 349.0007 | GPU 8159.8 MB\n",
      "Epoch 00015 | Step 00180 | Loss 0.2246 | Train Acc 0.9307 | Speed (samples/sec) 349.4012 | GPU 8159.8 MB\n",
      "Epoch Time(s): 475.1758\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 245/245 [00:38<00:00,  6.40it/s]\n",
      "100%|██████████| 245/245 [01:44<00:00,  2.35it/s]\n",
      "100%|██████████| 245/245 [01:35<00:00,  2.57it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Val acc 0.9223, Test Acc: 0.7885\n",
      "Best val acc 0.9223, best test acc: 0.7885\n",
      "Epoch 00016 | Step 00000 | Loss 0.2190 | Train Acc 0.9297 | Speed (samples/sec) 349.5479 | GPU 8159.8 MB\n",
      "Epoch 00016 | Step 00020 | Loss 0.2008 | Train Acc 0.9434 | Speed (samples/sec) 349.9758 | GPU 8159.8 MB\n",
      "Epoch 00016 | Step 00040 | Loss 0.2552 | Train Acc 0.9189 | Speed (samples/sec) 350.3480 | GPU 8159.8 MB\n",
      "Epoch 00016 | Step 00060 | Loss 0.2469 | Train Acc 0.9238 | Speed (samples/sec) 350.7215 | GPU 8159.8 MB\n",
      "Epoch 00016 | Step 00080 | Loss 0.2790 | Train Acc 0.9199 | Speed (samples/sec) 351.1228 | GPU 8159.8 MB\n",
      "Epoch 00016 | Step 00100 | Loss 0.2979 | Train Acc 0.9150 | Speed (samples/sec) 351.4499 | GPU 8159.8 MB\n",
      "Epoch 00016 | Step 00120 | Loss 0.2471 | Train Acc 0.9316 | Speed (samples/sec) 351.8449 | GPU 8159.8 MB\n",
      "Epoch 00016 | Step 00140 | Loss 0.2808 | Train Acc 0.9199 | Speed (samples/sec) 352.2556 | GPU 8159.8 MB\n",
      "Epoch 00016 | Step 00160 | Loss 0.2708 | Train Acc 0.9141 | Speed (samples/sec) 352.6433 | GPU 8159.8 MB\n",
      "Epoch 00016 | Step 00180 | Loss 0.2750 | Train Acc 0.9277 | Speed (samples/sec) 353.0737 | GPU 8159.8 MB\n",
      "Epoch Time(s): 476.1735\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 245/245 [00:39<00:00,  6.27it/s]\n",
      "100%|██████████| 245/245 [01:44<00:00,  2.35it/s]\n",
      "100%|██████████| 245/245 [01:29<00:00,  2.74it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Val acc 0.9181, Test Acc: 0.7863\n",
      "Epoch 00017 | Step 00000 | Loss 0.2490 | Train Acc 0.9248 | Speed (samples/sec) 353.2343 | GPU 8159.8 MB\n",
      "Epoch 00017 | Step 00020 | Loss 0.2143 | Train Acc 0.9326 | Speed (samples/sec) 353.7422 | GPU 8159.8 MB\n",
      "Epoch 00017 | Step 00040 | Loss 0.2865 | Train Acc 0.9141 | Speed (samples/sec) 354.1636 | GPU 8159.8 MB\n",
      "Epoch 00017 | Step 00060 | Loss 0.2329 | Train Acc 0.9346 | Speed (samples/sec) 354.5906 | GPU 8159.8 MB\n",
      "Epoch 00017 | Step 00080 | Loss 0.2656 | Train Acc 0.9160 | Speed (samples/sec) 355.0421 | GPU 8159.8 MB\n",
      "Epoch 00017 | Step 00100 | Loss 0.2415 | Train Acc 0.9160 | Speed (samples/sec) 355.5334 | GPU 8159.8 MB\n",
      "Epoch 00017 | Step 00120 | Loss 0.2762 | Train Acc 0.9160 | Speed (samples/sec) 356.0085 | GPU 8159.8 MB\n",
      "Epoch 00017 | Step 00140 | Loss 0.2519 | Train Acc 0.9277 | Speed (samples/sec) 356.4718 | GPU 8159.8 MB\n",
      "Epoch 00017 | Step 00160 | Loss 0.2473 | Train Acc 0.9141 | Speed (samples/sec) 356.9269 | GPU 8159.8 MB\n",
      "Epoch 00017 | Step 00180 | Loss 0.2186 | Train Acc 0.9375 | Speed (samples/sec) 357.3857 | GPU 8159.8 MB\n",
      "Epoch Time(s): 455.0644\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 245/245 [00:37<00:00,  6.56it/s]\n",
      "100%|██████████| 245/245 [01:41<00:00,  2.41it/s]\n",
      "100%|██████████| 245/245 [01:33<00:00,  2.61it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Val acc 0.9213, Test Acc: 0.7917\n",
      "Epoch 00018 | Step 00000 | Loss 0.2835 | Train Acc 0.9170 | Speed (samples/sec) 357.5810 | GPU 8159.8 MB\n",
      "Epoch 00018 | Step 00020 | Loss 0.2417 | Train Acc 0.9258 | Speed (samples/sec) 357.9221 | GPU 8159.8 MB\n",
      "Epoch 00018 | Step 00040 | Loss 0.2661 | Train Acc 0.9141 | Speed (samples/sec) 358.3325 | GPU 8159.8 MB\n",
      "Epoch 00018 | Step 00060 | Loss 0.2982 | Train Acc 0.9062 | Speed (samples/sec) 358.7283 | GPU 8159.8 MB\n",
      "Epoch 00018 | Step 00080 | Loss 0.3204 | Train Acc 0.9072 | Speed (samples/sec) 359.0490 | GPU 8159.8 MB\n",
      "Epoch 00018 | Step 00100 | Loss 0.2456 | Train Acc 0.9219 | Speed (samples/sec) 359.3945 | GPU 8159.8 MB\n",
      "Epoch 00018 | Step 00120 | Loss 0.2646 | Train Acc 0.9121 | Speed (samples/sec) 359.7576 | GPU 8159.8 MB\n",
      "Epoch 00018 | Step 00140 | Loss 0.2129 | Train Acc 0.9277 | Speed (samples/sec) 360.2033 | GPU 8159.8 MB\n",
      "Epoch 00018 | Step 00160 | Loss 0.2638 | Train Acc 0.9238 | Speed (samples/sec) 360.5990 | GPU 8159.8 MB\n",
      "Epoch 00018 | Step 00180 | Loss 0.2721 | Train Acc 0.9170 | Speed (samples/sec) 360.9965 | GPU 8159.8 MB\n",
      "Epoch Time(s): 461.3085\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 245/245 [00:35<00:00,  6.96it/s]\n",
      "100%|██████████| 245/245 [01:42<00:00,  2.39it/s]\n",
      "100%|██████████| 245/245 [01:33<00:00,  2.63it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Val acc 0.9222, Test Acc: 0.7939\n",
      "Epoch 00019 | Step 00000 | Loss 0.2356 | Train Acc 0.9238 | Speed (samples/sec) 361.1536 | GPU 8159.8 MB\n",
      "Epoch 00019 | Step 00020 | Loss 0.2486 | Train Acc 0.9141 | Speed (samples/sec) 361.4866 | GPU 8159.8 MB\n",
      "Epoch 00019 | Step 00040 | Loss 0.2841 | Train Acc 0.9131 | Speed (samples/sec) 361.8552 | GPU 8159.8 MB\n",
      "Epoch 00019 | Step 00060 | Loss 0.2481 | Train Acc 0.9219 | Speed (samples/sec) 362.2297 | GPU 8159.8 MB\n",
      "Epoch 00019 | Step 00080 | Loss 0.2452 | Train Acc 0.9170 | Speed (samples/sec) 362.5385 | GPU 8159.8 MB\n",
      "Epoch 00019 | Step 00100 | Loss 0.1988 | Train Acc 0.9336 | Speed (samples/sec) 362.8643 | GPU 8159.8 MB\n",
      "Epoch 00019 | Step 00120 | Loss 0.2393 | Train Acc 0.9219 | Speed (samples/sec) 363.2672 | GPU 8159.8 MB\n",
      "Epoch 00019 | Step 00140 | Loss 0.2558 | Train Acc 0.9307 | Speed (samples/sec) 363.6041 | GPU 8159.8 MB\n",
      "Epoch 00019 | Step 00160 | Loss 0.2388 | Train Acc 0.9277 | Speed (samples/sec) 364.0545 | GPU 8159.8 MB\n",
      "Epoch 00019 | Step 00180 | Loss 0.3059 | Train Acc 0.9082 | Speed (samples/sec) 364.5045 | GPU 8159.8 MB\n",
      "Epoch Time(s): 454.9714\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 245/245 [00:33<00:00,  7.28it/s]\n",
      "100%|██████████| 245/245 [01:39<00:00,  2.45it/s]\n",
      "100%|██████████| 245/245 [01:32<00:00,  2.65it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Val acc 0.9194, Test Acc: 0.7858\n",
      "Avg epoch time: 687.2379230499267\n",
      "Best val acc 0.9223, best test acc: 0.7885\n"
     ]
    }
   ],
   "source": [
    "g, num_labels = load_ogb('ogbn-products')\n",
    "g.ndata.pop('features')\n",
    "\n",
    "in_feats = 128\n",
    "eigen_vals, eigen_vecs = get_eigen(g, in_feats, 'ogbn-products')\n",
    "nfeat = th.tensor(eigen_vecs * np.sqrt(eigen_vals).reshape((1, len(eigen_vals))), dtype=th.float32)\n",
    "labels = g.ndata.pop('labels')\n",
    "\n",
    "hyperparams = {\n",
    "    'batch_size': 1024,\n",
    "    'num_workers': 0,\n",
    "    'num_hidden': 256,\n",
    "    'num_layers': 3,\n",
    "    'num_heads': 2,\n",
    "    'dropout': 0.5,\n",
    "    'lr': 0.003,\n",
    "    'sparse_lr': 0.0003,\n",
    "    'num_epochs': 20,\n",
    "    'fanouts': [15,10,5],\n",
    "    'eval_batch_size': 10000\n",
    "}\n",
    "device = th.device('cuda:6')\n",
    "\n",
    "# Pack data\n",
    "data = g, nfeat, labels\n",
    "# Define model\n",
    "model = GAT(in_feats, hyperparams['num_hidden'], num_labels,\n",
    "             hyperparams['num_layers'], hyperparams['num_heads'], F.relu, hyperparams['dropout'])\n",
    "train_ft_embed(model, data, hyperparams, device, 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}