resnext/resnest_baseline.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "from tqdm import trange, tqdm\n",
    "import torch\n",
    "from torch import nn\n",
    "from torchvision.datasets import CIFAR10 \n",
    "from torchvision import transforms\n",
    "from models.resnext import ResNeXt29_2x64d"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Files already downloaded and verified\n"
     ]
    }
   ],
   "source": [
    "transform_test = transforms.Compose([\n",
    "    transforms.ToTensor(),\n",
    "    transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)),\n",
    "])\n",
    "\n",
    "testset = CIFAR10(root='./data', train=False, download=True, transform=transform_test)\n",
    "testloader = torch.utils.data.DataLoader(testset, batch_size=128, shuffle=False, num_workers=2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[-0.7028, -1.1852,  1.0164]])"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "a = torch.randn(1, 3)\n",
    "a"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.return_types.max(\n",
       "values=tensor([1.0164]),\n",
       "indices=tensor([2]))"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "a.max(1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[False, False, False]])"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "b = torch.randn(1, 3)\n",
    "b.eq(a)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "NVIDIA A100-SXM4-40GB\n"
     ]
    }
   ],
   "source": [
    "if torch.cuda.is_available():\n",
    "    print(torch.cuda.get_device_name(0))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Downloading https://www.cs.toronto.edu/~kriz/cifar-10-python.tar.gz to ./data/cifar-10-python.tar.gz\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b6e86b8ce9a24872b4f4ce0b5e496159",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/170498071 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Extracting ./data/cifar-10-python.tar.gz to ./data/\n"
     ]
    }
   ],
   "source": [
    "num_epochs = 3\n",
    "batch_size = 128\n",
    "\n",
    "transform_train = transforms.Compose([\n",
    "    transforms.RandomCrop(32, padding=4),\n",
    "    transforms.RandomHorizontalFlip(),\n",
    "    transforms.ToTensor(),\n",
    "    transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)),\n",
    "])\n",
    "\n",
    "train_data = CIFAR10(root = \"./data/\", transform = transform_train, train = True, download = True)\n",
    "train_data_loader = torch.utils.data.DataLoader(dataset = train_data, batch_size = batch_size, shuffle = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "391"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(train_data_loader)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "391"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "for x, y in train_data_loader:\n",
    "    break\n",
    "len(train_data_loader)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAPsAAAD5CAYAAADhukOtAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAAsTAAALEwEAmpwYAAAUFklEQVR4nO3df5BfVXnH8fdTDGIgjUBAtpCIpmhkiCZxGxxJmQQbpDEOwSFosBo7wjoOKChaKRaCOp2iFTQ4Ai6SGisgofx0yIxhKJTGjtElQBJMUEIhxKwJMRICa2WBp398L3VJ73N29/vjfndzPq+ZzO6eZ8+9T272yd3vPd9zjrk7IrLv+5N2JyAi1VCxi2RCxS6SCRW7SCZU7CKZULGLZOI1jXQ2s1OApcB+wHfd/bJBvl/jfG3yzne+s90p7BNeSMT2T8T+kIj98qnny4835oCwz+v2e6m0fc9vt/L753ZZWczqHWc3s/2AXwJzga3Az4FF7v6LRB8Ve5vo/RTNsSURm5SIbU7E5p7309L2iR1Twj5Tx+8ubV9x2fvZ8eS60mJv5Nf4mcBj7v64u78A/BA4tYHjiUgLNVLsRwJPDfh6a9EmIiNQI6/Zy35V+H+/K5pZF9DVwHlEpAkaKfatwMQBXx8FbNv7m9y9G+gGvWYXaadGfo3/OXCMmb3JzPYHPgTc2Zy0RKTZ6r6zu/uLZnYu8GNqQ2/L3P2RpmUmdZrR7gRawj68NIzde/15YeyIoD1+zp2WeuKeMjkRe3zpu0rbL7057vP9y64obd+9c2fYp6FxdndfCaxs5BgiUg29g04kEyp2kUyo2EUyoWIXyYSKXSQTdU+EqetkelNN63UsLG1+y6R5YZdHf/qxFiUzPEfMvzaMbb8r8SbMBT+IY33BoNeqa+I+F5wehk5cMD+MzZoVH3JuHGJ2IjZcnZ2d9PT0NH0ijIiMIip2kUyo2EUyoWIXyYSKXSQTeho/Gk2Np3Hst7m3tP0/nnwm7HPChEYTag6zMxLRxKwQpsahs24tb//u3ySOl5rukoqNTcT649CZny1tPnHxYWGXecGT/2/N6mTrWj2NF8mail0kEyp2kUyo2EUyoWIXyYSKXSQTGnrbx4wL2p+teEeYvqD9E+etCPv84MryddVq1jSUz8jWMcx2gGitud/g/gcNvYnkTMUukgkVu0gmVOwimVCxi2RCxS6SiYZ2hDGzJ4A9wEvAi+7e2YykJK3r5Hgzoe/8+NuV5XHF7c+EsZW3l89Su2f5ksQRy2fsATAjmL0GsGl3HOv728T5Roro7524HnVoqNgLc9w93mBKREYE/RovkolGi92BVWb2gJkl1voVkXZr9Nf4E9x9m5kdDtxtZpvc/f6B31D8J6D/CETarKE7u7tvKz7uAG4DZpZ8T7e7d+rhnUh71V3sZnagmY175XPgZGBDsxITkeaqe9abmb2Z2t0cai8HbnD3fxykj2a9DdHGB34VxqbsvCOM/d17P1fa/rWKZ72Z/VkQqXM4acqP4thZ8ZZMJ5bvhsX9yxPnuuSSRPDyROz4MHLI1HhRzF3rr0wcc/jcvXTWW92v2d39ceAddWckIpXS0JtIJlTsIplQsYtkQsUukgkVu0gmmjERRuq0/PoFYWzKjHhPsc9Y+fAawDeD9q8NLaUmau6MLbYk5lrdG4d+M6e8fdz4uM8eFiUSmR5Gxh1/Whj7xsXxEcdPWVravuCk6+NOW1J71ZXTnV0kEyp2kUyo2EUyoWIXyYSKXSQTehrfYs88/y9hbPzYYJYGAKvDyFsTvd4/eEqjU1+0oRSwOQ71Bg/x9/QnT5aIxevd7VnzQBhbPD9O8oNfP6O0/fylHw77fPOSM8sDj/1F2Ed3dpFMqNhFMqFiF8mEil0kEyp2kUyo2EUyoaG3YRlT2vr8tnh4bezYxIyL/g8mTjUjDE2dEHd7cMTszRNN5NlS3+HGJP7SY54OQ3s4rDyQmqeTOBU74wlKTI635WJs3O++TeXtb0386NAfXEd/IeyiO7tIJlTsIplQsYtkQsUukgkVu0gmVOwimRh06M3MlgHzgR3uflzRdghwE3A08ARwhrv/rnVpVqfrkwvC2Heu6i4P9N0eH3D1dXFsVmqMJ555tToxvJaczFWpaLujOofeUuONZwbDa8A7Ti5vf3hN4lwTEkNoO4NxMoDeeEYcs94YhrYH/57bE7P52BTNinwu7DKUO/v3gFP2arsQuMfdjwHuKb4WkRFs0GIv9lvftVfzqcArW+MtBxY0Ny0RabZ6X7O/wd17AYqPhzcvJRFphZa/XdbMuoCuVp9HRNLqvbNvN7MOgOLjjugb3b3b3TvdvbPOc4lIE9Rb7HcCi4vPFwN3NCcdEWmVoQy93QjMBiaY2VZgCXAZsMLMPk5tLCW1cuKIc9eKC8LYvIWnxx03f768fWdiLGzWrEQm5bPoAOiNx10S866YkjhktdbW0ed9YWTJxcE+TsBxC+IjRhPHrgnWawS4u//1YWzPVxJDh333xbFVcYixs8vbtySGAHkqaI9nvQ1a7O4ebXz1nsH6isjIoXfQiWRCxS6SCRW7SCZU7CKZULGLZGKfXXByzW3x8NrMBXPjjr1fjWOTgnGtyanhtdQsr8TQSmL62qTEBLBDpyROV6EbntxW2n73DXGfZRVOp5qbuPQr4xFA1k+IZvPBlBnle7YBzEz8iERz5X6SmPV21knBz+JvLOyjO7tIJlTsIplQsYtkQsUukgkVu0gmVOwimTB3r+5kZk09Wdcn43Gm71z1D3HHvp/Esf7EDLbdwQZhkxIz5Xg8EXtrHFqzMgzdN/+u+IjB5LCOFdX9O9drS7zGJpPGVpfHaNbZ2UlPT0/p+Jvu7CKZULGLZELFLpIJFbtIJlTsIpkY1RNhpqcWXOu/L46NjR/tvttuDmPdXy7vd9zFH43PlVpnjt/GofXrw1Bf4qn17jHl5+tIZDFS6Il7a+nOLpIJFbtIJlTsIplQsYtkQsUukgkVu0gmhrL90zJgPrDD3Y8r2i4FzgaeLr7tInePZ260yIyxiYXaxsSLjPV97O/DWOKIHLcp2kwoNWYU9YF49TFgZ7x2XSrHZyeMhkE2aYeh3Nm/B5xS0v4Nd59W/Km80EVkeAYtdne/H9hVQS4i0kKNvGY/18zWmdkyMzu4aRmJSEvUW+xXU1v0fBrQC1wefaOZdZlZj5n11HkuEWmCuord3be7+0vu/jJwLTAz8b3d7t7p7p31Jikijaur2M1s4CPf04ANzUlHRFplKENvNwKzgQlmthVYAsw2s2mAA08An2hdirGZc45PROO15DasjXt1p044JxrOS60zlxheS2wNtTue9EZ/YtbbzDkjZP+nJkutTzchMfKpiXR/NGixu/uikubrWpCLiLSQ3kEnkgkVu0gmVOwimVCxi2RCxS6SiVG94CST5yaCD4aRmbctiLuNScxSmzQ7CDyayCOxnVQiNj6RxunvSxxySnNnva3cFMc+f95VYewXq85pah7Nd3Iilvo3S4zbjnC6s4tkQsUukgkVu0gmVOwimVCxi2RCxS6SidE99FavybMTwd8nYtEMttTMtmQicejTkxJpJM7XP/x5Xt+667E4jfnHDPt4afHf65wL4jmHfb3xcNj0yYmZfn3le999+vJ3xH3YN2cO6s4ukgkVu0gmVOwimVCxi2RCxS6SCXP36k5m1tSTuadWjPtZIpZ62npoIhbNTkmtQZc43qolYeg1743Xp/th4mzRUm0fTfw7p6Z9PJiYCDN3FD+0nnjaJWFs+oUfDWM/+sq34oPedWUjKTWNu1tZu+7sIplQsYtkQsUukgkVu0gmVOwimVCxi2RiKNs/TQS+DxwBvAx0u/tSMzsEuAk4mtoWUGe4++9al2qZxEQSfhKH+laHod1resPYhrXl41Arb44npty1Jk7j4TiUtLCOPp+x0tEYAH6bGpbb8nwYe/clF4Wxf/76l0rb16zaHPZZefONYey+VavC2LxZ8aJ8p8+ZXdq+9favhH16xycGI9fGPzsj3VDu7C8CF7j724B3AeeY2bHAhcA97n4McE/xtYiMUIMWu7v3uvva4vM9wEbgSOBUYHnxbcuBBS3KUUSaYFiv2c3saGA6sAZ4g7v3Qu0/BODwpmcnIk0z5MUrzOwg4BbgfHd/1hKvAffq1wV01ZeeiDTLkO7sZjaGWqFf7+63Fs3bzayjiHcAO8r6unu3u3e6e2czEhaR+gxa7Fa7hV8HbHT3KwaE7gQWF58vBu5ofnoi0iyDznozs1nAfwLrqQ29AVxE7XX7CmqLim0BFrr7rkGO1dRZby88/eswtv9hRzbzVPu0xzbG/yx//rahvVxrpwM6/imMffHLny1tv/js17YqnbaLZr0N+prd3VcD0b/4expJSkSqo3fQiWRCxS6SCRW7SCZU7CKZULGLZGJUb/90ze3lW/vI8Ew/6ex2p9CQ/1oRz8GaPqu8fcFZ8XDjVDspcbZ7h5jVyKM7u0gmVOwimVCxi2RCxS6SCRW7SCZU7CKZGNV7vY1+qQUz5yRi8aKN8c5tqR3d4kU2R4M38fUwtujMRaXt/VM6wj6HToln+k2dFOfRkYjFS5LC2GAEuS/auA/YGRzwCx/qZPMjPdrrTSRnKnaRTKjYRTKhYhfJhIpdJBOjeiLM6Jd6qp6KyUD/Tbw11BH3ll/HT1387bDPNTf/exj7yBlLwtgu4if8jJ8Shj6w8NTS9o6O+HjfvvK68sBz8ciK7uwimVCxi2RCxS6SCRW7SCZU7CKZULGLZGIo2z9NBL4PHEFt+6dud19qZpcCZwNPF996kbuvHORYTZ4IszgRW56ITU3E1teZS4Zm3RTHVn+wujwS1px8QWn7zB/Hk2d2JiagHHbgyN8Oq+7tn4AXgQvcfa2ZjQMeMLO7i9g33D2+aiIyYgxlr7deijmQ7r7HzDYC2jVRZJQZ1mt2MzsamE5tB1eAc81snZktM7ODm52ciDTPkIvdzA4CbgHOd/dngauprb4wjdqd//KgX5eZ9ZhZT+Ppiki9hlTsZjaGWqFf7+63Arj7dnd/yd1fBq4FZpb1dfdud+90985mJS0iwzdosZuZAdcBG939igHtA9+lfxqwofnpiUizDOVp/AnAR4D1ZvZQ0XYRsMjMpgEOPAF8ogX5DSI1vJZS5fDaJxOxqyvLoiXWP5oInhm039CCRMaGka+uL197b9EN68I+P2N84lxnJWJrE7EtiVi0Ql3894IJQfvWsMdQnsavBsrG7ZJj6iIysugddCKZULGLZELFLpIJFbtIJlTsIpnQgpMtd2+7E2id3amhw+q2lHrL+34Uxj518Uml7f398fH+dO22MLZ63bVhbGy8piSbE6NyY4KR4C398STRRymfmrfisr8M++jOLpIJFbtIJlTsIplQsYtkQsUukgkVu0gmBl1wsqkna/KCk0exMIxt/b/FdMqkZiDJqDMpnlnoT1417MNdc/lPw1hfX7wa5aXfjeeG7dlyc+KMw/95PGrynNL27Vt7eOF/ni1dcFJ3dpFMqNhFMqFiF8mEil0kEyp2kUyo2EUyMapnvZ0wY3YcHHN8HOuPFviD3bvj2JYg9oudqWG+TYmYNMWWePadWXMX9Tw2MbVtT191/9ZzO8qHAO/c/nLYR3d2kUyo2EUyoWIXyYSKXSQTKnaRTAw6EcbMDgDuB15L7en9v7n7EjM7BLgJOJra9k9nuPvvBjlWdbNuRDLl7qUTYYZS7AYc6O7PFbu5rgbOAz4A7HL3y8zsQuBgd//CIMdSsYu0WFTsg/4a7zXPFV+OKf44cCp/3FlxObCg8TRFpFWGuj/7fsUOrjuAu919DfAGd+8FKD4e3rIsRaRhQyp2d3/J3acBRwEzzey4oZ7AzLrMrMfMeurMUUSaYFhP4939GeA+4BRgu5l1ABQfdwR9ut290907G0tVRBoxaLGb2WFm9vri89cBf0XtDd93AouLb1sM3NGiHEWkCYbyNP7t1B7A7UftP4cV7v5lMzsUWAFMoraI1kJ33zXIsfQ0XqTF6h56ayYVu0jr1T30JiL7BhW7SCZU7CKZULGLZELFLpKJqteg2wk8WXw+ofi63ZTHqymPVxttebwxClQ69PaqE5v1jIR31SkP5ZFLHvo1XiQTKnaRTLSz2LvbeO6BlMerKY9X22fyaNtrdhGpln6NF8lEW4rdzE4xs0fN7LFi/bq2MLMnzGy9mT1U5eIaZrbMzHaY2YYBbYeY2d1m9qvi48FtyuNSM/t1cU0eMrN5FeQx0czuNbONZvaImZ1XtFd6TRJ5VHpNzOwAM/uZmT1c5PGlor2x6+Hulf6hNlV2M/BmYH/gYeDYqvMocnkCmNCG854IzAA2DGj7GnBh8fmFwFfblMelwOcqvh4dwIzi83HAL4Fjq74miTwqvSaAAQcVn48B1gDvavR6tOPOPhN4zN0fd/cXgB9SW7wyG+5+P7D33P/KF/AM8qicu/e6+9ri8z3ARuBIKr4miTwq5TVNX+S1HcV+JPDUgK+30oYLWnBglZk9YGZdbcrhFSNpAc9zzWxd8Wt+y19ODGRmRwPTqd3N2nZN9soDKr4mrVjktR3FXjaxvl1DAie4+wzgr4FzzOzENuUxklwNTAamAb3A5VWd2MwOAm4Bznf3Z6s67xDyqPyaeAOLvEbaUexbgYkDvj4K2NaGPHD3bcXHHcBt1F5itMuQFvBsNXffXvygvQxcS0XXpNiA5Bbgene/tWiu/JqU5dGua1Kc+xmGuchrpB3F/nPgGDN7k5ntD3yI2uKVlTKzA81s3CufAycDG9K9WmpELOD5yg9T4TQquCbFrkPXARvd/YoBoUqvSZRH1dekZYu8VvWEca+njfOoPencDHyxTTm8mdpIwMPAI1XmAdxI7dfBfmq/6XwcOBS4B/hV8fGQNuXxr8B6YF3xw9VRQR6zqL2UWwc8VPyZV/U1SeRR6TUB3g48WJxvA3BJ0d7Q9dA76EQyoXfQiWRCxS6SCRW7SCZU7CKZULGLZELFLpIJFbtIJlTsIpn4X6r4PasNQ3QjAAAAAElFTkSuQmCC",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.imshow(np.clip(train_data[1][0].numpy().transpose((1, 2, 0)), 0, 1))\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def init_weights(m):\n",
    "    if type(m) == nn.Linear or type(m) == nn.Conv2d:\n",
    "        nn.init.kaiming_uniform_(m.weight)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.float32\n",
      "torch.float32\n",
      "torch.float32\n",
      "torch.float32\n",
      "torch.float32\n",
      "torch.float32\n",
      "torch.float32\n",
      "torch.float32\n",
      "torch.float32\n",
      "torch.float32\n",
      "torch.float32\n",
      "torch.float32\n",
      "torch.float32\n",
      "torch.float32\n",
      "torch.float32\n",
      "torch.float32\n",
      "torch.float32\n",
      "torch.float32\n",
      "torch.float32\n",
      "torch.float32\n",
      "torch.float32\n",
      "torch.float32\n",
      "torch.float32\n",
      "torch.float32\n",
      "torch.float32\n",
      "torch.float32\n",
      "torch.float32\n",
      "torch.float32\n",
      "torch.float32\n",
      "torch.float32\n",
      "torch.float32\n",
      "torch.float32\n",
      "torch.float32\n",
      "torch.float32\n",
      "torch.float32\n",
      "torch.float32\n",
      "torch.float32\n",
      "torch.float32\n",
      "torch.float32\n",
      "torch.float32\n",
      "torch.float32\n",
      "torch.float32\n",
      "torch.float32\n",
      "torch.float32\n",
      "torch.float32\n",
      "torch.float32\n",
      "torch.float32\n",
      "torch.float32\n",
      "torch.float32\n",
      "torch.float32\n",
      "torch.float32\n",
      "torch.float32\n",
      "torch.float32\n",
      "torch.float32\n",
      "torch.float32\n",
      "torch.float32\n",
      "torch.float32\n",
      "torch.float32\n",
      "torch.float32\n",
      "torch.float32\n",
      "torch.float32\n",
      "torch.float32\n",
      "torch.float32\n",
      "torch.float32\n",
      "torch.float32\n",
      "torch.float32\n",
      "torch.float32\n",
      "torch.float32\n",
      "torch.float32\n",
      "torch.float32\n",
      "torch.float32\n",
      "torch.float32\n",
      "torch.float32\n",
      "torch.float32\n",
      "torch.float32\n",
      "torch.float32\n",
      "torch.float32\n",
      "torch.float32\n",
      "torch.float32\n",
      "torch.float32\n",
      "torch.float32\n",
      "torch.float32\n",
      "torch.float32\n",
      "torch.float32\n",
      "torch.float32\n",
      "torch.float32\n",
      "torch.float32\n",
      "torch.float32\n",
      "torch.float32\n",
      "torch.float32\n",
      "torch.float32\n",
      "torch.float32\n",
      "torch.float32\n",
      "torch.float32\n",
      "torch.float32\n",
      "torch.float32\n",
      "torch.float32\n",
      "torch.float32\n",
      "torch.float32\n",
      "torch.float32\n",
      "torch.float32\n",
      "torch.float32\n",
      "torch.float32\n",
      "torch.float32\n",
      "torch.float32\n",
      "torch.float32\n",
      "torch.float32\n",
      "torch.float32\n",
      "torch.float32\n",
      "torch.float32\n",
      "torch.float32\n",
      "torch.float32\n",
      "torch.float32\n",
      "torch.float32\n",
      "torch.float32\n",
      "torch.float32\n",
      "torch.float32\n",
      "torch.float32\n",
      "torch.float32\n",
      "torch.float32\n",
      "torch.float32\n",
      "torch.float32\n",
      "torch.float32\n",
      "torch.float32\n",
      "torch.float32\n",
      "torch.float32\n",
      "torch.float32\n",
      "torch.float32\n",
      "torch.float32\n",
      "torch.float32\n",
      "torch.float32\n",
      "torch.float32\n",
      "torch.float32\n",
      "torch.float32\n",
      "torch.float32\n",
      "torch.float32\n",
      "torch.float32\n",
      "torch.float32\n",
      "torch.float32\n",
      "torch.float32\n",
      "torch.float32\n",
      "torch.float32\n",
      "torch.float32\n",
      "torch.float32\n",
      "torch.float32\n",
      "torch.float32\n",
      "torch.float32\n",
      "torch.float32\n",
      "torch.float32\n",
      "torch.float32\n",
      "torch.float32\n",
      "torch.float32\n",
      "torch.float32\n",
      "torch.float32\n",
      "torch.float32\n",
      "torch.float32\n",
      "torch.float32\n",
      "torch.float32\n",
      "torch.float32\n",
      "torch.float32\n",
      "torch.float32\n",
      "torch.float32\n",
      "torch.float32\n",
      "torch.float32\n",
      "torch.float32\n",
      "torch.float32\n",
      "torch.float32\n",
      "torch.float32\n",
      "torch.float32\n",
      "torch.float32\n",
      "torch.float32\n",
      "torch.float32\n",
      "torch.float32\n",
      "torch.float32\n",
      "torch.float32\n",
      "torch.float32\n",
      "torch.float32\n",
      "torch.float32\n",
      "torch.float32\n",
      "torch.float32\n",
      "torch.float32\n",
      "torch.float32\n",
      "torch.float32\n",
      "torch.float32\n",
      "torch.float32\n",
      "torch.float32\n",
      "torch.float32\n",
      "torch.float32\n",
      "torch.float32\n",
      "torch.float32\n"
     ]
    }
   ],
   "source": [
    "model = ResNeXt29_2x64d()\n",
    "for param in model.parameters():\n",
    "    print(param.dtype)\n",
    "\n",
    "for param in model.parameters():\n",
    "    param = param.to(torch.float16)\n",
    "\n",
    "for param in model.parameters():\n",
    "    print(param.dtype)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 33%|███▎      | 1/3 [01:33<03:06, 93.42s/it]\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m/tmp/ipykernel_26015/2328044188.py\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m     21\u001b[0m         \u001b[0ml\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mloss\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0my_hat\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     22\u001b[0m         \u001b[0ml\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbackward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 23\u001b[0;31m         \u001b[0moptimizer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstep\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     24\u001b[0m         \u001b[0mrunning_loss\u001b[0m \u001b[0;34m+=\u001b[0m \u001b[0ml\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mitem\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m*\u001b[0m \u001b[0mx\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msize\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     25\u001b[0m         \u001b[0mcorrect\u001b[0m \u001b[0;34m+=\u001b[0m \u001b[0msum\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0margmax\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0my_hat\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0my\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mitem\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/opt/conda/lib/python3.7/site-packages/torch/optim/optimizer.py\u001b[0m in \u001b[0;36mwrapper\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m     86\u001b[0m                 \u001b[0mprofile_name\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m\"Optimizer.step#{}.step\"\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mformat\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mobj\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__class__\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__name__\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     87\u001b[0m                 \u001b[0;32mwith\u001b[0m \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mautograd\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mprofiler\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrecord_function\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mprofile_name\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 88\u001b[0;31m                     \u001b[0;32mreturn\u001b[0m \u001b[0mfunc\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     89\u001b[0m             \u001b[0;32mreturn\u001b[0m \u001b[0mwrapper\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     90\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/opt/conda/lib/python3.7/site-packages/torch/autograd/grad_mode.py\u001b[0m in \u001b[0;36mdecorate_context\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m     26\u001b[0m         \u001b[0;32mdef\u001b[0m \u001b[0mdecorate_context\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     27\u001b[0m             \u001b[0;32mwith\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__class__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 28\u001b[0;31m                 \u001b[0;32mreturn\u001b[0m \u001b[0mfunc\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     29\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mcast\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mF\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdecorate_context\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     30\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/opt/conda/lib/python3.7/site-packages/torch/optim/adam.py\u001b[0m in \u001b[0;36mstep\u001b[0;34m(self, closure)\u001b[0m\n\u001b[1;32m    142\u001b[0m                    \u001b[0mlr\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mgroup\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'lr'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    143\u001b[0m                    \u001b[0mweight_decay\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mgroup\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'weight_decay'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 144\u001b[0;31m                    eps=group['eps'])\n\u001b[0m\u001b[1;32m    145\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mloss\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/opt/conda/lib/python3.7/site-packages/torch/optim/_functional.py\u001b[0m in \u001b[0;36madam\u001b[0;34m(params, grads, exp_avgs, exp_avg_sqs, max_exp_avg_sqs, state_steps, amsgrad, beta1, beta2, lr, weight_decay, eps)\u001b[0m\n\u001b[1;32m     92\u001b[0m             \u001b[0mdenom\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mmax_exp_avg_sqs\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msqrt\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m/\u001b[0m \u001b[0mmath\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msqrt\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbias_correction2\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0madd_\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0meps\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     93\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 94\u001b[0;31m             \u001b[0mdenom\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mexp_avg_sq\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msqrt\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m/\u001b[0m \u001b[0mmath\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msqrt\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbias_correction2\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0madd_\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0meps\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     95\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     96\u001b[0m         \u001b[0mstep_size\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mlr\u001b[0m \u001b[0;34m/\u001b[0m \u001b[0mbias_correction1\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "model = ResNeXt29_2x64d()\n",
    "# model.fc = nn.Linear(in_features=model.fc.in_features, out_features=10)\n",
    "model.apply(init_weights)\n",
    "model = model.cuda()\n",
    "\n",
    "optimizer = torch.optim.Adam(model.parameters(), lr=0.01, weight_decay=5e-4)\n",
    "loss = nn.CrossEntropyLoss()\n",
    "training_loss = []\n",
    "training_acc = []\n",
    "\n",
    "for i in trange(num_epochs): \n",
    "    model.train()\n",
    "    running_loss = 0\n",
    "    correct = 0\n",
    "\n",
    "    for x, y in train_data_loader:\n",
    "        x = x.cuda()\n",
    "        y = y.cuda()\n",
    "        optimizer.zero_grad()\n",
    "        y_hat = model(x)\n",
    "        l = loss(y_hat, y)\n",
    "        l.backward()\n",
    "        optimizer.step()\n",
    "        running_loss += l.item() * x.size(0)\n",
    "        correct += sum(torch.argmax(y_hat) == y).item()\n",
    "    \n",
    "    training_loss.append(running_loss / 50000)\n",
    "    training_acc.append(correct / 50000)\n",
    "\n",
    "    if i % 10 == 9:\n",
    "        torch.save(model.state_dict(), f\"model_history/resnest_baseline_epoch{i+1}.pt\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "torch.save(model.state_dict(), \"model_history/resnest_baseline.pt\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "np.savez(\"model_history/resnest_baseline.npz\", loss=training_loss, beta=training_acc)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([0, 8, 1, 9, 1, 6, 9, 1, 6, 1, 1, 6, 1, 3, 5, 8, 9, 6, 7, 0, 8, 2, 6, 6,\n",
       "        0, 4, 5, 6, 9, 1, 0, 4, 1, 7, 2, 8, 8, 4, 7, 4, 6, 7, 0, 0, 1, 7, 0, 5,\n",
       "        2, 9, 8, 3, 3, 0, 9, 0, 5, 5, 2, 8, 9, 3, 8, 7, 5, 7, 6, 4, 2, 5, 5, 6,\n",
       "        4, 5, 0, 9, 8, 7, 9, 1, 5, 9, 1, 2, 5, 8, 8, 9, 5, 9, 1, 5, 2, 4, 1, 3,\n",
       "        1, 0, 4, 5, 2, 2, 8, 7, 2, 8, 4, 8, 1, 7, 1, 1, 0, 7, 8, 6, 4, 0, 0, 2,\n",
       "        2, 1, 8, 6, 1, 4, 0, 0])"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "for x, y in train_data_loader:\n",
    "    break\n",
    "\n",
    "y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.7.12 ('base')",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.12"
  },
  "orig_nbformat": 4,
  "vscode": {
   "interpreter": {
    "hash": "d4d1e4263499bec80672ea0156c357c1ee493ec2b1c70f0acce89fc37c4a6abe"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}