examples.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "PyTorchViz examples\n",
    "=========="
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "from torch import nn\n",
    "from torchviz import make_dot, make_dot_from_trace"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Visualize gradients of simple MLP\n",
    "\n",
    "The method below is for building directed graphs of PyTorch operations, built during forward propagation and showing which operations will be called on backward. It omits subgraphs which do not require gradients."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/svg+xml": [
       "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>\n",
       "<!DOCTYPE svg PUBLIC \"-//W3C//DTD SVG 1.1//EN\"\n",
       " \"http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd\">\n",
       "<!-- Generated by graphviz version 2.40.1 (20161225.0304)\n",
       " -->\n",
       "<!-- Title: %3 Pages: 1 -->\n",
       "<svg width=\"312pt\" height=\"276pt\"\n",
       " viewBox=\"0.00 0.00 312.14 276.00\" xmlns=\"http://www.w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\">\n",
       "<g id=\"graph0\" class=\"graph\" transform=\"scale(1 1) rotate(0) translate(4 272)\">\n",
       "<title>%3</title>\n",
       "<polygon fill=\"#ffffff\" stroke=\"transparent\" points=\"-4,4 -4,-272 308.1436,-272 308.1436,4 -4,4\"/>\n",
       "<!-- 4600717832 -->\n",
       "<g id=\"node1\" class=\"node\">\n",
       "<title>4600717832</title>\n",
       "<polygon fill=\"#caff70\" stroke=\"#000000\" points=\"222.4594,-20 104.8414,-20 104.8414,0 222.4594,0 222.4594,-20\"/>\n",
       "<text text-anchor=\"middle\" x=\"163.6504\" y=\"-6.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">ThAddmmBackward</text>\n",
       "</g>\n",
       "<!-- 4600717888 -->\n",
       "<g id=\"node2\" class=\"node\">\n",
       "<title>4600717888</title>\n",
       "<polygon fill=\"#d3d3d3\" stroke=\"#000000\" points=\"101.4516,-76 -.1508,-76 -.1508,-56 101.4516,-56 101.4516,-76\"/>\n",
       "<text text-anchor=\"middle\" x=\"50.6504\" y=\"-62.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">ExpandBackward</text>\n",
       "</g>\n",
       "<!-- 4600717888&#45;&gt;4600717832 -->\n",
       "<g id=\"edge1\" class=\"edge\">\n",
       "<title>4600717888&#45;&gt;4600717832</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M70.8525,-55.9883C88.499,-47.2431 114.3378,-34.4381 134.2716,-24.5594\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"135.8362,-27.6903 143.2422,-20.1138 132.7279,-21.4182 135.8362,-27.6903\"/>\n",
       "</g>\n",
       "<!-- 4600718112 -->\n",
       "<g id=\"node3\" class=\"node\">\n",
       "<title>4600718112</title>\n",
       "<polygon fill=\"#add8e6\" stroke=\"#000000\" points=\"78.3077,-144 22.9931,-144 22.9931,-112 78.3077,-112 78.3077,-144\"/>\n",
       "<text text-anchor=\"middle\" x=\"50.6504\" y=\"-130.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">W1.bias</text>\n",
       "<text text-anchor=\"middle\" x=\"50.6504\" y=\"-118.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\"> (1)</text>\n",
       "</g>\n",
       "<!-- 4600718112&#45;&gt;4600717888 -->\n",
       "<g id=\"edge2\" class=\"edge\">\n",
       "<title>4600718112&#45;&gt;4600717888</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M50.6504,-111.7102C50.6504,-104.0144 50.6504,-94.744 50.6504,-86.5691\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"54.1505,-86.3512 50.6504,-76.3512 47.1505,-86.3513 54.1505,-86.3512\"/>\n",
       "</g>\n",
       "<!-- 4600717944 -->\n",
       "<g id=\"node4\" class=\"node\">\n",
       "<title>4600717944</title>\n",
       "<polygon fill=\"#d3d3d3\" stroke=\"#000000\" points=\"208.1145,-76 119.1863,-76 119.1863,-56 208.1145,-56 208.1145,-76\"/>\n",
       "<text text-anchor=\"middle\" x=\"163.6504\" y=\"-62.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">TanhBackward</text>\n",
       "</g>\n",
       "<!-- 4600717944&#45;&gt;4600717832 -->\n",
       "<g id=\"edge3\" class=\"edge\">\n",
       "<title>4600717944&#45;&gt;4600717832</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M163.6504,-55.9883C163.6504,-48.9098 163.6504,-39.1714 163.6504,-30.4779\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"167.1505,-30.3038 163.6504,-20.3039 160.1505,-30.3039 167.1505,-30.3038\"/>\n",
       "</g>\n",
       "<!-- 4600718168 -->\n",
       "<g id=\"node5\" class=\"node\">\n",
       "<title>4600718168</title>\n",
       "<polygon fill=\"#d3d3d3\" stroke=\"#000000\" points=\"217.4594,-138 99.8414,-138 99.8414,-118 217.4594,-118 217.4594,-138\"/>\n",
       "<text text-anchor=\"middle\" x=\"158.6504\" y=\"-124.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">ThAddmmBackward</text>\n",
       "</g>\n",
       "<!-- 4600718168&#45;&gt;4600717944 -->\n",
       "<g id=\"edge4\" class=\"edge\">\n",
       "<title>4600718168&#45;&gt;4600717944</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M159.476,-117.762C160.1677,-109.185 161.1759,-96.6836 162.0282,-86.1154\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"165.5248,-86.2965 162.8401,-76.0475 158.5475,-85.7338 165.5248,-86.2965\"/>\n",
       "</g>\n",
       "<!-- 4600718280 -->\n",
       "<g id=\"node6\" class=\"node\">\n",
       "<title>4600718280</title>\n",
       "<polygon fill=\"#d3d3d3\" stroke=\"#000000\" points=\"156.4516,-200 54.8492,-200 54.8492,-180 156.4516,-180 156.4516,-200\"/>\n",
       "<text text-anchor=\"middle\" x=\"105.6504\" y=\"-186.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">ExpandBackward</text>\n",
       "</g>\n",
       "<!-- 4600718280&#45;&gt;4600718168 -->\n",
       "<g id=\"edge5\" class=\"edge\">\n",
       "<title>4600718280&#45;&gt;4600718168</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M114.4023,-179.762C122.2745,-170.553 134.0139,-156.82 143.4231,-145.8131\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"146.224,-147.923 150.0614,-138.0475 140.9031,-143.3745 146.224,-147.923\"/>\n",
       "</g>\n",
       "<!-- 4600718448 -->\n",
       "<g id=\"node7\" class=\"node\">\n",
       "<title>4600718448</title>\n",
       "<polygon fill=\"#add8e6\" stroke=\"#000000\" points=\"133.3077,-268 77.9931,-268 77.9931,-236 133.3077,-236 133.3077,-268\"/>\n",
       "<text text-anchor=\"middle\" x=\"105.6504\" y=\"-254.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">W0.bias</text>\n",
       "<text text-anchor=\"middle\" x=\"105.6504\" y=\"-242.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\"> (16)</text>\n",
       "</g>\n",
       "<!-- 4600718448&#45;&gt;4600718280 -->\n",
       "<g id=\"edge6\" class=\"edge\">\n",
       "<title>4600718448&#45;&gt;4600718280</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M105.6504,-235.7102C105.6504,-228.0144 105.6504,-218.744 105.6504,-210.5691\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"109.1505,-210.3512 105.6504,-200.3512 102.1505,-210.3513 109.1505,-210.3512\"/>\n",
       "</g>\n",
       "<!-- 4600718336 -->\n",
       "<g id=\"node8\" class=\"node\">\n",
       "<title>4600718336</title>\n",
       "<polygon fill=\"#d3d3d3\" stroke=\"#000000\" points=\"246.625,-200 174.6758,-200 174.6758,-180 246.625,-180 246.625,-200\"/>\n",
       "<text text-anchor=\"middle\" x=\"210.6504\" y=\"-186.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">TBackward</text>\n",
       "</g>\n",
       "<!-- 4600718336&#45;&gt;4600718168 -->\n",
       "<g id=\"edge7\" class=\"edge\">\n",
       "<title>4600718336&#45;&gt;4600718168</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M202.0636,-179.762C194.34,-170.553 182.822,-156.82 173.5904,-145.8131\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"176.1852,-143.4603 167.0774,-138.0475 170.8218,-147.9586 176.1852,-143.4603\"/>\n",
       "</g>\n",
       "<!-- 4600718504 -->\n",
       "<g id=\"node9\" class=\"node\">\n",
       "<title>4600718504</title>\n",
       "<polygon fill=\"#add8e6\" stroke=\"#000000\" points=\"245.1367,-268 176.1641,-268 176.1641,-236 245.1367,-236 245.1367,-268\"/>\n",
       "<text text-anchor=\"middle\" x=\"210.6504\" y=\"-254.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">W0.weight</text>\n",
       "<text text-anchor=\"middle\" x=\"210.6504\" y=\"-242.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\"> (16, 8)</text>\n",
       "</g>\n",
       "<!-- 4600718504&#45;&gt;4600718336 -->\n",
       "<g id=\"edge8\" class=\"edge\">\n",
       "<title>4600718504&#45;&gt;4600718336</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M210.6504,-235.7102C210.6504,-228.0144 210.6504,-218.744 210.6504,-210.5691\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"214.1505,-210.3512 210.6504,-200.3512 207.1505,-210.3513 214.1505,-210.3512\"/>\n",
       "</g>\n",
       "<!-- 4600718000 -->\n",
       "<g id=\"node10\" class=\"node\">\n",
       "<title>4600718000</title>\n",
       "<polygon fill=\"#d3d3d3\" stroke=\"#000000\" points=\"301.625,-76 229.6758,-76 229.6758,-56 301.625,-56 301.625,-76\"/>\n",
       "<text text-anchor=\"middle\" x=\"265.6504\" y=\"-62.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">TBackward</text>\n",
       "</g>\n",
       "<!-- 4600718000&#45;&gt;4600717832 -->\n",
       "<g id=\"edge9\" class=\"edge\">\n",
       "<title>4600718000&#45;&gt;4600717832</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M247.4148,-55.9883C231.7023,-47.3619 208.7945,-34.785 190.905,-24.9633\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"192.5222,-21.8584 182.072,-20.1138 189.1534,-27.9945 192.5222,-21.8584\"/>\n",
       "</g>\n",
       "<!-- 4600718224 -->\n",
       "<g id=\"node11\" class=\"node\">\n",
       "<title>4600718224</title>\n",
       "<polygon fill=\"#add8e6\" stroke=\"#000000\" points=\"304.1367,-144 235.1641,-144 235.1641,-112 304.1367,-112 304.1367,-144\"/>\n",
       "<text text-anchor=\"middle\" x=\"269.6504\" y=\"-130.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">W1.weight</text>\n",
       "<text text-anchor=\"middle\" x=\"269.6504\" y=\"-118.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\"> (1, 16)</text>\n",
       "</g>\n",
       "<!-- 4600718224&#45;&gt;4600718000 -->\n",
       "<g id=\"edge10\" class=\"edge\">\n",
       "<title>4600718224&#45;&gt;4600718000</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M268.5994,-111.7102C268.1029,-104.0144 267.5048,-94.744 266.9774,-86.5691\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"270.4549,-86.1051 266.3182,-76.3512 263.4694,-86.5558 270.4549,-86.1051\"/>\n",
       "</g>\n",
       "</g>\n",
       "</svg>\n"
      ],
      "text/plain": [
       "<graphviz.dot.Digraph at 0x1123961d0>"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model = nn.Sequential()\n",
    "model.add_module('W0', nn.Linear(8, 16))\n",
    "model.add_module('tanh', nn.Tanh())\n",
    "model.add_module('W1', nn.Linear(16, 1))\n",
    "\n",
    "x = torch.randn(1,8)\n",
    "\n",
    "make_dot(model(x), params=dict(model.named_parameters()))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### torch.jit.trace\n",
    "\n",
    "An alternative to the above is `torch.jit.trace`, which gives more information about operations, and shows all operations performed during forward"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/svg+xml": [
       "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>\n",
       "<!DOCTYPE svg PUBLIC \"-//W3C//DTD SVG 1.1//EN\"\n",
       " \"http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd\">\n",
       "<!-- Generated by graphviz version 2.40.1 (20161225.0304)\n",
       " -->\n",
       "<!-- Title: %3 Pages: 1 -->\n",
       "<svg width=\"352pt\" height=\"292pt\"\n",
       " viewBox=\"0.00 0.00 351.63 292.00\" xmlns=\"http://www.w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\">\n",
       "<g id=\"graph0\" class=\"graph\" transform=\"scale(1 1) rotate(0) translate(4 288)\">\n",
       "<title>%3</title>\n",
       "<polygon fill=\"#ffffff\" stroke=\"transparent\" points=\"-4,4 -4,-288 347.6309,-288 347.6309,4 -4,4\"/>\n",
       "<!-- Sequential/Linear[W0]/5 -->\n",
       "<g id=\"node1\" class=\"node\">\n",
       "<title>Sequential/Linear[W0]/5</title>\n",
       "<polygon fill=\"#d3d3d3\" stroke=\"#000000\" points=\"252.4472,-204 180.1836,-204 180.1836,-160 252.4472,-160 252.4472,-204\"/>\n",
       "<text text-anchor=\"middle\" x=\"216.3154\" y=\"-190.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">Sequential</text>\n",
       "<text text-anchor=\"middle\" x=\"216.3154\" y=\"-178.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">Linear[W0]</text>\n",
       "<text text-anchor=\"middle\" x=\"216.3154\" y=\"-166.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">5</text>\n",
       "</g>\n",
       "<!-- Sequential/Tanh[tanh]/6 -->\n",
       "<g id=\"node5\" class=\"node\">\n",
       "<title>Sequential/Tanh[tanh]/6</title>\n",
       "<polygon fill=\"#d3d3d3\" stroke=\"#000000\" points=\"250.7878,-124 181.8431,-124 181.8431,-80 250.7878,-80 250.7878,-124\"/>\n",
       "<text text-anchor=\"middle\" x=\"216.3154\" y=\"-110.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">Sequential</text>\n",
       "<text text-anchor=\"middle\" x=\"216.3154\" y=\"-98.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">Tanh[tanh]</text>\n",
       "<text text-anchor=\"middle\" x=\"216.3154\" y=\"-86.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">6</text>\n",
       "</g>\n",
       "<!-- Sequential/Linear[W0]/5&#45;&gt;Sequential/Tanh[tanh]/6 -->\n",
       "<g id=\"edge4\" class=\"edge\">\n",
       "<title>Sequential/Linear[W0]/5&#45;&gt;Sequential/Tanh[tanh]/6</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M216.3154,-159.7155C216.3154,-151.827 216.3154,-142.7865 216.3154,-134.2379\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"219.8155,-134.1649 216.3154,-124.165 212.8155,-134.165 219.8155,-134.1649\"/>\n",
       "</g>\n",
       "<!-- input/0 -->\n",
       "<g id=\"node2\" class=\"node\">\n",
       "<title>input/0</title>\n",
       "<polygon fill=\"#d3d3d3\" stroke=\"#000000\" points=\"162.3154,-278 108.3154,-278 108.3154,-246 162.3154,-246 162.3154,-278\"/>\n",
       "<text text-anchor=\"middle\" x=\"135.3154\" y=\"-264.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">input</text>\n",
       "<text text-anchor=\"middle\" x=\"135.3154\" y=\"-252.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">0</text>\n",
       "</g>\n",
       "<!-- input/0&#45;&gt;Sequential/Linear[W0]/5 -->\n",
       "<g id=\"edge1\" class=\"edge\">\n",
       "<title>input/0&#45;&gt;Sequential/Linear[W0]/5</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M151.707,-245.8108C161.7369,-235.9047 174.8684,-222.9353 186.6716,-211.2778\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"189.1542,-213.7453 193.8096,-204.228 184.2352,-208.7648 189.1542,-213.7453\"/>\n",
       "</g>\n",
       "<!-- Sequential/Linear[W0]/1 -->\n",
       "<g id=\"node3\" class=\"node\">\n",
       "<title>Sequential/Linear[W0]/1</title>\n",
       "<polygon fill=\"#d3d3d3\" stroke=\"#000000\" points=\"252.4472,-284 180.1836,-284 180.1836,-240 252.4472,-240 252.4472,-284\"/>\n",
       "<text text-anchor=\"middle\" x=\"216.3154\" y=\"-270.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">Sequential</text>\n",
       "<text text-anchor=\"middle\" x=\"216.3154\" y=\"-258.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">Linear[W0]</text>\n",
       "<text text-anchor=\"middle\" x=\"216.3154\" y=\"-246.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">1</text>\n",
       "</g>\n",
       "<!-- Sequential/Linear[W0]/1&#45;&gt;Sequential/Linear[W0]/5 -->\n",
       "<g id=\"edge2\" class=\"edge\">\n",
       "<title>Sequential/Linear[W0]/1&#45;&gt;Sequential/Linear[W0]/5</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M216.3154,-239.7155C216.3154,-231.827 216.3154,-222.7865 216.3154,-214.2379\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"219.8155,-214.1649 216.3154,-204.165 212.8155,-214.165 219.8155,-214.1649\"/>\n",
       "</g>\n",
       "<!-- Sequential/Linear[W0]/2 -->\n",
       "<g id=\"node4\" class=\"node\">\n",
       "<title>Sequential/Linear[W0]/2</title>\n",
       "<polygon fill=\"#d3d3d3\" stroke=\"#000000\" points=\"343.4472,-284 271.1836,-284 271.1836,-240 343.4472,-240 343.4472,-284\"/>\n",
       "<text text-anchor=\"middle\" x=\"307.3154\" y=\"-270.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">Sequential</text>\n",
       "<text text-anchor=\"middle\" x=\"307.3154\" y=\"-258.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">Linear[W0]</text>\n",
       "<text text-anchor=\"middle\" x=\"307.3154\" y=\"-246.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">2</text>\n",
       "</g>\n",
       "<!-- Sequential/Linear[W0]/2&#45;&gt;Sequential/Linear[W0]/5 -->\n",
       "<g id=\"edge3\" class=\"edge\">\n",
       "<title>Sequential/Linear[W0]/2&#45;&gt;Sequential/Linear[W0]/5</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M281.9669,-239.7155C271.8104,-230.7868 259.9752,-220.3822 249.1727,-210.8855\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"251.3494,-208.1389 241.5281,-204.165 246.7276,-213.3962 251.3494,-208.1389\"/>\n",
       "</g>\n",
       "<!-- Sequential/Linear[W1]/7 -->\n",
       "<g id=\"node6\" class=\"node\">\n",
       "<title>Sequential/Linear[W1]/7</title>\n",
       "<polygon fill=\"#d3d3d3\" stroke=\"#000000\" points=\"163.4472,-44 91.1836,-44 91.1836,0 163.4472,0 163.4472,-44\"/>\n",
       "<text text-anchor=\"middle\" x=\"127.3154\" y=\"-30.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">Sequential</text>\n",
       "<text text-anchor=\"middle\" x=\"127.3154\" y=\"-18.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">Linear[W1]</text>\n",
       "<text text-anchor=\"middle\" x=\"127.3154\" y=\"-6.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">7</text>\n",
       "</g>\n",
       "<!-- Sequential/Tanh[tanh]/6&#45;&gt;Sequential/Linear[W1]/7 -->\n",
       "<g id=\"edge5\" class=\"edge\">\n",
       "<title>Sequential/Tanh[tanh]/6&#45;&gt;Sequential/Linear[W1]/7</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M191.524,-79.7155C181.5907,-70.7868 170.0156,-60.3822 159.4505,-50.8855\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"161.7508,-48.247 151.9739,-44.165 157.0713,-53.453 161.7508,-48.247\"/>\n",
       "</g>\n",
       "<!-- Sequential/Linear[W1]/3 -->\n",
       "<g id=\"node7\" class=\"node\">\n",
       "<title>Sequential/Linear[W1]/3</title>\n",
       "<polygon fill=\"#d3d3d3\" stroke=\"#000000\" points=\"72.4472,-124 .1836,-124 .1836,-80 72.4472,-80 72.4472,-124\"/>\n",
       "<text text-anchor=\"middle\" x=\"36.3154\" y=\"-110.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">Sequential</text>\n",
       "<text text-anchor=\"middle\" x=\"36.3154\" y=\"-98.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">Linear[W1]</text>\n",
       "<text text-anchor=\"middle\" x=\"36.3154\" y=\"-86.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">3</text>\n",
       "</g>\n",
       "<!-- Sequential/Linear[W1]/3&#45;&gt;Sequential/Linear[W1]/7 -->\n",
       "<g id=\"edge6\" class=\"edge\">\n",
       "<title>Sequential/Linear[W1]/3&#45;&gt;Sequential/Linear[W1]/7</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M61.664,-79.7155C71.8205,-70.7868 83.6557,-60.3822 94.4582,-50.8855\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"96.9033,-53.3962 102.1028,-44.165 92.2815,-48.1389 96.9033,-53.3962\"/>\n",
       "</g>\n",
       "<!-- Sequential/Linear[W1]/4 -->\n",
       "<g id=\"node8\" class=\"node\">\n",
       "<title>Sequential/Linear[W1]/4</title>\n",
       "<polygon fill=\"#d3d3d3\" stroke=\"#000000\" points=\"163.4472,-124 91.1836,-124 91.1836,-80 163.4472,-80 163.4472,-124\"/>\n",
       "<text text-anchor=\"middle\" x=\"127.3154\" y=\"-110.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">Sequential</text>\n",
       "<text text-anchor=\"middle\" x=\"127.3154\" y=\"-98.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">Linear[W1]</text>\n",
       "<text text-anchor=\"middle\" x=\"127.3154\" y=\"-86.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">4</text>\n",
       "</g>\n",
       "<!-- Sequential/Linear[W1]/4&#45;&gt;Sequential/Linear[W1]/7 -->\n",
       "<g id=\"edge7\" class=\"edge\">\n",
       "<title>Sequential/Linear[W1]/4&#45;&gt;Sequential/Linear[W1]/7</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M127.3154,-79.7155C127.3154,-71.827 127.3154,-62.7865 127.3154,-54.2379\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"130.8155,-54.1649 127.3154,-44.165 123.8155,-54.165 130.8155,-54.1649\"/>\n",
       "</g>\n",
       "</g>\n",
       "</svg>\n"
      ],
      "text/plain": [
       "<graphviz.dot.Digraph at 0x112390a90>"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "with torch.onnx.set_training(model, False):\n",
    "    trace, _ = torch.jit.get_trace_graph(model, args=(x,))\n",
    "make_dot_from_trace(trace)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Double Backpropagation\n",
    "\n",
    "Implements \"Double Backpropagation\" from [Drucker and Lecun](http://yann.lecun.com/exdb/publis/pdf/drucker-lecun-92.pdf). The idea is to minimize the loss:\n",
    "\n",
    "$$f(x, \\theta) = f(x, \\theta) + g(\\frac{\\partial f(x, \\theta)}{\\partial x})$$\n",
    "\n",
    "where $x$ and $\\theta$ are input and parameter vectors, $f(x, \\theta)$ is the original loss function, and $g$ is a function of gradient w.r.t. input.\n",
    "\n",
    "This is used in [Improved Wasserstein GAN](https://arxiv.org/abs/1704.00028) and [Attention Transfer](https://arxiv.org/abs/1612.03928)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/svg+xml": [
       "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>\n",
       "<!DOCTYPE svg PUBLIC \"-//W3C//DTD SVG 1.1//EN\"\n",
       " \"http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd\">\n",
       "<!-- Generated by graphviz version 2.40.1 (20161225.0304)\n",
       " -->\n",
       "<!-- Title: %3 Pages: 1 -->\n",
       "<svg width=\"404pt\" height=\"512pt\"\n",
       " viewBox=\"0.00 0.00 403.97 512.00\" xmlns=\"http://www.w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\">\n",
       "<g id=\"graph0\" class=\"graph\" transform=\"scale(1 1) rotate(0) translate(4 508)\">\n",
       "<title>%3</title>\n",
       "<polygon fill=\"#ffffff\" stroke=\"transparent\" points=\"-4,4 -4,-508 399.9746,-508 399.9746,4 -4,4\"/>\n",
       "<!-- 4600931664 -->\n",
       "<g id=\"node1\" class=\"node\">\n",
       "<title>4600931664</title>\n",
       "<polygon fill=\"#caff70\" stroke=\"#000000\" points=\"295.1286,-20 196.846,-20 196.846,0 295.1286,0 295.1286,-20\"/>\n",
       "<text text-anchor=\"middle\" x=\"245.9873\" y=\"-6.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">ThAddBackward</text>\n",
       "</g>\n",
       "<!-- 4600931720 -->\n",
       "<g id=\"node2\" class=\"node\">\n",
       "<title>4600931720</title>\n",
       "<polygon fill=\"#d3d3d3\" stroke=\"#000000\" points=\"256.9541,-76 159.0205,-76 159.0205,-56 256.9541,-56 256.9541,-76\"/>\n",
       "<text text-anchor=\"middle\" x=\"207.9873\" y=\"-62.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">MeanBackward1</text>\n",
       "</g>\n",
       "<!-- 4600931720&#45;&gt;4600931664 -->\n",
       "<g id=\"edge1\" class=\"edge\">\n",
       "<title>4600931720&#45;&gt;4600931664</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M214.7809,-55.9883C219.9044,-48.4379 227.0815,-37.8611 233.2612,-28.7543\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"236.2765,-30.5439 238.9954,-20.3039 230.4842,-26.6134 236.2765,-30.5439\"/>\n",
       "</g>\n",
       "<!-- 4600931888 -->\n",
       "<g id=\"node3\" class=\"node\">\n",
       "<title>4600931888</title>\n",
       "<polygon fill=\"#d3d3d3\" stroke=\"#000000\" points=\"235.9717,-132 144.0029,-132 144.0029,-112 235.9717,-112 235.9717,-132\"/>\n",
       "<text text-anchor=\"middle\" x=\"189.9873\" y=\"-118.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">PowBackward0</text>\n",
       "</g>\n",
       "<!-- 4600931888&#45;&gt;4600931720 -->\n",
       "<g id=\"edge2\" class=\"edge\">\n",
       "<title>4600931888&#45;&gt;4600931720</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M193.2053,-111.9883C195.5311,-104.7525 198.7503,-94.7373 201.5909,-85.9\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"204.9473,-86.8952 204.6753,-76.3039 198.2831,-84.7531 204.9473,-86.8952\"/>\n",
       "</g>\n",
       "<!-- 4600932000 -->\n",
       "<g id=\"node4\" class=\"node\">\n",
       "<title>4600932000</title>\n",
       "<polygon fill=\"#d3d3d3\" stroke=\"#000000\" points=\"216.1365,-188 131.8381,-188 131.8381,-168 216.1365,-168 216.1365,-188\"/>\n",
       "<text text-anchor=\"middle\" x=\"173.9873\" y=\"-174.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">MmBackward</text>\n",
       "</g>\n",
       "<!-- 4600932000&#45;&gt;4600931888 -->\n",
       "<g id=\"edge3\" class=\"edge\">\n",
       "<title>4600932000&#45;&gt;4600931888</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M176.8478,-167.9883C178.8927,-160.8311 181.7145,-150.9547 184.2192,-142.1885\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"187.6614,-142.8807 187.0433,-132.3039 180.9307,-140.9576 187.6614,-142.8807\"/>\n",
       "</g>\n",
       "<!-- 4600888568 -->\n",
       "<g id=\"node5\" class=\"node\">\n",
       "<title>4600888568</title>\n",
       "<polygon fill=\"#d3d3d3\" stroke=\"#000000\" points=\"242.5948,-244 105.3798,-244 105.3798,-224 242.5948,-224 242.5948,-244\"/>\n",
       "<text text-anchor=\"middle\" x=\"173.9873\" y=\"-230.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">TanhBackwardBackward</text>\n",
       "</g>\n",
       "<!-- 4600888568&#45;&gt;4600932000 -->\n",
       "<g id=\"edge4\" class=\"edge\">\n",
       "<title>4600888568&#45;&gt;4600932000</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M173.9873,-223.9883C173.9873,-216.9098 173.9873,-207.1714 173.9873,-198.4779\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"177.4874,-198.3038 173.9873,-188.3039 170.4874,-198.3039 177.4874,-198.3038\"/>\n",
       "</g>\n",
       "<!-- 4600932224 -->\n",
       "<g id=\"node6\" class=\"node\">\n",
       "<title>4600932224</title>\n",
       "<polygon fill=\"#d3d3d3\" stroke=\"#000000\" points=\"148.1365,-300 63.8381,-300 63.8381,-280 148.1365,-280 148.1365,-300\"/>\n",
       "<text text-anchor=\"middle\" x=\"105.9873\" y=\"-286.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">MmBackward</text>\n",
       "</g>\n",
       "<!-- 4600932224&#45;&gt;4600888568 -->\n",
       "<g id=\"edge5\" class=\"edge\">\n",
       "<title>4600932224&#45;&gt;4600888568</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M118.1443,-279.9883C127.9812,-271.8873 142.0484,-260.3026 153.6027,-250.7873\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"155.9811,-253.3627 161.4755,-244.3039 151.5311,-247.9592 155.9811,-253.3627\"/>\n",
       "</g>\n",
       "<!-- 4600932392 -->\n",
       "<g id=\"node7\" class=\"node\">\n",
       "<title>4600932392</title>\n",
       "<polygon fill=\"#d3d3d3\" stroke=\"#000000\" points=\"289.9619,-362 218.0127,-362 218.0127,-342 289.9619,-342 289.9619,-362\"/>\n",
       "<text text-anchor=\"middle\" x=\"253.9873\" y=\"-348.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">TBackward</text>\n",
       "</g>\n",
       "<!-- 4600932392&#45;&gt;4600932224 -->\n",
       "<g id=\"edge6\" class=\"edge\">\n",
       "<title>4600932392&#45;&gt;4600932224</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M229.8792,-341.9006C205.2603,-331.5873 166.8512,-315.497 139.3266,-303.9664\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"140.6609,-300.7308 130.0851,-300.095 137.9562,-307.1871 140.6609,-300.7308\"/>\n",
       "</g>\n",
       "<!-- 4600932504 -->\n",
       "<g id=\"node8\" class=\"node\">\n",
       "<title>4600932504</title>\n",
       "<polygon fill=\"#d3d3d3\" stroke=\"#000000\" points=\"395.9619,-430 324.0127,-430 324.0127,-410 395.9619,-410 395.9619,-430\"/>\n",
       "<text text-anchor=\"middle\" x=\"359.9873\" y=\"-416.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">TBackward</text>\n",
       "</g>\n",
       "<!-- 4600932504&#45;&gt;4600932392 -->\n",
       "<g id=\"edge7\" class=\"edge\">\n",
       "<title>4600932504&#45;&gt;4600932392</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M344.3466,-409.9664C326.889,-398.7671 298.4284,-380.5094 278.1136,-367.4772\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"279.999,-364.5285 269.6921,-362.0748 276.2192,-370.4203 279.999,-364.5285\"/>\n",
       "</g>\n",
       "<!-- 4600931944 -->\n",
       "<g id=\"node19\" class=\"node\">\n",
       "<title>4600931944</title>\n",
       "<polygon fill=\"#d3d3d3\" stroke=\"#000000\" points=\"379.7963,-244 262.1783,-244 262.1783,-224 379.7963,-224 379.7963,-244\"/>\n",
       "<text text-anchor=\"middle\" x=\"320.9873\" y=\"-230.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">ThAddmmBackward</text>\n",
       "</g>\n",
       "<!-- 4600932504&#45;&gt;4600931944 -->\n",
       "<g id=\"edge23\" class=\"edge\">\n",
       "<title>4600932504&#45;&gt;4600931944</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M365.7521,-409.8683C378.5629,-385.9025 406.5497,-324.4108 383.9873,-280 377.3395,-266.9148 365.0297,-256.6647 353.043,-249.1942\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"354.4536,-245.9682 344.039,-244.0224 350.9671,-252.0382 354.4536,-245.9682\"/>\n",
       "</g>\n",
       "<!-- 4600932616 -->\n",
       "<g id=\"node9\" class=\"node\">\n",
       "<title>4600932616</title>\n",
       "<polygon fill=\"#add8e6\" stroke=\"#000000\" points=\"394.4736,-504 325.501,-504 325.501,-472 394.4736,-472 394.4736,-504\"/>\n",
       "<text text-anchor=\"middle\" x=\"359.9873\" y=\"-490.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">W1.weight</text>\n",
       "<text text-anchor=\"middle\" x=\"359.9873\" y=\"-478.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\"> (1, 16)</text>\n",
       "</g>\n",
       "<!-- 4600932616&#45;&gt;4600932504 -->\n",
       "<g id=\"edge8\" class=\"edge\">\n",
       "<title>4600932616&#45;&gt;4600932504</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M359.9873,-471.8849C359.9873,-462.5254 359.9873,-450.6379 359.9873,-440.6036\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"363.4874,-440.2954 359.9873,-430.2954 356.4874,-440.2954 363.4874,-440.2954\"/>\n",
       "</g>\n",
       "<!-- 4600932280 -->\n",
       "<g id=\"node10\" class=\"node\">\n",
       "<title>4600932280</title>\n",
       "<polygon fill=\"#d3d3d3\" stroke=\"#000000\" points=\"255.4514,-300 166.5232,-300 166.5232,-280 255.4514,-280 255.4514,-300\"/>\n",
       "<text text-anchor=\"middle\" x=\"210.9873\" y=\"-286.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">TanhBackward</text>\n",
       "</g>\n",
       "<!-- 4600932280&#45;&gt;4600888568 -->\n",
       "<g id=\"edge9\" class=\"edge\">\n",
       "<title>4600932280&#45;&gt;4600888568</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M204.3724,-279.9883C199.3838,-272.4379 192.3956,-261.8611 186.3785,-252.7543\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"189.228,-250.7178 180.7952,-244.3039 183.3877,-254.5767 189.228,-250.7178\"/>\n",
       "</g>\n",
       "<!-- 4600932280&#45;&gt;4600931944 -->\n",
       "<g id=\"edge22\" class=\"edge\">\n",
       "<title>4600932280&#45;&gt;4600931944</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M230.6531,-279.9883C247.7534,-271.2827 272.7564,-258.5539 292.1247,-248.6937\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"293.7972,-251.7698 301.1209,-244.1138 290.6213,-245.5316 293.7972,-251.7698\"/>\n",
       "</g>\n",
       "<!-- 4600932448 -->\n",
       "<g id=\"node11\" class=\"node\">\n",
       "<title>4600932448</title>\n",
       "<polygon fill=\"#d3d3d3\" stroke=\"#000000\" points=\"199.7963,-362 82.1783,-362 82.1783,-342 199.7963,-342 199.7963,-362\"/>\n",
       "<text text-anchor=\"middle\" x=\"140.9873\" y=\"-348.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">ThAddmmBackward</text>\n",
       "</g>\n",
       "<!-- 4600932448&#45;&gt;4600932280 -->\n",
       "<g id=\"edge10\" class=\"edge\">\n",
       "<title>4600932448&#45;&gt;4600932280</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M152.5464,-341.762C163.2494,-332.2821 179.3651,-318.0083 191.9641,-306.8492\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"194.478,-309.298 199.6433,-300.0475 189.8368,-304.0579 194.478,-309.298\"/>\n",
       "</g>\n",
       "<!-- 4600932672 -->\n",
       "<g id=\"node12\" class=\"node\">\n",
       "<title>4600932672</title>\n",
       "<polygon fill=\"#d3d3d3\" stroke=\"#000000\" points=\"191.7885,-430 90.1861,-430 90.1861,-410 191.7885,-410 191.7885,-430\"/>\n",
       "<text text-anchor=\"middle\" x=\"140.9873\" y=\"-416.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">ExpandBackward</text>\n",
       "</g>\n",
       "<!-- 4600932672&#45;&gt;4600932448 -->\n",
       "<g id=\"edge11\" class=\"edge\">\n",
       "<title>4600932672&#45;&gt;4600932448</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M140.9873,-409.9664C140.9873,-400.1231 140.9873,-384.827 140.9873,-372.4189\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"144.4874,-372.0748 140.9873,-362.0748 137.4874,-372.0748 144.4874,-372.0748\"/>\n",
       "</g>\n",
       "<!-- 4600932896 -->\n",
       "<g id=\"node13\" class=\"node\">\n",
       "<title>4600932896</title>\n",
       "<polygon fill=\"#add8e6\" stroke=\"#000000\" points=\"168.6446,-504 113.33,-504 113.33,-472 168.6446,-472 168.6446,-504\"/>\n",
       "<text text-anchor=\"middle\" x=\"140.9873\" y=\"-490.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">W0.bias</text>\n",
       "<text text-anchor=\"middle\" x=\"140.9873\" y=\"-478.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\"> (16)</text>\n",
       "</g>\n",
       "<!-- 4600932896&#45;&gt;4600932672 -->\n",
       "<g id=\"edge12\" class=\"edge\">\n",
       "<title>4600932896&#45;&gt;4600932672</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M140.9873,-471.8849C140.9873,-462.5254 140.9873,-450.6379 140.9873,-440.6036\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"144.4874,-440.2954 140.9873,-430.2954 137.4874,-440.2954 144.4874,-440.2954\"/>\n",
       "</g>\n",
       "<!-- 4600932784 -->\n",
       "<g id=\"node14\" class=\"node\">\n",
       "<title>4600932784</title>\n",
       "<polygon fill=\"#add8e6\" stroke=\"#000000\" points=\"263.9873,-436 209.9873,-436 209.9873,-404 263.9873,-404 263.9873,-436\"/>\n",
       "<text text-anchor=\"middle\" x=\"236.9873\" y=\"-422.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">x</text>\n",
       "<text text-anchor=\"middle\" x=\"236.9873\" y=\"-410.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\"> (1, 8)</text>\n",
       "</g>\n",
       "<!-- 4600932784&#45;&gt;4600932448 -->\n",
       "<g id=\"edge13\" class=\"edge\">\n",
       "<title>4600932784&#45;&gt;4600932448</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M214.2366,-403.8849C198.9413,-393.0507 178.8641,-378.8294 163.598,-368.0159\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"165.4214,-365.0184 155.238,-362.0943 161.3752,-370.7306 165.4214,-365.0184\"/>\n",
       "</g>\n",
       "<!-- 4600932728 -->\n",
       "<g id=\"node15\" class=\"node\">\n",
       "<title>4600932728</title>\n",
       "<polygon fill=\"#d3d3d3\" stroke=\"#000000\" points=\"71.9619,-430 .0127,-430 .0127,-410 71.9619,-410 71.9619,-430\"/>\n",
       "<text text-anchor=\"middle\" x=\"35.9873\" y=\"-416.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">TBackward</text>\n",
       "</g>\n",
       "<!-- 4600932728&#45;&gt;4600932448 -->\n",
       "<g id=\"edge14\" class=\"edge\">\n",
       "<title>4600932728&#45;&gt;4600932448</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M51.4804,-409.9664C68.6958,-398.8174 96.7129,-380.673 116.8174,-367.6529\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"118.9396,-370.4484 125.4306,-362.0748 115.1345,-364.5729 118.9396,-370.4484\"/>\n",
       "</g>\n",
       "<!-- 4600932112 -->\n",
       "<g id=\"node17\" class=\"node\">\n",
       "<title>4600932112</title>\n",
       "<polygon fill=\"#d3d3d3\" stroke=\"#000000\" points=\"81.9619,-244 10.0127,-244 10.0127,-224 81.9619,-224 81.9619,-244\"/>\n",
       "<text text-anchor=\"middle\" x=\"45.9873\" y=\"-230.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">TBackward</text>\n",
       "</g>\n",
       "<!-- 4600932728&#45;&gt;4600932112 -->\n",
       "<g id=\"edge17\" class=\"edge\">\n",
       "<title>4600932728&#45;&gt;4600932112</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M36.5249,-409.9998C38.1087,-380.5426 42.7783,-293.6873 44.9073,-254.0872\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"48.405,-254.2231 45.447,-244.0496 41.4151,-253.8473 48.405,-254.2231\"/>\n",
       "</g>\n",
       "<!-- 4600933008 -->\n",
       "<g id=\"node16\" class=\"node\">\n",
       "<title>4600933008</title>\n",
       "<polygon fill=\"#add8e6\" stroke=\"#000000\" points=\"70.4736,-504 1.501,-504 1.501,-472 70.4736,-472 70.4736,-504\"/>\n",
       "<text text-anchor=\"middle\" x=\"35.9873\" y=\"-490.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">W0.weight</text>\n",
       "<text text-anchor=\"middle\" x=\"35.9873\" y=\"-478.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\"> (16, 8)</text>\n",
       "</g>\n",
       "<!-- 4600933008&#45;&gt;4600932728 -->\n",
       "<g id=\"edge15\" class=\"edge\">\n",
       "<title>4600933008&#45;&gt;4600932728</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M35.9873,-471.8849C35.9873,-462.5254 35.9873,-450.6379 35.9873,-440.6036\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"39.4874,-440.2954 35.9873,-430.2954 32.4874,-440.2954 39.4874,-440.2954\"/>\n",
       "</g>\n",
       "<!-- 4600932112&#45;&gt;4600932000 -->\n",
       "<g id=\"edge16\" class=\"edge\">\n",
       "<title>4600932112&#45;&gt;4600932000</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M68.8711,-223.9883C89.1315,-215.1244 118.9251,-202.0897 141.6245,-192.1587\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"143.1113,-195.3286 150.87,-188.1138 140.3056,-188.9155 143.1113,-195.3286\"/>\n",
       "</g>\n",
       "<!-- 4600931776 -->\n",
       "<g id=\"node18\" class=\"node\">\n",
       "<title>4600931776</title>\n",
       "<polygon fill=\"#d3d3d3\" stroke=\"#000000\" points=\"359.9541,-132 262.0205,-132 262.0205,-112 359.9541,-112 359.9541,-132\"/>\n",
       "<text text-anchor=\"middle\" x=\"310.9873\" y=\"-118.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">MeanBackward1</text>\n",
       "</g>\n",
       "<!-- 4600931776&#45;&gt;4600931664 -->\n",
       "<g id=\"edge18\" class=\"edge\">\n",
       "<title>4600931776&#45;&gt;4600931664</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M304.9888,-111.6641C294.1055,-92.9113 270.9541,-53.0197 257.108,-29.1619\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"260.018,-27.2031 251.9713,-20.311 253.9637,-30.7168 260.018,-27.2031\"/>\n",
       "</g>\n",
       "<!-- 4600931944&#45;&gt;4600931776 -->\n",
       "<g id=\"edge19\" class=\"edge\">\n",
       "<title>4600931944&#45;&gt;4600931776</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M320.0645,-223.6641C318.4199,-205.2447 314.9544,-166.4314 312.8131,-142.4493\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"316.2835,-141.96 311.9079,-132.311 309.3112,-142.5826 316.2835,-141.96\"/>\n",
       "</g>\n",
       "<!-- 4600932168 -->\n",
       "<g id=\"node20\" class=\"node\">\n",
       "<title>4600932168</title>\n",
       "<polygon fill=\"#d3d3d3\" stroke=\"#000000\" points=\"374.7885,-300 273.1861,-300 273.1861,-280 374.7885,-280 374.7885,-300\"/>\n",
       "<text text-anchor=\"middle\" x=\"323.9873\" y=\"-286.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">ExpandBackward</text>\n",
       "</g>\n",
       "<!-- 4600932168&#45;&gt;4600931944 -->\n",
       "<g id=\"edge20\" class=\"edge\">\n",
       "<title>4600932168&#45;&gt;4600931944</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M323.451,-279.9883C323.0718,-272.9098 322.5501,-263.1714 322.0843,-254.4779\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"325.5693,-254.1023 321.5393,-244.3039 318.5793,-254.4768 325.5693,-254.1023\"/>\n",
       "</g>\n",
       "<!-- 4600932560 -->\n",
       "<g id=\"node21\" class=\"node\">\n",
       "<title>4600932560</title>\n",
       "<polygon fill=\"#add8e6\" stroke=\"#000000\" points=\"363.6446,-368 308.33,-368 308.33,-336 363.6446,-336 363.6446,-368\"/>\n",
       "<text text-anchor=\"middle\" x=\"335.9873\" y=\"-354.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">W1.bias</text>\n",
       "<text text-anchor=\"middle\" x=\"335.9873\" y=\"-342.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\"> (1)</text>\n",
       "</g>\n",
       "<!-- 4600932560&#45;&gt;4600932168 -->\n",
       "<g id=\"edge21\" class=\"edge\">\n",
       "<title>4600932560&#45;&gt;4600932168</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M332.8344,-335.7102C331.328,-327.9269 329.5098,-318.5331 327.9146,-310.2908\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"331.3273,-309.5039 325.9908,-300.3512 324.4548,-310.8341 331.3273,-309.5039\"/>\n",
       "</g>\n",
       "</g>\n",
       "</svg>\n"
      ],
      "text/plain": [
       "<graphviz.dot.Digraph at 0x1123ca470>"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x = torch.randn(1,8).requires_grad_(True)\n",
    "\n",
    "def double_backprop(inputs, net):\n",
    "    y = net(x).mean()\n",
    "    grad,  = torch.autograd.grad(y, x, create_graph=True, retain_graph=True)\n",
    "    return grad.pow(2).mean() + y\n",
    "\n",
    "make_dot(double_backprop(x, model), params=dict(list(model.named_parameters()) + [('x', x)]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## AlexNet"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/svg+xml": [
       "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>\n",
       "<!DOCTYPE svg PUBLIC \"-//W3C//DTD SVG 1.1//EN\"\n",
       " \"http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd\">\n",
       "<!-- Generated by graphviz version 2.40.1 (20161225.0304)\n",
       " -->\n",
       "<!-- Title: %3 Pages: 1 -->\n",
       "<svg width=\"668pt\" height=\"940pt\"\n",
       " viewBox=\"0.00 0.00 668.00 940.00\" xmlns=\"http://www.w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\">\n",
       "<g id=\"graph0\" class=\"graph\" transform=\"scale(.7231 .7231) rotate(0) translate(4 1296)\">\n",
       "<title>%3</title>\n",
       "<polygon fill=\"#ffffff\" stroke=\"transparent\" points=\"-4,4 -4,-1296 919.8262,-1296 919.8262,4 -4,4\"/>\n",
       "<!-- 4539553944 -->\n",
       "<g id=\"node1\" class=\"node\">\n",
       "<title>4539553944</title>\n",
       "<polygon fill=\"#caff70\" stroke=\"#000000\" points=\"798.3051,-20 680.6871,-20 680.6871,0 798.3051,0 798.3051,-20\"/>\n",
       "<text text-anchor=\"middle\" x=\"739.4961\" y=\"-6.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">ThAddmmBackward</text>\n",
       "</g>\n",
       "<!-- 4539554840 -->\n",
       "<g id=\"node2\" class=\"node\">\n",
       "<title>4539554840</title>\n",
       "<polygon fill=\"#d3d3d3\" stroke=\"#000000\" points=\"662.2973,-76 560.6949,-76 560.6949,-56 662.2973,-56 662.2973,-76\"/>\n",
       "<text text-anchor=\"middle\" x=\"611.4961\" y=\"-62.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">ExpandBackward</text>\n",
       "</g>\n",
       "<!-- 4539554840&#45;&gt;4539553944 -->\n",
       "<g id=\"edge1\" class=\"edge\">\n",
       "<title>4539554840&#45;&gt;4539553944</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M634.3799,-55.9883C654.6403,-47.1244 684.4339,-34.0897 707.1333,-24.1587\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"708.6201,-27.3286 716.3788,-20.1138 705.8143,-20.9155 708.6201,-27.3286\"/>\n",
       "</g>\n",
       "<!-- 4810466248 -->\n",
       "<g id=\"node3\" class=\"node\">\n",
       "<title>4810466248</title>\n",
       "<polygon fill=\"#add8e6\" stroke=\"#000000\" points=\"656.3268,-144 566.6654,-144 566.6654,-112 656.3268,-112 656.3268,-144\"/>\n",
       "<text text-anchor=\"middle\" x=\"611.4961\" y=\"-130.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">classifier.6.bias</text>\n",
       "<text text-anchor=\"middle\" x=\"611.4961\" y=\"-118.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\"> (1000)</text>\n",
       "</g>\n",
       "<!-- 4810466248&#45;&gt;4539554840 -->\n",
       "<g id=\"edge2\" class=\"edge\">\n",
       "<title>4810466248&#45;&gt;4539554840</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M611.4961,-111.7102C611.4961,-104.0144 611.4961,-94.744 611.4961,-86.5691\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"614.9962,-86.3512 611.4961,-76.3512 607.9962,-86.3513 614.9962,-86.3512\"/>\n",
       "</g>\n",
       "<!-- 4539554728 -->\n",
       "<g id=\"node4\" class=\"node\">\n",
       "<title>4539554728</title>\n",
       "<polygon fill=\"#d3d3d3\" stroke=\"#000000\" points=\"799.2973,-76 679.6949,-76 679.6949,-56 799.2973,-56 799.2973,-76\"/>\n",
       "<text text-anchor=\"middle\" x=\"739.4961\" y=\"-62.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">ThresholdBackward1</text>\n",
       "</g>\n",
       "<!-- 4539554728&#45;&gt;4539553944 -->\n",
       "<g id=\"edge3\" class=\"edge\">\n",
       "<title>4539554728&#45;&gt;4539553944</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M739.4961,-55.9883C739.4961,-48.9098 739.4961,-39.1714 739.4961,-30.4779\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"742.9962,-30.3038 739.4961,-20.3039 735.9962,-30.3039 742.9962,-30.3038\"/>\n",
       "</g>\n",
       "<!-- 4810466696 -->\n",
       "<g id=\"node5\" class=\"node\">\n",
       "<title>4810466696</title>\n",
       "<polygon fill=\"#d3d3d3\" stroke=\"#000000\" points=\"794.3051,-138 676.6871,-138 676.6871,-118 794.3051,-118 794.3051,-138\"/>\n",
       "<text text-anchor=\"middle\" x=\"735.4961\" y=\"-124.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">ThAddmmBackward</text>\n",
       "</g>\n",
       "<!-- 4810466696&#45;&gt;4539554728 -->\n",
       "<g id=\"edge4\" class=\"edge\">\n",
       "<title>4810466696&#45;&gt;4539554728</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M736.1566,-117.762C736.71,-109.185 737.5165,-96.6836 738.1983,-86.1154\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"741.6967,-86.2522 738.8479,-76.0475 734.7112,-85.8014 741.6967,-86.2522\"/>\n",
       "</g>\n",
       "<!-- 4810466808 -->\n",
       "<g id=\"node6\" class=\"node\">\n",
       "<title>4810466808</title>\n",
       "<polygon fill=\"#d3d3d3\" stroke=\"#000000\" points=\"665.2973,-200 563.6949,-200 563.6949,-180 665.2973,-180 665.2973,-200\"/>\n",
       "<text text-anchor=\"middle\" x=\"614.4961\" y=\"-186.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">ExpandBackward</text>\n",
       "</g>\n",
       "<!-- 4810466808&#45;&gt;4810466696 -->\n",
       "<g id=\"edge5\" class=\"edge\">\n",
       "<title>4810466808&#45;&gt;4810466696</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M634.2061,-179.9006C653.8942,-169.8125 684.3698,-154.1969 706.7525,-142.7281\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"708.4909,-145.7702 715.7945,-138.095 705.2987,-139.5404 708.4909,-145.7702\"/>\n",
       "</g>\n",
       "<!-- 4810466976 -->\n",
       "<g id=\"node7\" class=\"node\">\n",
       "<title>4810466976</title>\n",
       "<polygon fill=\"#add8e6\" stroke=\"#000000\" points=\"658.3268,-268 568.6654,-268 568.6654,-236 658.3268,-236 658.3268,-268\"/>\n",
       "<text text-anchor=\"middle\" x=\"613.4961\" y=\"-254.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">classifier.4.bias</text>\n",
       "<text text-anchor=\"middle\" x=\"613.4961\" y=\"-242.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\"> (4096)</text>\n",
       "</g>\n",
       "<!-- 4810466976&#45;&gt;4810466808 -->\n",
       "<g id=\"edge6\" class=\"edge\">\n",
       "<title>4810466976&#45;&gt;4810466808</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M613.7588,-235.7102C613.883,-228.0144 614.0325,-218.744 614.1643,-210.5691\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"617.6673,-210.4064 614.3291,-200.3512 610.6682,-210.2934 617.6673,-210.4064\"/>\n",
       "</g>\n",
       "<!-- 4810253824 -->\n",
       "<g id=\"node8\" class=\"node\">\n",
       "<title>4810253824</title>\n",
       "<polygon fill=\"#d3d3d3\" stroke=\"#000000\" points=\"787.6373,-200 683.3549,-200 683.3549,-180 787.6373,-180 787.6373,-200\"/>\n",
       "<text text-anchor=\"middle\" x=\"735.4961\" y=\"-186.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">DropoutBackward</text>\n",
       "</g>\n",
       "<!-- 4810253824&#45;&gt;4810466696 -->\n",
       "<g id=\"edge7\" class=\"edge\">\n",
       "<title>4810253824&#45;&gt;4810466696</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M735.4961,-179.762C735.4961,-171.185 735.4961,-158.6836 735.4961,-148.1154\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"738.9962,-148.0475 735.4961,-138.0475 731.9962,-148.0476 738.9962,-148.0475\"/>\n",
       "</g>\n",
       "<!-- 4810467032 -->\n",
       "<g id=\"node9\" class=\"node\">\n",
       "<title>4810467032</title>\n",
       "<polygon fill=\"#d3d3d3\" stroke=\"#000000\" points=\"795.2973,-262 675.6949,-262 675.6949,-242 795.2973,-242 795.2973,-262\"/>\n",
       "<text text-anchor=\"middle\" x=\"735.4961\" y=\"-248.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">ThresholdBackward1</text>\n",
       "</g>\n",
       "<!-- 4810467032&#45;&gt;4810253824 -->\n",
       "<g id=\"edge8\" class=\"edge\">\n",
       "<title>4810467032&#45;&gt;4810253824</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M735.4961,-241.762C735.4961,-233.185 735.4961,-220.6836 735.4961,-210.1154\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"738.9962,-210.0475 735.4961,-200.0475 731.9962,-210.0476 738.9962,-210.0475\"/>\n",
       "</g>\n",
       "<!-- 4810467144 -->\n",
       "<g id=\"node10\" class=\"node\">\n",
       "<title>4810467144</title>\n",
       "<polygon fill=\"#d3d3d3\" stroke=\"#000000\" points=\"794.3051,-324 676.6871,-324 676.6871,-304 794.3051,-304 794.3051,-324\"/>\n",
       "<text text-anchor=\"middle\" x=\"735.4961\" y=\"-310.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">ThAddmmBackward</text>\n",
       "</g>\n",
       "<!-- 4810467144&#45;&gt;4810467032 -->\n",
       "<g id=\"edge9\" class=\"edge\">\n",
       "<title>4810467144&#45;&gt;4810467032</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M735.4961,-303.762C735.4961,-295.185 735.4961,-282.6836 735.4961,-272.1154\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"738.9962,-272.0475 735.4961,-262.0475 731.9962,-272.0476 738.9962,-272.0475\"/>\n",
       "</g>\n",
       "<!-- 4810467256 -->\n",
       "<g id=\"node11\" class=\"node\">\n",
       "<title>4810467256</title>\n",
       "<polygon fill=\"#d3d3d3\" stroke=\"#000000\" points=\"665.2973,-380 563.6949,-380 563.6949,-360 665.2973,-360 665.2973,-380\"/>\n",
       "<text text-anchor=\"middle\" x=\"614.4961\" y=\"-366.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">ExpandBackward</text>\n",
       "</g>\n",
       "<!-- 4810467256&#45;&gt;4810467144 -->\n",
       "<g id=\"edge10\" class=\"edge\">\n",
       "<title>4810467256&#45;&gt;4810467144</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M636.1285,-359.9883C655.1098,-351.2036 682.9429,-338.3222 704.3268,-328.4255\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"706.0379,-331.4903 713.643,-324.1138 703.0978,-325.1376 706.0379,-331.4903\"/>\n",
       "</g>\n",
       "<!-- 4810467424 -->\n",
       "<g id=\"node12\" class=\"node\">\n",
       "<title>4810467424</title>\n",
       "<polygon fill=\"#add8e6\" stroke=\"#000000\" points=\"659.3268,-448 569.6654,-448 569.6654,-416 659.3268,-416 659.3268,-448\"/>\n",
       "<text text-anchor=\"middle\" x=\"614.4961\" y=\"-434.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">classifier.1.bias</text>\n",
       "<text text-anchor=\"middle\" x=\"614.4961\" y=\"-422.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\"> (4096)</text>\n",
       "</g>\n",
       "<!-- 4810467424&#45;&gt;4810467256 -->\n",
       "<g id=\"edge11\" class=\"edge\">\n",
       "<title>4810467424&#45;&gt;4810467256</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M614.4961,-415.7102C614.4961,-408.0144 614.4961,-398.744 614.4961,-390.5691\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"617.9962,-390.3512 614.4961,-380.3512 610.9962,-390.3513 617.9962,-390.3512\"/>\n",
       "</g>\n",
       "<!-- 4810253336 -->\n",
       "<g id=\"node13\" class=\"node\">\n",
       "<title>4810253336</title>\n",
       "<polygon fill=\"#d3d3d3\" stroke=\"#000000\" points=\"787.6373,-380 683.3549,-380 683.3549,-360 787.6373,-360 787.6373,-380\"/>\n",
       "<text text-anchor=\"middle\" x=\"735.4961\" y=\"-366.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">DropoutBackward</text>\n",
       "</g>\n",
       "<!-- 4810253336&#45;&gt;4810467144 -->\n",
       "<g id=\"edge12\" class=\"edge\">\n",
       "<title>4810253336&#45;&gt;4810467144</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M735.4961,-359.9883C735.4961,-352.9098 735.4961,-343.1714 735.4961,-334.4779\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"738.9962,-334.3038 735.4961,-324.3039 731.9962,-334.3039 738.9962,-334.3038\"/>\n",
       "</g>\n",
       "<!-- 4810467480 -->\n",
       "<g id=\"node14\" class=\"node\">\n",
       "<title>4810467480</title>\n",
       "<polygon fill=\"#d3d3d3\" stroke=\"#000000\" points=\"778.418,-442 688.5742,-442 688.5742,-422 778.418,-422 778.418,-442\"/>\n",
       "<text text-anchor=\"middle\" x=\"733.4961\" y=\"-428.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">ViewBackward</text>\n",
       "</g>\n",
       "<!-- 4810467480&#45;&gt;4810253336 -->\n",
       "<g id=\"edge13\" class=\"edge\">\n",
       "<title>4810467480&#45;&gt;4810253336</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M733.8264,-421.762C734.103,-413.185 734.5063,-400.6836 734.8472,-390.1154\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"738.3477,-390.1552 735.172,-380.0475 731.3513,-389.9295 738.3477,-390.1552\"/>\n",
       "</g>\n",
       "<!-- 4810467592 -->\n",
       "<g id=\"node15\" class=\"node\">\n",
       "<title>4810467592</title>\n",
       "<polygon fill=\"#d3d3d3\" stroke=\"#000000\" points=\"824.4831,-504 642.5091,-504 642.5091,-484 824.4831,-484 824.4831,-504\"/>\n",
       "<text text-anchor=\"middle\" x=\"733.4961\" y=\"-490.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">MaxPool2DWithIndicesBackward</text>\n",
       "</g>\n",
       "<!-- 4810467592&#45;&gt;4810467480 -->\n",
       "<g id=\"edge14\" class=\"edge\">\n",
       "<title>4810467592&#45;&gt;4810467480</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M733.4961,-483.762C733.4961,-475.185 733.4961,-462.6836 733.4961,-452.1154\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"736.9962,-452.0475 733.4961,-442.0475 729.9962,-452.0476 736.9962,-452.0475\"/>\n",
       "</g>\n",
       "<!-- 4810467704 -->\n",
       "<g id=\"node16\" class=\"node\">\n",
       "<title>4810467704</title>\n",
       "<polygon fill=\"#d3d3d3\" stroke=\"#000000\" points=\"793.2973,-560 673.6949,-560 673.6949,-540 793.2973,-540 793.2973,-560\"/>\n",
       "<text text-anchor=\"middle\" x=\"733.4961\" y=\"-546.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">ThresholdBackward1</text>\n",
       "</g>\n",
       "<!-- 4810467704&#45;&gt;4810467592 -->\n",
       "<g id=\"edge15\" class=\"edge\">\n",
       "<title>4810467704&#45;&gt;4810467592</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M733.4961,-539.9883C733.4961,-532.9098 733.4961,-523.1714 733.4961,-514.4779\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"736.9962,-514.3038 733.4961,-504.3039 729.9962,-514.3039 736.9962,-514.3038\"/>\n",
       "</g>\n",
       "<!-- 4810467816 -->\n",
       "<g id=\"node17\" class=\"node\">\n",
       "<title>4810467816</title>\n",
       "<polygon fill=\"#d3d3d3\" stroke=\"#000000\" points=\"798.6411,-616 668.3511,-616 668.3511,-596 798.6411,-596 798.6411,-616\"/>\n",
       "<text text-anchor=\"middle\" x=\"733.4961\" y=\"-602.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">ThnnConv2DBackward</text>\n",
       "</g>\n",
       "<!-- 4810467816&#45;&gt;4810467704 -->\n",
       "<g id=\"edge16\" class=\"edge\">\n",
       "<title>4810467816&#45;&gt;4810467704</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M733.4961,-595.9883C733.4961,-588.9098 733.4961,-579.1714 733.4961,-570.4779\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"736.9962,-570.3038 733.4961,-560.3039 729.9962,-570.3039 736.9962,-570.3038\"/>\n",
       "</g>\n",
       "<!-- 4810467928 -->\n",
       "<g id=\"node18\" class=\"node\">\n",
       "<title>4810467928</title>\n",
       "<polygon fill=\"#d3d3d3\" stroke=\"#000000\" points=\"663.2973,-678 543.6949,-678 543.6949,-658 663.2973,-658 663.2973,-678\"/>\n",
       "<text text-anchor=\"middle\" x=\"603.4961\" y=\"-664.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">ThresholdBackward1</text>\n",
       "</g>\n",
       "<!-- 4810467928&#45;&gt;4810467816 -->\n",
       "<g id=\"edge17\" class=\"edge\">\n",
       "<title>4810467928&#45;&gt;4810467816</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M624.6722,-657.9006C646.0136,-647.7224 679.1527,-631.9176 703.2565,-620.4219\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"704.8097,-623.5589 712.3291,-616.095 701.7964,-617.2407 704.8097,-623.5589\"/>\n",
       "</g>\n",
       "<!-- 4810468152 -->\n",
       "<g id=\"node19\" class=\"node\">\n",
       "<title>4810468152</title>\n",
       "<polygon fill=\"#d3d3d3\" stroke=\"#000000\" points=\"668.6411,-740 538.3511,-740 538.3511,-720 668.6411,-720 668.6411,-740\"/>\n",
       "<text text-anchor=\"middle\" x=\"603.4961\" y=\"-726.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">ThnnConv2DBackward</text>\n",
       "</g>\n",
       "<!-- 4810468152&#45;&gt;4810467928 -->\n",
       "<g id=\"edge18\" class=\"edge\">\n",
       "<title>4810468152&#45;&gt;4810467928</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M603.4961,-719.762C603.4961,-711.185 603.4961,-698.6836 603.4961,-688.1154\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"606.9962,-688.0475 603.4961,-678.0475 599.9962,-688.0476 606.9962,-688.0475\"/>\n",
       "</g>\n",
       "<!-- 4810468264 -->\n",
       "<g id=\"node20\" class=\"node\">\n",
       "<title>4810468264</title>\n",
       "<polygon fill=\"#d3d3d3\" stroke=\"#000000\" points=\"536.2973,-802 416.6949,-802 416.6949,-782 536.2973,-782 536.2973,-802\"/>\n",
       "<text text-anchor=\"middle\" x=\"476.4961\" y=\"-788.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">ThresholdBackward1</text>\n",
       "</g>\n",
       "<!-- 4810468264&#45;&gt;4810468152 -->\n",
       "<g id=\"edge19\" class=\"edge\">\n",
       "<title>4810468264&#45;&gt;4810468152</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M497.1835,-781.9006C517.9401,-771.7675 550.1205,-756.0574 573.6413,-744.5748\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"575.3667,-747.6273 582.8176,-740.095 572.2957,-741.3369 575.3667,-747.6273\"/>\n",
       "</g>\n",
       "<!-- 4810468488 -->\n",
       "<g id=\"node21\" class=\"node\">\n",
       "<title>4810468488</title>\n",
       "<polygon fill=\"#d3d3d3\" stroke=\"#000000\" points=\"541.6411,-864 411.3511,-864 411.3511,-844 541.6411,-844 541.6411,-864\"/>\n",
       "<text text-anchor=\"middle\" x=\"476.4961\" y=\"-850.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">ThnnConv2DBackward</text>\n",
       "</g>\n",
       "<!-- 4810468488&#45;&gt;4810468264 -->\n",
       "<g id=\"edge20\" class=\"edge\">\n",
       "<title>4810468488&#45;&gt;4810468264</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M476.4961,-843.762C476.4961,-835.185 476.4961,-822.6836 476.4961,-812.1154\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"479.9962,-812.0475 476.4961,-802.0475 472.9962,-812.0476 479.9962,-812.0475\"/>\n",
       "</g>\n",
       "<!-- 4810468600 -->\n",
       "<g id=\"node22\" class=\"node\">\n",
       "<title>4810468600</title>\n",
       "<polygon fill=\"#d3d3d3\" stroke=\"#000000\" points=\"409.4831,-926 227.5091,-926 227.5091,-906 409.4831,-906 409.4831,-926\"/>\n",
       "<text text-anchor=\"middle\" x=\"318.4961\" y=\"-912.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">MaxPool2DWithIndicesBackward</text>\n",
       "</g>\n",
       "<!-- 4810468600&#45;&gt;4810468488 -->\n",
       "<g id=\"edge21\" class=\"edge\">\n",
       "<title>4810468600&#45;&gt;4810468488</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M344.2332,-905.9006C370.6302,-895.5423 411.8782,-879.3564 441.2885,-867.8157\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"442.7396,-871.0061 450.77,-864.095 440.1826,-864.4898 442.7396,-871.0061\"/>\n",
       "</g>\n",
       "<!-- 4810468824 -->\n",
       "<g id=\"node23\" class=\"node\">\n",
       "<title>4810468824</title>\n",
       "<polygon fill=\"#d3d3d3\" stroke=\"#000000\" points=\"378.2973,-988 258.6949,-988 258.6949,-968 378.2973,-968 378.2973,-988\"/>\n",
       "<text text-anchor=\"middle\" x=\"318.4961\" y=\"-974.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">ThresholdBackward1</text>\n",
       "</g>\n",
       "<!-- 4810468824&#45;&gt;4810468600 -->\n",
       "<g id=\"edge22\" class=\"edge\">\n",
       "<title>4810468824&#45;&gt;4810468600</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M318.4961,-967.762C318.4961,-959.185 318.4961,-946.6836 318.4961,-936.1154\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"321.9962,-936.0475 318.4961,-926.0475 314.9962,-936.0476 321.9962,-936.0475\"/>\n",
       "</g>\n",
       "<!-- 4810468936 -->\n",
       "<g id=\"node24\" class=\"node\">\n",
       "<title>4810468936</title>\n",
       "<polygon fill=\"#d3d3d3\" stroke=\"#000000\" points=\"383.6411,-1044 253.3511,-1044 253.3511,-1024 383.6411,-1024 383.6411,-1044\"/>\n",
       "<text text-anchor=\"middle\" x=\"318.4961\" y=\"-1030.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">ThnnConv2DBackward</text>\n",
       "</g>\n",
       "<!-- 4810468936&#45;&gt;4810468824 -->\n",
       "<g id=\"edge23\" class=\"edge\">\n",
       "<title>4810468936&#45;&gt;4810468824</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M318.4961,-1023.9883C318.4961,-1016.9098 318.4961,-1007.1714 318.4961,-998.4779\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"321.9962,-998.3038 318.4961,-988.3039 314.9962,-998.3039 321.9962,-998.3038\"/>\n",
       "</g>\n",
       "<!-- 4810469048 -->\n",
       "<g id=\"node25\" class=\"node\">\n",
       "<title>4810469048</title>\n",
       "<polygon fill=\"#d3d3d3\" stroke=\"#000000\" points=\"251.4831,-1106 69.5091,-1106 69.5091,-1086 251.4831,-1086 251.4831,-1106\"/>\n",
       "<text text-anchor=\"middle\" x=\"160.4961\" y=\"-1092.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">MaxPool2DWithIndicesBackward</text>\n",
       "</g>\n",
       "<!-- 4810469048&#45;&gt;4810468936 -->\n",
       "<g id=\"edge24\" class=\"edge\">\n",
       "<title>4810469048&#45;&gt;4810468936</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M186.2332,-1085.9006C212.6302,-1075.5423 253.8782,-1059.3564 283.2885,-1047.8157\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"284.7396,-1051.0061 292.77,-1044.095 282.1826,-1044.4898 284.7396,-1051.0061\"/>\n",
       "</g>\n",
       "<!-- 4810469272 -->\n",
       "<g id=\"node26\" class=\"node\">\n",
       "<title>4810469272</title>\n",
       "<polygon fill=\"#d3d3d3\" stroke=\"#000000\" points=\"220.2973,-1168 100.6949,-1168 100.6949,-1148 220.2973,-1148 220.2973,-1168\"/>\n",
       "<text text-anchor=\"middle\" x=\"160.4961\" y=\"-1154.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">ThresholdBackward1</text>\n",
       "</g>\n",
       "<!-- 4810469272&#45;&gt;4810469048 -->\n",
       "<g id=\"edge25\" class=\"edge\">\n",
       "<title>4810469272&#45;&gt;4810469048</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M160.4961,-1147.762C160.4961,-1139.185 160.4961,-1126.6836 160.4961,-1116.1154\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"163.9962,-1116.0475 160.4961,-1106.0475 156.9962,-1116.0476 163.9962,-1116.0475\"/>\n",
       "</g>\n",
       "<!-- 4970037320 -->\n",
       "<g id=\"node27\" class=\"node\">\n",
       "<title>4970037320</title>\n",
       "<polygon fill=\"#d3d3d3\" stroke=\"#000000\" points=\"225.6411,-1224 95.3511,-1224 95.3511,-1204 225.6411,-1204 225.6411,-1224\"/>\n",
       "<text text-anchor=\"middle\" x=\"160.4961\" y=\"-1210.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">ThnnConv2DBackward</text>\n",
       "</g>\n",
       "<!-- 4970037320&#45;&gt;4810469272 -->\n",
       "<g id=\"edge26\" class=\"edge\">\n",
       "<title>4970037320&#45;&gt;4810469272</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M160.4961,-1203.9883C160.4961,-1196.9098 160.4961,-1187.1714 160.4961,-1178.4779\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"163.9962,-1178.3038 160.4961,-1168.3039 156.9962,-1178.3039 163.9962,-1178.3038\"/>\n",
       "</g>\n",
       "<!-- 4970037488 -->\n",
       "<g id=\"node28\" class=\"node\">\n",
       "<title>4970037488</title>\n",
       "<polygon fill=\"#add8e6\" stroke=\"#000000\" points=\"92.9883,-1292 .0039,-1292 .0039,-1260 92.9883,-1260 92.9883,-1292\"/>\n",
       "<text text-anchor=\"middle\" x=\"46.4961\" y=\"-1278.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">x</text>\n",
       "<text text-anchor=\"middle\" x=\"46.4961\" y=\"-1266.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\"> (1, 3, 227, 227)</text>\n",
       "</g>\n",
       "<!-- 4970037488&#45;&gt;4970037320 -->\n",
       "<g id=\"edge27\" class=\"edge\">\n",
       "<title>4970037488&#45;&gt;4970037320</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M76.1507,-1259.8721C93.7404,-1250.3057 115.8834,-1238.263 133.0674,-1228.9174\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"134.9308,-1231.8881 142.0434,-1224.0357 131.5863,-1225.7387 134.9308,-1231.8881\"/>\n",
       "</g>\n",
       "<!-- 4970037544 -->\n",
       "<g id=\"node29\" class=\"node\">\n",
       "<title>4970037544</title>\n",
       "<polygon fill=\"#add8e6\" stroke=\"#000000\" points=\"209.6315,-1292 111.3607,-1292 111.3607,-1260 209.6315,-1260 209.6315,-1292\"/>\n",
       "<text text-anchor=\"middle\" x=\"160.4961\" y=\"-1278.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">features.0.weight</text>\n",
       "<text text-anchor=\"middle\" x=\"160.4961\" y=\"-1266.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\"> (64, 3, 11, 11)</text>\n",
       "</g>\n",
       "<!-- 4970037544&#45;&gt;4970037320 -->\n",
       "<g id=\"edge28\" class=\"edge\">\n",
       "<title>4970037544&#45;&gt;4970037320</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M160.4961,-1259.7102C160.4961,-1252.0144 160.4961,-1242.744 160.4961,-1234.5691\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"163.9962,-1234.3512 160.4961,-1224.3512 156.9962,-1234.3513 163.9962,-1234.3512\"/>\n",
       "</g>\n",
       "<!-- 4970037600 -->\n",
       "<g id=\"node30\" class=\"node\">\n",
       "<title>4970037600</title>\n",
       "<polygon fill=\"#add8e6\" stroke=\"#000000\" points=\"313.3013,-1292 227.6909,-1292 227.6909,-1260 313.3013,-1260 313.3013,-1292\"/>\n",
       "<text text-anchor=\"middle\" x=\"270.4961\" y=\"-1278.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">features.0.bias</text>\n",
       "<text text-anchor=\"middle\" x=\"270.4961\" y=\"-1266.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\"> (64)</text>\n",
       "</g>\n",
       "<!-- 4970037600&#45;&gt;4970037320 -->\n",
       "<g id=\"edge29\" class=\"edge\">\n",
       "<title>4970037600&#45;&gt;4970037320</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M241.882,-1259.8721C225.0666,-1250.3943 203.9387,-1238.4858 187.4242,-1229.1777\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"188.7314,-1225.8968 178.3013,-1224.0357 185.2943,-1231.9949 188.7314,-1225.8968\"/>\n",
       "</g>\n",
       "<!-- 4810469104 -->\n",
       "<g id=\"node31\" class=\"node\">\n",
       "<title>4810469104</title>\n",
       "<polygon fill=\"#add8e6\" stroke=\"#000000\" points=\"367.6315,-1112 269.3607,-1112 269.3607,-1080 367.6315,-1080 367.6315,-1112\"/>\n",
       "<text text-anchor=\"middle\" x=\"318.4961\" y=\"-1098.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">features.3.weight</text>\n",
       "<text text-anchor=\"middle\" x=\"318.4961\" y=\"-1086.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\"> (192, 64, 5, 5)</text>\n",
       "</g>\n",
       "<!-- 4810469104&#45;&gt;4810468936 -->\n",
       "<g id=\"edge30\" class=\"edge\">\n",
       "<title>4810469104&#45;&gt;4810468936</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M318.4961,-1079.7102C318.4961,-1072.0144 318.4961,-1062.744 318.4961,-1054.5691\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"321.9962,-1054.3512 318.4961,-1044.3512 314.9962,-1054.3513 321.9962,-1054.3512\"/>\n",
       "</g>\n",
       "<!-- 4810469160 -->\n",
       "<g id=\"node32\" class=\"node\">\n",
       "<title>4810469160</title>\n",
       "<polygon fill=\"#add8e6\" stroke=\"#000000\" points=\"471.3013,-1112 385.6909,-1112 385.6909,-1080 471.3013,-1080 471.3013,-1112\"/>\n",
       "<text text-anchor=\"middle\" x=\"428.4961\" y=\"-1098.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">features.3.bias</text>\n",
       "<text text-anchor=\"middle\" x=\"428.4961\" y=\"-1086.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\"> (192)</text>\n",
       "</g>\n",
       "<!-- 4810469160&#45;&gt;4810468936 -->\n",
       "<g id=\"edge31\" class=\"edge\">\n",
       "<title>4810469160&#45;&gt;4810468936</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M399.882,-1079.8721C383.0666,-1070.3943 361.9387,-1058.4858 345.4242,-1049.1777\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"346.7314,-1045.8968 336.3013,-1044.0357 343.2943,-1051.9949 346.7314,-1045.8968\"/>\n",
       "</g>\n",
       "<!-- 4810468656 -->\n",
       "<g id=\"node33\" class=\"node\">\n",
       "<title>4810468656</title>\n",
       "<polygon fill=\"#add8e6\" stroke=\"#000000\" points=\"525.6315,-932 427.3607,-932 427.3607,-900 525.6315,-900 525.6315,-932\"/>\n",
       "<text text-anchor=\"middle\" x=\"476.4961\" y=\"-918.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">features.6.weight</text>\n",
       "<text text-anchor=\"middle\" x=\"476.4961\" y=\"-906.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\"> (384, 192, 3, 3)</text>\n",
       "</g>\n",
       "<!-- 4810468656&#45;&gt;4810468488 -->\n",
       "<g id=\"edge32\" class=\"edge\">\n",
       "<title>4810468656&#45;&gt;4810468488</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M476.4961,-899.7102C476.4961,-892.0144 476.4961,-882.744 476.4961,-874.5691\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"479.9962,-874.3512 476.4961,-864.3512 472.9962,-874.3513 479.9962,-874.3512\"/>\n",
       "</g>\n",
       "<!-- 4810468712 -->\n",
       "<g id=\"node34\" class=\"node\">\n",
       "<title>4810468712</title>\n",
       "<polygon fill=\"#add8e6\" stroke=\"#000000\" points=\"629.3013,-932 543.6909,-932 543.6909,-900 629.3013,-900 629.3013,-932\"/>\n",
       "<text text-anchor=\"middle\" x=\"586.4961\" y=\"-918.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">features.6.bias</text>\n",
       "<text text-anchor=\"middle\" x=\"586.4961\" y=\"-906.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\"> (384)</text>\n",
       "</g>\n",
       "<!-- 4810468712&#45;&gt;4810468488 -->\n",
       "<g id=\"edge33\" class=\"edge\">\n",
       "<title>4810468712&#45;&gt;4810468488</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M557.882,-899.8721C541.0666,-890.3943 519.9387,-878.4858 503.4242,-869.1777\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"504.7314,-865.8968 494.3013,-864.0357 501.2943,-871.9949 504.7314,-865.8968\"/>\n",
       "</g>\n",
       "<!-- 4810468320 -->\n",
       "<g id=\"node35\" class=\"node\">\n",
       "<title>4810468320</title>\n",
       "<polygon fill=\"#add8e6\" stroke=\"#000000\" points=\"652.6315,-808 554.3607,-808 554.3607,-776 652.6315,-776 652.6315,-808\"/>\n",
       "<text text-anchor=\"middle\" x=\"603.4961\" y=\"-794.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">features.8.weight</text>\n",
       "<text text-anchor=\"middle\" x=\"603.4961\" y=\"-782.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\"> (256, 384, 3, 3)</text>\n",
       "</g>\n",
       "<!-- 4810468320&#45;&gt;4810468152 -->\n",
       "<g id=\"edge34\" class=\"edge\">\n",
       "<title>4810468320&#45;&gt;4810468152</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M603.4961,-775.7102C603.4961,-768.0144 603.4961,-758.744 603.4961,-750.5691\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"606.9962,-750.3512 603.4961,-740.3512 599.9962,-750.3513 606.9962,-750.3512\"/>\n",
       "</g>\n",
       "<!-- 4810468376 -->\n",
       "<g id=\"node36\" class=\"node\">\n",
       "<title>4810468376</title>\n",
       "<polygon fill=\"#add8e6\" stroke=\"#000000\" points=\"756.3013,-808 670.6909,-808 670.6909,-776 756.3013,-776 756.3013,-808\"/>\n",
       "<text text-anchor=\"middle\" x=\"713.4961\" y=\"-794.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">features.8.bias</text>\n",
       "<text text-anchor=\"middle\" x=\"713.4961\" y=\"-782.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\"> (256)</text>\n",
       "</g>\n",
       "<!-- 4810468376&#45;&gt;4810468152 -->\n",
       "<g id=\"edge35\" class=\"edge\">\n",
       "<title>4810468376&#45;&gt;4810468152</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M684.882,-775.8721C668.0666,-766.3943 646.9387,-754.4858 630.4242,-745.1777\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"631.7314,-741.8968 621.3013,-740.0357 628.2943,-747.9949 631.7314,-741.8968\"/>\n",
       "</g>\n",
       "<!-- 4810467984 -->\n",
       "<g id=\"node37\" class=\"node\">\n",
       "<title>4810467984</title>\n",
       "<polygon fill=\"#add8e6\" stroke=\"#000000\" points=\"785.6315,-684 681.3607,-684 681.3607,-652 785.6315,-652 785.6315,-684\"/>\n",
       "<text text-anchor=\"middle\" x=\"733.4961\" y=\"-670.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">features.10.weight</text>\n",
       "<text text-anchor=\"middle\" x=\"733.4961\" y=\"-658.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\"> (256, 256, 3, 3)</text>\n",
       "</g>\n",
       "<!-- 4810467984&#45;&gt;4810467816 -->\n",
       "<g id=\"edge36\" class=\"edge\">\n",
       "<title>4810467984&#45;&gt;4810467816</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M733.4961,-651.7102C733.4961,-644.0144 733.4961,-634.744 733.4961,-626.5691\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"736.9962,-626.3512 733.4961,-616.3512 729.9962,-626.3513 736.9962,-626.3512\"/>\n",
       "</g>\n",
       "<!-- 4810468040 -->\n",
       "<g id=\"node38\" class=\"node\">\n",
       "<title>4810468040</title>\n",
       "<polygon fill=\"#add8e6\" stroke=\"#000000\" points=\"895.3013,-684 803.6909,-684 803.6909,-652 895.3013,-652 895.3013,-684\"/>\n",
       "<text text-anchor=\"middle\" x=\"849.4961\" y=\"-670.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">features.10.bias</text>\n",
       "<text text-anchor=\"middle\" x=\"849.4961\" y=\"-658.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\"> (256)</text>\n",
       "</g>\n",
       "<!-- 4810468040&#45;&gt;4810467816 -->\n",
       "<g id=\"edge37\" class=\"edge\">\n",
       "<title>4810468040&#45;&gt;4810467816</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M819.3213,-651.8721C801.4229,-642.3057 778.8914,-630.263 761.406,-620.9174\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"762.7417,-617.6627 752.2725,-616.0357 759.442,-623.8363 762.7417,-617.6627\"/>\n",
       "</g>\n",
       "<!-- 4810467312 -->\n",
       "<g id=\"node39\" class=\"node\">\n",
       "<title>4810467312</title>\n",
       "<polygon fill=\"#d3d3d3\" stroke=\"#000000\" points=\"881.4707,-380 809.5215,-380 809.5215,-360 881.4707,-360 881.4707,-380\"/>\n",
       "<text text-anchor=\"middle\" x=\"845.4961\" y=\"-366.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">TBackward</text>\n",
       "</g>\n",
       "<!-- 4810467312&#45;&gt;4810467144 -->\n",
       "<g id=\"edge38\" class=\"edge\">\n",
       "<title>4810467312&#45;&gt;4810467144</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M825.8303,-359.9883C808.73,-351.2827 783.727,-338.5539 764.3587,-328.6937\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"765.8621,-325.5316 755.3625,-324.1138 762.6862,-331.7698 765.8621,-325.5316\"/>\n",
       "</g>\n",
       "<!-- 4810467536 -->\n",
       "<g id=\"node40\" class=\"node\">\n",
       "<title>4810467536</title>\n",
       "<polygon fill=\"#add8e6\" stroke=\"#000000\" points=\"898.6568,-448 796.3354,-448 796.3354,-416 898.6568,-416 898.6568,-448\"/>\n",
       "<text text-anchor=\"middle\" x=\"847.4961\" y=\"-434.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">classifier.1.weight</text>\n",
       "<text text-anchor=\"middle\" x=\"847.4961\" y=\"-422.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\"> (4096, 9216)</text>\n",
       "</g>\n",
       "<!-- 4810467536&#45;&gt;4810467312 -->\n",
       "<g id=\"edge39\" class=\"edge\">\n",
       "<title>4810467536&#45;&gt;4810467312</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M846.9706,-415.7102C846.7224,-408.0144 846.4233,-398.744 846.1596,-390.5691\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"849.6507,-390.2331 845.83,-380.3512 842.6543,-390.4589 849.6507,-390.2331\"/>\n",
       "</g>\n",
       "<!-- 4810466864 -->\n",
       "<g id=\"node41\" class=\"node\">\n",
       "<title>4810466864</title>\n",
       "<polygon fill=\"#d3d3d3\" stroke=\"#000000\" points=\"888.4707,-200 816.5215,-200 816.5215,-180 888.4707,-180 888.4707,-200\"/>\n",
       "<text text-anchor=\"middle\" x=\"852.4961\" y=\"-186.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">TBackward</text>\n",
       "</g>\n",
       "<!-- 4810466864&#45;&gt;4810466696 -->\n",
       "<g id=\"edge40\" class=\"edge\">\n",
       "<title>4810466864&#45;&gt;4810466696</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M833.4376,-179.9006C814.4853,-169.8576 785.1951,-154.3362 763.5798,-142.882\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"765.0213,-139.6848 754.5464,-138.095 761.7436,-145.8701 765.0213,-139.6848\"/>\n",
       "</g>\n",
       "<!-- 4810467088 -->\n",
       "<g id=\"node42\" class=\"node\">\n",
       "<title>4810467088</title>\n",
       "<polygon fill=\"#add8e6\" stroke=\"#000000\" points=\"915.6568,-268 813.3354,-268 813.3354,-236 915.6568,-236 915.6568,-268\"/>\n",
       "<text text-anchor=\"middle\" x=\"864.4961\" y=\"-254.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">classifier.4.weight</text>\n",
       "<text text-anchor=\"middle\" x=\"864.4961\" y=\"-242.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\"> (4096, 4096)</text>\n",
       "</g>\n",
       "<!-- 4810467088&#45;&gt;4810466864 -->\n",
       "<g id=\"edge41\" class=\"edge\">\n",
       "<title>4810467088&#45;&gt;4810466864</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M861.3432,-235.7102C859.8368,-227.9269 858.0186,-218.5331 856.4233,-210.2908\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"859.8361,-209.5039 854.4996,-200.3512 852.9636,-210.8341 859.8361,-209.5039\"/>\n",
       "</g>\n",
       "<!-- 4539553496 -->\n",
       "<g id=\"node43\" class=\"node\">\n",
       "<title>4539553496</title>\n",
       "<polygon fill=\"#d3d3d3\" stroke=\"#000000\" points=\"894.4707,-76 822.5215,-76 822.5215,-56 894.4707,-56 894.4707,-76\"/>\n",
       "<text text-anchor=\"middle\" x=\"858.4961\" y=\"-62.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">TBackward</text>\n",
       "</g>\n",
       "<!-- 4539553496&#45;&gt;4539553944 -->\n",
       "<g id=\"edge42\" class=\"edge\">\n",
       "<title>4539553496&#45;&gt;4539553944</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M837.2213,-55.9883C818.5537,-47.2036 791.1807,-34.3222 770.1502,-24.4255\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"771.5265,-21.205 760.9879,-20.1138 768.5458,-27.5387 771.5265,-21.205\"/>\n",
       "</g>\n",
       "<!-- 4810466752 -->\n",
       "<g id=\"node44\" class=\"node\">\n",
       "<title>4810466752</title>\n",
       "<polygon fill=\"#add8e6\" stroke=\"#000000\" points=\"914.6568,-144 812.3354,-144 812.3354,-112 914.6568,-112 914.6568,-144\"/>\n",
       "<text text-anchor=\"middle\" x=\"863.4961\" y=\"-130.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">classifier.6.weight</text>\n",
       "<text text-anchor=\"middle\" x=\"863.4961\" y=\"-118.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\"> (1000, 4096)</text>\n",
       "</g>\n",
       "<!-- 4810466752&#45;&gt;4539553496 -->\n",
       "<g id=\"edge43\" class=\"edge\">\n",
       "<title>4810466752&#45;&gt;4539553496</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M862.1824,-111.7102C861.5618,-104.0144 860.8142,-94.744 860.1549,-86.5691\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"863.6235,-86.0375 859.3309,-76.3512 856.6461,-86.6002 863.6235,-86.0375\"/>\n",
       "</g>\n",
       "</g>\n",
       "</svg>\n"
      ],
      "text/plain": [
       "<graphviz.dot.Digraph at 0x112396080>"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from torchvision.models import AlexNet\n",
    "\n",
    "model = AlexNet()\n",
    "\n",
    "x = torch.randn(1, 3, 227, 227).requires_grad_(True)\n",
    "y = model(x)\n",
    "make_dot(y, params=dict(list(model.named_parameters()) + [('x', x)]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/svg+xml": [
       "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>\n",
       "<!DOCTYPE svg PUBLIC \"-//W3C//DTD SVG 1.1//EN\"\n",
       " \"http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd\">\n",
       "<!-- Generated by graphviz version 2.40.1 (20161225.0304)\n",
       " -->\n",
       "<!-- Title: %3 Pages: 1 -->\n",
       "<svg width=\"554pt\" height=\"972pt\"\n",
       " viewBox=\"0.00 0.00 554.09 972.00\" xmlns=\"http://www.w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\">\n",
       "<g id=\"graph0\" class=\"graph\" transform=\"scale(.4204 .4204) rotate(0) translate(4 2308)\">\n",
       "<title>%3</title>\n",
       "<polygon fill=\"#ffffff\" stroke=\"transparent\" points=\"-4,4 -4,-2308 1313.9619,-2308 1313.9619,4 -4,4\"/>\n",
       "<!-- AlexNet/Sequential[features]/Conv2d[0]/17 -->\n",
       "<g id=\"node1\" class=\"node\">\n",
       "<title>AlexNet/Sequential[features]/Conv2d[0]/17</title>\n",
       "<polygon fill=\"#d3d3d3\" stroke=\"#000000\" points=\"1178.7731,-2212 1066.5296,-2212 1066.5296,-2156 1178.7731,-2156 1178.7731,-2212\"/>\n",
       "<text text-anchor=\"middle\" x=\"1122.6514\" y=\"-2198.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">AlexNet</text>\n",
       "<text text-anchor=\"middle\" x=\"1122.6514\" y=\"-2186.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">Sequential[features]</text>\n",
       "<text text-anchor=\"middle\" x=\"1122.6514\" y=\"-2174.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">Conv2d[0]</text>\n",
       "<text text-anchor=\"middle\" x=\"1122.6514\" y=\"-2162.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">17</text>\n",
       "</g>\n",
       "<!-- AlexNet/Sequential[features]/ReLU[1]/18 -->\n",
       "<g id=\"node5\" class=\"node\">\n",
       "<title>AlexNet/Sequential[features]/ReLU[1]/18</title>\n",
       "<polygon fill=\"#d3d3d3\" stroke=\"#000000\" points=\"1178.7731,-2120 1066.5296,-2120 1066.5296,-2064 1178.7731,-2064 1178.7731,-2120\"/>\n",
       "<text text-anchor=\"middle\" x=\"1122.6514\" y=\"-2106.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">AlexNet</text>\n",
       "<text text-anchor=\"middle\" x=\"1122.6514\" y=\"-2094.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">Sequential[features]</text>\n",
       "<text text-anchor=\"middle\" x=\"1122.6514\" y=\"-2082.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">ReLU[1]</text>\n",
       "<text text-anchor=\"middle\" x=\"1122.6514\" y=\"-2070.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">18</text>\n",
       "</g>\n",
       "<!-- AlexNet/Sequential[features]/Conv2d[0]/17&#45;&gt;AlexNet/Sequential[features]/ReLU[1]/18 -->\n",
       "<g id=\"edge4\" class=\"edge\">\n",
       "<title>AlexNet/Sequential[features]/Conv2d[0]/17&#45;&gt;AlexNet/Sequential[features]/ReLU[1]/18</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M1122.6514,-2155.8971C1122.6514,-2147.8575 1122.6514,-2138.9539 1122.6514,-2130.4096\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1126.1515,-2130.2478 1122.6514,-2120.2479 1119.1515,-2130.2479 1126.1515,-2130.2478\"/>\n",
       "</g>\n",
       "<!-- input/0 -->\n",
       "<g id=\"node2\" class=\"node\">\n",
       "<title>input/0</title>\n",
       "<polygon fill=\"#d3d3d3\" stroke=\"#000000\" points=\"1048.6514,-2292 994.6514,-2292 994.6514,-2260 1048.6514,-2260 1048.6514,-2292\"/>\n",
       "<text text-anchor=\"middle\" x=\"1021.6514\" y=\"-2278.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">input</text>\n",
       "<text text-anchor=\"middle\" x=\"1021.6514\" y=\"-2266.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">0</text>\n",
       "</g>\n",
       "<!-- input/0&#45;&gt;AlexNet/Sequential[features]/Conv2d[0]/17 -->\n",
       "<g id=\"edge1\" class=\"edge\">\n",
       "<title>input/0&#45;&gt;AlexNet/Sequential[features]/Conv2d[0]/17</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M1039.2442,-2259.9748C1051.6782,-2248.6489 1068.8263,-2233.0288 1084.3416,-2218.896\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1086.7634,-2221.4245 1091.7992,-2212.1029 1082.0495,-2216.2495 1086.7634,-2221.4245\"/>\n",
       "</g>\n",
       "<!-- AlexNet/Sequential[features]/Conv2d[0]/1 -->\n",
       "<g id=\"node3\" class=\"node\">\n",
       "<title>AlexNet/Sequential[features]/Conv2d[0]/1</title>\n",
       "<polygon fill=\"#d3d3d3\" stroke=\"#000000\" points=\"1178.7731,-2304 1066.5296,-2304 1066.5296,-2248 1178.7731,-2248 1178.7731,-2304\"/>\n",
       "<text text-anchor=\"middle\" x=\"1122.6514\" y=\"-2290.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">AlexNet</text>\n",
       "<text text-anchor=\"middle\" x=\"1122.6514\" y=\"-2278.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">Sequential[features]</text>\n",
       "<text text-anchor=\"middle\" x=\"1122.6514\" y=\"-2266.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">Conv2d[0]</text>\n",
       "<text text-anchor=\"middle\" x=\"1122.6514\" y=\"-2254.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">1</text>\n",
       "</g>\n",
       "<!-- AlexNet/Sequential[features]/Conv2d[0]/1&#45;&gt;AlexNet/Sequential[features]/Conv2d[0]/17 -->\n",
       "<g id=\"edge2\" class=\"edge\">\n",
       "<title>AlexNet/Sequential[features]/Conv2d[0]/1&#45;&gt;AlexNet/Sequential[features]/Conv2d[0]/17</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M1122.6514,-2247.8971C1122.6514,-2239.8575 1122.6514,-2230.9539 1122.6514,-2222.4096\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1126.1515,-2222.2478 1122.6514,-2212.2479 1119.1515,-2222.2479 1126.1515,-2222.2478\"/>\n",
       "</g>\n",
       "<!-- AlexNet/Sequential[features]/Conv2d[0]/2 -->\n",
       "<g id=\"node4\" class=\"node\">\n",
       "<title>AlexNet/Sequential[features]/Conv2d[0]/2</title>\n",
       "<polygon fill=\"#d3d3d3\" stroke=\"#000000\" points=\"1309.7731,-2304 1197.5296,-2304 1197.5296,-2248 1309.7731,-2248 1309.7731,-2304\"/>\n",
       "<text text-anchor=\"middle\" x=\"1253.6514\" y=\"-2290.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">AlexNet</text>\n",
       "<text text-anchor=\"middle\" x=\"1253.6514\" y=\"-2278.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">Sequential[features]</text>\n",
       "<text text-anchor=\"middle\" x=\"1253.6514\" y=\"-2266.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">Conv2d[0]</text>\n",
       "<text text-anchor=\"middle\" x=\"1253.6514\" y=\"-2254.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">2</text>\n",
       "</g>\n",
       "<!-- AlexNet/Sequential[features]/Conv2d[0]/2&#45;&gt;AlexNet/Sequential[features]/Conv2d[0]/17 -->\n",
       "<g id=\"edge3\" class=\"edge\">\n",
       "<title>AlexNet/Sequential[features]/Conv2d[0]/2&#45;&gt;AlexNet/Sequential[features]/Conv2d[0]/17</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M1213.6352,-2247.8971C1200.3007,-2238.5323 1185.2967,-2227.9952 1171.3765,-2218.2192\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1173.0689,-2215.1309 1162.8739,-2212.2479 1169.0458,-2220.8593 1173.0689,-2215.1309\"/>\n",
       "</g>\n",
       "<!-- AlexNet/Sequential[features]/MaxPool2d[2]/19 -->\n",
       "<g id=\"node6\" class=\"node\">\n",
       "<title>AlexNet/Sequential[features]/MaxPool2d[2]/19</title>\n",
       "<polygon fill=\"#d3d3d3\" stroke=\"#000000\" points=\"1178.7731,-2028 1066.5296,-2028 1066.5296,-1972 1178.7731,-1972 1178.7731,-2028\"/>\n",
       "<text text-anchor=\"middle\" x=\"1122.6514\" y=\"-2014.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">AlexNet</text>\n",
       "<text text-anchor=\"middle\" x=\"1122.6514\" y=\"-2002.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">Sequential[features]</text>\n",
       "<text text-anchor=\"middle\" x=\"1122.6514\" y=\"-1990.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">MaxPool2d[2]</text>\n",
       "<text text-anchor=\"middle\" x=\"1122.6514\" y=\"-1978.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">19</text>\n",
       "</g>\n",
       "<!-- AlexNet/Sequential[features]/ReLU[1]/18&#45;&gt;AlexNet/Sequential[features]/MaxPool2d[2]/19 -->\n",
       "<g id=\"edge5\" class=\"edge\">\n",
       "<title>AlexNet/Sequential[features]/ReLU[1]/18&#45;&gt;AlexNet/Sequential[features]/MaxPool2d[2]/19</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M1122.6514,-2063.8971C1122.6514,-2055.8575 1122.6514,-2046.9539 1122.6514,-2038.4096\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1126.1515,-2038.2478 1122.6514,-2028.2479 1119.1515,-2038.2479 1126.1515,-2038.2478\"/>\n",
       "</g>\n",
       "<!-- AlexNet/Sequential[features]/Conv2d[3]/20 -->\n",
       "<g id=\"node7\" class=\"node\">\n",
       "<title>AlexNet/Sequential[features]/Conv2d[3]/20</title>\n",
       "<polygon fill=\"#d3d3d3\" stroke=\"#000000\" points=\"1047.7731,-1936 935.5296,-1936 935.5296,-1880 1047.7731,-1880 1047.7731,-1936\"/>\n",
       "<text text-anchor=\"middle\" x=\"991.6514\" y=\"-1922.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">AlexNet</text>\n",
       "<text text-anchor=\"middle\" x=\"991.6514\" y=\"-1910.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">Sequential[features]</text>\n",
       "<text text-anchor=\"middle\" x=\"991.6514\" y=\"-1898.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">Conv2d[3]</text>\n",
       "<text text-anchor=\"middle\" x=\"991.6514\" y=\"-1886.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">20</text>\n",
       "</g>\n",
       "<!-- AlexNet/Sequential[features]/MaxPool2d[2]/19&#45;&gt;AlexNet/Sequential[features]/Conv2d[3]/20 -->\n",
       "<g id=\"edge6\" class=\"edge\">\n",
       "<title>AlexNet/Sequential[features]/MaxPool2d[2]/19&#45;&gt;AlexNet/Sequential[features]/Conv2d[3]/20</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M1082.6352,-1971.8971C1069.3007,-1962.5323 1054.2967,-1951.9952 1040.3765,-1942.2192\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1042.0689,-1939.1309 1031.8739,-1936.2479 1038.0458,-1944.8593 1042.0689,-1939.1309\"/>\n",
       "</g>\n",
       "<!-- AlexNet/Sequential[features]/ReLU[4]/21 -->\n",
       "<g id=\"node10\" class=\"node\">\n",
       "<title>AlexNet/Sequential[features]/ReLU[4]/21</title>\n",
       "<polygon fill=\"#d3d3d3\" stroke=\"#000000\" points=\"1047.7731,-1844 935.5296,-1844 935.5296,-1788 1047.7731,-1788 1047.7731,-1844\"/>\n",
       "<text text-anchor=\"middle\" x=\"991.6514\" y=\"-1830.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">AlexNet</text>\n",
       "<text text-anchor=\"middle\" x=\"991.6514\" y=\"-1818.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">Sequential[features]</text>\n",
       "<text text-anchor=\"middle\" x=\"991.6514\" y=\"-1806.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">ReLU[4]</text>\n",
       "<text text-anchor=\"middle\" x=\"991.6514\" y=\"-1794.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">21</text>\n",
       "</g>\n",
       "<!-- AlexNet/Sequential[features]/Conv2d[3]/20&#45;&gt;AlexNet/Sequential[features]/ReLU[4]/21 -->\n",
       "<g id=\"edge9\" class=\"edge\">\n",
       "<title>AlexNet/Sequential[features]/Conv2d[3]/20&#45;&gt;AlexNet/Sequential[features]/ReLU[4]/21</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M991.6514,-1879.8971C991.6514,-1871.8575 991.6514,-1862.9539 991.6514,-1854.4096\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"995.1515,-1854.2478 991.6514,-1844.2479 988.1515,-1854.2479 995.1515,-1854.2478\"/>\n",
       "</g>\n",
       "<!-- AlexNet/Sequential[features]/Conv2d[3]/3 -->\n",
       "<g id=\"node8\" class=\"node\">\n",
       "<title>AlexNet/Sequential[features]/Conv2d[3]/3</title>\n",
       "<polygon fill=\"#d3d3d3\" stroke=\"#000000\" points=\"916.7731,-2028 804.5296,-2028 804.5296,-1972 916.7731,-1972 916.7731,-2028\"/>\n",
       "<text text-anchor=\"middle\" x=\"860.6514\" y=\"-2014.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">AlexNet</text>\n",
       "<text text-anchor=\"middle\" x=\"860.6514\" y=\"-2002.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">Sequential[features]</text>\n",
       "<text text-anchor=\"middle\" x=\"860.6514\" y=\"-1990.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">Conv2d[3]</text>\n",
       "<text text-anchor=\"middle\" x=\"860.6514\" y=\"-1978.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">3</text>\n",
       "</g>\n",
       "<!-- AlexNet/Sequential[features]/Conv2d[3]/3&#45;&gt;AlexNet/Sequential[features]/Conv2d[3]/20 -->\n",
       "<g id=\"edge7\" class=\"edge\">\n",
       "<title>AlexNet/Sequential[features]/Conv2d[3]/3&#45;&gt;AlexNet/Sequential[features]/Conv2d[3]/20</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M900.6675,-1971.8971C914.0021,-1962.5323 929.0061,-1951.9952 942.9263,-1942.2192\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"945.2569,-1944.8593 951.4289,-1936.2479 941.2338,-1939.1309 945.2569,-1944.8593\"/>\n",
       "</g>\n",
       "<!-- AlexNet/Sequential[features]/Conv2d[3]/4 -->\n",
       "<g id=\"node9\" class=\"node\">\n",
       "<title>AlexNet/Sequential[features]/Conv2d[3]/4</title>\n",
       "<polygon fill=\"#d3d3d3\" stroke=\"#000000\" points=\"1047.7731,-2028 935.5296,-2028 935.5296,-1972 1047.7731,-1972 1047.7731,-2028\"/>\n",
       "<text text-anchor=\"middle\" x=\"991.6514\" y=\"-2014.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">AlexNet</text>\n",
       "<text text-anchor=\"middle\" x=\"991.6514\" y=\"-2002.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">Sequential[features]</text>\n",
       "<text text-anchor=\"middle\" x=\"991.6514\" y=\"-1990.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">Conv2d[3]</text>\n",
       "<text text-anchor=\"middle\" x=\"991.6514\" y=\"-1978.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">4</text>\n",
       "</g>\n",
       "<!-- AlexNet/Sequential[features]/Conv2d[3]/4&#45;&gt;AlexNet/Sequential[features]/Conv2d[3]/20 -->\n",
       "<g id=\"edge8\" class=\"edge\">\n",
       "<title>AlexNet/Sequential[features]/Conv2d[3]/4&#45;&gt;AlexNet/Sequential[features]/Conv2d[3]/20</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M991.6514,-1971.8971C991.6514,-1963.8575 991.6514,-1954.9539 991.6514,-1946.4096\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"995.1515,-1946.2478 991.6514,-1936.2479 988.1515,-1946.2479 995.1515,-1946.2478\"/>\n",
       "</g>\n",
       "<!-- AlexNet/Sequential[features]/MaxPool2d[5]/22 -->\n",
       "<g id=\"node11\" class=\"node\">\n",
       "<title>AlexNet/Sequential[features]/MaxPool2d[5]/22</title>\n",
       "<polygon fill=\"#d3d3d3\" stroke=\"#000000\" points=\"1047.7731,-1752 935.5296,-1752 935.5296,-1696 1047.7731,-1696 1047.7731,-1752\"/>\n",
       "<text text-anchor=\"middle\" x=\"991.6514\" y=\"-1738.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">AlexNet</text>\n",
       "<text text-anchor=\"middle\" x=\"991.6514\" y=\"-1726.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">Sequential[features]</text>\n",
       "<text text-anchor=\"middle\" x=\"991.6514\" y=\"-1714.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">MaxPool2d[5]</text>\n",
       "<text text-anchor=\"middle\" x=\"991.6514\" y=\"-1702.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">22</text>\n",
       "</g>\n",
       "<!-- AlexNet/Sequential[features]/ReLU[4]/21&#45;&gt;AlexNet/Sequential[features]/MaxPool2d[5]/22 -->\n",
       "<g id=\"edge10\" class=\"edge\">\n",
       "<title>AlexNet/Sequential[features]/ReLU[4]/21&#45;&gt;AlexNet/Sequential[features]/MaxPool2d[5]/22</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M991.6514,-1787.8971C991.6514,-1779.8575 991.6514,-1770.9539 991.6514,-1762.4096\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"995.1515,-1762.2478 991.6514,-1752.2479 988.1515,-1762.2479 995.1515,-1762.2478\"/>\n",
       "</g>\n",
       "<!-- AlexNet/Sequential[features]/Conv2d[6]/23 -->\n",
       "<g id=\"node12\" class=\"node\">\n",
       "<title>AlexNet/Sequential[features]/Conv2d[6]/23</title>\n",
       "<polygon fill=\"#d3d3d3\" stroke=\"#000000\" points=\"916.7731,-1660 804.5296,-1660 804.5296,-1604 916.7731,-1604 916.7731,-1660\"/>\n",
       "<text text-anchor=\"middle\" x=\"860.6514\" y=\"-1646.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">AlexNet</text>\n",
       "<text text-anchor=\"middle\" x=\"860.6514\" y=\"-1634.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">Sequential[features]</text>\n",
       "<text text-anchor=\"middle\" x=\"860.6514\" y=\"-1622.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">Conv2d[6]</text>\n",
       "<text text-anchor=\"middle\" x=\"860.6514\" y=\"-1610.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">23</text>\n",
       "</g>\n",
       "<!-- AlexNet/Sequential[features]/MaxPool2d[5]/22&#45;&gt;AlexNet/Sequential[features]/Conv2d[6]/23 -->\n",
       "<g id=\"edge11\" class=\"edge\">\n",
       "<title>AlexNet/Sequential[features]/MaxPool2d[5]/22&#45;&gt;AlexNet/Sequential[features]/Conv2d[6]/23</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M951.6352,-1695.8971C938.3007,-1686.5323 923.2967,-1675.9952 909.3765,-1666.2192\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"911.0689,-1663.1309 900.8739,-1660.2479 907.0458,-1668.8593 911.0689,-1663.1309\"/>\n",
       "</g>\n",
       "<!-- AlexNet/Sequential[features]/ReLU[7]/24 -->\n",
       "<g id=\"node15\" class=\"node\">\n",
       "<title>AlexNet/Sequential[features]/ReLU[7]/24</title>\n",
       "<polygon fill=\"#d3d3d3\" stroke=\"#000000\" points=\"916.7731,-1568 804.5296,-1568 804.5296,-1512 916.7731,-1512 916.7731,-1568\"/>\n",
       "<text text-anchor=\"middle\" x=\"860.6514\" y=\"-1554.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">AlexNet</text>\n",
       "<text text-anchor=\"middle\" x=\"860.6514\" y=\"-1542.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">Sequential[features]</text>\n",
       "<text text-anchor=\"middle\" x=\"860.6514\" y=\"-1530.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">ReLU[7]</text>\n",
       "<text text-anchor=\"middle\" x=\"860.6514\" y=\"-1518.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">24</text>\n",
       "</g>\n",
       "<!-- AlexNet/Sequential[features]/Conv2d[6]/23&#45;&gt;AlexNet/Sequential[features]/ReLU[7]/24 -->\n",
       "<g id=\"edge14\" class=\"edge\">\n",
       "<title>AlexNet/Sequential[features]/Conv2d[6]/23&#45;&gt;AlexNet/Sequential[features]/ReLU[7]/24</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M860.6514,-1603.8971C860.6514,-1595.8575 860.6514,-1586.9539 860.6514,-1578.4096\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"864.1515,-1578.2478 860.6514,-1568.2479 857.1515,-1578.2479 864.1515,-1578.2478\"/>\n",
       "</g>\n",
       "<!-- AlexNet/Sequential[features]/Conv2d[6]/5 -->\n",
       "<g id=\"node13\" class=\"node\">\n",
       "<title>AlexNet/Sequential[features]/Conv2d[6]/5</title>\n",
       "<polygon fill=\"#d3d3d3\" stroke=\"#000000\" points=\"785.7731,-1752 673.5296,-1752 673.5296,-1696 785.7731,-1696 785.7731,-1752\"/>\n",
       "<text text-anchor=\"middle\" x=\"729.6514\" y=\"-1738.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">AlexNet</text>\n",
       "<text text-anchor=\"middle\" x=\"729.6514\" y=\"-1726.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">Sequential[features]</text>\n",
       "<text text-anchor=\"middle\" x=\"729.6514\" y=\"-1714.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">Conv2d[6]</text>\n",
       "<text text-anchor=\"middle\" x=\"729.6514\" y=\"-1702.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">5</text>\n",
       "</g>\n",
       "<!-- AlexNet/Sequential[features]/Conv2d[6]/5&#45;&gt;AlexNet/Sequential[features]/Conv2d[6]/23 -->\n",
       "<g id=\"edge12\" class=\"edge\">\n",
       "<title>AlexNet/Sequential[features]/Conv2d[6]/5&#45;&gt;AlexNet/Sequential[features]/Conv2d[6]/23</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M769.6675,-1695.8971C783.0021,-1686.5323 798.0061,-1675.9952 811.9263,-1666.2192\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"814.2569,-1668.8593 820.4289,-1660.2479 810.2338,-1663.1309 814.2569,-1668.8593\"/>\n",
       "</g>\n",
       "<!-- AlexNet/Sequential[features]/Conv2d[6]/6 -->\n",
       "<g id=\"node14\" class=\"node\">\n",
       "<title>AlexNet/Sequential[features]/Conv2d[6]/6</title>\n",
       "<polygon fill=\"#d3d3d3\" stroke=\"#000000\" points=\"916.7731,-1752 804.5296,-1752 804.5296,-1696 916.7731,-1696 916.7731,-1752\"/>\n",
       "<text text-anchor=\"middle\" x=\"860.6514\" y=\"-1738.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">AlexNet</text>\n",
       "<text text-anchor=\"middle\" x=\"860.6514\" y=\"-1726.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">Sequential[features]</text>\n",
       "<text text-anchor=\"middle\" x=\"860.6514\" y=\"-1714.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">Conv2d[6]</text>\n",
       "<text text-anchor=\"middle\" x=\"860.6514\" y=\"-1702.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">6</text>\n",
       "</g>\n",
       "<!-- AlexNet/Sequential[features]/Conv2d[6]/6&#45;&gt;AlexNet/Sequential[features]/Conv2d[6]/23 -->\n",
       "<g id=\"edge13\" class=\"edge\">\n",
       "<title>AlexNet/Sequential[features]/Conv2d[6]/6&#45;&gt;AlexNet/Sequential[features]/Conv2d[6]/23</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M860.6514,-1695.8971C860.6514,-1687.8575 860.6514,-1678.9539 860.6514,-1670.4096\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"864.1515,-1670.2478 860.6514,-1660.2479 857.1515,-1670.2479 864.1515,-1670.2478\"/>\n",
       "</g>\n",
       "<!-- AlexNet/Sequential[features]/Conv2d[8]/25 -->\n",
       "<g id=\"node16\" class=\"node\">\n",
       "<title>AlexNet/Sequential[features]/Conv2d[8]/25</title>\n",
       "<polygon fill=\"#d3d3d3\" stroke=\"#000000\" points=\"785.7731,-1476 673.5296,-1476 673.5296,-1420 785.7731,-1420 785.7731,-1476\"/>\n",
       "<text text-anchor=\"middle\" x=\"729.6514\" y=\"-1462.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">AlexNet</text>\n",
       "<text text-anchor=\"middle\" x=\"729.6514\" y=\"-1450.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">Sequential[features]</text>\n",
       "<text text-anchor=\"middle\" x=\"729.6514\" y=\"-1438.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">Conv2d[8]</text>\n",
       "<text text-anchor=\"middle\" x=\"729.6514\" y=\"-1426.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">25</text>\n",
       "</g>\n",
       "<!-- AlexNet/Sequential[features]/ReLU[7]/24&#45;&gt;AlexNet/Sequential[features]/Conv2d[8]/25 -->\n",
       "<g id=\"edge15\" class=\"edge\">\n",
       "<title>AlexNet/Sequential[features]/ReLU[7]/24&#45;&gt;AlexNet/Sequential[features]/Conv2d[8]/25</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M820.6352,-1511.8971C807.3007,-1502.5323 792.2967,-1491.9952 778.3765,-1482.2192\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"780.0689,-1479.1309 769.8739,-1476.2479 776.0458,-1484.8593 780.0689,-1479.1309\"/>\n",
       "</g>\n",
       "<!-- AlexNet/Sequential[features]/ReLU[9]/26 -->\n",
       "<g id=\"node19\" class=\"node\">\n",
       "<title>AlexNet/Sequential[features]/ReLU[9]/26</title>\n",
       "<polygon fill=\"#d3d3d3\" stroke=\"#000000\" points=\"785.7731,-1384 673.5296,-1384 673.5296,-1328 785.7731,-1328 785.7731,-1384\"/>\n",
       "<text text-anchor=\"middle\" x=\"729.6514\" y=\"-1370.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">AlexNet</text>\n",
       "<text text-anchor=\"middle\" x=\"729.6514\" y=\"-1358.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">Sequential[features]</text>\n",
       "<text text-anchor=\"middle\" x=\"729.6514\" y=\"-1346.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">ReLU[9]</text>\n",
       "<text text-anchor=\"middle\" x=\"729.6514\" y=\"-1334.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">26</text>\n",
       "</g>\n",
       "<!-- AlexNet/Sequential[features]/Conv2d[8]/25&#45;&gt;AlexNet/Sequential[features]/ReLU[9]/26 -->\n",
       "<g id=\"edge18\" class=\"edge\">\n",
       "<title>AlexNet/Sequential[features]/Conv2d[8]/25&#45;&gt;AlexNet/Sequential[features]/ReLU[9]/26</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M729.6514,-1419.8971C729.6514,-1411.8575 729.6514,-1402.9539 729.6514,-1394.4096\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"733.1515,-1394.2478 729.6514,-1384.2479 726.1515,-1394.2479 733.1515,-1394.2478\"/>\n",
       "</g>\n",
       "<!-- AlexNet/Sequential[features]/Conv2d[8]/7 -->\n",
       "<g id=\"node17\" class=\"node\">\n",
       "<title>AlexNet/Sequential[features]/Conv2d[8]/7</title>\n",
       "<polygon fill=\"#d3d3d3\" stroke=\"#000000\" points=\"654.7731,-1568 542.5296,-1568 542.5296,-1512 654.7731,-1512 654.7731,-1568\"/>\n",
       "<text text-anchor=\"middle\" x=\"598.6514\" y=\"-1554.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">AlexNet</text>\n",
       "<text text-anchor=\"middle\" x=\"598.6514\" y=\"-1542.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">Sequential[features]</text>\n",
       "<text text-anchor=\"middle\" x=\"598.6514\" y=\"-1530.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">Conv2d[8]</text>\n",
       "<text text-anchor=\"middle\" x=\"598.6514\" y=\"-1518.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">7</text>\n",
       "</g>\n",
       "<!-- AlexNet/Sequential[features]/Conv2d[8]/7&#45;&gt;AlexNet/Sequential[features]/Conv2d[8]/25 -->\n",
       "<g id=\"edge16\" class=\"edge\">\n",
       "<title>AlexNet/Sequential[features]/Conv2d[8]/7&#45;&gt;AlexNet/Sequential[features]/Conv2d[8]/25</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M638.6675,-1511.8971C652.0021,-1502.5323 667.0061,-1491.9952 680.9263,-1482.2192\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"683.2569,-1484.8593 689.4289,-1476.2479 679.2338,-1479.1309 683.2569,-1484.8593\"/>\n",
       "</g>\n",
       "<!-- AlexNet/Sequential[features]/Conv2d[8]/8 -->\n",
       "<g id=\"node18\" class=\"node\">\n",
       "<title>AlexNet/Sequential[features]/Conv2d[8]/8</title>\n",
       "<polygon fill=\"#d3d3d3\" stroke=\"#000000\" points=\"785.7731,-1568 673.5296,-1568 673.5296,-1512 785.7731,-1512 785.7731,-1568\"/>\n",
       "<text text-anchor=\"middle\" x=\"729.6514\" y=\"-1554.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">AlexNet</text>\n",
       "<text text-anchor=\"middle\" x=\"729.6514\" y=\"-1542.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">Sequential[features]</text>\n",
       "<text text-anchor=\"middle\" x=\"729.6514\" y=\"-1530.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">Conv2d[8]</text>\n",
       "<text text-anchor=\"middle\" x=\"729.6514\" y=\"-1518.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">8</text>\n",
       "</g>\n",
       "<!-- AlexNet/Sequential[features]/Conv2d[8]/8&#45;&gt;AlexNet/Sequential[features]/Conv2d[8]/25 -->\n",
       "<g id=\"edge17\" class=\"edge\">\n",
       "<title>AlexNet/Sequential[features]/Conv2d[8]/8&#45;&gt;AlexNet/Sequential[features]/Conv2d[8]/25</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M729.6514,-1511.8971C729.6514,-1503.8575 729.6514,-1494.9539 729.6514,-1486.4096\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"733.1515,-1486.2478 729.6514,-1476.2479 726.1515,-1486.2479 733.1515,-1486.2478\"/>\n",
       "</g>\n",
       "<!-- AlexNet/Sequential[features]/Conv2d[10]/27 -->\n",
       "<g id=\"node20\" class=\"node\">\n",
       "<title>AlexNet/Sequential[features]/Conv2d[10]/27</title>\n",
       "<polygon fill=\"#d3d3d3\" stroke=\"#000000\" points=\"654.7731,-1292 542.5296,-1292 542.5296,-1236 654.7731,-1236 654.7731,-1292\"/>\n",
       "<text text-anchor=\"middle\" x=\"598.6514\" y=\"-1278.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">AlexNet</text>\n",
       "<text text-anchor=\"middle\" x=\"598.6514\" y=\"-1266.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">Sequential[features]</text>\n",
       "<text text-anchor=\"middle\" x=\"598.6514\" y=\"-1254.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">Conv2d[10]</text>\n",
       "<text text-anchor=\"middle\" x=\"598.6514\" y=\"-1242.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">27</text>\n",
       "</g>\n",
       "<!-- AlexNet/Sequential[features]/ReLU[9]/26&#45;&gt;AlexNet/Sequential[features]/Conv2d[10]/27 -->\n",
       "<g id=\"edge19\" class=\"edge\">\n",
       "<title>AlexNet/Sequential[features]/ReLU[9]/26&#45;&gt;AlexNet/Sequential[features]/Conv2d[10]/27</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M689.6352,-1327.8971C676.3007,-1318.5323 661.2967,-1307.9952 647.3765,-1298.2192\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"649.0689,-1295.1309 638.8739,-1292.2479 645.0458,-1300.8593 649.0689,-1295.1309\"/>\n",
       "</g>\n",
       "<!-- AlexNet/Sequential[features]/ReLU[11]/28 -->\n",
       "<g id=\"node23\" class=\"node\">\n",
       "<title>AlexNet/Sequential[features]/ReLU[11]/28</title>\n",
       "<polygon fill=\"#d3d3d3\" stroke=\"#000000\" points=\"654.7731,-1200 542.5296,-1200 542.5296,-1144 654.7731,-1144 654.7731,-1200\"/>\n",
       "<text text-anchor=\"middle\" x=\"598.6514\" y=\"-1186.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">AlexNet</text>\n",
       "<text text-anchor=\"middle\" x=\"598.6514\" y=\"-1174.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">Sequential[features]</text>\n",
       "<text text-anchor=\"middle\" x=\"598.6514\" y=\"-1162.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">ReLU[11]</text>\n",
       "<text text-anchor=\"middle\" x=\"598.6514\" y=\"-1150.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">28</text>\n",
       "</g>\n",
       "<!-- AlexNet/Sequential[features]/Conv2d[10]/27&#45;&gt;AlexNet/Sequential[features]/ReLU[11]/28 -->\n",
       "<g id=\"edge22\" class=\"edge\">\n",
       "<title>AlexNet/Sequential[features]/Conv2d[10]/27&#45;&gt;AlexNet/Sequential[features]/ReLU[11]/28</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M598.6514,-1235.8971C598.6514,-1227.8575 598.6514,-1218.9539 598.6514,-1210.4096\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"602.1515,-1210.2478 598.6514,-1200.2479 595.1515,-1210.2479 602.1515,-1210.2478\"/>\n",
       "</g>\n",
       "<!-- AlexNet/Sequential[features]/Conv2d[10]/9 -->\n",
       "<g id=\"node21\" class=\"node\">\n",
       "<title>AlexNet/Sequential[features]/Conv2d[10]/9</title>\n",
       "<polygon fill=\"#d3d3d3\" stroke=\"#000000\" points=\"523.7731,-1384 411.5296,-1384 411.5296,-1328 523.7731,-1328 523.7731,-1384\"/>\n",
       "<text text-anchor=\"middle\" x=\"467.6514\" y=\"-1370.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">AlexNet</text>\n",
       "<text text-anchor=\"middle\" x=\"467.6514\" y=\"-1358.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">Sequential[features]</text>\n",
       "<text text-anchor=\"middle\" x=\"467.6514\" y=\"-1346.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">Conv2d[10]</text>\n",
       "<text text-anchor=\"middle\" x=\"467.6514\" y=\"-1334.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">9</text>\n",
       "</g>\n",
       "<!-- AlexNet/Sequential[features]/Conv2d[10]/9&#45;&gt;AlexNet/Sequential[features]/Conv2d[10]/27 -->\n",
       "<g id=\"edge20\" class=\"edge\">\n",
       "<title>AlexNet/Sequential[features]/Conv2d[10]/9&#45;&gt;AlexNet/Sequential[features]/Conv2d[10]/27</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M507.6675,-1327.8971C521.0021,-1318.5323 536.0061,-1307.9952 549.9263,-1298.2192\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"552.2569,-1300.8593 558.4289,-1292.2479 548.2338,-1295.1309 552.2569,-1300.8593\"/>\n",
       "</g>\n",
       "<!-- AlexNet/Sequential[features]/Conv2d[10]/10 -->\n",
       "<g id=\"node22\" class=\"node\">\n",
       "<title>AlexNet/Sequential[features]/Conv2d[10]/10</title>\n",
       "<polygon fill=\"#d3d3d3\" stroke=\"#000000\" points=\"654.7731,-1384 542.5296,-1384 542.5296,-1328 654.7731,-1328 654.7731,-1384\"/>\n",
       "<text text-anchor=\"middle\" x=\"598.6514\" y=\"-1370.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">AlexNet</text>\n",
       "<text text-anchor=\"middle\" x=\"598.6514\" y=\"-1358.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">Sequential[features]</text>\n",
       "<text text-anchor=\"middle\" x=\"598.6514\" y=\"-1346.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">Conv2d[10]</text>\n",
       "<text text-anchor=\"middle\" x=\"598.6514\" y=\"-1334.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">10</text>\n",
       "</g>\n",
       "<!-- AlexNet/Sequential[features]/Conv2d[10]/10&#45;&gt;AlexNet/Sequential[features]/Conv2d[10]/27 -->\n",
       "<g id=\"edge21\" class=\"edge\">\n",
       "<title>AlexNet/Sequential[features]/Conv2d[10]/10&#45;&gt;AlexNet/Sequential[features]/Conv2d[10]/27</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M598.6514,-1327.8971C598.6514,-1319.8575 598.6514,-1310.9539 598.6514,-1302.4096\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"602.1515,-1302.2478 598.6514,-1292.2479 595.1515,-1302.2479 602.1515,-1302.2478\"/>\n",
       "</g>\n",
       "<!-- AlexNet/Sequential[features]/MaxPool2d[12]/29 -->\n",
       "<g id=\"node24\" class=\"node\">\n",
       "<title>AlexNet/Sequential[features]/MaxPool2d[12]/29</title>\n",
       "<polygon fill=\"#d3d3d3\" stroke=\"#000000\" points=\"654.7731,-1108 542.5296,-1108 542.5296,-1052 654.7731,-1052 654.7731,-1108\"/>\n",
       "<text text-anchor=\"middle\" x=\"598.6514\" y=\"-1094.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">AlexNet</text>\n",
       "<text text-anchor=\"middle\" x=\"598.6514\" y=\"-1082.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">Sequential[features]</text>\n",
       "<text text-anchor=\"middle\" x=\"598.6514\" y=\"-1070.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">MaxPool2d[12]</text>\n",
       "<text text-anchor=\"middle\" x=\"598.6514\" y=\"-1058.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">29</text>\n",
       "</g>\n",
       "<!-- AlexNet/Sequential[features]/ReLU[11]/28&#45;&gt;AlexNet/Sequential[features]/MaxPool2d[12]/29 -->\n",
       "<g id=\"edge23\" class=\"edge\">\n",
       "<title>AlexNet/Sequential[features]/ReLU[11]/28&#45;&gt;AlexNet/Sequential[features]/MaxPool2d[12]/29</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M598.6514,-1143.8971C598.6514,-1135.8575 598.6514,-1126.9539 598.6514,-1118.4096\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"602.1515,-1118.2478 598.6514,-1108.2479 595.1515,-1118.2479 602.1515,-1118.2478\"/>\n",
       "</g>\n",
       "<!-- AlexNet/30 -->\n",
       "<g id=\"node25\" class=\"node\">\n",
       "<title>AlexNet/30</title>\n",
       "<polygon fill=\"#d3d3d3\" stroke=\"#000000\" points=\"598.8048,-1016 542.498,-1016 542.498,-984 598.8048,-984 598.8048,-1016\"/>\n",
       "<text text-anchor=\"middle\" x=\"570.6514\" y=\"-1002.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">AlexNet</text>\n",
       "<text text-anchor=\"middle\" x=\"570.6514\" y=\"-990.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">30</text>\n",
       "</g>\n",
       "<!-- AlexNet/Sequential[features]/MaxPool2d[12]/29&#45;&gt;AlexNet/30 -->\n",
       "<g id=\"edge24\" class=\"edge\">\n",
       "<title>AlexNet/Sequential[features]/MaxPool2d[12]/29&#45;&gt;AlexNet/30</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M588.7008,-1051.5697C585.7506,-1043.1406 582.5379,-1033.9614 579.6515,-1025.7148\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"582.9394,-1024.5137 576.3323,-1016.2314 576.3324,-1026.8262 582.9394,-1024.5137\"/>\n",
       "</g>\n",
       "<!-- AlexNet/37 -->\n",
       "<g id=\"node32\" class=\"node\">\n",
       "<title>AlexNet/37</title>\n",
       "<polygon fill=\"#d3d3d3\" stroke=\"#000000\" points=\"626.8048,-676 570.498,-676 570.498,-644 626.8048,-644 626.8048,-676\"/>\n",
       "<text text-anchor=\"middle\" x=\"598.6514\" y=\"-662.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">AlexNet</text>\n",
       "<text text-anchor=\"middle\" x=\"598.6514\" y=\"-650.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">37</text>\n",
       "</g>\n",
       "<!-- AlexNet/Sequential[features]/MaxPool2d[12]/29&#45;&gt;AlexNet/37 -->\n",
       "<g id=\"edge31\" class=\"edge\">\n",
       "<title>AlexNet/Sequential[features]/MaxPool2d[12]/29&#45;&gt;AlexNet/37</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M607.2044,-1051.9261C615.4387,-1022.3775 626.6514,-974.3634 626.6514,-932 626.6514,-932 626.6514,-932 626.6514,-796 626.6514,-757.0551 615.5003,-713.1019 607.2724,-685.9716\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"610.512,-684.6047 604.1833,-676.1078 603.832,-686.6968 610.512,-684.6047\"/>\n",
       "</g>\n",
       "<!-- AlexNet/31 -->\n",
       "<g id=\"node26\" class=\"node\">\n",
       "<title>AlexNet/31</title>\n",
       "<polygon fill=\"#d3d3d3\" stroke=\"#000000\" points=\"598.8048,-948 542.498,-948 542.498,-916 598.8048,-916 598.8048,-948\"/>\n",
       "<text text-anchor=\"middle\" x=\"570.6514\" y=\"-934.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">AlexNet</text>\n",
       "<text text-anchor=\"middle\" x=\"570.6514\" y=\"-922.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">31</text>\n",
       "</g>\n",
       "<!-- AlexNet/30&#45;&gt;AlexNet/31 -->\n",
       "<g id=\"edge25\" class=\"edge\">\n",
       "<title>AlexNet/30&#45;&gt;AlexNet/31</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M570.6514,-983.8849C570.6514,-976.3737 570.6514,-967.2345 570.6514,-958.7315\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"574.1515,-958.4717 570.6514,-948.4717 567.1515,-958.4718 574.1515,-958.4717\"/>\n",
       "</g>\n",
       "<!-- AlexNet/32 -->\n",
       "<g id=\"node27\" class=\"node\">\n",
       "<title>AlexNet/32</title>\n",
       "<polygon fill=\"#d3d3d3\" stroke=\"#000000\" points=\"598.8048,-880 542.498,-880 542.498,-848 598.8048,-848 598.8048,-880\"/>\n",
       "<text text-anchor=\"middle\" x=\"570.6514\" y=\"-866.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">AlexNet</text>\n",
       "<text text-anchor=\"middle\" x=\"570.6514\" y=\"-854.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">32</text>\n",
       "</g>\n",
       "<!-- AlexNet/31&#45;&gt;AlexNet/32 -->\n",
       "<g id=\"edge26\" class=\"edge\">\n",
       "<title>AlexNet/31&#45;&gt;AlexNet/32</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M570.6514,-915.8849C570.6514,-908.3737 570.6514,-899.2345 570.6514,-890.7315\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"574.1515,-890.4717 570.6514,-880.4717 567.1515,-890.4718 574.1515,-890.4717\"/>\n",
       "</g>\n",
       "<!-- AlexNet/34 -->\n",
       "<g id=\"node29\" class=\"node\">\n",
       "<title>AlexNet/34</title>\n",
       "<polygon fill=\"#d3d3d3\" stroke=\"#000000\" points=\"598.8048,-812 542.498,-812 542.498,-780 598.8048,-780 598.8048,-812\"/>\n",
       "<text text-anchor=\"middle\" x=\"570.6514\" y=\"-798.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">AlexNet</text>\n",
       "<text text-anchor=\"middle\" x=\"570.6514\" y=\"-786.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">34</text>\n",
       "</g>\n",
       "<!-- AlexNet/32&#45;&gt;AlexNet/34 -->\n",
       "<g id=\"edge27\" class=\"edge\">\n",
       "<title>AlexNet/32&#45;&gt;AlexNet/34</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M570.6514,-847.8849C570.6514,-840.3737 570.6514,-831.2345 570.6514,-822.7315\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"574.1515,-822.4717 570.6514,-812.4717 567.1515,-822.4718 574.1515,-822.4717\"/>\n",
       "</g>\n",
       "<!-- AlexNet/33 -->\n",
       "<g id=\"node28\" class=\"node\">\n",
       "<title>AlexNet/33</title>\n",
       "<polygon fill=\"#d3d3d3\" stroke=\"#000000\" points=\"523.8048,-880 467.498,-880 467.498,-848 523.8048,-848 523.8048,-880\"/>\n",
       "<text text-anchor=\"middle\" x=\"495.6514\" y=\"-866.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">AlexNet</text>\n",
       "<text text-anchor=\"middle\" x=\"495.6514\" y=\"-854.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">33</text>\n",
       "</g>\n",
       "<!-- AlexNet/35 -->\n",
       "<g id=\"node30\" class=\"node\">\n",
       "<title>AlexNet/35</title>\n",
       "<polygon fill=\"#d3d3d3\" stroke=\"#000000\" points=\"523.8048,-812 467.498,-812 467.498,-780 523.8048,-780 523.8048,-812\"/>\n",
       "<text text-anchor=\"middle\" x=\"495.6514\" y=\"-798.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">AlexNet</text>\n",
       "<text text-anchor=\"middle\" x=\"495.6514\" y=\"-786.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">35</text>\n",
       "</g>\n",
       "<!-- AlexNet/33&#45;&gt;AlexNet/35 -->\n",
       "<g id=\"edge28\" class=\"edge\">\n",
       "<title>AlexNet/33&#45;&gt;AlexNet/35</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M495.6514,-847.8849C495.6514,-840.3737 495.6514,-831.2345 495.6514,-822.7315\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"499.1515,-822.4717 495.6514,-812.4717 492.1515,-822.4718 499.1515,-822.4717\"/>\n",
       "</g>\n",
       "<!-- AlexNet/36 -->\n",
       "<g id=\"node31\" class=\"node\">\n",
       "<title>AlexNet/36</title>\n",
       "<polygon fill=\"#d3d3d3\" stroke=\"#000000\" points=\"598.8048,-744 542.498,-744 542.498,-712 598.8048,-712 598.8048,-744\"/>\n",
       "<text text-anchor=\"middle\" x=\"570.6514\" y=\"-730.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">AlexNet</text>\n",
       "<text text-anchor=\"middle\" x=\"570.6514\" y=\"-718.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">36</text>\n",
       "</g>\n",
       "<!-- AlexNet/34&#45;&gt;AlexNet/36 -->\n",
       "<g id=\"edge29\" class=\"edge\">\n",
       "<title>AlexNet/34&#45;&gt;AlexNet/36</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M570.6514,-779.8849C570.6514,-772.3737 570.6514,-763.2345 570.6514,-754.7315\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"574.1515,-754.4717 570.6514,-744.4717 567.1515,-754.4718 574.1515,-754.4717\"/>\n",
       "</g>\n",
       "<!-- AlexNet/35&#45;&gt;AlexNet/36 -->\n",
       "<g id=\"edge30\" class=\"edge\">\n",
       "<title>AlexNet/35&#45;&gt;AlexNet/36</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M513.4253,-779.8849C522.8834,-771.3096 534.6819,-760.6123 545.1045,-751.1625\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"547.6876,-753.5449 552.7451,-744.235 542.9858,-748.359 547.6876,-753.5449\"/>\n",
       "</g>\n",
       "<!-- AlexNet/36&#45;&gt;AlexNet/37 -->\n",
       "<g id=\"edge32\" class=\"edge\">\n",
       "<title>AlexNet/36&#45;&gt;AlexNet/37</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M577.287,-711.8849C580.4829,-704.1233 584.3947,-694.6234 587.9935,-685.8834\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"591.2977,-687.0512 591.8689,-676.4717 584.8249,-684.3859 591.2977,-687.0512\"/>\n",
       "</g>\n",
       "<!-- AlexNet/Sequential[classifier]/Dropout[0]/38 -->\n",
       "<g id=\"node33\" class=\"node\">\n",
       "<title>AlexNet/Sequential[classifier]/Dropout[0]/38</title>\n",
       "<polygon fill=\"#d3d3d3\" stroke=\"#000000\" points=\"657.4545,-608 539.8482,-608 539.8482,-552 657.4545,-552 657.4545,-608\"/>\n",
       "<text text-anchor=\"middle\" x=\"598.6514\" y=\"-594.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">AlexNet</text>\n",
       "<text text-anchor=\"middle\" x=\"598.6514\" y=\"-582.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">Sequential[classifier]</text>\n",
       "<text text-anchor=\"middle\" x=\"598.6514\" y=\"-570.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">Dropout[0]</text>\n",
       "<text text-anchor=\"middle\" x=\"598.6514\" y=\"-558.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">38</text>\n",
       "</g>\n",
       "<!-- AlexNet/37&#45;&gt;AlexNet/Sequential[classifier]/Dropout[0]/38 -->\n",
       "<g id=\"edge33\" class=\"edge\">\n",
       "<title>AlexNet/37&#45;&gt;AlexNet/Sequential[classifier]/Dropout[0]/38</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M598.6514,-643.8108C598.6514,-636.3802 598.6514,-627.2259 598.6514,-618.1947\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"602.1515,-618.1487 598.6514,-608.1488 595.1515,-618.1488 602.1515,-618.1487\"/>\n",
       "</g>\n",
       "<!-- AlexNet/Sequential[classifier]/Linear[1]/40 -->\n",
       "<g id=\"node34\" class=\"node\">\n",
       "<title>AlexNet/Sequential[classifier]/Linear[1]/40</title>\n",
       "<polygon fill=\"#d3d3d3\" stroke=\"#000000\" points=\"522.4545,-516 404.8482,-516 404.8482,-460 522.4545,-460 522.4545,-516\"/>\n",
       "<text text-anchor=\"middle\" x=\"463.6514\" y=\"-502.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">AlexNet</text>\n",
       "<text text-anchor=\"middle\" x=\"463.6514\" y=\"-490.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">Sequential[classifier]</text>\n",
       "<text text-anchor=\"middle\" x=\"463.6514\" y=\"-478.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">Linear[1]</text>\n",
       "<text text-anchor=\"middle\" x=\"463.6514\" y=\"-466.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">40</text>\n",
       "</g>\n",
       "<!-- AlexNet/Sequential[classifier]/Dropout[0]/38&#45;&gt;AlexNet/Sequential[classifier]/Linear[1]/40 -->\n",
       "<g id=\"edge34\" class=\"edge\">\n",
       "<title>AlexNet/Sequential[classifier]/Dropout[0]/38&#45;&gt;AlexNet/Sequential[classifier]/Linear[1]/40</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M557.4134,-551.8971C543.542,-542.444 527.9176,-531.7962 513.4586,-521.9427\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"515.3366,-518.9871 505.102,-516.2479 511.3946,-524.7716 515.3366,-518.9871\"/>\n",
       "</g>\n",
       "<!-- AlexNet/Sequential[classifier]/ReLU[2]/41 -->\n",
       "<g id=\"node37\" class=\"node\">\n",
       "<title>AlexNet/Sequential[classifier]/ReLU[2]/41</title>\n",
       "<polygon fill=\"#d3d3d3\" stroke=\"#000000\" points=\"522.4545,-424 404.8482,-424 404.8482,-368 522.4545,-368 522.4545,-424\"/>\n",
       "<text text-anchor=\"middle\" x=\"463.6514\" y=\"-410.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">AlexNet</text>\n",
       "<text text-anchor=\"middle\" x=\"463.6514\" y=\"-398.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">Sequential[classifier]</text>\n",
       "<text text-anchor=\"middle\" x=\"463.6514\" y=\"-386.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">ReLU[2]</text>\n",
       "<text text-anchor=\"middle\" x=\"463.6514\" y=\"-374.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">41</text>\n",
       "</g>\n",
       "<!-- AlexNet/Sequential[classifier]/Linear[1]/40&#45;&gt;AlexNet/Sequential[classifier]/ReLU[2]/41 -->\n",
       "<g id=\"edge37\" class=\"edge\">\n",
       "<title>AlexNet/Sequential[classifier]/Linear[1]/40&#45;&gt;AlexNet/Sequential[classifier]/ReLU[2]/41</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M463.6514,-459.8971C463.6514,-451.8575 463.6514,-442.9539 463.6514,-434.4096\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"467.1515,-434.2478 463.6514,-424.2479 460.1515,-434.2479 467.1515,-434.2478\"/>\n",
       "</g>\n",
       "<!-- AlexNet/Sequential[classifier]/Linear[1]/11 -->\n",
       "<g id=\"node35\" class=\"node\">\n",
       "<title>AlexNet/Sequential[classifier]/Linear[1]/11</title>\n",
       "<polygon fill=\"#d3d3d3\" stroke=\"#000000\" points=\"387.4545,-608 269.8482,-608 269.8482,-552 387.4545,-552 387.4545,-608\"/>\n",
       "<text text-anchor=\"middle\" x=\"328.6514\" y=\"-594.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">AlexNet</text>\n",
       "<text text-anchor=\"middle\" x=\"328.6514\" y=\"-582.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">Sequential[classifier]</text>\n",
       "<text text-anchor=\"middle\" x=\"328.6514\" y=\"-570.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">Linear[1]</text>\n",
       "<text text-anchor=\"middle\" x=\"328.6514\" y=\"-558.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">11</text>\n",
       "</g>\n",
       "<!-- AlexNet/Sequential[classifier]/Linear[1]/11&#45;&gt;AlexNet/Sequential[classifier]/Linear[1]/40 -->\n",
       "<g id=\"edge35\" class=\"edge\">\n",
       "<title>AlexNet/Sequential[classifier]/Linear[1]/11&#45;&gt;AlexNet/Sequential[classifier]/Linear[1]/40</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M369.8894,-551.8971C383.7607,-542.444 399.3851,-531.7962 413.8442,-521.9427\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"415.9082,-524.7716 422.2007,-516.2479 411.9661,-518.9871 415.9082,-524.7716\"/>\n",
       "</g>\n",
       "<!-- AlexNet/Sequential[classifier]/Linear[1]/12 -->\n",
       "<g id=\"node36\" class=\"node\">\n",
       "<title>AlexNet/Sequential[classifier]/Linear[1]/12</title>\n",
       "<polygon fill=\"#d3d3d3\" stroke=\"#000000\" points=\"522.4545,-608 404.8482,-608 404.8482,-552 522.4545,-552 522.4545,-608\"/>\n",
       "<text text-anchor=\"middle\" x=\"463.6514\" y=\"-594.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">AlexNet</text>\n",
       "<text text-anchor=\"middle\" x=\"463.6514\" y=\"-582.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">Sequential[classifier]</text>\n",
       "<text text-anchor=\"middle\" x=\"463.6514\" y=\"-570.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">Linear[1]</text>\n",
       "<text text-anchor=\"middle\" x=\"463.6514\" y=\"-558.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">12</text>\n",
       "</g>\n",
       "<!-- AlexNet/Sequential[classifier]/Linear[1]/12&#45;&gt;AlexNet/Sequential[classifier]/Linear[1]/40 -->\n",
       "<g id=\"edge36\" class=\"edge\">\n",
       "<title>AlexNet/Sequential[classifier]/Linear[1]/12&#45;&gt;AlexNet/Sequential[classifier]/Linear[1]/40</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M463.6514,-551.8971C463.6514,-543.8575 463.6514,-534.9539 463.6514,-526.4096\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"467.1515,-526.2478 463.6514,-516.2479 460.1515,-526.2479 467.1515,-526.2478\"/>\n",
       "</g>\n",
       "<!-- AlexNet/Sequential[classifier]/Dropout[3]/42 -->\n",
       "<g id=\"node38\" class=\"node\">\n",
       "<title>AlexNet/Sequential[classifier]/Dropout[3]/42</title>\n",
       "<polygon fill=\"#d3d3d3\" stroke=\"#000000\" points=\"522.4545,-332 404.8482,-332 404.8482,-276 522.4545,-276 522.4545,-332\"/>\n",
       "<text text-anchor=\"middle\" x=\"463.6514\" y=\"-318.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">AlexNet</text>\n",
       "<text text-anchor=\"middle\" x=\"463.6514\" y=\"-306.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">Sequential[classifier]</text>\n",
       "<text text-anchor=\"middle\" x=\"463.6514\" y=\"-294.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">Dropout[3]</text>\n",
       "<text text-anchor=\"middle\" x=\"463.6514\" y=\"-282.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">42</text>\n",
       "</g>\n",
       "<!-- AlexNet/Sequential[classifier]/ReLU[2]/41&#45;&gt;AlexNet/Sequential[classifier]/Dropout[3]/42 -->\n",
       "<g id=\"edge38\" class=\"edge\">\n",
       "<title>AlexNet/Sequential[classifier]/ReLU[2]/41&#45;&gt;AlexNet/Sequential[classifier]/Dropout[3]/42</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M463.6514,-367.8971C463.6514,-359.8575 463.6514,-350.9539 463.6514,-342.4096\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"467.1515,-342.2478 463.6514,-332.2479 460.1515,-342.2479 467.1515,-342.2478\"/>\n",
       "</g>\n",
       "<!-- AlexNet/Sequential[classifier]/Linear[4]/44 -->\n",
       "<g id=\"node39\" class=\"node\">\n",
       "<title>AlexNet/Sequential[classifier]/Linear[4]/44</title>\n",
       "<polygon fill=\"#d3d3d3\" stroke=\"#000000\" points=\"387.4545,-240 269.8482,-240 269.8482,-184 387.4545,-184 387.4545,-240\"/>\n",
       "<text text-anchor=\"middle\" x=\"328.6514\" y=\"-226.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">AlexNet</text>\n",
       "<text text-anchor=\"middle\" x=\"328.6514\" y=\"-214.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">Sequential[classifier]</text>\n",
       "<text text-anchor=\"middle\" x=\"328.6514\" y=\"-202.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">Linear[4]</text>\n",
       "<text text-anchor=\"middle\" x=\"328.6514\" y=\"-190.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">44</text>\n",
       "</g>\n",
       "<!-- AlexNet/Sequential[classifier]/Dropout[3]/42&#45;&gt;AlexNet/Sequential[classifier]/Linear[4]/44 -->\n",
       "<g id=\"edge39\" class=\"edge\">\n",
       "<title>AlexNet/Sequential[classifier]/Dropout[3]/42&#45;&gt;AlexNet/Sequential[classifier]/Linear[4]/44</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M422.4134,-275.8971C408.542,-266.444 392.9176,-255.7962 378.4586,-245.9427\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"380.3366,-242.9871 370.102,-240.2479 376.3946,-248.7716 380.3366,-242.9871\"/>\n",
       "</g>\n",
       "<!-- AlexNet/Sequential[classifier]/ReLU[5]/45 -->\n",
       "<g id=\"node42\" class=\"node\">\n",
       "<title>AlexNet/Sequential[classifier]/ReLU[5]/45</title>\n",
       "<polygon fill=\"#d3d3d3\" stroke=\"#000000\" points=\"387.4545,-148 269.8482,-148 269.8482,-92 387.4545,-92 387.4545,-148\"/>\n",
       "<text text-anchor=\"middle\" x=\"328.6514\" y=\"-134.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">AlexNet</text>\n",
       "<text text-anchor=\"middle\" x=\"328.6514\" y=\"-122.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">Sequential[classifier]</text>\n",
       "<text text-anchor=\"middle\" x=\"328.6514\" y=\"-110.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">ReLU[5]</text>\n",
       "<text text-anchor=\"middle\" x=\"328.6514\" y=\"-98.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">45</text>\n",
       "</g>\n",
       "<!-- AlexNet/Sequential[classifier]/Linear[4]/44&#45;&gt;AlexNet/Sequential[classifier]/ReLU[5]/45 -->\n",
       "<g id=\"edge42\" class=\"edge\">\n",
       "<title>AlexNet/Sequential[classifier]/Linear[4]/44&#45;&gt;AlexNet/Sequential[classifier]/ReLU[5]/45</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M328.6514,-183.8971C328.6514,-175.8575 328.6514,-166.9539 328.6514,-158.4096\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"332.1515,-158.2478 328.6514,-148.2479 325.1515,-158.2479 332.1515,-158.2478\"/>\n",
       "</g>\n",
       "<!-- AlexNet/Sequential[classifier]/Linear[4]/13 -->\n",
       "<g id=\"node40\" class=\"node\">\n",
       "<title>AlexNet/Sequential[classifier]/Linear[4]/13</title>\n",
       "<polygon fill=\"#d3d3d3\" stroke=\"#000000\" points=\"252.4545,-332 134.8482,-332 134.8482,-276 252.4545,-276 252.4545,-332\"/>\n",
       "<text text-anchor=\"middle\" x=\"193.6514\" y=\"-318.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">AlexNet</text>\n",
       "<text text-anchor=\"middle\" x=\"193.6514\" y=\"-306.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">Sequential[classifier]</text>\n",
       "<text text-anchor=\"middle\" x=\"193.6514\" y=\"-294.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">Linear[4]</text>\n",
       "<text text-anchor=\"middle\" x=\"193.6514\" y=\"-282.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">13</text>\n",
       "</g>\n",
       "<!-- AlexNet/Sequential[classifier]/Linear[4]/13&#45;&gt;AlexNet/Sequential[classifier]/Linear[4]/44 -->\n",
       "<g id=\"edge40\" class=\"edge\">\n",
       "<title>AlexNet/Sequential[classifier]/Linear[4]/13&#45;&gt;AlexNet/Sequential[classifier]/Linear[4]/44</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M234.8894,-275.8971C248.7607,-266.444 264.3851,-255.7962 278.8442,-245.9427\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"280.9082,-248.7716 287.2007,-240.2479 276.9661,-242.9871 280.9082,-248.7716\"/>\n",
       "</g>\n",
       "<!-- AlexNet/Sequential[classifier]/Linear[4]/14 -->\n",
       "<g id=\"node41\" class=\"node\">\n",
       "<title>AlexNet/Sequential[classifier]/Linear[4]/14</title>\n",
       "<polygon fill=\"#d3d3d3\" stroke=\"#000000\" points=\"387.4545,-332 269.8482,-332 269.8482,-276 387.4545,-276 387.4545,-332\"/>\n",
       "<text text-anchor=\"middle\" x=\"328.6514\" y=\"-318.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">AlexNet</text>\n",
       "<text text-anchor=\"middle\" x=\"328.6514\" y=\"-306.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">Sequential[classifier]</text>\n",
       "<text text-anchor=\"middle\" x=\"328.6514\" y=\"-294.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">Linear[4]</text>\n",
       "<text text-anchor=\"middle\" x=\"328.6514\" y=\"-282.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">14</text>\n",
       "</g>\n",
       "<!-- AlexNet/Sequential[classifier]/Linear[4]/14&#45;&gt;AlexNet/Sequential[classifier]/Linear[4]/44 -->\n",
       "<g id=\"edge41\" class=\"edge\">\n",
       "<title>AlexNet/Sequential[classifier]/Linear[4]/14&#45;&gt;AlexNet/Sequential[classifier]/Linear[4]/44</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M328.6514,-275.8971C328.6514,-267.8575 328.6514,-258.9539 328.6514,-250.4096\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"332.1515,-250.2478 328.6514,-240.2479 325.1515,-250.2479 332.1515,-250.2478\"/>\n",
       "</g>\n",
       "<!-- AlexNet/Sequential[classifier]/Linear[6]/46 -->\n",
       "<g id=\"node43\" class=\"node\">\n",
       "<title>AlexNet/Sequential[classifier]/Linear[6]/46</title>\n",
       "<polygon fill=\"#d3d3d3\" stroke=\"#000000\" points=\"252.4545,-56 134.8482,-56 134.8482,0 252.4545,0 252.4545,-56\"/>\n",
       "<text text-anchor=\"middle\" x=\"193.6514\" y=\"-42.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">AlexNet</text>\n",
       "<text text-anchor=\"middle\" x=\"193.6514\" y=\"-30.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">Sequential[classifier]</text>\n",
       "<text text-anchor=\"middle\" x=\"193.6514\" y=\"-18.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">Linear[6]</text>\n",
       "<text text-anchor=\"middle\" x=\"193.6514\" y=\"-6.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">46</text>\n",
       "</g>\n",
       "<!-- AlexNet/Sequential[classifier]/ReLU[5]/45&#45;&gt;AlexNet/Sequential[classifier]/Linear[6]/46 -->\n",
       "<g id=\"edge43\" class=\"edge\">\n",
       "<title>AlexNet/Sequential[classifier]/ReLU[5]/45&#45;&gt;AlexNet/Sequential[classifier]/Linear[6]/46</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M287.4134,-91.8971C273.542,-82.444 257.9176,-71.7962 243.4586,-61.9427\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"245.3366,-58.9871 235.102,-56.2479 241.3946,-64.7716 245.3366,-58.9871\"/>\n",
       "</g>\n",
       "<!-- AlexNet/Sequential[classifier]/Linear[6]/15 -->\n",
       "<g id=\"node44\" class=\"node\">\n",
       "<title>AlexNet/Sequential[classifier]/Linear[6]/15</title>\n",
       "<polygon fill=\"#d3d3d3\" stroke=\"#000000\" points=\"117.4545,-148 -.1518,-148 -.1518,-92 117.4545,-92 117.4545,-148\"/>\n",
       "<text text-anchor=\"middle\" x=\"58.6514\" y=\"-134.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">AlexNet</text>\n",
       "<text text-anchor=\"middle\" x=\"58.6514\" y=\"-122.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">Sequential[classifier]</text>\n",
       "<text text-anchor=\"middle\" x=\"58.6514\" y=\"-110.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">Linear[6]</text>\n",
       "<text text-anchor=\"middle\" x=\"58.6514\" y=\"-98.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">15</text>\n",
       "</g>\n",
       "<!-- AlexNet/Sequential[classifier]/Linear[6]/15&#45;&gt;AlexNet/Sequential[classifier]/Linear[6]/46 -->\n",
       "<g id=\"edge44\" class=\"edge\">\n",
       "<title>AlexNet/Sequential[classifier]/Linear[6]/15&#45;&gt;AlexNet/Sequential[classifier]/Linear[6]/46</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M99.8894,-91.8971C113.7607,-82.444 129.3851,-71.7962 143.8442,-61.9427\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"145.9082,-64.7716 152.2007,-56.2479 141.9661,-58.9871 145.9082,-64.7716\"/>\n",
       "</g>\n",
       "<!-- AlexNet/Sequential[classifier]/Linear[6]/16 -->\n",
       "<g id=\"node45\" class=\"node\">\n",
       "<title>AlexNet/Sequential[classifier]/Linear[6]/16</title>\n",
       "<polygon fill=\"#d3d3d3\" stroke=\"#000000\" points=\"252.4545,-148 134.8482,-148 134.8482,-92 252.4545,-92 252.4545,-148\"/>\n",
       "<text text-anchor=\"middle\" x=\"193.6514\" y=\"-134.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">AlexNet</text>\n",
       "<text text-anchor=\"middle\" x=\"193.6514\" y=\"-122.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">Sequential[classifier]</text>\n",
       "<text text-anchor=\"middle\" x=\"193.6514\" y=\"-110.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">Linear[6]</text>\n",
       "<text text-anchor=\"middle\" x=\"193.6514\" y=\"-98.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">16</text>\n",
       "</g>\n",
       "<!-- AlexNet/Sequential[classifier]/Linear[6]/16&#45;&gt;AlexNet/Sequential[classifier]/Linear[6]/46 -->\n",
       "<g id=\"edge45\" class=\"edge\">\n",
       "<title>AlexNet/Sequential[classifier]/Linear[6]/16&#45;&gt;AlexNet/Sequential[classifier]/Linear[6]/46</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M193.6514,-91.8971C193.6514,-83.8575 193.6514,-74.9539 193.6514,-66.4096\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"197.1515,-66.2478 193.6514,-56.2479 190.1515,-66.2479 197.1515,-66.2478\"/>\n",
       "</g>\n",
       "</g>\n",
       "</svg>\n"
      ],
      "text/plain": [
       "<graphviz.dot.Digraph at 0x1123904a8>"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "with torch.onnx.set_training(model, False):\n",
    "    trace, _ = torch.jit.get_trace_graph(model, args=(x,))\n",
    "make_dot_from_trace(trace)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "And AlexNet double backprop for fun:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/svg+xml": [
       "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>\n",
       "<!DOCTYPE svg PUBLIC \"-//W3C//DTD SVG 1.1//EN\"\n",
       " \"http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd\">\n",
       "<!-- Generated by graphviz version 2.40.1 (20161225.0304)\n",
       " -->\n",
       "<!-- Title: %3 Pages: 1 -->\n",
       "<svg width=\"961pt\" height=\"1793pt\"\n",
       " viewBox=\"0.00 0.00 961.29 1793.00\" xmlns=\"http://www.w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\">\n",
       "<g id=\"graph0\" class=\"graph\" transform=\"scale(.7081 .7081) rotate(0) translate(4 2528)\">\n",
       "<title>%3</title>\n",
       "<polygon fill=\"#ffffff\" stroke=\"transparent\" points=\"-4,4 -4,-2528 1353.4961,-2528 1353.4961,4 -4,4\"/>\n",
       "<!-- 4600695048 -->\n",
       "<g id=\"node1\" class=\"node\">\n",
       "<title>4600695048</title>\n",
       "<polygon fill=\"#caff70\" stroke=\"#000000\" points=\"370.1413,-20 271.8587,-20 271.8587,0 370.1413,0 370.1413,-20\"/>\n",
       "<text text-anchor=\"middle\" x=\"321\" y=\"-6.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">ThAddBackward</text>\n",
       "</g>\n",
       "<!-- 4539554672 -->\n",
       "<g id=\"node2\" class=\"node\">\n",
       "<title>4539554672</title>\n",
       "<polygon fill=\"#d3d3d3\" stroke=\"#000000\" points=\"691.9668,-76 594.0332,-76 594.0332,-56 691.9668,-56 691.9668,-76\"/>\n",
       "<text text-anchor=\"middle\" x=\"643\" y=\"-62.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">MeanBackward1</text>\n",
       "</g>\n",
       "<!-- 4539554672&#45;&gt;4600695048 -->\n",
       "<g id=\"edge1\" class=\"edge\">\n",
       "<title>4539554672&#45;&gt;4600695048</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M593.7446,-57.4338C536.3362,-47.4498 441.5241,-30.9607 380.1729,-20.2909\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"380.7017,-16.8305 370.2499,-18.5652 379.5023,-23.7269 380.7017,-16.8305\"/>\n",
       "</g>\n",
       "<!-- 4810468488 -->\n",
       "<g id=\"node3\" class=\"node\">\n",
       "<title>4810468488</title>\n",
       "<polygon fill=\"#d3d3d3\" stroke=\"#000000\" points=\"971.9844,-132 880.0156,-132 880.0156,-112 971.9844,-112 971.9844,-132\"/>\n",
       "<text text-anchor=\"middle\" x=\"926\" y=\"-118.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">PowBackward0</text>\n",
       "</g>\n",
       "<!-- 4810468488&#45;&gt;4539554672 -->\n",
       "<g id=\"edge2\" class=\"edge\">\n",
       "<title>4810468488&#45;&gt;4539554672</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M879.9013,-112.878C831.492,-103.2988 755.1978,-88.2017 702.3267,-77.7396\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"702.8991,-74.285 692.4099,-75.7772 701.5402,-81.1519 702.8991,-74.285\"/>\n",
       "</g>\n",
       "<!-- 4810466248 -->\n",
       "<g id=\"node4\" class=\"node\">\n",
       "<title>4810466248</title>\n",
       "<polygon fill=\"#d3d3d3\" stroke=\"#000000\" points=\"1299.7893,-188 1120.2107,-188 1120.2107,-168 1299.7893,-168 1299.7893,-188\"/>\n",
       "<text text-anchor=\"middle\" x=\"1210\" y=\"-174.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">ThnnConv2DBackwardBackward</text>\n",
       "</g>\n",
       "<!-- 4810466248&#45;&gt;4810468488 -->\n",
       "<g id=\"edge3\" class=\"edge\">\n",
       "<title>4810466248&#45;&gt;4810468488</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M1159.2265,-167.9883C1109.4177,-158.1669 1033.6399,-143.2248 982.0799,-133.058\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"982.594,-129.5921 972.1057,-131.0913 981.2397,-136.4598 982.594,-129.5921\"/>\n",
       "</g>\n",
       "<!-- 4810467704 -->\n",
       "<g id=\"node5\" class=\"node\">\n",
       "<title>4810467704</title>\n",
       "<polygon fill=\"#d3d3d3\" stroke=\"#000000\" points=\"1236.9453,-244 1075.0547,-244 1075.0547,-224 1236.9453,-224 1236.9453,-244\"/>\n",
       "<text text-anchor=\"middle\" x=\"1156\" y=\"-230.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">ThresholdBackwardBackward</text>\n",
       "</g>\n",
       "<!-- 4810467704&#45;&gt;4810466248 -->\n",
       "<g id=\"edge4\" class=\"edge\">\n",
       "<title>4810467704&#45;&gt;4810466248</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M1165.6541,-223.9883C1173.2383,-216.1233 1183.9889,-204.9745 1193.0055,-195.6239\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1195.6422,-197.9318 1200.0641,-188.3039 1190.6033,-193.0729 1195.6422,-197.9318\"/>\n",
       "</g>\n",
       "<!-- 4810467816 -->\n",
       "<g id=\"node6\" class=\"node\">\n",
       "<title>4810467816</title>\n",
       "<polygon fill=\"#d3d3d3\" stroke=\"#000000\" points=\"1159.1309,-300 928.8691,-300 928.8691,-280 1159.1309,-280 1159.1309,-300\"/>\n",
       "<text text-anchor=\"middle\" x=\"1044\" y=\"-286.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">MaxPool2DWithIndicesBackwardBackward</text>\n",
       "</g>\n",
       "<!-- 4810467816&#45;&gt;4810467704 -->\n",
       "<g id=\"edge5\" class=\"edge\">\n",
       "<title>4810467816&#45;&gt;4810467704</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M1064.0234,-279.9883C1081.4346,-271.2827 1106.8922,-258.5539 1126.6127,-248.6937\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1128.3934,-251.7165 1135.7724,-244.1138 1125.2628,-245.4555 1128.3934,-251.7165\"/>\n",
       "</g>\n",
       "<!-- 4810468320 -->\n",
       "<g id=\"node7\" class=\"node\">\n",
       "<title>4810468320</title>\n",
       "<polygon fill=\"#d3d3d3\" stroke=\"#000000\" points=\"1133.7893,-356 954.2107,-356 954.2107,-336 1133.7893,-336 1133.7893,-356\"/>\n",
       "<text text-anchor=\"middle\" x=\"1044\" y=\"-342.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">ThnnConv2DBackwardBackward</text>\n",
       "</g>\n",
       "<!-- 4810468320&#45;&gt;4810467816 -->\n",
       "<g id=\"edge6\" class=\"edge\">\n",
       "<title>4810468320&#45;&gt;4810467816</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M1044,-335.9883C1044,-328.9098 1044,-319.1714 1044,-310.4779\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1047.5001,-310.3038 1044,-300.3039 1040.5001,-310.3039 1047.5001,-310.3038\"/>\n",
       "</g>\n",
       "<!-- 4810466696 -->\n",
       "<g id=\"node8\" class=\"node\">\n",
       "<title>4810466696</title>\n",
       "<polygon fill=\"#d3d3d3\" stroke=\"#000000\" points=\"1124.9453,-412 963.0547,-412 963.0547,-392 1124.9453,-392 1124.9453,-412\"/>\n",
       "<text text-anchor=\"middle\" x=\"1044\" y=\"-398.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">ThresholdBackwardBackward</text>\n",
       "</g>\n",
       "<!-- 4810466696&#45;&gt;4810468320 -->\n",
       "<g id=\"edge7\" class=\"edge\">\n",
       "<title>4810466696&#45;&gt;4810468320</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M1044,-391.9883C1044,-384.9098 1044,-375.1714 1044,-366.4779\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1047.5001,-366.3038 1044,-356.3039 1040.5001,-366.3039 1047.5001,-366.3038\"/>\n",
       "</g>\n",
       "<!-- 4810467424 -->\n",
       "<g id=\"node9\" class=\"node\">\n",
       "<title>4810467424</title>\n",
       "<polygon fill=\"#d3d3d3\" stroke=\"#000000\" points=\"1123.1309,-468 892.8691,-468 892.8691,-448 1123.1309,-448 1123.1309,-468\"/>\n",
       "<text text-anchor=\"middle\" x=\"1008\" y=\"-454.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">MaxPool2DWithIndicesBackwardBackward</text>\n",
       "</g>\n",
       "<!-- 4810467424&#45;&gt;4810466696 -->\n",
       "<g id=\"edge8\" class=\"edge\">\n",
       "<title>4810467424&#45;&gt;4810466696</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M1014.4361,-447.9883C1019.2899,-440.4379 1026.0893,-429.8611 1031.9437,-420.7543\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1034.9126,-422.6083 1037.3761,-412.3039 1029.0243,-418.823 1034.9126,-422.6083\"/>\n",
       "</g>\n",
       "<!-- 4810466864 -->\n",
       "<g id=\"node10\" class=\"node\">\n",
       "<title>4810466864</title>\n",
       "<polygon fill=\"#d3d3d3\" stroke=\"#000000\" points=\"392.7893,-524 213.2107,-524 213.2107,-504 392.7893,-504 392.7893,-524\"/>\n",
       "<text text-anchor=\"middle\" x=\"303\" y=\"-510.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">ThnnConv2DBackwardBackward</text>\n",
       "</g>\n",
       "<!-- 4810466864&#45;&gt;4810467424 -->\n",
       "<g id=\"edge9\" class=\"edge\">\n",
       "<title>4810466864&#45;&gt;4810467424</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M392.6465,-506.8791C516.8718,-497.0116 741.9297,-479.1347 882.7177,-467.9515\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"883.0148,-471.439 892.7063,-467.1581 882.4605,-464.461 883.0148,-471.439\"/>\n",
       "</g>\n",
       "<!-- 4810466976 -->\n",
       "<g id=\"node11\" class=\"node\">\n",
       "<title>4810466976</title>\n",
       "<polygon fill=\"#d3d3d3\" stroke=\"#000000\" points=\"492.9453,-580 331.0547,-580 331.0547,-560 492.9453,-560 492.9453,-580\"/>\n",
       "<text text-anchor=\"middle\" x=\"412\" y=\"-566.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">ThresholdBackwardBackward</text>\n",
       "</g>\n",
       "<!-- 4810466976&#45;&gt;4810466864 -->\n",
       "<g id=\"edge10\" class=\"edge\">\n",
       "<title>4810466976&#45;&gt;4810466864</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M392.513,-559.9883C375.5681,-551.2827 350.7924,-538.5539 331.6002,-528.6937\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"333.18,-525.5705 322.6858,-524.1138 329.9812,-531.7968 333.18,-525.5705\"/>\n",
       "</g>\n",
       "<!-- 4810469104 -->\n",
       "<g id=\"node12\" class=\"node\">\n",
       "<title>4810469104</title>\n",
       "<polygon fill=\"#d3d3d3\" stroke=\"#000000\" points=\"1051.7893,-636 872.2107,-636 872.2107,-616 1051.7893,-616 1051.7893,-636\"/>\n",
       "<text text-anchor=\"middle\" x=\"962\" y=\"-622.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">ThnnConv2DBackwardBackward</text>\n",
       "</g>\n",
       "<!-- 4810469104&#45;&gt;4810466976 -->\n",
       "<g id=\"edge11\" class=\"edge\">\n",
       "<title>4810469104&#45;&gt;4810466976</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M872.1019,-616.8467C770.4104,-606.4927 605.6886,-589.721 503.3868,-579.3048\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"503.6823,-575.8169 493.3792,-578.2859 502.9732,-582.7809 503.6823,-575.8169\"/>\n",
       "</g>\n",
       "<!-- 4810468880 -->\n",
       "<g id=\"node13\" class=\"node\">\n",
       "<title>4810468880</title>\n",
       "<polygon fill=\"#d3d3d3\" stroke=\"#000000\" points=\"1008.9453,-692 847.0547,-692 847.0547,-672 1008.9453,-672 1008.9453,-692\"/>\n",
       "<text text-anchor=\"middle\" x=\"928\" y=\"-678.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">ThresholdBackwardBackward</text>\n",
       "</g>\n",
       "<!-- 4810468880&#45;&gt;4810469104 -->\n",
       "<g id=\"edge12\" class=\"edge\">\n",
       "<title>4810468880&#45;&gt;4810469104</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M934.0785,-671.9883C938.615,-664.5165 944.9507,-654.0812 950.4404,-645.0393\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"953.546,-646.6682 955.7441,-636.3039 947.5625,-643.0353 953.546,-646.6682\"/>\n",
       "</g>\n",
       "<!-- 4810467032 -->\n",
       "<g id=\"node14\" class=\"node\">\n",
       "<title>4810467032</title>\n",
       "<polygon fill=\"#d3d3d3\" stroke=\"#000000\" points=\"955.7893,-748 776.2107,-748 776.2107,-728 955.7893,-728 955.7893,-748\"/>\n",
       "<text text-anchor=\"middle\" x=\"866\" y=\"-734.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">ThnnConv2DBackwardBackward</text>\n",
       "</g>\n",
       "<!-- 4810467032&#45;&gt;4810468880 -->\n",
       "<g id=\"edge13\" class=\"edge\">\n",
       "<title>4810467032&#45;&gt;4810468880</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M877.0844,-727.9883C885.9662,-719.966 898.6306,-708.5272 909.1066,-699.065\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"911.5171,-701.6041 916.5921,-692.3039 906.8251,-696.4094 911.5171,-701.6041\"/>\n",
       "</g>\n",
       "<!-- 4970041128 -->\n",
       "<g id=\"node15\" class=\"node\">\n",
       "<title>4970041128</title>\n",
       "<polygon fill=\"#d3d3d3\" stroke=\"#000000\" points=\"932.9453,-804 771.0547,-804 771.0547,-784 932.9453,-784 932.9453,-804\"/>\n",
       "<text text-anchor=\"middle\" x=\"852\" y=\"-790.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">ThresholdBackwardBackward</text>\n",
       "</g>\n",
       "<!-- 4970041128&#45;&gt;4810467032 -->\n",
       "<g id=\"edge14\" class=\"edge\">\n",
       "<title>4970041128&#45;&gt;4810467032</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M854.5029,-783.9883C856.2922,-776.8311 858.7613,-766.9547 860.9529,-758.1885\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"864.3941,-758.8542 863.424,-748.3039 857.6031,-757.1564 864.3941,-758.8542\"/>\n",
       "</g>\n",
       "<!-- 4970041240 -->\n",
       "<g id=\"node16\" class=\"node\">\n",
       "<title>4970041240</title>\n",
       "<polygon fill=\"#d3d3d3\" stroke=\"#000000\" points=\"895.1309,-860 664.8691,-860 664.8691,-840 895.1309,-840 895.1309,-860\"/>\n",
       "<text text-anchor=\"middle\" x=\"780\" y=\"-846.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">MaxPool2DWithIndicesBackwardBackward</text>\n",
       "</g>\n",
       "<!-- 4970041240&#45;&gt;4970041128 -->\n",
       "<g id=\"edge15\" class=\"edge\">\n",
       "<title>4970041240&#45;&gt;4970041128</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M792.8722,-839.9883C803.3888,-831.8087 818.472,-820.0773 830.7719,-810.5108\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"833.0074,-813.2061 838.7522,-804.3039 828.7098,-807.6806 833.0074,-813.2061\"/>\n",
       "</g>\n",
       "<!-- 4970039560 -->\n",
       "<g id=\"node17\" class=\"node\">\n",
       "<title>4970039560</title>\n",
       "<polygon fill=\"#d3d3d3\" stroke=\"#000000\" points=\"895.1451,-916 782.8549,-916 782.8549,-896 895.1451,-896 895.1451,-916\"/>\n",
       "<text text-anchor=\"middle\" x=\"839\" y=\"-902.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">AsStridedBackward</text>\n",
       "</g>\n",
       "<!-- 4970039560&#45;&gt;4970041240 -->\n",
       "<g id=\"edge16\" class=\"edge\">\n",
       "<title>4970039560&#45;&gt;4970041240</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M828.452,-895.9883C820.0827,-888.0446 808.1843,-876.7512 798.273,-867.3439\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"800.5185,-864.6496 790.8559,-860.3039 795.6995,-869.7268 800.5185,-864.6496\"/>\n",
       "</g>\n",
       "<!-- 4970039224 -->\n",
       "<g id=\"node18\" class=\"node\">\n",
       "<title>4970039224</title>\n",
       "<polygon fill=\"#d3d3d3\" stroke=\"#000000\" points=\"887.9785,-972 790.0215,-972 790.0215,-952 887.9785,-952 887.9785,-972\"/>\n",
       "<text text-anchor=\"middle\" x=\"839\" y=\"-958.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">ThMulBackward</text>\n",
       "</g>\n",
       "<!-- 4970039224&#45;&gt;4970039560 -->\n",
       "<g id=\"edge17\" class=\"edge\">\n",
       "<title>4970039224&#45;&gt;4970039560</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M839,-951.9883C839,-944.9098 839,-935.1714 839,-926.4779\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"842.5001,-926.3038 839,-916.3039 835.5001,-926.3039 842.5001,-926.3038\"/>\n",
       "</g>\n",
       "<!-- 4970040064 -->\n",
       "<g id=\"node19\" class=\"node\">\n",
       "<title>4970040064</title>\n",
       "<polygon fill=\"#d3d3d3\" stroke=\"#000000\" points=\"881.1492,-1028 796.8508,-1028 796.8508,-1008 881.1492,-1008 881.1492,-1028\"/>\n",
       "<text text-anchor=\"middle\" x=\"839\" y=\"-1014.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">MmBackward</text>\n",
       "</g>\n",
       "<!-- 4970040064&#45;&gt;4970039224 -->\n",
       "<g id=\"edge18\" class=\"edge\">\n",
       "<title>4970040064&#45;&gt;4970039224</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M839,-1007.9883C839,-1000.9098 839,-991.1714 839,-982.4779\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"842.5001,-982.3038 839,-972.3039 835.5001,-982.3039 842.5001,-982.3038\"/>\n",
       "</g>\n",
       "<!-- 4600563864 -->\n",
       "<g id=\"node20\" class=\"node\">\n",
       "<title>4600563864</title>\n",
       "<polygon fill=\"#d3d3d3\" stroke=\"#000000\" points=\"766.9453,-1084 605.0547,-1084 605.0547,-1064 766.9453,-1064 766.9453,-1084\"/>\n",
       "<text text-anchor=\"middle\" x=\"686\" y=\"-1070.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">ThresholdBackwardBackward</text>\n",
       "</g>\n",
       "<!-- 4600563864&#45;&gt;4970040064 -->\n",
       "<g id=\"edge19\" class=\"edge\">\n",
       "<title>4600563864&#45;&gt;4970040064</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M713.3533,-1063.9883C738.0798,-1054.9381 774.6855,-1041.5399 802.0158,-1031.5367\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"803.4382,-1034.7432 811.626,-1028.0192 801.0322,-1028.1697 803.4382,-1034.7432\"/>\n",
       "</g>\n",
       "<!-- 4970037992 -->\n",
       "<g id=\"node21\" class=\"node\">\n",
       "<title>4970037992</title>\n",
       "<polygon fill=\"#d3d3d3\" stroke=\"#000000\" points=\"728.9785,-1140 631.0215,-1140 631.0215,-1120 728.9785,-1120 728.9785,-1140\"/>\n",
       "<text text-anchor=\"middle\" x=\"680\" y=\"-1126.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">ThMulBackward</text>\n",
       "</g>\n",
       "<!-- 4970037992&#45;&gt;4600563864 -->\n",
       "<g id=\"edge20\" class=\"edge\">\n",
       "<title>4970037992&#45;&gt;4600563864</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M681.0727,-1119.9883C681.8311,-1112.9098 682.8745,-1103.1714 683.8059,-1094.4779\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"687.3107,-1094.6199 684.896,-1084.3039 680.3505,-1093.8741 687.3107,-1094.6199\"/>\n",
       "</g>\n",
       "<!-- 4970038496 -->\n",
       "<g id=\"node22\" class=\"node\">\n",
       "<title>4970038496</title>\n",
       "<polygon fill=\"#d3d3d3\" stroke=\"#000000\" points=\"722.1492,-1196 637.8508,-1196 637.8508,-1176 722.1492,-1176 722.1492,-1196\"/>\n",
       "<text text-anchor=\"middle\" x=\"680\" y=\"-1182.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">MmBackward</text>\n",
       "</g>\n",
       "<!-- 4970038496&#45;&gt;4970037992 -->\n",
       "<g id=\"edge21\" class=\"edge\">\n",
       "<title>4970038496&#45;&gt;4970037992</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M680,-1175.9883C680,-1168.9098 680,-1159.1714 680,-1150.4779\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"683.5001,-1150.3038 680,-1140.3039 676.5001,-1150.3039 683.5001,-1150.3038\"/>\n",
       "</g>\n",
       "<!-- 4970038328 -->\n",
       "<g id=\"node23\" class=\"node\">\n",
       "<title>4970038328</title>\n",
       "<polygon fill=\"#d3d3d3\" stroke=\"#000000\" points=\"639.9453,-1252 478.0547,-1252 478.0547,-1232 639.9453,-1232 639.9453,-1252\"/>\n",
       "<text text-anchor=\"middle\" x=\"559\" y=\"-1238.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">ThresholdBackwardBackward</text>\n",
       "</g>\n",
       "<!-- 4970038328&#45;&gt;4970038496 -->\n",
       "<g id=\"edge22\" class=\"edge\">\n",
       "<title>4970038328&#45;&gt;4970038496</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M580.6324,-1231.9883C599.6137,-1223.2036 627.4468,-1210.3222 648.8307,-1200.4255\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"650.5418,-1203.4903 658.1469,-1196.1138 647.6017,-1197.1376 650.5418,-1203.4903\"/>\n",
       "</g>\n",
       "<!-- 4970037544 -->\n",
       "<g id=\"node24\" class=\"node\">\n",
       "<title>4970037544</title>\n",
       "<polygon fill=\"#d3d3d3\" stroke=\"#000000\" points=\"384.1492,-1308 299.8508,-1308 299.8508,-1288 384.1492,-1288 384.1492,-1308\"/>\n",
       "<text text-anchor=\"middle\" x=\"342\" y=\"-1294.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">MmBackward</text>\n",
       "</g>\n",
       "<!-- 4970037544&#45;&gt;4970038328 -->\n",
       "<g id=\"edge23\" class=\"edge\">\n",
       "<title>4970037544&#45;&gt;4970038328</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M380.7953,-1287.9883C416.9414,-1278.6603 470.9865,-1264.7132 510.0774,-1254.6252\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"511.3672,-1257.9071 520.1754,-1252.0192 509.618,-1251.1291 511.3672,-1257.9071\"/>\n",
       "</g>\n",
       "<!-- 4970037936 -->\n",
       "<g id=\"node25\" class=\"node\">\n",
       "<title>4970037936</title>\n",
       "<polygon fill=\"#d3d3d3\" stroke=\"#000000\" points=\"374.9746,-1370 303.0254,-1370 303.0254,-1350 374.9746,-1350 374.9746,-1370\"/>\n",
       "<text text-anchor=\"middle\" x=\"339\" y=\"-1356.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">TBackward</text>\n",
       "</g>\n",
       "<!-- 4970037936&#45;&gt;4970037544 -->\n",
       "<g id=\"edge24\" class=\"edge\">\n",
       "<title>4970037936&#45;&gt;4970037544</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M339.4954,-1349.762C339.9104,-1341.185 340.5153,-1328.6836 341.0267,-1318.1154\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"344.5263,-1318.205 341.5138,-1308.0475 337.5345,-1317.8667 344.5263,-1318.205\"/>\n",
       "</g>\n",
       "<!-- 4970040120 -->\n",
       "<g id=\"node26\" class=\"node\">\n",
       "<title>4970040120</title>\n",
       "<polygon fill=\"#d3d3d3\" stroke=\"#000000\" points=\"307.9746,-1432 236.0254,-1432 236.0254,-1412 307.9746,-1412 307.9746,-1432\"/>\n",
       "<text text-anchor=\"middle\" x=\"272\" y=\"-1418.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">TBackward</text>\n",
       "</g>\n",
       "<!-- 4970040120&#45;&gt;4970037936 -->\n",
       "<g id=\"edge25\" class=\"edge\">\n",
       "<title>4970040120&#45;&gt;4970037936</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M283.0637,-1411.762C293.308,-1402.2821 308.733,-1388.0083 320.792,-1376.8492\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"323.1797,-1379.4083 328.1422,-1370.0475 318.4254,-1374.2706 323.1797,-1379.4083\"/>\n",
       "</g>\n",
       "<!-- 4810468656 -->\n",
       "<g id=\"node70\" class=\"node\">\n",
       "<title>4810468656</title>\n",
       "<polygon fill=\"#d3d3d3\" stroke=\"#000000\" points=\"320.809,-1252 203.191,-1252 203.191,-1232 320.809,-1232 320.809,-1252\"/>\n",
       "<text text-anchor=\"middle\" x=\"262\" y=\"-1238.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">ThAddmmBackward</text>\n",
       "</g>\n",
       "<!-- 4970040120&#45;&gt;4810468656 -->\n",
       "<g id=\"edge94\" class=\"edge\">\n",
       "<title>4970040120&#45;&gt;4810468656</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M271.4337,-1411.8067C269.8386,-1383.0949 265.2854,-1301.137 263.142,-1262.5555\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"266.6095,-1261.8719 262.5601,-1252.0815 259.6203,-1262.2603 266.6095,-1261.8719\"/>\n",
       "</g>\n",
       "<!-- 4970041184 -->\n",
       "<g id=\"node27\" class=\"node\">\n",
       "<title>4970041184</title>\n",
       "<polygon fill=\"#add8e6\" stroke=\"#000000\" points=\"322.1607,-1500 219.8393,-1500 219.8393,-1468 322.1607,-1468 322.1607,-1500\"/>\n",
       "<text text-anchor=\"middle\" x=\"271\" y=\"-1486.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">classifier.6.weight</text>\n",
       "<text text-anchor=\"middle\" x=\"271\" y=\"-1474.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\"> (1000, 4096)</text>\n",
       "</g>\n",
       "<!-- 4970041184&#45;&gt;4970040120 -->\n",
       "<g id=\"edge26\" class=\"edge\">\n",
       "<title>4970041184&#45;&gt;4970040120</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M271.2627,-1467.7102C271.3869,-1460.0144 271.5364,-1450.744 271.6682,-1442.5691\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"275.1712,-1442.4064 271.833,-1432.3512 268.1721,-1442.2934 275.1712,-1442.4064\"/>\n",
       "</g>\n",
       "<!-- 4970037320 -->\n",
       "<g id=\"node28\" class=\"node\">\n",
       "<title>4970037320</title>\n",
       "<polygon fill=\"#d3d3d3\" stroke=\"#000000\" points=\"618.8012,-1308 499.1988,-1308 499.1988,-1288 618.8012,-1288 618.8012,-1308\"/>\n",
       "<text text-anchor=\"middle\" x=\"559\" y=\"-1294.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">ThresholdBackward1</text>\n",
       "</g>\n",
       "<!-- 4970037320&#45;&gt;4970038328 -->\n",
       "<g id=\"edge27\" class=\"edge\">\n",
       "<title>4970037320&#45;&gt;4970038328</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M559,-1287.9883C559,-1280.9098 559,-1271.1714 559,-1262.4779\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"562.5001,-1262.3038 559,-1252.3039 555.5001,-1262.3039 562.5001,-1262.3038\"/>\n",
       "</g>\n",
       "<!-- 4970037320&#45;&gt;4810468656 -->\n",
       "<g id=\"edge93\" class=\"edge\">\n",
       "<title>4970037320&#45;&gt;4810468656</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M505.9024,-1287.9883C455.2725,-1278.442 378.985,-1264.0578 325.2362,-1253.9233\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"325.6132,-1250.4328 315.1378,-1252.0192 324.3161,-1257.3116 325.6132,-1250.4328\"/>\n",
       "</g>\n",
       "<!-- 4970040344 -->\n",
       "<g id=\"node29\" class=\"node\">\n",
       "<title>4970040344</title>\n",
       "<polygon fill=\"#d3d3d3\" stroke=\"#000000\" points=\"624.809,-1370 507.191,-1370 507.191,-1350 624.809,-1350 624.809,-1370\"/>\n",
       "<text text-anchor=\"middle\" x=\"566\" y=\"-1356.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">ThAddmmBackward</text>\n",
       "</g>\n",
       "<!-- 4970040344&#45;&gt;4970037320 -->\n",
       "<g id=\"edge28\" class=\"edge\">\n",
       "<title>4970040344&#45;&gt;4970037320</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M564.8441,-1349.762C563.8757,-1341.185 562.4643,-1328.6836 561.2711,-1318.1154\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"564.7343,-1317.5917 560.1344,-1308.0475 557.7785,-1318.3771 564.7343,-1317.5917\"/>\n",
       "</g>\n",
       "<!-- 4970040624 -->\n",
       "<g id=\"node30\" class=\"node\">\n",
       "<title>4970040624</title>\n",
       "<polygon fill=\"#d3d3d3\" stroke=\"#000000\" points=\"517.8012,-1432 416.1988,-1432 416.1988,-1412 517.8012,-1412 517.8012,-1432\"/>\n",
       "<text text-anchor=\"middle\" x=\"467\" y=\"-1418.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">ExpandBackward</text>\n",
       "</g>\n",
       "<!-- 4970040624&#45;&gt;4970040344 -->\n",
       "<g id=\"edge29\" class=\"edge\">\n",
       "<title>4970040624&#45;&gt;4970040344</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M483.3478,-1411.762C499.0617,-1401.921 523.0248,-1386.9138 541.1097,-1375.5878\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"543.3389,-1378.3216 549.9564,-1370.0475 539.6235,-1372.3889 543.3389,-1378.3216\"/>\n",
       "</g>\n",
       "<!-- 4970040008 -->\n",
       "<g id=\"node31\" class=\"node\">\n",
       "<title>4970040008</title>\n",
       "<polygon fill=\"#add8e6\" stroke=\"#000000\" points=\"506.8307,-1500 417.1693,-1500 417.1693,-1468 506.8307,-1468 506.8307,-1500\"/>\n",
       "<text text-anchor=\"middle\" x=\"462\" y=\"-1486.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">classifier.4.bias</text>\n",
       "<text text-anchor=\"middle\" x=\"462\" y=\"-1474.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\"> (4096)</text>\n",
       "</g>\n",
       "<!-- 4970040008&#45;&gt;4970040624 -->\n",
       "<g id=\"edge30\" class=\"edge\">\n",
       "<title>4970040008&#45;&gt;4970040624</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M463.3137,-1467.7102C463.9343,-1460.0144 464.6819,-1450.744 465.3412,-1442.5691\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"468.85,-1442.6002 466.1652,-1432.3512 461.8726,-1442.0375 468.85,-1442.6002\"/>\n",
       "</g>\n",
       "<!-- 4970096128 -->\n",
       "<g id=\"node32\" class=\"node\">\n",
       "<title>4970096128</title>\n",
       "<polygon fill=\"#d3d3d3\" stroke=\"#000000\" points=\"730.1412,-1432 625.8588,-1432 625.8588,-1412 730.1412,-1412 730.1412,-1432\"/>\n",
       "<text text-anchor=\"middle\" x=\"678\" y=\"-1418.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">DropoutBackward</text>\n",
       "</g>\n",
       "<!-- 4970096128&#45;&gt;4970040344 -->\n",
       "<g id=\"edge31\" class=\"edge\">\n",
       "<title>4970096128&#45;&gt;4970040344</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M659.5055,-1411.762C641.402,-1401.7404 613.6204,-1386.3613 593.0393,-1374.9682\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"594.5944,-1371.8286 584.1504,-1370.0475 591.2042,-1377.9529 594.5944,-1371.8286\"/>\n",
       "</g>\n",
       "<!-- 4970038720 -->\n",
       "<g id=\"node33\" class=\"node\">\n",
       "<title>4970038720</title>\n",
       "<polygon fill=\"#d3d3d3\" stroke=\"#000000\" points=\"764.8012,-1494 645.1988,-1494 645.1988,-1474 764.8012,-1474 764.8012,-1494\"/>\n",
       "<text text-anchor=\"middle\" x=\"705\" y=\"-1480.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">ThresholdBackward1</text>\n",
       "</g>\n",
       "<!-- 4970038720&#45;&gt;4600563864 -->\n",
       "<g id=\"edge68\" class=\"edge\">\n",
       "<title>4970038720&#45;&gt;4600563864</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M713.0877,-1473.8189C720.7456,-1463.7296 732.0735,-1447.5713 739,-1432 752.4509,-1401.7613 758,-1393.0954 758,-1360 758,-1360 758,-1360 758,-1298 758,-1218.3911 774.7065,-1190.6414 738,-1120 731.7912,-1108.0512 721.0982,-1097.7708 711.0174,-1089.9783\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"712.9585,-1087.0627 702.8053,-1084.0356 708.8547,-1092.7336 712.9585,-1087.0627\"/>\n",
       "</g>\n",
       "<!-- 4970038720&#45;&gt;4970096128 -->\n",
       "<g id=\"edge32\" class=\"edge\">\n",
       "<title>4970038720&#45;&gt;4970096128</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M700.5415,-1473.762C696.7277,-1465.0044 691.1322,-1452.1555 686.4704,-1441.4506\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"689.5772,-1439.8184 682.3755,-1432.0475 683.1593,-1442.6133 689.5772,-1439.8184\"/>\n",
       "</g>\n",
       "<!-- 4970039504 -->\n",
       "<g id=\"node34\" class=\"node\">\n",
       "<title>4970039504</title>\n",
       "<polygon fill=\"#d3d3d3\" stroke=\"#000000\" points=\"763.809,-1556 646.191,-1556 646.191,-1536 763.809,-1536 763.809,-1556\"/>\n",
       "<text text-anchor=\"middle\" x=\"705\" y=\"-1542.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">ThAddmmBackward</text>\n",
       "</g>\n",
       "<!-- 4970039504&#45;&gt;4970038720 -->\n",
       "<g id=\"edge33\" class=\"edge\">\n",
       "<title>4970039504&#45;&gt;4970038720</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M705,-1535.762C705,-1527.185 705,-1514.6836 705,-1504.1154\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"708.5001,-1504.0475 705,-1494.0475 701.5001,-1504.0476 708.5001,-1504.0475\"/>\n",
       "</g>\n",
       "<!-- 4970038048 -->\n",
       "<g id=\"node35\" class=\"node\">\n",
       "<title>4970038048</title>\n",
       "<polygon fill=\"#d3d3d3\" stroke=\"#000000\" points=\"755.8012,-1612 654.1988,-1612 654.1988,-1592 755.8012,-1592 755.8012,-1612\"/>\n",
       "<text text-anchor=\"middle\" x=\"705\" y=\"-1598.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">ExpandBackward</text>\n",
       "</g>\n",
       "<!-- 4970038048&#45;&gt;4970039504 -->\n",
       "<g id=\"edge34\" class=\"edge\">\n",
       "<title>4970038048&#45;&gt;4970039504</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M705,-1591.9883C705,-1584.9098 705,-1575.1714 705,-1566.4779\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"708.5001,-1566.3038 705,-1556.3039 701.5001,-1566.3039 708.5001,-1566.3038\"/>\n",
       "</g>\n",
       "<!-- 4970037712 -->\n",
       "<g id=\"node36\" class=\"node\">\n",
       "<title>4970037712</title>\n",
       "<polygon fill=\"#add8e6\" stroke=\"#000000\" points=\"736.8307,-1680 647.1693,-1680 647.1693,-1648 736.8307,-1648 736.8307,-1680\"/>\n",
       "<text text-anchor=\"middle\" x=\"692\" y=\"-1666.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">classifier.1.bias</text>\n",
       "<text text-anchor=\"middle\" x=\"692\" y=\"-1654.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\"> (4096)</text>\n",
       "</g>\n",
       "<!-- 4970037712&#45;&gt;4970038048 -->\n",
       "<g id=\"edge35\" class=\"edge\">\n",
       "<title>4970037712&#45;&gt;4970038048</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M695.4156,-1647.7102C697.0476,-1639.9269 699.0173,-1630.5331 700.7455,-1622.2908\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"704.2029,-1622.8567 702.8296,-1612.3512 697.3518,-1621.4201 704.2029,-1622.8567\"/>\n",
       "</g>\n",
       "<!-- 4970095640 -->\n",
       "<g id=\"node37\" class=\"node\">\n",
       "<title>4970095640</title>\n",
       "<polygon fill=\"#d3d3d3\" stroke=\"#000000\" points=\"636.1412,-1612 531.8588,-1612 531.8588,-1592 636.1412,-1592 636.1412,-1612\"/>\n",
       "<text text-anchor=\"middle\" x=\"584\" y=\"-1598.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">DropoutBackward</text>\n",
       "</g>\n",
       "<!-- 4970095640&#45;&gt;4970039504 -->\n",
       "<g id=\"edge36\" class=\"edge\">\n",
       "<title>4970095640&#45;&gt;4970039504</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M605.6324,-1591.9883C624.6137,-1583.2036 652.4468,-1570.3222 673.8307,-1560.4255\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"675.5418,-1563.4903 683.1469,-1556.1138 672.6017,-1557.1376 675.5418,-1563.4903\"/>\n",
       "</g>\n",
       "<!-- 4970038216 -->\n",
       "<g id=\"node38\" class=\"node\">\n",
       "<title>4970038216</title>\n",
       "<polygon fill=\"#d3d3d3\" stroke=\"#000000\" points=\"628.9219,-1674 539.0781,-1674 539.0781,-1654 628.9219,-1654 628.9219,-1674\"/>\n",
       "<text text-anchor=\"middle\" x=\"584\" y=\"-1660.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">ViewBackward</text>\n",
       "</g>\n",
       "<!-- 4970038216&#45;&gt;4970095640 -->\n",
       "<g id=\"edge37\" class=\"edge\">\n",
       "<title>4970038216&#45;&gt;4970095640</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M584,-1653.762C584,-1645.185 584,-1632.6836 584,-1622.1154\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"587.5001,-1622.0475 584,-1612.0475 580.5001,-1622.0476 587.5001,-1622.0475\"/>\n",
       "</g>\n",
       "<!-- 4970037880 -->\n",
       "<g id=\"node39\" class=\"node\">\n",
       "<title>4970037880</title>\n",
       "<polygon fill=\"#d3d3d3\" stroke=\"#000000\" points=\"705.987,-1736 524.013,-1736 524.013,-1716 705.987,-1716 705.987,-1736\"/>\n",
       "<text text-anchor=\"middle\" x=\"615\" y=\"-1722.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">MaxPool2DWithIndicesBackward</text>\n",
       "</g>\n",
       "<!-- 4970037880&#45;&gt;4970038216 -->\n",
       "<g id=\"edge38\" class=\"edge\">\n",
       "<title>4970037880&#45;&gt;4970038216</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M609.881,-1715.762C605.457,-1706.9141 598.9451,-1693.8901 593.5601,-1683.1202\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"596.6265,-1681.4265 589.0238,-1674.0475 590.3655,-1684.5571 596.6265,-1681.4265\"/>\n",
       "</g>\n",
       "<!-- 4970040288 -->\n",
       "<g id=\"node40\" class=\"node\">\n",
       "<title>4970040288</title>\n",
       "<polygon fill=\"#d3d3d3\" stroke=\"#000000\" points=\"824.8012,-1792 705.1988,-1792 705.1988,-1772 824.8012,-1772 824.8012,-1792\"/>\n",
       "<text text-anchor=\"middle\" x=\"765\" y=\"-1778.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">ThresholdBackward1</text>\n",
       "</g>\n",
       "<!-- 4970040288&#45;&gt;4970041128 -->\n",
       "<g id=\"edge72\" class=\"edge\">\n",
       "<title>4970040288&#45;&gt;4970041128</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M802.8449,-1771.9011C849.5878,-1756.8657 923,-1723.7421 923,-1664 923,-1664 923,-1664 923,-906 923,-875.4754 920.3027,-865.8065 904,-840 896.7579,-828.5361 885.8388,-818.2644 875.8729,-810.351\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"877.9191,-807.5104 867.8275,-804.284 873.7045,-813.0994 877.9191,-807.5104\"/>\n",
       "</g>\n",
       "<!-- 4970040288&#45;&gt;4970041240 -->\n",
       "<g id=\"edge71\" class=\"edge\">\n",
       "<title>4970040288&#45;&gt;4970041240</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M766.2926,-1771.8954C772.1595,-1725.1237 796,-1524.9206 796,-1360 796,-1360 796,-1360 796,-1298 796,-1127.9409 778.8909,-1085.9888 774,-916 773.7444,-907.1148 773.4239,-904.8702 774,-896 774.5526,-887.491 775.6744,-878.1522 776.8,-870.1254\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"780.2676,-870.6016 778.2843,-860.1941 773.3445,-869.5668 780.2676,-870.6016\"/>\n",
       "</g>\n",
       "<!-- 4970040288&#45;&gt;4970037880 -->\n",
       "<g id=\"edge39\" class=\"edge\">\n",
       "<title>4970040288&#45;&gt;4970037880</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M738.183,-1771.9883C713.9414,-1762.9381 678.0534,-1749.5399 651.259,-1739.5367\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"652.4299,-1736.2379 641.8373,-1736.0192 649.9815,-1742.7958 652.4299,-1736.2379\"/>\n",
       "</g>\n",
       "<!-- 4970039000 -->\n",
       "<g id=\"node41\" class=\"node\">\n",
       "<title>4970039000</title>\n",
       "<polygon fill=\"#d3d3d3\" stroke=\"#000000\" points=\"830.145,-1848 699.855,-1848 699.855,-1828 830.145,-1828 830.145,-1848\"/>\n",
       "<text text-anchor=\"middle\" x=\"765\" y=\"-1834.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">ThnnConv2DBackward</text>\n",
       "</g>\n",
       "<!-- 4970039000&#45;&gt;4970040288 -->\n",
       "<g id=\"edge40\" class=\"edge\">\n",
       "<title>4970039000&#45;&gt;4970040288</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M765,-1827.9883C765,-1820.9098 765,-1811.1714 765,-1802.4779\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"768.5001,-1802.3038 765,-1792.3039 761.5001,-1802.3039 768.5001,-1802.3038\"/>\n",
       "</g>\n",
       "<!-- 4810469216 -->\n",
       "<g id=\"node42\" class=\"node\">\n",
       "<title>4810469216</title>\n",
       "<polygon fill=\"#d3d3d3\" stroke=\"#000000\" points=\"1008.8012,-1910 889.1988,-1910 889.1988,-1890 1008.8012,-1890 1008.8012,-1910\"/>\n",
       "<text text-anchor=\"middle\" x=\"949\" y=\"-1896.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">ThresholdBackward1</text>\n",
       "</g>\n",
       "<!-- 4810469216&#45;&gt;4810468880 -->\n",
       "<g id=\"edge75\" class=\"edge\">\n",
       "<title>4810469216&#45;&gt;4810468880</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M956.5939,-1889.8802C970.5813,-1870.2033 999,-1824.9412 999,-1782 999,-1782 999,-1782 999,-794 999,-755.417 968.572,-719.3514 947.5552,-699.0425\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"949.7507,-696.3046 940.0442,-692.058 944.9839,-701.4308 949.7507,-696.3046\"/>\n",
       "</g>\n",
       "<!-- 4810469216&#45;&gt;4810467032 -->\n",
       "<g id=\"edge73\" class=\"edge\">\n",
       "<title>4810469216&#45;&gt;4810467032</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M950.7193,-1889.9331C954.0374,-1869.6013 961,-1822.154 961,-1782 961,-1782 961,-1782 961,-850 961,-819.4754 960.5558,-808.2371 942,-784 931.6172,-770.4383 915.9954,-760.0018 901.6101,-752.5245\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"902.9254,-749.2719 892.403,-748.035 899.8574,-755.5638 902.9254,-749.2719\"/>\n",
       "</g>\n",
       "<!-- 4810469216&#45;&gt;4970039000 -->\n",
       "<g id=\"edge41\" class=\"edge\">\n",
       "<title>4810469216&#45;&gt;4970039000</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M919.0277,-1889.9006C887.8859,-1879.4072 838.9952,-1862.9332 804.6682,-1851.3665\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"805.5536,-1847.9715 794.9595,-1848.095 803.3183,-1854.605 805.5536,-1847.9715\"/>\n",
       "</g>\n",
       "<!-- 4970039392 -->\n",
       "<g id=\"node43\" class=\"node\">\n",
       "<title>4970039392</title>\n",
       "<polygon fill=\"#d3d3d3\" stroke=\"#000000\" points=\"1047.145,-1972 916.855,-1972 916.855,-1952 1047.145,-1952 1047.145,-1972\"/>\n",
       "<text text-anchor=\"middle\" x=\"982\" y=\"-1958.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">ThnnConv2DBackward</text>\n",
       "</g>\n",
       "<!-- 4970039392&#45;&gt;4810469216 -->\n",
       "<g id=\"edge42\" class=\"edge\">\n",
       "<title>4970039392&#45;&gt;4810469216</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M976.5507,-1951.762C971.8414,-1942.9141 964.9093,-1929.8901 959.1769,-1919.1202\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"962.136,-1917.2305 954.3479,-1910.0475 955.9568,-1920.5195 962.136,-1917.2305\"/>\n",
       "</g>\n",
       "<!-- 4810469328 -->\n",
       "<g id=\"node44\" class=\"node\">\n",
       "<title>4810469328</title>\n",
       "<polygon fill=\"#d3d3d3\" stroke=\"#000000\" points=\"637.8012,-2034 518.1988,-2034 518.1988,-2014 637.8012,-2014 637.8012,-2034\"/>\n",
       "<text text-anchor=\"middle\" x=\"578\" y=\"-2020.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">ThresholdBackward1</text>\n",
       "</g>\n",
       "<!-- 4810469328&#45;&gt;4810466976 -->\n",
       "<g id=\"edge78\" class=\"edge\">\n",
       "<title>4810469328&#45;&gt;4810466976</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M518.0957,-2017.3542C450.5432,-2006.1469 350,-1976.5998 350,-1900 350,-1900 350,-1900 350,-1484 350,-1433.6776 371.2825,-1424.6889 384,-1376 399.376,-1317.1331 412,-1302.8418 412,-1242 412,-1242 412,-1242 412,-682 412,-650.0428 412,-612.9564 412,-590.4982\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"415.5001,-590.1921 412,-580.1921 408.5001,-590.1921 415.5001,-590.1921\"/>\n",
       "</g>\n",
       "<!-- 4810469328&#45;&gt;4810469104 -->\n",
       "<g id=\"edge76\" class=\"edge\">\n",
       "<title>4810469328&#45;&gt;4810469104</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M624.548,-2013.9346C734.8282,-1989.9062 1003.8819,-1930.143 1018,-1916 1043.2076,-1890.748 1037,-1873.6803 1037,-1838 1037,-1838 1037,-1838 1037,-738 1037,-707.4754 1034.7719,-697.504 1018,-672 1010.145,-660.0554 998.2308,-649.6519 987.4082,-641.7746\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"989.2802,-638.814 979.0611,-636.0176 985.3059,-644.5764 989.2802,-638.814\"/>\n",
       "</g>\n",
       "<!-- 4810469328&#45;&gt;4970039392 -->\n",
       "<g id=\"edge43\" class=\"edge\">\n",
       "<title>4810469328&#45;&gt;4970039392</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M638.0466,-2014.7849C709.4651,-2003.8247 828.7534,-1985.518 906.3948,-1973.6028\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"907.2393,-1977.0142 916.5927,-1972.0378 906.1774,-1970.0952 907.2393,-1977.0142\"/>\n",
       "</g>\n",
       "<!-- 4970039616 -->\n",
       "<g id=\"node45\" class=\"node\">\n",
       "<title>4970039616</title>\n",
       "<polygon fill=\"#d3d3d3\" stroke=\"#000000\" points=\"491.145,-2096 360.855,-2096 360.855,-2076 491.145,-2076 491.145,-2096\"/>\n",
       "<text text-anchor=\"middle\" x=\"426\" y=\"-2082.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">ThnnConv2DBackward</text>\n",
       "</g>\n",
       "<!-- 4970039616&#45;&gt;4810469328 -->\n",
       "<g id=\"edge44\" class=\"edge\">\n",
       "<title>4970039616&#45;&gt;4810469328</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M450.7597,-2075.9006C476.044,-2065.5873 515.4911,-2049.497 543.7597,-2037.9664\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"545.3134,-2041.1127 553.2509,-2034.095 542.6696,-2034.6312 545.3134,-2041.1127\"/>\n",
       "</g>\n",
       "<!-- 4810466808 -->\n",
       "<g id=\"node46\" class=\"node\">\n",
       "<title>4810466808</title>\n",
       "<polygon fill=\"#d3d3d3\" stroke=\"#000000\" points=\"478.987,-2158 297.013,-2158 297.013,-2138 478.987,-2138 478.987,-2158\"/>\n",
       "<text text-anchor=\"middle\" x=\"388\" y=\"-2144.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">MaxPool2DWithIndicesBackward</text>\n",
       "</g>\n",
       "<!-- 4810466808&#45;&gt;4810466864 -->\n",
       "<g id=\"edge79\" class=\"edge\">\n",
       "<title>4810466808&#45;&gt;4810466864</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M344.8861,-2137.9483C267.7239,-2118.6963 114,-2074.0601 114,-2024 114,-2024 114,-2024 114,-738 114,-638.8794 223.4401,-560.9041 275.7629,-529.2886\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"277.7086,-532.2042 284.5338,-524.1007 274.1448,-526.1793 277.7086,-532.2042\"/>\n",
       "</g>\n",
       "<!-- 4810466808&#45;&gt;4970039616 -->\n",
       "<g id=\"edge45\" class=\"edge\">\n",
       "<title>4810466808&#45;&gt;4970039616</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M394.2749,-2137.762C399.7531,-2128.8238 407.8434,-2115.6239 414.4828,-2104.7912\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"417.6003,-2106.4025 419.8418,-2096.0475 411.6321,-2102.7446 417.6003,-2106.4025\"/>\n",
       "</g>\n",
       "<!-- 4810468712 -->\n",
       "<g id=\"node47\" class=\"node\">\n",
       "<title>4810468712</title>\n",
       "<polygon fill=\"#d3d3d3\" stroke=\"#000000\" points=\"1124.8012,-2220 1005.1988,-2220 1005.1988,-2200 1124.8012,-2200 1124.8012,-2220\"/>\n",
       "<text text-anchor=\"middle\" x=\"1065\" y=\"-2206.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">ThresholdBackward1</text>\n",
       "</g>\n",
       "<!-- 4810468712&#45;&gt;4810466696 -->\n",
       "<g id=\"edge82\" class=\"edge\">\n",
       "<title>4810468712&#45;&gt;4810466696</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M1078.8174,-2199.7551C1103.1276,-2180.4359 1151,-2136.0393 1151,-2086 1151,-2086 1151,-2086 1151,-514 1151,-483.4754 1151.3054,-471.6443 1132,-448 1120.2427,-433.6002 1102.776,-423.1696 1086.4961,-415.9194\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1087.6773,-412.6201 1077.1,-412.0121 1084.9895,-419.0835 1087.6773,-412.6201\"/>\n",
       "</g>\n",
       "<!-- 4810468712&#45;&gt;4810467424 -->\n",
       "<g id=\"edge81\" class=\"edge\">\n",
       "<title>4810468712&#45;&gt;4810467424</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M1071.8772,-2199.9013C1085.1497,-2179.399 1113,-2131.0146 1113,-2086 1113,-2086 1113,-2086 1113,-570 1113,-525.4856 1068.3097,-491.3837 1037.2407,-473.0544\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1038.9192,-469.9827 1028.4943,-468.0926 1035.4652,-476.0712 1038.9192,-469.9827\"/>\n",
       "</g>\n",
       "<!-- 4810468712&#45;&gt;4810466808 -->\n",
       "<g id=\"edge46\" class=\"edge\">\n",
       "<title>4810468712&#45;&gt;4810466808</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M1005.3489,-2206.8467C900.322,-2200.9394 675.6419,-2186.7126 487,-2164 476.6515,-2162.754 465.729,-2161.2158 455.0668,-2159.5901\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"455.3715,-2156.0954 444.9514,-2158.0114 454.292,-2163.0117 455.3715,-2156.0954\"/>\n",
       "</g>\n",
       "<!-- 4970040568 -->\n",
       "<g id=\"node48\" class=\"node\">\n",
       "<title>4970040568</title>\n",
       "<polygon fill=\"#d3d3d3\" stroke=\"#000000\" points=\"1058.145,-2276 927.855,-2276 927.855,-2256 1058.145,-2256 1058.145,-2276\"/>\n",
       "<text text-anchor=\"middle\" x=\"993\" y=\"-2262.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">ThnnConv2DBackward</text>\n",
       "</g>\n",
       "<!-- 4970040568&#45;&gt;4810468712 -->\n",
       "<g id=\"edge47\" class=\"edge\">\n",
       "<title>4970040568&#45;&gt;4810468712</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M1005.8722,-2255.9883C1016.3888,-2247.8087 1031.472,-2236.0773 1043.7719,-2226.5108\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1046.0074,-2229.2061 1051.7522,-2220.3039 1041.7098,-2223.6806 1046.0074,-2229.2061\"/>\n",
       "</g>\n",
       "<!-- 4810468376 -->\n",
       "<g id=\"node49\" class=\"node\">\n",
       "<title>4810468376</title>\n",
       "<polygon fill=\"#d3d3d3\" stroke=\"#000000\" points=\"1198.987,-2338 1017.013,-2338 1017.013,-2318 1198.987,-2318 1198.987,-2338\"/>\n",
       "<text text-anchor=\"middle\" x=\"1108\" y=\"-2324.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">MaxPool2DWithIndicesBackward</text>\n",
       "</g>\n",
       "<!-- 4810468376&#45;&gt;4810468320 -->\n",
       "<g id=\"edge83\" class=\"edge\">\n",
       "<title>4810468376&#45;&gt;4810468320</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M1121.3755,-2317.9288C1144.3818,-2299.3429 1189,-2257.2116 1189,-2210 1189,-2210 1189,-2210 1189,-458 1189,-406.6525 1131.3454,-375.5281 1088.7467,-359.5394\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1089.695,-356.1612 1079.1005,-356.0762 1087.3296,-362.7494 1089.695,-356.1612\"/>\n",
       "</g>\n",
       "<!-- 4810468376&#45;&gt;4970040568 -->\n",
       "<g id=\"edge48\" class=\"edge\">\n",
       "<title>4810468376&#45;&gt;4970040568</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M1089.2673,-2317.9006C1070.639,-2307.8576 1041.8495,-2292.3362 1020.6037,-2280.882\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1022.1879,-2277.7599 1011.7247,-2276.095 1018.866,-2283.9214 1022.1879,-2277.7599\"/>\n",
       "</g>\n",
       "<!-- 4810468040 -->\n",
       "<g id=\"node50\" class=\"node\">\n",
       "<title>4810468040</title>\n",
       "<polygon fill=\"#d3d3d3\" stroke=\"#000000\" points=\"1183.8012,-2400 1064.1988,-2400 1064.1988,-2380 1183.8012,-2380 1183.8012,-2400\"/>\n",
       "<text text-anchor=\"middle\" x=\"1124\" y=\"-2386.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">ThresholdBackward1</text>\n",
       "</g>\n",
       "<!-- 4810468040&#45;&gt;4810467704 -->\n",
       "<g id=\"edge86\" class=\"edge\">\n",
       "<title>4810468040&#45;&gt;4810467704</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M1155.8692,-2379.8345C1173.7179,-2372.5656 1194.9453,-2361.0181 1208,-2344 1229.7169,-2315.6899 1227,-2301.6803 1227,-2266 1227,-2266 1227,-2266 1227,-346 1227,-307.417 1196.572,-271.3514 1175.5552,-251.0425\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1177.7507,-248.3046 1168.0442,-244.058 1172.9839,-253.4308 1177.7507,-248.3046\"/>\n",
       "</g>\n",
       "<!-- 4810468040&#45;&gt;4810467816 -->\n",
       "<g id=\"edge85\" class=\"edge\">\n",
       "<title>4810468040&#45;&gt;4810467816</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M1064.017,-2388.7387C940.198,-2385.5699 647.5142,-2375.1586 404,-2344 222.6074,-2320.7901 0,-2448.8715 0,-2266 0,-2266 0,-2266 0,-570 0,-381.7388 641.3265,-316.5818 918.6729,-297.2063\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"919.051,-300.6886 928.7865,-296.5088 918.5694,-293.7052 919.051,-300.6886\"/>\n",
       "</g>\n",
       "<!-- 4810468040&#45;&gt;4810468376 -->\n",
       "<g id=\"edge49\" class=\"edge\">\n",
       "<title>4810468040&#45;&gt;4810468376</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M1121.3579,-2379.762C1119.1212,-2371.0947 1115.8503,-2358.42 1113.1051,-2347.7823\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1116.4807,-2346.8557 1110.5929,-2338.0475 1109.7028,-2348.6049 1116.4807,-2346.8557\"/>\n",
       "</g>\n",
       "<!-- 4970039952 -->\n",
       "<g id=\"node51\" class=\"node\">\n",
       "<title>4970039952</title>\n",
       "<polygon fill=\"#d3d3d3\" stroke=\"#000000\" points=\"1221.145,-2456 1090.855,-2456 1090.855,-2436 1221.145,-2436 1221.145,-2456\"/>\n",
       "<text text-anchor=\"middle\" x=\"1156\" y=\"-2442.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">ThnnConv2DBackward</text>\n",
       "</g>\n",
       "<!-- 4970039952&#45;&gt;4810468040 -->\n",
       "<g id=\"edge50\" class=\"edge\">\n",
       "<title>4970039952&#45;&gt;4810468040</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M1150.279,-2435.9883C1146.0095,-2428.5165 1140.0464,-2418.0812 1134.8796,-2409.0393\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1137.8882,-2407.2498 1129.8879,-2400.3039 1131.8105,-2410.7228 1137.8882,-2407.2498\"/>\n",
       "</g>\n",
       "<!-- 4810467984 -->\n",
       "<g id=\"node52\" class=\"node\">\n",
       "<title>4810467984</title>\n",
       "<polygon fill=\"#add8e6\" stroke=\"#000000\" points=\"1349.4922,-2524 1256.5078,-2524 1256.5078,-2492 1349.4922,-2492 1349.4922,-2524\"/>\n",
       "<text text-anchor=\"middle\" x=\"1303\" y=\"-2510.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">x</text>\n",
       "<text text-anchor=\"middle\" x=\"1303\" y=\"-2498.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\"> (1, 3, 227, 227)</text>\n",
       "</g>\n",
       "<!-- 4810467984&#45;&gt;4810466248 -->\n",
       "<g id=\"edge87\" class=\"edge\">\n",
       "<title>4810467984&#45;&gt;4810466248</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M1303,-2491.9677C1303,-2469.2692 1303,-2426.4653 1303,-2390 1303,-2390 1303,-2390 1303,-290 1303,-247.8092 1263.4551,-213.0184 1235.9385,-193.9168\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1237.7525,-190.9195 1227.4926,-188.2768 1233.8651,-196.7409 1237.7525,-190.9195\"/>\n",
       "</g>\n",
       "<!-- 4810467984&#45;&gt;4970039952 -->\n",
       "<g id=\"edge51\" class=\"edge\">\n",
       "<title>4810467984&#45;&gt;4970039952</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M1264.7612,-2491.8721C1241.3446,-2481.9957 1211.6701,-2469.4799 1189.2349,-2460.0174\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1190.3684,-2456.697 1179.7942,-2456.0357 1187.6481,-2463.1468 1190.3684,-2456.697\"/>\n",
       "</g>\n",
       "<!-- 4810467592 -->\n",
       "<g id=\"node53\" class=\"node\">\n",
       "<title>4810467592</title>\n",
       "<polygon fill=\"#add8e6\" stroke=\"#000000\" points=\"1238.1354,-2524 1139.8646,-2524 1139.8646,-2492 1238.1354,-2492 1238.1354,-2524\"/>\n",
       "<text text-anchor=\"middle\" x=\"1189\" y=\"-2510.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">features.0.weight</text>\n",
       "<text text-anchor=\"middle\" x=\"1189\" y=\"-2498.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\"> (64, 3, 11, 11)</text>\n",
       "</g>\n",
       "<!-- 4810467592&#45;&gt;4810466248 -->\n",
       "<g id=\"edge88\" class=\"edge\">\n",
       "<title>4810467592&#45;&gt;4810466248</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M1208.178,-2491.7611C1230.6924,-2470.7719 1265,-2431.9121 1265,-2390 1265,-2390 1265,-2390 1265,-290 1265,-259.4754 1259.8833,-251.1847 1246,-224 1240.8651,-213.9455 1233.3881,-203.9365 1226.5771,-195.835\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1229.0717,-193.3712 1219.8466,-188.1608 1223.809,-197.9868 1229.0717,-193.3712\"/>\n",
       "</g>\n",
       "<!-- 4810467592&#45;&gt;4970039952 -->\n",
       "<g id=\"edge52\" class=\"edge\">\n",
       "<title>4810467592&#45;&gt;4970039952</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M1180.3296,-2491.7102C1176.0007,-2483.5771 1170.7358,-2473.6855 1166.2123,-2465.1867\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1169.2977,-2463.5342 1161.5095,-2456.3512 1163.1184,-2466.8232 1169.2977,-2463.5342\"/>\n",
       "</g>\n",
       "<!-- 4970090568 -->\n",
       "<g id=\"node54\" class=\"node\">\n",
       "<title>4970090568</title>\n",
       "<polygon fill=\"#add8e6\" stroke=\"#000000\" points=\"1121.8052,-2524 1036.1948,-2524 1036.1948,-2492 1121.8052,-2492 1121.8052,-2524\"/>\n",
       "<text text-anchor=\"middle\" x=\"1079\" y=\"-2510.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">features.0.bias</text>\n",
       "<text text-anchor=\"middle\" x=\"1079\" y=\"-2498.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\"> (64)</text>\n",
       "</g>\n",
       "<!-- 4970090568&#45;&gt;4970039952 -->\n",
       "<g id=\"edge53\" class=\"edge\">\n",
       "<title>4970090568&#45;&gt;4970039952</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M1099.2308,-2491.7102C1110.3091,-2482.7901 1124.0146,-2471.7545 1135.1835,-2462.7613\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1137.5506,-2465.349 1143.1444,-2456.3512 1133.1604,-2459.8967 1137.5506,-2465.349\"/>\n",
       "</g>\n",
       "<!-- 4810466752 -->\n",
       "<g id=\"node55\" class=\"node\">\n",
       "<title>4810466752</title>\n",
       "<polygon fill=\"#add8e6\" stroke=\"#000000\" points=\"511.1354,-2344 412.8646,-2344 412.8646,-2312 511.1354,-2312 511.1354,-2344\"/>\n",
       "<text text-anchor=\"middle\" x=\"462\" y=\"-2330.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">features.3.weight</text>\n",
       "<text text-anchor=\"middle\" x=\"462\" y=\"-2318.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\"> (192, 64, 5, 5)</text>\n",
       "</g>\n",
       "<!-- 4810466752&#45;&gt;4810468320 -->\n",
       "<g id=\"edge84\" class=\"edge\">\n",
       "<title>4810466752&#45;&gt;4810468320</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M412.532,-2324.6999C300.6855,-2315.8832 38,-2286.9817 38,-2210 38,-2210 38,-2210 38,-626 38,-439.2522 689.5291,-371.7397 943.9345,-352.4961\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"944.5704,-355.9585 954.282,-351.7235 944.0491,-348.9779 944.5704,-355.9585\"/>\n",
       "</g>\n",
       "<!-- 4810466752&#45;&gt;4970040568 -->\n",
       "<g id=\"edge54\" class=\"edge\">\n",
       "<title>4810466752&#45;&gt;4970040568</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M511.4784,-2322.2229C604.7295,-2311.3348 807.0134,-2287.716 917.4035,-2274.8267\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"917.9407,-2278.2879 927.4673,-2273.6516 917.1289,-2271.3351 917.9407,-2278.2879\"/>\n",
       "</g>\n",
       "<!-- 4970040848 -->\n",
       "<g id=\"node56\" class=\"node\">\n",
       "<title>4970040848</title>\n",
       "<polygon fill=\"#add8e6\" stroke=\"#000000\" points=\"999.8052,-2344 914.1948,-2344 914.1948,-2312 999.8052,-2312 999.8052,-2344\"/>\n",
       "<text text-anchor=\"middle\" x=\"957\" y=\"-2330.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">features.3.bias</text>\n",
       "<text text-anchor=\"middle\" x=\"957\" y=\"-2318.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\"> (192)</text>\n",
       "</g>\n",
       "<!-- 4970040848&#45;&gt;4970040568 -->\n",
       "<g id=\"edge55\" class=\"edge\">\n",
       "<title>4970040848&#45;&gt;4970040568</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M966.4586,-2311.7102C971.181,-2303.5771 976.9246,-2293.6855 981.8593,-2285.1867\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"984.995,-2286.7566 986.9896,-2276.3512 978.9414,-2283.2416 984.995,-2286.7566\"/>\n",
       "</g>\n",
       "<!-- 4810469048 -->\n",
       "<g id=\"node57\" class=\"node\">\n",
       "<title>4810469048</title>\n",
       "<polygon fill=\"#add8e6\" stroke=\"#000000\" points=\"203.1354,-2164 104.8646,-2164 104.8646,-2132 203.1354,-2132 203.1354,-2164\"/>\n",
       "<text text-anchor=\"middle\" x=\"154\" y=\"-2150.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">features.6.weight</text>\n",
       "<text text-anchor=\"middle\" x=\"154\" y=\"-2138.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\"> (384, 192, 3, 3)</text>\n",
       "</g>\n",
       "<!-- 4810469048&#45;&gt;4810466864 -->\n",
       "<g id=\"edge80\" class=\"edge\">\n",
       "<title>4810469048&#45;&gt;4810466864</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M135.5151,-2131.981C112.4494,-2110.11 76,-2068.5072 76,-2024 76,-2024 76,-2024 76,-682 76,-594.7929 185.584,-547.6672 252.4738,-527.0248\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"253.8378,-530.2702 262.4169,-524.0533 251.8335,-523.5633 253.8378,-530.2702\"/>\n",
       "</g>\n",
       "<!-- 4810469048&#45;&gt;4970039616 -->\n",
       "<g id=\"edge56\" class=\"edge\">\n",
       "<title>4810469048&#45;&gt;4970039616</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M203.2571,-2136.7723C250.9509,-2125.9009 323.0717,-2109.4616 372.2353,-2098.2552\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"373.0821,-2101.652 382.0541,-2096.0171 371.5263,-2094.8271 373.0821,-2101.652\"/>\n",
       "</g>\n",
       "<!-- 4970040456 -->\n",
       "<g id=\"node58\" class=\"node\">\n",
       "<title>4970040456</title>\n",
       "<polygon fill=\"#add8e6\" stroke=\"#000000\" points=\"581.8052,-2164 496.1948,-2164 496.1948,-2132 581.8052,-2132 581.8052,-2164\"/>\n",
       "<text text-anchor=\"middle\" x=\"539\" y=\"-2150.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">features.6.bias</text>\n",
       "<text text-anchor=\"middle\" x=\"539\" y=\"-2138.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\"> (384)</text>\n",
       "</g>\n",
       "<!-- 4970040456&#45;&gt;4970039616 -->\n",
       "<g id=\"edge57\" class=\"edge\">\n",
       "<title>4970040456&#45;&gt;4970039616</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M509.6056,-2131.8721C492.1701,-2122.3057 470.2213,-2110.263 453.1881,-2100.9174\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"454.7415,-2097.7775 444.2908,-2096.0357 451.3743,-2103.9144 454.7415,-2097.7775\"/>\n",
       "</g>\n",
       "<!-- 4810469160 -->\n",
       "<g id=\"node59\" class=\"node\">\n",
       "<title>4810469160</title>\n",
       "<polygon fill=\"#add8e6\" stroke=\"#000000\" points=\"1085.1354,-2040 986.8646,-2040 986.8646,-2008 1085.1354,-2008 1085.1354,-2040\"/>\n",
       "<text text-anchor=\"middle\" x=\"1036\" y=\"-2026.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">features.8.weight</text>\n",
       "<text text-anchor=\"middle\" x=\"1036\" y=\"-2014.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\"> (256, 384, 3, 3)</text>\n",
       "</g>\n",
       "<!-- 4810469160&#45;&gt;4810469104 -->\n",
       "<g id=\"edge77\" class=\"edge\">\n",
       "<title>4810469160&#45;&gt;4810469104</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M1044.6554,-2007.6679C1056.1451,-1984.4276 1075,-1940.17 1075,-1900 1075,-1900 1075,-1900 1075,-738 1075,-706.7875 1071.6107,-695.4397 1051,-672 1038.5187,-657.8055 1020.5962,-647.3729 1004.1012,-640.0662\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1005.1837,-636.7259 994.6061,-636.1206 1002.4975,-643.19 1005.1837,-636.7259\"/>\n",
       "</g>\n",
       "<!-- 4810469160&#45;&gt;4970039392 -->\n",
       "<g id=\"edge58\" class=\"edge\">\n",
       "<title>4810469160&#45;&gt;4970039392</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M1021.8121,-2007.7102C1014.4238,-1999.2273 1005.3692,-1988.8313 997.7629,-1980.0981\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1000.2227,-1977.5933 991.0156,-1972.3512 994.9441,-1982.1908 1000.2227,-1977.5933\"/>\n",
       "</g>\n",
       "<!-- 4970039840 -->\n",
       "<g id=\"node60\" class=\"node\">\n",
       "<title>4970039840</title>\n",
       "<polygon fill=\"#add8e6\" stroke=\"#000000\" points=\"968.8052,-2040 883.1948,-2040 883.1948,-2008 968.8052,-2008 968.8052,-2040\"/>\n",
       "<text text-anchor=\"middle\" x=\"926\" y=\"-2026.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">features.8.bias</text>\n",
       "<text text-anchor=\"middle\" x=\"926\" y=\"-2014.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\"> (256)</text>\n",
       "</g>\n",
       "<!-- 4970039840&#45;&gt;4970039392 -->\n",
       "<g id=\"edge59\" class=\"edge\">\n",
       "<title>4970039840&#45;&gt;4970039392</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M940.7133,-2007.7102C948.4543,-1999.1399 957.9591,-1988.6168 965.8968,-1979.8285\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"968.545,-1982.1183 972.6505,-1972.3512 963.3502,-1977.4263 968.545,-1982.1183\"/>\n",
       "</g>\n",
       "<!-- 4970037600 -->\n",
       "<g id=\"node61\" class=\"node\">\n",
       "<title>4970037600</title>\n",
       "<polygon fill=\"#add8e6\" stroke=\"#000000\" points=\"592.1354,-1916 487.8646,-1916 487.8646,-1884 592.1354,-1884 592.1354,-1916\"/>\n",
       "<text text-anchor=\"middle\" x=\"540\" y=\"-1902.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">features.10.weight</text>\n",
       "<text text-anchor=\"middle\" x=\"540\" y=\"-1890.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\"> (256, 256, 3, 3)</text>\n",
       "</g>\n",
       "<!-- 4970037600&#45;&gt;4810467032 -->\n",
       "<g id=\"edge74\" class=\"edge\">\n",
       "<title>4970037600&#45;&gt;4810467032</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M488.5397,-1883.9072C444.4434,-1866.5331 388,-1834.1155 388,-1782 388,-1782 388,-1782 388,-1484 388,-1450.9046 396.7638,-1443.4727 407,-1412 412.3097,-1395.6745 416.0579,-1392.4406 421,-1376 438.5417,-1317.6453 450,-1302.9343 450,-1242 450,-1242 450,-1242 450,-850 450,-784.7989 642.9525,-756.4594 766.0372,-745.0144\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"766.6033,-748.4775 776.2464,-744.0888 765.9711,-741.5061 766.6033,-748.4775\"/>\n",
       "</g>\n",
       "<!-- 4970037600&#45;&gt;4970039000 -->\n",
       "<g id=\"edge60\" class=\"edge\">\n",
       "<title>4970037600&#45;&gt;4970039000</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M592.1851,-1885.6201C630.3944,-1875.0913 681.7828,-1860.931 718.5712,-1850.7937\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"719.8263,-1854.0784 728.5372,-1848.0475 717.9667,-1847.3299 719.8263,-1854.0784\"/>\n",
       "</g>\n",
       "<!-- 4970039168 -->\n",
       "<g id=\"node62\" class=\"node\">\n",
       "<title>4970039168</title>\n",
       "<polygon fill=\"#add8e6\" stroke=\"#000000\" points=\"810.8052,-1916 719.1948,-1916 719.1948,-1884 810.8052,-1884 810.8052,-1916\"/>\n",
       "<text text-anchor=\"middle\" x=\"765\" y=\"-1902.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">features.10.bias</text>\n",
       "<text text-anchor=\"middle\" x=\"765\" y=\"-1890.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\"> (256)</text>\n",
       "</g>\n",
       "<!-- 4970039168&#45;&gt;4970039000 -->\n",
       "<g id=\"edge61\" class=\"edge\">\n",
       "<title>4970039168&#45;&gt;4970039000</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M765,-1883.7102C765,-1876.0144 765,-1866.744 765,-1858.5691\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"768.5001,-1858.3512 765,-1848.3512 761.5001,-1858.3513 768.5001,-1858.3512\"/>\n",
       "</g>\n",
       "<!-- 4970037768 -->\n",
       "<g id=\"node63\" class=\"node\">\n",
       "<title>4970037768</title>\n",
       "<polygon fill=\"#d3d3d3\" stroke=\"#000000\" points=\"883.9746,-1612 812.0254,-1612 812.0254,-1592 883.9746,-1592 883.9746,-1612\"/>\n",
       "<text text-anchor=\"middle\" x=\"848\" y=\"-1598.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">TBackward</text>\n",
       "</g>\n",
       "<!-- 4970037768&#45;&gt;4970039504 -->\n",
       "<g id=\"edge62\" class=\"edge\">\n",
       "<title>4970037768&#45;&gt;4970039504</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M822.4345,-1591.9883C799.4967,-1583.0057 765.6214,-1569.7398 740.1407,-1559.7614\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"741.4141,-1556.5013 730.8263,-1556.1138 738.8615,-1563.0193 741.4141,-1556.5013\"/>\n",
       "</g>\n",
       "<!-- 4970038384 -->\n",
       "<g id=\"node68\" class=\"node\">\n",
       "<title>4970038384</title>\n",
       "<polygon fill=\"#d3d3d3\" stroke=\"#000000\" points=\"894.9746,-1196 823.0254,-1196 823.0254,-1176 894.9746,-1176 894.9746,-1196\"/>\n",
       "<text text-anchor=\"middle\" x=\"859\" y=\"-1182.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">TBackward</text>\n",
       "</g>\n",
       "<!-- 4970037768&#45;&gt;4970038384 -->\n",
       "<g id=\"edge70\" class=\"edge\">\n",
       "<title>4970037768&#45;&gt;4970038384</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M848.7164,-1591.9041C850.0989,-1571.5202 853,-1523.9838 853,-1484 853,-1484 853,-1484 853,-1298 853,-1265.8501 855.4957,-1228.5833 857.2679,-1206.1727\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"860.7581,-1206.4354 858.0887,-1196.1823 853.7816,-1205.8621 860.7581,-1206.4354\"/>\n",
       "</g>\n",
       "<!-- 4970038552 -->\n",
       "<g id=\"node64\" class=\"node\">\n",
       "<title>4970038552</title>\n",
       "<polygon fill=\"#add8e6\" stroke=\"#000000\" points=\"895.1607,-1680 792.8393,-1680 792.8393,-1648 895.1607,-1648 895.1607,-1680\"/>\n",
       "<text text-anchor=\"middle\" x=\"844\" y=\"-1666.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">classifier.1.weight</text>\n",
       "<text text-anchor=\"middle\" x=\"844\" y=\"-1654.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\"> (4096, 9216)</text>\n",
       "</g>\n",
       "<!-- 4970038552&#45;&gt;4970037768 -->\n",
       "<g id=\"edge63\" class=\"edge\">\n",
       "<title>4970038552&#45;&gt;4970037768</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M845.051,-1647.7102C845.5475,-1640.0144 846.1455,-1630.744 846.673,-1622.5691\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"850.181,-1622.5558 847.3322,-1612.3512 843.1955,-1622.1051 850.181,-1622.5558\"/>\n",
       "</g>\n",
       "<!-- 4970040960 -->\n",
       "<g id=\"node65\" class=\"node\">\n",
       "<title>4970040960</title>\n",
       "<polygon fill=\"#d3d3d3\" stroke=\"#000000\" points=\"607.9746,-1432 536.0254,-1432 536.0254,-1412 607.9746,-1412 607.9746,-1432\"/>\n",
       "<text text-anchor=\"middle\" x=\"572\" y=\"-1418.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">TBackward</text>\n",
       "</g>\n",
       "<!-- 4970040960&#45;&gt;4970040344 -->\n",
       "<g id=\"edge64\" class=\"edge\">\n",
       "<title>4970040960&#45;&gt;4970040344</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M571.0092,-1411.762C570.1792,-1403.185 568.9694,-1390.6836 567.9467,-1380.1154\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"571.4194,-1379.6639 566.9723,-1370.0475 564.4519,-1380.3382 571.4194,-1379.6639\"/>\n",
       "</g>\n",
       "<!-- 4970038160 -->\n",
       "<g id=\"node67\" class=\"node\">\n",
       "<title>4970038160</title>\n",
       "<polygon fill=\"#d3d3d3\" stroke=\"#000000\" points=\"729.9746,-1252 658.0254,-1252 658.0254,-1232 729.9746,-1232 729.9746,-1252\"/>\n",
       "<text text-anchor=\"middle\" x=\"694\" y=\"-1238.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">TBackward</text>\n",
       "</g>\n",
       "<!-- 4970040960&#45;&gt;4970038160 -->\n",
       "<g id=\"edge67\" class=\"edge\">\n",
       "<title>4970040960&#45;&gt;4970038160</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M590.2773,-1411.9303C603.8069,-1403.7207 621.9132,-1391.0469 634,-1376 662.4662,-1340.5623 680.3882,-1289.4733 688.6431,-1261.7342\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"692.0385,-1262.5901 691.4214,-1252.0132 685.308,-1260.6664 692.0385,-1262.5901\"/>\n",
       "</g>\n",
       "<!-- 4970041072 -->\n",
       "<g id=\"node66\" class=\"node\">\n",
       "<title>4970041072</title>\n",
       "<polygon fill=\"#add8e6\" stroke=\"#000000\" points=\"627.1607,-1500 524.8393,-1500 524.8393,-1468 627.1607,-1468 627.1607,-1500\"/>\n",
       "<text text-anchor=\"middle\" x=\"576\" y=\"-1486.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">classifier.4.weight</text>\n",
       "<text text-anchor=\"middle\" x=\"576\" y=\"-1474.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\"> (4096, 4096)</text>\n",
       "</g>\n",
       "<!-- 4970041072&#45;&gt;4970040960 -->\n",
       "<g id=\"edge65\" class=\"edge\">\n",
       "<title>4970041072&#45;&gt;4970040960</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M574.949,-1467.7102C574.4525,-1460.0144 573.8545,-1450.744 573.327,-1442.5691\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"576.8045,-1442.1051 572.6678,-1432.3512 569.819,-1442.5558 576.8045,-1442.1051\"/>\n",
       "</g>\n",
       "<!-- 4970038160&#45;&gt;4970038496 -->\n",
       "<g id=\"edge66\" class=\"edge\">\n",
       "<title>4970038160&#45;&gt;4970038496</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M691.4971,-1231.9883C689.7078,-1224.8311 687.2387,-1214.9547 685.0471,-1206.1885\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"688.3969,-1205.1564 682.576,-1196.3039 681.6059,-1206.8542 688.3969,-1205.1564\"/>\n",
       "</g>\n",
       "<!-- 4970038384&#45;&gt;4970040064 -->\n",
       "<g id=\"edge69\" class=\"edge\">\n",
       "<title>4970038384&#45;&gt;4970040064</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M857.8088,-1175.9939C854.6066,-1149.0953 845.7638,-1074.8157 841.4418,-1038.5108\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"844.8679,-1037.6821 840.2102,-1028.166 837.917,-1038.5097 844.8679,-1037.6821\"/>\n",
       "</g>\n",
       "<!-- 4539554840 -->\n",
       "<g id=\"node69\" class=\"node\">\n",
       "<title>4539554840</title>\n",
       "<polygon fill=\"#d3d3d3\" stroke=\"#000000\" points=\"279.9668,-1140 182.0332,-1140 182.0332,-1120 279.9668,-1120 279.9668,-1140\"/>\n",
       "<text text-anchor=\"middle\" x=\"231\" y=\"-1126.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">MeanBackward1</text>\n",
       "</g>\n",
       "<!-- 4539554840&#45;&gt;4600695048 -->\n",
       "<g id=\"edge89\" class=\"edge\">\n",
       "<title>4539554840&#45;&gt;4600695048</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M217.2393,-1119.96C194.6052,-1102.1675 152,-1062.7343 152,-1018 152,-1018 152,-1018 152,-738 152,-574.097 0,-565.903 0,-402 0,-402 0,-402 0,-122 0,-67.2422 168.6246,-33.0247 261.4413,-18.3378\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"262.2028,-21.7616 271.5472,-16.7684 261.1285,-14.8445 262.2028,-21.7616\"/>\n",
       "</g>\n",
       "<!-- 4810468656&#45;&gt;4539554840 -->\n",
       "<g id=\"edge90\" class=\"edge\">\n",
       "<title>4810468656&#45;&gt;4539554840</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M259.1392,-1231.6641C254.0179,-1213.1614 243.2006,-1174.0797 236.5702,-1150.1246\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"239.8947,-1149.0149 233.8539,-1140.311 233.1484,-1150.8823 239.8947,-1149.0149\"/>\n",
       "</g>\n",
       "<!-- 4810467928 -->\n",
       "<g id=\"node71\" class=\"node\">\n",
       "<title>4810467928</title>\n",
       "<polygon fill=\"#d3d3d3\" stroke=\"#000000\" points=\"243.8012,-1308 142.1988,-1308 142.1988,-1288 243.8012,-1288 243.8012,-1308\"/>\n",
       "<text text-anchor=\"middle\" x=\"193\" y=\"-1294.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">ExpandBackward</text>\n",
       "</g>\n",
       "<!-- 4810467928&#45;&gt;4810468656 -->\n",
       "<g id=\"edge91\" class=\"edge\">\n",
       "<title>4810467928&#45;&gt;4810468656</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M205.3358,-1287.9883C215.3174,-1279.8873 229.5914,-1268.3026 241.3156,-1258.7873\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"243.7452,-1261.3232 249.3042,-1252.3039 239.3339,-1255.888 243.7452,-1261.3232\"/>\n",
       "</g>\n",
       "<!-- 4810468264 -->\n",
       "<g id=\"node72\" class=\"node\">\n",
       "<title>4810468264</title>\n",
       "<polygon fill=\"#add8e6\" stroke=\"#000000\" points=\"237.8307,-1376 148.1693,-1376 148.1693,-1344 237.8307,-1344 237.8307,-1376\"/>\n",
       "<text text-anchor=\"middle\" x=\"193\" y=\"-1362.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">classifier.6.bias</text>\n",
       "<text text-anchor=\"middle\" x=\"193\" y=\"-1350.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\"> (1000)</text>\n",
       "</g>\n",
       "<!-- 4810468264&#45;&gt;4810467928 -->\n",
       "<g id=\"edge92\" class=\"edge\">\n",
       "<title>4810468264&#45;&gt;4810467928</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M193,-1343.7102C193,-1336.0144 193,-1326.744 193,-1318.5691\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"196.5001,-1318.3512 193,-1308.3512 189.5001,-1318.3513 196.5001,-1318.3512\"/>\n",
       "</g>\n",
       "</g>\n",
       "</svg>\n"
      ],
      "text/plain": [
       "<graphviz.dot.Digraph at 0x112390b00>"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "make_dot(double_backprop(x, model), params=dict(list(model.named_parameters()) + [('x', x)]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## LSTM cell"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/svg+xml": [
       "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>\n",
       "<!DOCTYPE svg PUBLIC \"-//W3C//DTD SVG 1.1//EN\"\n",
       " \"http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd\">\n",
       "<!-- Generated by graphviz version 2.40.1 (20161225.0304)\n",
       " -->\n",
       "<!-- Title: %3 Pages: 1 -->\n",
       "<svg width=\"444pt\" height=\"544pt\"\n",
       " viewBox=\"0.00 0.00 443.88 544.00\" xmlns=\"http://www.w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\">\n",
       "<g id=\"graph0\" class=\"graph\" transform=\"scale(1 1) rotate(0) translate(4 540)\">\n",
       "<title>%3</title>\n",
       "<polygon fill=\"#ffffff\" stroke=\"transparent\" points=\"-4,4 -4,-540 439.8818,-540 439.8818,4 -4,4\"/>\n",
       "<!-- 4970090680 -->\n",
       "<g id=\"node1\" class=\"node\">\n",
       "<title>4970090680</title>\n",
       "<polygon fill=\"#caff70\" stroke=\"#000000\" points=\"181.6289,-20 83.6719,-20 83.6719,0 181.6289,0 181.6289,-20\"/>\n",
       "<text text-anchor=\"middle\" x=\"132.6504\" y=\"-6.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">ThMulBackward</text>\n",
       "</g>\n",
       "<!-- 4970090736 -->\n",
       "<g id=\"node2\" class=\"node\">\n",
       "<title>4970090736</title>\n",
       "<polygon fill=\"#d3d3d3\" stroke=\"#000000\" points=\"125.4712,-132 19.8296,-132 19.8296,-112 125.4712,-112 125.4712,-132\"/>\n",
       "<text text-anchor=\"middle\" x=\"72.6504\" y=\"-118.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">SigmoidBackward</text>\n",
       "</g>\n",
       "<!-- 4970090736&#45;&gt;4970090680 -->\n",
       "<g id=\"edge1\" class=\"edge\">\n",
       "<title>4970090736&#45;&gt;4970090680</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M78.1875,-111.6641C88.2336,-92.9113 109.6041,-53.0197 122.3851,-29.1619\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"125.4896,-30.7786 127.1267,-20.311 119.3192,-27.473 125.4896,-30.7786\"/>\n",
       "</g>\n",
       "<!-- 4970091016 -->\n",
       "<g id=\"node3\" class=\"node\">\n",
       "<title>4970091016</title>\n",
       "<polygon fill=\"#d3d3d3\" stroke=\"#000000\" points=\"258.4713,-300 170.8295,-300 170.8295,-280 258.4713,-280 258.4713,-300\"/>\n",
       "<text text-anchor=\"middle\" x=\"214.6504\" y=\"-286.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">SplitBackward</text>\n",
       "</g>\n",
       "<!-- 4970091016&#45;&gt;4970090736 -->\n",
       "<g id=\"edge2\" class=\"edge\">\n",
       "<title>4970091016&#45;&gt;4970090736</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M170.9202,-284.8147C143.9888,-279.2624 111.0201,-267.772 91.6504,-244 67.9547,-214.9187 67.7767,-168.5864 70.0364,-142.1781\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"73.5217,-142.5011 71.0926,-132.1884 66.5605,-141.765 73.5217,-142.5011\"/>\n",
       "</g>\n",
       "<!-- 4970091912 -->\n",
       "<g id=\"node18\" class=\"node\">\n",
       "<title>4970091912</title>\n",
       "<polygon fill=\"#d3d3d3\" stroke=\"#000000\" points=\"206.4712,-244 100.8296,-244 100.8296,-224 206.4712,-224 206.4712,-244\"/>\n",
       "<text text-anchor=\"middle\" x=\"153.6504\" y=\"-230.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">SigmoidBackward</text>\n",
       "</g>\n",
       "<!-- 4970091016&#45;&gt;4970091912 -->\n",
       "<g id=\"edge18\" class=\"edge\">\n",
       "<title>4970091016&#45;&gt;4970091912</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M203.7448,-279.9883C195.0919,-272.0446 182.7901,-260.7512 172.5429,-251.3439\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"174.6078,-248.4883 164.8743,-244.3039 169.8738,-253.6449 174.6078,-248.4883\"/>\n",
       "</g>\n",
       "<!-- 4970092024 -->\n",
       "<g id=\"node20\" class=\"node\">\n",
       "<title>4970092024</title>\n",
       "<polygon fill=\"#d3d3d3\" stroke=\"#000000\" points=\"329.4712,-244 223.8296,-244 223.8296,-224 329.4712,-224 329.4712,-244\"/>\n",
       "<text text-anchor=\"middle\" x=\"276.6504\" y=\"-230.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">SigmoidBackward</text>\n",
       "</g>\n",
       "<!-- 4970091016&#45;&gt;4970092024 -->\n",
       "<g id=\"edge21\" class=\"edge\">\n",
       "<title>4970091016&#45;&gt;4970092024</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M225.7347,-279.9883C234.6166,-271.966 247.281,-260.5272 257.7569,-251.065\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"260.1675,-253.6041 265.2425,-244.3039 255.4755,-248.4094 260.1675,-253.6041\"/>\n",
       "</g>\n",
       "<!-- 4970092080 -->\n",
       "<g id=\"node21\" class=\"node\">\n",
       "<title>4970092080</title>\n",
       "<polygon fill=\"#d3d3d3\" stroke=\"#000000\" points=\"436.1145,-244 347.1863,-244 347.1863,-224 436.1145,-224 436.1145,-244\"/>\n",
       "<text text-anchor=\"middle\" x=\"391.6504\" y=\"-230.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">TanhBackward</text>\n",
       "</g>\n",
       "<!-- 4970091016&#45;&gt;4970092080 -->\n",
       "<g id=\"edge23\" class=\"edge\">\n",
       "<title>4970091016&#45;&gt;4970092080</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M246.2944,-279.9883C275.2759,-270.819 318.3641,-257.1866 350.1071,-247.1436\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"351.504,-250.3728 359.9824,-244.0192 349.3924,-243.6988 351.504,-250.3728\"/>\n",
       "</g>\n",
       "<!-- 4970091128 -->\n",
       "<g id=\"node4\" class=\"node\">\n",
       "<title>4970091128</title>\n",
       "<polygon fill=\"#d3d3d3\" stroke=\"#000000\" points=\"263.7917,-356 165.5091,-356 165.5091,-336 263.7917,-336 263.7917,-356\"/>\n",
       "<text text-anchor=\"middle\" x=\"214.6504\" y=\"-342.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">ThAddBackward</text>\n",
       "</g>\n",
       "<!-- 4970091128&#45;&gt;4970091016 -->\n",
       "<g id=\"edge3\" class=\"edge\">\n",
       "<title>4970091128&#45;&gt;4970091016</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M214.6504,-335.9883C214.6504,-328.9098 214.6504,-319.1714 214.6504,-310.4779\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"218.1505,-310.3038 214.6504,-300.3039 211.1505,-310.3039 218.1505,-310.3038\"/>\n",
       "</g>\n",
       "<!-- 4970091240 -->\n",
       "<g id=\"node5\" class=\"node\">\n",
       "<title>4970091240</title>\n",
       "<polygon fill=\"#d3d3d3\" stroke=\"#000000\" points=\"206.4594,-412 88.8414,-412 88.8414,-392 206.4594,-392 206.4594,-412\"/>\n",
       "<text text-anchor=\"middle\" x=\"147.6504\" y=\"-398.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">ThAddmmBackward</text>\n",
       "</g>\n",
       "<!-- 4970091240&#45;&gt;4970091128 -->\n",
       "<g id=\"edge4\" class=\"edge\">\n",
       "<title>4970091240&#45;&gt;4970091128</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M159.6286,-391.9883C169.3209,-383.8873 183.1812,-372.3026 194.5656,-362.7873\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"196.8943,-365.4025 202.3225,-356.3039 192.4051,-360.0315 196.8943,-365.4025\"/>\n",
       "</g>\n",
       "<!-- 4970091408 -->\n",
       "<g id=\"node6\" class=\"node\">\n",
       "<title>4970091408</title>\n",
       "<polygon fill=\"#d3d3d3\" stroke=\"#000000\" points=\"101.4516,-468 -.1508,-468 -.1508,-448 101.4516,-448 101.4516,-468\"/>\n",
       "<text text-anchor=\"middle\" x=\"50.6504\" y=\"-454.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">ExpandBackward</text>\n",
       "</g>\n",
       "<!-- 4970091408&#45;&gt;4970091240 -->\n",
       "<g id=\"edge5\" class=\"edge\">\n",
       "<title>4970091408&#45;&gt;4970091240</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M67.992,-447.9883C82.7972,-439.441 104.3201,-427.0155 121.2624,-417.2343\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"123.2214,-420.1448 130.1318,-412.1138 119.7215,-414.0825 123.2214,-420.1448\"/>\n",
       "</g>\n",
       "<!-- 4970091576 -->\n",
       "<g id=\"node7\" class=\"node\">\n",
       "<title>4970091576</title>\n",
       "<polygon fill=\"#add8e6\" stroke=\"#000000\" points=\"77.6504,-536 23.6504,-536 23.6504,-504 77.6504,-504 77.6504,-536\"/>\n",
       "<text text-anchor=\"middle\" x=\"50.6504\" y=\"-522.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">bias_ih</text>\n",
       "<text text-anchor=\"middle\" x=\"50.6504\" y=\"-510.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\"> (512)</text>\n",
       "</g>\n",
       "<!-- 4970091576&#45;&gt;4970091408 -->\n",
       "<g id=\"edge6\" class=\"edge\">\n",
       "<title>4970091576&#45;&gt;4970091408</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M50.6504,-503.7102C50.6504,-496.0144 50.6504,-486.744 50.6504,-478.5691\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"54.1505,-478.3512 50.6504,-468.3512 47.1505,-478.3513 54.1505,-478.3512\"/>\n",
       "</g>\n",
       "<!-- 4970091464 -->\n",
       "<g id=\"node8\" class=\"node\">\n",
       "<title>4970091464</title>\n",
       "<polygon fill=\"#d3d3d3\" stroke=\"#000000\" points=\"191.625,-468 119.6758,-468 119.6758,-448 191.625,-448 191.625,-468\"/>\n",
       "<text text-anchor=\"middle\" x=\"155.6504\" y=\"-454.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">TBackward</text>\n",
       "</g>\n",
       "<!-- 4970091464&#45;&gt;4970091240 -->\n",
       "<g id=\"edge7\" class=\"edge\">\n",
       "<title>4970091464&#45;&gt;4970091240</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M154.2202,-447.9883C153.2089,-440.9098 151.8177,-431.1714 150.5758,-422.4779\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"154.0015,-421.7084 149.1224,-412.3039 147.0718,-422.6984 154.0015,-421.7084\"/>\n",
       "</g>\n",
       "<!-- 4970091632 -->\n",
       "<g id=\"node9\" class=\"node\">\n",
       "<title>4970091632</title>\n",
       "<polygon fill=\"#add8e6\" stroke=\"#000000\" points=\"190.1426,-536 121.1582,-536 121.1582,-504 190.1426,-504 190.1426,-536\"/>\n",
       "<text text-anchor=\"middle\" x=\"155.6504\" y=\"-522.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">weight_ih</text>\n",
       "<text text-anchor=\"middle\" x=\"155.6504\" y=\"-510.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\"> (512, 128)</text>\n",
       "</g>\n",
       "<!-- 4970091632&#45;&gt;4970091464 -->\n",
       "<g id=\"edge8\" class=\"edge\">\n",
       "<title>4970091632&#45;&gt;4970091464</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M155.6504,-503.7102C155.6504,-496.0144 155.6504,-486.744 155.6504,-478.5691\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"159.1505,-478.3512 155.6504,-468.3512 152.1505,-478.3513 159.1505,-478.3512\"/>\n",
       "</g>\n",
       "<!-- 4970091296 -->\n",
       "<g id=\"node10\" class=\"node\">\n",
       "<title>4970091296</title>\n",
       "<polygon fill=\"#d3d3d3\" stroke=\"#000000\" points=\"341.4594,-412 223.8414,-412 223.8414,-392 341.4594,-392 341.4594,-412\"/>\n",
       "<text text-anchor=\"middle\" x=\"282.6504\" y=\"-398.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">ThAddmmBackward</text>\n",
       "</g>\n",
       "<!-- 4970091296&#45;&gt;4970091128 -->\n",
       "<g id=\"edge9\" class=\"edge\">\n",
       "<title>4970091296&#45;&gt;4970091128</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M270.4934,-391.9883C260.6564,-383.8873 246.5893,-372.3026 235.035,-362.7873\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"237.1066,-359.9592 227.1622,-356.3039 232.6566,-365.3627 237.1066,-359.9592\"/>\n",
       "</g>\n",
       "<!-- 4970091520 -->\n",
       "<g id=\"node11\" class=\"node\">\n",
       "<title>4970091520</title>\n",
       "<polygon fill=\"#d3d3d3\" stroke=\"#000000\" points=\"326.4516,-468 224.8492,-468 224.8492,-448 326.4516,-448 326.4516,-468\"/>\n",
       "<text text-anchor=\"middle\" x=\"275.6504\" y=\"-454.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">ExpandBackward</text>\n",
       "</g>\n",
       "<!-- 4970091520&#45;&gt;4970091296 -->\n",
       "<g id=\"edge10\" class=\"edge\">\n",
       "<title>4970091520&#45;&gt;4970091296</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M276.9019,-447.9883C277.7867,-440.9098 279.004,-431.1714 280.0906,-422.4779\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"283.5949,-422.6608 281.3624,-412.3039 276.649,-421.7925 283.5949,-422.6608\"/>\n",
       "</g>\n",
       "<!-- 4970091856 -->\n",
       "<g id=\"node12\" class=\"node\">\n",
       "<title>4970091856</title>\n",
       "<polygon fill=\"#add8e6\" stroke=\"#000000\" points=\"302.6504,-536 248.6504,-536 248.6504,-504 302.6504,-504 302.6504,-536\"/>\n",
       "<text text-anchor=\"middle\" x=\"275.6504\" y=\"-522.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">bias_hh</text>\n",
       "<text text-anchor=\"middle\" x=\"275.6504\" y=\"-510.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\"> (512)</text>\n",
       "</g>\n",
       "<!-- 4970091856&#45;&gt;4970091520 -->\n",
       "<g id=\"edge11\" class=\"edge\">\n",
       "<title>4970091856&#45;&gt;4970091520</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M275.6504,-503.7102C275.6504,-496.0144 275.6504,-486.744 275.6504,-478.5691\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"279.1505,-478.3512 275.6504,-468.3512 272.1505,-478.3513 279.1505,-478.3512\"/>\n",
       "</g>\n",
       "<!-- 4970091688 -->\n",
       "<g id=\"node13\" class=\"node\">\n",
       "<title>4970091688</title>\n",
       "<polygon fill=\"#d3d3d3\" stroke=\"#000000\" points=\"416.625,-468 344.6758,-468 344.6758,-448 416.625,-448 416.625,-468\"/>\n",
       "<text text-anchor=\"middle\" x=\"380.6504\" y=\"-454.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">TBackward</text>\n",
       "</g>\n",
       "<!-- 4970091688&#45;&gt;4970091296 -->\n",
       "<g id=\"edge12\" class=\"edge\">\n",
       "<title>4970091688&#45;&gt;4970091296</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M363.13,-447.9883C348.1721,-439.441 326.4274,-427.0155 309.3104,-417.2343\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"310.7685,-414.0364 300.3496,-412.1138 307.2955,-420.1141 310.7685,-414.0364\"/>\n",
       "</g>\n",
       "<!-- 4970091744 -->\n",
       "<g id=\"node14\" class=\"node\">\n",
       "<title>4970091744</title>\n",
       "<polygon fill=\"#add8e6\" stroke=\"#000000\" points=\"415.1426,-536 346.1582,-536 346.1582,-504 415.1426,-504 415.1426,-536\"/>\n",
       "<text text-anchor=\"middle\" x=\"380.6504\" y=\"-522.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">weight_hh</text>\n",
       "<text text-anchor=\"middle\" x=\"380.6504\" y=\"-510.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\"> (512, 128)</text>\n",
       "</g>\n",
       "<!-- 4970091744&#45;&gt;4970091688 -->\n",
       "<g id=\"edge13\" class=\"edge\">\n",
       "<title>4970091744&#45;&gt;4970091688</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M380.6504,-503.7102C380.6504,-496.0144 380.6504,-486.744 380.6504,-478.5691\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"384.1505,-478.3512 380.6504,-468.3512 377.1505,-478.3513 384.1505,-478.3512\"/>\n",
       "</g>\n",
       "<!-- 4970090904 -->\n",
       "<g id=\"node15\" class=\"node\">\n",
       "<title>4970090904</title>\n",
       "<polygon fill=\"#d3d3d3\" stroke=\"#000000\" points=\"225.1145,-76 136.1863,-76 136.1863,-56 225.1145,-56 225.1145,-76\"/>\n",
       "<text text-anchor=\"middle\" x=\"180.6504\" y=\"-62.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">TanhBackward</text>\n",
       "</g>\n",
       "<!-- 4970090904&#45;&gt;4970090680 -->\n",
       "<g id=\"edge14\" class=\"edge\">\n",
       "<title>4970090904&#45;&gt;4970090680</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M172.069,-55.9883C165.3949,-48.2019 155.9622,-37.1971 147.9975,-27.9049\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"150.6476,-25.6187 141.4823,-20.3039 145.3328,-30.1742 150.6476,-25.6187\"/>\n",
       "</g>\n",
       "<!-- 4970090848 -->\n",
       "<g id=\"node16\" class=\"node\">\n",
       "<title>4970090848</title>\n",
       "<polygon fill=\"#caff70\" stroke=\"#000000\" points=\"241.7917,-132 143.5091,-132 143.5091,-112 241.7917,-112 241.7917,-132\"/>\n",
       "<text text-anchor=\"middle\" x=\"192.6504\" y=\"-118.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">ThAddBackward</text>\n",
       "</g>\n",
       "<!-- 4970090848&#45;&gt;4970090904 -->\n",
       "<g id=\"edge15\" class=\"edge\">\n",
       "<title>4970090848&#45;&gt;4970090904</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M190.505,-111.9883C188.9713,-104.8311 186.855,-94.9547 184.9765,-86.1885\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"188.376,-85.3485 182.8584,-76.3039 181.5314,-86.8153 188.376,-85.3485\"/>\n",
       "</g>\n",
       "<!-- 4970091184 -->\n",
       "<g id=\"node17\" class=\"node\">\n",
       "<title>4970091184</title>\n",
       "<polygon fill=\"#d3d3d3\" stroke=\"#000000\" points=\"205.6289,-188 107.6719,-188 107.6719,-168 205.6289,-168 205.6289,-188\"/>\n",
       "<text text-anchor=\"middle\" x=\"156.6504\" y=\"-174.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">ThMulBackward</text>\n",
       "</g>\n",
       "<!-- 4970091184&#45;&gt;4970090848 -->\n",
       "<g id=\"edge16\" class=\"edge\">\n",
       "<title>4970091184&#45;&gt;4970090848</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M163.0865,-167.9883C167.9403,-160.4379 174.7397,-149.8611 180.5941,-140.7543\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"183.563,-142.6083 186.0265,-132.3039 177.6747,-138.823 183.563,-142.6083\"/>\n",
       "</g>\n",
       "<!-- 4970091912&#45;&gt;4970091184 -->\n",
       "<g id=\"edge17\" class=\"edge\">\n",
       "<title>4970091912&#45;&gt;4970091184</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M154.1867,-223.9883C154.5659,-216.9098 155.0876,-207.1714 155.5534,-198.4779\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"159.0583,-198.4768 156.0984,-188.3039 152.0684,-198.1023 159.0583,-198.4768\"/>\n",
       "</g>\n",
       "<!-- 4970091352 -->\n",
       "<g id=\"node19\" class=\"node\">\n",
       "<title>4970091352</title>\n",
       "<polygon fill=\"#d3d3d3\" stroke=\"#000000\" points=\"325.6289,-188 227.6719,-188 227.6719,-168 325.6289,-168 325.6289,-188\"/>\n",
       "<text text-anchor=\"middle\" x=\"276.6504\" y=\"-174.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">ThMulBackward</text>\n",
       "</g>\n",
       "<!-- 4970091352&#45;&gt;4970090848 -->\n",
       "<g id=\"edge19\" class=\"edge\">\n",
       "<title>4970091352&#45;&gt;4970090848</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M261.6329,-167.9883C249.0493,-159.5993 230.8619,-147.4743 216.321,-137.7804\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"218.0831,-134.7487 207.8211,-132.1138 214.2002,-140.573 218.0831,-134.7487\"/>\n",
       "</g>\n",
       "<!-- 4970092024&#45;&gt;4970091352 -->\n",
       "<g id=\"edge20\" class=\"edge\">\n",
       "<title>4970092024&#45;&gt;4970091352</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M276.6504,-223.9883C276.6504,-216.9098 276.6504,-207.1714 276.6504,-198.4779\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"280.1505,-198.3038 276.6504,-188.3039 273.1505,-198.3039 280.1505,-198.3038\"/>\n",
       "</g>\n",
       "<!-- 4970092080&#45;&gt;4970091352 -->\n",
       "<g id=\"edge22\" class=\"edge\">\n",
       "<title>4970092080&#45;&gt;4970091352</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M371.0907,-223.9883C353.1318,-215.2431 326.8358,-202.4381 306.5491,-192.5594\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"307.9429,-189.3452 297.4198,-188.1138 304.8782,-195.6387 307.9429,-189.3452\"/>\n",
       "</g>\n",
       "</g>\n",
       "</svg>\n"
      ],
      "text/plain": [
       "<graphviz.dot.Digraph at 0x10e941c50>"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lstm_cell = nn.LSTMCell(128, 128)\n",
    "x = torch.randn(1, 128)\n",
    "make_dot(lstm_cell(x), params=dict(list(lstm_cell.named_parameters())))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}