pytorch-dynamic-network.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Dynamic Graph and Weight Sharing in Pytorch\n",
    "A fully connected ReLU that on each forward pass chooses a random number betweeen 1 and 4 and uses that many hidden layers, reusing the same weight multiple times to compute the innermost hiddenlayers."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import random\n",
    "import torch\n",
    "from torch.autograd import Variable\n",
    "from torch import nn"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "class DynamicNet(nn.Module):\n",
    "    def __init__(self, input_dim, hidden_dim, output_dim):\n",
    "        \"\"\"\n",
    "        We construct three nn.Linear instances that we will use in the forward pass\n",
    "        \"\"\"\n",
    "        super(DynamicNet, self).__init__()\n",
    "        self.input_linear = torch.nn.Linear(input_dim, hidden_dim)\n",
    "        self.middle_linear = torch.nn.Linear(hidden_dim, hidden_dim)\n",
    "        self.output_linear = torch.nn.Linear(hidden_dim, output_dim)\n",
    "    \n",
    "    def forward(self, x):\n",
    "        \"\"\"\n",
    "        Randomly choose either 0, 1, 2 or 3 and reuse the middle_linear Module\n",
    "        that many times to compute hidden lyaer representations.\n",
    "        Since each forward pass builds a dynamic computation graph, we can use normal\n",
    "        Python control-flow operators like loops or conditional statements when defining\n",
    "        the forward pass of the model.\n",
    "        Here we also see that it is perfectly safe to reuse the same Module many times\n",
    "        when defining a computational graph.\n",
    "        \"\"\"\n",
    "        h_relu = self.input_linear(x).clamp(min=0)\n",
    "        for _ in range(random.randint(0, 3)):\n",
    "            h_relu = self.middle_linear(h_relu).clamp(min=0)\n",
    "        y_pred = self.output_linear(h_relu)\n",
    "        return y_pred"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "batch_size = 64\n",
    "input_dim = 1000\n",
    "hidden_dim = 100\n",
    "output_dim = 10"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Create tensors wrapped with Variables to hold inputs and outputs\n",
    "x = Variable(torch.randn(batch_size, input_dim))\n",
    "y = Variable(torch.randn(batch_size, output_dim), requires_grad=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Initialize our model\n",
    "model = DynamicNet(input_dim, hidden_dim, output_dim)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Define our loss function and optimizer\n",
    "criterion = torch.nn.MSELoss(size_average=False)\n",
    "optimizer = torch.optim.SGD(model.parameters(), lr=1e-4, momentum=0.9)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 753.000732421875\n",
      "1 781.9813842773438\n",
      "2 753.4469604492188\n",
      "3 745.1839599609375\n",
      "4 740.6723022460938\n",
      "5 602.9732666015625\n",
      "6 717.0496826171875\n",
      "7 495.111328125\n",
      "8 434.4676818847656\n",
      "9 741.2548217773438\n",
      "10 718.1215209960938\n",
      "11 284.787841796875\n",
      "12 244.49330139160156\n",
      "13 657.5676879882812\n",
      "14 169.89157104492188\n",
      "15 137.93592834472656\n",
      "16 688.9822387695312\n",
      "17 94.38672637939453\n",
      "18 83.29523468017578\n",
      "19 73.72959899902344\n",
      "20 716.45947265625\n",
      "21 638.3557739257812\n",
      "22 619.5986938476562\n",
      "23 71.51226806640625\n",
      "24 69.9638442993164\n",
      "25 59.480987548828125\n",
      "26 547.053955078125\n",
      "27 490.8528137207031\n",
      "28 450.5050048828125\n",
      "29 39.074092864990234\n",
      "30 590.2769165039062\n",
      "31 429.1327819824219\n",
      "32 520.2168579101562\n",
      "33 470.17388916015625\n",
      "34 338.3823547363281\n",
      "35 51.85633850097656\n",
      "36 320.9876403808594\n",
      "37 51.585113525390625\n",
      "38 374.6116943359375\n",
      "39 48.96507263183594\n",
      "40 493.4587097167969\n",
      "41 41.06834030151367\n",
      "42 69.48420715332031\n",
      "43 78.45011138916016\n",
      "44 608.5751953125\n",
      "45 52.18977355957031\n",
      "46 123.8389663696289\n",
      "47 345.6250305175781\n",
      "48 322.7888488769531\n",
      "49 347.9517517089844\n",
      "50 276.2320251464844\n",
      "51 286.53314208984375\n",
      "52 264.2723693847656\n",
      "53 263.0390319824219\n",
      "54 411.5144348144531\n",
      "55 177.36761474609375\n",
      "56 185.54092407226562\n",
      "57 211.45973205566406\n",
      "58 109.5706787109375\n",
      "59 138.36558532714844\n",
      "60 85.75788879394531\n",
      "61 120.42704010009766\n",
      "62 67.86219787597656\n",
      "63 54.65778350830078\n",
      "64 165.31246948242188\n",
      "65 195.57579040527344\n",
      "66 38.37827682495117\n",
      "67 87.15435028076172\n",
      "68 256.7325439453125\n",
      "69 73.45977783203125\n",
      "70 134.3487091064453\n",
      "71 69.94019317626953\n",
      "72 121.59098815917969\n",
      "73 172.1791534423828\n",
      "74 52.36682891845703\n",
      "75 63.658870697021484\n",
      "76 142.5111846923828\n",
      "77 37.98030471801758\n",
      "78 60.07545471191406\n",
      "79 108.2693862915039\n",
      "80 40.70830154418945\n",
      "81 121.95209503173828\n",
      "82 60.09835433959961\n",
      "83 92.8600845336914\n",
      "84 82.66680908203125\n",
      "85 47.29707336425781\n",
      "86 30.413516998291016\n",
      "87 19.49070930480957\n",
      "88 39.03681182861328\n",
      "89 39.6722412109375\n",
      "90 173.1539306640625\n",
      "91 44.13147735595703\n",
      "92 19.336254119873047\n",
      "93 14.012284278869629\n",
      "94 31.722444534301758\n",
      "95 40.450721740722656\n",
      "96 38.25864791870117\n",
      "97 80.40978240966797\n",
      "98 60.131290435791016\n",
      "99 47.408321380615234\n",
      "100 14.664841651916504\n",
      "101 53.82725524902344\n",
      "102 17.910459518432617\n",
      "103 31.568702697753906\n",
      "104 19.499187469482422\n",
      "105 48.650665283203125\n",
      "106 15.423171043395996\n",
      "107 19.47341537475586\n",
      "108 46.32317352294922\n",
      "109 23.548465728759766\n",
      "110 28.206790924072266\n",
      "111 11.857417106628418\n",
      "112 19.68158531188965\n",
      "113 17.36689567565918\n",
      "114 12.132612228393555\n",
      "115 30.56975555419922\n",
      "116 9.230422019958496\n",
      "117 11.778141021728516\n",
      "118 12.282915115356445\n",
      "119 41.89468765258789\n",
      "120 29.628265380859375\n",
      "121 16.33526611328125\n",
      "122 17.78495979309082\n",
      "123 21.8283748626709\n",
      "124 10.987488746643066\n",
      "125 9.294793128967285\n",
      "126 8.597335815429688\n",
      "127 12.114715576171875\n",
      "128 47.02675247192383\n",
      "129 11.460339546203613\n",
      "130 8.518847465515137\n",
      "131 15.811339378356934\n",
      "132 13.412745475769043\n",
      "133 8.745634078979492\n",
      "134 13.035041809082031\n",
      "135 5.080920219421387\n",
      "136 6.308061599731445\n",
      "137 22.034255981445312\n",
      "138 7.287370681762695\n",
      "139 16.058259963989258\n",
      "140 12.737470626831055\n",
      "141 2.640249013900757\n",
      "142 2.7151472568511963\n",
      "143 31.485692977905273\n",
      "144 6.316053867340088\n",
      "145 6.448642730712891\n",
      "146 10.440910339355469\n",
      "147 6.3814592361450195\n",
      "148 5.272261619567871\n",
      "149 19.855146408081055\n",
      "150 10.871561050415039\n",
      "151 10.275073051452637\n",
      "152 15.714147567749023\n",
      "153 26.21265411376953\n",
      "154 6.6076273918151855\n",
      "155 10.159985542297363\n",
      "156 3.0218236446380615\n",
      "157 24.8105525970459\n",
      "158 10.090217590332031\n",
      "159 4.349621772766113\n",
      "160 1.888251781463623\n",
      "161 13.375933647155762\n",
      "162 12.921570777893066\n",
      "163 4.785999298095703\n",
      "164 2.8810877799987793\n",
      "165 7.6422834396362305\n",
      "166 9.354177474975586\n",
      "167 10.821127891540527\n",
      "168 7.151147842407227\n",
      "169 5.75758171081543\n",
      "170 4.100106239318848\n",
      "171 4.849876880645752\n",
      "172 5.328088760375977\n",
      "173 3.7236688137054443\n",
      "174 3.1663196086883545\n",
      "175 2.902552604675293\n",
      "176 3.2326865196228027\n",
      "177 1.9443981647491455\n",
      "178 4.40763521194458\n",
      "179 4.963263034820557\n",
      "180 1.9580456018447876\n",
      "181 2.0706193447113037\n",
      "182 1.7678884267807007\n",
      "183 1.646574854850769\n",
      "184 2.0036025047302246\n",
      "185 1.3901386260986328\n",
      "186 8.159029960632324\n",
      "187 5.049837589263916\n",
      "188 4.64187479019165\n",
      "189 6.902841567993164\n",
      "190 5.539373397827148\n",
      "191 2.1829545497894287\n",
      "192 2.601435422897339\n",
      "193 12.179495811462402\n",
      "194 1.7004846334457397\n",
      "195 2.4516584873199463\n",
      "196 2.061924695968628\n",
      "197 1.7899569272994995\n",
      "198 22.220582962036133\n",
      "199 0.8258351683616638\n",
      "200 2.484921455383301\n",
      "201 1.9923644065856934\n",
      "202 2.396775722503662\n",
      "203 2.070665121078491\n",
      "204 6.0382304191589355\n",
      "205 22.961788177490234\n",
      "206 1.8633712530136108\n",
      "207 5.504815578460693\n",
      "208 14.90666675567627\n",
      "209 34.17441177368164\n",
      "210 5.364364147186279\n",
      "211 5.2590765953063965\n",
      "212 15.651840209960938\n",
      "213 1.669350266456604\n",
      "214 1.7024089097976685\n",
      "215 5.610174179077148\n",
      "216 6.361705303192139\n",
      "217 0.6510224938392639\n",
      "218 5.886091232299805\n",
      "219 7.3061113357543945\n",
      "220 7.971462249755859\n",
      "221 0.696617841720581\n",
      "222 1.2792216539382935\n",
      "223 10.941497802734375\n",
      "224 1.4325147867202759\n",
      "225 6.9009013175964355\n",
      "226 3.6495954990386963\n",
      "227 2.3624565601348877\n",
      "228 2.8387813568115234\n",
      "229 5.6659464836120605\n",
      "230 4.310866832733154\n",
      "231 1.852573037147522\n",
      "232 2.1786975860595703\n",
      "233 1.5746476650238037\n",
      "234 7.520528793334961\n",
      "235 6.6097307205200195\n",
      "236 6.921037673950195\n",
      "237 1.9165689945220947\n",
      "238 2.125396490097046\n",
      "239 1.76966392993927\n",
      "240 1.1399812698364258\n",
      "241 6.652859687805176\n",
      "242 1.6402742862701416\n",
      "243 3.1317195892333984\n",
      "244 3.2662510871887207\n",
      "245 2.81954026222229\n",
      "246 1.0505144596099854\n",
      "247 0.9425686001777649\n",
      "248 1.2857102155685425\n",
      "249 1.588710069656372\n",
      "250 2.2005856037139893\n",
      "251 1.111427903175354\n",
      "252 0.7306831479072571\n",
      "253 1.1387983560562134\n",
      "254 4.716137886047363\n",
      "255 2.2275326251983643\n",
      "256 2.62353253364563\n",
      "257 0.8973050713539124\n",
      "258 4.062382221221924\n",
      "259 0.7442890405654907\n",
      "260 0.6004570722579956\n",
      "261 1.5411418676376343\n",
      "262 1.921446681022644\n",
      "263 0.48436036705970764\n",
      "264 0.456564337015152\n",
      "265 0.40376517176628113\n",
      "266 0.33333298563957214\n",
      "267 0.2633797228336334\n",
      "268 1.9854778051376343\n",
      "269 0.16602464020252228\n",
      "270 5.099719524383545\n",
      "271 1.483310341835022\n",
      "272 4.0275750160217285\n",
      "273 3.83449649810791\n",
      "274 1.3913449048995972\n",
      "275 1.9280285835266113\n",
      "276 0.8899971842765808\n",
      "277 11.004687309265137\n",
      "278 1.4759472608566284\n",
      "279 2.9273593425750732\n",
      "280 5.364153861999512\n",
      "281 3.1750547885894775\n",
      "282 2.258035898208618\n",
      "283 1.1720343828201294\n",
      "284 1.2182453870773315\n",
      "285 3.471085786819458\n",
      "286 6.1924357414245605\n",
      "287 1.3204522132873535\n",
      "288 1.3034838438034058\n",
      "289 1.168338656425476\n",
      "290 0.646409809589386\n",
      "291 0.6491054892539978\n",
      "292 28.93299102783203\n",
      "293 1.0294286012649536\n",
      "294 9.155486106872559\n",
      "295 2.725666046142578\n",
      "296 18.66292381286621\n",
      "297 3.2747676372528076\n",
      "298 2.672046661376953\n",
      "299 9.370023727416992\n",
      "300 10.330788612365723\n",
      "301 4.2797746658325195\n",
      "302 0.49460533261299133\n",
      "303 1.175969123840332\n",
      "304 2.037815570831299\n",
      "305 23.683273315429688\n",
      "306 0.9791551828384399\n",
      "307 3.126950979232788\n",
      "308 1.6493492126464844\n",
      "309 2.307908773422241\n",
      "310 9.729482650756836\n",
      "311 23.31981658935547\n",
      "312 0.7887316346168518\n",
      "313 13.237662315368652\n",
      "314 1.1129027605056763\n",
      "315 37.55201721191406\n",
      "316 10.832111358642578\n",
      "317 1.5823403596878052\n",
      "318 4.442019939422607\n",
      "319 26.191213607788086\n",
      "320 56.2115364074707\n",
      "321 2.0720553398132324\n",
      "322 15.72870922088623\n",
      "323 17.730499267578125\n",
      "324 34.857276916503906\n",
      "325 29.845808029174805\n",
      "326 3.032186508178711\n",
      "327 5.517999649047852\n",
      "328 15.899139404296875\n",
      "329 68.44563293457031\n",
      "330 8.034839630126953\n",
      "331 7.762413501739502\n",
      "332 13.530072212219238\n",
      "333 11.78797435760498\n",
      "334 34.17622375488281\n",
      "335 8.035533905029297\n",
      "336 3.8663549423217773\n",
      "337 1.6652740240097046\n",
      "338 5.43595552444458\n",
      "339 9.149157524108887\n",
      "340 9.245430946350098\n",
      "341 5.884003639221191\n",
      "342 8.82254695892334\n",
      "343 2.830547571182251\n",
      "344 2.647005319595337\n",
      "345 6.208058834075928\n",
      "346 8.163989067077637\n",
      "347 6.164583206176758\n",
      "348 2.636134147644043\n",
      "349 1.241769790649414\n",
      "350 20.38216781616211\n",
      "351 3.3018693923950195\n",
      "352 1.514542818069458\n",
      "353 3.354240894317627\n",
      "354 1.5807915925979614\n",
      "355 4.566961765289307\n",
      "356 1.8389058113098145\n",
      "357 8.234028816223145\n",
      "358 3.662841558456421\n",
      "359 2.8299849033355713\n",
      "360 3.8135340213775635\n",
      "361 3.205599308013916\n",
      "362 2.2201104164123535\n",
      "363 1.610234022140503\n",
      "364 5.276986598968506\n",
      "365 5.221022605895996\n",
      "366 1.018385648727417\n",
      "367 2.387845516204834\n",
      "368 1.9932843446731567\n",
      "369 1.6209725141525269\n",
      "370 1.8417716026306152\n",
      "371 2.2634856700897217\n",
      "372 1.5264670848846436\n",
      "373 1.0080969333648682\n",
      "374 1.0073858499526978\n",
      "375 2.7119078636169434\n",
      "376 2.214397668838501\n",
      "377 1.1431801319122314\n",
      "378 1.8347560167312622\n",
      "379 1.9414381980895996\n",
      "380 1.2746083736419678\n",
      "381 0.9948926568031311\n",
      "382 0.669525682926178\n",
      "383 0.6887656450271606\n",
      "384 5.468475818634033\n",
      "385 0.5061923861503601\n",
      "386 1.3302431106567383\n",
      "387 1.238879680633545\n",
      "388 2.5641939640045166\n",
      "389 0.42740482091903687\n",
      "390 0.32454541325569153\n",
      "391 0.2668313682079315\n",
      "392 0.2565133273601532\n",
      "393 2.6527109146118164\n",
      "394 0.39513471722602844\n",
      "395 1.0482618808746338\n",
      "396 1.2547763586044312\n",
      "397 0.7214126586914062\n",
      "398 1.347601294517517\n",
      "399 1.3923907279968262\n",
      "400 1.009940266609192\n",
      "401 0.9700447916984558\n",
      "402 0.9788423776626587\n",
      "403 1.3486427068710327\n",
      "404 0.2805522680282593\n",
      "405 1.0686274766921997\n",
      "406 0.6814342141151428\n",
      "407 1.3168617486953735\n",
      "408 0.3088904917240143\n",
      "409 0.979232132434845\n",
      "410 0.5474603176116943\n",
      "411 0.27575570344924927\n",
      "412 0.9868480563163757\n",
      "413 0.7782059907913208\n",
      "414 0.8757583498954773\n",
      "415 0.7819598317146301\n",
      "416 0.7642139196395874\n",
      "417 0.9116191864013672\n",
      "418 0.5227716565132141\n",
      "419 0.943610429763794\n",
      "420 0.8945363759994507\n",
      "421 0.8677513599395752\n",
      "422 0.7499887347221375\n",
      "423 0.5578003525733948\n",
      "424 0.7354749441146851\n",
      "425 0.6026622653007507\n",
      "426 0.6692773699760437\n",
      "427 0.6379887461662292\n",
      "428 0.6097679138183594\n",
      "429 0.410738080739975\n",
      "430 0.9325309991836548\n",
      "431 0.28659752011299133\n",
      "432 0.603763222694397\n",
      "433 0.5630149841308594\n",
      "434 0.19547738134860992\n",
      "435 0.16791735589504242\n",
      "436 0.133005753159523\n",
      "437 1.1109341382980347\n",
      "438 0.5584028959274292\n",
      "439 0.4522409737110138\n",
      "440 1.0432584285736084\n",
      "441 1.1827887296676636\n",
      "442 0.9341022372245789\n",
      "443 0.2597249746322632\n",
      "444 0.6215726733207703\n",
      "445 0.6034061312675476\n",
      "446 1.0656880140304565\n",
      "447 0.15580633282661438\n",
      "448 0.5938395857810974\n",
      "449 0.43965768814086914\n",
      "450 0.1335611790418625\n",
      "451 0.8330302834510803\n",
      "452 0.1107773408293724\n",
      "453 0.0993771106004715\n",
      "454 0.09217098355293274\n",
      "455 2.1174674034118652\n",
      "456 0.5830348134040833\n",
      "457 0.23876231908798218\n",
      "458 0.32484567165374756\n",
      "459 1.2549060583114624\n",
      "460 0.18327942490577698\n",
      "461 0.08090700954198837\n",
      "462 0.7239789962768555\n",
      "463 0.5047436952590942\n",
      "464 0.414983868598938\n",
      "465 0.8602089285850525\n",
      "466 0.5640901327133179\n",
      "467 0.9649055600166321\n",
      "468 0.5337298512458801\n",
      "469 1.0969408750534058\n",
      "470 0.37906092405319214\n",
      "471 0.3780907988548279\n",
      "472 0.41850343346595764\n",
      "473 0.575017511844635\n",
      "474 1.694735050201416\n",
      "475 1.0724208354949951\n",
      "476 0.9472945928573608\n",
      "477 1.6622148752212524\n",
      "478 1.1617841720581055\n",
      "479 0.4991232752799988\n",
      "480 0.8940041661262512\n",
      "481 0.15948376059532166\n",
      "482 1.216066837310791\n",
      "483 0.13993269205093384\n",
      "484 0.1151719018816948\n",
      "485 0.09713590145111084\n",
      "486 0.5604636073112488\n",
      "487 0.08149044960737228\n",
      "488 0.08107579499483109\n",
      "489 0.5919966101646423\n",
      "490 0.5416550040245056\n",
      "491 1.0281840562820435\n",
      "492 0.4255167245864868\n",
      "493 0.5156342387199402\n",
      "494 0.564496636390686\n",
      "495 0.9047347903251648\n",
      "496 0.28434640169143677\n",
      "497 0.5715476274490356\n",
      "498 0.6635839939117432\n",
      "499 0.27372726798057556\n"
     ]
    }
   ],
   "source": [
    "for t in range(500):\n",
    "    y_pred = model(x)\n",
    "    loss = criterion(y_pred, y)\n",
    "    print(t, loss.data[0])\n",
    "    optimizer.zero_grad()\n",
    "    loss.backward()\n",
    "    optimizer.step()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "DynamicNet (\n",
      "  (input_linear): Linear (1000 -> 100)\n",
      "  (middle_linear): Linear (100 -> 100)\n",
      "  (output_linear): Linear (100 -> 10)\n",
      ")\n",
      "6\n",
      "torch.Size([100, 1000])\n",
      "torch.Size([100])\n",
      "torch.Size([100, 100])\n",
      "torch.Size([100])\n",
      "torch.Size([10, 100])\n",
      "torch.Size([10])\n"
     ]
    }
   ],
   "source": [
    "print(model)\n",
    "params = list(model.parameters())\n",
    "print(len(params))\n",
    "print(params[0].size())\n",
    "print(params[1].size())\n",
    "print(params[2].size())\n",
    "print(params[3].size())\n",
    "print(params[4].size())\n",
    "print(params[5].size())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2+"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}