simple_rewiring.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Rewire gradients for less memory"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Utilities and imports"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'1.1.0'"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# for instructions of how to enable Log memory usage see\n",
    "# https://github.com/yaroslavvb/notebooks/blob/master/mnist-memory.ipynb\n",
    "import os\n",
    "os.environ[\"CUDA_VISIBLE_DEVICES\"]=\"\"\n",
    "%matplotlib inline\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "import tensorflow as tf\n",
    "import numpy as np\n",
    "import re, os, sys\n",
    "tf.__version__"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Graph visualizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style>.container { width:100% !important; }</style>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# make things wide\n",
    "from IPython.core.display import display, HTML\n",
    "display(HTML(\"<style>.container { width:100% !important; }</style>\"))\n",
    "\n",
    "from IPython.display import clear_output, Image, display, HTML\n",
    "\n",
    "def strip_consts(graph_def, max_const_size=32):\n",
    "    \"\"\"Strip large constant values from graph_def.\"\"\"\n",
    "    strip_def = tf.GraphDef()\n",
    "    for n0 in graph_def.node:\n",
    "        n = strip_def.node.add() \n",
    "        n.MergeFrom(n0)\n",
    "        if n.op == 'Const':\n",
    "            tensor = n.attr['value'].tensor\n",
    "            size = len(tensor.tensor_content)\n",
    "            if size > max_const_size:\n",
    "                tensor.tensor_content = \"<stripped %d bytes>\"%size\n",
    "    return strip_def\n",
    "\n",
    "def show_graph(graph_def=None, width=1200, height=800, max_const_size=32, ungroup_gradients=False):\n",
    "    if not graph_def:\n",
    "        graph_def = tf.get_default_graph().as_graph_def()\n",
    "        \n",
    "    \"\"\"Visualize TensorFlow graph.\"\"\"\n",
    "    if hasattr(graph_def, 'as_graph_def'):\n",
    "        graph_def = graph_def.as_graph_def()\n",
    "    strip_def = strip_consts(graph_def, max_const_size=max_const_size)\n",
    "    data = str(strip_def)\n",
    "    if ungroup_gradients:\n",
    "        data = data.replace('\"gradients/', '\"b_')\n",
    "        #print(data)\n",
    "    code = \"\"\"\n",
    "        <script>\n",
    "          function load() {{\n",
    "            document.getElementById(\"{id}\").pbtxt = {data};\n",
    "          }}\n",
    "        </script>\n",
    "        <link rel=\"import\" href=\"https://tensorboard.appspot.com/tf-graph-basic.build.html\" onload=load()>\n",
    "        <div style=\"height:600px\">\n",
    "          <tf-graph-basic id=\"{id}\"></tf-graph-basic>\n",
    "        </div>\n",
    "    \"\"\".format(data=repr(data), id='graph'+str(np.random.rand()))\n",
    "\n",
    "    iframe = \"\"\"\n",
    "        <iframe seamless style=\"width:{}px;height:{}px;border:0\" srcdoc=\"{}\"></iframe>\n",
    "    \"\"\".format(width, height, code.replace('\"', '&quot;'))\n",
    "    display(HTML(iframe))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Input redirector"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Better solution here: http://stackoverflow.com/questions/41216215/using-pipes-to-capture-things-printed-to-stderr-into-python-variable-from-jupyte"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import os\n",
    "import sys\n",
    "\n",
    "STDOUT = 1\n",
    "STDERR = 2\n",
    "\n",
    "# this hangs when pipe buffer is full\n",
    "\n",
    "class FDRedirector(object):\n",
    "    \"\"\" Class to redirect output (stdout or stderr) at the OS level using\n",
    "        file descriptors.\n",
    "    \"\"\" \n",
    "\n",
    "    def __init__(self, fd=STDOUT):\n",
    "        \"\"\" fd is the file descriptor of the outpout you want to capture.\n",
    "            It can be STDOUT or STERR.\n",
    "        \"\"\"\n",
    "        self.fd = fd\n",
    "        self.started = False\n",
    "        self.piper = None\n",
    "        self.pipew = None\n",
    "\n",
    "    def start(self):\n",
    "        \"\"\" Setup the redirection.\n",
    "        \"\"\"\n",
    "        if not self.started:\n",
    "            self.oldhandle = os.dup(self.fd)\n",
    "            self.piper, self.pipew = os.pipe()\n",
    "            os.dup2(self.pipew, self.fd)\n",
    "            os.close(self.pipew)\n",
    "\n",
    "            self.started = True\n",
    "\n",
    "    def flush(self):\n",
    "        \"\"\" Flush the captured output, similar to the flush method of any\n",
    "        stream.\n",
    "        \"\"\"\n",
    "        if self.fd == STDOUT:\n",
    "            sys.stdout.flush()\n",
    "        elif self.fd == STDERR:\n",
    "            sys.stderr.flush()\n",
    "\n",
    "    def stop(self):\n",
    "        \"\"\" Unset the redirection and return the captured output. \n",
    "        \"\"\"\n",
    "        if self.started:\n",
    "            self.flush()\n",
    "            os.dup2(self.oldhandle, self.fd)\n",
    "            os.close(self.oldhandle)\n",
    "            f = os.fdopen(self.piper, 'r')\n",
    "            output = f.read()\n",
    "            f.close()\n",
    "\n",
    "            self.started = False\n",
    "            return output\n",
    "        else:\n",
    "            return ''\n",
    "\n",
    "    def getvalue(self):\n",
    "        \"\"\" Return the output captured since the last getvalue, or the\n",
    "        start of the redirection.\n",
    "        \"\"\"\n",
    "        output = self.stop()\n",
    "        self.start()\n",
    "        return output\n",
    "\n",
    "redirect=FDRedirector(STDERR)\n",
    "stderr = \"\"\n",
    "\n",
    "from contextlib import contextmanager\n",
    "@contextmanager\n",
    "def grab_stderr():\n",
    "    global stderr\n",
    "    redirect.start();\n",
    "    yield\n",
    "    stderr = redirect.stop()\n",
    "    \n",
    "def open_tag(tag):\n",
    "    sess.run(tf.Print(tf.constant(1), [tf.constant(1)], tag+\"<BEGIN>\"))\n",
    "    \n",
    "def close_tag(tag):\n",
    "    sess.run(tf.Print(tf.constant(1), [tf.constant(1)], tag+\"<END>\"))\n",
    "    \n",
    "def grab_output(tag, filename=\"/tmp/jupyter.txt\"):\n",
    "    s = open(filename).read()\n",
    "    opening_tag = tag+\"<BEGIN>\"\n",
    "    ending_tag = tag+\"<END>\"\n",
    "    starting_pos = s.rfind(opening_tag)\n",
    "    ending_pos = s.rfind(ending_tag)\n",
    "    return s[starting_pos:ending_pos]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Memory timeline utils"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import re\n",
    "import tensorflow as tf\n",
    "\n",
    "# Setup utilities to parse __LOG_MEMORY__ statements in command-line\n",
    "tensor_allocation_regex = re.compile(r\"allocated_bytes: (?P<allocated_bytes>\\d+).*.*allocation_id: (?P<allocation_id>\\d+).*timestamp (?P<timestamp>\\d+)\")\n",
    "tensor_output_regex = re.compile(\"\"\"MemoryLogTensorOutput.* step_id: (?P<step_id>[-0123456789]+) kernel_name: \\\"(?P<kernel_name>[^\"]+).*allocated_bytes: (?P<allocated_bytes>\\d+).*allocation_id: (?P<allocation_id>\\d+).*timestamp (?P<timestamp>\\d+)\"\"\")   \n",
    "tensor_deallocation_regex = re.compile(r\"allocation_id: (?P<allocation_id>\\d+).*timestamp (?P<timestamp>\\d+)\")\n",
    "\n",
    "def _parse_logline(l):\n",
    "    # There are 4 kinds of lines with __LOG_MEMORY__ tag\n",
    "    # MemoryLogTensorDeallocation, MemoryLogTensorAllocation, MemoryLogStep,\n",
    "    #\n",
    "    # Example lines for debugging:\n",
    "    # MemoryLogTensorDeallocation\n",
    "    # 5143420588.000000 file tensorflow/core/framework/log_memory.cc:41] __LOG_MEMORY__ MemoryLogTensorDeallocation { allocation_id: 2 allocator_name: \"cpu\" } timestamp 5143420588459765\n",
    "\n",
    "    # MemoryLogTensorAllocation\n",
    "    # I 5143420588.000000 file tensorflow/core/framework/log_memory.cc:41] __LOG_MEMORY__ MemoryLogTensorAllocation { step_id: -6 kernel_name: \"Unknown (from Proto)\" tensor { dtype: DT_INT32 shape { dim { size: 3 } } allocation_description { requested_bytes: 12 allocated_bytes: 12 allocator_name: \"cpu\" allocation_id: 3 has_single_reference: true ptr: 29496256 } } } timestamp 5143420588526034\n",
    "\n",
    "    # MemoryLogStep\n",
    "    # I 5143420588.000000 file tensorflow/core/framework/log_memory.cc:41] __LOG_MEMORY__ MemoryLogStep { step_id: 1 handle: \"->Print:0//0/;0\" } timestamp 5143420588718348\n",
    "\n",
    "    # MemoryLogTensorOutput\n",
    "    # I 5143420588.000000 file tensorflow/core/framework/log_memory.cc:41] __LOG_MEMORY__ MemoryLogTensorOutput { step_id: 1 kernel_name: \"Const\" tensor { dtype: DT_INT32 shape { dim { size: 3 } } allocation_description { requested_bytes: 12 allocated_bytes: 12 allocator_name: \"cpu\" allocation_id: 3 ptr: 29496256 } } } timestamp 5143420588932740\n",
    "     \n",
    "    # some weird initialization step \n",
    "    # if \"step_id: -6\" in l:\n",
    "    #    return {}\n",
    "    \n",
    "    if 'MemoryLogTensorOutput' in l:\n",
    "        m = tensor_output_regex.search(l)\n",
    "        # assert m, l\n",
    "        if not m:  # some Shape lines are missing bytes info, ie\n",
    "            # I 5162643141.000000 file tensorflow/core/framework/log_memory.cc:41] __LOG_MEMORY__ MemoryLogTensorOutput { step_id: 5 kernel_name: \"gradients/Shape\" tensor { dtype: DT_INT32 shape { dim { } } } } timestamp 5162643141310124\n",
    "            return {}\n",
    "        d = {f: m.group(f) for f in [\"timestamp\", \"allocation_id\",\n",
    "                                     \"allocated_bytes\", \"kernel_name\"]}\n",
    "        d[\"timestamp\"] = int(d[\"timestamp\"])\n",
    "        d[\"allocation_id\"] = int(d[\"allocation_id\"])\n",
    "        d[\"allocated_bytes\"] = int(d[\"allocated_bytes\"])\n",
    "    elif 'MemoryLogTensorAllocation' in l:\n",
    "        m = tensor_allocation_regex.search(l)\n",
    "        d = {f: int(m.group(f)) for f in [\"timestamp\", \"allocation_id\",\n",
    "                                          \"allocated_bytes\"]}\n",
    "        d[\"kernel_name\"] = \"<unknown>\"\n",
    "    elif 'MemoryLogTensorDeallocation' in l:\n",
    "        m = tensor_deallocation_regex.search(l)\n",
    "        assert m, l\n",
    "        d = {f: int(m.group(f)) for f in [\"timestamp\", \"allocation_id\"]}\n",
    "    else:\n",
    "        d = {}\n",
    "    return d\n",
    "\n",
    "def memory_timeline(output):\n",
    "    \"\"\"Returns array of tuples representing memory actions\n",
    "    (timestamp_nanoseconds, kernel_name, memory_bytes)\n",
    "    memory is positive for allocations and negative for deallocations.\n",
    "    timestamps are relative to the first memory allocation line found\n",
    "    \"\"\"\n",
    "    \n",
    "    parsed_lines = [_parse_logline(l) for l in output.split(\"\\n\")]\n",
    "    parsed_lines = [l for l in parsed_lines if l] # remove Nones\n",
    "    parsed_lines.sort(key=lambda l: l[\"timestamp\"])\n",
    "    first_timestamp = parsed_lines[0][\"timestamp\"]\n",
    "    \n",
    "    # Tensors that are output have two records (allocation and output)\n",
    "    # since output comes after, overwrite allocation_map with output\n",
    "    # info (it has kernel_name)\n",
    "    allocation_map = {} # map of allocation_id->parsed_logline of allocation\n",
    "    double_records = {} # map of ids which have both allocation and output\n",
    "    for line in parsed_lines:\n",
    "        if \"kernel_name\" in line:\n",
    "            if line[\"allocation_id\"] in allocation_map:\n",
    "                double_records[line[\"allocation_id\"]] = True\n",
    "            line[\"allocation_id\"]\n",
    "            allocation_map[line[\"allocation_id\"]] = line\n",
    "            \n",
    "\n",
    "    result = []\n",
    "    for line in parsed_lines:\n",
    "        if \"kernel_name\" in line:\n",
    "            # ignore tensorallocation lines which will later have tensoroutput\n",
    "            if (line[\"allocation_id\"] in double_records and\n",
    "                line[\"kernel_name\"] == \"<unknown>\"):\n",
    "                continue\n",
    "            kernel_name = line[\"kernel_name\"]\n",
    "            allocated_bytes = line[\"allocated_bytes\"]\n",
    "        else: # deallocation\n",
    "            allocation_line = allocation_map[line[\"allocation_id\"]]\n",
    "            kernel_name = allocation_line[\"kernel_name\"]\n",
    "            allocated_bytes = -allocation_line[\"allocated_bytes\"]\n",
    "        result.append((line[\"timestamp\"]-first_timestamp, kernel_name,\n",
    "                       allocated_bytes))\n",
    "    return result\n",
    "\n",
    "\n",
    "def print_memory_timeline(stderr):\n",
    "    total_memory = 0\n",
    "    for record in memory_timeline(stderr):\n",
    "        timestamp, kernel_name, allocated_bytes = record\n",
    "        if abs(allocated_bytes)<1000:\n",
    "            continue  # ignore small allocations\n",
    "        total_memory += allocated_bytes\n",
    "        print(\"%9d %40s %11d %11d\"%(timestamp, kernel_name, allocated_bytes, total_memory))\n",
    "\n",
    "def plot_memory_timeline(stderr):\n",
    "    total_memory = 0\n",
    "    timestamps = []\n",
    "    data = []\n",
    "    for record in memory_timeline(stderr):\n",
    "        timestamps.append(record[0]-1)\n",
    "        data.append(total_memory)\n",
    "        total_memory += record[-1]\n",
    "        timestamps.append(record[0])\n",
    "        data.append(total_memory)\n",
    "    plt.plot(timestamps, data)\n",
    "    \n",
    "    \n",
    "run_metadata = tf.RunMetadata()\n",
    "def summarize(run_metadata=run_metadata):\n",
    "    ss = tf.contrib.stat_summarizer.NewStatSummarizer(tf.get_default_graph().as_graph_def().SerializeToString())\n",
    "    ss.ProcessStepStatsStr(run_metadata.step_stats.SerializeToString())\n",
    "    print(ss.GetOutputString())\n",
    "    \n",
    "def create_session():\n",
    "    config = tf.ConfigProto(graph_options=tf.GraphOptions(\n",
    "        optimizer_options=tf.OptimizerOptions(\n",
    "            opt_level=tf.OptimizerOptions.L0)))\n",
    "    return tf.Session(config=config)\n",
    "\n",
    "def printops(ops):\n",
    "    print([op.name for op in ops])\n",
    "    \n",
    "run_options = tf.RunOptions(trace_level=tf.RunOptions.FULL_TRACE)\n",
    "def sessrun(*args, **kwargs):\n",
    "    sess = create_session()\n",
    "    sess.run(tf.global_variables_initializer())\n",
    "    kwargs[\"run_metadata\"] = run_metadata\n",
    "    kwargs[\"options\"] = run_options\n",
    "    return sess.run(*args, **kwargs)\n",
    "\n",
    "import tensorflow.contrib.graph_editor as ge\n",
    "def run_after(a_tensor, b_tensor):\n",
    "    \"\"\"Force a to run after b\"\"\"\n",
    "    ge.reroute.add_control_inputs(a_tensor.op, [b_tensor.op])\n",
    "    "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Simple graph"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "n=1000*1000*25\n",
    "tf.reset_default_graph()\n",
    "a1 = tf.placeholder(tf.float32, name=\"a1\", shape=(n,))\n",
    "a2 = tf.tanh(a1, name=\"a2\")\n",
    "a3 = tf.tanh(a2, name=\"a3\")\n",
    "cost = tf.reduce_sum(a3, name=\"cost\")\n",
    "xs = [a1]\n",
    "ys = [cost]\n",
    "fwd_ops = ge.get_walks_intersection_ops(forward_seed_ops=[x.op for x in xs], backward_seed_ops=[y.op for y in ys])\n",
    "grads = tf.gradients(ys, xs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'redirect' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-8-d8aeefa213b8>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0msess\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcreate_session\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 2\u001b[0;31m \u001b[0mredirect\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstart\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m;\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      3\u001b[0m \u001b[0msess\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrun\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mgrads\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mop\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfeed_dict\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m{\u001b[0m\u001b[0ma1\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mzeros\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mn\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m}\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      4\u001b[0m \u001b[0mstderr\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mredirect\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstop\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      5\u001b[0m \u001b[0mplot_memory_timeline\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mstderr\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mNameError\u001b[0m: name 'redirect' is not defined"
     ]
    }
   ],
   "source": [
    "sess = create_session()\n",
    "redirect.start();\n",
    "sess.run(grads[0].op, feed_dict={a1:np.zeros((n,))})\n",
    "stderr = redirect.stop()\n",
    "plot_memory_timeline(stderr)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " 39971463                                       a2   100000000   100000000\n",
      " 40383258                 gradients/cost_grad/Tile   100000000   200000000\n",
      " 48615535                                       a3   100000000   300000000\n",
      " 62861878               gradients/a3_grad/TanhGrad   100000000   400000000\n",
      " 62953267                                       a3  -100000000   300000000\n",
      " 63341227                 gradients/cost_grad/Tile  -100000000   200000000\n",
      " 74271814               gradients/a2_grad/TanhGrad   100000000   300000000\n",
      " 74369276                                       a2  -100000000   200000000\n",
      " 74920551               gradients/a3_grad/TanhGrad  -100000000   100000000\n",
      " 75289873               gradients/a2_grad/TanhGrad  -100000000           0\n"
     ]
    }
   ],
   "source": [
    "print_memory_timeline(stderr)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "        <iframe seamless style=\"width:1200px;height:800px;border:0\" srcdoc=\"\n",
       "        <script>\n",
       "          function load() {\n",
       "            document.getElementById(&quot;graph0.2321073006319052&quot;).pbtxt = 'node {\\n  name: &quot;a1&quot;\\n  op: &quot;Placeholder&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;shape&quot;\\n    value {\\n      shape {\\n        dim {\\n          size: 25000000\\n        }\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;a2&quot;\\n  op: &quot;Tanh&quot;\\n  input: &quot;a1&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;a3&quot;\\n  op: &quot;Tanh&quot;\\n  input: &quot;a2&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;Const&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n          dim {\\n            size: 1\\n          }\\n        }\\n        int_val: 0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;cost&quot;\\n  op: &quot;Sum&quot;\\n  input: &quot;a3&quot;\\n  input: &quot;Const&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;Tidx&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;keep_dims&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;b_Shape&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n          dim {\\n          }\\n        }\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;b_Const&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n        }\\n        float_val: 1.0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;b_Fill&quot;\\n  op: &quot;Fill&quot;\\n  input: &quot;b_Shape&quot;\\n  input: &quot;b_Const&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;b_cost_grad/Reshape/shape&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n          dim {\\n            size: 1\\n          }\\n        }\\n        int_val: 1\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;b_cost_grad/Reshape&quot;\\n  op: &quot;Reshape&quot;\\n  input: &quot;b_Fill&quot;\\n  input: &quot;b_cost_grad/Reshape/shape&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;Tshape&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;b_cost_grad/Tile/multiples&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n          dim {\\n            size: 1\\n          }\\n        }\\n        int_val: 25000000\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;b_cost_grad/Tile&quot;\\n  op: &quot;Tile&quot;\\n  input: &quot;b_cost_grad/Reshape&quot;\\n  input: &quot;b_cost_grad/Tile/multiples&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;Tmultiples&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;b_a3_grad/TanhGrad&quot;\\n  op: &quot;TanhGrad&quot;\\n  input: &quot;a3&quot;\\n  input: &quot;b_cost_grad/Tile&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;b_a2_grad/TanhGrad&quot;\\n  op: &quot;TanhGrad&quot;\\n  input: &quot;a2&quot;\\n  input: &quot;b_a3_grad/TanhGrad&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\n';\n",
       "          }\n",
       "        </script>\n",
       "        <link rel=&quot;import&quot; href=&quot;https://tensorboard.appspot.com/tf-graph-basic.build.html&quot; onload=load()>\n",
       "        <div style=&quot;height:600px&quot;>\n",
       "          <tf-graph-basic id=&quot;graph0.2321073006319052&quot;></tf-graph-basic>\n",
       "        </div>\n",
       "    \"></iframe>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "show_graph(ungroup_gradients=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['a2']\n"
     ]
    }
   ],
   "source": [
    "# save everything between a3 and a1\n",
    "remember_ts = ge.filter_ts_from_regex(fwd_ops, \"a1|a3\")\n",
    "intermediate_ops = ge.get_walks_intersection_ops(forward_seed_ops=[remember_ts[0].op], backward_seed_ops=[remember_ts[1].op])\n",
    "intermediate_ops = intermediate_ops[1:-1]  # remove starting/endpoints\n",
    "printops(intermediate_ops)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:VARIABLES collection name is deprecated, please use GLOBAL_VARIABLES instead; VARIABLES will be removed after 2017-03-02.\n",
      "WARNING:tensorflow:VARIABLES collection name is deprecated, please use GLOBAL_VARIABLES instead; VARIABLES will be removed after 2017-03-02.\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "        <iframe seamless style=\"width:1200px;height:800px;border:0\" srcdoc=\"\n",
       "        <script>\n",
       "          function load() {\n",
       "            document.getElementById(&quot;graph0.11356853209468576&quot;).pbtxt = 'node {\\n  name: &quot;a1&quot;\\n  op: &quot;Placeholder&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;shape&quot;\\n    value {\\n      shape {\\n        dim {\\n          size: 25000000\\n        }\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;a2&quot;\\n  op: &quot;Tanh&quot;\\n  input: &quot;a1&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;a3&quot;\\n  op: &quot;Tanh&quot;\\n  input: &quot;a2&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;Const&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n          dim {\\n            size: 1\\n          }\\n        }\\n        int_val: 0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;cost&quot;\\n  op: &quot;Sum&quot;\\n  input: &quot;a3&quot;\\n  input: &quot;Const&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;Tidx&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;keep_dims&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;b_Shape&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n          dim {\\n          }\\n        }\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;b_Const&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n        }\\n        float_val: 1.0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;b_Fill&quot;\\n  op: &quot;Fill&quot;\\n  input: &quot;b_Shape&quot;\\n  input: &quot;b_Const&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;b_cost_grad/Reshape/shape&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n          dim {\\n            size: 1\\n          }\\n        }\\n        int_val: 1\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;b_cost_grad/Reshape&quot;\\n  op: &quot;Reshape&quot;\\n  input: &quot;b_Fill&quot;\\n  input: &quot;b_cost_grad/Reshape/shape&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;Tshape&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;b_cost_grad/Tile/multiples&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n          dim {\\n            size: 1\\n          }\\n        }\\n        int_val: 25000000\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;b_cost_grad/Tile&quot;\\n  op: &quot;Tile&quot;\\n  input: &quot;b_cost_grad/Reshape&quot;\\n  input: &quot;b_cost_grad/Tile/multiples&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;Tmultiples&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;b_a3_grad/TanhGrad&quot;\\n  op: &quot;TanhGrad&quot;\\n  input: &quot;a3&quot;\\n  input: &quot;b_cost_grad/Tile&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;b_a2_grad/TanhGrad&quot;\\n  op: &quot;TanhGrad&quot;\\n  input: &quot;a2&quot;\\n  input: &quot;b_a3_grad/TanhGrad&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;a2_1&quot;\\n  op: &quot;Tanh&quot;\\n  input: &quot;a1&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\n';\n",
       "          }\n",
       "        </script>\n",
       "        <link rel=&quot;import&quot; href=&quot;https://tensorboard.appspot.com/tf-graph-basic.build.html&quot; onload=load()>\n",
       "        <div style=&quot;height:600px&quot;>\n",
       "          <tf-graph-basic id=&quot;graph0.11356853209468576&quot;></tf-graph-basic>\n",
       "        </div>\n",
       "    \"></iframe>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "copied_sgv, info = ge.copy_with_input_replacements(ge.sgv(intermediate_ops), {})\n",
    "show_graph(ungroup_gradients=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "        <iframe seamless style=\"width:1200px;height:800px;border:0\" srcdoc=\"\n",
       "        <script>\n",
       "          function load() {\n",
       "            document.getElementById(&quot;graph0.5132368925062993&quot;).pbtxt = 'node {\\n  name: &quot;a1&quot;\\n  op: &quot;Placeholder&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;shape&quot;\\n    value {\\n      shape {\\n        dim {\\n          size: 25000000\\n        }\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;a2&quot;\\n  op: &quot;Tanh&quot;\\n  input: &quot;a1&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;a3&quot;\\n  op: &quot;Tanh&quot;\\n  input: &quot;a2&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;Const&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n          dim {\\n            size: 1\\n          }\\n        }\\n        int_val: 0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;cost&quot;\\n  op: &quot;Sum&quot;\\n  input: &quot;a3&quot;\\n  input: &quot;Const&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;Tidx&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;keep_dims&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;gradients/Shape&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n          dim {\\n          }\\n        }\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;gradients/Const&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n        }\\n        float_val: 1.0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;gradients/Fill&quot;\\n  op: &quot;Fill&quot;\\n  input: &quot;gradients/Shape&quot;\\n  input: &quot;gradients/Const&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;gradients/cost_grad/Reshape/shape&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n          dim {\\n            size: 1\\n          }\\n        }\\n        int_val: 1\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;gradients/cost_grad/Reshape&quot;\\n  op: &quot;Reshape&quot;\\n  input: &quot;gradients/Fill&quot;\\n  input: &quot;gradients/cost_grad/Reshape/shape&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;Tshape&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;gradients/cost_grad/Tile/multiples&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n          dim {\\n            size: 1\\n          }\\n        }\\n        int_val: 25000000\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;gradients/cost_grad/Tile&quot;\\n  op: &quot;Tile&quot;\\n  input: &quot;gradients/cost_grad/Reshape&quot;\\n  input: &quot;gradients/cost_grad/Tile/multiples&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;Tmultiples&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;gradients/a3_grad/TanhGrad&quot;\\n  op: &quot;TanhGrad&quot;\\n  input: &quot;a3&quot;\\n  input: &quot;gradients/cost_grad/Tile&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;gradients/a2_grad/TanhGrad&quot;\\n  op: &quot;TanhGrad&quot;\\n  input: &quot;a2_1&quot;\\n  input: &quot;gradients/a3_grad/TanhGrad&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;a2_1&quot;\\n  op: &quot;Tanh&quot;\\n  input: &quot;a1&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\n';\n",
       "          }\n",
       "        </script>\n",
       "        <link rel=&quot;import&quot; href=&quot;https://tensorboard.appspot.com/tf-graph-basic.build.html&quot; onload=load()>\n",
       "        <div style=&quot;height:600px&quot;>\n",
       "          <tf-graph-basic id=&quot;graph0.5132368925062993&quot;></tf-graph-basic>\n",
       "        </div>\n",
       "    \"></iframe>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "bwd_ops = ge.filter_ops_from_regex(tf.get_default_graph(), \"gradients/\")\n",
    "intermediate_copied_ops = [info._transformed_ops[y] for y in intermediate_ops]\n",
    "intermediate_copied_ts = [op.outputs[0] for op in intermediate_copied_ops]\n",
    "intermediate_ts = [op.outputs[0] for op in intermediate_ops]\n",
    "# b left dangling\n",
    "ge.reroute_ts(intermediate_copied_ts, intermediate_ts, can_modify=bwd_ops)\n",
    "show_graph()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def run_op_after(a_op, b_op):\n",
    "    \"\"\"Force a to run after b\"\"\"\n",
    "    print(\"Adding %s to run after %s\"%(a_op.name, b_op.name))\n",
    "    ge.reroute.add_control_inputs(a_op, [b_op])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Adding a2_1 to run after gradients/a3_grad/TanhGrad\n"
     ]
    }
   ],
   "source": [
    "# force copied intermediate ops to run after earliest backprop op\n",
    "cost_grad = ge.filter_ops_from_regex(tf.get_default_graph(), \"gradients/a3_grad/TanhGrad\")\n",
    "for cost_op in cost_grad:\n",
    "    for intermediate_op in intermediate_copied_ops:\n",
    "        run_op_after(intermediate_op, cost_op)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAgsAAAF+CAYAAAAMWFkhAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAAPYQAAD2EBqD+naQAAHvtJREFUeJzt3XuUZVVh5/HvDzAQHjZOHGgNBEYephldYJWvjgGMCIjJ\nQNT4KEGJTnAQspZpVoLK4DDJGuNyjHQwCYnKxNBBKjExQSYGeQQDozSypgsw0YZulzA8ImCUKVAe\nDvSeP84puRRVu/rcqrr3Fnw/a51VfffZ+559dlXf+7v7nHtOSilIkiTNZ4dhd0CSJI02w4IkSaoy\nLEiSpCrDgiRJqjIsSJKkKsOCJEmqMixIkqQqw4IkSaoyLEiSpCrDgiRJqhrpsJDk8CSXJrk7ybYk\nx/fxHMcm2ZjkgST3JfnrJPstR38lSXo6GumwAOwG3AScDnS+iUWS/YFLgKuAQ4FjgOcCn1+yHkqS\n9DSXlXIjqSTbgF8upVzaU/YTwO8CbwP2BP4J+EAp5Zp2/ZuAi0spO/e0+SWaALFzKeXxAe6CJEkr\n0qjPLCzkj4BXAG8BXgz8FXBZkgPa9ZuAbUnelWSHJKuAdwBXGhQkSdo+K3ZmIcm+wLeBfUsp9/TU\nuxL4Winl7PbxEcDngJ8CdgQ2AseVUh4Y8C5IkrQireSZhRfTvPlvSfLgzAIcARwAkGRv4NPAZ4CX\ntusexXMWJEnabjsNuwOLsDvwGDAGbJu17gftz9OB6VLKB2dWJHkHcGeSl5dSbhhITyVJWsFWcli4\nkWZmYe9SylfnqbMrMPvchJlgsZJnVSRJGphOb5hJTk1yc5Lpdrkuyesq9U9ur4/wePtzW5KHOmxv\ntySHJjmsLXpB+3jfUspW4GJgQ5I3JNk/ycuTfCDJcW39LwIvS/KhJAcmGaM5JHEbTdiQJEkL6Prp\n+k7g/cB4u1wNfCHJmkqbaWB1z9LlgkgvpXlT30RznYWPA1PAb7frfxXYAPwecAvwt22bOwBKKV8G\n3g6c0Lb7e+BhmhMcH+3QD0mSnrEW/W2IJN8DfrOU8pk51p0MrC+l/JtFbUSSJA1N38ft2+sWvI3m\nvICNlaq7J7k9yR1JLklySL/blCRJg9f5BMckL6IJB7sADwJvKKXcMk/1W4F3A18HVgG/BVyX5N+X\nUu6ubOOngGOB24FHuvZRkqRnsF2A/YHLSynfW4on7HwYIslOwM/QXF75TcApwBGVwDC77WaaSzCf\nU6n3duCznTomSZJ6nVhKuXgpnqjzzEIp5TGaKycCTCV5OfA+4L3b0zbJjcCBC1S9HeCiiy5izZra\nuZNPf+vWrWP9+vXD7sbQOQ5PcCwajsMTHIuG49DYvHkzJ510ErTvpUthKa6zsAOw84K1aM5zAF5E\n862EmkcA1qxZw9jY2OJ6t8KtWrXqGT8G4Dj0ciwajsMTHIuG4/AUS3YYv1NYSPJh4DKar1DuAZwI\nHElz62eSbADuKqWc1T7+EHA98C2awxZn0nx18oIl6r8kSVpmXWcW9qa5rsHzaK6f8HXgmFLK1e36\nfWguwTzjOcCnaK6vcD/N9RLWbs/5DZIkaTR0CgullF9bYP1rZj0+Azijj35JkqQR4f0RRtzExMSw\nuzASHIcnOBYNx+EJjkXDcVg+i76C43Jo7+GwadOmTZ6sIklSB1NTU4yPjwOMl1KmluI5nVmQJElV\nhgVJklRlWJAkSVWGBUmSVGVYkCRJVYYFSZJUZViQJElVhgVJklRlWJAkSVWGBUmSVGVYkCRJVYYF\nSZJUZViQJElVhgVJklRlWJAkSVWGBUmSVGVYkCRJVYYFSZJUZViQJElVhgVJklRlWJAkSVWGBUmS\nVGVYkCRJVYYFSZJUZViQJElVhgVJklRlWJAkSVWGBUmSVGVYkCRJVYYFSZJUZViQJElVncJCklOT\n3Jxkul2uS/K6Bdq8OcnmJA+3bY9bXJclSdIgdZ1ZuBN4PzDeLlcDX0iyZq7KSdYCFwOfBg4DLgEu\nSXJI3z2WJEkD1SkslFK+WEr5UinlW+1yNvAD4JXzNHkfcFkp5dxSyq2llHOAKeDXF9dtSZI0KH2f\ns5BkhyRvA3YFNs5TbS1w1ayyy9tySZK0AuzUtUGSF9GEg12AB4E3lFJumaf6auDeWWX3tuWSJGkF\n6Gdm4RbgUOAVwB8DG5L8bIf2AUof25Wkobplvo9FQzaq/Voptm5tFs2v88xCKeUx4Nvtw6kkL6c5\nN+G9c1S/B9h7VtlePHW2YU7r1q1j1apVTyqbmJhgYmKiU58labGuugqOPrr5edRRw+7NE0a1XyvF\n1q1w8MHNv7dsgYMOGm5/upqcnGRycvJJZdPT00u+nc5hYQ47ADvPs24jcBTwiZ6yo5n/HIcnWb9+\nPWNjY4vrnSQtgdtvb37edttQu/EUo9qvleLBB+f+90ox1wfoqakpxsfHl3Q7ncJCkg8Dl9F8hXIP\n4ETgSOCYdv0G4K5Sylltk/OAa5KcAXwRmKD5yuUpS9J7SZK07LrOLOwNbACeB0wDXweOKaVc3a7f\nB3hspnIpZWOSCeDD7bIVOKGU8s3FdlySJA1Gp7BQSvm1Bda/Zo6yzwOf79gvSZI0Irw3hCRJqjIs\nSJKkKsOCJEmqMixIkqQqw4IkSaoyLEiSpCrDgiRJqjIsSJKkKsOCJEmqMixIkqQqw4IkSaoyLEiS\npCrDgiRJqjIsSJKkKsOCJEmqMixIkqQqw4IkSaoyLEiSpCrDgiRJqjIsSJKkKsOCJEmqMixIkqQq\nw4IkSaoyLEiSpCrDgiRJqjIsSJKkKsOCJEmqMixIkqQqw4IkSaoyLEiSpCrDgiRJqjIsSJKkKsOC\nJEmqMixIkqSqTmEhyQeT3JDkgST3JvnbJAcv0ObkJNuSPN7+3JbkocV1W5IkDUrXmYXDgT8AXgG8\nFngWcEWSn1yg3TSwumfZr+N2JUnSkOzUpXIp5fW9j5P8KnAfMA58pd60fLdz7yRJ0tAt9pyFPYEC\nfH+BersnuT3JHUkuSXLIIrcrSZIGpO+wkCTA7wNfKaV8s1L1VuDdwPHAie02r0vy0/1uW5IkDU6n\nwxCznA8cAryqVqmUcj1w/czjJBuBzcB7gHNqbdetW8eqVaueVDYxMcHExESfXZYk6eljcnKSycnJ\nJ5VNT08v+Xb6CgtJ/hB4PXB4KeU7XdqWUh5LciNw4EJ1169fz9jYWD9dlCTpaW+uD9BTU1OMj48v\n6XY6H4Zog8IJwC+UUu7oo/0OwIuATiFDkiQNR6eZhSTnAxM05x/8MMne7arpUsojbZ0LgbtLKWe1\njz9EcxjiWzQnRJ5J89XJC5ZkDyRJ0rLqehjiVJpvP/zjrPJ3ARvaf+8LPN6z7jnAp2iur3A/sAlY\nW0q5pWtnJUnS4HW9zsKChy1KKa+Z9fgM4IyO/ZIkSSPCe0NIkqQqw4IkSaoyLEiSpCrDgiRJqjIs\nSJKkKsOCJEmqMixIkqQqw4IkSaoyLEiSpCrDgiRJqjIsSJKkKsOCJEmqMixIkqQqw4IkSaoyLEiS\npCrDgiRJqjIsSJKkKsOCJEmqMixIkqQqw4IkSaoyLEiSpCrDgiRJqjIsSJKkKsOCJEmqMixIkqQq\nw4IkSaoyLEiSpCrDgiRJqjIsSJKkKsOCJEmqMixIkqQqw4IkSaoyLEiSpKpOYSHJB5PckOSBJPcm\n+dskB29Huzcn2Zzk4SQ3Jzmu/y5LkqRB6jqzcDjwB8ArgNcCzwKuSPKT8zVIsha4GPg0cBhwCXBJ\nkkP66rEkSRqonbpULqW8vvdxkl8F7gPGga/M0+x9wGWllHPbx+ckOQb4deC0Tr2VJEkDt9hzFvYE\nCvD9Sp21wFWzyi5vyyVJ0ojrNLPQK0mA3we+Ukr5ZqXqauDeWWX3tuWStCy2boUHH1y659u8+Ymf\nU1NL97yL1duvrVvhoIOG259eS/07WA4z4zfz7z32GK0xHBUppfTXMPlj4FjgVaWU71TqPQq8s5Ty\nlz1lpwFnl1KeP0+bMWDTEUccwapVq560bmJigomJib76LOmZYetWOHjBU6+fnrZsGY03u5X8OxiV\nMdwek5OTTE5OPqlsenqaa6+9FmC8lLIk0bavmYUkfwi8Hji8FhRa9wB7zyrbi6fONjzF+vXrGRsb\n66eLkp7BZj7NXnQRrFmzdM97zz2wegTnRO+5B+6/H046aXQ+yS/X72A57LFH8/OGG0ZrDLfHXB+g\np6amGB8fX9LtdA4LbVA4ATiylHLHdjTZCBwFfKKn7Oi2XJKWzZo18Ez5vDFKh0Z6raTfwUoKCYPW\nKSwkOR+YAI4HfphkZsZgupTySFvnQuDuUspZ7brzgGuSnAF8sW0/DpyyBP2XJEnLrOu3IU4Fng38\nI/AvPctbeursS8/Ji6WUjTQB4T3ATcAbgRMWOClSkiSNiK7XWVgwXJRSXjNH2eeBz3fZliRJGg3e\nG0KSJFUZFiRJUpVhQZIkVRkWJElSlWFBkiRVGRYkSVKVYUGSJFUZFiRJUpVhQZIkVRkWJElSlWFB\nkiRVGRYkSVKVYUGSJFUZFiRJUpVhQZIkVRkWJElSlWFBkiRVGRYkSVKVYUGSJFUZFiRJUpVhQZIk\nVRkWJElSlWFBkiRVGRYkSVKVYUGSJFUZFiRJUpVhQZIkVRkWJElSlWFBkiRVGRYkSVKVYUGSJFUZ\nFiRJUpVhQZIkVXUOC0kOT3JpkruTbEty/AL1j2zr9S6PJ9mr/25LkqRB6WdmYTfgJuB0oGxnmwIc\nBKxul+eVUu7rY9uSJGnAduraoJTyJeBLAEnSoel3SykPdN2eJEkarkGdsxDgpiT/kuSKJD83oO1K\nkqRFGkRY+A7wn4A3AW8E7gT+MclhA9i2JElapM6HIboqpWwBtvQUXZ/kAGAdcHKt7bp161i1atWT\nyiYmJpiYmFjyfkqStNJMTk4yOTn5pLLp6ekl386yh4V53AC8aqFK69evZ2xsbADdkSRp5ZnrA/TU\n1BTj4+NLup1hXWfhMJrDE5IkacR1nllIshtwIM1JiwAvSHIo8P1Syp1JPgI8v5Ryclv/fcBtwDeA\nXYBTgF8Ajl6C/kuSpGXWz2GIlwJfprl2QgE+3pZfCLyb5joK+/bU/4m2zvOBh4CvA0eVUq7ts8+S\nJGmA+rnOwjVUDl+UUt416/HHgI9175okSRoF3htCkiRVGRYkSVKVYUGSJFUZFiRJUpVhQZIkVRkW\nJElSlWFBkiRVGRYkSVKVYUGSJFUZFiRJUpVhQZIkVRkWJElSlWFBkiRVGRYkSVKVYUGSJFUZFiRJ\nUpVhQZIkVRkWJElSlWFBkiRVGRYkSVKVYUGSJFUZFiRJUpVhQZIkVRkWJElSlWFBkiRVGRYkSVKV\nYUGSJFUZFiRJUpVhQZIkVRkWJElSlWFBkiRVGRYkSVKVYUGSJFUZFiRJUlXnsJDk8CSXJrk7ybYk\nx29Hm1cn2ZTkkSRbkpzcX3clSdKg9TOzsBtwE3A6UBaqnGR/4O+AfwAOBc4DLkhydB/bliRJA7ZT\n1wallC8BXwJIku1o8l7g26WUM9vHtyb5eWAdcGXX7UuSpMEaxDkLrwSumlV2ObB2ANuWJEmL1Hlm\noQ+rgXtnld0LPDvJzqWURwfQB0lLbOtWePDBYfdibps3D7sHwzMq+z4q/ejHXXfB2NiwezFaBhEW\n5jJz+KJ6zsO6detYtWrVk8omJiaYmJhYrn5J2g5bt8LBBw+7FwvbY49h92BwZvb1pJOG24/ZVtLv\nYKavJ5wAW7bAQQcNtz/bY3JyksnJySeVTU9PL/l2BhEW7gH2nlW2F/BAKeVHtYbr169nzHgnjZyZ\nGYWLLoI1a4bbl/nsscfKeLFfKgcd1LzBjdJsz0r7HRx0EHzhC01YGKVxrJnrA/TU1BTj4+NLup1B\nhIWNwHGzyo5pyyWtYGvWOF07SlbSG/Oo2mefYfdgNPVznYXdkhya5LC26AXt433b9R9JcmFPkz8B\nDkjy0SQvTHIa8CvAuYvuvSRJWnb9fBvipcCNwCaacw4+DkwBv92uXw3sO1O5lHI78IvAa2muz7AO\n+I+llNnfkJAkSSOon+ssXEMlZJRS3jVPm6U9gCJJkgbCe0NIkqQqw4IkSaoyLEiSpCrDgiRJqjIs\nSJKkKsOCJEmqMixIkqQqw4IkSaoyLEiSpCrDgiRJqjIsSJKkKsOCJEmqMixIkqQqw4IkSaoyLEiS\npCrDgiRJqjIsSJKkKsOCJEmqMixIkqQqw4IkSaoyLEiSpCrDgiRJqjIsSJKkKsOCJEmqMixIkqQq\nw4IkSaoyLEiSpCrDgiRJqjIsSJKkKsOCJEmqMixIkqQqw4IkSaoyLEiSpKq+wkKS05PcluThJNcn\neVml7slJtiV5vP25LclD/XdZkiQNUuewkOStwMeBc4CXADcDlyd5bqXZNLC6Z9mve1clSdIw9DOz\nsA74ZCllQynlFuBU4CHg3ZU2pZTy3VLKfe3y3X46K0mSBq9TWEjyLGAc+IeZslJKAa4C1laa7p7k\n9iR3JLkkySF99VaSJA1c15mF5wI7AvfOKr+X5vDCXG6lmXU4Hjix3eZ1SX6647YlSdIQ7LREzxOg\nzLWilHI9cP2PKyYbgc3Ae2jOe5jXunXrWLVq1ZPKJiYmmJiYWGx/JUla8SYnJ5mcnHxS2fT09JJv\np2tY+FfgcWDvWeV78dTZhjmVUh5LciNw4EJ1169fz9jYWMcuSpL0zDDXB+ipqSnGx8eXdDudDkOU\nUv4fsAk4aqYsSdrH123PcyTZAXgR8J0u25YkScPRz2GIc4ELk2wCbqD5dsSuwJ8BJNkA3FVKOat9\n/CGawxDfAvYEzqT56uQFi+28JElafp3DQinlc+01FX6H5nDETcCxPV+H3Ad4rKfJc4BP0ZwAeT/N\nzMTa9muXkiRpxPV1gmMp5Xzg/HnWvWbW4zOAM/rZjiRJGj7vDSFJkqoMC5IkqcqwIEmSqgwLkiSp\nyrAgSZKqDAuSJKnKsCBJkqoMC5IkqcqwIEmSqgwLkiSpyrAgSZKqDAuSJKnKsCBJkqoMC5Ikqcqw\nIEmSqgwLkiSpyrAgSZKqDAuSJKnKsCBJkqoMC5IkqcqwIEmSqgwLkiSpyrAgSZKqDAuSJKnKsCBJ\nkqoMC5IkqcqwIEmSqgwLkiSpyrAgSZKqDAuSJKnKsCBJkqoMC5IkqcqwMOImJyeH3YWR4Dg8wbFo\nOA5PcCwajsPy6SssJDk9yW1JHk5yfZKXLVD/zUk2t/VvTnJcf9195vGPv+E4PMGxaDgOT3AsGo7D\n8ukcFpK8Ffg4cA7wEuBm4PIkz52n/lrgYuDTwGHAJcAlSQ7pt9OSJGlw+plZWAd8spSyoZRyC3Aq\n8BDw7nnqvw+4rJRybinl1lLKOcAU8Ot99ViSJA1Up7CQ5FnAOPAPM2WllAJcBaydp9nadn2vyyv1\nJUnSCNmpY/3nAjsC984qvxd44TxtVs9Tf3VlO7sAbN68uWP3nl4efhjuumuaz352athdGTrH4Qmj\nMBa33db8HOZ/0enpaaam/JsAx2LGUozDzN/0Sn776Xnv3GWpnjPNxMB2Vk6eB9wNrC2lfK2n/L8D\nP19K+bk52jwKvLOU8pc9ZacBZ5dSnj/Pdt4OfHa7OyZJkmY7sZRy8VI8UdeZhX8FHgf2nlW+F0+d\nPZhxT8f60BymOBG4HXikYx8lSXom2wXYn+a9dEl0mlkASHI98LVSyvvaxwHuAD5RSvnYHPX/AvjJ\nUsoJPWVfBW4upZy2mM5LkqTl13VmAeBc4MIkm4AbaL4dsSvwZwBJNgB3lVLOauufB1yT5Azgi8AE\nzUmSpyyu65IkaRA6h4VSyufaayr8Ds3hhZuAY0sp322r7AM81lN/Y5IJ4MPtshU4oZTyzcV2XpIk\nLb/OhyEkSdIzi/eGkCRJVYYFSZJUNRJhIclzknw2yXSS+5NckGS3Bep/IsktSX6Y5P8kOS/JswfZ\n76XgTbkaXcYhya8luTbJ99vlyoXGbSXp+jfR0+5tSbYl+Zvl7uMg9PF/Y1WSP0ryL22bW5K8blD9\nXU59jMVvtPv/UJI7kpybZOdB9Xc5JDk8yaVJ7m7/zo/fjjavTrIpySNJtiQ5eRB9XU5dxyHJG5Jc\nkeS+9j32uiTHdN3uSIQFmhtNrQGOAn4ROAL4ZKX+84HnAWcALwJOBl4HXLC83Vxa3pSr0XUcgCNp\nxuHVwCuBO4Er2ouGrWh9jMVMu/2AjwHXLnsnB6CP/xvPorms/M8Ab6S5ouwpNBeRW9H6GIu3Ax9p\n6/8szX173kpzgvlKthvNCfWnAwuebJdkf+DvaG5PcCjNN/MuSHL08nVxIDqNA8376RXAccAY8GXg\nfyY5tNNWSylDXWj+mLcBL+kpO5bmGxWrOzzPrwAPAzsMe5869Pl64LyexwHuAs6cp/5fAJfOKtsI\nnD/sfRnkOMzRfgdgGjhp2PsyjLFo9/9/Ae8CPgP8zbD3Y9DjQHNDu63AjsPu+wiMxR8AV84q+z3g\n2mHvyxKOyTbg+AXqfBT4+qyySeDvh93/QY7DPO3+meYqytvdZhRmFtYC95dSbuwpu4omMb2iw/Ps\nCTxQStm2lJ1bLt6Uq9HnOMy2G/As4PtL3sEBWsRYnAPcV0r5zPL2cDD6HIf/QBuck9yT5J+SfDDJ\nKLzG9a3PsbgOGJ85VJHkBcDraa5z80zySp5mr5dLob2Q4h50fL3s56JMS201cF9vQSnl8STfp36z\nqR9rp+POpn7oYtQM6qZco66fcZjtozTTzbNfGFaazmOR5FU0MwrdphRHWz9/Ey8AXgNcRDPdehBw\nfvs8/215ujkQnceilDLZviZ+pX1j2BH4k1LKR5e1p6NnvtfLZyfZuZTy6BD6NAp+i+YD1ue6NFq2\n1J3kI+3JF/Mtjyc5uPYUbN9xqT1oEvM/A7+9RN0fpu3a70XUXym29/f/AeAtwC+XUn607L0ajjnH\nIsnuwJ8Dp5RS7h94rwav9jexA80bwXtKKTeWUj5Hc4z+vYPq3IDNOxZJXg2cRXNo5iU053D8UpKz\nB9a70ZX259PxNXNB7fksHwLeXEr51y5tl3Nm4fdojp/WfJvmRlN79RYm2RF4DvWbTc28WF4O/F/g\njaWUx/vu7eAN6qZco66fcQAgyW8CZwJHlVK+sTzdG6iuY3EAsB/NyUozL4I7ACT5EfDCUspty9TX\n5dTP38R3gB+1U/QzNgOrk+xUSnlsnnajrp+x+B1gQ89hqW+0r5WfZGXPsnQ13+vlA0/jDxbzSvI2\n4FPAr5RSvty1/bLNLJRSvldK2bLA8hjNccY9k7ykp/lRNAnwa3M+OT+eUbiC5qTG41faL7+U8v+A\nTTT7Cvz4WNJRNMcc57Kxt37r6LZ8RepzHEjyW8B/prnU+I3z1VtJ+hiLzcCLab4Zc2i7XApc3f77\nzmXu8rLo82/iq8CBs8peCHxnBQeFfsdiV5oT33pta5tmjvpPV3O9Xh7DCn697FeaWy78D2CilPKl\nvp5k2Gdzth8E/h7438DLgFcBtwJ/3rP++TQvjC9tH+9Oc4bwTcC/o0mPM8tK+jbEW2jCzjtpvhXy\nSeB7wL9t128Afren/lrgRzRfGX0h8F9pbuF9yLD3ZcDjcGa732+Y9bvfbdj7MuixmKP90+XbEF3/\nJvah+UbMeTTnK/wizSfLDwx7X4YwFufQzLa+leY2xUfTfFPk4mHvyyLHYTeaEHwYTfj5jfbxvu36\njwAX9tTfH/gBzTlNLwROa18/XzvsfRnwOEy0+33qrNfLZ3fa7rB3vN2ZPWlOTJoG7qe5jsCuPev3\no5mKO6J9fGT7uHfZ1v78mWHvT8d9Pw24vX0x2EgbiNp1VwN/Oqv+m4Bb2vpfp/lkPfT9GOQ4ALfN\n8ft/HPgvw96PYfxNzGr7tAgL/YwDzbenrgMeat8c3097/5uVvnT8/7EDzXHpLcAP23af6PrmMGpL\n+7o/8zrfu/xpu/4zwNVztNnUjttW4B3D3o9BjwPNdRXmer2c93VkrsUbSUmSpKoV/R1kSZK0/AwL\nkiSpyrAgSZKqDAuSJKnKsCBJkqoMC5IkqcqwIEmSqgwLkiQNUJLDk1ya5O72xorH9/EcxybZmOSB\nJPcl+esk+y1Hf8GwIEnSoO1Gc7uC0+njDphJ9gcuAa6iudTzMTS3M//8kvVw9ja9gqMkScORZBvw\ny6WUS3vKfgL4XeBtNLdD+Cea+5xc065/E829PnbuafNLNAFi57IMd2B2ZkGSpNHyRzT3OnkLzZ1l\n/wq4LMkB7fpNwLYk70qyQ5JVwDuAK5cjKIAzC5IkDc3smYUk+wLfprmL5D099a4EvlZKObt9fATw\nOeCngB1pbjB2XCnlgeXopzMLkiSNjhfTvPlvSfLgzAIcARwAkGRvmrszfwZ4abvuUZbxnIWdluuJ\nJUlSZ7sDjwFjNLei7vWD9ufpwHQp5YMzK5K8A7gzyctLKTcsdacMC5IkjY4baWYW9i6lfHWeOrsC\ns89NmAkWy3LEwMMQkiQNUJLdkhya5LC26AXt431LKVuBi4ENSd6QZP8kL0/ygSTHtfW/CLwsyYeS\nHJhkjOaQxG00YWPp++wJjpIkDU6SI4Ev89RrLFxYSnl3kh2Bs4F3Aj8NfI/mBMZzSinfaJ/jLcCZ\nwMHAQ+3695dStixLnw0LkiSpxsMQkiSpyrAgSZKqDAuSJKnKsCBJkqoMC5IkqcqwIEmSqgwLkiSp\nyrAgSZKqDAuSJKnKsCBJkqoMC5Ikqer/A6ND1FJab/CRAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7fb150684518>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "sess = create_session()\n",
    "redirect.start();\n",
    "sess.run(grads[0].op, feed_dict={a1:np.zeros((n,))})\n",
    "stderr = redirect.stop()\n",
    "plot_memory_timeline(stderr)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " 50385696                                       a2   100000000   100000000\n",
      " 62436162                 gradients/cost_grad/Tile   100000000   200000000\n",
      " 68648386                                       a3   100000000   300000000\n",
      " 68695205                                       a2  -100000000   200000000\n",
      " 80062870               gradients/a3_grad/TanhGrad   100000000   300000000\n",
      " 80151949                                       a3  -100000000   200000000\n",
      " 80628014                 gradients/cost_grad/Tile  -100000000   100000000\n",
      " 91433279                                     a2_1   100000000   200000000\n",
      "104269881               gradients/a2_grad/TanhGrad   100000000   300000000\n",
      "104355949                                     a2_1  -100000000   200000000\n",
      "104935598               gradients/a3_grad/TanhGrad  -100000000   100000000\n",
      "105331258               gradients/a2_grad/TanhGrad  -100000000           0\n"
     ]
    }
   ],
   "source": [
    "print_memory_timeline(stderr)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python [default]",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  },
  "toc": {
   "colors": {
    "hover_highlight": "#DAA520",
    "running_highlight": "#FF0000",
    "selected_highlight": "#FFD700"
   },
   "moveMenuLeft": true,
   "nav_menu": {
    "height": "120px",
    "width": "252px"
   },
   "navigate_menu": true,
   "number_sections": false,
   "sideBar": true,
   "threshold": 4,
   "toc_cell": false,
   "toc_section_display": "block",
   "toc_window_display": true
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}