adaptive-computation.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import keras\n",
    "import keras.backend as K"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "(X_train,Y_train),(X_test,Y_test) = keras.datasets.reuters.load_data()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYAAAAD9CAYAAAC1DKAUAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvNQv5yAAAGzZJREFUeJzt3XuQXGd55/HvM93TM5qLpJnReKzLyJJlObZs8IXxDcPGibEthIPMbvDaC7bwGpRK7K2wSW2VYdkyhZMqsgmkYJeYCKzFEF9wAjEKaPEaQeLarcjWGIQsybdBWNeRZqyRRnPv7tPP/tGnR22j8cxoevoy5/ep6urT73m7+311eZ9znvc9fczdERGR6KkqdQNERKQ0FABERCJKAUBEJKIUAEREIkoBQEQkohQAREQiatIAYGbtZvYzM9trZnvM7I/D8s+b2WEz2xk+1uW95zNm1mVmr5rZLXnla8OyLjN7YHa6JCIiU2GTXQdgZouBxe7+czNrBF4EbgNuBwbd/a/eVn8N8ARwNbAE+AlwYbj7NeAm4BCwA7jT3fcWrjsiIjJV8ckquHs30B1uD5jZy8DSd3jLeuBJdx8Dfm1mXWSDAUCXu+8DMLMnw7oKACIiJTCtOQAzWwFcATwfFt1vZrvMbLOZNYVlS4GDeW87FJZNVC4iIiUw6RlAjpk1AN8DPu3up8zsYeAhwMPnLwH/caYNMrONwEaA+vr691x00UUz/UgRkUh58cUX33T31snqTSkAmFk12cH/MXf/PoC7H8vb/w3gh+HLw0B73tuXhWW8Q/k4d98EbALo6Ojwzs7OqTRRRERCZrZ/KvWmsgrIgEeAl939y3nli/OqfQTYHW5vAe4wsxozWwmsBl4gO+m72sxWmlkCuCOsKyIiJTCVM4DrgbuAl8xsZ1j2WeBOM7ucbAroDeAPANx9j5k9RXZyNw3c5+4BgJndDzwDxIDN7r6ngH0REZFpmHQZaCkpBSQiMn1m9qK7d0xWT1cCi4hElAKAiEhEKQCIiESUAoCISEQpAIiIlJnvvXiIJ144MOvfowAgIlJmnt55mL/vPDh5xRlSABARKTPJdIbq2OwPzwoAIiJlJhUoAIiIRFIqcKpjNuvfowAgIlJmdAYgIhJRCgAiIhGlFJCISESldQYgIhJNycCpjisAiIhETirIUF2lFJCISOQoBSQiElEppYBERKLH3UkqBSQiEj3pTPY2vUoBiYhETDoIA4BSQCIi0ZIMMgDElQISEYmWVBgAEjoDEBGJlvEUkOYARESiJXcGoAAgIhIxyfEAoDkAEZFIUQpIRCSilAISEYmo8WWgSgGJiERLKh0uA9UZgIhItOinIEREIkqrgEREIiqXAtIZgIhIxJRVCsjM2s3sZ2a218z2mNkfh+XNZvasmb0ePjeF5WZmXzWzLjPbZWZX5n3WhrD+62a2Yfa6JSJSmVJllgJKA3/q7muAa4H7zGwN8ACwzd1XA9vC1wAfBFaHj43Aw5ANGMCDwDXA1cCDuaAhIiJZyXJKAbl7t7v/PNweAF4GlgLrgUfDao8Ct4Xb64Fve9Z2YKGZLQZuAZ519z53PwE8C6wtaG9ERCpcWaWA8pnZCuAK4Hmgzd27w11HgbZweylwMO9th8KyicpFRCRUbikgAMysAfge8Gl3P5W/z90d8EI0yMw2mlmnmXX29vYW4iNFRCpGLgUUL5czADOrJjv4P+bu3w+Lj4WpHcLnnrD8MNCe9/ZlYdlE5W/h7pvcvcPdO1pbW6fTFxGRipcKfwyuLK4ENjMDHgFedvcv5+3aAuRW8mwAfpBXfne4GuhaoD9MFT0D3GxmTeHk781hmYiIhNJFTAHFp1DneuAu4CUz2xmWfRb4IvCUmd0L7AduD/dtBdYBXcAwcA+Au/eZ2UPAjrDeF9y9ryC9EBGZI1JBBjOIFeGewJMGAHf/v8BELbnxDPUduG+Cz9oMbJ5OA0VEoiQZONVVVWSTL7NLVwKLiJSRdJApSvoHFABERMpKKshQHS/O0KwAICJSRpKBE69SABARiZx0kCGhFJCISPQoBSQiElGpwIkXYQkoKACIiJSVZJApyg/BgQKAiEhZSQcZEkoBiYhETypwnQGIiERRMshoDkBEJIqUAhIRiSilgEREIiqlFJCISDTpQjARkYhKBV6Uu4GBAoCISFlRCkhEJKKUAhIRiSilgEREIiqlO4KJiERTKsgQ1xmAiEi0uLsuBBMRiaJ0xgF0RzARkahJBRkApYBERKImFWTPAJQCEhGJmNwZgFJAIiIRoxSQiEhEpdJKAYmIRFIqkz0D0IVgIiIRc3oOQGcAIiKRkksBaQ5ARCRilAISEYmoVLrMUkBmttnMesxsd17Z583ssJntDB/r8vZ9xsy6zOxVM7slr3xtWNZlZg8UvisiIpUtdyFYOaWAvgWsPUP5X7v75eFjK4CZrQHuAC4J3/M3ZhYzsxjwNeCDwBrgzrCuiIiEip0Cik9Wwd2fM7MVU/y89cCT7j4G/NrMuoCrw31d7r4PwMyeDOvunXaLRUTmqFwKqBKuA7jfzHaFKaKmsGwpcDCvzqGwbKJyEREJVcpvAT0MrAIuB7qBLxWqQWa20cw6zayzt7e3UB8rIlL2ctcBlPUqIHc/5u6Bu2eAb3A6zXMYaM+ruiwsm6j8TJ+9yd073L2jtbX1bJonIlKRTgeAMj4DMLPFeS8/AuRWCG0B7jCzGjNbCawGXgB2AKvNbKWZJchOFG85+2aLiMw9uRRQIl6cADDpJLCZPQHcACwys0PAg8ANZnY54MAbwB8AuPseM3uK7ORuGrjP3YPwc+4HngFiwGZ331Pw3oiIVLDxXwOtKp9VQHeeofiRd6j/58Cfn6F8K7B1Wq0TEYmQ8RRQkc4AdCWwiEiZGE8BlfMcgIiIFF6xU0AKACIiZSIdZDCDmAKAiEi0JAOnOlaFmQKAiEikpIIM1UU6+gcFABGRspEKMkVbAQQKACIiZSMVpoCKRQFARKRMpIJM0ZaAggKAiEjZSAUZ4kX6IThQABARKRtppYBERKIpGWQUAEREoigVZIp2LwBQABARKRtKAYmIRFRSZwAiItGU0hyAiEg0KQCIiERUdg5AKSARkcjRMlARkYhSCkhEJKKUAhIRiSidAYiIRFQyrQAgIhJJ6YxSQCIikaQUkIhIBLk7qcCJKwCIiERLOuMAJJQCEhGJllSQAVAKSEQkalLp7BmAAoCISMQkx88AlAISEYmUdEYpIBGRSFIKSEQkonIpoLhSQCIi0ZJLASXK6QzAzDabWY+Z7c4razazZ83s9fC5KSw3M/uqmXWZ2S4zuzLvPRvC+q+b2YbZ6Y6ISGUq1xTQt4C1byt7ANjm7quBbeFrgA8Cq8PHRuBhyAYM4EHgGuBq4MFc0BARkTJNAbn7c0Df24rXA4+G248Ct+WVf9uztgMLzWwxcAvwrLv3ufsJ4Fl+M6iIiERWOijDFNAE2ty9O9w+CrSF20uBg3n1DoVlE5WLiAiQCsIUULz8A8A4d3fAC9AWAMxso5l1mllnb29voT5WRKSsVdJPQRwLUzuEzz1h+WGgPa/esrBsovLf4O6b3L3D3TtaW1vPsnkiIpVlfA6gqozmACawBcit5NkA/CCv/O5wNdC1QH+YKnoGuNnMmsLJ35vDMhERIXs/YIBEEVNA8ckqmNkTwA3AIjM7RHY1zxeBp8zsXmA/cHtYfSuwDugChoF7ANy9z8weAnaE9b7g7m+fWBYRiaxSpIAmDQDufucEu248Q10H7pvgczYDm6fVOhGRiKikFJCIiBRQKVJACgAiImWgklYBiYhIAaXK8UpgERGZfbkLwSrhSmARESkgpYBERCIqFWSoMohpFZCISLQkgwzxIh79gwKAiEhZSAde1Pw/KACIiJSFVJChuogrgEABQESkLKSUAhIRiaaUUkAiItGkFJCISEQpBSQiElGpwIt6ERgoAIiIlIVUkCGhFJCISPRk5wB0BiAiEjmptBf1l0BBAUBEpCykMjoDEBGJnNFUwL7eIVobaor6vQoAIiIltvWlbvpHUvz+e5YV9XsVAERESuzvtu/n/EX1XLeqpajfqwAgIlJCe4+c4ucHTvIfrlmOmSaBRUQi47Hn91MTryp6+gcUAERESmZwLM3TvzjMre9ewsK6RNG/XwFARKREfrDzMEPJgI9du7wk368AICJSApmM851/3c/Fi+dzRfvCkrRBAUBEpASe2XOUV44O8Kn3ryz65G+OAoCISJEFGeevf/Iaq1rrWX/50pK1QwFARKTIfrjrCK8dG+TTH7iQWFVpjv5BAUBEpKjSQYav/OR1Ljq3kQ+9a3FJ26IAICJSRE/vPMK+N4f49AcupKqER/+gACAiUjSjqYCvbHuNS5bM55ZL2krdHOKlboCISFT8z592cbBvhMc/+e6SrfzJN6MzADN7w8xeMrOdZtYZljWb2bNm9nr43BSWm5l91cy6zGyXmV1ZiA6IiFSC144N8PV/+RX/9sqlvPeCRaVuDlCYFNDvuPvl7t4Rvn4A2Obuq4Ft4WuADwKrw8dG4OECfLeISNnLZJzPfP8lGmvjfO5Da0rdnHGzMQewHng03H4UuC2v/NuetR1YaGalnQIXESmCJ3Yc4MX9J/jsuotpri/+b/5MZKYBwIH/Y2YvmtnGsKzN3bvD7aNAbqZjKXAw772HwrK3MLONZtZpZp29vb0zbJ6ISGntPtzPF7e+wnXnt5TkFz/fyUwngd/n7ofN7BzgWTN7JX+nu7uZ+XQ+0N03AZsAOjo6pvVeEZFy8urRAe565Hnmz6vmLz9aHhO/+WZ0BuDuh8PnHuAfgauBY7nUTvjcE1Y/DLTnvX1ZWCYiMud09QzysW9uJxGv4vFPXcOyprpSN+k3nHUAMLN6M2vMbQM3A7uBLcCGsNoG4Afh9hbg7nA10LVAf16qSERkzjjYN8zHvrkdMB7/1LWc11Jf6iad0UxSQG3AP4anNHHgcXf/sZntAJ4ys3uB/cDtYf2twDqgCxgG7pnBd4uIlKWegVE+/sjzjKUzfHfjdaxqbSh1kyZ01gHA3fcBl52h/Dhw4xnKHbjvbL9PRKTc9Y+k2LB5B70DYzz2yWv4rXMbS92kd6SfghARKYChsTSffHQHXT0D/O1d7+GK5U2lbtKk9FMQIiIz1D+S4p7/9QI7D57kf9x5Je9f3VrqJk2JAoCIyAy8OTjGXY+8QFfPAH/zsStZe2nlXN+qACAicpa6egbZ+J1Ojpwc4ZsbruK3L6yMI/8cBQARkWlydx57/gB/9qO91FbH+M6913DViuZSN2vaFABERKahZ2CUz35/Nz95+RjvX72Iv/roZbTNry11s86KAoCIyBRkMs5jLxzgv//4FcZSGf7brWu4570rSn5Xr5lQABARmcSeI/187und/OLASd67qoU/u+1Szi/jC7ymSgFARGQC/cMpvvTsq/zd9v001SX48u2X8ZErlpbdj7qdLQUAEZE87s6eI6f48e6jPP7CAU4OJ7nr2vP4k5t+iwV11aVuXkEpAIhI5Lk7L3cPsOWXR9j6UjcH+oapMnjf6lYeWHsRa5bML3UTZ4UCgIhE1uvHBtj60lH+adcRunoGiVUZ11+wiD+6YRU3rWmjpaGm1E2cVQoAIhIp+3oH2fLLI/xwVzddPYOYwVXnNfPQbZey7tJz5/ygn08BQETmNHfnV71DbHv5GD96qZtdh/oxg6tXNHP3+ku45ZJzK3Yd/0wpAIjInJNMZ9jxRh8/faWHbS8f443jwwBcunQ+n/vQxdz67iWcuyCag34+BQARmRNGUwH//Gov/7TrCP/yai+DY2kS8SquO7+Fe9+3kt+9uI2lC+eVupllRQFARCpSOsjw2rFBfn7gBJ1v9LHtlR4GRtO01Ce49d2LufHiNq6/oIW6hIa5iehPRkTKnrtz6MQIe46cYufBk+w8eIJdh/oZTgYAtNQnuGlNG+svX8r1q1qIx3Svq6lQABCRspHJOIdPjvB6zwD7eofY9+YQv+oZ5OXuU5waTQNQHTPWLJ7PR9+zjCvPa+KK9ibam+fNmatzi0kBQESKajQVcKBvmP3Hhzl0Ypju/lG6+0c52DfM68cGGAqP6gEWzKvm/NZ6br1sCWsWz+eSJfO5ePF8aqtjJezB3KEAICIF5e70DSU5fHKE7v5RjvaPcqBvmK6eQbp6Bjl8cuQt9RPxKpYsqGXJwnl8tKOdC9saubCtgVWtDTTVJ0rUi2hQABCRaUsFGY6cHGH/8WEO9IWP48Ps7xvmYN8wg2Ppt9Svra5iVWsDV61o4t+3tnNeSx3Lm7OP5vqE0jclogAgIhPK5eS7egbZ232KvUdOsbf7FAf6hgkyPl4vEa+ivWke57XUc83KZpY317GsaR5LFs7j3AW1NNclKvp38+cqBQAR4eRwcjxF88bxYQ70DbH/+DD7eocYSZ3Oybc3z2PN4vl86F2Ls0fwLXWc11JHW2OtBvgKpAAgMse5O6dG0xztH6W7f4Sj/aMcCSdds5OxQ7w5mByvXx0z2pvqaG+u45qVLaxua+CCcxq4sK2RBfPm1s8hR50CgEiFcXeGkwFDY2kGxtIMjqbpH0nRN5SkbyjJm4NjHD2VnXw9Gq6wyT+KBzCDJQvmsby5jg9c3Maq1uwgf8E5DSxZOI+YjuYjQQFAZJa5O+mMM5bOkExnGBpLM5j/GD39nBvQc3Wyr1Nv2T80liYv/f4b4lVG2/xazl1Qy8VL5vO7F53DuQtqaZtfy5KF2edzGmtJxHWxVNQpAEgkBRknGQ7Iw6nsoDowmmYkGTAWZMb3JdMZksFbt8dSwfhAPZwMGEsHpwf35OkBfSwVvjfI4O8wYL9dfSJGY201DbVxGmqyj7b5tdSH241heX24nX1U01yfoKU+wfzaauXjZUoUAKTsBBkfPzoeHkszls6Ej4ChsdOpj7FU8NbBORygc3UGx9KMhnXGUrmBPmBwNE0yyJx1+8ygIRGnoTZOXSJGTTxGIl5FTbyKtsZaVrXGqUvEqQnLEvEqErHwOV6VHbjfNoDnBvf6RFyDtxSNAoBMWSrIpi9yR7tj6QwjyYCBsRSDo2lGUsH4IDySDMbTFsO58rcdUY+msnUGwvfmBvRUMI3D5VCsysYH2dxRc31NjHmJGI21capjVacH2ZrswJ2IV1Edq2JedbZOQ22cuurT5eOD99sG8ESsSuvWZU5QAJgD8tMZY8HpwXY0dTolMZQ8nZIYSwUMJQMGwlzzWDoYH9CHkgGDoymGxgJG06c/ayiZZjQ1/aPmukQsO9jmD6DhIFpbHWN5fR0Ntdkj39y+mnAQb6w9fSSd29dYk02N1Ndkj7xrwsFak5Yi06cAUESZjDOcCsIccXaQHU4GDCezR8CjqexR8dCZJgjHsvnp3NHzSPg5A2NpkumzS2ck4lXU56UwEnkD7zmNtdRUnz7yzT96rs0vD1MhDTVxaqtj44N1bXWMhpq4BmaRMlb0AGBma4GvADHgm+7+xWK3YabG0gF9Q0mODybHl96dHE7mrdrIW9UxmqZv+HSdd1q9kS+XZ87lh3Npi4baOIlYFfMS2QE2m7aIv+XouiZ2+kg6N5mYzVXnymPjR9AiEl1FDQBmFgO+BtwEHAJ2mNkWd987m987Eh5lAziQDsKUSRDQP5KmbyjJiaEkx4eS9A2NcXwoyamRVDZFkkwzPJZd5ZEKMgyHue2J5B9F53LRq89poLk+QXO4QqM+zE831GRTHHWJbK66Nh6jtrpqfLBXnllEZlOxzwCuBrrcfR+AmT0JrAcKHgDcnc79J3hs+362vnR0yqs+aquraKmvYf68ahpr47Q11lLbEhtf0TGvOk5zfTXN9TV5z9nBvaEmrrXVIlIxih0AlgIH814fAq4p9Jcc7Bvm3kd38NqxQRpr4txxdTsXnNMwvj9elTepWBunpT5BU12CloaEbh8nIpFRdqOdmW0ENgIsX778rD5j8YJaljXVce/7VvJ7ly3RoC4icgbFHhkPA+15r5eFZePcfROwCaCjo2P6C8KBeKyKzZ+46mzbKCISCcVOWO8AVpvZSjNLAHcAW4rcBhERochnAO6eNrP7gWfILgPd7O57itkGERHJKnpy3N23AluL/b0iIvJWWrMoIhJRCgAiIhGlACAiElEKACIiEaUAICISUebTuVddkZlZL7B/Bh+xCHizQM2pFFHrc9T6C+pzVMykz+e5e+tklco6AMyUmXW6e0ep21FMUetz1PoL6nNUFKPPSgGJiESUAoCISETN9QCwqdQNKIGo9Tlq/QX1OSpmvc9zeg5AREQmNtfPAEREZAIVHwDMbK2ZvWpmXWb2wBn215jZd8P9z5vZiuK3srCm0Oc/MbO9ZrbLzLaZ2XmlaGchTdbnvHr/zszczCp+xchU+mxmt4d/13vM7PFit7HQpvBve7mZ/czMfhH++15XinYWipltNrMeM9s9wX4zs6+Gfx67zOzKgjbA3Sv2QfYnpX8FnA8kgF8Ca95W54+Ar4fbdwDfLXW7i9Dn3wHqwu0/jEKfw3qNwHPAdqCj1O0uwt/zauAXQFP4+pxSt7sIfd4E/GG4vQZ4o9TtnmGf/w1wJbB7gv3rgP8NGHAt8Hwhv7/SzwDGbzLv7kkgd5P5fOuBR8PtfwBuNDMrYhsLbdI+u/vP3H04fLmd7J3XKtlU/p4BHgL+AhgtZuNmyVT6/Cnga+5+AsDde4rcxkKbSp8dmB9uLwCOFLF9BefuzwF971BlPfBtz9oOLDSzxYX6/koPAGe6yfzSieq4exroB1qK0rrZMZU+57uX7BFEJZu0z+Gpcbu7/6iYDZtFU/l7vhC40Mz+n5ltN7O1RWvd7JhKnz8PfNzMDpG9r8h/Kk7TSma6/9+nRXdLn8PM7ONAB/DbpW7LbDKzKuDLwCdK3JRii5NNA91A9izvOTN7l7ufLGmrZtedwLfc/Utmdh3wHTO71N0zpW5YJar0M4BJbzKfX8fM4mRPG48XpXWzYyp9xsw+APxX4MPuPlakts2WyfrcCFwK/LOZvUE2V7qlwieCp/L3fAjY4u4pd/818BrZgFCpptLne4GnANz9X4Fasr+ZM1dN6f/72ar0ADCVm8xvATaE278P/NTD2ZUKNWmfzewK4G/JDv6VnheGSfrs7v3uvsjdV7j7CrLzHh92987SNLcgpvJv+2myR/+Y2SKyKaF9xWxkgU2lzweAGwHM7GKyAaC3qK0sri3A3eFqoGuBfnfvLtSHV3QKyCe4ybyZfQHodPctwCNkTxO7yE623FG6Fs/cFPv8l0AD8PfhfPcBd/9wyRo9Q1Ps85wyxT4/A9xsZnuBAPgv7l6xZ7dT7POfAt8ws/9MdkL4E5V8QGdmT5AN4ovCeY0HgWoAd/862XmOdUAXMAzcU9Dvr+A/OxERmYFKTwGJiMhZUgAQEYkoBQARkYhSABARiSgFABGRiFIAEBGJKAUAEZGIUgAQEYmo/w+9RodNO6bdLgAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f2c03194be0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "data_input_len_quantile_x = np.linspace(0, 1, num=100)\n",
    "data_input_len_quantile_y = np.quantile(\n",
    "    [len(x) for x in X_train] + [len(x) for x in X_test], data_input_len_quantile_x)\n",
    "plt.plot(data_input_len_quantile_x, data_input_len_quantile_y)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((8982, 64), (8982,), (2246, 64), (2246,))"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train = keras.preprocessing.sequence.pad_sequences(X_train, maxlen=64, padding='post', truncating='post')\n",
    "X_test = keras.preprocessing.sequence.pad_sequences(X_test, maxlen=64, padding='post', truncating='post')\n",
    "Y_train = np.array(Y_train)\n",
    "Y_test = np.array(Y_test)\n",
    "X_train.shape, Y_train.shape, X_test.shape, Y_test.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "30979"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_input_dim = max(np.max(X_train), np.max(X_test)) + 1\n",
    "data_input_dim"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "46"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_output_dim = max(np.max(Y_train), np.max(Y_test)) + 1\n",
    "data_output_dim"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "class AdaptiveComputationLSTMCell(keras.layers.LSTMCell):\n",
    "    \n",
    "    def __init__(self, *args, substeps=8, **kwargs):\n",
    "        self.substeps = substeps\n",
    "        super().__init__(*args, **kwargs)\n",
    "        \n",
    "    def build(self, input_shape):\n",
    "        super().build(input_shape)\n",
    "        self.kernel_halt_c = self.add_weight(\n",
    "            shape=(self.units, 1),\n",
    "            initializer='glorot_normal', name='kernel_halt_c')\n",
    "        self.kernel_halt_inputs = self.add_weight(\n",
    "            shape=(input_shape[-1], 1),\n",
    "            initializer='glorot_normal', name='kernel_halt_inputs')\n",
    "        self.bias_halt = self.add_weight(\n",
    "            shape=(1,),\n",
    "            initializer='zeros', name='bias_halt')\n",
    "    \n",
    "    def call(self, inputs, states, **kwargs):\n",
    "        state_list = []\n",
    "        zero_inputs = K.zeros_like(inputs)\n",
    "        for i in range(self.substeps):\n",
    "            _,states = super().call(\n",
    "                inputs if i == 0 else zero_inputs,\n",
    "                states, **kwargs)\n",
    "            state_list.append(states)\n",
    "        halts = list(zip(*state_list))[1]\n",
    "        halts = K.stack(halts, axis=1)\n",
    "        halts = K.dot(halts, self.kernel_halt_c)\n",
    "        halts = halts + K.expand_dims(\n",
    "            inputs @ self.kernel_halt_inputs)\n",
    "        halts = halts + self.bias_halt\n",
    "        halts = K.softmax(halts, axis=1)\n",
    "        state_halted = []\n",
    "        for s in zip(*state_list):\n",
    "            s = K.stack(s, axis=1)\n",
    "            s = K.sum(s*halts, axis=1)\n",
    "            state_halted.append(s)\n",
    "        return state_halted[0], state_halted"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_1 (InputLayer)         (None, None)              0         \n",
      "_________________________________________________________________\n",
      "embedding_1 (Embedding)      (None, None, 32)          991328    \n",
      "_________________________________________________________________\n",
      "rnn_1 (RNN)                  (None, 16)                3185      \n",
      "_________________________________________________________________\n",
      "dense_1 (Dense)              (None, 46)                782       \n",
      "=================================================================\n",
      "Total params: 995,295\n",
      "Trainable params: 995,295\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "X = X_inputs = keras.layers.Input((None,), dtype='int32')\n",
    "X = keras.layers.Embedding(data_input_dim, 32, mask_zero=True)(X)\n",
    "X = keras.layers.RNN(AdaptiveComputationLSTMCell(16, substeps=4))(X)\n",
    "X = keras.layers.Dense(data_output_dim, activation='softmax')(X)\n",
    "M = keras.Model(X_inputs, X)\n",
    "M.compile('nadam', 'sparse_categorical_crossentropy', ['acc'])\n",
    "M.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 8982 samples, validate on 2246 samples\n",
      "Epoch 1/25\n",
      "8982/8982 [==============================] - 25s 3ms/step - loss: 2.7941 - acc: 0.3468 - val_loss: 2.4045 - val_acc: 0.3620\n",
      "Epoch 2/25\n",
      "8982/8982 [==============================] - 22s 2ms/step - loss: 2.3397 - acc: 0.3517 - val_loss: 2.2877 - val_acc: 0.3620\n",
      "Epoch 3/25\n",
      "8982/8982 [==============================] - 22s 2ms/step - loss: 2.1278 - acc: 0.4395 - val_loss: 2.0398 - val_acc: 0.5089\n",
      "Epoch 4/25\n",
      "8982/8982 [==============================] - 23s 3ms/step - loss: 1.8517 - acc: 0.5140 - val_loss: 1.8142 - val_acc: 0.5561\n",
      "Epoch 5/25\n",
      "8982/8982 [==============================] - 22s 2ms/step - loss: 1.6662 - acc: 0.5825 - val_loss: 1.7416 - val_acc: 0.5583\n",
      "Epoch 6/25\n",
      "8982/8982 [==============================] - 22s 2ms/step - loss: 1.5429 - acc: 0.6173 - val_loss: 1.6476 - val_acc: 0.5913\n",
      "Epoch 7/25\n",
      "8982/8982 [==============================] - 22s 2ms/step - loss: 1.4315 - acc: 0.6472 - val_loss: 1.6006 - val_acc: 0.6126\n",
      "Epoch 8/25\n",
      "8982/8982 [==============================] - 22s 2ms/step - loss: 1.3273 - acc: 0.6688 - val_loss: 1.5670 - val_acc: 0.6153\n",
      "Epoch 9/25\n",
      "8982/8982 [==============================] - 22s 2ms/step - loss: 1.2361 - acc: 0.6836 - val_loss: 1.6035 - val_acc: 0.5948\n",
      "Epoch 10/25\n",
      "8982/8982 [==============================] - 22s 2ms/step - loss: 1.3237 - acc: 0.6670 - val_loss: 1.5522 - val_acc: 0.6202\n",
      "Epoch 11/25\n",
      "8982/8982 [==============================] - 22s 2ms/step - loss: 1.1730 - acc: 0.6939 - val_loss: 1.5887 - val_acc: 0.6140\n",
      "Epoch 12/25\n",
      "8982/8982 [==============================] - 22s 2ms/step - loss: 1.0989 - acc: 0.7007 - val_loss: 1.5400 - val_acc: 0.6247\n",
      "Epoch 13/25\n",
      "8982/8982 [==============================] - 23s 3ms/step - loss: 1.0408 - acc: 0.7149 - val_loss: 1.5473 - val_acc: 0.6269\n",
      "Epoch 14/25\n",
      "8982/8982 [==============================] - 26s 3ms/step - loss: 0.9952 - acc: 0.7261 - val_loss: 1.5701 - val_acc: 0.6238\n",
      "Epoch 15/25\n",
      "8982/8982 [==============================] - 25s 3ms/step - loss: 0.9481 - acc: 0.7523 - val_loss: 1.7059 - val_acc: 0.6220\n",
      "Epoch 16/25\n",
      "8982/8982 [==============================] - 25s 3ms/step - loss: 0.9073 - acc: 0.7652 - val_loss: 1.5757 - val_acc: 0.6309\n",
      "Epoch 17/25\n",
      "8982/8982 [==============================] - 24s 3ms/step - loss: 0.8714 - acc: 0.7756 - val_loss: 1.5774 - val_acc: 0.6380\n",
      "Epoch 18/25\n",
      "8982/8982 [==============================] - 25s 3ms/step - loss: 0.8290 - acc: 0.7909 - val_loss: 1.6241 - val_acc: 0.6291\n",
      "Epoch 19/25\n",
      "8982/8982 [==============================] - 23s 3ms/step - loss: 0.7917 - acc: 0.8017 - val_loss: 1.6080 - val_acc: 0.6327\n",
      "Epoch 20/25\n",
      "8982/8982 [==============================] - 24s 3ms/step - loss: 0.7612 - acc: 0.8091 - val_loss: 1.6184 - val_acc: 0.6340\n",
      "Epoch 21/25\n",
      "8982/8982 [==============================] - 26s 3ms/step - loss: 0.7368 - acc: 0.8167 - val_loss: 1.7141 - val_acc: 0.6104\n",
      "Epoch 22/25\n",
      "8982/8982 [==============================] - 25s 3ms/step - loss: 0.7092 - acc: 0.8224 - val_loss: 1.6056 - val_acc: 0.6443\n",
      "\n",
      "Epoch 00022: ReduceLROnPlateau reducing learning rate to 0.00020000000949949026.\n",
      "Epoch 23/25\n",
      "8982/8982 [==============================] - 23s 3ms/step - loss: 0.6639 - acc: 0.8346 - val_loss: 1.6084 - val_acc: 0.6407\n",
      "Epoch 24/25\n",
      "8982/8982 [==============================] - 24s 3ms/step - loss: 0.6531 - acc: 0.8381 - val_loss: 1.6105 - val_acc: 0.6407\n",
      "Epoch 25/25\n",
      "8982/8982 [==============================] - 23s 3ms/step - loss: 0.6460 - acc: 0.8403 - val_loss: 1.6134 - val_acc: 0.6398\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0x7f2c0208d588>"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "M.fit(X_train, Y_train, validation_data=(X_test,Y_test), batch_size=128, epochs=25, callbacks=[\n",
    "    keras.callbacks.ReduceLROnPlateau(verbose=1)\n",
    "])"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "ml",
   "language": "python",
   "name": "ml"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}