embedding-only-classification.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import keras\n",
    "import keras.backend as K"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "(X_train,Y_train),(X_test,Y_test) = keras.datasets.reuters.load_data()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYAAAAD9CAYAAAC1DKAUAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvNQv5yAAAGzZJREFUeJzt3XuQXGd55/HvM93TM5qLpJnReKzLyJJlObZs8IXxDcPGibEthIPMbvDaC7bwGpRK7K2wSW2VYdkyhZMqsgmkYJeYCKzFEF9wAjEKaPEaQeLarcjWGIQsybdBWNeRZqyRRnPv7tPP/tGnR22j8cxoevoy5/ep6urT73m7+311eZ9znvc9fczdERGR6KkqdQNERKQ0FABERCJKAUBEJKIUAEREIkoBQEQkohQAREQiatIAYGbtZvYzM9trZnvM7I/D8s+b2WEz2xk+1uW95zNm1mVmr5rZLXnla8OyLjN7YHa6JCIiU2GTXQdgZouBxe7+czNrBF4EbgNuBwbd/a/eVn8N8ARwNbAE+AlwYbj7NeAm4BCwA7jT3fcWrjsiIjJV8ckquHs30B1uD5jZy8DSd3jLeuBJdx8Dfm1mXWSDAUCXu+8DMLMnw7oKACIiJTCtOQAzWwFcATwfFt1vZrvMbLOZNYVlS4GDeW87FJZNVC4iIiUw6RlAjpk1AN8DPu3up8zsYeAhwMPnLwH/caYNMrONwEaA+vr691x00UUz/UgRkUh58cUX33T31snqTSkAmFk12cH/MXf/PoC7H8vb/w3gh+HLw0B73tuXhWW8Q/k4d98EbALo6Ojwzs7OqTRRRERCZrZ/KvWmsgrIgEeAl939y3nli/OqfQTYHW5vAe4wsxozWwmsBl4gO+m72sxWmlkCuCOsKyIiJTCVM4DrgbuAl8xsZ1j2WeBOM7ucbAroDeAPANx9j5k9RXZyNw3c5+4BgJndDzwDxIDN7r6ngH0REZFpmHQZaCkpBSQiMn1m9qK7d0xWT1cCi4hElAKAiEhEKQCIiESUAoCISEQpAIiIlJnvvXiIJ144MOvfowAgIlJmnt55mL/vPDh5xRlSABARKTPJdIbq2OwPzwoAIiJlJhUoAIiIRFIqcKpjNuvfowAgIlJmdAYgIhJRCgAiIhGlFJCISESldQYgIhJNycCpjisAiIhETirIUF2lFJCISOQoBSQiElEppYBERKLH3UkqBSQiEj3pTPY2vUoBiYhETDoIA4BSQCIi0ZIMMgDElQISEYmWVBgAEjoDEBGJlvEUkOYARESiJXcGoAAgIhIxyfEAoDkAEZFIUQpIRCSilAISEYmo8WWgSgGJiERLKh0uA9UZgIhItOinIEREIkqrgEREIiqXAtIZgIhIxJRVCsjM2s3sZ2a218z2mNkfh+XNZvasmb0ePjeF5WZmXzWzLjPbZWZX5n3WhrD+62a2Yfa6JSJSmVJllgJKA3/q7muAa4H7zGwN8ACwzd1XA9vC1wAfBFaHj43Aw5ANGMCDwDXA1cCDuaAhIiJZyXJKAbl7t7v/PNweAF4GlgLrgUfDao8Ct4Xb64Fve9Z2YKGZLQZuAZ519z53PwE8C6wtaG9ERCpcWaWA8pnZCuAK4Hmgzd27w11HgbZweylwMO9th8KyicpFRCRUbikgAMysAfge8Gl3P5W/z90d8EI0yMw2mlmnmXX29vYW4iNFRCpGLgUUL5czADOrJjv4P+bu3w+Lj4WpHcLnnrD8MNCe9/ZlYdlE5W/h7pvcvcPdO1pbW6fTFxGRipcKfwyuLK4ENjMDHgFedvcv5+3aAuRW8mwAfpBXfne4GuhaoD9MFT0D3GxmTeHk781hmYiIhNJFTAHFp1DneuAu4CUz2xmWfRb4IvCUmd0L7AduD/dtBdYBXcAwcA+Au/eZ2UPAjrDeF9y9ryC9EBGZI1JBBjOIFeGewJMGAHf/v8BELbnxDPUduG+Cz9oMbJ5OA0VEoiQZONVVVWSTL7NLVwKLiJSRdJApSvoHFABERMpKKshQHS/O0KwAICJSRpKBE69SABARiZx0kCGhFJCISPQoBSQiElGpwIkXYQkoKACIiJSVZJApyg/BgQKAiEhZSQcZEkoBiYhETypwnQGIiERRMshoDkBEJIqUAhIRiSilgEREIiqlFJCISDTpQjARkYhKBV6Uu4GBAoCISFlRCkhEJKKUAhIRiSilgEREIiqlO4KJiERTKsgQ1xmAiEi0uLsuBBMRiaJ0xgF0RzARkahJBRkApYBERKImFWTPAJQCEhGJmNwZgFJAIiIRoxSQiEhEpdJKAYmIRFIqkz0D0IVgIiIRc3oOQGcAIiKRkksBaQ5ARCRilAISEYmoVLrMUkBmttnMesxsd17Z583ssJntDB/r8vZ9xsy6zOxVM7slr3xtWNZlZg8UvisiIpUtdyFYOaWAvgWsPUP5X7v75eFjK4CZrQHuAC4J3/M3ZhYzsxjwNeCDwBrgzrCuiIiEip0Cik9Wwd2fM7MVU/y89cCT7j4G/NrMuoCrw31d7r4PwMyeDOvunXaLRUTmqFwKqBKuA7jfzHaFKaKmsGwpcDCvzqGwbKJyEREJVcpvAT0MrAIuB7qBLxWqQWa20cw6zayzt7e3UB8rIlL2ctcBlPUqIHc/5u6Bu2eAb3A6zXMYaM+ruiwsm6j8TJ+9yd073L2jtbX1bJonIlKRTgeAMj4DMLPFeS8/AuRWCG0B7jCzGjNbCawGXgB2AKvNbKWZJchOFG85+2aLiMw9uRRQIl6cADDpJLCZPQHcACwys0PAg8ANZnY54MAbwB8AuPseM3uK7ORuGrjP3YPwc+4HngFiwGZ331Pw3oiIVLDxXwOtKp9VQHeeofiRd6j/58Cfn6F8K7B1Wq0TEYmQ8RRQkc4AdCWwiEiZGE8BlfMcgIiIFF6xU0AKACIiZSIdZDCDmAKAiEi0JAOnOlaFmQKAiEikpIIM1UU6+gcFABGRspEKMkVbAQQKACIiZSMVpoCKRQFARKRMpIJM0ZaAggKAiEjZSAUZ4kX6IThQABARKRtppYBERKIpGWQUAEREoigVZIp2LwBQABARKRtKAYmIRFRSZwAiItGU0hyAiEg0KQCIiERUdg5AKSARkcjRMlARkYhSCkhEJKKUAhIRiSidAYiIRFQyrQAgIhJJ6YxSQCIikaQUkIhIBLk7qcCJKwCIiERLOuMAJJQCEhGJllSQAVAKSEQkalLp7BmAAoCISMQkx88AlAISEYmUdEYpIBGRSFIKSEQkonIpoLhSQCIi0ZJLASXK6QzAzDabWY+Z7c4razazZ83s9fC5KSw3M/uqmXWZ2S4zuzLvPRvC+q+b2YbZ6Y6ISGUq1xTQt4C1byt7ANjm7quBbeFrgA8Cq8PHRuBhyAYM4EHgGuBq4MFc0BARkTJNAbn7c0Df24rXA4+G248Ct+WVf9uztgMLzWwxcAvwrLv3ufsJ4Fl+M6iIiERWOijDFNAE2ty9O9w+CrSF20uBg3n1DoVlE5WLiAiQCsIUULz8A8A4d3fAC9AWAMxso5l1mllnb29voT5WRKSsVdJPQRwLUzuEzz1h+WGgPa/esrBsovLf4O6b3L3D3TtaW1vPsnkiIpVlfA6gqozmACawBcit5NkA/CCv/O5wNdC1QH+YKnoGuNnMmsLJ35vDMhERIXs/YIBEEVNA8ckqmNkTwA3AIjM7RHY1zxeBp8zsXmA/cHtYfSuwDugChoF7ANy9z8weAnaE9b7g7m+fWBYRiaxSpIAmDQDufucEu248Q10H7pvgczYDm6fVOhGRiKikFJCIiBRQKVJACgAiImWgklYBiYhIAaXK8UpgERGZfbkLwSrhSmARESkgpYBERCIqFWSoMohpFZCISLQkgwzxIh79gwKAiEhZSAde1Pw/KACIiJSFVJChuogrgEABQESkLKSUAhIRiaaUUkAiItGkFJCISEQpBSQiElGpwIt6ERgoAIiIlIVUkCGhFJCISPRk5wB0BiAiEjmptBf1l0BBAUBEpCykMjoDEBGJnNFUwL7eIVobaor6vQoAIiIltvWlbvpHUvz+e5YV9XsVAERESuzvtu/n/EX1XLeqpajfqwAgIlJCe4+c4ucHTvIfrlmOmSaBRUQi47Hn91MTryp6+gcUAERESmZwLM3TvzjMre9ewsK6RNG/XwFARKREfrDzMEPJgI9du7wk368AICJSApmM851/3c/Fi+dzRfvCkrRBAUBEpASe2XOUV44O8Kn3ryz65G+OAoCISJEFGeevf/Iaq1rrWX/50pK1QwFARKTIfrjrCK8dG+TTH7iQWFVpjv5BAUBEpKjSQYav/OR1Ljq3kQ+9a3FJ26IAICJSRE/vPMK+N4f49AcupKqER/+gACAiUjSjqYCvbHuNS5bM55ZL2krdHOKlboCISFT8z592cbBvhMc/+e6SrfzJN6MzADN7w8xeMrOdZtYZljWb2bNm9nr43BSWm5l91cy6zGyXmV1ZiA6IiFSC144N8PV/+RX/9sqlvPeCRaVuDlCYFNDvuPvl7t4Rvn4A2Obuq4Ft4WuADwKrw8dG4OECfLeISNnLZJzPfP8lGmvjfO5Da0rdnHGzMQewHng03H4UuC2v/NuetR1YaGalnQIXESmCJ3Yc4MX9J/jsuotpri/+b/5MZKYBwIH/Y2YvmtnGsKzN3bvD7aNAbqZjKXAw772HwrK3MLONZtZpZp29vb0zbJ6ISGntPtzPF7e+wnXnt5TkFz/fyUwngd/n7ofN7BzgWTN7JX+nu7uZ+XQ+0N03AZsAOjo6pvVeEZFy8urRAe565Hnmz6vmLz9aHhO/+WZ0BuDuh8PnHuAfgauBY7nUTvjcE1Y/DLTnvX1ZWCYiMud09QzysW9uJxGv4vFPXcOyprpSN+k3nHUAMLN6M2vMbQM3A7uBLcCGsNoG4Afh9hbg7nA10LVAf16qSERkzjjYN8zHvrkdMB7/1LWc11Jf6iad0UxSQG3AP4anNHHgcXf/sZntAJ4ys3uB/cDtYf2twDqgCxgG7pnBd4uIlKWegVE+/sjzjKUzfHfjdaxqbSh1kyZ01gHA3fcBl52h/Dhw4xnKHbjvbL9PRKTc9Y+k2LB5B70DYzz2yWv4rXMbS92kd6SfghARKYChsTSffHQHXT0D/O1d7+GK5U2lbtKk9FMQIiIz1D+S4p7/9QI7D57kf9x5Je9f3VrqJk2JAoCIyAy8OTjGXY+8QFfPAH/zsStZe2nlXN+qACAicpa6egbZ+J1Ojpwc4ZsbruK3L6yMI/8cBQARkWlydx57/gB/9qO91FbH+M6913DViuZSN2vaFABERKahZ2CUz35/Nz95+RjvX72Iv/roZbTNry11s86KAoCIyBRkMs5jLxzgv//4FcZSGf7brWu4570rSn5Xr5lQABARmcSeI/187und/OLASd67qoU/u+1Szi/jC7ymSgFARGQC/cMpvvTsq/zd9v001SX48u2X8ZErlpbdj7qdLQUAEZE87s6eI6f48e6jPP7CAU4OJ7nr2vP4k5t+iwV11aVuXkEpAIhI5Lk7L3cPsOWXR9j6UjcH+oapMnjf6lYeWHsRa5bML3UTZ4UCgIhE1uvHBtj60lH+adcRunoGiVUZ11+wiD+6YRU3rWmjpaGm1E2cVQoAIhIp+3oH2fLLI/xwVzddPYOYwVXnNfPQbZey7tJz5/ygn08BQETmNHfnV71DbHv5GD96qZtdh/oxg6tXNHP3+ku45ZJzK3Yd/0wpAIjInJNMZ9jxRh8/faWHbS8f443jwwBcunQ+n/vQxdz67iWcuyCag34+BQARmRNGUwH//Gov/7TrCP/yai+DY2kS8SquO7+Fe9+3kt+9uI2lC+eVupllRQFARCpSOsjw2rFBfn7gBJ1v9LHtlR4GRtO01Ce49d2LufHiNq6/oIW6hIa5iehPRkTKnrtz6MQIe46cYufBk+w8eIJdh/oZTgYAtNQnuGlNG+svX8r1q1qIx3Svq6lQABCRspHJOIdPjvB6zwD7eofY9+YQv+oZ5OXuU5waTQNQHTPWLJ7PR9+zjCvPa+KK9ibam+fNmatzi0kBQESKajQVcKBvmP3Hhzl0Ypju/lG6+0c52DfM68cGGAqP6gEWzKvm/NZ6br1sCWsWz+eSJfO5ePF8aqtjJezB3KEAICIF5e70DSU5fHKE7v5RjvaPcqBvmK6eQbp6Bjl8cuQt9RPxKpYsqGXJwnl8tKOdC9saubCtgVWtDTTVJ0rUi2hQABCRaUsFGY6cHGH/8WEO9IWP48Ps7xvmYN8wg2Ppt9Svra5iVWsDV61o4t+3tnNeSx3Lm7OP5vqE0jclogAgIhPK5eS7egbZ232KvUdOsbf7FAf6hgkyPl4vEa+ivWke57XUc83KZpY317GsaR5LFs7j3AW1NNclKvp38+cqBQAR4eRwcjxF88bxYQ70DbH/+DD7eocYSZ3Oybc3z2PN4vl86F2Ls0fwLXWc11JHW2OtBvgKpAAgMse5O6dG0xztH6W7f4Sj/aMcCSdds5OxQ7w5mByvXx0z2pvqaG+u45qVLaxua+CCcxq4sK2RBfPm1s8hR50CgEiFcXeGkwFDY2kGxtIMjqbpH0nRN5SkbyjJm4NjHD2VnXw9Gq6wyT+KBzCDJQvmsby5jg9c3Maq1uwgf8E5DSxZOI+YjuYjQQFAZJa5O+mMM5bOkExnGBpLM5j/GD39nBvQc3Wyr1Nv2T80liYv/f4b4lVG2/xazl1Qy8VL5vO7F53DuQtqaZtfy5KF2edzGmtJxHWxVNQpAEgkBRknGQ7Iw6nsoDowmmYkGTAWZMb3JdMZksFbt8dSwfhAPZwMGEsHpwf35OkBfSwVvjfI4O8wYL9dfSJGY201DbVxGmqyj7b5tdSH241heX24nX1U01yfoKU+wfzaauXjZUoUAKTsBBkfPzoeHkszls6Ej4ChsdOpj7FU8NbBORygc3UGx9KMhnXGUrmBPmBwNE0yyJx1+8ygIRGnoTZOXSJGTTxGIl5FTbyKtsZaVrXGqUvEqQnLEvEqErHwOV6VHbjfNoDnBvf6RFyDtxSNAoBMWSrIpi9yR7tj6QwjyYCBsRSDo2lGUsH4IDySDMbTFsO58rcdUY+msnUGwvfmBvRUMI3D5VCsysYH2dxRc31NjHmJGI21capjVacH2ZrswJ2IV1Edq2JedbZOQ22cuurT5eOD99sG8ESsSuvWZU5QAJgD8tMZY8HpwXY0dTolMZQ8nZIYSwUMJQMGwlzzWDoYH9CHkgGDoymGxgJG06c/ayiZZjQ1/aPmukQsO9jmD6DhIFpbHWN5fR0Ntdkj39y+mnAQb6w9fSSd29dYk02N1Ndkj7xrwsFak5Yi06cAUESZjDOcCsIccXaQHU4GDCezR8CjqexR8dCZJgjHsvnp3NHzSPg5A2NpkumzS2ck4lXU56UwEnkD7zmNtdRUnz7yzT96rs0vD1MhDTVxaqtj44N1bXWMhpq4BmaRMlb0AGBma4GvADHgm+7+xWK3YabG0gF9Q0mODybHl96dHE7mrdrIW9UxmqZv+HSdd1q9kS+XZ87lh3Npi4baOIlYFfMS2QE2m7aIv+XouiZ2+kg6N5mYzVXnymPjR9AiEl1FDQBmFgO+BtwEHAJ2mNkWd987m987Eh5lAziQDsKUSRDQP5KmbyjJiaEkx4eS9A2NcXwoyamRVDZFkkwzPJZd5ZEKMgyHue2J5B9F53LRq89poLk+QXO4QqM+zE831GRTHHWJbK66Nh6jtrpqfLBXnllEZlOxzwCuBrrcfR+AmT0JrAcKHgDcnc79J3hs+362vnR0yqs+aquraKmvYf68ahpr47Q11lLbEhtf0TGvOk5zfTXN9TV5z9nBvaEmrrXVIlIxih0AlgIH814fAq4p9Jcc7Bvm3kd38NqxQRpr4txxdTsXnNMwvj9elTepWBunpT5BU12CloaEbh8nIpFRdqOdmW0ENgIsX778rD5j8YJaljXVce/7VvJ7ly3RoC4icgbFHhkPA+15r5eFZePcfROwCaCjo2P6C8KBeKyKzZ+46mzbKCISCcVOWO8AVpvZSjNLAHcAW4rcBhERochnAO6eNrP7gWfILgPd7O57itkGERHJKnpy3N23AluL/b0iIvJWWrMoIhJRCgAiIhGlACAiElEKACIiEaUAICISUebTuVddkZlZL7B/Bh+xCHizQM2pFFHrc9T6C+pzVMykz+e5e+tklco6AMyUmXW6e0ep21FMUetz1PoL6nNUFKPPSgGJiESUAoCISETN9QCwqdQNKIGo9Tlq/QX1OSpmvc9zeg5AREQmNtfPAEREZAIVHwDMbK2ZvWpmXWb2wBn215jZd8P9z5vZiuK3srCm0Oc/MbO9ZrbLzLaZ2XmlaGchTdbnvHr/zszczCp+xchU+mxmt4d/13vM7PFit7HQpvBve7mZ/czMfhH++15XinYWipltNrMeM9s9wX4zs6+Gfx67zOzKgjbA3Sv2QfYnpX8FnA8kgF8Ca95W54+Ar4fbdwDfLXW7i9Dn3wHqwu0/jEKfw3qNwHPAdqCj1O0uwt/zauAXQFP4+pxSt7sIfd4E/GG4vQZ4o9TtnmGf/w1wJbB7gv3rgP8NGHAt8Hwhv7/SzwDGbzLv7kkgd5P5fOuBR8PtfwBuNDMrYhsLbdI+u/vP3H04fLmd7J3XKtlU/p4BHgL+AhgtZuNmyVT6/Cnga+5+AsDde4rcxkKbSp8dmB9uLwCOFLF9BefuzwF971BlPfBtz9oOLDSzxYX6/koPAGe6yfzSieq4exroB1qK0rrZMZU+57uX7BFEJZu0z+Gpcbu7/6iYDZtFU/l7vhC40Mz+n5ltN7O1RWvd7JhKnz8PfNzMDpG9r8h/Kk7TSma6/9+nRXdLn8PM7ONAB/DbpW7LbDKzKuDLwCdK3JRii5NNA91A9izvOTN7l7ufLGmrZtedwLfc/Utmdh3wHTO71N0zpW5YJar0M4BJbzKfX8fM4mRPG48XpXWzYyp9xsw+APxX4MPuPlakts2WyfrcCFwK/LOZvUE2V7qlwieCp/L3fAjY4u4pd/818BrZgFCpptLne4GnANz9X4Fasr+ZM1dN6f/72ar0ADCVm8xvATaE278P/NTD2ZUKNWmfzewK4G/JDv6VnheGSfrs7v3uvsjdV7j7CrLzHh92987SNLcgpvJv+2myR/+Y2SKyKaF9xWxkgU2lzweAGwHM7GKyAaC3qK0sri3A3eFqoGuBfnfvLtSHV3QKyCe4ybyZfQHodPctwCNkTxO7yE623FG6Fs/cFPv8l0AD8PfhfPcBd/9wyRo9Q1Ps85wyxT4/A9xsZnuBAPgv7l6xZ7dT7POfAt8ws/9MdkL4E5V8QGdmT5AN4ovCeY0HgWoAd/862XmOdUAXMAzcU9Dvr+A/OxERmYFKTwGJiMhZUgAQEYkoBQARkYhSABARiSgFABGRiFIAEBGJKAUAEZGIUgAQEYmo/w+9RodNO6bdLgAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f7a07715be0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "data_input_len_quantile_x = np.linspace(0, 1, num=100)\n",
    "data_input_len_quantile_y = np.quantile(\n",
    "    [len(x) for x in X_train] + [len(x) for x in X_test], data_input_len_quantile_x)\n",
    "plt.plot(data_input_len_quantile_x, data_input_len_quantile_y)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((8982, 256), (8982,), (2246, 256), (2246,))"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train = keras.preprocessing.sequence.pad_sequences(X_train, maxlen=256, padding='post', truncating='post')\n",
    "X_test = keras.preprocessing.sequence.pad_sequences(X_test, maxlen=256, padding='post', truncating='post')\n",
    "Y_train = np.array(Y_train)\n",
    "Y_test = np.array(Y_test)\n",
    "X_train.shape, Y_train.shape, X_test.shape, Y_test.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "30982"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_input_dim = max(np.max(X_train), np.max(X_test)) + 1\n",
    "data_input_dim"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "46"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_output_dim = max(np.max(Y_train), np.max(Y_test)) + 1\n",
    "data_output_dim"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_1 (InputLayer)         (None, None)              0         \n",
      "_________________________________________________________________\n",
      "embedding_1 (Embedding)      (None, None, 16)          495712    \n",
      "_________________________________________________________________\n",
      "lambda_1 (Lambda)            (None, 16)                0         \n",
      "_________________________________________________________________\n",
      "dense_1 (Dense)              (None, 46)                782       \n",
      "=================================================================\n",
      "Total params: 496,494\n",
      "Trainable params: 496,494\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "X = X_inputs = keras.layers.Input((None,), dtype='int32')\n",
    "X = keras.layers.Embedding(data_input_dim, 16, mask_zero=True)(X)\n",
    "X = keras.layers.Lambda(lambda x: K.mean(x, axis=-2))(X)\n",
    "X = keras.layers.Dense(data_output_dim, activation='softmax')(X)\n",
    "M = keras.Model(X_inputs, X)\n",
    "M.compile('nadam', 'sparse_categorical_crossentropy', ['acc'])\n",
    "M.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 8982 samples, validate on 2246 samples\n",
      "Epoch 1/50\n",
      "8982/8982 [==============================] - 3s 306us/step - loss: 3.0887 - acc: 0.3493 - val_loss: 2.3566 - val_acc: 0.3620\n",
      "Epoch 2/50\n",
      "8982/8982 [==============================] - 2s 181us/step - loss: 2.2725 - acc: 0.3520 - val_loss: 2.2272 - val_acc: 0.3660\n",
      "Epoch 3/50\n",
      "8982/8982 [==============================] - 2s 180us/step - loss: 2.1678 - acc: 0.3716 - val_loss: 2.1273 - val_acc: 0.3972\n",
      "Epoch 4/50\n",
      "8982/8982 [==============================] - 2s 182us/step - loss: 2.0590 - acc: 0.3926 - val_loss: 2.0183 - val_acc: 0.4154\n",
      "Epoch 5/50\n",
      "8982/8982 [==============================] - 2s 180us/step - loss: 1.9465 - acc: 0.4449 - val_loss: 1.9155 - val_acc: 0.4915\n",
      "Epoch 6/50\n",
      "8982/8982 [==============================] - 2s 181us/step - loss: 1.8364 - acc: 0.5176 - val_loss: 1.8188 - val_acc: 0.5414\n",
      "Epoch 7/50\n",
      "8982/8982 [==============================] - 2s 178us/step - loss: 1.7294 - acc: 0.5752 - val_loss: 1.7276 - val_acc: 0.5913\n",
      "Epoch 8/50\n",
      "8982/8982 [==============================] - 2s 180us/step - loss: 1.6270 - acc: 0.6202 - val_loss: 1.6445 - val_acc: 0.6282\n",
      "Epoch 9/50\n",
      "8982/8982 [==============================] - 2s 182us/step - loss: 1.5322 - acc: 0.6550 - val_loss: 1.5705 - val_acc: 0.6451\n",
      "Epoch 10/50\n",
      "8982/8982 [==============================] - 2s 179us/step - loss: 1.4478 - acc: 0.6713 - val_loss: 1.5061 - val_acc: 0.6638\n",
      "Epoch 11/50\n",
      "8982/8982 [==============================] - 2s 181us/step - loss: 1.3739 - acc: 0.6858 - val_loss: 1.4515 - val_acc: 0.6750\n",
      "Epoch 12/50\n",
      "8982/8982 [==============================] - 2s 180us/step - loss: 1.3088 - acc: 0.6969 - val_loss: 1.4038 - val_acc: 0.6799\n",
      "Epoch 13/50\n",
      "8982/8982 [==============================] - 2s 180us/step - loss: 1.2514 - acc: 0.7055 - val_loss: 1.3628 - val_acc: 0.6866\n",
      "Epoch 14/50\n",
      "8982/8982 [==============================] - 2s 183us/step - loss: 1.1992 - acc: 0.7162 - val_loss: 1.3265 - val_acc: 0.6879\n",
      "Epoch 15/50\n",
      "8982/8982 [==============================] - 2s 182us/step - loss: 1.1514 - acc: 0.7266 - val_loss: 1.2926 - val_acc: 0.6981\n",
      "Epoch 16/50\n",
      "8982/8982 [==============================] - 2s 180us/step - loss: 1.1066 - acc: 0.7415 - val_loss: 1.2624 - val_acc: 0.7070\n",
      "Epoch 17/50\n",
      "8982/8982 [==============================] - 2s 182us/step - loss: 1.0651 - acc: 0.7545 - val_loss: 1.2352 - val_acc: 0.7128\n",
      "Epoch 18/50\n",
      "8982/8982 [==============================] - 2s 181us/step - loss: 1.0258 - acc: 0.7654 - val_loss: 1.2094 - val_acc: 0.7208\n",
      "Epoch 19/50\n",
      "8982/8982 [==============================] - 2s 181us/step - loss: 0.9891 - acc: 0.7741 - val_loss: 1.1850 - val_acc: 0.7271\n",
      "Epoch 20/50\n",
      "8982/8982 [==============================] - 2s 183us/step - loss: 0.9541 - acc: 0.7828 - val_loss: 1.1633 - val_acc: 0.7324\n",
      "Epoch 21/50\n",
      "8982/8982 [==============================] - 2s 185us/step - loss: 0.9216 - acc: 0.7890 - val_loss: 1.1425 - val_acc: 0.7382\n",
      "Epoch 22/50\n",
      "8982/8982 [==============================] - 2s 179us/step - loss: 0.8905 - acc: 0.7949 - val_loss: 1.1237 - val_acc: 0.7444\n",
      "Epoch 23/50\n",
      "8982/8982 [==============================] - 2s 180us/step - loss: 0.8608 - acc: 0.8015 - val_loss: 1.1060 - val_acc: 0.7471\n",
      "Epoch 24/50\n",
      "8982/8982 [==============================] - 2s 181us/step - loss: 0.8326 - acc: 0.8069 - val_loss: 1.0887 - val_acc: 0.7516\n",
      "Epoch 25/50\n",
      "8982/8982 [==============================] - 2s 179us/step - loss: 0.8058 - acc: 0.8151 - val_loss: 1.0740 - val_acc: 0.7533\n",
      "Epoch 26/50\n",
      "8982/8982 [==============================] - 2s 182us/step - loss: 0.7797 - acc: 0.8218 - val_loss: 1.0597 - val_acc: 0.7565\n",
      "Epoch 27/50\n",
      "8982/8982 [==============================] - 2s 185us/step - loss: 0.7549 - acc: 0.8264 - val_loss: 1.0463 - val_acc: 0.7609\n",
      "Epoch 28/50\n",
      "8982/8982 [==============================] - 2s 183us/step - loss: 0.7310 - acc: 0.8324 - val_loss: 1.0335 - val_acc: 0.7649\n",
      "Epoch 29/50\n",
      "8982/8982 [==============================] - 2s 180us/step - loss: 0.7078 - acc: 0.8380 - val_loss: 1.0219 - val_acc: 0.7640\n",
      "Epoch 30/50\n",
      "8982/8982 [==============================] - 2s 182us/step - loss: 0.6856 - acc: 0.8425 - val_loss: 1.0111 - val_acc: 0.7685\n",
      "Epoch 31/50\n",
      "8982/8982 [==============================] - 2s 183us/step - loss: 0.6641 - acc: 0.8491 - val_loss: 1.0006 - val_acc: 0.7689\n",
      "Epoch 32/50\n",
      "8982/8982 [==============================] - 2s 179us/step - loss: 0.6432 - acc: 0.8544 - val_loss: 0.9914 - val_acc: 0.7743\n",
      "Epoch 33/50\n",
      "8982/8982 [==============================] - 2s 186us/step - loss: 0.6232 - acc: 0.8596 - val_loss: 0.9816 - val_acc: 0.7756\n",
      "Epoch 34/50\n",
      "8982/8982 [==============================] - 2s 179us/step - loss: 0.6038 - acc: 0.8647 - val_loss: 0.9728 - val_acc: 0.7778\n",
      "Epoch 35/50\n",
      "8982/8982 [==============================] - 2s 182us/step - loss: 0.5847 - acc: 0.8702 - val_loss: 0.9650 - val_acc: 0.7814\n",
      "Epoch 36/50\n",
      "8982/8982 [==============================] - 2s 182us/step - loss: 0.5665 - acc: 0.8750 - val_loss: 0.9579 - val_acc: 0.7836\n",
      "Epoch 37/50\n",
      "8982/8982 [==============================] - 2s 181us/step - loss: 0.5485 - acc: 0.8800 - val_loss: 0.9514 - val_acc: 0.7827\n",
      "Epoch 38/50\n",
      "8982/8982 [==============================] - 2s 182us/step - loss: 0.5313 - acc: 0.8828 - val_loss: 0.9463 - val_acc: 0.7863\n",
      "Epoch 39/50\n",
      "8982/8982 [==============================] - 2s 187us/step - loss: 0.5149 - acc: 0.8858 - val_loss: 0.9391 - val_acc: 0.7916\n",
      "Epoch 40/50\n",
      "8982/8982 [==============================] - 2s 180us/step - loss: 0.4988 - acc: 0.8892 - val_loss: 0.9337 - val_acc: 0.7925\n",
      "Epoch 41/50\n",
      "8982/8982 [==============================] - 2s 183us/step - loss: 0.4830 - acc: 0.8929 - val_loss: 0.9298 - val_acc: 0.7925\n",
      "Epoch 42/50\n",
      "8982/8982 [==============================] - 2s 184us/step - loss: 0.4676 - acc: 0.8963 - val_loss: 0.9250 - val_acc: 0.7934\n",
      "Epoch 43/50\n",
      "8982/8982 [==============================] - 2s 181us/step - loss: 0.4530 - acc: 0.8995 - val_loss: 0.9205 - val_acc: 0.7952\n",
      "Epoch 44/50\n",
      "8982/8982 [==============================] - 2s 183us/step - loss: 0.4387 - acc: 0.9028 - val_loss: 0.9177 - val_acc: 0.7943\n",
      "Epoch 45/50\n",
      "8982/8982 [==============================] - 2s 184us/step - loss: 0.4250 - acc: 0.9066 - val_loss: 0.9154 - val_acc: 0.7934\n",
      "Epoch 46/50\n",
      "8982/8982 [==============================] - 2s 186us/step - loss: 0.4116 - acc: 0.9093 - val_loss: 0.9114 - val_acc: 0.7974\n",
      "Epoch 47/50\n",
      "8982/8982 [==============================] - 2s 183us/step - loss: 0.3984 - acc: 0.9114 - val_loss: 0.9099 - val_acc: 0.7979\n",
      "Epoch 48/50\n",
      "8982/8982 [==============================] - 2s 183us/step - loss: 0.3862 - acc: 0.9151 - val_loss: 0.9084 - val_acc: 0.7992\n",
      "Epoch 49/50\n",
      "8982/8982 [==============================] - 2s 181us/step - loss: 0.3740 - acc: 0.9166 - val_loss: 0.9053 - val_acc: 0.8005\n",
      "Epoch 50/50\n",
      "8982/8982 [==============================] - 2s 182us/step - loss: 0.3623 - acc: 0.9203 - val_loss: 0.9044 - val_acc: 0.8010\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0x7f7a077156d8>"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "M.fit(X_train, Y_train, validation_data=(X_test,Y_test), batch_size=64, epochs=50, callbacks=[\n",
    "    keras.callbacks.ReduceLROnPlateau(verbose=1)\n",
    "])"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "ml",
   "language": "python",
   "name": "ml"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}