Neural_Network_Part1_Solutions.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "# Neural Network"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "from __future__ import print_function\n",
    "import numpy as np\n",
    "import tensorflow as tf\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "datetime.date(2017, 2, 28)"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from datetime import date\n",
    "date.today()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "author = \"kyubyong. https://github.com/Kyubyong/tensorflow-exercises\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'1.0.0'"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tf.__version__"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'1.12.0'"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.__version__"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "# Activation Functions"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "Q1. Apply `relu`, `elu`, and `softplus` to `x`."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAD8CAYAAABn919SAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3Xd8U/X+x/HXt4uyN8iSJSjIvmUJClcEFbiiCA5AhgiK\noqKCongRlftDFNErTlyoLBFBEXAgDoaDJRuEqiAFoaVAaSmd+f7+SOQWaGlpk5wmfT8fjzySnJHz\nzsnpJ6fffPONsdYiIiKBL8TpACIi4h0q6CIiQUIFXUQkSKigi4gECRV0EZEgoYIuIhIkVNBFRIKE\nCrqISJBQQRcRCRJh/txYpUqVbJ06dfy5SRGRgLd+/frD1trKuS3n14Jep04d1q1b589NiogEPGPM\n3rwspyYXEZEgoYIuIhIkVNBFRIKEX9vQxXfS09OJiYkhJSXF6SiFWmRkJDVr1iQ8PNzpKCJep4Ie\nJGJiYihdujR16tTBGON0nELJWkt8fDwxMTHUrVvX6TgiXpdrk4sx5h1jTKwxZmuWaRWMMcuMMbs9\n1+V9G1Nyk5KSQsWKFVXMz8EYQ8WKFfVfjAStvLShzwCuOWPaWGC5tbYBsNxzXxymYp477SMJZrkW\ndGvtCuDIGZN7Ae95br8HXO/lXCIiQeH3fdt44K1uHEs87PNt5beXS1Vr7V+e2weBqjktaIwZboxZ\nZ4xZFxcXl8/NSTDp3LmzvmAmRUJC0hEe/rwfK0MPsHbbcp9vr8DdFq37V6Zz/KVpa+10a22UtTaq\ncuVcv7kqQcJai8vlcjqGiGNcmZmMnt2TX4u5GFruX3Rtd7PPt5nfgn7IGFMNwHMd671IEqj27NnD\nxRdfzMCBA2nSpAkffPAB7du3p1WrVvTt25ekpKSz1ilVqtSp2/Pnz2fw4MF+TCziO/9+vy8/FUuk\nl23IiBsm+WWb+e22uAgYBDzjuf7Ua4mkwJ78bBvbDxz36mM2rl6GJ/51aa7L7d69m/fee4+LLrqI\n3r178/XXX1OyZEkmT57M1KlTGT9+vFdziRRGrywYw6KQ3VyWWpanhs7z23ZzLejGmDlAZ6CSMSYG\neAJ3IZ9njBkK7AVu8mVICRy1a9emXbt2LF68mO3bt9OhQwcA0tLSaN++vcPpRHxv0Yq3eSfhcy5J\nD2NK/8WEhIb6bdu5FnRr7a05zOri5SziJXk5k/aVkiVLAu429K5duzJnzpxzLp+1G6H6h0ug+2Xn\nSqZET6Wiy/Bsj7mULlnOr9vXWC7iE+3atWP16tVER0cDcOLECXbt2nXWclWrVmXHjh24XC4WLlzo\n75giXnPw8D7+vfJuMoDxUc9St8Ylfs+ggi4+UblyZWbMmMGtt95Ks2bNaN++PTt37jxruWeeeYae\nPXty2WWXUa1aNQeSihRcSmoyD318PfvDLffWGErHFt0dyWHcvQ79Iyoqyqr/sW/s2LGDRo0aOR0j\nIGhfibfd/2YXvomIZWDEZYy59Q2vP74xZr21Niq35XSGLiJSAJNmDeGbiFi6pVf3STE/HyroIiL5\n9P7SScxNX0urlEgmDVzkdBwVdBGR/Ph+/Se8cmgmtdMNU276lIiIYk5H0njoIiLna/efW3h64+MU\nB/7T+U0ql6/udCRAZ+giIuflWGI8j3wxgIQQy9hLHqVpg3ZORzpFBV1EJI8yMzN4aE5PoiMyGV7h\nBq7pMMDpSKdRQRefqlOnDocP+34caBF/GPd+H9YUS+IG05hhvSY6HecsKugiInnw0scPsiTkNzqm\nluOJAece0sIpKujiNTNnzqRNmza0aNGCO++8k8zMzFPz9uzZQ5MmTU7dnzJlChMmTHAgpcj5W/jd\nG8xI/IpLU0OZMmCpXwfcOh/q5RKMPh8LB7d49zEvaArXPpPj7B07dvDhhx+yevVqwsPDufvuu5k1\na5Z3M4g4YP3273n+95eo4jI823MeJUuUdjpSjlTQxSuWL1/O+vXrad26NQAnT56kSpUqDqcSKZj9\ncXt4/Id7cIXAE62ncGH1hk5HOicV9GB0jjNpX7HWMmjQICZNOv2XWWbMmAFAWFjYaT9Jp6FypbA7\nmXqC0Qt7czACHq0xjPbNr3E6Uq7Uhi5e0aVLF+bPn09srPvXCI8cOcLevXtPza9atSqxsbHEx8eT\nmprK4sWLnYoqkitrLWPe/xdbi6VzW+Tl3NR1lNOR8kRn6OIVjRs3ZuLEiXTr1g2Xy0V4eDivvPLK\nqfnh4eGMHz+eNm3aUKNGDS65xP9jRYvk1X9mDeL7iDiuSa/Jg4NeczpOnmn43CChIWHzTvtKzmXG\nkolMjZvLP1JLMP32VYSHRzgdKc/D5+oMXUTE45u183k1di710kN4/ubPCkUxPx8q6CIiwK97NzFx\n8xOUAv7vqneoUK6q05HOmz4UFZEi79jxwzzy1W0khcDYxuNoXC/X1o1CSQVdRIq0jIx0Hpjbk9/D\nXdxZoQ/d2vdzOlK+qaCLSJH22Pu9WVfsBH1MU4b2etLpOAWigi4iRdaLH93P56F76JRanscHzHQ6\nToGpoIvfrVy5kksvvZQWLVqwY8cOZs+enaf1SpUq5eNkUpR8/O1rvHdiOU1Twph8W+EdcOt8qKCL\n382aNYtHH32UjRs3cujQoTwXdBFvWbPtG6b+8TLV0uHZXvMpWTw4ThZU0MUrTpw4QY8ePWjevDlN\nmjThww8/ZPny5bRs2ZKmTZty++23k5qayltvvcW8efP497//Tf/+/Rk7diwrV66kRYsWvPDCC8yY\nMYNevXrRuXNnGjRowJNPnt2m+d1339GzZ89T90eOHHlqzJixY8fSuHFjmjVrxujRo/319CWA7Dv0\nO+N/vA8DTGj3IjUvqO90JK8pUD90Y8wDwB2ABbYAQ6y1GnXJYZPXTGbnkZ1efcxLKlzCI20eyXH+\nF198QfXq1VmyZAkACQkJNGnShOXLl9OwYUMGDhzIa6+9xqhRo1i1ahU9e/akT58+fPfdd0yZMuXU\n2C4zZsxgzZo1bN26lRIlStC6dWt69OhBVFTu3cji4+NZuHAhO3fuxBjDsWPHvPPkJWgkn0xi9Kd9\niI2AcTXvpk3Tq5yO5FX5PkM3xtQA7gOirLVNgFDgFm8Fk8DStGlTli1bxiOPPMLKlSvZs2cPdevW\npWFD93CjgwYNYsWKFXl6rK5du1KxYkWKFy9O7969WbVqVZ7WK1u2LJGRkQwdOpQFCxZQokSJfD8f\nCT6uzEwemtmT7cXSGVT8n9x41T1OR/K6gn5TNAwoboxJB0oABwoeSQrqXGfSvtKwYUM2bNjA0qVL\nefzxx7nyyivz/VjGmHPez2ko3rCwMNasWcPy5cuZP38+L7/8Mt98802+c0hweXr2IFZFxNMj40Lu\nv3ma03F8It9n6Nba/cAU4E/gLyDBWvvVmcsZY4YbY9YZY9bFxcXlP6kUagcOHKBEiRIMGDCAMWPG\n8OOPP7Jnzx6io6MB+OCDD+jUqdNZ65UuXZrExMTTpi1btowjR45w8uRJPvnkEzp06HDa/Nq1a7N9\n+3ZSU1M5duwYy5cvByApKYmEhAS6d+/OCy+8wKZNm3z0bCXQvL34ST7O3EiblBJMHPSJ03F8Jt9n\n6MaY8kAvoC5wDPjIGDPAWntaZ05r7XRgOrhHWyxAVinEtmzZwpgxYwgJCSE8PJzXXnuNhIQE+vbt\nS0ZGBq1bt+auu+46a71mzZoRGhpK8+bNGTx4MOXLl6dNmzbceOONxMTEMGDAgLPaz2vVqsVNN91E\nkyZNqFu3Li1btgQgMTGRXr16kZKSgrWWqVOn+uW5S+G2bM08Xo/7iIvSQ3juls8ICwt3OpLP5Hv4\nXGNMX+Aaa+1Qz/2BQDtr7d05raPhc30nWIaEnTFjBuvWrePll1/22TaCZV9J7nb8vp4R3w4kFHj1\nyplcXLel05HyJa/D5xak2+KfQDtjTAnjbuTsAuwowOOJiHjNkYRYHlk+hJMh8Oil4wO2mJ+PfDe5\nWGt/NsbMBzYAGcAveJpWRPJr8ODBDB482OkYEuAyMtJ54MN/sTfCxQMVbuKqdjc7HckvCtTLxVr7\nBPCEl7JIAVlrz+oRIqfz5y90iXMeef96NhRL5hbTnMHXFZ0SpW+KBonIyEji4+NVsM7BWkt8fDyR\nkZFORxEfen7eSL4K/ZN/plbk0f7vOx3Hr/SLRUGiZs2axMTEoK6h5xYZGUnNmjWdjiE+Mm/5y3yQ\n/B3NUiOYPGhJUAy4dT5U0INEeHg4devWdTqGiGN+2vwVL+x9jRqZhmev/5jikSWdjuR3anIRkYD3\n58Foxq95kDDgyfYvUaNq0Ty5UUEXkYB24mQiDy3qS3woPFTnXqKa5H/YiUCngi4iAcuVmcmDM3uw\ns1gGg0texfVXnv1t5KJEBV1EAtaE2QP4IeIo/8qow703veh0HMepoItIQJq+aDwLXVtpn1KSpwct\ndDpOoaBeLiIScL74cTbT4xdwcXoIz966mNAwlTJQQReRALPtt7VM2v4fyluY1O0DypWp5HSkQkNN\nLiISMA4fO8gj3wwlNQTGNXuaBnWaOx2pUFFBF5GAkJ6exqh51xET7uKeyv3o3Ka305EKHRV0EQkI\nD7/fi03FTnJL6D+4rec4p+MUSiroIlLoPfvhCL4Oi6FLaiUe6f+u03EKLRV0ESnU5ix7kdknV9Ii\nJZxnBi7BhKhs5US9XESk0Fq9+Qte2vcmtTINz/b+hMjIEk5HKtT0VicihdKeA78yYc1oIiw81eEV\nqlW+0OlIhZ4KuogUOknJxxm9+GaOhsLo+qNo2biT05ECggq6iBQqrsxMHpjVg1+LZTK0VDf+1XmY\n05EChgq6iBQq42fdyk8Rx+iVUY8Rfac6HSegqKCLSKHx2qfj+NTuoENKKZ4c9LHTcQKOermISKGw\n5If3eevIpzRKC+HZfks04FY+aI+JiOM2R//EMzsnU8kFk66dRZnSFZyOFJDU5CIijoo9eoBHvx1G\nJjCuxX+of2FTpyMFLBV0EXFMWnoqD3zUiwPhlnsuGMAVUdc7HSmgqaCLiGNGv38dm4ulcGtYa/p3\nf9TpOAGvQAXdGFPOGDPfGLPTGLPDGNPeW8FEJLhNmjOcb8MO0C21CmP6ve10nKBQ0A9F/wt8Ya3t\nY4yJADTQgojkauZXU5ib+gOtUovxf4MXa8AtL8l3QTfGlAWuAAYDWGvTgDTvxBKRYLVy42Km7X+X\nOhmGZ3p/QrFixZ2OFDQK8rZYF4gD3jXG/GKMecsYU9JLuUQkCP1xYCcT1o+luAue7Pg61SrXcjpS\nUClIQQ8DWgGvWWtbAieAsWcuZIwZboxZZ4xZFxcXV4DNiUggO37iGA8tuYWEEMuYix6kRaOOTkcK\nOgUp6DFAjLX2Z8/9+bgL/GmstdOttVHW2qjKlSsXYHMiEqgyMzMYNbsn0eEZ3FG6Oz06DXU6UlDK\nd0G31h4E9hljLvZM6gJs90oqEQkq42bezNqIBHq7GnJXn+ecjhO0CtrL5V5glqeHy+/AkIJHEpFg\nMu2Th1nCLq5IKc34Oz5yOk5QK1BBt9ZuBKK8lEVEgswnq97hnWNLaZIayuT+SwkJDXU6UlDT4Fwi\n4hPrd63kuV1TqZZp+b/ucyhVqpzTkYKeevOLiNcdOLyXx1bcTQiWf//jWerWaux0pCJBBV1EvCol\n9ST3LezN4VDLfdWH0r5VD6cjFRkq6CLiNdZa7pvZk18j0hgYfhl9r3nQ6UhFigq6iHjNhLlD+DEs\nll6p1bl/wHSn4xQ5Kugi4hVvfj6RBWnr6ZAcyRODFzkdp0hSLxcRKbAv183n9YNzaZxmmHjTIsIj\nijkdqUhSQReRAtm+dwNPb55AZZeLCf+cQaWK1ZyOVGSpyUVE8i3+eCwPLRuCC8uYSx6nUcM2Tkcq\n0lTQRSRf0jJSuWfedRwMy2RE+Rvp0rGf05GKPBV0ETlv1loemHUj28JP0N825bYbnnI6kqCCLiL5\nMGnBvaxgLz1OluehQbOcjiMeKugicl5mfjeNuYnf0TY5lPEDl+j3QAsR9XIRkTxbuf1L/vvHGzRM\ntzzZaz4lSpR2OpJkoYIuInnyx6FdPP7jaMpaF+PaTqNG9YucjiRn0P9KIpKrxJPHufezm0kNcTHq\nwpG0bN7F6UiSDRV0ETmnTFcmI2b3ZF9YOsOKXUXPbnc7HUlyoIIuIuc0Zk4/NoUd5dbUOgzt91+n\n48g5qKCLSI6mLn6MZRnbufpECUbfvsDpOJILFXQRydaCn97j/cOLaJ0Mj/dbTFh4hNORJBfq5SIi\nZ1n/+w9M3vEcddNdPN5tNuXKVXY6kuSBCrqInGb/0X2M/vYuSuBiTNOJ1KvfwulIkkdqchGRU06m\nJXP3x705EZLJyEr9uax9b6cjyXlQQRcRAFzWxYjZ1/NH2EkGE8WNvcY5HUnOkwq6iADw6EdDWW/+\n4qbkKowY9K7TcSQfVNBFhJe/msjSk+volhTG6CGfacCtAKUPRUWKuEUb5vLmgbm0OenikRuXEFm8\npNORJJ8K/DZsjAk1xvxijFnsjUAi4j+/7FvDxE3/oX5aBqOvmE6VCy50OpIUgDf+r7of2OGFxxER\nPzpwfD/3LxtGaVcG91/0KI0u7eh0JCmgAhV0Y0xNoAfwlnfiiIg/nEg/wR3ze5NqMhhR8no6XTnI\n6UjiBQU9Q38ReBhweSGLiPhBhiuDYXNv5EDICYalNabPLZOcjiReku+CbozpCcRaa9fnstxwY8w6\nY8y6uLi4/G5ORLzAWsuYhcPY4trPoOPlGDJ0jtORxIsKcobeAbjOGLMHmAtcaYyZeeZC1trp1too\na21U5coaD0LESS9++x++TlpH7wQYNvgzQsPU0S2Y5LugW2sftdbWtNbWAW4BvrHWDvBaMhHxqoWb\n5vLun3PplJTOnb3mU6pMeacjiZfp7VmkCFgX8zNP//IfmqSmc2fbaVSvfbHTkcQHvFLQrbXfAd95\n47FExLv2JfzJfcvupGpmOsNr3kfTqG5ORxIf0fd7RYJYQmoCty+8CWPTuDOsG5173ON0JPEhFXSR\nIJWemc4dH99MPEmMOFGPXgNfdDqS+JgKukgQstYy6rPh7Ezfz11HStB32DwNuFUE6BUWCULPfv80\nKxLWMfBoBn1v+5RikSWcjiR+oF4uIkFmzuaZzNz7Ed0TU+h7zTzKV67mdCTxExV0kSCyYu93PLNh\nMu1OpnBrs8nUuaSl05HEj9TkIhIktsdv58Fv76dhWhoDKg2lxRX6PdCiRgVdJAjsT9rPsMWDqJCZ\nxtDMDnTqO9bpSOIAFXSRAHcs5RhDPu2PzUzmvmM16DpsutORxCEq6CIBLCUjhWFLhhCfdpixh8P4\n5/D5GnCrCNMrLxKgMl2ZPLjsfn5N3M34uBTa9vuCkqXLOR1LHKSCLhKArLVM+vH/WBn7Aw/EJ9K8\n6wdUrXmR07HEYWpyEQlA722bwYfR87gt4TgtmzxFg5ZXOB1JCgEVdJEA8/kfn/P8+ql0SzpBh7L9\naXm1fg9U3NTkIhJA1h5cy2MrxtIqJYUb06Nof9tEpyNJIaIzdJEA8euRX7ln2d3USkvlniOViRrx\nvgbcktPoDF0kAOxL3MewL4ZSKi2ZCYeg3l0LiCgW6XQsKWRU0EUKucMnDzPsyzvISDnGtIPHKX/T\nZ5SrdIHTsaQQUkEXKcSS0pIYsewuDice4I2DcYR2fosLG7ZwOpYUUiroIoVUamYq931zH7uP7GJa\nbCyui8fSpMO/nI4lhZgKukghlOnKZOyKsaw9tJZJcYcJL3sjrW98wOlYUsipoIsUMtZaJv48ka//\n/Joxh49S07ag2R0vOR1LAoAKukgh88rGV5i/az6Djp7gsuQqVH9gLiEacEvyQEeJSCEyc/tM3tj8\nBtckZnDbMYsZ9jElSpV1OpYECH0rQaSQ+HjXx0xeO5l2ySH8Oy6exBs+oEqNuk7HkgCigi5SCCz9\nfSlP/vgkzVKL8/LBvfzW4Xkuat7R6VgSYFTQRRz2zZ/f8Niqx2jgKsvbf+3il/r30rLbbU7HkgCU\n74JujKlljPnWGLPdGLPNGHO/N4OJFAU/HPiB0d+PpnZIRd7/cwuby15L2wFPOh1LAlRBPhTNAB6y\n1m4wxpQG1htjlllrt3spm0hQ23BoA6O+HUX18Eq8/esG9oQ3o8WIdzXgluRbvo8ca+1f1toNntuJ\nwA6ghreCiQSzbfHbuGf5PVQIL8d/d+3kJJWpeedHGnBLCsQrpwLGmDpAS+BnbzyeSDDbEb+D4V8N\np1RYSSbu3k8lVwb0n0fZilWdjiYBrsAF3RhTCvgYGGWtPZ7N/OHGmHXGmHVxcXEF3ZxIQNsRv4M7\nvrqDkuEleWxPKs3S/mJ/tzepdVFTp6NJEChQQTfGhOMu5rOstQuyW8ZaO91aG2WtjapcuXJBNicS\n0P4u5qXCSzHqYHn+mbyZjS0m0PiyHk5HkyBRkF4uBngb2GGtneq9SCLBJ2sxvze1Jd0Pf8GP1QbS\n5ob7nI4mQaQgZ+gdgNuAK40xGz2X7l7KJRI0shbz0aVvoPuu1/mlZEfa3vGi09EkyOS726K1dhVg\nvJhFJOhsPbyVO5fdSanwUjxR515afHY7v4fV45IRcwgJDXU6ngQZdXgV8ZG1B9cy9MuhlI4ozfMt\nn6b+4vtIMiUpO3QBxUuVcTqeBCEVdBEfWBGzghFfj6BayWq80elVis25m9I2ieM3zKRy9TpOx5Mg\npYIu4mVf/PEF939zP/XL1eftrm8RP+N+6qdHs6vji1zUvIPT8SSIqaCLeNH8XfN5eMXDNK/SnLe7\nvc2u2U/TKmkFaxqMokXXfk7HkyCnH7gQ8QJrLa9vfp1XN75Kxxodmdp5KlsXv037/TNYU6EnbfuN\ndzqiFAEq6CIFlO5KZ+JPE1mwewHX1b+OCe0nEL3ma5r/Mp6tkS1oedc7GnBL/EIFXaQATqSf4KHv\nH2L1/tUMbzackS1Gsv+3bVT78g4Ohl5ArTvnEx5RzOmYUkSooIvkU2xyLCOXj2TX0V080f4J+jTs\nQ8KROFyzbwIgpN88ylbQcBfiPyroIvmwOW4zo74dRVJ6EtOunMblNS8nPS2VfW/0oWHmQaKvmUXj\ni5o4HVOKGDXsiZynT6M/ZfAXg4kIjWBm95lcXvNyrMvFL68PpUnqRja1fIrG7a91OqYUQTpDF8mj\nDFcGU9dP5YPtH9D2grZM6TSFcpHlAFgz+ynaHvmMn2oMot31Ix1OKkWVCrpIHhw8cZBHVjzChtgN\n9G/Un9FRowkLcf/5bFw2i9a7X2RD6Stoc/sLDieVokwFXSQXK2NW8tiqx0jNTGXS5ZPoWa/nqXm/\nbV5Nw1UPEB3egEYjZmvALXGUCrpIDtJd6Uz7ZRrvbn2XhuUbMqXTFOqWrXtqftyBPZReMIBEU4oK\nt8+neMnSDqYVUUEXydauo7sYt2ocO4/spG/Dvjzc+mEiw/73A87JSQkce/tGatgT/NVnEfWr13Yw\nrYibCrpIFhmuDN7d+i6vbnqVMhFleLHzi3Sp3eW0ZVyZmex8rT8tMn5jy+Wv0bxpO4fSipxOBV3E\nY1v8Nib+OJGt8VvpVrsbj7d7nPKR5c9abs3bo2h3YiU/NXyIdlfd6kBSkeypoEuRl5CawLRfpjHv\n13lUiKzAc52e45o612S77JqFL9HuwPv8XLEXbW993M9JRc5NBV2KrAxXBp9Ef8K0X6ZxLPUY/Rr1\n454W91A6IvsPN7f9sJQWGyewNbIlre58UwNuSaGjgi5FjrWWr//8mpc2vMSe43toUbkFb3R9g0sq\nXJLjOvuit1Djq2H8FVqNWndpwC0pnFTQpchwWRcrYlYwffN0thzeQr2y9Xjxny9yZa0rMSbn3ztP\niD8Es27CYggf8BFly1fyY2qRvFNBl6CX7kpn6e9LeXfru/yW8BvVS1bnqcue4rr61xEacu4vAqWl\nphDzRh8auGL5rftsGtVr7KfUIudPBV2C1l9Jf7EgegELdi8gNjmWBuUbMOnySVxd52rCQ8JzXd+6\nXGx67XZap21m3T8mEdX2aj+kFsk/FXQJKsnpyazYv4JF0YtYtX8VAB1rdGRC+wl0rNHxnE0rZ/p5\n1gTaHVvCTzWH0O66u30VWcRrVNAl4CWnJ7P6wGq+3PMlK2JWcDLjJFWKV2F4s+H0btCb6qWqn/dj\n/vLVTNpEv8SG0p1oM+R5H6QW8T4VdAk4Luti19FdrN6/mh8O/MCG2A1kuDKoEFmB6+pfx9V1rqZV\nlVa5to/nJHrTKi5e/SDR4Q1ofLcG3JLAoYIuhd7xtONsO7yNjXEb2RS3ic1xm0lMSwSgYfmG3Nbo\nNjrW6Eirqq1ODWmbX7H7/6DMwts4bspQ4Y6PiSxRyhtPQcQvCnT0G2OuAf4LhAJvWWuf8UoqKXKs\ntRxLPUZMYgx7ju9h97Hd7D66m+hj0Rw8cRAAg6F+ufp0q92NllVa0r56e6qUqOK1DMlJCRx/50aq\n2WQO9V1EvQsu9Npji/hDvgu6MSYUeAXoCsQAa40xi6y1270VToJDhiuDIylHiD8Zz+GTh4lP8Vyf\njGd/0n72J+0nJjGG5IzkU+uEhYRRr2w9/lH1HzQo14BGFRrRtHLTHL/FWVCujAx+ffVWmmX8zrZO\nb9CsSVufbEfElwpyht4GiLbW/g5gjJkL9AJU0Aspl3WRaTPJdGXisi4ybAYul+faush0Zbrne5ZJ\nc6WRkpFCWmYaKZlnXGeZnpKRQlJ6EolpiSSlJ5GUlkRieqL7Oi2RE+knsNiz8pQIK0H1UtWpUaoG\nrS9oTY1SNahRqga1y9TmwjIX5qlrobesfWskbZNX8/MlY2h75c1+266INxWkoNcA9mW5HwP45LTm\n+rmj+TN52xlTbQ63c55m87FObtOsyes6WafnfZ3sCmFenrs1LsB9sWS6l8s2qxfYEEJscUJscQzF\nPbdLEGIrYmwkpW0JQm0ZQl1lCLWlCXWVJcSWJoRipMXDH7gv/3PAc/GPbslLuD9lDj9W6kO7W8b5\nbbsi3ubzD0WNMcOB4QAXXpi/NsmyEeUpmer+urUhp37E5ozrM+dmN/3sdUwu88+edz7r5D1H9uvk\n9tz/t7yYaDZ+AAAJYElEQVQhBAjBEIohJMv9/007/b6BLMu6L+GEEEEI4YQQRggRWaaFnZpnTGjO\n0Qq5JslrGJnyBtHlOtL6ztfPq5+6SGFTkIK+H6iV5X5Nz7TTWGunA9MBoqKi8nWK+F5vnTWJD/z+\nPcx+Ci5owkVDPoRw/zXxiPhCQcb/XAs0MMbUNcZEALcAi7wTS8TH9v4Ac26BCvXgtk+gmLonSuDL\n9xm6tTbDGDMS+BJ3t8V3rLVnNnSLFD47l8L8IVDuQhj4KZSs6HQiEa8oUBu6tXYpsNRLWUR8y1pY\n9zYsHQPVW0K/eVBSQ+FK8NA3RaVoSEuGpaNh4yxocDX0fRciSjqdSsSrVNAl+MXugI/vgEPboNMj\n7ks+x3kRKcxU0CV4ZabDqhfg+2chsgz0nw8NrnI6lYjPqKBL8LEWdn8Fy8ZD3E5ociNc+6zayyXo\nqaBLcPnzZ/h2IvyxAirUh1vnwsXXOp1KxC9U0CXwuTJh1xew+iXY9xOUqAjXPgdRQyBUXxaSokMF\nXQJX/G/wy0zYNBcSD0DZC91NKy0HqAeLFEkq6BI4rIWDm2HnEveXgw5tARMCF3WFa5+Bi3tAqA5p\nKbp09Evh5cqE2O3ur+nvXe2+PhHnLuK12kK3idCkD5Sp5nRSkUJBBV2cZy0kHoSjf7j7jB/c4r4c\n2gYZJ93LlL0Q6neBuldAw6vVY0UkGyro4luuTEiOh6RD7kviof/dProXju5xX/4u3ACRZeGCZu4P\nNas1h9qXucddEZFzUkEPdtaCdbkLqysDrOfa5Trjfqb7cup+BqSnQEaWS9b76SchI9VdiNNOQMpx\nSEmAVM911vvZ/QhHRCl3ka5QDy7qAuXrQIW6UKkhlK0FGpdc5LwFRkFf/ADsWc1phcFm8+s/55yW\nZXq208jjcl6elmvurNPyuu4ZBdyXQsLdPUoiy7q/jVmsLJSr7b4dWRaKlYFSVTyXqv+7Vi8UEa8L\njIJetiZUaeS+fdqZmzm/aadNz880bz9eAZ5LbtNCQiEkDIznOiQ0m2kh2S/z9/3wSAiLhLDiWW5H\nQnhxCCvmvq0xUUQKjcAo6Jc/5HQCEZFCryC/WCQiIoWICrqISJBQQRcRCRIq6CIiQUIFXUQkSKig\ni4gECRV0EZEgoYIuIhIkjLXZjLPhq40ZEwfszefqlYDDXozjLcp1fpTr/CjX+QnWXLWttZVzW8iv\nBb0gjDHrrLVRTuc4k3KdH+U6P8p1fop6LjW5iIgECRV0EZEgEUgFfbrTAXKgXOdHuc6Pcp2fIp0r\nYNrQRUTk3ALpDF1ERM6hUBV0Y0xfY8w2Y4zLGBN1xrxHjTHRxphfjTFX57B+XWPMz57lPjTGRPgg\n44fGmI2eyx5jzMYclttjjNniWW6dt3Nks70Jxpj9WbJ1z2G5azz7MNoYM9YPuZ4zxuw0xmw2xiw0\nxpTLYTm/7K/cnr8xppjnNY72HEt1fJUlyzZrGWO+NcZs9xz/92ezTGdjTEKW13e8r3N5tnvO18W4\nveTZX5uNMa38kOniLPthozHmuDFm1BnL+GV/GWPeMcbEGmO2ZplWwRizzBiz23NdPod1B3mW2W2M\nGeSVQNbaQnMBGgEXA98BUVmmNwY2AcWAusBvQGg2688DbvHcfh0Y4eO8zwPjc5i3B6jkx303ARid\nyzKhnn1XD4jw7NPGPs7VDQjz3J4MTHZqf+Xl+QN3A697bt8CfOiH164a0MpzuzSwK5tcnYHF/jqe\n8vq6AN2Bz3H/bFY74Gc/5wsFDuLup+33/QVcAbQCtmaZ9iww1nN7bHbHPFAB+N1zXd5zu3xB8xSq\nM3Rr7Q5r7a/ZzOoFzLXWplpr/wCigTZZFzDGGOBKYL5n0nvA9b7K6tneTcAcX23DB9oA0dba3621\nacBc3PvWZ6y1X1lrMzx3fwJq+nJ7ucjL8++F+9gB97HUxfNa+4y19i9r7QbP7URgB1DDl9v0ol7A\n+9btJ6CcMaaaH7ffBfjNWpvfLywWiLV2BXDkjMlZj6Gc6tDVwDJr7RFr7VFgGXBNQfMUqoJ+DjWA\nfVnux3D2AV8ROJaleGS3jDddDhyy1u7OYb4FvjLGrDfGDPdhjqxGev7tfSeHf/Pysh996XbcZ3PZ\n8cf+ysvzP7WM51hKwH1s+YWniacl8HM2s9sbYzYZYz43xlzqp0i5vS5OH1O3kPNJlRP7C6CqtfYv\nz+2DQNVslvHJfvP7b4oaY74GLshm1jhr7af+zpOdPGa8lXOfnXe01u43xlQBlhljdnrezX2SC3gN\neBr3H+DTuJuDbi/I9ryR6+/9ZYwZB2QAs3J4GK/vr0BjjCkFfAyMstYeP2P2BtzNCkmez0c+ARr4\nIVahfV08n5FdBzyazWyn9tdprLXWGOO3roR+L+jW2qvysdp+oFaW+zU907KKx/3vXpjnzCq7ZbyS\n0RgTBvQG/nGOx9jvuY41xizE/e9+gf4Q8rrvjDFvAouzmZWX/ej1XMaYwUBPoIv1NCBm8xhe31/Z\nyMvz/3uZGM/rXBb3seVTxphw3MV8lrV2wZnzsxZ4a+1SY8yrxphK1lqfjluSh9fFJ8dUHl0LbLDW\nHjpzhlP7y+OQMaaatfYvT/NTbDbL7Mfdzv+3mrg/OyyQQGlyWQTc4umBUBf3O+2arAt4CsW3QB/P\npEGAr874rwJ2WmtjsptpjClpjCn9923cHwxuzW5Zbzmj3fKGHLa3Fmhg3L2BInD/u7rIx7muAR4G\nrrPWJuewjL/2V16e/yLcxw64j6VvcnoT8hZPG/3bwA5r7dQclrng77Z8Y0wb3H+7Pn2jyePrsggY\n6Ont0g5IyNLc4Gs5/pfsxP7KIusxlFMd+hLoZowp72ke7eaZVjC+/hT4fC64C1EMkAocAr7MMm8c\n7h4KvwLXZpm+FKjuuV0Pd6GPBj4Civko5wzgrjOmVQeWZsmxyXPZhrvpwdf77gNgC7DZc0BVOzOX\n53533L0ofvNTrmjcbYUbPZfXz8zlz/2V3fMHnsL9hgMQ6Tl2oj3HUj0/7KOOuJvKNmfZT92Bu/4+\nzoCRnn2zCfeHy5f5IVe2r8sZuQzwimd/biFL7zQfZyuJu0CXzTLN7/sL9xvKX0C6p3YNxf2Zy3Jg\nN/A1UMGzbBTwVpZ1b/ccZ9HAEG/k0TdFRUSCRKA0uYiISC5U0EVEgoQKuohIkFBBFxEJEiroIiJB\nQgVdRCRIqKCLiAQJFXQRkSDx/w6be+3xGYsrAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f598001da50>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "_x = np.linspace(-10., 10., 1000)\n",
    "x = tf.convert_to_tensor(_x)\n",
    "\n",
    "relu = tf.nn.relu(x)\n",
    "elu = tf.nn.elu(x)\n",
    "softplus = tf.nn.softplus(x)\n",
    "\n",
    "with tf.Session() as sess:\n",
    "    _relu, _elu, _softplus = sess.run([relu, elu, softplus])\n",
    "    plt.plot(_x, _relu, label='relu')\n",
    "    plt.plot(_x, _elu, label='elu')\n",
    "    plt.plot(_x, _softplus, label='softplus')\n",
    "    plt.legend(bbox_to_anchor=(0.5, 1.0))\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "Q2. Apply `sigmoid` and `tanh` to x."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYYAAAD8CAYAAABzTgP2AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3Xl8VPW5+PHPk50QCIGQiIAsgiKIAmERVxD1orXivtxq\nbV34deH2dtG6ta5XL/Z2ubdXa7VqhbpgrRsqXgRN1IrIoggYRMImASQkkJCQdWae3x/nBCZhskxm\nTfK8X6/D2b7fM8+cOcyTc75nzldUFWOMMaZRQqwDMMYYE18sMRhjjGnCEoMxxpgmLDEYY4xpwhKD\nMcaYJiwxGGOMacISgzHGmCYsMRhjjGnCEoMxxpgmkmIdQEdkZ2fr0KFDO1T34MGD9OzZM7wBhYHF\nFRyLKzgWV3C6alyrV68uVdX+bRZU1U435OXlaUfl5+d3uG4kWVzBsbiCY3EFp6vGBazSdnzH2qUk\nY4wxTVhiMMYY04QlBmOMMU10ysZnEzkNDQ0UFxdTW1sLQGZmJhs2bIhxVEeKdVxpaWkMGjSI5OTk\nmMVgTKRYYjBNFBcX06tXL4YOHYqIUFlZSa9evWId1hFiGZeqUlZWRnFxMcOGDYtJDMZEUlguJYnI\n0yJSIiLrW1gvIvJHESkSkbUiMsFv3fUisskdrg9HPKbjamtr6devHyIS61DilojQr1+/Q2dVxnQ1\n4WpjeAaY2cr684GR7jAbeAxARPoC9wBTgMnAPSKSFaaYTAdZUmib7SPTlYXlUpKqfiAiQ1spMguY\n795Hu1xE+ojIAGAasERV9wGIyBKcBPNCOOIyxrSDKjRUQ10V1FdBXSXUHwRvPXgbwNfgjP2nfQ3g\n8zp11QeoM918fMQ6mswP3boV9OPDoaB4fYrHp/h8ik/B61N86gxed5lPnfVeVXw+Z17d96LuW3LG\niqLOvNuN8eH1znIay+nhdZ79+1m97c2mu6mFmUOTLXST3Ga9JqtbL1BbUc6+0SPpmzMw4GuFS7Ta\nGAYCO/zmi91lLS0/gojMxjnbIDc3l4KCgg4FUlVV1eG6kRQvcWVmZlJZWXlo3uv1NpmPhTlz5jBn\nzhxGjRp1aFm447rssst46qmn6NOnT5PlDz30EBkZGfzkJz85ok5tbe0Rn1m8fI7+Ej3VJJVuYsOC\nfNKri0mtKyOlfr87lJPcUIngi0lsQwG2H54XnC+luGj8rAi+ik8jfya56P1pZPQ/JqKvERf7vz1U\n9QngCYCJEyfqtGnTOrSdgoICOlo3kuIlrg0bNjRp1I2Hxud58+YdsSzccb3zzjsBl6emppKamhrw\ntdLS0hg/fnyTZXHxOVbvg03vwLZ/wo4VULrx8DpJhN4DISMHck50xun9ILUXpPR0xxnOdFIaJCZD\nQhIkJuOTZEpqfBRX1LOjwsOO8np2lNdScqCOkqp6SirrqfP4UMQdwOderU5ISCAjNYnePVLo1SOF\ntOREeqQkUVlRzjFHH0WPlER6JCfSIyWRtOREUpMSSElKICkhgaREISXRGSclJJCcKCQlJpCcICQn\nJZCU4CxPSIAEEXdwLvclyOFlIiDSRpkEJzn985//5MwzznB2md93vXB4JtDVxPaUlSblJcCyI9c3\nKigo4MIoHF/RSgw7gcF+84PcZTtxLif5Ly+IUkwmDh08eJArr7yS4uJivF4vv/71r3nsscf47W9/\ny8SJE3nqqad4+OGH6dWrFxMmTCA1NZVHHnmE733ve/To0YPPPvuMkpISnn76aebPn8/HH3/MlClT\neOaZZwB44YUXeOihh1BVvvWtb/Hwww8DMHToUFatWkV2djYPPvgg8+bNIycnh8GDB5OXlxfDPdJO\nnnoofB0+mw/bPgL1QlofGDwFTrqCdSU+xk67FLKGOl/2baht8LJ+ZwWFOw6wYfcBCneXs/GbA9Q2\nHD6zSE4UBvbpwYDMTI4bksrpvVLJ6ZVGTu9U+vdKpU+PFDLTk8nskUzPlMSA7TJOIh0Xzj0RFj2S\nhJ6pnebv5rCL1jtfCMwRkQU4Dc0VqrpbRBYDD/k1OJ8H3BGlmEwb7nvjC9bt2E9iYmLYtjn66N7c\n8+0xLa7/v//7P44++mjeeustACoqKnjssccA2LVrFw888ACffvopALNmzeLkk08+VHf//v18/PHH\nLFy4kIsuuoiPPvqIJ598kkmTJrFmzRpycnK47bbbWL16NVlZWZx33nm89tprXHzxxYe2sXr1ahYs\nWMCaNWvweDxMmDAhvhNDQw188jh8/CgcLIGsYXDav8MJF8KA8ZDg/MVeVlAA2SNb3Ex1vYflW8pY\nsXU/K7ftY11xBfVeJwn0SU/mhKN6c83kYxiRk8GQvj0Z0i+dAZlpJCXab2S7orAkBhF5Aecv/2wR\nKca50ygZQFX/DCwCLgCKgGrg++66fSLyALDS3dT9jQ3RpnsaO3Ysv/jFL7jtttu48MILOcM9nQdY\nsWIFZ511Fn379qWyspIrrriCr7766tD6b3/724gIY8eOJTc3l7FjxwIwZswYtm3bxvbt25k2bRr9\n+zsPl/zOd77DBx980CQxfPjhh1xyySWkp6cDcNFFF0XjbXfMxv+Dt34BB4rh2LNh6o9h+NmHkkFb\ndlfUsLRwD+9+WcKyzWXUe3wkJwpjB2by/dOGkjcki7GDMjmqd5rdhdXNhOuupGvaWK/Aj1tY9zTw\ndDjiMOF1z7fHRL2N4bjjjuPTTz9l0aJF/OpXv2LGjBntrpuamgo417MbpxvnPR5P1/mVcn01vP1L\n+OxvkDMGLv4TDD+rXVUraxt4e903vPJZMZ9s3YcqDO2XzrVThnD2qBwmDs0iLTl8Z4imc7LzQBNX\ndu3aRXp6Otdeey233nrroctGAJMmTeL9999n//79eDweXn755aC2PXnyZN5//31KS0vxer288MIL\nnHVW0y/UM888k9dee42amhoqKyt54403wvK+wqaqBOZdCJ89C6f/DGbntysp7Kzycccr65j04FJ+\n+fJavqmo5aczjuPdX5xFwa3Tufvbozl9ZLYlBQN0oruSTPewbt06br31VhISEkhOTuaxxx7jlltu\nAWDgwIHceeedTJ48mczMTMaMGUNmZma7tz1gwADmzp3L9OnTDzU+z5o1q0mZCRMmcNVVV3HyySeT\nk5PDpEmTwvr+QlK+A575lpMcrnrWaUdow6pt+/ifdzfx4aYaUpOKuXjcQK6aPJjxg/vY5SHTsvZ0\n2hBvg3XUEzmFhYVN5g8cOBCjSAKrrKxUVdV9+/bphRdeqK+88krMYmm+r1Qj+DlW7lH9n/GqDw1S\n3bGqzeKf79iv333qEx1y25ua98AS/fmTi7Wsqi4ysYUgXo775rpqXLSzox47YzCdyr333svSpUup\nrq5m5syZTRqOuyxPHSz4V6jcDde9BoNavktqb2Ud//n2Bl75dCd90pO54/xRfHfqUD5Z9iF9e6ZE\nMWjTmVliMJ3Kb3/7WyA+fngXNW/fBsUr4cr5cMyUgEV8PuVvy7fz28UbqfV4+eG0Y/nRtGPpldZF\nGtxNVFliMCaebXgTVv8VTvspjJ4VsMjO8hpu+fvnfLyljNNHZHPfrDEc2z8jyoGarsQSgzHxqnof\nvPkzOGosnP2rgEVeX7OTX726Hp8qD182lisnDrZGZRMySwzGxKsld0PNPrj25SMeY9Hg9fHgWxt4\nZtk2Jg7J4vdXjuOYfukxCtR0NZYYjIlH36xzfqsw9ccw4KQmq/ZW1vHj5z5lxbZ93Hj6MG4/fxTJ\n9mgKE0Z2NJm4Ul5ezp/+9KcO1582bRqrVq0KY0QxoArv/Ap69IEzb2myamvpQS597CPW7iznf64e\nx68vHG1JwYSdHVEmroSaGLqE7R/BlgI485fQ43CHhmt2lHPZY8s4WOdlweypzBoX2c5aTPdlicHE\nldtvv53Nmzczbtw4fvaznzFjxgwmTJjA2LFjef311wHYtm0bEydO5Oabb2bMmDGcd9551NTUHNrG\nSy+9xOTJkznuuOP48MMPY/VWOu7D30PP/jDx+4cWLSsq5ZonlpORmsQrPzyVcYP7tLIBY0JjbQym\nZW/fTo+dn0FiGA+To8bC+XNbXD137lzWr19/6LHX1dXV9O7dm9LSUk455ZRDTzvdvHkzL774In/5\ny1+48sorefnll7n22msB8Hg8rFixgkWLFnHfffexdOnS8MUfabvWwOZ3YcY9kNwDgGWbS7lh3kqG\n9O3JszdNoX+v1DY2YkxoLDGYuKWq3HnnnXzwwQckJCSwc+dO9uzZA8CQIUMYN87p4CUvL49t27Yd\nqnfppZcGXN4pLPtfSO0Nk24E4OPNZdzwzEqO6ZvOczdPITvDkoKJPEsMpmXnz6Umhr8wfu6559i7\ndy+rV68mOTmZoUOHUltbC9DksdqJiYlNLiU1rktMTMTj8UQ36FAcLHV6YZt0I6Rlsra4nBvnrWRw\nVjrP33yKJQUTNWFpYxCRmSKyUUSKROT2AOv/ICJr3OErESn3W+f1W7cwHPGYzqtXr15UVlYCTu9t\nOTk5JCcnk5+fz/bt29uo3cmteQ58DZD3fbaVHuT7f11J354pPHeTnSmY6Ar5jEFEEoFHgXOBYmCl\niCxU1cLGMqr6M7/y/wb496Beo6rx1+mriYl+/fpx2mmnceKJJzJp0iS+/PJLxo4dy8SJExk1alSs\nw4scnw9WPwPHTGVvj2F897Fl+FSZf8NkcnqnxTo6082E41LSZKBIVbcAuP06zwIKWyh/DU7Xn8YE\n9Pzzz7dZ5pNPPjk03dhfAzidyzfKzs7uPG0MO5bDvi3Un34rN81fxd7KOp6/eQrD7ZlHJgbCcSlp\nILDDb77YXXYEERkCDAPe81ucJiKrRGS5iHSDZygbE8D6l9GkHtzz1TA+31HOH64ax/hjstquZ0wE\niNN3QwgbELkcmKmqN7nz1wFTVHVOgLK3AYNU9d/8lg1U1Z0iMhwnYcxQ1c0B6s4GZgPk5ubmLViw\noEPxVlVVkZERf3+FxUtcmZmZjBgx4tC81+slMTH+unuMh7iKioqoqKhosqwjn6P4vEz9+Pt8mTyG\ni/bNYdaxyVwyMrx9J8TL8dWcxRWcUOOaPn36alWd2GbB9vTm09oATAUW+83fAdzRQtnPgFNb2dYz\nwOVtvab14BY5hYWF6vP5Ds3HWw9ujWIdl8/nC18PbpuWqt7TW39w571687yV6vX62q4TpHg5vpqz\nuIITrR7cwnEpaSUwUkSGiUgKcDVwxN1FIjIKyAI+9luWJSKp7nQ2cBott02YKEhLS6OsrKwxUZsA\nVJWysjLS0sLTKFzz2UtUks72vqfy+6vGkZBgj802sRVy47OqekRkDrAYSASeVtUvROR+nOzUmCSu\nBhZo02+cE4DHRcSH094xV/3uZjLRN2jQIIqLi9m7dy8AtbW1YfsCDKdYx5WWlsagQYNC3o7X66V+\nw9t86BvPH687hYxU+2mRib2wHIWqughY1GzZ3c3m7w1QbxkwNhwxmPBITk5m2LBhh+YLCgoYP358\nKzViI17jCtbLC1/nSl85/fJmMSKnm3RVauKePUTPmBhZsXUfJatfx0sCp5x7ZazDMeYQSwzGxEBF\ndQP/vuAzZqasgWOmIul2a6qJH5YYjImBexauJ6lyJyN820g8fmaswzGmCUsMxkTZ2+t289qaXdw9\nxmngZ8Q5sQ3ImGYsMRgTRaVVddz12nrGDsxkRtqXToc8OSfEOixjmrDEYEyUqCp3vrKOqjoPv7vi\nJBK2fQjDzgSx3y2Y+GKJwZgoeW3NTt4p3MMt5x3HcUl7oHK3kxiMiTOWGIyJgtKqOu5dWMjEIVnc\nePpw2Pq+s8ISg4lDlhiMiYKH3tpAdb2HuZeNJTFBYOsHkDkYsoa1XdmYKLPEYEyE/XNTKa98tpMf\nnnWs8+tmnw+2WvuCiV+WGIyJoNoGL796bR3Dsnvyo+nu48zLNkHNPhhyamyDM6YF9sQuYyLokfeK\n2FZWzfM3TSEt2e0/Yofb+9zgKbELzJhW2BmDMRHy1Z5KHv9gM5dOGMipI7IPr9ixAnpkQb8RLVc2\nJoYsMRgTAT6fcter68hITeJX3xrddGXxShg0ydoXTNyyxGBMBLy4agcrt+3nzgtOoG9Pv246a8ph\n75cweHLsgjOmDZYYjAmzvZV1/OeiDZwyvC+X5zXrzKd4lTMeZInBxK+wJAYRmSkiG0WkSERuD7D+\neyKyV0TWuMNNfuuuF5FN7nB9OOIxJpYeeLOQ2gYfD14yFml+uah4BUgCDMyLTXDGtEPIdyWJSCLw\nKHAuUAysFJGFAbrofFFV5zSr2xe4B5gIKLDarbs/1LiMiYV1ez0s/HwXPz1nJMf2zziywI4VkDsG\nUgOsMyZOhOOMYTJQpKpbVLUeWADMamfdfwGWqOo+NxksAezh9KZTqqn3Mr+wnuHZPfnhtGOPLKAK\nuz6FgROjH5wxQQhHYhgI7PCbL3aXNXeZiKwVkX+IyOAg6xoT9/743ib21igPXjKW1KTEIwvs3wa1\nFXD0uKjHZkwwovUDtzeAF1S1TkT+HzAPODuYDYjIbGA2QG5uLgUFBR0KpKqqqsN1I8niCk68xbWj\n0scTy2o4JUep27GOgh1Hlulf8hFjgFW7vFRVFkQ1vnjbX40sruBELS5VDWkApgKL/ebvAO5opXwi\nUOFOXwM87rfuceCatl4zLy9POyo/P7/DdSPJ4gpOPMXl9fr0kkf/qePvf0ffWPxeywXfuVv1vn6q\nDbXRC84VT/vLn8UVnFDjAlZpO77Xw3EpaSUwUkSGiUgKcDWw0L+AiAzwm70I2OBOLwbOE5EsEckC\nznOXGdNpPL/iaz79upy7LjiBjJRWfrS2+3PIHQ1JqdELzpgOCDkxqKoHmIPzhb4B+LuqfiEi94vI\nRW6xn4jIFyLyOfAT4Htu3X3AAzjJZSVwv7vMmE5hz4FaHn77S049th+XTmileUwVdq+BAda+YOJf\nWNoYVHURsKjZsrv9pu/AucQUqO7TwNPhiMOYaLvvjS+o87bwmwV/5V9DzX4YcHL0gjOmg+yXz8Z0\n0NLCPSxa9w3/PmMkw7J7tl549+fO2O5IMp2AJQZjOqCqzsPdr6/n+Nxe3HzG8LYr7F4DCUmQMyby\nwRkTIksMxnTA797ZyO4DtTx06VhSktrx32j3Wug/CpLTIh+cMSGyxGBMkNbsKOeZZdu4dsoQ8oZk\nta9SSaHzKAxjOgFLDMYEocHr445X1pHTK5VbZx7fvko1++HATsgZ3XZZY+KAde1pTBCe+udWNuw+\nwJ+vzaN3WnL7Ku1xnydpZwymk7AzBmPaqaikkt8v+YrzRucy88Sj2l+xxE0MdsZgOglLDMa0g8fr\n4xcvraVnSiIPXjI2uMp7voC0TOh9dGSCMybM7FKSMe3wxIdb+HxHOf97zXj69wrykRYlhZB7ovXx\nbDoNO2Mwpg0bv6nkv5ds4oKxR3HhSQParuBPFUo22GUk06lYYjCmFQ1eH7e89Dm90pJ4YNaJrT/2\nIpCKHVB3wHl4njGdhF1KMqYV//vuJtbtrOBP35lAv4wOPBV1zxfO2H7xbDoRO2MwpgXLt5Txv/lF\nXJ43iAvGBnkJqdGhxHBC+AIzJsIsMRgTwP6D9fzsxTUM7deT+y4K4a/9kkLIPAbSeocvOGMizC4l\nGdOMqnLby2sprarjlR+eRs/UEP6b7Cm09gXT6dgZgzHNPLt8O+8U7uG2maMYOyiz4xvy1EPZJrsj\nyXQ6YUkMIjJTRDaKSJGI3B5g/c9FpFBE1orIuyIyxG+dV0TWuMPC5nWNiabV2/dz/5uFTDu+Pzec\nNiy0jZUVgc9jicF0OiFfShKRROBR4FygGFgpIgtVtdCv2GfARFWtFpEfAr8BrnLX1aiq9V5iYq7k\nQC0/fHY1AzJ78N9XjSMhIcQfpJV+5Yz7Hxd6cMZEUTjOGCYDRaq6RVXrgQXALP8CqpqvqtXu7HJg\nUBhe15iwqff4+NFzn1JZ6+Hx6/Lok54S+kYbE0O/EaFvy5goElUNbQMilwMzVfUmd/46YIqqzmmh\n/CPAN6r6H+68B1gDeIC5qvpaC/VmA7MBcnNz8xYsWNCheKuqqsjIyOhQ3UiyuIITzrhUlfmF9eTv\n8PCDk1M5ZUDHT6T94zqh8HdkVnzJ8ql/CUucoegOn2M4ddW4pk+fvlpVJ7ZZUFVDGoDLgSf95q8D\nHmmh7LU4ZwypfssGuuPhwDbg2LZeMy8vTzsqPz+/w3UjyeIKTjjjevz9Ih1y25v60KLCkLfVJK4/\nn6E6/5KQtxkO3eFzDKeuGhewStvxvR6OS0k7gcF+84PcZU2IyDnAXcBFqlrnl5h2uuMtQAEwPgwx\nGdMub67dxUOLvuRbJw3gtn8ZFb4N+3xQugmyrX3BdD7hSAwrgZEiMkxEUoCrgSZ3F4nIeOBxnKRQ\n4rc8S0RS3els4DTAv9HamIhZvqWMn7/4OZOGZvG7K04OvbHZ34Gd0FBtDc+mUwr5riRV9YjIHGAx\nkAg8rapfiMj9OKctC4H/AjKAl9yHkH2tqhcBJwCPi4gPJ0nN1aZ3MxkTEau37+fGZ1YyuG8Pnrhu\nImnJieF9gcaGZztjMJ1QWH75rKqLgEXNlt3tN31OC/WWAUH2emJMaNYWl/O9p1fQv1cqz998Clk9\nw3AHUnOlm5yxJQbTCdkvn0238vmOcq57agWZ6ck8f/Mp5PZOi8wLlW6EtD7Qs39ktm9MBFliMN3G\nPzeVcs1fltO7RxIv3HwKR/fpEbkXa2x4tl7bTCdkicF0C2+t3c0Nz6zkmL7pvPyDUxncNz2yL1j6\nlV1GMp2WPV3VdGmqyiPvFfG7JV+RNySLp6+fRGZ6cmRftKYcqvbYHUmm07LEYLqs6noPt760lrfW\n7ebicUcz97KTwn/3USDW8Gw6OUsMpkvasPsAP3nhMzbvreLOC0Zx8xnDg++vuaPsVlXTyVliMF2K\nz6f8ddk2Hn77SzLTk5l/wxROH5kd3SBKN0JiCvQZ0nZZY+KQJQbTZWzaU8ldr65nxbZ9nHNCLr+5\n/CT6RuI3Cm0p3QR9j4VE++9lOic7ck2nV1Xn4bGCIp74YAs9U5P4zWUnccXEQdG7dNRc6VfWOY/p\n1CwxmE6rzuPl+U++5pH3iig7WM9lEwZx5wWj6JeRGrOYxNcA+7bC6ItjFoMxobLEYDqd6noPS7c3\ncNfy99lZXsPU4f345czjGX9MVqxDo0fNblAv9D8+1qEY02GWGEynUby/mgUrdvDsJ9spr25g3OA+\n/OelYzljZHbsLhs1k17tPnE+e2RsAzEmBJYYTFw7WOdh8Rff8I/VxSzbXIYInDc6l4kZFdx08alx\nkxAapVcXOxP9LDGYzssSg4k7JQdqWbJhD0sL9/DR5jLqPT6O6ZvOz889jkvGD2Rw33QKCgriLimA\nmxh6D4LU+OsW0pj2ssRgYu6bilpWbNvHJ1vK+GTrPopKqgAY3LcH104ZwswTj2LS0Ky4TATNpVcX\nQ387WzCdW1gSg4jMBP4Hp6OeJ1V1brP1qcB8IA8oA65S1W3uujuAGwEv8BNVXRyOmEz8qW3wsmNf\nNRv3VPLFrgMU7jpA4e4D7K10enrtmZLIxKF9uXTCQGaMyuW43IxOkQwOUXUSQ/aMWEdiTEhCTgwi\nkgg8CpwLFAMrRWRhs57YbgT2q+oIEbkaeBi4SkRG43QFOgY4GlgqIsepqjfUuEx0+XzKvup6Sg7U\nUVJZS0llHXsr6/i6rJrt+w6yvayabw7UouqUT0oQRub24syR/Rl9dG8mDc1i9IDeJCV24gf+HthF\nkrfWHp5nOr1wnDFMBopUdQuAiCwAZtG07+ZZwL3u9D+AR8T5U3AWsEBV64CtIlLkbu/jMMRlAvD5\nFK8qXp/S4PVR5/FR2+CltsEZ13kOT9c2+Fizs4Gdn2znYJ2HAzUeDtQ2cKCmgQO1HipqnOmKmgbK\nDtbj9ekRr5edkcqQfulMPbYfQ/r2ZGh2Osf2z2BkbgapSVF4oF002TOSTBcRjsQwENjhN18MTGmp\njNtHdAXQz12+vFndgWGIKaBH84t4b00tL+xYdegvV4VD06DNluuhadxyh6cPfwk6y/XQdKBlfi9x\nZFmgoqKGPxZ+1CQe9SvUNAZnzucDr98Xvcfnw+tVPL7G+cPLfT6c8ZHf3W1btx6ABIHePZLpnZZM\n7x5J9E5LZkROBr3TksnulUJOrzRyeqWS0zuVnF5p9O+VGp2nmcYLe6qq6SI6TeOziMwGZgPk5uZS\nUFAQ9DZWF9axq9LDnoMljds8vP1Dr0OTZdKsgP8VbwlQPtA2aLY80LJEvNQdPOCulyav5x+DcLh3\npSQgMQESEiAxERJESBTnC7xx7AyJznxCYrP1QnICJCdCSoKQkggpCZCSKIeWNdRW06dXOmlJQlqi\n/z7zukPt4TdTD5RBZRlUApuJnKqqqg4dA5E08qv3yElM56NVG0C+jHU4TcTj/gKLK1hRi0tVQxqA\nqcBiv/k7gDualVkMTHWnk4BSnO+4JmX9y7U25OXlaUfl5+d3uG4kWVzBicu4nrlQK347MdZRBBSX\n+0strmCFGhewStvxvR6Olr6VwEgRGSYiKTiNyQublVkIXO9OXw685wa5ELhaRFJFZBgwElgRhpiM\nib7STVSnD4p1FMaELORLSeq0GczB+Ws/EXhaVb8QkftxstNC4Cngb27j8j6c5IFb7u84DdUe4Mdq\ndySZzqj2AFTu5mD/c2MdiTEhC0sbg6ouAhY1W3a333QtcEULdR8EHgxHHMbEjNvwbGcMpivoxDeN\nGxNH3FtVq9MjdlOdMVFjicGYcCj9ChKSqU07KtaRGBMySwzGhEPpV9B3OJrQae4AN6ZFlhiMCYe9\nG+1RGKbLsMRgTKg89bBvi/3i2XQZlhiMCdX+rU53ntnWnafpGiwxGBOqvRudsV1KMl2EJQZjQlXq\nJgbrztN0EZYYjAlV6SbrztN0KZYYjAmV3ZFkuhhLDMaEwudzzhjsjiTThVhiMCYUB3ZCw0FLDKZL\nscRgTCgau/Psb7eqmq7DEoMxobB+nk0XZInBmFDs3QhpfaBn/1hHYkzYWGIwJhSlm5zLSP4dfRvT\nyYWUGETslc4JAAASCklEQVSkr4gsEZFN7jgrQJlxIvKxiHwhImtF5Cq/dc+IyFYRWeMO40KJx5io\nK90I2fbDNtO1hHrGcDvwrqqOBN5155urBr6rqmOAmcB/i0gfv/W3quo4d1gTYjzGRE/1Pji4156R\nZLqcUBPDLGCeOz0PuLh5AVX9SlU3udO7gBLALsiazs/tztPuSDJdTaiJIVdVd7vT3wC5rRUWkclA\nCrDZb/GD7iWmP4hIaojxGBM9jc9IsjuSTBcjqtp6AZGlQKD+Cu8C5qlqH7+y+1X1iHYGd90AoAC4\nXlWX+y37BidZPAFsVtX7W6g/G5gNkJubm7dgwYLW31kLqqqqyMiIv2faWFzBiYe4ji36KwN3vsUH\nZ74Ikhg3cQVicQWnq8Y1ffr01ao6sc2CqtrhAdgIDHCnBwAbWyjXG/gUuLyVbU0D3mzP6+bl5WlH\n5efnd7huJFlcwYmLuP52meqfTm2yKC7iCsDiCk5XjQtYpe34jg31UtJC4Hp3+nrg9eYFRCQFeBWY\nr6r/aLZugDsWnPaJ9SHGY0z0lGyAnNGxjsKYsAs1McwFzhWRTcA57jwiMlFEnnTLXAmcCXwvwG2p\nz4nIOmAdkA38R4jxGBMdtRVwoBhyToh1JMaEXVIolVW1DJgRYPkq4CZ3+lng2Rbqnx3K6xsTMyVf\nOmM7YzBdkP3y2ZiOKPnCGedaYjBdjyUGYzqiZAOkZEDm4FhHYkzYWWIwpiNKNjjtC/aMJNMFWWIw\nJliqsOcLa3g2XZYlBmOCdXAv1OyzhmfTZVliMCZYe9yGZztjMF2UJQZjglWywRnnjIltHMZEiCUG\nY4JVUgjp2ZBhDwk2XZMlBmOCVVJol5FMl2aJwZhgeD1OG8NRJ8U6EmMixhKDMcEo2wSeWhhgicF0\nXZYYjAnGN+ucsZ0xmC7MEoMxwdj9OSSmQvbIWEdiTMRYYjAmGN+sdR6cl5gc60iMiRhLDMa0lyrs\nXmuXkUyXZ4nBmPaq2AG15dbwbLq8kBKDiPQVkSUisskdZ7VQzuvXe9tCv+XDROQTESkSkRfdbkCN\niU+HGp5Pjm0cxkRYqGcMtwPvqupI4F13PpAaVR3nDhf5LX8Y+IOqjgD2AzeGGI8xkbN7LUgC5Nqj\nMEzXFmpimAXMc6fnARe3t6KICHA28I+O1Dcm6nZ9BtnHQUp6rCMxJqJEVTteWaRcVfu40wLsb5xv\nVs4DrAE8wFxVfU1EsoHl7tkCIjIYeFtVT2zhtWYDswFyc3PzFixY0KGYq6qqyMjI6FDdSLK4ghP1\nuFQ5ddl3Kes3iY2jfhI/cbWTxRWcrhrX9OnTV6vqxDYLqmqrA7AUWB9gmAWUNyu7v4VtDHTHw4Ft\nwLFANlDkV2YwsL6teFSVvLw87aj8/PwO140kiys4UY+rbIvqPb1VVz7VajHbX8GxuIITalzAKm3H\nd2xSOxLHOS2tE5E9IjJAVXeLyACgpIVt7HTHW0SkABgPvAz0EZEkVfUAg4CdbcVjTEzsXO2MB7b9\nx5YxnV2obQwLgevd6euB15sXEJEsEUl1p7OB04BCN3vlA5e3Vt+YuFC8CpJ6WK9tplsINTHMBc4V\nkU3AOe48IjJRRJ50y5wArBKRz3ESwVxVLXTX3Qb8XESKgH7AUyHGY0xk7FwFR4+DxDZPso3p9EI6\nylW1DJgRYPkq4CZ3ehkwtoX6W4DJocRgTMR56p1bVSffHOtIjIkK++WzMW3Zsx68dTDI2hdM92CJ\nwZi2FK9yxtbwbLoJSwzGtOXrZdB7EGQOinUkxkSFJQZjWqMK2z6CIaeCSKyjMSYqLDEY05qyzXCw\nBIaeFutIjIkaSwzGtGb7R854iCUG031YYjCmNduXQc/+0G9ErCMxJmosMRjTElXnjMHaF0w3Y4nB\nmJbs2+L02jb0jFhHYkxUWWIwpiVF7zrjEUf8uN+YLs0SgzEt2fweZA2FvsNjHYkxUWWJwZhAPPWw\n7UM41s4WTPdjicGYQIpXQH2VXUYy3ZIlBmMC2fQOJCRZw7PpliwxGNOcKmx4E4adCWm9Yx2NMVFn\nicGY5vZ+Cfs2w6gLYx2JMTERUmIQkb4iskRENrnjrABlpovIGr+hVkQudtc9IyJb/daNCyUeY8Ji\nwxvO+PgLYhuHMTES6hnD7cC7qjoSeNedb0JV81V1nKqOA84GqoF3/Irc2rheVdeEGI8xodvwBgya\nBL0HxDoSY2Ii1MQwC5jnTs8DLm6j/OXA26paHeLrGhMZpUXwzVo44aJYR2JMzIiqdryySLmq9nGn\nBdjfON9C+feA36vqm+78M8BUoA73jENV61qoOxuYDZCbm5u3YMGCDsVcVVVFRkZGh+pGksUVnEjF\nNXTrcwzZ/g8+nvok9an94iauUFlcwemqcU2fPn21qrbdFaGqtjoAS4H1AYZZQHmzsvtb2c4AYC+Q\n3GyZAKk4Zxx3txWPqpKXl6cdlZ+f3+G6kWRxBScicXm9qr8/UXX+xR3eRLfaX2FgcQUn1LiAVdqO\n79ikdiSOc1paJyJ7RGSAqu4WkQFASSubuhJ4VVUb/La9252sE5G/Are0FY8xEfP1Mqj4Gmb8OtaR\nGBNTobYxLASud6evB15vpew1wAv+C9xk0ngZ6mKcMxFjYuPT+ZDSy25TNd1eqIlhLnCuiGwCznHn\nEZGJIvJkYyERGQoMBt5vVv85EVkHrAOygf8IMR5jOqZyD6x/BcZ/B1LSYx2NMTHV5qWk1qhqGXDE\nw2RUdRVwk9/8NmBggHJnh/L6xoTN6mfA1wCTbo51JMbEnP3y2RhPHax6CkacC9nWhacxlhiM+XQ+\nVO2BU+fEOhJj4oIlBtO9NdTCh7+DY06FYWfFOhpj4kJIbQzGdHqrnobK3XDpX0Ak1tEYExfsjMF0\nX1V7oWAuDJ8Ow6zfBWMaWWIw3dfSe6GhGs7/TawjMSauWGIw3dPm92DNszD1R9D/uFhHY0xcscRg\nup+DpfDqD6D/KJh2R6yjMSbuWOOz6V68DfDyTVCzH659BZJ7xDoiY+KOJQbTfajC27+ELflw0SNw\n1ImxjsiYuGSXkkz3oApLfu3cnnraT2HCdbGOyJi4ZWcMpuvz1MPbtzrPQ5o8G865N8YBGRPfLDGY\nru3ALnjp+7BjOZz+c5hxt/2QzZg2WGIwXZPPC6v/CkvvA58HLv8rnHhprKMyplOwxGC6Fm+D06/C\nB/8FZZuc5x99+7+h7/BYR2ZMpxFS47OIXCEiX4iIT0Ra7GBaRGaKyEYRKRKR2/2WDxORT9zlL4pI\nSijxmG7K54XiVbD4Lvj9aHh1NiSlwpXz4buvW1IwJkihnjGsBy4FHm+pgIgkAo8C5wLFwEoRWaiq\nhcDDwB9UdYGI/Bm4EXgsxJhMV1eznz7718KydU5C2Pq+87uEhCQ4biaMvw5GngcJdtOdMR0Rag9u\nGwCk9ca8yUCRqm5xyy4AZonIBuBs4F/dcvOAe7HE0PX5vE7nON4659KPpw689c64vgpqyqG2/PC4\neh9U7IDyr51xzX7GNW6r90A47nwYMQOOPRvS+8bynRnTJUSjjWEgsMNvvhiYAvQDylXV47f8iO4/\nw+rNnzGp8B1Y3xPQpuvUf15bWB6udUcuP6W2Fj5LazmO1mJsbV2I8Z/W0ACfNDtMWo2jxRmnEdhT\nB+olKCm9IHMQ9BkMgyZB1hA+3+Pj5POuhYz+wW3LGNOmNhODiCwFjgqw6i5VfT38IbUYx2xgNkBu\nbi4FBQVBb+OYsgbSUgdyUAK8bTn0DwDa5CyotTOipuu01bOnwNtUAU+Sh6TkpFbLNXfka3WkXuvx\nNjQ0kJycHMT7ajqvTVYl4ktIwpeQjIozbhxUnLE3MQ1PUgYNyT3xJGXgSeqJJiQ33XwDVKVUsX/V\nF63EFBtVVVUdOjYjzeIKTrePS1VDHoACYGIL66YCi/3m73AHAUqBpEDlWhvy8vK0o/Lz8ztcN5Is\nruBYXMGxuILTVeMCVmk7vmOj0Tq3Ehjp3oGUAlwNLHSDzAcud8tdD0TtDMQYY0xgod6ueomIFOP8\ntf+WiCx2lx8tIosA1GlDmAMsBjYAf1fVxmsAtwE/F5EinDaHp0KJxxhjTOhCvSvpVeDVAMt3ARf4\nzS8CFgUotwXnriVjjDFxwm70NsYY04QlBmOMMU1YYjDGGNOEJQZjjDFNWGIwxhjThOgRj0WIfyKy\nF9jewerZOD+sizcWV3AsruBYXMHpqnENUdU2nyPTKRNDKERklaq2+IjwWLG4gmNxBcfiCk53j8su\nJRljjGnCEoMxxpgmumNieCLWAbTA4gqOxRUciys43TqubtfGYIwxpnXd8YzBGGNMK7pkYhCRK0Tk\nCxHxicjEZuvuEJEiEdkoIv/SQv1hIvKJW+5F93Hh4Y7xRRFZ4w7bRGRNC+W2icg6t9yqcMcR4PXu\nFZGdfrFd0EK5me4+LBKR26MQ13+JyJcislZEXhWRPi2Ui8r+auv9i0iq+xkXucfS0EjF4veag0Uk\nX0QK3eP/3wOUmSYiFX6f792Rjst93VY/F3H80d1fa0VkQhRiOt5vP6wRkQMi8tNmZaKyv0TkaREp\nEZH1fsv6isgSEdnkjrNaqHu9W2aTiFwfloDa02lDZxuAE4DjadaBEDAa+BxIBYYBm4HEAPX/Dlzt\nTv8Z+GGE4/0dcHcL67YB2VHcd/cCt7RRJtHdd8OBFHefjo5wXOdxuFOnh4GHY7W/2vP+gR8Bf3an\nrwZejMJnNwCY4E73Ar4KENc04M1oHU/t/Vxwnsb8Nk4HXqcAn0Q5vkTgG5z7/KO+v4AzgQnAer9l\nvwFud6dvD3TMA32BLe44y53OCjWeLnnGoKobVHVjgFWzgAWqWqeqW4Eimj32W0QEOBv4h7toHnBx\npGJ1X+9K4IVIvUYETAaKVHWLqtYDC3D2bcSo6jt6uH/w5cCgSL5eG9rz/mfhHDvgHEsz3M86YlR1\nt6p+6k5X4vR/Etl+1MNnFjBfHcuBPiIyIIqvPwPYrKod/eFsSFT1A2Bfs8X+x1BL30P/AixR1X2q\nuh9YAswMNZ4umRhaMRDY4TdfzJH/cfoB5X5fQoHKhNMZwB5V3dTCegXeEZHVbr/X0TDHPZ1/uoXT\n1/bsx0i6Aeevy0Cisb/a8/4PlXGPpQqcYysq3EtX44FPAqyeKiKfi8jbIjImSiG19bnE+pi6mpb/\nOIvF/gLIVdXd7vQ3QG6AMhHZbyF11BNLIrIUOCrAqrtUNS66CG1njNfQ+tnC6aq6U0RygCUi8qX7\n10VE4gIeAx7A+Y/8AM5lrhtCeb1wxNW4v0TkLsADPNfCZsK+vzobEckAXgZ+qqoHmq3+FOdySZXb\nfvQaMDIKYcXt5+K2IV6E0xd9c7HaX02oqopI1G4h7bSJQVXP6UC1ncBgv/lB7jJ/ZTinsUnuX3qB\nyoQlRhFJAi4F8lrZxk53XCIir+JcxgjpP1R7952I/AV4M8Cq9uzHsMclIt8DLgRmqHuBNcA2wr6/\nAmjP+28sU+x+zpk4x1ZEiUgyTlJ4TlVfab7eP1Go6iIR+ZOIZKtqRJ8L1I7PJSLHVDudD3yqqnua\nr4jV/nLtEZEBqrrbvaxWEqDMTpx2kEaDcNpWQ9LdLiUtBK527xgZhpP5V/gXcL9w8oHL3UXXA5E6\nAzkH+FJViwOtFJGeItKrcRqnAXZ9oLLh0uy67iUtvN5KYKQ4d2+l4JyGL4xwXDOBXwIXqWp1C2Wi\ntb/a8/4X4hw74BxL77WUzMLFbcN4Ctigqr9vocxRjW0dIjIZ5zsgogmrnZ/LQuC77t1JpwAVfpdR\nIq3Fs/ZY7C8//sdQS99Di4HzRCTLvex7nrssNJFubY/FgPOFVgzUAXuAxX7r7sK5o2QjcL7f8kXA\n0e70cJyEUQS8BKRGKM5ngB80W3Y0sMgvjs/d4QucSyqR3nd/A9YBa90Dc0DzuNz5C3DuetkcpbiK\ncK6lrnGHPzePK5r7K9D7B+7HSVwAae6xU+QeS8OjsI9Ox7kEuNZvP10A/KDxOAPmuPvmc5xG/FOj\nEFfAz6VZXAI86u7PdfjdTRjh2HrifNFn+i2L+v7CSUy7gQb3u+tGnDapd4FNwFKgr1t2IvCkX90b\n3OOsCPh+OOKxXz4bY4xportdSjLGGNMGSwzGGGOasMRgjDGmCUsMxhhjmrDEYIwxpglLDMYYY5qw\nxGCMMaYJSwzGGGOa+P97Iv/Z4Pf01AAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f597fce72d0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "_x = np.linspace(-10., 10., 1000)\n",
    "x = tf.convert_to_tensor(_x)\n",
    "\n",
    "sigmoid = tf.nn.sigmoid(x)\n",
    "tanh = tf.nn.tanh(x)\n",
    "\n",
    "with tf.Session() as sess:\n",
    "    _sigmoid, _tanh = sess.run([sigmoid, tanh])\n",
    "    plt.plot(_x, _sigmoid, label='sigmoid')\n",
    "    plt.plot(_x, _tanh, label='tanh')\n",
    "    plt.legend(bbox_to_anchor=(0.5, 1.0))\n",
    "    plt.grid()\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "Q3. Apply `softmax` to x."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[  8.92509008e-04   2.42609112e-03   1.79265216e-02   9.78754938e-01]\n",
      " [  2.14400887e-03   1.58422012e-02   1.17058903e-01   8.64954829e-01]]\n"
     ]
    }
   ],
   "source": [
    "_x = np.array([[1, 2, 4, 8], [2, 4, 6, 8]], dtype=np.float32)\n",
    "x = tf.convert_to_tensor(_x)\n",
    "out = tf.nn.softmax(x, dim=-1)\n",
    "with tf.Session() as sess:\n",
    "    _out = sess.run(out)\n",
    "    print(_out)    \n",
    "    assert np.allclose(np.sum(_out, axis=-1), 1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "Q4. Apply `dropout` with keep_prob=.5 to x."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_x =\n",
      " [[ 1.  2.  4.  8.]\n",
      " [ 2.  4.  6.  8.]]\n",
      "_out =\n",
      " [[  2.   4.   8.  16.]\n",
      " [  4.   0.   0.  16.]]\n"
     ]
    }
   ],
   "source": [
    "_x = np.array([[1, 2, 4, 8], [2, 4, 6, 8]], dtype=np.float32)\n",
    "print(\"_x =\\n\" , _x)\n",
    "x = tf.convert_to_tensor(_x)\n",
    "out = tf.nn.dropout(x, keep_prob=0.5)\n",
    "with tf.Session() as sess:\n",
    "    _out = sess.run(out)\n",
    "    print(\"_out =\\n\", _out) "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "# Fully Connected"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "Q5. Apply a fully connected layer to x with 2 outputs and then an sigmoid function."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[ 0.51716274  0.419792  ]\n",
      " [ 0.58838969  0.48844361]\n",
      " [ 0.18143947  0.52119553]\n",
      " [ 0.61469793  0.58985186]\n",
      " [ 0.27449742  0.75117522]\n",
      " [ 0.13121325  0.21781211]\n",
      " [ 0.86445647  0.85839492]\n",
      " [ 0.63927263  0.67295653]]\n"
     ]
    }
   ],
   "source": [
    "x = tf.random_normal([8, 10])\n",
    "out = tf.contrib.layers.fully_connected(inputs=x, num_outputs=2, \n",
    "                                        activation_fn=tf.nn.sigmoid,\n",
    "                                        weights_initializer=tf.contrib.layers.xavier_initializer())\n",
    "with tf.Session() as sess:\n",
    "    sess.run(tf.global_variables_initializer())\n",
    "    print(sess.run(out))\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "## Convolution"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "Q6. Apply 2 kernels of width-height (2, 2), stride 1, and same padding to x."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "tf.reset_default_graph()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(2, 3, 3, 2)\n"
     ]
    }
   ],
   "source": [
    "x = tf.random_uniform(shape=(2, 3, 3, 3), dtype=tf.float32)\n",
    "filter = tf.get_variable(\"filter\", shape=(2, 2, 3, 2), dtype=tf.float32, \n",
    "                    initializer=tf.random_uniform_initializer())\n",
    "out = tf.nn.conv2d(x, filter, strides=[1, 1, 1, 1], padding=\"SAME\")\n",
    "init = tf.global_variables_initializer()\n",
    "with tf.Session() as sess:\n",
    "    sess.run(init)\n",
    "    _out = sess.run(out)\n",
    "    print(_out.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "Q7. Apply 3 kernels of width-height (2, 2), stride 1, dilation_rate 2 and valid padding to x."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "tf.reset_default_graph()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(4, 8, 8, 2)\n"
     ]
    }
   ],
   "source": [
    "x = tf.random_uniform(shape=(4, 10, 10, 3), dtype=tf.float32)\n",
    "filter = tf.get_variable(\"filter\", shape=(2, 2, 3, 2), dtype=tf.float32, \n",
    "                    initializer=tf.random_uniform_initializer())\n",
    "out = tf.nn.atrous_conv2d(x, filter, padding=\"VALID\", rate=2)\n",
    "init = tf.global_variables_initializer()\n",
    "with tf.Session() as sess:\n",
    "    sess.run(init)\n",
    "    _out = sess.run(out)\n",
    "    print(_out.shape)\n",
    "# Do we really have to distinguish between these two functions? \n",
    "# Unless you want to use stride of 2 or more,\n",
    "# You can just use tf.nn.atrous_conv2d. For normal convolution, set rate 1."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "Q8. Apply 4 kernels of width-height (3, 3), stride 2, and same padding to x."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "tf.reset_default_graph()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(4, 5, 5, 4)\n"
     ]
    }
   ],
   "source": [
    "x = tf.random_uniform(shape=(4, 10, 10, 5), dtype=tf.float32)\n",
    "filter = tf.get_variable(\"filter\", shape=(3, 3, 5, 4), dtype=tf.float32, \n",
    "                    initializer=tf.random_uniform_initializer())\n",
    "out = tf.nn.conv2d(x, filter, strides=[1, 2, 2, 1], padding=\"SAME\")\n",
    "init = tf.global_variables_initializer()\n",
    "with tf.Session() as sess:\n",
    "    sess.run(init)\n",
    "    _out = sess.run(out)\n",
    "    print(_out.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "Q9. Apply 4 times of kernels of width-height (3, 3), stride 2, and same padding to x, depth-wise."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "tf.reset_default_graph()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(4, 5, 5, 20)\n"
     ]
    }
   ],
   "source": [
    "x = tf.random_uniform(shape=(4, 10, 10, 5), dtype=tf.float32)\n",
    "filter = tf.get_variable(\"filter\", shape=(3, 3, 5, 4), dtype=tf.float32, \n",
    "                    initializer=tf.random_uniform_initializer())\n",
    "out = tf.nn.depthwise_conv2d(x, filter, strides=[1, 2, 2, 1], padding=\"SAME\")\n",
    "init = tf.global_variables_initializer()\n",
    "with tf.Session() as sess:\n",
    "    sess.run(init)\n",
    "    _out = sess.run(out)\n",
    "    print(_out.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "Q10. Apply 5 kernels of height 3, stride 2, and valid padding to x."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "tf.reset_default_graph()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(4, 4, 5)\n"
     ]
    }
   ],
   "source": [
    "x = tf.random_uniform(shape=(4, 10, 5), dtype=tf.float32)\n",
    "filter = tf.get_variable(\"filter\", shape=(3, 5, 5), dtype=tf.float32, \n",
    "                    initializer=tf.random_uniform_initializer())\n",
    "out = tf.nn.conv1d(x, filter, stride=2, padding=\"VALID\")\n",
    "init = tf.global_variables_initializer()\n",
    "with tf.Session() as sess:\n",
    "    sess.run(init)\n",
    "    _out = sess.run(out)\n",
    "    print(_out.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "Q11. Apply conv2d transpose with 5 kernels of width-height (3, 3), stride 2, and same padding to x."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "tf.reset_default_graph()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(4, 10, 10, 5)\n"
     ]
    }
   ],
   "source": [
    "x = tf.random_uniform(shape=(4, 5, 5, 4), dtype=tf.float32)\n",
    "filter = tf.get_variable(\"filter\", shape=(3, 3, 5, 4), dtype=tf.float32, \n",
    "                    initializer=tf.random_uniform_initializer())\n",
    "shp = x.get_shape().as_list()\n",
    "output_shape = [shp[0], shp[1]*2, shp[2]*2, 5]\n",
    "out = tf.nn.conv2d_transpose(x, filter, strides=[1, 2, 2, 1], output_shape=output_shape, padding=\"SAME\")\n",
    "init = tf.global_variables_initializer()\n",
    "with tf.Session() as sess:\n",
    "    sess.run(init)\n",
    "    _out = sess.run(out)\n",
    "    print(_out.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "Q12. Apply conv2d transpose with 5 kernels of width-height (3, 3), stride 2, and valid padding to x."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "tf.reset_default_graph()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(4, 11, 11, 5)\n"
     ]
    }
   ],
   "source": [
    "x = tf.random_uniform(shape=(4, 5, 5, 4), dtype=tf.float32)\n",
    "filter = tf.get_variable(\"filter\", shape=(3, 3, 5, 4), dtype=tf.float32, \n",
    "                    initializer=tf.random_uniform_initializer())\n",
    "shp = x.get_shape().as_list()\n",
    "output_shape = [shp[0], (shp[1]-1)*2+3, (shp[2]-1)*2+3, 5]\n",
    "out = tf.nn.conv2d_transpose(x, filter, strides=[1, 2, 2, 1], output_shape=output_shape, padding=\"VALID\")\n",
    "init = tf.global_variables_initializer()\n",
    "with tf.Session() as sess:\n",
    "    sess.run(init)\n",
    "    _out = sess.run(out)\n",
    "    print(_out.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "Q13. Apply max pooling and average pooling of window size 2, stride 1, and valid padding to x."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1st channel of x =\n",
      " [[[ 1.  2.  3.]\n",
      "  [ 4.  5.  6.]\n",
      "  [ 7.  8.  9.]]]\n",
      "\n",
      "2nd channel of x =\n",
      " [[[ 10.  11.  12.]\n",
      "  [ 13.  14.  15.]\n",
      "  [ 16.  17.  18.]]]\n",
      "\n",
      "3rd channel of x =\n",
      " [[[ 19.  20.  21.]\n",
      "  [ 22.  23.  24.]\n",
      "  [ 25.  26.  27.]]]\n",
      "\n",
      "1st channel of max pooling =\n",
      " [[[ 5.  6.]\n",
      "  [ 8.  9.]]]\n",
      "\n",
      "2nd channel of max pooling =\n",
      " [[[ 14.  15.]\n",
      "  [ 17.  18.]]]\n",
      "\n",
      "3rd channel of max pooling =\n",
      " [[[ 23.  24.]\n",
      "  [ 26.  27.]]]\n",
      "\n",
      "1st channel of avg pooling =\n",
      " [[[ 3.  4.]\n",
      "  [ 6.  7.]]]\n",
      "\n",
      "2nd channel of avg pooling =\n",
      " [[[ 12.  13.]\n",
      "  [ 15.  16.]]]\n",
      "\n",
      "3rd channel of avg pooling =\n",
      " [[[ 21.  22.]\n",
      "  [ 24.  25.]]]\n"
     ]
    }
   ],
   "source": [
    "_x = np.zeros((1, 3, 3, 3), dtype=np.float32)\n",
    "_x[0, :, :, 0] = np.arange(1, 10, dtype=np.float32).reshape(3, 3)\n",
    "_x[0, :, :, 1] = np.arange(10, 19, dtype=np.float32).reshape(3, 3)\n",
    "_x[0, :, :, 2] = np.arange(19, 28, dtype=np.float32).reshape(3, 3)\n",
    "print(\"1st channel of x =\\n\", _x[:, :, :, 0])\n",
    "print(\"\\n2nd channel of x =\\n\", _x[:, :, :, 1])\n",
    "print(\"\\n3rd channel of x =\\n\", _x[:, :, :, 2])\n",
    "x = tf.constant(_x)\n",
    "\n",
    "maxpool = tf.nn.max_pool(x, [1, 2, 2, 1], [1, 1, 1, 1], padding=\"VALID\")\n",
    "avgpool = tf.nn.avg_pool(x, [1, 2, 2, 1], [1, 1, 1, 1], padding=\"VALID\")\n",
    "with tf.Session() as sess:\n",
    "    _maxpool, _avgpool = sess.run([maxpool, avgpool])\n",
    "    print(\"\\n1st channel of max pooling =\\n\", _maxpool[:, :, :, 0])\n",
    "    print(\"\\n2nd channel of max pooling =\\n\", _maxpool[:, :, :, 1])\n",
    "    print(\"\\n3rd channel of max pooling =\\n\", _maxpool[:, :, :, 2])\n",
    "    print(\"\\n1st channel of avg pooling =\\n\", _avgpool[:, :, :, 0])\n",
    "    print(\"\\n2nd channel of avg pooling =\\n\", _avgpool[:, :, :, 1])\n",
    "    print(\"\\n3rd channel of avg pooling =\\n\", _avgpool[:, :, :, 2])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}