qlearning.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Q-learning\n",
    "\n",
    "This notebook will guide you through implementation of vanilla Q-learning algorithm.\n",
    "\n",
    "You need to implement QLearningAgent (follow instructions for each method) and use it on a number of tests below."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The autoreload extension is already loaded. To reload it, use:\n",
      "  %reload_ext autoreload\n"
     ]
    }
   ],
   "source": [
    "#XVFB will be launched if you run on a server\n",
    "import os\n",
    "if type(os.environ.get(\"DISPLAY\")) is not str or len(os.environ.get(\"DISPLAY\"))==0:\n",
    "    !bash ../xvfb start\n",
    "    %env DISPLAY=:1\n",
    "        \n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "%load_ext autoreload\n",
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Overwriting qlearning.py\n"
     ]
    }
   ],
   "source": [
    "%%writefile qlearning.py\n",
    "from collections import defaultdict\n",
    "import random, math\n",
    "import numpy as np\n",
    "\n",
    "class QLearningAgent:\n",
    "    def __init__(self, alpha, epsilon, discount, get_legal_actions):\n",
    "        \"\"\"\n",
    "        Q-Learning Agent\n",
    "        based on http://inst.eecs.berkeley.edu/~cs188/sp09/pacman.html\n",
    "        Instance variables you have access to\n",
    "          - self.epsilon (exploration prob)\n",
    "          - self.alpha (learning rate)\n",
    "          - self.discount (discount rate aka gamma)\n",
    "\n",
    "        Functions you should use\n",
    "          - self.get_legal_actions(state) {state, hashable -> list of actions, each is hashable}\n",
    "            which returns legal actions for a state\n",
    "          - self.get_qvalue(state,action)\n",
    "            which returns Q(state,action)\n",
    "          - self.set_qvalue(state,action,value)\n",
    "            which sets Q(state,action) := value\n",
    "\n",
    "        !!!Important!!!\n",
    "        Note: please avoid using self._qValues directly. \n",
    "            There's a special self.get_qvalue/set_qvalue for that.\n",
    "        \"\"\"\n",
    "\n",
    "        self.get_legal_actions = get_legal_actions\n",
    "        self._qvalues = defaultdict(lambda: defaultdict(lambda: 0))\n",
    "        self.alpha = alpha\n",
    "        self.epsilon = epsilon\n",
    "        self.discount = discount\n",
    "\n",
    "    def get_qvalue(self, state, action):\n",
    "        \"\"\" Returns Q(state,action) \"\"\"\n",
    "        return self._qvalues[state][action]\n",
    "\n",
    "    def set_qvalue(self,state,action,value):\n",
    "        \"\"\" Sets the Qvalue for [state,action] to the given value \"\"\"\n",
    "        self._qvalues[state][action] = value\n",
    "\n",
    "    #---------------------START OF YOUR CODE---------------------#\n",
    "\n",
    "    def get_value(self, state):\n",
    "        \"\"\"\n",
    "        Compute your agent's estimate of V(s) using current q-values\n",
    "        V(s) = max_over_action Q(state,action) over possible actions.\n",
    "        Note: please take into account that q-values can be negative.\n",
    "        \"\"\"\n",
    "        possible_actions = self.get_legal_actions(state)\n",
    "\n",
    "        #If there are no legal actions, return 0.0\n",
    "        if len(possible_actions) == 0:\n",
    "            return 0.0\n",
    "\n",
    "        value = max([self.get_qvalue(state,action) for action in possible_actions])\n",
    "\n",
    "        return value\n",
    "\n",
    "    def update(self, state, action, reward, next_state):\n",
    "        \"\"\"\n",
    "        You should do your Q-Value update here:\n",
    "           Q(s,a) := (1 - alpha) * Q(s,a) + alpha * (r + gamma * V(s'))\n",
    "        \"\"\"\n",
    "\n",
    "        #agent parameters\n",
    "        gamma = self.discount\n",
    "        learning_rate = self.alpha\n",
    "\n",
    "        qvalue = (1-learning_rate)*self.get_qvalue(state,action)+learning_rate*(reward+gamma*self.get_value(next_state))\n",
    "        \n",
    "        self.set_qvalue(state, action, qvalue)\n",
    "\n",
    "    \n",
    "    def get_best_action(self, state):\n",
    "        \"\"\"\n",
    "        Compute the best action to take in a state (using current q-values). \n",
    "        \"\"\"\n",
    "        possible_actions = self.get_legal_actions(state)\n",
    "\n",
    "        #If there are no legal actions, return None\n",
    "        if len(possible_actions) == 0:\n",
    "            return None\n",
    "\n",
    "        #value = self.get_value(state)\n",
    "        value = max([self.get_qvalue(state,action) for action in possible_actions])\n",
    "        best_action = None\n",
    "        \n",
    "        for a in possible_actions:\n",
    "            if self.get_qvalue(state, a) == value:\n",
    "                best_action = a\n",
    "                break\n",
    "\n",
    "        return best_action\n",
    "\n",
    "    def get_action(self, state):\n",
    "        \"\"\"\n",
    "        Compute the action to take in the current state, including exploration.  \n",
    "        With probability self.epsilon, we should take a random action.\n",
    "            otherwise - the best policy action (self.getPolicy).\n",
    "        \n",
    "        Note: To pick randomly from a list, use random.choice(list). \n",
    "              To pick True or False with a given probablity, generate uniform number in [0, 1]\n",
    "              and compare it with your probability\n",
    "        \"\"\"\n",
    "\n",
    "        # Pick Action\n",
    "        possible_actions = self.get_legal_actions(state)\n",
    "        action = None\n",
    "\n",
    "        #If there are no legal actions, return None\n",
    "        if len(possible_actions) == 0:\n",
    "            return None\n",
    "\n",
    "        #agent parameters:\n",
    "        epsilon = self.epsilon\n",
    "\n",
    "        if random.uniform(0, 1) <= epsilon:\n",
    "            chosen_action = random.choice(possible_actions)\n",
    "        else:\n",
    "            chosen_action = self.get_best_action(state)\n",
    "        \n",
    "        return chosen_action"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Try it on taxi\n",
    "\n",
    "Here we use the qlearning agent on taxi env from openai gym.\n",
    "You will need to insert a few agent functions here."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import gym\n",
    "env = gym.make(\"Taxi-v2\")\n",
    "\n",
    "n_actions = env.action_space.n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from qlearning import QLearningAgent\n",
    "\n",
    "agent = QLearningAgent(alpha=0.5, epsilon=0.25, discount=0.99,\n",
    "                       get_legal_actions = lambda s: range(n_actions))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def play_and_train(env,agent,t_max=10**4):\n",
    "    \"\"\"\n",
    "    This function should \n",
    "    - run a full game, actions given by agent's e-greedy policy\n",
    "    - train agent using agent.update(...) whenever it is possible\n",
    "    - return total reward\n",
    "    \"\"\"\n",
    "    total_reward = 0.0\n",
    "    s = env.reset()\n",
    "    \n",
    "    for t in range(t_max):\n",
    "        # get agent to pick action given state s.\n",
    "        a = agent.get_action(s)\n",
    "        \n",
    "        next_s, r, done, _ = env.step(a)\n",
    "        \n",
    "        # train (update) agent for state s\n",
    "        agent.update(action=a, next_state=next_s, reward=r, state=s)\n",
    "        \n",
    "        s = next_s\n",
    "        total_reward +=r\n",
    "        if done: break\n",
    "        \n",
    "    return total_reward\n",
    "    \n",
    "        \n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "eps = 2.9191091959171894e-05 mean reward = 8.8\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAD8CAYAAAB6paOMAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAIABJREFUeJzt3XecVPW9//HXZ2cbu5TdpbMFlqJI\nLyugqBFFQYwhGo0ao16NkoLpyY3Em2sa+ZnE9OINMaZ4TUhyYxKuciWaqImxIcYCKrgKCgjSlF62\nfH9/zJnZmdmpO7OFOe/n47EPZr7nO+d89+zw/ZxvOd9jzjlERMS/Crq7ACIi0r0UCEREfE6BQETE\n5xQIRER8ToFARMTnFAhERHxOgUBExOcUCEREfE6BQETE5wq7uwDpGDBggBsxYkR3F0NE5LiyZs2a\nXc65ganyHReBYMSIETz11FPdXQwRkeOKmb2WTj51DYmI+JwCgYiIzykQiIj4nAKBiIjPKRCIiPic\nAoGIiM8pEIiI+JwCQRfae7iJo80t3Xb8g0ebOXysa46/+8BRWlqTPwZ15/6j7Nh3JO19HjjazIGj\nzSnzHWlq4e1Dx/jbS2+yadfBtPZ9+FgLB9PYd3fZuf9o0u0H0zw36Xhz3xFiH2Gby/3Ham117Ngf\n/B445zL6TgDhz3am5pbWlH+DeP764pts3nMIgLcPHePwsRZ+u/p1tr59uF3eeOd9297DvPzm/o4V\nOgPHxQ1lx5s1r+1h5/5jzJ8wJCp98pf+wuTaCv68eHZa+2lqaWX56s08vH4HX1o4geqKXmzcdZAh\nfUvZ8OZ+3nj7MOdNHArA5j2H6FdWRN/Sonb7+dhv/kV1ZS9ue+gVyosDrPvy/PB/nkF9SlOW487H\nNnH/izv41bUzADh0rJkd+44yYkA5La2OHz/YSFXvYur7l/PCtn0ECowv/e8LXHdaPZc01HLikD5R\n+zt0rJn33PYYL27bB8Bvrp/FxJp+7Nof3Gek+9Zuo7nVcdroAZz//UfY+vZhPnPuCVw4rYbqil4A\nvHXwGEebWzlwtJlhFaV88M41/OPlXQBUV/TinzeexYGjzWzfe4TRg3oDsGnXQX7x6CZqKntxxczh\nnPSf9zH3pMHcfnUDAGu37uXGu5/jrutm0a9X8Jxe+4vVzBs/mEtPruN/1myhtrIXg/qWUu+VuXHH\nAZ7cuIcLp1bzxt7DjBrYm/vWbmfT7oNMra3gW3/ZwK8+MIOSwgJ+9shGaqvK6F9ezK4DRzltzEB+\n/shG5o4bTE1lL9462ERFeRGv7TpEi3O8+0f/BKBvaSHLF53CkH6lVJUXh8/TWd96iDf3HeXyGXUM\n71/GhGH9OHC0Oeo7uH77fgoDxqiBvdl/pIlrf7Ga2qoyvnD+OCrLi1m7dS8FZiz4/j+4Yc5oKsuL\nuXb2CJb9/VX+3/+9BMA1s0dw3ekjKS0sYNveI1SUFbHk7ue5eHoNk2oqGNC7mN4lhfzuqc04B5fN\nqIv6e77wxj7GDulDQYGF037y91f5+n0v8afFs/nt6s385snXuXh6DVedMpwCMyZU92Pv4Sb2HW6i\ntqqMD925hn1Hmrj+jJEUBwq44vYn+PZ7J1NWHKCwoIC54wYD8Jd121n7xj4+cfYYHLDhzf2cNLRv\n+Lh7DzXxy8c28aF3jKK4MHhN/PruQ1SUF/H67kOMH9aX57fuZdzQvnz5nhf41WOvsXjOKD51zom8\nvudQ+O8e0tzSyt1Pb2XtG3vpVRRg3LC+fHz5MwBUlhXx1qGmqPxfefcEZo/qz8iBvXl99yHO+OaD\nlBcH+PX1s5hcW8HR5hbe+5PHKC0McN8nziAQcc5yzbrr4fVmNh/4HhAAbnfO3ZIob0NDgzue7iwe\nceO9ALz6tQX885VdnDZ6AP94eRdX3fEkAJtuOR8IVjYPb9hJaVGAr9zzAq9+bQGHm1p4cds+GkZU\n8cUV6/jFo5sAqKsq4/5PncGJ/3EfCyYOYeXz28P72rH/CDOW/pX6AeUsOmMkN/3xeVodXDK9hm9e\nMjlcnnhWfeIM5n337yy9cAJXzBzOo427eN/tT/D4krMZ3LeEnz2yka/e+2L4WM456pesBKC4sIDa\nyl68sjP5VXddVRm7Dhzl7o+cyomD+4Q/H6m6ohdb3z7Mv506Ivw7p1JXVcYl02v41v0bwmkTqvuy\nduu+qHwFBqHGSf/yYn56dQMX/fjR8PbrTqvn9kc2JjzOdy+dwpyxg5j8pb/E3f7ojWex9N4Xuff5\nbVHpgQJr1yq6/vR6fvqP9sca1KeEHRlecc4aWcXLbx5g98FjCfPceslkPv/H57m0oZY7H3+NPiWF\n/PmG2Zz1rYfTOsbM+iqe2Lgn7TL1KgpwuKmt1XnFzDoOHWvh/IlDWbVuO79fswWAk0dUsvdwExve\nPJByn9efXs/9L7zJpt2HEn4/Iv/GU+sqGD+sL//9+OsAjB/Wl3VvBL8Tt1w0kdKiAPuPNvOFP60N\nf760qICPnjWGb65a327fQ/uVsm1vW6tjal0F/3r9bQBuvmAcV8wczoPrd/DBO9ek/F0yETqX37l0\nMhdOrenQPsxsjXOuIWW+7ggEZhYANgDnAFuA1cDlzrkX4uU/XgPB5xeM5WsrX2LOiQN5cP3O8PZQ\nIAjlKwoYTS2O9V+dz+K7/sUDL77J0184h8uWPRb1H+Whz5zJmbc+RP/y4vB//k23nM/Em1ex32u2\nX3ZyLctXb446VrJAMLm2gmc3v015cYCvXjiB3zyxmSc37aF+QDkbY7pVhvYr5dxxg/nlY2ndtS4+\nVVhgNKfoFuwMxYECjrW0dvlxY2Xz+4cuiCI9/YVzolp/mUg3EHTXGMEMoNE596pz7hiwHFjYTWXJ\nqcjA+sqOYEX6+KvJr6giY/G6N/YCwX7uY83RX+otbwW/IJUxX4r9EX23pUWBjMo7vKoMgIPHWvjk\nb5/lyU3BssYGAYBte490exD45NwTuHLW8LTzNwyvDL++9ZLJlBcnPj/nTxqaUVkK02iqFwfi/xe7\naFo1d3/k1HbpT3/hHCZW9wu//9z8sVHbY9/H+uH7pnLXdTM5dVT/cNrs0f0Z0reU/3r/tHb5b7+q\nrY44bfQAnr35XF752oKoz8eK9zudPmYAHzxjJKVFBfzkyums/+p8PviOkVw+ozacp29psCf6oqnV\nAJxxwkDu+8Tp3HrJZBZMDHZhDegd/G7XVvVi0y3ns3jOKN45aSilRdHHvGnBSfzy2hms/NjpvHPS\nUJZeOIFnbj6Hez92Gh88YyQ1lcFuw99cP4ufXd3A9y6bwqA+JVH7GOd1E509dhB9SuL3kt8wZ3S4\nC/K3i2Zx8wXjKC8O8Nl5J7bLO7O+ihU3zOalr8zn2++dTE1lL2aMqApvXzxnFLdd0fY3+O6lU5hY\n3Y8ZI6q4cGo13710Cv+88SzeNzPYnXbRtGqe+o+5HQ4CmeiuMYJqYHPE+y3AzG4qS049+sru8OsW\nr4aPN2j6+u5D4detXr7YxllTS3TC7oPBroPGHYmb060xO3ksojzxFAY6r98xl+77xOkYFh5vuPPx\nxAGptqoXm/cEg+adH5jJSf95HwAj+pfx5E1zue6XT3HgaDPPb90b/syU2grOmzCEe59r695ZvmgW\nly17PO4xLp9Rx80XjGPsF+5LWI5Qy+/vG3aGuwUheNX37fdO4Q3vyi/UIgTo16uI//7ATCZ/OdgN\nde74wXz9vmD//JWzhvPhM0eF30OwUl2y4CROXvoAHz97DO+cNAyAyrJiFnz/H5w7bjDLIir7r7x7\nAnVVZVztlSfUnw7w3cumhMdDfn39rHD6G28fZtnfX2VgnxI+/I5RFBQYjTsOMPfbD1MUMF5euiCc\n98bzxmIW/E4tOe8kAE4fM5Dlqzfzi387OTw28O1Lp4Q/M3ZIX6bWVbDy+e0UFhTw9BfOCV8EfXZe\nW+C7++kt/PmZN/jB+6ZGjYX98H1tlev4Yf0YP6wfnzr3BDbuOsjYIW1jAgunVLPhzf3UVZWx4c39\nTKqpiPp7hVrOv75+JqeMDAZCM+MzEZX+zJH9uWZ2Pc45AgXGuyYPY8Wzb7BgwlDq+pe1/V2m1XDR\ntBqcc6x7Yx8TIoL7PR89jX9tfpt3T63m3V5QjPS1CyfytQsntkvvTD12sNjMFgGLAOrq6lLk7hma\nWlq54vYnwu9bvQDQEqf7bdW67W35XOjftnwOOBrTIoit5OOJDTqX/zR+RRay/0j2M0EumlbNWWMH\nsfTeF6P6UtMV2W8+f/wQ3j21mg/9d3R/a3VFL/pE/OcvLw5wMMEMqKF92wJB5JVkr+IA5SWF/GbR\nLJxz/H7NFt45aShlxcH/Bg+t3xH8fL9SHltyNs45Lp5ew9ljB/Hhu54O7+c/zj+J604fCQTHYX6/\nZgtjBvXm5QQButq7Oo3V37v6nVxTwVOvvQUExxX6lbX9ngPK265iP3NusEL66Fmj+cHfGnnkc3Oo\nruiFmfHkTWfTPyLvuGF9w4Eo0pWzhoe/lyGhfvf+Ca48h1X04ovvGh+VFrpyj+0CCQWBSAsmDmXB\nxOStraH9gpMWrjxleMIr4FDlmo6SwkBUEAg5YXDwQiI2CAD85MrpPP7qbk4dNSDl/s2MD71jFED4\n30T5IoMAwITqfu3Sult3BYKtQG3E+xovLcw5twxYBsExgq4rWmaaW1rZ8tbh8AyaSBt2BKd9xWsR\nxBvga3UQ+d+oKaa/s7kl9WlY8cwbaZS6zb7DTakzEX2V3W5bZRnvnDSMpd6gcqY+N38sE6r78Wjj\nbpYsGEtFWVtF8OMrplFVXhwVBABuec8kPvqbf8XdX2RFGlkxhSr8UPp7G2qjPhfqVrOIPLdeMrnd\n368gYp/fuHgS88YPYeeBoyy5+3neM62GPzy9JSr/qIG9ufMDMygvKeSiHz9K6OMlhQF+98FTOGFw\nb6Z8+f64v0vfXm1l7uV1a33qnBP42NljKIrookln9le4/N5VeWjWy80XjOOm80+KW4knEroiX3Je\n8q6qdJUVF/Ly0vPS6m7rLPPGD2He+CGpM+ah7hojWA2MMbN6MysGLgNWdFNZ0vLfj7/GqM+vbFcp\n3PqXDZx560PhucKRYmevRNpzsP0Mkdgr/tgxgn1pXL3vT3Oud2i6XLozQuaeNDjhtrLi6Ao0kZsW\nnBQ3vTBQwMIp1Xz94klRQQCCXRyzRrbvr75g8jAevfGsuPsL9WFProm+6uqVYvwkHAhiKsRAgfHA\np87gomnBZnxkXWVmzB03OJxmBrddMY3fLpoVtY/Txwykqqz9le6M+ioqyor5w4dP4QeXTw2n33zB\nOE4eUYmZhfvWQ38zM4sKAh3x8GfP5E8fmd3h/RUUGJtuOZ9FZyS+Gs5UUaAgo2AkudMtLQLnXLOZ\n3QCsIjh99A7n3LruKEu6vvy/L9DS6mhqaSVQ0FahPLEx2Ae/Y/8RHn1lV9r723Ow/ZW4i6j3nXPt\nWgS7DmR+Q0siRQVG4kmH7RUk+Q9almQANtL1Z4xk6crMWg3JxjBC/dmxRg4sZ/VNc6OupqHtijqR\nUKAoiFMnjh7UJxwoCuJctYZiuEH43o5YNZW9mDd+cNyuhOnDq5geMQZ+zex6rpldD8C9Hzs9fM9F\nrgzvX546k/hGt40ROOdWAu0nlPdQoX7+2Pow1JRtbnF87g/Pp72/2EoewOHCV0TOte9/bWrO3dS4\nwkABkP5dxsla7CWF8a+kcyHZTTTlJYX8+vqZ/OjBRv7ZuJsxg3ozra6SD75jFL3jzAJJFbCKvKBj\nCdo2755Sza+feJ3Zo9v3IYf+UslOQWGggJ9cmXImXzu1VWXUVpWlzijSQT12sLinCXUJxY7Xhiqq\nVMspxIo38Pvwhp1xcraJFzze9cNHMjpuSFGGs4UStQhGDiyPulszXd+4eBKtrY6JNckHzVL1GZ86\nagB3elNaP3nOCUkHJVN1f4T+IokOOaO+Ku4ALMAJg4N3LJ8cMV1Q5HihQJClQq8fId7MoGTiZf/4\n8mfCc5bjBYpjcQaLn9uyt11aOgrj9X8kkehq/2+fPjP8OpNb4C+eVhO3iyVWLm6rP3/i0HZ3/cZT\nV1XG6WMG8Im5YzI+xvThVeFZPCLHGy06l6FELYJ0ZvS07cOlnAr69qH2YwjNObxrMtP7B9Kpj392\ndfrdHukEAUgvYKWKwd+/fCrrvzo/5X6KAgXc+YGZTB/esav6msoyDXbKcUmBIEOO6Fon1HWRqlsn\nah8u9T0BR5ra99/H6xrqqHjdJANj7ryMlGywOGTM4D5UelM3czUNMBctgkCBhccxRKQ9BYIMxQ4F\nhO4kTnehtOA+HK0J6vTQOiPxxhxi7zTORryK+m+ffkfClVHTvdANBYwfe7fSJ1piIV3dOa9cxC8U\nCDIUuZbQm/uORK20mK7WNFoE8cYccrmgVmGcCrpPaVHCVoGZhZdw/n7EfPd4+SA4oweC68lkI5MW\ngUKGSMdosDhDkdVz7A1f6WpNY4wg3uqFuewaip2XPnt08KatRFfgBQZ/+NCpbN93JLy8QDyh+FJR\nVsSKG2YzZlCfrB7Ik85YxrWn1XPfuu1MH1GZMq+ItKdAkKHI+jvdAc94+0jVyxNv8DmXgSDWXdcF\n74RNdAVeYME1cPqVFbE3zkB2ZD4IjkGMHxZc1yXVjVzJpNMiSDatU0RSU9dQpiLq50AHZ4i0ONfu\nkXTt8sQZRGhq7vgYwSlxlmkIiaxrE83SicyTbCJPKBAkqsDnnDgwo+6eTKe5ikjm1CLIUOSsoY7W\nUR3tGspmjKB3afSfeuGUYfzZW6Du3VPalsINJOiKiZwWmaxyDm0qSpDn59fMSKu84fJosFik0+ly\nK0NR9XcHL9BdKwlnDYXEmzXUnORDoQXREomtT0Pr1kNwJc+QxGMEbenptAhip9l2lGYNiXQ+BYIM\n5SAOpNciiDdGkKRrKN5D6yPF3gcQWsky9nVsvtDTpCyN7iNo6y7L1ZMK1SIQ6XwKBBmKenhMksou\nWf3V6lzKu2Hj30eQuEXQpzR5L1/swHai+f2xV+DxBsST/W6nerOP4i361hFqEYh0PgWCDEVW4Mm6\nP5LdiZvOfQT//ofn2qU1JekaSjUzJ1mLICpfbCCI82skW0bh5gvG87dPvyPpXcrpCHV1qUUg0vkU\nCDIUWfknbREkqcDSWWsonmRdQ6mWUIgdA0539dFMK+KiQAEjB/bO6DPxfOM9k3j2P8/V2j0iXUCB\nIFMu7st2kk0tbUmjayieZF1Dia7wQw7FPN83WQUfOSe/uyriwkBB1CMnRaTzKBBkKGqwOEltnqyi\nTadrKJ6k00dT7O/1mEdppnulr54ZkfynQJChqDGCJHVvsgvp1laX8fMLIPlS1/HuO4j0rinDot6n\nOwhbEPHENBHJT7qhLEOhMYIvrljH9r1HEuZLVs06l/o+gnjidQ0VGHzqnBNSPiHtw+8Yxf+s2cKr\nOw8CUBxIb9mHdJafFpHjmwJBhkJXxpksOx2rNY0lJuKJd9X/6XNPZPGc0Sz7+ytJP2tm3PmBmfzy\n0U0M6lNCXf/0noGrQCCS/xQIMpRu336yQdYzb32IIX1LO3T84sKCqFVPQ4eJDRJV5cXsOXgsKq26\nohefX3BS+P3QfqXsO5x4ATnQGIGIH2Q1RmBml5jZOjNrNbOGmG1LzKzRzNab2byI9PleWqOZ3ZjN\n8btDrvrKOzJYDFAScyOYeZ1Q9f3Lo9LfN6Mu5b4e+dxZPPfFeUnzhKbB5mrJCBHpebIdLF4LXAT8\nPTLRzMYBlwHjgfnAj80sYGYB4EfAecA44HIvb4+2adfBjD+Tquuno0swJFqf/7yJQ6OeLpbug+FT\nzR5Sg0Ak/2XVNeScexHidoMsBJY7544CG82sEQgtO9nonHvV+9xyL+8L2ZSjs51560Ph193dIngr\n5lkAkad+cm1F+HVHl8huJ8luJlb3y80xRKRbddYYQTXweMT7LV4awOaY9JmdVIZOkW4XSaobsTr6\ndLN2x4l5/6fFsxnYp4Q/Pr0lR/uPP310w1fP0/IPInkiZSAwsweAIXE23eSc+3PuixQ+7iJgEUBd\nXer+7q6SqxZBRx/dGCs23kzxWgUdfXpaqv2HpLqTWUSOHykDgXNubgf2uxWojXhf46WRJD32uMuA\nZQANDQ1ZV79Hmlp49JVdnDV2cNJ8tz30CpNq+jF79IC423M1ZNqU6lmVEQos8zGFnHUNiUje66zL\nuhXAZWZWYmb1wBjgSWA1MMbM6s2smOCA8opOKkOUm/+8jmt/8RTr3tibNN/X73uJK25/IuF25xzf\nXPVSTsv282tOTro92Vx+S9CJr24bEUlXVmMEZnYh8ANgIHCvmT3jnJvnnFtnZr8jOAjcDCx2zrV4\nn7kBWAUEgDucc+uy+g3StHF3cObP/iPNWe2n1cGPHkx+8xaknjWUiWAgiL+/RDFCN4KJSLqynTX0\nR+CPCbYtBZbGSV8JrMzmuN0rvQo+ozCQInNBAZDhkIJaBCKSLo34ZSjdC/1cLtKWtGsowbZcDRaL\nSP5TIMhQuvV7Jl1DqaakdqSbR4PFIpIu/wSCHF2hp90iyM3hgORLWifalOCRxJkfOze7EZEezD+B\nIEf+b+22tPJlcudwqqzJ+vs1WCwi2fJPIMhRvfjdB15OK19XjREkosFiEUmXfwJBFy+emUkgSJU3\n+X0EmX9GRCSSfwJBF8tk2eYTh/RJuj3ysZI/vSpqtW/NGhKRrPknEHRxvZhJi6C2qoxlV05PuD2y\nm2dGfVXUtlQX/mOH9OGfN56VfmFExHf884SyDnYNtXbwwQGZLjOdarXSkNgL/VSfGj2oN9UVvTIq\nC8D/3nAam986xNqtwSU5cnmntIj0LP5pEXgybRi0dLAC7Kxqs13ff4IAkm3FPbGmHwsmDk3Z4hCR\n45/vAkGm1WNLB1sEmdbD6da3mVbM6bY0RMS/fBcIMtXRJ4l1ltgWQWdX85VlxQD061XUyUcSke7i\nnzECT6YVZ0efLdxZYi/wB/UpSZ4/y+P926kjKC8p5L0Ntakzi8hxyXeBIFMd7RrKVLo9OJHPH/jZ\n1Q2cNXZQJ5UoqDBQwOUzes4T4kQk9xQIIsQbYM1k1pBZbu8ojidy1tDZJyV+2loP69ESkR7M92ME\nj76yixE33sv67fvjVp6ZzBoqzOImrnRbBJneMayxYhFJxfeB4P+e3w7AExt3x51RlEmLIJv1fRI9\ncrJdvjQPkcmdzSLib74PBCHOxZ8hlFmLoPNPZ6bTQdUgEJFUfB8IIuvVuF1DGbQIunJ5n0s1i0dE\ncsQ3g8Wpukqcc3FbBJkMuvbtVcS+I81J81wzewQ//+em9hsyCCIvfWU+xSmePKPBYhFJl1oE3r9f\n/N8X+MnDr7bbnkmL4K7rZqbMs3jO6LT3l0hpUSDl6qKnjhoAwJWnjMj6eCKS37IKBGb2TTN7ycye\nM7M/mllFxLYlZtZoZuvNbF5E+nwvrdHMbszm+BmVNcEld2Sf+3ce2NBueyZjBMP7l6fMk2jWT7zU\na2fXp33sWEP6lbLplvOZPryyw/sQEX/ItkVwPzDBOTcJ2AAsATCzccBlwHhgPvBjMwuYWQD4EXAe\nMA643Mvb6To6i6ajq48mkmoc4YwTBoZfXz5D4wAi0vmyCgTOub8450Kd4o8DNd7rhcBy59xR59xG\noBGY4f00Oudedc4dA5Z7eXusjq4+moiZcc9HT+PmC8a1S4fom9p0D4CIdIVcjhFcC/yf97oa2Byx\nbYuXlii906U7Tz9Wc0vuWwQTqvtxTUy3T/odRiIiuZVy1pCZPQAMibPpJufcn708NwHNwF25KpiZ\nLQIWAdTVZb/WTaKuoVRX3bleayiTm87UIhCRrpAyEDjn5ibbbmb/BrwTONu19WtsBSI7uGu8NJKk\nxx53GbAMoKGhIWe1cbo3ZP1hzRbeM72G5pyPESQ/vnPw8GfPpMCMppbWcPrPrzmZfYebcloWERHI\nftbQfODfgXc55w5FbFoBXGZmJWZWD4wBngRWA2PMrN7MigkOKK/IpgyZil1YLlGX0ad//ywAzRGV\ncS4kigN9SoMxeWi/Uob3L6e2qiwqaM05cRALp3RJL5qI+Ey2N5T9ECgB7vcqrcedcx9yzq0zs98B\nLxDsMlrsnGsBMLMbgFVAALjDObcuyzJ0qlx3DSVqEUytq+QHl0/l7JPalpVWz5CIdIWsAoFzLuHd\nUc65pcDSOOkrgZXZHDcbma7V05VdQxdMHhb1XmMEItIVfHdncbuuoRwPFg/uG3xi2J8Wz+a/3j+9\n3fZM1iPq6EwnEZFM+GatoY5wzkUN2Kbj4c/OobnV0bukMHpY3JNJi0QtAhHpCr4LBLEVcbK6ttVl\n3iIoLQp0oFQiIt3Hd11DsZJddbe0upyPEWRCLQIR6QoKBElq21bncjJr6KKpmvYpIj2X77qGMtHS\nmvkYQaxXvraAAoO7/xX3vjkRkW6nQJBESw5aBNk8x1hEpCuoayjJttaIMYKSQt+fKhHJU6rdkmhu\nbWsRLLuqoZtLIyLSORQIkjQJWnMwRiAi0tP5JhB05PkyuRgjEBHp6XwTCBJJtoxDuvcR9OtVxIj+\nZUnzXDK9RgPHItIj+WbWUOh2gdi1hpJpbW27szjZ5+7+yKmMGtg76b6+eclkvnnJ5LSPLSLSVXzT\nIuho19ChYy3Bz+e4PCIiPYVvWgQhsRV6smUcXty2j/96+BUAZo8awMXTa+hfXsy2vUdY8ewbbftU\nlBCR45hvAkFb11D6n1m7dW/4dXFhAbd6XTsPrt8RFQhERI5nvu8aSjZ8++OHXknzM2oSiMjxyzeB\nIMSR2YNp4oldqE5dQyJyPPNdIIjVkaeAddUk0AG9g087++Q5J3TREUXEj3wzRhCWg6v32FZEZzUI\nSosCbLrl/E7au4hIkO9aBOrFERGJ5rtAkAux3UkaIxCR41lWgcDMvmJmz5nZM2b2FzMb5qWbmX3f\nzBq97dMiPnO1mb3s/Vyd7S+QqdhKO3bwOB3tu4YUCUTk+JVti+CbzrlJzrkpwD3Af3rp5wFjvJ9F\nwG0AZlYF3AzMBGYAN5tZZZZlyEhspd2Rq3mtGCQi+SSrQOCc2xfxtpy2LviFwK9c0ONAhZkNBeYB\n9zvn9jjn3gLuB+ZnU4ZMffZ/BGQpAAAOUElEQVT3z0W979DiojGRoLCDi8l97cKJ/OraGR36rIhI\nrmQ9a8jMlgJXAXuBOV5yNbA5ItsWLy1Rerz9LiLYmqCuri7bYoZt33ck6n06i9B9dt6JcdNHDizn\nfTPqUi44l8j7Zubu9xIR6aiULQIze8DM1sb5WQjgnLvJOVcL3AXckKuCOeeWOecanHMNAwcOzNVu\noxxrbuXgsea42yrKisKvp9VF916FBosH9i7hutNHtrvBTETkeJKyReCcm5vmvu4CVhIcA9gK1EZs\nq/HStgJnxqQ/lOb+sxLvuv9dP3yEl7bvj5s/srsnUT2vIWIRyQfZzhoaE/F2IfCS93oFcJU3e2gW\nsNc5tw1YBZxrZpXeIPG5Xlq3SBQEAAoL2k5NbBwIBwZFAhHJA9mOEdxiZicCrcBrwIe89JXAAqAR\nOARcA+Cc22NmXwFWe/m+7Jzbk2UZ0pJp501hILJFEP1pdQSJSD7JKhA4596TIN0BixNsuwO4I5vj\ndkSmF+9FgbYWQaJJQbp/QETyge4sTiDZGEGohaA7ikUkH/gmEGTanRN6RGW8T4cfcpNViUREegbf\nrD4aWWkfONpMU3Nr0vxb3z4cfh3bNRQeK1aTQETygG8CQaTTvv433j7UlHb+doPFGi0WkTzim66h\nSJkEAUjcraT2gIjkA18Ggky1bwFosFhE8ocCQRoKEnQNKQ6ISD5QIOiAcFhQk0BE8oACQRoS3Ucg\nIpIPFAjSENs1FKL2gIjkAwWCNMTGgRMH92HkgHJuWnBS9xRIRCSHfHkfQaZiH1bfqzjA3z5zZvcU\nRkQkx9QiSEMHn0QpInJcUCBIg8aGRSSfKRCkRZFARPKXAkEa1CIQkXymQJCGRNNHRUTygQJBGhQG\nRCSfKRCkQQ0CEclnCgRpUNeQiOQz3wQCPU1MRCS+nAQCM/u0mTkzG+C9NzP7vpk1mtlzZjYtIu/V\nZvay93N1Lo7f2dQgEJF8lvUSE2ZWC5wLvB6RfB4wxvuZCdwGzDSzKuBmoIHgmm1rzGyFc+6tbMuR\nRjm75bMiIj1dLloE3wH+nejFOBcCv3JBjwMVZjYUmAfc75zb41X+9wPzc1CGlLLpGtISEyKSz7IK\nBGa2ENjqnHs2ZlM1sDni/RYvLVF6vH0vMrOnzOypnTt3ZlPMrMUuOicikk9Sdg2Z2QPAkDibbgI+\nT7BbKOecc8uAZQANDQ3dOtKrniERyWcpA4Fzbm68dDObCNQDz3p96DXA02Y2A9gK1EZkr/HStgJn\nxqQ/1IFydykFAhHJZx3uGnLOPe+cG+ScG+GcG0Gwm2eac247sAK4yps9NAvY65zbBqwCzjWzSjOr\nJNiaWJX9r9G51DUkIvmssx5MsxJYADQCh4BrAJxze8zsK8BqL9+XnXN7OqkMOaMWgYjks5wFAq9V\nEHrtgMUJ8t0B3JGr43YF3VksIvnMN3cWZ0NhQETymQJBGtQgEJF8pkAQYUZ9Vdx0DRaLSD5TIIgw\nf3y82yXAdJZEJI/5porL5o40tQdEJJ/5JhBkQ4vOiUg+800gyKYq16JzIpLPfBMIsusaUiQQkfzl\nm0CQjkTBQj1DIpLPfBMIVJeLiMTnm0CQTdeQlpgQkXzmm0CQDcUBEclnvgkE2dTligMiks98EwjU\nNSQiEp9vAkE6Ej3gXnFARPKZAkGMc8cNBuC00QPCabqzWETyWWc9oey4ddv7p3O0uYWSwgCjPr+y\nu4sjItLpFAhiBAqMsmKdFhHxD3UNiYj4nAKBiIjP5XUg2H3gKBO/uIrlT77eoc/3LlEXkYjkv6wC\ngZl90cy2mtkz3s+CiG1LzKzRzNab2byI9PleWqOZ3ZjN8VMpMGP/kWaONLV06PMrP3Y6P7h8ao5L\nJSLSs+Tikvc7zrlbIxPMbBxwGTAeGAY8YGYneJt/BJwDbAFWm9kK59wLOShHO4FAcNpnc2vHbier\n619GXf+yXBZJRKTH6ay+j4XAcufcUWCjmTUCM7xtjc65VwHMbLmXt1MCQaH3RJmWDgYCERE/yMUY\nwQ1m9pyZ3WFmlV5aNbA5Is8WLy1RejtmtsjMnjKzp3bu3NmhggUKsmsRiIj4QcpAYGYPmNnaOD8L\ngduAUcAUYBvwrVwVzDm3zDnX4JxrGDhwYIf2UVgQ/PXUIhARSSxl15Bzbm46OzKznwL3eG+3ArUR\nm2u8NJKk51zoWcNqEYiIJJbtrKGhEW8vBNZ6r1cAl5lZiZnVA2OAJ4HVwBgzqzezYoIDyiuyKUOK\n8lFYYLS0tqaVP8GacyIieS3bweJvmNkUgqs8bwI+COCcW2dmvyM4CNwMLHbOtQCY2Q3AKiAA3OGc\nW5dlGZIKFBjNrU6VvIhIAlkFAufclUm2LQWWxklfCXTZam6FBUZLi+vwvQQiIvkur+8shrYWwduH\nmrq7KCIiPVLer6FQGCjgd09t5tCx1C2Cwf1Ku6BEIiI9S94HgkCBpRUEfn7NyZx5QsemqYqIHM/y\nPhCE7i5OZc6Jgzq5JCIiPZMvxghSuWR6TReURESkZ/J9i6Bx6XlpBQsRkXyV94EgVSVfGMj7RpGI\nSFJ5XwuG1hsSEZH48r6WjGwRfOfSyd1YEhGRninvA0FhoC0QlBYGurEkIiI9U94HgsgWQWmxAoGI\nSKy8DwSRs4ZKCvP+1xURyVje14yRzyIo1gwhEZF28r5mfP/M4eHXRV4gUMtARKRN3teIk2r6hV8X\newGguqJXdxVHRKTHyftAMLBPSfh1qEWgR1eKiLTJ+0BQHNENFOoS0sPsRUTa5H0giLyzOHRPQVNL\nes8wFhHxg7wPBEURN5SFgoJaBCIibfI+EJi1BYJQUNAYgYhIm7wPBJFKvCUmJlT37eaSiIj0HFkH\nAjP7qJm9ZGbrzOwbEelLzKzRzNab2byI9PleWqOZ3Zjt8TPRqzjA3R85lf96//SuPKyISI+W1fMI\nzGwOsBCY7Jw7amaDvPRxwGXAeGAY8ICZneB97EfAOcAWYLWZrXDOvZBNOTIxra6yqw4lInJcyPbB\nNB8GbnHOHQVwzu3w0hcCy730jWbWCMzwtjU6514FMLPlXt4uCwQiIhIt266hE4DTzewJM3vYzE72\n0quBzRH5tnhpidJFRKSbpGwRmNkDwJA4m27yPl8FzAJOBn5nZiNzUTAzWwQsAqirq8vFLkVEJI6U\ngcA5NzfRNjP7MHC3c84BT5pZKzAA2ArURmSt8dJIkh573GXAMoCGhgbN9xQR6STZdg39CZgD4A0G\nFwO7gBXAZWZWYmb1wBjgSWA1MMbM6s2smOCA8oosyyAiIlnIdrD4DuAOM1sLHAOu9loH68zsdwQH\ngZuBxc65FgAzuwFYBQSAO5xz67Isg4iIZCGrQOCcOwa8P8G2pcDSOOkrgZXZHFdERHLHV3cWi4hI\newoEIiI+l+0YwXHhuS+ei9O8IxGRuHwRCPqWFnV3EUREeix1DYmI+JwCgYiIzykQiIj4nC/GCOL5\n9fUzeXPfke4uhohIt/NtIDh11IDuLoKISI+griEREZ9TIBAR8TkFAhERn1MgEBHxOQUCERGfUyAQ\nEfE5BQIREZ9TIBAR8Tlzx8H6zGa2E3gti10MIPgsZdG5iKXzEU3no00+nIvhzrmBqTIdF4EgW2b2\nlHOuobvL0RPoXETT+Yim89HGT+dCXUMiIj6nQCAi4nN+CQTLursAPYjORTSdj2g6H218cy58MUYg\nIiKJ+aVFICIiCeR1IDCz+Wa23swazezG7i5PVzCzWjN70MxeMLN1ZvZxL73KzO43s5e9fyu9dDOz\n73vn6Dkzm9a9v0HumVnAzP5lZvd47+vN7Anvd/6tmRV76SXe+0Zv+4juLHdnMLMKM/sfM3vJzF40\ns1P8+t0ws096/0fWmtlvzKzUr9+NvA0EZhYAfgScB4wDLjezcd1bqi7RDHzaOTcOmAUs9n7vG4G/\nOufGAH/13kPw/IzxfhYBt3V9kTvdx4EXI95/HfiOc2408BbwAS/9A8BbXvp3vHz55nvAfc65scBk\ngufFd98NM6sGPgY0OOcmAAHgMvz63XDO5eUPcAqwKuL9EmBJd5erG87Dn4FzgPXAUC9tKLDee/0T\n4PKI/OF8+fAD1BCs3M4C7gGM4E1ChbHfE2AVcIr3utDLZ939O+TwXPQDNsb+Tn78bgDVwGagyvtb\n3wPM8+t3I29bBLT9oUO2eGm+4TVfpwJPAIOdc9u8TduBwd7rfD9P3wX+HWj13vcH3nbONXvvI3/f\n8Lnwtu/18ueLemAn8HOvq+x2MyvHh98N59xW4FbgdWAbwb/1Gnz63cjnQOBrZtYb+APwCefcvsht\nLnhZk/fTxczsncAO59ya7i5LD1EITANuc85NBQ7S1g0E+Oq7UQksJBgchwHlwPxuLVQ3yudAsBWo\njXhf46XlPTMrIhgE7nLO3e0lv2lmQ73tQ4EdXno+n6fZwLvMbBOwnGD30PeACjMr9PJE/r7hc+Ft\n7wfs7soCd7ItwBbn3BPe+/8hGBj8+N2YC2x0zu10zjUBdxP8vvjyu5HPgWA1MMabBVBMcCBoRTeX\nqdOZmQE/A150zn07YtMK4Grv9dUExw5C6Vd5M0RmAXsjugmOa865Jc65GufcCIJ//785564AHgQu\n9rLFnovQObrYy583V8fOue3AZjM70Us6G3gBH343CHYJzTKzMu//TOhc+PK70e2DFJ35AywANgCv\nADd1d3m66Hc+jWDT/jngGe9nAcH+zL8CLwMPAFVefiM4u+oV4HmCsyi6/ffohPNyJnCP93ok8CTQ\nCPweKPHSS733jd72kd1d7k44D1OAp7zvx5+ASr9+N4AvAS8Ba4E7gRK/fjd0Z7GIiM/lc9eQiIik\nQYFARMTnFAhERHxOgUBExOcUCEREfE6BQETE5xQIRER8ToFARMTn/j+dg2KVUM4zSAAAAABJRU5E\nrkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from IPython.display import clear_output\n",
    "\n",
    "rewards = []\n",
    "for i in range(1000):\n",
    "    rewards.append(play_and_train(env, agent))\n",
    "    agent.epsilon *= 0.99\n",
    "    \n",
    "    if i %100 ==0:\n",
    "        clear_output(True)\n",
    "        print('eps =', agent.epsilon, 'mean reward =', np.mean(rewards[-10:]))\n",
    "        plt.plot(rewards)\n",
    "        plt.show()\n",
    "        "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Submit to Coursera I: Preparation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "submit_rewards1 = rewards.copy()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "# Binarized state spaces\n",
    "\n",
    "Use agent to train efficiently on CartPole-v0.\n",
    "This environment has a continuous set of possible states, so you will have to group them into bins somehow.\n",
    "\n",
    "The simplest way is to use `round(x,n_digits)` (or numpy round) to round real number to a given amount of digits.\n",
    "\n",
    "The tricky part is to get the n_digits right for each state to train effectively.\n",
    "\n",
    "Note that you don't need to convert state to integers, but to __tuples__ of any kind of values."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "first state:[ 0.00382666 -0.03441743 -0.00646399 -0.03318883]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<matplotlib.image.AxesImage at 0x7f3e43821780>"
      ]
     },
     "execution_count": 87,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXgAAAD8CAYAAAB9y7/cAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAEk5JREFUeJzt3X+MnVd95/H3p3FIWKB1QmYtr3/U\naXGL0tXiZKchEahKE9EmWVSnUouSViVCkYaVggQq2m3SSi1IG6mVWtKi7Ua4TcBUlJAN0FhRWpqa\nSBV/kDAGY+yYlAEc2ZYTO5AEKGq6Dt/9Y47DxRl77syd6/Ec3i/p0X2e85znueckV5955sw5vqkq\nJEn9+YnlboAkaTwMeEnqlAEvSZ0y4CWpUwa8JHXKgJekTo0t4JNcm+SJJDNJbhvX+0iS5pZxzINP\ncg7wL8BbgEPAF4CbqurxJX8zSdKcxvUEfzkwU1XfqKp/B+4Fto7pvSRJc1g1pvuuAw4OHB8C3niq\nyhdddFFt2rRpTE2RpJXnwIEDPPPMMxnlHuMK+HklmQKmADZu3Mj09PRyNUWSzjqTk5Mj32NcQzSH\ngQ0Dx+tb2UuqaltVTVbV5MTExJiaIUk/vsYV8F8ANie5OMkrgBuBHWN6L0nSHMYyRFNVx5O8C/gM\ncA5wT1XtG8d7SZLmNrYx+Kp6CHhoXPeXJJ2eK1klqVMGvCR1yoCXpE4Z8JLUKQNekjplwEtSpwx4\nSeqUAS9JnTLgJalTBrwkdcqAl6ROGfCS1CkDXpI6ZcBLUqcMeEnqlAEvSZ0y4CWpUwa8JHVqpK/s\nS3IA+C7wInC8qiaTXAh8AtgEHADeVlXPjtZMSdJCLcUT/C9X1ZaqmmzHtwE7q2ozsLMdS5LOsHEM\n0WwFtrf97cANY3gPSdI8Rg34Av4xya4kU61sTVUdaftPAWtGfA9J0iKMNAYPvLmqDif5j8DDSb46\neLKqKknNdWH7gTAFsHHjxhGbIUk62UhP8FV1uL0eBT4NXA48nWQtQHs9eoprt1XVZFVNTkxMjNIM\nSdIcFh3wSV6V5DUn9oFfAfYCO4CbW7WbgQdGbaQkaeFGGaJZA3w6yYn7/G1V/UOSLwD3JbkFeBJ4\n2+jNlCQt1KIDvqq+AbxhjvJvAdeM0ihJ0uhcySpJnTLgJalTBrwkdcqAl6ROGfCS1CkDXpI6ZcBL\nUqcMeEnqlAEvSZ0y4CWpUwa8JHXKgJekThnwktQpA16SOmXAS1KnDHhJ6pQBL0mdMuAlqVMGvCR1\nat6AT3JPkqNJ9g6UXZjk4SRfa68XtPIk+WCSmSR7klw2zsZLkk5tmCf4jwDXnlR2G7CzqjYDO9sx\nwHXA5rZNAXctTTMlSQs1b8BX1T8D3z6peCuwve1vB24YKP9ozfo8sDrJ2qVqrCRpeIsdg19TVUfa\n/lPAmra/Djg4UO9QK3uZJFNJppNMHzt2bJHNkCSdysh/ZK2qAmoR122rqsmqmpyYmBi1GZKkkyw2\n4J8+MfTSXo+28sPAhoF661uZJOkMW2zA7wBubvs3Aw8MlL+9zaa5Anh+YChHknQGrZqvQpKPA1cB\nFyU5BPwR8MfAfUluAZ4E3taqPwRcD8wA3wfeMYY2S5KGMG/AV9VNpzh1zRx1C7h11EZJkkbnSlZJ\n6pQBL0mdMuAlqVMGvCR1yoCXpE4Z8JLUKQNekjplwEtSpwx4SeqUAS9JnTLgJalTBrwkdcqAl6RO\nGfCS1CkDXpI6ZcBLUqcMeEnqlAEvSZ2aN+CT3JPkaJK9A2XvS3I4ye62XT9w7vYkM0meSPKr42q4\nJOn0hnmC/whw7Rzld1bVlrY9BJDkEuBG4BfaNf8nyTlL1VhJ0vDmDfiq+mfg20Pebytwb1W9UFXf\nBGaAy0donyRpkUYZg39Xkj1tCOeCVrYOODhQ51Are5kkU0mmk0wfO3ZshGZIkuay2IC/C/hZYAtw\nBPizhd6gqrZV1WRVTU5MTCyyGZKkU1lUwFfV01X1YlX9APgrfjgMcxjYMFB1fSuTJJ1hiwr4JGsH\nDn8dODHDZgdwY5LzklwMbAYeG62JkqTFWDVfhSQfB64CLkpyCPgj4KokW4ACDgDvBKiqfUnuAx4H\njgO3VtWL42m6JOl05g34qrppjuK7T1P/DuCOURolSRqdK1klqVMGvCR1yoCXpE4Z8JLUKQNekjpl\nwEtSp+adJin9ONi17Z0vK/uvUx9ahpZIS8cneEnqlAEvSZ0y4CWpUwa8JHXKgJekThnwktQpA16S\nOmXAS1KnDHhJ6pQBL0mdMuAlqVPzBnySDUkeSfJ4kn1J3t3KL0zycJKvtdcLWnmSfDDJTJI9SS4b\ndyckSS83zBP8ceC9VXUJcAVwa5JLgNuAnVW1GdjZjgGuAza3bQq4a8lbLUma17wBX1VHquqLbf+7\nwH5gHbAV2N6qbQduaPtbgY/WrM8Dq5OsXfKWS5JOa0Fj8Ek2AZcCjwJrqupIO/UUsKbtrwMODlx2\nqJWdfK+pJNNJpo8dO7bAZkuS5jN0wCd5NfBJ4D1V9Z3Bc1VVQC3kjatqW1VNVtXkxMTEQi6VJA1h\nqIBPci6z4f6xqvpUK376xNBLez3ayg8DGwYuX9/KJEln0DCzaALcDeyvqg8MnNoB3Nz2bwYeGCh/\ne5tNcwXw/MBQjiTpDBnmK/veBPwO8JUku1vZ7wN/DNyX5BbgSeBt7dxDwPXADPB94B1L2mJJ0lDm\nDfiq+hyQU5y+Zo76Bdw6YrskSSNyJaskdcqAl6ROGfCS1CkDXpI6ZcBLUqcMeEnqlAEvSZ0y4CWp\nUwa8JHXKgJekThnwktQpA16SOmXAS1KnDHhJ6pQBL0mdMuAlqVMGvCR1yoCXpE4N86XbG5I8kuTx\nJPuSvLuVvy/J4SS723b9wDW3J5lJ8kSSXx1nByRJcxvmS7ePA++tqi8meQ2wK8nD7dydVfWng5WT\nXALcCPwC8J+Af0ryc1X14lI2XJJ0evM+wVfVkar6Ytv/LrAfWHeaS7YC91bVC1X1TWAGuHwpGitJ\nGt6CxuCTbAIuBR5tRe9KsifJPUkuaGXrgIMDlx3i9D8QJEljMHTAJ3k18EngPVX1HeAu4GeBLcAR\n4M8W8sZJppJMJ5k+duzYQi6VJA1hqIBPci6z4f6xqvoUQFU9XVUvVtUPgL/ih8Mwh4ENA5evb2U/\noqq2VdVkVU1OTEyM0gdJ0hyGmUUT4G5gf1V9YKB87UC1Xwf2tv0dwI1JzktyMbAZeGzpmixJGsYw\ns2jeBPwO8JUku1vZ7wM3JdkCFHAAeCdAVe1Lch/wOLMzcG51Bo0knXnzBnxVfQ7IHKceOs01dwB3\njNAuSdKIXMkqSZ0y4CWpUwa8JHXKgJekThnwktQpA16SOmXAS1KnDHhJ6pQBL0mdMuAlqVMGvCR1\nyoCXpE4Z8JLUKQNe3Uoy9DaO66XlZsBLUqeG+cIP6cfCg0emXtp/69pty9gSaWn4BC/xo+E+17G0\nEhnwktSpYb50+/wkjyX5cpJ9Sd7fyi9O8miSmSSfSPKKVn5eO55p5zeNtwuSpLkM8wT/AnB1Vb0B\n2AJcm+QK4E+AO6vqdcCzwC2t/i3As638zlZPOqudPOb+1rXbmP6QwzRa2Yb50u0CvtcOz21bAVcD\nv9XKtwPvA+4CtrZ9gPuB/50k7T7SWWnynduAH4b8+5atJdLSGWoWTZJzgF3A64C/BL4OPFdVx1uV\nQ8C6tr8OOAhQVceTPA+8FnjmVPfftWuXc4m1ovn51dloqICvqheBLUlWA58GXj/qGyeZAqYANm7c\nyJNPPjnqLaUfcSZD119QtdQmJydHvseCZtFU1XPAI8CVwOokJ35ArAcOt/3DwAaAdv6ngG/Nca9t\nVTVZVZMTExOLbL4k6VSGmUUz0Z7cSfJK4C3AfmaD/jdatZuBB9r+jnZMO/9Zx98l6cwbZohmLbC9\njcP/BHBfVT2Y5HHg3iT/C/gScHerfzfwN0lmgG8DN46h3ZKkeQwzi2YPcOkc5d8ALp+j/N+A31yS\n1kmSFs2VrJLUKQNekjplwEtSp/zngtUtJ2/px51P8JLUKQNekjplwEtSpwx4SeqUAS9JnTLgJalT\nBrwkdcqAl6ROGfCS1CkDXpI6ZcBLUqcMeEnqlAEvSZ0y4CWpU8N86fb5SR5L8uUk+5K8v5V/JMk3\nk+xu25ZWniQfTDKTZE+Sy8bdCUnSyw3z78G/AFxdVd9Lci7wuSR/3879j6q6/6T61wGb2/ZG4K72\nKkk6g+Z9gq9Z32uH57btdN+ksBX4aLvu88DqJGtHb6okaSGGGoNPck6S3cBR4OGqerSduqMNw9yZ\n5LxWtg44OHD5oVYmSTqDhgr4qnqxqrYA64HLk/xn4Hbg9cAvAhcCv7eQN04ylWQ6yfSxY8cW2GxJ\n0nwWNIumqp4DHgGuraojbRjmBeDDwOWt2mFgw8Bl61vZyffaVlWTVTU5MTGxuNZLkk5pmFk0E0lW\nt/1XAm8BvnpiXD1JgBuAve2SHcDb22yaK4Dnq+rIWFovSTqlYWbRrAW2JzmH2R8I91XVg0k+m2QC\nCLAb+O+t/kPA9cAM8H3gHUvfbEnSfOYN+KraA1w6R/nVp6hfwK2jN02SNApXskpSpwx4SeqUAS9J\nnTLgJalTBrwkdcqAl6ROGfCS1CkDXpI6ZcBLUqcMeEnqlAEvSZ0y4CWpUwa8JHXKgJekThnwktQp\nA16SOmXAS1KnDHhJ6pQBL0mdGjrgk5yT5EtJHmzHFyd5NMlMkk8keUUrP68dz7Tzm8bTdEnS6Szk\nCf7dwP6B4z8B7qyq1wHPAre08luAZ1v5na2eJOkMGyrgk6wH/hvw1+04wNXA/a3KduCGtr+1HdPO\nX9PqS5LOoFVD1vtz4H8Cr2nHrwWeq6rj7fgQsK7trwMOAlTV8STPt/rPDN4wyRQw1Q5fSLJ3UT04\n+13ESX3vRK/9gn77Zr9Wlp9OMlVV2xZ7g3kDPslbgaNVtSvJVYt9o5O1Rm9r7zFdVZNLde+zSa99\n67Vf0G/f7NfKk2SalpOLMcwT/JuAX0tyPXA+8JPAXwCrk6xqT/HrgcOt/mFgA3AoySrgp4BvLbaB\nkqTFmXcMvqpur6r1VbUJuBH4bFX9NvAI8But2s3AA21/Rzumnf9sVdWStlqSNK9R5sH/HvC7SWaY\nHWO/u5XfDby2lf8ucNsQ91r0ryArQK9967Vf0G/f7NfKM1Lf4sO1JPXJlayS1KllD/gk1yZ5oq18\nHWY456yS5J4kRweneSa5MMnDSb7WXi9o5UnywdbXPUkuW76Wn16SDUkeSfJ4kn1J3t3KV3Tfkpyf\n5LEkX279en8r72Jldq8rzpMcSPKVJLvbzJIV/1kESLI6yf1Jvppkf5Irl7JfyxrwSc4B/hK4DrgE\nuCnJJcvZpkX4CHDtSWW3ATurajOwkx/+HeI6YHPbpoC7zlAbF+M48N6qugS4Ari1/b9Z6X17Abi6\nqt4AbAGuTXIF/azM7nnF+S9X1ZaBKZEr/bMIszMS/6GqXg+8gdn/d0vXr6patg24EvjMwPHtwO3L\n2aZF9mMTsHfg+AlgbdtfCzzR9j8E3DRXvbN9Y3aW1Ft66hvwH4AvAm9kdqHMqlb+0ucS+AxwZdtf\n1epludt+iv6sb4FwNfAgkB761dp4ALjopLIV/Vlkdgr5N0/+776U/VruIZqXVr02gytiV7I1VXWk\n7T8FrGn7K7K/7df3S4FH6aBvbRhjN3AUeBj4OkOuzAZOrMw+G51Ycf6Ddjz0inPO7n4BFPCPSXa1\nVfCw8j+LFwPHgA+3YbW/TvIqlrBfyx3w3avZH7UrdqpSklcDnwTeU1XfGTy3UvtWVS9W1RZmn3gv\nB16/zE0aWQZWnC93W8bkzVV1GbPDFLcm+aXBkyv0s7gKuAy4q6ouBf6Vk6aVj9qv5Q74E6teTxhc\nEbuSPZ1kLUB7PdrKV1R/k5zLbLh/rKo+1Yq76BtAVT3H7IK9K2krs9upuVZmc5avzD6x4vwAcC+z\nwzQvrThvdVZivwCoqsPt9SjwaWZ/MK/0z+Ih4FBVPdqO72c28JesX8sd8F8ANre/9L+C2ZWyO5a5\nTUthcDXvyat8397+Gn4F8PzAr2JnlSRhdtHa/qr6wMCpFd23JBNJVrf9VzL7d4X9rPCV2dXxivMk\nr0rymhP7wK8Ae1nhn8Wqego4mOTnW9E1wOMsZb/Ogj80XA/8C7PjoH+w3O1ZRPs/DhwB/h+zP5Fv\nYXYscyfwNeCfgAtb3TA7a+jrwFeAyeVu/2n69WZmfzXcA+xu2/UrvW/AfwG+1Pq1F/jDVv4zwGPA\nDPB/gfNa+fnteKad/5nl7sMQfbwKeLCXfrU+fLlt+07kxEr/LLa2bgGm2+fx74ALlrJfrmSVpE4t\n9xCNJGlMDHhJ6pQBL0mdMuAlqVMGvCR1yoCXpE4Z8JLUKQNekjr1/wFwJ3uUqSJEOgAAAABJRU5E\nrkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "env = gym.make(\"CartPole-v0\")\n",
    "n_actions = env.action_space.n\n",
    "\n",
    "print(\"first state:%s\" % (env.reset()))\n",
    "plt.imshow(env.render('rgb_array'))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Play a few games\n",
    "\n",
    "We need to estimate observation distributions. To do so, we'll play a few games and record all states."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYcAAAD8CAYAAACcjGjIAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAE21JREFUeJzt3X+sX/V93/Hna7iQpltiE+4Is53a\na6xsJNoWegVUkaYoVGBIhZFGI0fVcDK33la6dlul1LTSLJEikW0aK2pD54GHiSJ+iLXDW5wyDxJl\nkwrhkh/8LOOWhNgWhNvYcdbRJnP63h/34/LFn3t9L/d7fb/X3OdD+up7zvvzOed8vsdX9+Xz43tu\nqgpJkgb9lVEPQJK0/BgOkqSO4SBJ6hgOkqSO4SBJ6hgOkqSO4SBJ6hgOkqTOnOGQZE+SV5I8NUPb\nryapJOe1+SS5NclkkieSXDTQd1uS59tr20D9J5M82Za5NUkW68NJkhZm1Tz63An8NnDXYDHJeuBy\n4FsD5SuBTe11CXAbcEmSc4FdwDhQwONJ9lXV0dbnF4BHgf3AZuDzcw3qvPPOqw0bNsxj+JKkEx5/\n/PE/qaqxufrNGQ5V9aUkG2ZougX4BPDAQG0LcFdNP5PjkSSrk1wAfBA4UFVHAJIcADYn+SLwtqp6\npNXvAq5hHuGwYcMGJiYm5uomSRqQ5MX59FvQNYckW4DDVfX1k5rWAgcH5g+12qnqh2aoS5JGaD6n\nlV4nyVuBX2f6lNKSSrID2AHwrne9a6k3L0krxkKOHH4C2Ah8Pck3gXXAV5K8EzgMrB/ou67VTlVf\nN0N9RlW1u6rGq2p8bGzOU2aSpAV6w+FQVU9W1V+vqg1VtYHpU0EXVdXLwD7gunbX0qXAsap6CXgQ\nuDzJmiRrmD7qeLC1fS/Jpe0upet4/TUMSdIIzOdW1ruBPwTek+RQku2n6L4feAGYBP4j8IsA7UL0\nJ4HH2uvGExenW5/b2zJ/zDwuRkuSTq+cqX/sZ3x8vLxbSZLemCSPV9X4XP38hrQkqWM4SJI6hoMk\nqfOGv+eglWfDzs8teNlv3vzhRRyJpKXikYMkqWM4SJI6hoMkqWM4SJI6hoMkqWM4SJI6hoMkqWM4\nSJI6hoMkqWM4SJI6hoMkqWM4SJI6hoMkqWM4SJI6hoMkqWM4SJI6hoMkqWM4SJI6c4ZDkj1JXkny\n1EDt3yT5oyRPJPn9JKsH2m5IMpnkuSRXDNQ3t9pkkp0D9Y1JHm31e5OcvZgfUJL0xs3nyOFOYPNJ\ntQPA+6rq7wD/G7gBIMmFwFbgvW2ZTyc5K8lZwO8AVwIXAh9tfQE+BdxSVe8GjgLbh/pEkqShzRkO\nVfUl4MhJtf9eVcfb7CPAuja9Bbinqr5fVd8AJoGL22uyql6oqh8A9wBbkgT4EHB/W34vcM2Qn0mS\nNKTFuObwj4DPt+m1wMGBtkOtNlv9HcB3B4LmRF2SNEJDhUOS3wCOA59dnOHMub0dSSaSTExNTS3F\nJiVpRVpwOCT5GPAzwM9VVbXyYWD9QLd1rTZb/TvA6iSrTqrPqKp2V9V4VY2PjY0tdOiSpDksKByS\nbAY+AVxdVa8ONO0DtiY5J8lGYBPwZeAxYFO7M+lspi9a72uh8gXg2rb8NuCBhX0USdJimc+trHcD\nfwi8J8mhJNuB3wb+GnAgydeS/C5AVT0N3Ac8A/wBcH1V/bBdU/gl4EHgWeC+1hfg14B/mWSS6WsQ\ndyzqJ5QkvWGr5upQVR+doTzrL/Cqugm4aYb6fmD/DPUXmL6bSZK0TPgNaUlSx3CQJHUMB0lSx3CQ\nJHUMB0lSx3CQJHUMB0lSx3CQJHUMB0lSx3CQJHUMB0lSx3CQJHUMB0lSx3CQJHUMB0lSx3CQJHUM\nB0lSx3CQJHUMB0lSx3CQJHUMB0lSx3CQJHXmDIcke5K8kuSpgdq5SQ4keb69r2n1JLk1yWSSJ5Jc\nNLDMttb/+STbBuo/meTJtsytSbLYH1KS9MbM58jhTmDzSbWdwENVtQl4qM0DXAlsaq8dwG0wHSbA\nLuAS4GJg14lAaX1+YWC5k7clSVpic4ZDVX0JOHJSeQuwt03vBa4ZqN9V0x4BVie5ALgCOFBVR6rq\nKHAA2Nza3lZVj1RVAXcNrEuSNCILveZwflW91KZfBs5v02uBgwP9DrXaqeqHZqjPKMmOJBNJJqam\nphY4dEnSXIa+IN3+x1+LMJb5bGt3VY1X1fjY2NhSbFKSVqSFhsO32ykh2vsrrX4YWD/Qb12rnaq+\nboa6JGmEFhoO+4ATdxxtAx4YqF/X7lq6FDjWTj89CFyeZE27EH058GBr+16SS9tdStcNrEuSNCKr\n5uqQ5G7gg8B5SQ4xfdfRzcB9SbYDLwIfad33A1cBk8CrwMcBqupIkk8Cj7V+N1bViYvcv8j0HVE/\nCny+vSRJIzRnOFTVR2dpumyGvgVcP8t69gB7ZqhPAO+baxySpKXjN6QlSR3DQZLUMRwkSR3DQZLU\nMRwkSR3DQZLUMRwkSR3DQZLUMRwkSR3DQZLUMRwkSR3DQZLUMRwkSR3DQZLUMRwkSR3DQZLUMRwk\nSR3DQZLUMRwkSR3DQZLUMRwkSZ2hwiHJv0jydJKnktyd5C1JNiZ5NMlkknuTnN36ntPmJ1v7hoH1\n3NDqzyW5YriPJEka1oLDIcla4JeB8ap6H3AWsBX4FHBLVb0bOApsb4tsB462+i2tH0kubMu9F9gM\nfDrJWQsdlyRpeMOeVloF/GiSVcBbgZeADwH3t/a9wDVtekubp7VfliStfk9Vfb+qvgFMAhcPOS5J\n0hAWHA5VdRj4t8C3mA6FY8DjwHer6njrdghY26bXAgfbssdb/3cM1mdYRpI0AsOcVlrD9P/6NwJ/\nA/gxpk8LnTZJdiSZSDIxNTV1OjclSSvaMKeVfhr4RlVNVdX/A34P+ACwup1mAlgHHG7Th4H1AK39\n7cB3BuszLPM6VbW7qsaranxsbGyIoUuSTmWYcPgWcGmSt7ZrB5cBzwBfAK5tfbYBD7TpfW2e1v5w\nVVWrb213M20ENgFfHmJckqQhrZq7y8yq6tEk9wNfAY4DXwV2A58D7knym612R1vkDuAzSSaBI0zf\noURVPZ3kPqaD5ThwfVX9cKHjkiQNb8HhAFBVu4BdJ5VfYIa7jarqz4GfnWU9NwE3DTMWSdLi8RvS\nkqSO4SBJ6hgOkqSO4SBJ6hgOkqSO4SBJ6hgOkqSO4SBJ6hgOkqSO4SBJ6hgOkqSO4SBJ6hgOkqSO\n4SBJ6hgOkqSO4SBJ6hgOkqSO4SBJ6hgOkqSO4SBJ6hgOkqSO4SBJ6gwVDklWJ7k/yR8leTbJTyU5\nN8mBJM+39zWtb5LcmmQyyRNJLhpYz7bW//kk24b9UJKk4Qx75PBbwB9U1d8C/i7wLLATeKiqNgEP\ntXmAK4FN7bUDuA0gybnALuAS4GJg14lAkSSNxoLDIcnbgb8P3AFQVT+oqu8CW4C9rdte4Jo2vQW4\nq6Y9AqxOcgFwBXCgqo5U1VHgALB5oeOSJA1vmCOHjcAU8J+SfDXJ7Ul+DDi/ql5qfV4Gzm/Ta4GD\nA8sfarXZ6pKkERkmHFYBFwG3VdX7gf/La6eQAKiqAmqIbbxOkh1JJpJMTE1NLdZqJUknGSYcDgGH\nqurRNn8/02Hx7Xa6iPb+Sms/DKwfWH5dq81W71TV7qoar6rxsbGxIYYuSTqVBYdDVb0MHEzynla6\nDHgG2AecuONoG/BAm94HXNfuWroUONZOPz0IXJ5kTbsQfXmrSZJGZNWQy/8z4LNJzgZeAD7OdODc\nl2Q78CLwkdZ3P3AVMAm82vpSVUeSfBJ4rPW7saqODDkuSdIQhgqHqvoaMD5D02Uz9C3g+lnWswfY\nM8xYJEmLx29IS5I6hoMkqWM4SJI6hoMkqWM4SJI6hoMkqWM4SJI6hoMkqWM4SJI6hoMkqWM4SJI6\nhoMkqWM4SJI6hoMkqTPs33PQGWLDzs+NegiSziAeOUiSOoaDJKljOEiSOoaDJKljOEiSOoaDJKlj\nOEiSOkOHQ5Kzknw1yX9r8xuTPJpkMsm9Sc5u9XPa/GRr3zCwjhta/bkkVww7JknScBbjyOFXgGcH\n5j8F3FJV7waOAttbfTtwtNVvaf1IciGwFXgvsBn4dJKzFmFckqQFGiockqwDPgzc3uYDfAi4v3XZ\nC1zTpre0eVr7Za3/FuCeqvp+VX0DmAQuHmZckqThDHvk8O+BTwB/0ebfAXy3qo63+UPA2ja9FjgI\n0NqPtf5/WZ9hmddJsiPJRJKJqampIYcuSZrNgsMhyc8Ar1TV44s4nlOqqt1VNV5V42NjY0u1WUla\ncYZ58N4HgKuTXAW8BXgb8FvA6iSr2tHBOuBw638YWA8cSrIKeDvwnYH6CYPLSJJGYMFHDlV1Q1Wt\nq6oNTF9Qfriqfg74AnBt67YNeKBN72vztPaHq6pafWu7m2kjsAn48kLHJUka3ul4ZPevAfck+U3g\nq8AdrX4H8Jkkk8ARpgOFqno6yX3AM8Bx4Pqq+uFpGJckaZ4WJRyq6ovAF9v0C8xwt1FV/Tnws7Ms\nfxNw02KMRZI0PP/Yj06rYf7I0Ddv/vAijkTSG+HjMyRJHcNBktQxHCRJHcNBktQxHCRJHcNBktQx\nHCRJHcNBktQxHCRJHcNBktQxHCRJHcNBktQxHCRJHcNBktQxHCRJHcNBktQxHCRJHcNBktQxHCRJ\nHcNBktRZcDgkWZ/kC0meSfJ0kl9p9XOTHEjyfHtf0+pJcmuSySRPJLloYF3bWv/nk2wb/mNJkoYx\nzJHDceBXq+pC4FLg+iQXAjuBh6pqE/BQmwe4EtjUXjuA22A6TIBdwCXAxcCuE4EiSRqNBYdDVb1U\nVV9p0/8HeBZYC2wB9rZue4Fr2vQW4K6a9giwOskFwBXAgao6UlVHgQPA5oWOS5I0vEW55pBkA/B+\n4FHg/Kp6qTW9DJzfptcCBwcWO9Rqs9UlSSMydDgk+avAfwb+eVV9b7CtqgqoYbcxsK0dSSaSTExN\nTS3WaiVJJxkqHJL8CNPB8Nmq+r1W/nY7XUR7f6XVDwPrBxZf12qz1TtVtbuqxqtqfGxsbJihS5JO\nYZi7lQLcATxbVf9uoGkfcOKOo23AAwP169pdS5cCx9rppweBy5OsaReiL281SdKIrBpi2Q8A/xB4\nMsnXWu3XgZuB+5JsB14EPtLa9gNXAZPAq8DHAarqSJJPAo+1fjdW1ZEhxiVJGtKCw6Gq/heQWZov\nm6F/AdfPsq49wJ6FjkWStLj8hrQkqWM4SJI6hoMkqWM4SJI6hoMkqWM4SJI6hoMkqWM4SJI6hoMk\nqWM4SJI6hoMkqWM4SJI6hoMkqWM4SJI6hoMkqTPMH/vREtuw83OjHoKkFcIjB0lSx3CQJHU8raRl\na9jTaN+8+cOLNBJp5fHIQZLUMRwkSR3DQZLUWTbhkGRzkueSTCbZOerxSNJKtizCIclZwO8AVwIX\nAh9NcuFoRyVJK9dyuVvpYmCyql4ASHIPsAV4ZqSjWmR+iW1pDbO/vdNJK91yCYe1wMGB+UPAJSMa\nyyn5C35lOFP/nQ01LZblEg7zkmQHsKPN/mmS507j5s4D/uQ0rv/Nwv00P0uyn/Kp072F086fp/kZ\nZj/9+Hw6LZdwOAysH5hf12qvU1W7gd1LMaAkE1U1vhTbOpO5n+bH/TQ/7qf5WYr9tCwuSAOPAZuS\nbExyNrAV2DfiMUnSirUsjhyq6niSXwIeBM4C9lTV0yMeliStWMsiHACqaj+wf9TjGLAkp6/eBNxP\n8+N+mh/30/yc9v2Uqjrd25AknWGWyzUHSdIyYjg0Sc5NciDJ8+19zSz9fpjka+21Yi6az/V4kyTn\nJLm3tT+aZMPSj3K05rGPPpZkauDn5+dHMc5RS7InyStJnpqlPUlubfvxiSQXLfUYl4N57KcPJjk2\n8PP0rxZz+4bDa3YCD1XVJuChNj+TP6uqv9deVy/d8EZnno832Q4crap3A7cAZ/4d92/AG3gEzL0D\nPz+3L+kgl487gc2naL8S2NReO4DblmBMy9GdnHo/AfzPgZ+nGxdz44bDa7YAe9v0XuCaEY5lufnL\nx5tU1Q+AE483GTS4/+4HLkuSJRzjqM1nHwmoqi8BR07RZQtwV017BFid5IKlGd3yMY/9dFoZDq85\nv6peatMvA+fP0u8tSSaSPJJkpQTITI83WTtbn6o6DhwD3rEko1se5rOPAP5BO1Vyf5L1M7Rr/vtS\n8FNJvp7k80neu5grXja3si6FJP8DeOcMTb8xOFNVlWS227h+vKoOJ/mbwMNJnqyqP17ssepN6b8C\nd1fV95P8Y6aPtD404jHpzPUVpn8f/WmSq4D/wvSpuEWxosKhqn56trYk305yQVW91A5hX5llHYfb\n+wtJvgi8H3izh8N8Hm9yos+hJKuAtwPfWZrhLQtz7qOqGtwftwP/egnGdSaa1+N0Vrqq+t7A9P4k\nn05yXlUtyrOpPK30mn3Atja9DXjg5A5J1iQ5p02fB3yAN9ljxWcxn8ebDO6/a4GHa2V9iWbOfXTS\nefOrgWeXcHxnkn3Ade2upUuBYwOnfNUkeeeJ63pJLmb69/mi/YdsRR05zOFm4L4k24EXgY8AJBkH\n/klV/Tzwt4H/kOQvmP6HuLmq3vThMNvjTZLcCExU1T7gDuAzSSaZvoi2dXQjXnrz3Ee/nORq4DjT\n++hjIxvwCCW5G/ggcF6SQ8Au4EcAqup3mX5SwlXAJPAq8PHRjHS05rGfrgX+aZLjwJ8BWxfzP2R+\nQ1qS1PG0kiSpYzhIkjqGgySpYzhIkjqGgySpYzhIkjqGgySpYzhIkjr/H8mivBis1xEYAAAAAElF\nTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYAAAAD8CAYAAAB+UHOxAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAADZRJREFUeJzt3XGonXd9x/H3x1bdwGGrvXSShN2C\nYSOOTSXUiv+MdrbRiumGSmVo5gL5pwMFwaXzjzK1UBnYTTYdZQ1GEWtRR4utdFmtyGDWptp1tlnX\nO600odpoalVER/S7P+4v5VoT7znJveck+b5fcLnP83uec+7voe199znnOc9NVSFJ6uc5856AJGk+\nDIAkNWUAJKkpAyBJTRkASWrKAEhSUwZAkpoyAJLUlAGQpKbOnfcEfp0LLrigFhcX5z0NSTqj3H//\n/d+rqoXV9jutA7C4uMj+/fvnPQ1JOqMk+fYk+/kSkCQ1ZQAkqSkDIElNGQBJasoASFJTBkCSmjIA\nktSUAZCkpgyAJDV1Wn8SWFpvi7vvOOnHPnbDlWs4E2n2PAOQpKYMgCQ1ZQAkqSkDIElNGQBJasqr\ngKST5BVEOtN5BiBJTRkASWrKAEhSUwZAkpoyAJLUlAGQpKYMgCQ1ZQAkqSkDIElNGQBJasoASFJT\nBkCSmjIAktSUAZCkpiYOQJJzknw9yefH+kVJ7k2ylOTTSZ43xp8/1pfG9sUVz3HtGH8kyRVrfTCS\npMlNcwbwTuDAivUPAjdW1UuBp4CdY3wn8NQYv3HsR5ItwNXAy4BtwEeSnHNq05cknayJApBkI3Al\n8M9jPcClwGfGLnuBq8by9rHO2H7Z2H87cEtV/ayqvgUsARevxUFIkqY36RnA3wHvAX4x1l8M/KCq\njo71g8CGsbwBeBxgbH967P/M+HEeI0masVUDkOQNwJNVdf8M5kOSXUn2J9l/+PDhWfxISWppkjOA\n1wBvTPIYcAvLL/38PXBekmN/U3gjcGgsHwI2AYztLwS+v3L8OI95RlXdVFVbq2rrwsLC1AckSZrM\nqgGoqmuramNVLbL8Ju4Xq+rPgHuAN43ddgC3jeXbxzpj+xerqsb41eMqoYuAzcBX1+xIJElTOXf1\nXU7or4BbknwA+Dpw8xi/GfhEkiXgCMvRoKoeSnIr8DBwFLimqn5+Cj9fknQKpgpAVX0J+NJY/ibH\nuYqnqn4KvPkEj78euH7aSUqS1p6fBJakpgyAJDVlACSpKQMgSU0ZAElqygBIUlMGQJKaMgCS1JQB\nkKSmDIAkNWUAJKkpAyBJTRkASWrKAEhSUwZAkpoyAJLUlAGQpKYMgCQ1ZQAkqSkDIElNGQBJasoA\nSFJT5857AtKpWNx9x7ynIJ2xPAOQpKYMgCQ1ZQAkqSkDIElNGQBJasoASFJTBkCSmjIAktSUAZCk\npgyAJDVlACSpKe8FpLnzfj7SfHgGIElNrRqAJL+R5KtJ/jPJQ0n+ZoxflOTeJEtJPp3keWP8+WN9\naWxfXPFc147xR5JcsV4HJUla3SRnAD8DLq2qPwReDmxLcgnwQeDGqnop8BSwc+y/E3hqjN849iPJ\nFuBq4GXANuAjSc5Zy4ORJE1u1QDUsh+P1eeOrwIuBT4zxvcCV43l7WOdsf2yJBnjt1TVz6rqW8AS\ncPGaHIUkaWoTvQeQ5JwkDwBPAvuA/wV+UFVHxy4HgQ1jeQPwOMDY/jTw4pXjx3mMJGnGJgpAVf28\nql4ObGT5/9p/b70mlGRXkv1J9h8+fHi9fowktTfVVUBV9QPgHuDVwHlJjl1GuhE4NJYPAZsAxvYX\nAt9fOX6cx6z8GTdV1daq2rqwsDDN9CRJU5jkKqCFJOeN5d8EXgscYDkEbxq77QBuG8u3j3XG9i9W\nVY3xq8dVQhcBm4GvrtWBSJKmM8kHwV4C7B1X7DwHuLWqPp/kYeCWJB8Avg7cPPa/GfhEkiXgCMtX\n/lBVDyW5FXgYOApcU1U/X9vDkSRNatUAVNWDwCuOM/5NjnMVT1X9FHjzCZ7reuD66acpSVprfhJY\nkpoyAJLUlAGQpKYMgCQ1ZQAkqSkDIElNGQBJasoASFJTBkCSmjIAktSUAZCkpgyAJDVlACSpKQMg\nSU0ZAElqygBIUlMGQJKaMgCS1JQBkKSmDIAkNbXqH4WXtPYWd99x0o997IYr13Am6swzAElqygBI\nUlMGQJKaMgCS1JQBkKSmDIAkNWUAJKkpAyBJTRkASWrKAEhSUwZAkpoyAJLUlAGQpKYMgCQ1ZQAk\nqSkDIElNrRqAJJuS3JPk4SQPJXnnGH9Rkn1JHh3fzx/jSfLhJEtJHkzyyhXPtWPs/2iSHet3WJKk\n1UxyBnAUeHdVbQEuAa5JsgXYDdxdVZuBu8c6wOuAzeNrF/BRWA4GcB3wKuBi4Lpj0ZAkzd6qAaiq\nJ6rqa2P5R8ABYAOwHdg7dtsLXDWWtwMfr2VfAc5L8hLgCmBfVR2pqqeAfcC2NT0aSdLEpnoPIMki\n8ArgXuDCqnpibPoOcOFY3gA8vuJhB8fYicYlSXMwcQCSvAD4LPCuqvrhym1VVUCtxYSS7EqyP8n+\nw4cPr8VTSpKOY6IAJHkuy7/8P1lVnxvD3x0v7TC+PznGDwGbVjx84xg70fgvqaqbqmprVW1dWFiY\n5lgkSVOY5CqgADcDB6rqQys23Q4cu5JnB3DbivG3j6uBLgGeHi8V3QVcnuT88ebv5WNMkjQH506w\nz2uAtwH/leSBMfbXwA3ArUl2At8G3jK23Qm8HlgCfgK8A6CqjiR5P3Df2O99VXVkTY5CkjS1VQNQ\nVf8O5ASbLzvO/gVcc4Ln2gPsmWaCkqT14SeBJakpAyBJTRkASWrKAEhSUwZAkpoyAJLU1CSfA5BW\ntbj7jnlPQdKUPAOQpKYMgCQ1ZQAkqSkDIElNGQBJasoASFJTBkCSmjIAktSUAZCkpgyAJDVlACSp\nKQMgSU0ZAElqygBIUlMGQJKaMgCS1JQBkKSmDIAkNWUAJKkpAyBJTRkASWrKAEhSUwZAkpoyAJLU\nlAGQpKYMgCQ1ZQAkqSkDIElNnTvvCUiazuLuO076sY/dcOUazkRnOs8AJKmpVQOQZE+SJ5N8Y8XY\ni5LsS/Lo+H7+GE+SDydZSvJgkleueMyOsf+jSXasz+FIkiY1yRnAx4BtzxrbDdxdVZuBu8c6wOuA\nzeNrF/BRWA4GcB3wKuBi4Lpj0ZAkzceqAaiqLwNHnjW8Hdg7lvcCV60Y/3gt+wpwXpKXAFcA+6rq\nSFU9BezjV6MiSZqhk30P4MKqemIsfwe4cCxvAB5fsd/BMXai8V+RZFeS/Un2Hz58+CSnJ0lazSm/\nCVxVBdQazOXY891UVVurauvCwsJaPa0k6VlONgDfHS/tML4/OcYPAZtW7LdxjJ1oXJI0JycbgNuB\nY1fy7ABuWzH+9nE10CXA0+OloruAy5OcP978vXyMSZLmZNUPgiX5FPBHwAVJDrJ8Nc8NwK1JdgLf\nBt4ydr8TeD2wBPwEeAdAVR1J8n7gvrHf+6rq2W8sS5JmaNUAVNVbT7DpsuPsW8A1J3iePcCeqWYn\nSVo3fhJYkpoyAJLUlAGQpKYMgCQ1ZQAkqSn/HoDUyKn8LQHw7wmcbTwDkKSmDIAkNWUAJKkpAyBJ\nTRkASWrKq4D0jFO9QkTSmcUzAElqygBIUlMGQJKaMgCS1JQBkKSmDIAkNWUAJKkpAyBJTflBMEkT\nO5UPC3or6dOPZwCS1JQBkKSmfAnoLOK9fCRNwzMASWrKAEhSUwZAkpoyAJLUlAGQpKYMgCQ1ZQAk\nqSkDIElN+UGw04wf5pI0KwZA0kx4I7nTjy8BSVJTBkCSmpp5AJJsS/JIkqUku2f98yVJy2b6HkCS\nc4B/BF4LHATuS3J7VT08y3lIOrP4/sH6mPWbwBcDS1X1TYAktwDbgbMqAF7JI+lMMOsAbAAeX7F+\nEHjVev0wfxFL8uzhxE67y0CT7AJ2jdUfJ3lknvN5lguA7817EjPmMffR8bh/7THngzOcydr6nUl2\nmnUADgGbVqxvHGPPqKqbgJtmOalJJdlfVVvnPY9Z8pj76HjcHY95pVlfBXQfsDnJRUmeB1wN3D7j\nOUiSmPEZQFUdTfKXwF3AOcCeqnpolnOQJC2b+XsAVXUncOesf+4aOS1fmlpnHnMfHY+74zE/I1U1\n7zlIkubAW0FIUlMGYEpJ/jbJfyd5MMm/JDlv3nNab0nenOShJL9IclZfMdHxViVJ9iR5Msk35j2X\nWUmyKck9SR4e/26/c95zmgcDML19wO9X1R8A/wNcO+f5zMI3gD8FvjzviaynFbcqeR2wBXhrki3z\nndVMfAzYNu9JzNhR4N1VtQW4BLimyT/rX2IAplRV/1pVR8fqV1j+LMNZraoOVNXp9IG89fLMrUqq\n6v+AY7cqOatV1ZeBI/OexyxV1RNV9bWx/CPgAMt3KmjFAJyavwC+MO9JaM0c71Yl7X4pdJNkEXgF\ncO98ZzJ7p92tIE4HSf4N+O3jbHpvVd029nkvy6eRn5zl3NbLJMcsnW2SvAD4LPCuqvrhvOczawbg\nOKrqj3/d9iR/DrwBuKzOkutoVzvmJla9VYnOHkmey/Iv/09W1efmPZ958CWgKSXZBrwHeGNV/WTe\n89Ga8lYlTSQJcDNwoKo+NO/5zIsBmN4/AL8F7EvyQJJ/mveE1luSP0lyEHg1cEeSu+Y9p/Uw3tw/\ndquSA8CtHW5VkuRTwH8Av5vkYJKd857TDLwGeBtw6fjv+IEkr5/3pGbNTwJLUlOeAUhSUwZAkpoy\nAJLUlAGQpKYMgCQ1ZQAkqSkDIElNGQBJaur/AUTjSQflUya9AAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYAAAAD8CAYAAAB+UHOxAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAERdJREFUeJzt3X+s3fVdx/HnSzam8RcwrrWWYtHV\nGGa0M1fEzOgUhcKMxWROSLZ1k9j9AVGj/7DNhLlJgj9xJpNYpbEYN4b7EZqtil03o0tkUDZkKzi5\nY520dlAHwymKFt/+cT6dx3pv77n3nntOez/PR3Jyvt/39/P9ns8nTe/rfH+eVBWSpP581bQ7IEma\nDgNAkjplAEhSpwwASeqUASBJnTIAJKlTBoAkdcoAkKROGQCS1KkXTLsDp3L++efXpk2bpt0NSTqj\nPPDAA/9cVTOLtTutA2DTpk0cOHBg2t2QpDNKks+P0s5DQJLUKQNAkjplAEhSpwwASeqUASBJnTIA\nJKlTBoAkdcoAkKROGQCS1KnT+k5g6XS26cYPLXvdQ7e8cow9kZZn0T2AJF+d5L4kf5fkYJJfbfWL\nknw8yVyS9yQ5u9Vf1Obn2vJNQ9t6U6t/JskVqzUoSdLiRjkE9Bzwo1X1PcAWYGuSS4FfB26tqpcA\nTwPXtfbXAU+3+q2tHUkuBq4BXgpsBX4/yVnjHIwkaXSLBkAN/GubfWF7FfCjwHtbfTdwdZve1uZp\nyy9Lkla/s6qeq6rPAXPAJWMZhSRpyUY6B9C+qT8AvAR4J/BZ4EtVdbw1OQxsaNMbgMcBqup4kmeA\nF7f6vUObHV5HmoqVHMeXznQjXQVUVc9X1RbgAgbf2r9ztTqUZEeSA0kOHDt2bLU+RpK6t6TLQKvq\nS8BHgR8AzklyYg/iAuBImz4CbARoy78R+OJwfZ51hj9jZ1XNVtXszMyiv2cgSVqmUa4CmklyTpv+\nGuDHgUcYBMGrWrPtwN1tek+bpy3/SFVVq1/TrhK6CNgM3DeugUiSlmaUcwDrgd3tPMBXAXdV1QeT\nPAzcmeTXgE8Ct7f2twN/kmQOeIrBlT9U1cEkdwEPA8eB66vq+fEOR5I0qkUDoKoeAl42T/0x5rmK\np6r+A/jpBbZ1M3Dz0rspSRo3HwUhSZ0yACSpUwaAJHXKAJCkThkAktQpA0CSOmUASFKnDABJ6pQB\nIEmd8ichdUbzcc7S8rkHIEmdMgAkqVMGgCR1ygCQpE4ZAJLUKQNAkjplAEhSpwwASeqUASBJnTIA\nJKlTBoAkdcoAkKROGQCS1CkDQJI6ZQBIUqcW/T2AJBuBO4B1QAE7q+odSd4K/BxwrDV9c1Xtbeu8\nCbgOeB74+aq6p9W3Au8AzgL+qKpuGe9wpDPDSn7H4NAtrxxjT9SzUX4Q5jjwy1X1iSRfDzyQZF9b\ndmtV/dZw4yQXA9cALwW+Bfhwku9oi98J/DhwGLg/yZ6qengcA5EkLc2iAVBVR4GjbfrLSR4BNpxi\nlW3AnVX1HPC5JHPAJW3ZXFU9BpDkztbWAJCkKVjSOYAkm4CXAR9vpRuSPJRkV5JzW20D8PjQaodb\nbaH6yZ+xI8mBJAeOHTt28mJJ0piMHABJvg54H/CLVfUvwG3AtwNbGOwh/PY4OlRVO6tqtqpmZ2Zm\nxrFJSdI8RvpR+CQvZPDH/0+r6v0AVfXE0PI/BD7YZo8AG4dWv6DVOEVdkjRhi+4BJAlwO/BIVf3O\nUH39ULOfAj7dpvcA1yR5UZKLgM3AfcD9wOYkFyU5m8GJ4j3jGYYkaalG2QN4OfBa4FNJHmy1NwPX\nJtnC4NLQQ8AbAarqYJK7GJzcPQ5cX1XPAyS5AbiHwWWgu6rq4BjHIklaglGuAvoYkHkW7T3FOjcD\nN89T33uq9SRJk+OdwJLUKQNAkjplAEhSpwwASeqUASBJnTIAJKlTBoAkdcoAkKROGQCS1CkDQJI6\nZQBIUqcMAEnqlAEgSZ0yACSpUwaAJHXKAJCkThkAktQpA0CSOmUASFKnDABJ6pQBIEmdMgAkqVMG\ngCR1ygCQpE4tGgBJNib5aJKHkxxM8gutfl6SfUkebe/ntnqS/F6SuSQPJfneoW1tb+0fTbJ99YYl\nSVrMKHsAx4FfrqqLgUuB65NcDNwI7K+qzcD+Ng9wJbC5vXYAt8EgMICbgO8HLgFuOhEakqTJWzQA\nqupoVX2iTX8ZeATYAGwDdrdmu4Gr2/Q24I4auBc4J8l64ApgX1U9VVVPA/uArWMdjSRpZEs6B5Bk\nE/Ay4OPAuqo62hZ9AVjXpjcAjw+tdrjVFqpLkqZg5ABI8nXA+4BfrKp/GV5WVQXUODqUZEeSA0kO\nHDt2bByblCTNY6QASPJCBn/8/7Sq3t/KT7RDO7T3J1v9CLBxaPULWm2h+v9RVTuraraqZmdmZpYy\nFknSEoxyFVCA24FHqup3hhbtAU5cybMduHuo/rp2NdClwDPtUNE9wOVJzm0nfy9vNUnSFLxghDYv\nB14LfCrJg632ZuAW4K4k1wGfB17dlu0FrgLmgGeBNwBU1VNJ3g7c39q9raqeGssoJElLtmgAVNXH\ngCyw+LJ52hdw/QLb2gXsWkoHJUmrwzuBJalTBoAkdcoAkKROGQCS1CkDQJI6ZQBIUqcMAEnqlAEg\nSZ0yACSpUwaAJHXKAJCkThkAktQpA0CSOmUASFKnDABJ6pQBIEmdMgAkqVMGgCR1ygCQpE4ZAJLU\nKQNAkjplAEhSpwwASeqUASBJnTIAJKlTiwZAkl1Jnkzy6aHaW5McSfJge101tOxNSeaSfCbJFUP1\nra02l+TG8Q9FkrQUo+wB/DGwdZ76rVW1pb32AiS5GLgGeGlb5/eTnJXkLOCdwJXAxcC1ra0kaUpe\nsFiDqvrrJJtG3N424M6qeg74XJI54JK2bK6qHgNIcmdr+/CSeyxJGouVnAO4IclD7RDRua22AXh8\nqM3hVluoLkmakkX3ABZwG/B2oNr7bwM/O44OJdkB7AC48MILx7FJneY23fihaXdB6tKy9gCq6omq\ner6q/hv4Q/73MM8RYONQ0wtabaH6fNveWVWzVTU7MzOznO5JkkawrABIsn5o9qeAE1cI7QGuSfKi\nJBcBm4H7gPuBzUkuSnI2gxPFe5bfbUnSSi16CCjJu4FXAOcnOQzcBLwiyRYGh4AOAW8EqKqDSe5i\ncHL3OHB9VT3ftnMDcA9wFrCrqg6OfTSSpJGNchXQtfOUbz9F+5uBm+ep7wX2Lql3kqRV453AktQp\nA0CSOmUASFKnDABJ6tRybwSTNCUrvXHu0C2vHFNPdKZzD0CSOmUASFKnDABJ6pQBIEmdMgAkqVMG\ngCR1ygCQpE4ZAJLUKQNAkjplAEhSpwwASeqUASBJnTIAJKlTBoAkdcoAkKRO+XsAUmdW8nsC/pbA\n2uIegCR1ygCQpE4ZAJLUKQNAkjq16EngJLuAnwCerKrvarXzgPcAm4BDwKur6ukkAd4BXAU8C7y+\nqj7R1tkO/Erb7K9V1e7xDkXTtNIfKpc0eaPsAfwxsPWk2o3A/qraDOxv8wBXApvbawdwG3wlMG4C\nvh+4BLgpybkr7bwkafkWDYCq+mvgqZPK24AT3+B3A1cP1e+ogXuBc5KsB64A9lXVU1X1NLCP/x8q\nkqQJWu45gHVVdbRNfwFY16Y3AI8PtTvcagvV/58kO5IcSHLg2LFjy+yeJGkxKz4JXFUF1Bj6cmJ7\nO6tqtqpmZ2ZmxrVZSdJJlnsn8BNJ1lfV0XaI58lWPwJsHGp3QasdAV5xUv2vlvnZkqbEu4jXluXu\nAewBtrfp7cDdQ/XXZeBS4Jl2qOge4PIk57aTv5e3miRpSka5DPTdDL69n5/kMIOreW4B7kpyHfB5\n4NWt+V4Gl4DOMbgM9A0AVfVUkrcD97d2b6uqk08sS5ImaNEAqKprF1h02TxtC7h+ge3sAnYtqXeS\npFXjncCS1CkfB62v8G5eqS/uAUhSpwwASeqUASBJnTIAJKlTBoAkdcqrgCRNhI+ROP24ByBJnTIA\nJKlTHgJaY7yZS9Ko3AOQpE4ZAJLUKQ8BSTrteQXR6nAPQJI6ZQBIUqcMAEnqlAEgSZ0yACSpU14F\ndJrxRi5Jk2IASFrTVvqlai1fRuohIEnqlHsAq8DDOJLOBO4BSFKnVhQASQ4l+VSSB5McaLXzkuxL\n8mh7P7fVk+T3kswleSjJ945jAJKk5RnHHsCPVNWWqppt8zcC+6tqM7C/zQNcCWxurx3AbWP4bEnS\nMq3GIaBtwO42vRu4eqh+Rw3cC5yTZP0qfL4kaQQrDYAC/jLJA0l2tNq6qjrapr8ArGvTG4DHh9Y9\n3GqSpClY6VVAP1hVR5J8E7Avyd8PL6yqSlJL2WALkh0AF1544Qq7J0layIr2AKrqSHt/EvgAcAnw\nxIlDO+39ydb8CLBxaPULWu3kbe6sqtmqmp2ZmVlJ9yRJp7DsAEjytUm+/sQ0cDnwaWAPsL012w7c\n3ab3AK9rVwNdCjwzdKhIkjRhKzkEtA74QJIT23lXVf1FkvuBu5JcB3weeHVrvxe4CpgDngXesILP\nliSt0LIDoKoeA75nnvoXgcvmqRdw/XI/b9K8m1fSWuedwJLUKQNAkjrlw+Ak6RRWcjj4dH+UtHsA\nktQpA0CSOmUASFKnDABJ6pQBIEmdMgAkqVMGgCR1ygCQpE4ZAJLUKQNAkjplAEhSpwwASeqUASBJ\nnTIAJKlTBoAkdcoAkKROGQCS1CkDQJI6ZQBIUqcMAEnq1Jr+UfiV/JizJK11azoAJGmaVvIl9NAt\nrxxjT+Y38UNASbYm+UySuSQ3TvrzJUkDEw2AJGcB7wSuBC4Grk1y8ST7IEkamPQewCXAXFU9VlX/\nCdwJbJtwHyRJTD4ANgCPD80fbjVJ0oSddieBk+wAdrTZf03ymWn2Z4zOB/552p2YAMe5tjjOKcmv\nr2j1bx2l0aQD4AiwcWj+glb7iqraCeycZKcmIcmBqpqddj9Wm+NcWxzn2jbpQ0D3A5uTXJTkbOAa\nYM+E+yBJYsJ7AFV1PMkNwD3AWcCuqjo4yT5IkgYmfg6gqvYCeyf9uaeBNXdYawGOc21xnGtYqmra\nfZAkTYEPg5OkThkAqyTJeUn2JXm0vZ87T5stSf42ycEkDyX5mWn0dSVGGWdr9xdJvpTkg5Pu40os\n9uiSJC9K8p62/ONJNk2+lys3wjh/KMknkhxP8qpp9HEcRhjnLyV5uP1/3J9kpMspz1QGwOq5Edhf\nVZuB/W3+ZM8Cr6uqlwJbgd9Ncs4E+zgOo4wT4DeB106sV2Mw4qNLrgOerqqXALcCK7t6ewpGHOc/\nAq8H3jXZ3o3PiOP8JDBbVd8NvBf4jcn2crIMgNWzDdjdpncDV5/coKr+oaoebdP/BDwJzEysh+Ox\n6DgBqmo/8OVJdWpMRnl0yfD43wtcliQT7OM4LDrOqjpUVQ8B/z2NDo7JKOP8aFU922bvZXCv0ppl\nAKyedVV1tE1/AVh3qsZJLgHOBj672h0bsyWN8wwzyqNLvtKmqo4DzwAvnkjvxqeXR7QsdZzXAX++\nqj2astPuURBnkiQfBr55nkVvGZ6pqkqy4OVWSdYDfwJsr6rT7hvWuMYpnSmSvAaYBX542n1ZTQbA\nClTVjy20LMkTSdZX1dH2B/7JBdp9A/Ah4C1Vde8qdXVFxjHOM9Sijy4ZanM4yQuAbwS+OJnujc0o\n41wLRhpnkh9j8OXmh6vquQn1bSo8BLR69gDb2/R24O6TG7THYXwAuKOq3jvBvo3TouM8g43y6JLh\n8b8K+EideTfX9PKIlkXHmeRlwB8AP1lVa+nLzPyqytcqvBgcB94PPAp8GDiv1WeBP2rTrwH+C3hw\n6LVl2n0f9zjb/N8Ax4B/Z3Ds9Ypp933E8V0F/AODczNvabW3MfgDAfDVwJ8Bc8B9wLdNu8+rNM7v\na/9u/8ZgD+fgtPu8SuP8MPDE0P/HPdPu82q+vBNYkjrlISBJ6pQBIEmdMgAkqVMGgCR1ygCQpE4Z\nAJLUKQNAkjplAEhSp/4H6BicblAWDlgAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYAAAAD8CAYAAAB+UHOxAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAADtxJREFUeJzt3X+o3Xd9x/Hny7T+QJ2t9lK6JOwW\nDNvq2FRC63AMsbONRkwHVupkRlcIQmUVNjQqrPNHIWVg1W3KigmLUqzFOhpsh2a24oT1x22t1TZ2\nvdOUNlQTTVstoiP1vT/OJ90lJt5zcs89595+ng+43O/38/2c73l/SHJe+X6/n+/3pKqQJPXnWdMu\nQJI0HQaAJHXKAJCkThkAktQpA0CSOmUASFKnDABJ6pQBIEmdMgAkqVOnTLuA3+SMM86o2dnZaZch\nSavKXXfd9eOqmlms34oOgNnZWebm5qZdhiStKkkeGqafp4AkqVMGgCR1ygCQpE4ZAJLUKQNAkjpl\nAEhSpwwASeqUASBJnTIAJKlTK/pOYGm5zW6/6aRfu3/H5jFWIk2eRwCS1CkDQJI6ZQBIUqcMAEnq\nlAEgSZ0yACSpUwaAJHXKAJCkThkAktQpA0CSOmUASFKnhg6AJGuSfCvJl9v62UluTzKf5AtJnt3a\nn9PW59v22QX7eH9rfyDJheMejCRpeKMcAVwO7FuwfhVwdVW9FHgMuLS1Xwo81tqvbv1Icg5wCfAy\nYBPwqSRrlla+JOlkDRUASdYBm4HPtPUArwW+2LrsBi5qy1vaOm37+a3/FuC6qvplVf0AmAfOHccg\nJEmjG/YI4OPAe4FftfWXAI9X1ZG2/giwti2vBR4GaNufaP2fbj/OayRJE7ZoACR5I3Cwqu6aQD0k\n2ZZkLsncoUOHJvGWktSlYY4AXg28Kcl+4DoGp34+AZyW5OgXyqwDDrTlA8B6gLb9RcBPFrYf5zVP\nq6prqmpjVW2cmZkZeUCSpOEsGgBV9f6qWldVswwu4t5SVW8DbgXe3LptBW5sy3vaOm37LVVVrf2S\nNkvobGADcMfYRiJJGslSvhLyfcB1ST4KfAvY2dp3Ap9LMg8cZhAaVNV9Sa4H7geOAJdV1VNLeH9J\n0hKMFABV9XXg6235+xxnFk9V/QK4+ASvvxK4ctQiJUnj553AktQpA0CSOmUASFKnDABJ6pQBIEmd\nMgAkqVMGgCR1ygCQpE4ZAJLUKQNAkjplAEhSpwwASerUUp4GKnVtdvtNJ/3a/Ts2j7ES6eR4BCBJ\nnTIAJKlTBoAkdcoAkKROGQCS1CkDQJI6ZQBIUqcMAEnqlAEgSZ0yACSpUwaAJHXKAJCkThkAktQp\nA0CSOmUASFKnDABJ6pQBIEmdMgAkqVMGgCR1ygCQpE4ZAJLUKQNAkjplAEhSpwwASeqUASBJnTIA\nJKlTiwZAkucmuSPJt5Pcl+RDrf3sJLcnmU/yhSTPbu3Paevzbfvsgn29v7U/kOTC5RqUJGlxwxwB\n/BJ4bVX9EfByYFOSVwFXAVdX1UuBx4BLW/9Lgcda+9WtH0nOAS4BXgZsAj6VZM04ByNJGt4pi3Wo\nqgKebKuntp8CXgv8RWvfDfw98GlgS1sG+CLwT0nS2q+rql8CP0gyD5wL/Nc4BqI+zW6/adolSKvW\nUNcAkqxJcg9wENgL/A/weFUdaV0eAda25bXAwwBt+xPASxa2H+c1kqQJGyoAquqpqno5sI7B/9p/\nb7kKSrItyVySuUOHDi3X20hS90aaBVRVjwO3An8MnJbk6CmkdcCBtnwAWA/Qtr8I+MnC9uO8ZuF7\nXFNVG6tq48zMzCjlSZJGMMwsoJkkp7Xl5wGvA/YxCII3t25bgRvb8p62Ttt+S7uOsAe4pM0SOhvY\nANwxroFIkkaz6EVg4Cxgd5ux8yzg+qr6cpL7geuSfBT4FrCz9d8JfK5d5D3MYOYPVXVfkuuB+4Ej\nwGVV9dR4hyNJGtYws4DuBV5xnPbvM7gecGz7L4CLT7CvK4ErRy9TkjRu3gksSZ0yACSpUwaAJHXK\nAJCkThkAktQpA0CSOmUASFKnDABJ6pQBIEmdMgAkqVMGgCR1ygCQpE4ZAJLUqWEeBy1pzJbyXcb7\nd2weYyXqmUcAktQpA0CSOmUASFKnDABJ6pQBIEmdMgAkqVMGgCR1ygCQpE4ZAJLUKQNAkjplAEhS\npwwASeqUASBJnTIAJKlTBoAkdcoAkKROGQCS1CkDQJI6ZQBIUqcMAEnqlAEgSZ0yACSpUwaAJHXK\nAJCkThkAktSpRQMgyfoktya5P8l9SS5v7S9OsjfJg+336a09ST6ZZD7JvUleuWBfW1v/B5NsXb5h\nSZIWc8oQfY4Af1NVdyd5IXBXkr3AO4CvVdWOJNuB7cD7gNcDG9rPecCngfOSvBi4AtgIVNvPnqp6\nbNyD0uoyu/2maZcgdWnRI4CqerSq7m7LPwP2AWuBLcDu1m03cFFb3gJ8tgZuA05LchZwIbC3qg63\nD/29wKaxjkaSNLSRrgEkmQVeAdwOnFlVj7ZNPwTObMtrgYcXvOyR1nai9mPfY1uSuSRzhw4dGqU8\nSdIIhg6AJC8AbgDeU1U/XbitqorBaZ0lq6prqmpjVW2cmZkZxy4lSccxVAAkOZXBh/+1VfWl1vyj\ndmqH9vtgaz8ArF/w8nWt7UTtkqQpGGYWUICdwL6q+tiCTXuAozN5tgI3Lmh/e5sN9CrgiXaq6CvA\nBUlObzOGLmhtkqQpGGYW0KuBvwS+k+Se1vYBYAdwfZJLgYeAt7RtNwNvAOaBnwPvBKiqw0k+AtzZ\n+n24qg6PZRSSpJEtGgBV9U0gJ9h8/nH6F3DZCfa1C9g1SoGSpOXhncCS1CkDQJI6Ncw1AEkryFLu\nnN6/Y/MYK9Fq5xGAJHXKAJCkThkAktQpA0CSOmUASFKnDABJ6pQBIEmdMgAkqVMGgCR1ygCQpE4Z\nAJLUKQNAkjplAEhSpwwASeqUASBJnTIAJKlTBoAkdcoAkKROGQCS1CkDQJI6ZQBIUqdOmXYBemaY\n3X7TtEuQNCKPACSpUwaAJHXKAJCkThkAktQpA0CSOmUASFKnnAYqdWSp03X379g8pkq0EngEIEmd\nMgAkqVMGgCR1ygCQpE4ZAJLUKQNAkjq1aAAk2ZXkYJLvLmh7cZK9SR5sv09v7UnyySTzSe5N8soF\nr9na+j+YZOvyDEeSNKxhjgD+Fdh0TNt24GtVtQH4WlsHeD2wof1sAz4Ng8AArgDOA84FrjgaGpKk\n6Vg0AKrqG8DhY5q3ALvb8m7gogXtn62B24DTkpwFXAjsrarDVfUYsJdfDxVJ0gSd7DWAM6vq0bb8\nQ+DMtrwWeHhBv0da24naJUlTsuSLwFVVQI2hFgCSbEsyl2Tu0KFD49qtJOkYJxsAP2qndmi/D7b2\nA8D6Bf3WtbYTtf+aqrqmqjZW1caZmZmTLE+StJiTDYA9wNGZPFuBGxe0v73NBnoV8EQ7VfQV4IIk\np7eLvxe0NknSlCz6NNAknwdeA5yR5BEGs3l2ANcnuRR4CHhL634z8AZgHvg58E6Aqjqc5CPAna3f\nh6vq2AvLkqQJWjQAquqtJ9h0/nH6FnDZCfazC9g1UnWaqKU+KljS6uKdwJLUKQNAkjplAEhSpwwA\nSeqUASBJnTIAJKlTi04DlaSjljJVeP+OzWOsROPgEYAkdcoAkKROGQCS1CkDQJI6ZQBIUqcMAEnq\nlAEgSZ0yACSpUwaAJHXKAJCkThkAktQpnwX0DOJXOkoahUcAktQpA0CSOmUASFKnDABJ6pQXgSVN\nhF8ms/J4BCBJnTIAJKlTBoAkdcoAkKROGQCS1CkDQJI6ZQBIUqe8D2CF8YFukibFIwBJ6pQBIEmd\n8hSQpBXPx0gsD48AJKlTBoAkdcoAkKROeQ1gGTiVU9JqMPEASLIJ+ASwBvhMVe2YdA2S+uEF5BOb\n6CmgJGuAfwZeD5wDvDXJOZOsQZI0MOkjgHOB+ar6PkCS64AtwP0TrmNRnsaRtNTPgZV+BDHpAFgL\nPLxg/RHgvOV6Mz/EJU3TSj/9tOIuAifZBmxrq08meWCMuz8D+PEY9zdpq71+cAwrxWofw2qvHxYZ\nQ65a0r5/Z5hOkw6AA8D6BevrWtvTquoa4JrlePMkc1W1cTn2PQmrvX5wDCvFah/Daq8fVsYYJn0f\nwJ3AhiRnJ3k2cAmwZ8I1SJKY8BFAVR1J8m7gKwymge6qqvsmWYMkaWDi1wCq6mbg5km/b7Msp5Ym\naLXXD45hpVjtY1jt9cMKGEOqato1SJKmwGcBSVKnuguAJB9Jcm+Se5J8NclvT7umUST5hyTfa2P4\ntySnTbumUSW5OMl9SX6VZNXM5EiyKckDSeaTbJ92PaNKsivJwSTfnXYtJyvJ+iS3Jrm//R26fNo1\njSLJc5PckeTbrf4PTbWe3k4BJfmtqvppW/5r4JyqeteUyxpakguAW9oF9asAqup9Uy5rJEl+H/gV\n8C/A31bV3JRLWlR7jMl/A69jcAPjncBbq2rF3cV+Ikn+FHgS+GxV/cG06zkZSc4Czqqqu5O8ELgL\nuGi1/DkkCfD8qnoyyanAN4HLq+q2adTT3RHA0Q//5vnAqkrAqvpqVR1pq7cxuJdiVamqfVU1zhv8\nJuHpx5hU1f8CRx9jsmpU1TeAw9OuYymq6tGqurst/wzYx+AJA6tCDTzZVk9tP1P7DOouAACSXJnk\nYeBtwN9Nu54l+Cvg36ddRCeO9xiTVfPB80yUZBZ4BXD7dCsZTZI1Se4BDgJ7q2pq9T8jAyDJfyT5\n7nF+tgBU1Qeraj1wLfDu6Vb76xarv/X5IHCEwRhWnGHGIJ2sJC8AbgDec8xR/YpXVU9V1csZHL2f\nm2Rqp+NW3LOAxqGq/mzIrtcyuCfhimUsZ2SL1Z/kHcAbgfNrhV7EGeHPYLVY9DEmmox27vwG4Nqq\n+tK06zlZVfV4kluBTcBULsw/I48AfpMkGxasbgG+N61aTkb7Qp33Am+qqp9Pu56O+BiTFaBdRN0J\n7Kuqj027nlElmTk6cy/J8xhMKpjaZ1CPs4BuAH6XwSyUh4B3VdWq+Z9cknngOcBPWtNtq2kWE0CS\nPwf+EZgBHgfuqaoLp1vV4pK8Afg4//8YkyunXNJIknweeA2Dp1D+CLiiqnZOtagRJfkT4D+B7zD4\nNwzwgfaEgRUvyR8Cuxn8HXoWcH1VfXhq9fQWAJKkge5OAUmSBgwASeqUASBJnTIAJKlTBoAkdcoA\nkKROGQCS1CkDQJI69X9mPdqyIfl7ywAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "all_states = []\n",
    "for _ in range(1000):\n",
    "    all_states.append(env.reset())\n",
    "    done = False\n",
    "    while not done:\n",
    "        s, r, done, _ = env.step(env.action_space.sample())\n",
    "        all_states.append(s)\n",
    "        if done: break\n",
    "            \n",
    "all_states = np.array(all_states)\n",
    "\n",
    "for obs_i in range(env.observation_space.shape[0]):\n",
    "    plt.hist(all_states[:, obs_i], bins=20)\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Binarize environment"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 177,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from gym.core import ObservationWrapper\n",
    "class Binarizer(ObservationWrapper):\n",
    "    \n",
    "    def _observation(self, state):    \n",
    "        \n",
    "        #state = <round state to some amount digits.>\n",
    "        #hint: you can do that with round(x,n_digits)\n",
    "        #you will need to pick a different n_digits for each dimension\n",
    "        state[0] = np.round(state[0], 0)\n",
    "        state[1] = np.round(state[1], 1)\n",
    "        state[2] = np.round(state[2], 2)\n",
    "        state[3] = np.round(state[3], 1)\n",
    "        return tuple(state)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 178,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "env = Binarizer(gym.make(\"CartPole-v0\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 179,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYcAAAD8CAYAAACcjGjIAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAEUdJREFUeJzt3X+s3XV9x/HnSzrQzTmKdB0CWti6\nOeYyxAbJXOYPTCkssZgxVxJHdcz6A5ct2xJx/oHRmemSzYTM4Zh2wH6IDGfoYllXEWOWWKVuyA8d\n9oIS2lVaLeKMGYq+98f5XPe1n3t7b+899wf0+UhOzve8v7/e93Nu7+uc7/d7TlNVSJI09JSlbkCS\ntPwYDpKkjuEgSeoYDpKkjuEgSeoYDpKkjuEgSeoYDpKkjuEgSeqsWOoG5urkk0+uNWvWLHUbkvSE\n8rnPfe5rVbVqpuWesOGwZs0adu/evdRtSNITSpIHZ7Och5UkSR3DQZLUMRwkSR3DQZLUMRwkSR3D\nQZLUMRwkSR3DQZLUMRwkSZ0n7CekpeVqzZUfm/O6X3n3r42xE2nufOcgSeoYDpKkjuEgSeoYDpKk\njuEgSeoYDpKkjuEgSeoYDpKkjuEgSeoYDpKkjuEgSeoYDpKkjuEgSeoYDpKkjuEgSeoYDpKkjuEg\nSeoYDpKkjuEgSeoYDpKkjuEgSeoYDpKkjuEgSerMGA5JTk9ye5IvJLk3ye+1+klJdibZ0+5XtnqS\nXJ1kIsldSc4ZbGtzW35Pks2D+guS3N3WuTpJFuKHlSTNzmzeOTwO/GFVnQWcB1yR5CzgSuC2qloL\n3NYeA1wIrG23LcA1MAoT4CrghcC5wFWTgdKWed1gvQ3z/9EkSXM1YzhU1f6q+o82/T/AF4FTgY3A\n9W2x64GL2/RG4IYa2QWcmOQU4AJgZ1UdqqpHgJ3AhjbvGVW1q6oKuGGwLUnSEjiqcw5J1gDPBz4D\nrK6q/W3WV4HVbfpU4KHBantb7Uj1vVPUJUlLZNbhkOTpwEeA36+qbw7ntVf8NebepuphS5LdSXYf\nPHhwoXcnScesWYVDkh9hFAz/UFX/3MoPt0NCtPsDrb4POH2w+mmtdqT6aVPUO1V1bVWtq6p1q1at\nmk3rkqQ5mM3VSgE+CHyxqv5iMGsbMHnF0WbglkH9snbV0nnAo+3w0w5gfZKV7UT0emBHm/fNJOe1\nfV022JYkaQmsmMUyLwJ+C7g7yZ2t9sfAu4GbklwOPAi8qs3bDlwETADfBl4LUFWHkrwTuKMt946q\nOtSm3wRcBzwNuLXdJElLZMZwqKp/B6b73MH5UyxfwBXTbGsrsHWK+m7geTP1IklaHH5CWpLUMRwk\nSR3DQZLUMRwkSR3DQZLUMRwkSR3DQZLUMRwkSR3DQZLUMRwkSR3DQZLUMRwkSR3DQZLUMRwkSR3D\nQZLUMRwkSR3DQZLUMRwkSR3DQZLUMRwkSR3DQZLUMRwkSR3DQZLUMRwkSR3DQZLUMRwkSR3DQZLU\nMRwkSR3DQZLUMRwkSR3DQZLUMRwkSR3DQZLUMRwkSR3DQZLUMRwkSZ0ZwyHJ1iQHktwzqL09yb4k\nd7bbRYN5b00ykeS+JBcM6htabSLJlYP6GUk+0+ofTnL8OH9ASdLRm807h+uADVPU31tVZ7fbdoAk\nZwGbgF9o6/xVkuOSHAe8D7gQOAu4tC0L8J62rZ8BHgEun88PJEmavxnDoao+BRya5fY2AjdW1WNV\n9WVgAji33Saq6oGq+g5wI7AxSYCXATe39a8HLj7Kn0GSNGbzOefw5iR3tcNOK1vtVOChwTJ7W226\n+jOBb1TV44fVJUlLaK7hcA3w08DZwH7gz8fW0REk2ZJkd5LdBw8eXIxdStIxaU7hUFUPV9X3qur7\nwN8wOmwEsA84fbDoaa02Xf3rwIlJVhxWn26/11bVuqpat2rVqrm0LkmahTmFQ5JTBg9fCUxeybQN\n2JTkhCRnAGuBzwJ3AGvblUnHMzppva2qCrgduKStvxm4ZS49SZLGZ8VMCyT5EPAS4OQke4GrgJck\nORso4CvA6wGq6t4kNwFfAB4Hrqiq77XtvBnYARwHbK2qe9su3gLcmORPgP8EPji2n06SNCczhkNV\nXTpFedo/4FX1LuBdU9S3A9unqD/A/x+WkiQtA35CWpLUMRwkSR3DQZLUMRwkSR3DQZLUMRwkSR3D\nQZLUMRwkSR3DQZLUMRwkSR3DQZLUMRwkSR3DQZLUMRwkSR3DQZLUMRwkSR3DQZLUMRwkSR3DQZLU\nMRwkSR3DQZLUMRwkSR3DQZLUMRwkSR3DQZLUMRwkSR3DQZLUMRwkSR3DQZLUMRwkSR3DQZLUMRwk\nSR3DQZLUMRwkSR3DQZLUMRwkSZ0ZwyHJ1iQHktwzqJ2UZGeSPe1+ZasnydVJJpLcleScwTqb2/J7\nkmwe1F+Q5O62ztVJMu4fUpJ0dGbzzuE6YMNhtSuB26pqLXBbewxwIbC23bYA18AoTICrgBcC5wJX\nTQZKW+Z1g/UO35ckaZHNGA5V9Sng0GHljcD1bfp64OJB/YYa2QWcmOQU4AJgZ1UdqqpHgJ3Ahjbv\nGVW1q6oKuGGwLUnSEpnrOYfVVbW/TX8VWN2mTwUeGiy3t9WOVN87RX1KSbYk2Z1k98GDB+fYuiRp\nJvM+Id1e8dcYepnNvq6tqnVVtW7VqlWLsUtJOibNNRweboeEaPcHWn0fcPpgudNa7Uj106aoS5KW\n0FzDYRswecXRZuCWQf2ydtXSecCj7fDTDmB9kpXtRPR6YEeb980k57WrlC4bbEuStERWzLRAkg8B\nLwFOTrKX0VVH7wZuSnI58CDwqrb4duAiYAL4NvBagKo6lOSdwB1tuXdU1eRJ7jcxuiLqacCt7SZJ\nWkIzhkNVXTrNrPOnWLaAK6bZzlZg6xT13cDzZupDkrR4/IS0JKljOEiSOoaDJKljOEiSOoaDJKlj\nOEiSOoaDJKljOEiSOoaDJKljOEiSOoaDJKljOEiSOoaDJKljOEiSOoaDJKljOEiSOoaDJKljOEiS\nOoaDJKljOEiSOoaDJKljOEiSOoaDJKljOEiSOoaDJKljOEiSOoaDJKljOEiSOoaDJKljOEiSOoaD\nJKljOEiSOoaDJKljOEiSOoaDJKkzr3BI8pUkdye5M8nuVjspyc4ke9r9ylZPkquTTCS5K8k5g+1s\nbsvvSbJ5fj+SJGm+xvHO4aVVdXZVrWuPrwRuq6q1wG3tMcCFwNp22wJcA6MwAa4CXgicC1w1GSiS\npKWxEIeVNgLXt+nrgYsH9RtqZBdwYpJTgAuAnVV1qKoeAXYCGxagL0nSLM03HAr4tySfS7Kl1VZX\n1f42/VVgdZs+FXhosO7eVpuuLklaIivmuf6vVNW+JD8J7EzyX8OZVVVJap77+IEWQFsAnv3sZ49r\ns5Kkw8zrnUNV7Wv3B4CPMjpn8HA7XES7P9AW3wecPlj9tFabrj7V/q6tqnVVtW7VqlXzaV2SdARz\nDockP5bkxyengfXAPcA2YPKKo83ALW16G3BZu2rpPODRdvhpB7A+ycp2Inp9q0mSlsh8DiutBj6a\nZHI7/1hV/5rkDuCmJJcDDwKvastvBy4CJoBvA68FqKpDSd4J3NGWe0dVHZpHX5KkeZpzOFTVA8Av\nTVH/OnD+FPUCrphmW1uBrXPtRZI0Xn5CWpLUMRwkSR3DQZLUMRwkSR3DQZLUMRwkSR3DQZLUMRwk\nSR3DQZLUMRwkSR3DQZLUMRwkSR3DQZLUMRwkSR3DQZLUMRwkSR3DQZLUMRwkSR3DQZLUMRwkSR3D\nQZLUMRwkSR3DQZLUMRwkSR3DQZLUMRwkSR3DQZLUMRwkSR3DQZLUMRwkSR3DQZLUMRwkSR3DQZLU\nMRwkSR3DQZLUMRwkSZ1lEw5JNiS5L8lEkiuXuh9JOpYti3BIchzwPuBC4Czg0iRnLW1XknTsWhbh\nAJwLTFTVA1X1HeBGYOMS9yRJx6zlEg6nAg8NHu9tNUnSElix1A0cjSRbgC3t4beS3DfHTZ0MfG08\nXY2VfR2dJ11fec+YO/lhT7rxWmBP1r6eM5uFlks47ANOHzw+rdV+SFVdC1w7350l2V1V6+a7nXGz\nr6NjX0fHvo7Osd7XcjmsdAewNskZSY4HNgHblrgnSTpmLYt3DlX1eJI3AzuA44CtVXXvErclSces\nZREOAFW1Hdi+SLub96GpBWJfR8e+jo59HZ1juq9U1WLsR5L0BLJczjlIkpaRJ204JPmNJPcm+X6S\nac/sT/e1He3k+Gda/cPtRPk4+jopyc4ke9r9yimWeWmSOwe3/01ycZt3XZIvD+advVh9teW+N9j3\ntkF9Kcfr7CSfbs/3XUl+czBvrOM109e8JDmh/fwTbTzWDOa9tdXvS3LBfPqYQ19/kOQLbXxuS/Kc\nwbwpn9NF6us1SQ4O9v87g3mb2/O+J8nmRe7rvYOevpTkG4N5CzJeSbYmOZDknmnmJ8nVree7kpwz\nmDf+saqqJ+UN+Hng54BPAuumWeY44H7gTOB44PPAWW3eTcCmNv1+4I1j6uvPgCvb9JXAe2ZY/iTg\nEPCj7fF1wCULMF6z6gv41jT1JRsv4GeBtW36WcB+4MRxj9eRfl8Gy7wJeH+b3gR8uE2f1ZY/ATij\nbee4RezrpYPfoTdO9nWk53SR+noN8JdTrHsS8EC7X9mmVy5WX4ct/7uMLpJZ6PH6VeAc4J5p5l8E\n3AoEOA/4zEKO1ZP2nUNVfbGqZvqQ3JRf25EkwMuAm9ty1wMXj6m1jW17s93uJcCtVfXtMe1/Okfb\n1w8s9XhV1Zeqak+b/m/gALBqTPsfms3XvAz7vRk4v43PRuDGqnqsqr4MTLTtLUpfVXX74HdoF6PP\nEi20+XwtzgXAzqo6VFWPADuBDUvU16XAh8a072lV1acYvRCczkbghhrZBZyY5BQWaKyetOEwS9N9\nbcczgW9U1eOH1cdhdVXtb9NfBVbPsPwm+l/Md7W3le9NcsIi9/XUJLuT7Jo81MUyGq8k5zJ6NXj/\noDyu8ZrN17z8YJk2Ho8yGp+F/IqYo9325YxegU6a6jldzL5+vT0/NyeZ/DDsshivdvjtDOATg/JC\njddMput7QcZq2VzKOhdJPg781BSz3lZVtyx2P5OO1NfwQVVVkmkvF2uvCn6R0ec/Jr2V0R/J4xld\n0vYW4B2L2NdzqmpfkjOBTyS5m9EfwDkb83j9HbC5qr7fynMeryejJK8G1gEvHpS757Sq7p96C2P3\nL8CHquqxJK9n9K7rZYu079nYBNxcVd8b1JZyvBbNEzocqurl89zEdF/b8XVGb9lWtFd/U36dx1z6\nSvJwklOqan/7Y3bgCJt6FfDRqvruYNuTr6IfS/K3wB8tZl9Vta/dP5Dkk8DzgY+wxOOV5BnAxxi9\nMNg12Pacx2sKs/mal8ll9iZZAfwEo9+nWX1FzAL2RZKXMwrcF1fVY5P1aZ7Tcfyxm7Gvqvr64OEH\nGJ1jmlz3JYet+8kx9DSrvgY2AVcMCws4XjOZru8FGatj/bDSlF/bUaOzPLczOt4PsBkY1zuRbW17\ns9lud6yz/YGcPM5/MTDllQ0L0VeSlZOHZZKcDLwI+MJSj1d77j7K6HjszYfNG+d4zeZrXob9XgJ8\noo3PNmBTRlcznQGsBT47j16Oqq8kzwf+GnhFVR0Y1Kd8Thexr1MGD18BfLFN7wDWt/5WAuv54XfQ\nC9pX6+25jE7wfnpQW8jxmsk24LJ21dJ5wKPtxc/CjNU4z7YvpxvwSkbH3h4DHgZ2tPqzgO2D5S4C\nvsQo+d82qJ/J6B/vBPBPwAlj6uuZwG3AHuDjwEmtvg74wGC5NYxeETzlsPU/AdzN6I/c3wNPX6y+\ngF9u+/58u798OYwX8Grgu8Cdg9vZCzFeU/2+MDpM9Yo2/dT280+08ThzsO7b2nr3AReO+fd9pr4+\n3v4dTI7Ptpme00Xq60+Be9v+bweeO1j3t9s4TgCvXcy+2uO3A+8+bL0FGy9GLwT3t9/lvYzODb0B\neEObH0b/Kdr9bd/rBuuOfaz8hLQkqXOsH1aSJE3BcJAkdQwHSVLHcJAkdQwHSVLHcJAkdQwHSVLH\ncJAkdf4PEeXNjVKUY/MAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYAAAAD8CAYAAAB+UHOxAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAEc9JREFUeJzt3XGsXvV93/H3pxDaKu1iE+48ZDsz\nUq1UdGoIs4Ao07TFqzGkitnUIKKp3DFL3h/elEiVOmf9wxo0EtGkZkVbmazi1VRpqJc2wmpY6Z2T\nKpo0CJeE0oBDfUuDbAvwba5D2qKmIv3uj+d3kwf33tznsa/vc83v/ZIePed8z++c53ce2f74nPM7\n50lVIUnqzw9NugOSpMkwACSpUwaAJHXKAJCkThkAktQpA0CSOmUASFKnDABJ6pQBIEmdunLSHfhB\nrrnmmtq2bdukuyFJl5Wnn376z6tqaqV26zoAtm3bxuzs7KS7IUmXlSQvjdLOU0CS1CkDQJI6ZQBI\nUqcMAEnqlAEgSZ0yACSpUwaAJHXKAJCkThkAktSpdX0nsLSebTvw+Qte9xv3f3AVeyJdGI8AJKlT\nBoAkdcoAkKROGQCS1KkVAyDJu5M8M/T6dpKPJbk6yUySk+19Y2ufJA8kmUvybJIbh7Y13dqfTDJ9\nKXdMkvSDrRgAVfVCVd1QVTcA/xh4HfgccAA4XlXbgeNtHuA2YHt77QMeBEhyNXAQuBm4CTi4GBqS\npLU37imgncCfVtVLwB7gSKsfAe5o03uAh2vgCWBDkmuBW4GZqlqoqnPADLD7ovdAknRBxg2Au4DP\ntOlNVfVym34F2NSmNwOnhtY53WrL1d8kyb4ks0lm5+fnx+yeJGlUIwdAkquADwH/6/xlVVVArUaH\nqupQVe2oqh1TUyv+pKUk6QKNcwRwG/CVqnq1zb/aTu3Q3s+2+hlg69B6W1ptubokaQLGCYCP8P3T\nPwDHgMWRPNPAo0P1u9tooFuA19qposeBXUk2tou/u1pNkjQBIz0LKMnbgZ8B/t1Q+X7gaJK9wEvA\nna3+GHA7MMdgxNA9AFW1kOQ+4KnW7t6qWrjoPZAkXZCRAqCq/gp453m1bzIYFXR+2wL2L7Odw8Dh\n8bspSVpt3gksSZ0yACSpUwaAJHXKAJCkThkAktQpA0CSOmUASFKnDABJ6pQBIEmdMgAkqVMGgCR1\nygCQpE4ZAJLUKQNAkjplAEhSpwwASeqUASBJnTIAJKlTBoAkdWqkAEiyIclnk3w9yYkk70tydZKZ\nJCfb+8bWNkkeSDKX5NkkNw5tZ7q1P5lk+lLtlCRpZaMeAfwq8PtV9ZPAe4ATwAHgeFVtB463eYDb\ngO3ttQ94ECDJ1cBB4GbgJuDgYmhIktbeigGQ5B3APwUeAqiqv6mqbwF7gCOt2RHgjja9B3i4Bp4A\nNiS5FrgVmKmqhao6B8wAu1d1byRJIxvlCOA6YB74n0m+muTXk7wd2FRVL7c2rwCb2vRm4NTQ+qdb\nbbm6JGkCRgmAK4EbgQer6r3AX/H90z0AVFUBtRodSrIvyWyS2fn5+dXYpCRpCaMEwGngdFU92eY/\nyyAQXm2ndmjvZ9vyM8DWofW3tNpy9TepqkNVtaOqdkxNTY2zL5KkMawYAFX1CnAqybtbaSfwPHAM\nWBzJMw082qaPAXe30UC3AK+1U0WPA7uSbGwXf3e1miRpAq4csd1/AD6d5CrgReAeBuFxNMle4CXg\nztb2MeB2YA54vbWlqhaS3Ac81drdW1ULq7IXkqSxjRQAVfUMsGOJRTuXaFvA/mW2cxg4PE4HJUmX\nhncCS1KnDABJ6pQBIEmdMgAkqVMGgCR1ygCQpE4ZAJLUKQNAkjplAEhSpwwASeqUASBJnTIAJKlT\nBoAkdcoAkKROGQCS1CkDQJI6ZQBIUqcMAEnqlAEgSZ0aKQCSfCPJHyd5Jslsq12dZCbJyfa+sdWT\n5IEkc0meTXLj0HamW/uTSaYvzS5JkkYxzhHAP6+qG6pq8cfhDwDHq2o7cLzNA9wGbG+vfcCDMAgM\n4CBwM3ATcHAxNCRJa+9iTgHtAY606SPAHUP1h2vgCWBDkmuBW4GZqlqoqnPADLD7Ij5fknQRRg2A\nAv4gydNJ9rXapqp6uU2/Amxq05uBU0Prnm615eqSpAm4csR2/6SqziT5+8BMkq8PL6yqSlKr0aEW\nMPsA3vWud63GJiVJSxjpCKCqzrT3s8DnGJzDf7Wd2qG9n23NzwBbh1bf0mrL1c//rENVtaOqdkxN\nTY23N5Kkka0YAEnenuTHF6eBXcDXgGPA4kieaeDRNn0MuLuNBroFeK2dKnoc2JVkY7v4u6vVJEkT\nMMopoE3A55Istv+tqvr9JE8BR5PsBV4C7mztHwNuB+aA14F7AKpqIcl9wFOt3b1VtbBqeyJJGsuK\nAVBVLwLvWaL+TWDnEvUC9i+zrcPA4fG7KUlabd4JLEmdMgAkqVMGgCR1ygCQpE4ZAJLUKQNAkjpl\nAEhSpwwASeqUASBJnTIAJKlTBoAkdcoAkKROGQCS1CkDQJI6ZQBIUqcMAEnqlAEgSZ0yACSpUwaA\nJHXKAJCkTo0cAEmuSPLVJL/X5q9L8mSSuSS/neSqVv/hNj/Xlm8b2sbHW/2FJLeu9s5IkkY3zhHA\nR4ETQ/OfBD5VVT8BnAP2tvpe4Fyrf6q1I8n1wF3ATwG7gV9LcsXFdV+SdKFGCoAkW4APAr/e5gN8\nAPhsa3IEuKNN72nztOU7W/s9wCNV9Z2q+jNgDrhpNXZCkjS+UY8A/ivwi8Dftvl3At+qqjfa/Glg\nc5veDJwCaMtfa+2/V19ine9Jsi/JbJLZ+fn5MXZFkjSOFQMgyc8CZ6vq6TXoD1V1qKp2VNWOqamp\ntfhISerSlSO0eT/woSS3Az8C/D3gV4ENSa5s/8vfApxp7c8AW4HTSa4E3gF8c6i+aHgdSdIaW/EI\noKo+XlVbqmobg4u4X6iqfw18Efi51mwaeLRNH2vztOVfqKpq9bvaKKHrgO3Al1dtTyRJYxnlCGA5\n/xF4JMkvA18FHmr1h4DfTDIHLDAIDarquSRHgeeBN4D9VfXdi/h8SdJFGCsAquoPgT9s0y+yxCie\nqvpr4MPLrP8J4BPjdlKStPq8E1iSOmUASFKnDABJ6pQBIEmdMgAkqVMGgCR1ygCQpE4ZAJLUKQNA\nkjplAEhSpwwASeqUASBJnTIAJKlTBoAkdcoAkKROGQCS1KmL+UUw6bK37cDnJ90FaWI8ApCkThkA\nktSpFQMgyY8k+XKSP0ryXJL/3OrXJXkyyVyS305yVav/cJufa8u3DW3r463+QpJbL9VOSZJWNsoR\nwHeAD1TVe4AbgN1JbgE+CXyqqn4COAfsbe33Auda/VOtHUmuB+4CfgrYDfxakitWc2ckSaNbMQBq\n4C/b7Nvaq4APAJ9t9SPAHW16T5unLd+ZJK3+SFV9p6r+DJgDblqVvZAkjW2kawBJrkjyDHAWmAH+\nFPhWVb3RmpwGNrfpzcApgLb8NeCdw/Ul1pEkrbGRAqCqvltVNwBbGPyv/ScvVYeS7Esym2R2fn7+\nUn2MJHVvrFFAVfUt4IvA+4ANSRbvI9gCnGnTZ4CtAG35O4BvDteXWGf4Mw5V1Y6q2jE1NTVO9yRJ\nYxhlFNBUkg1t+keBnwFOMAiCn2vNpoFH2/SxNk9b/oWqqla/q40Sug7YDnx5tXZEkjSeUe4EvhY4\n0kbs/BBwtKp+L8nzwCNJfhn4KvBQa/8Q8JtJ5oAFBiN/qKrnkhwFngfeAPZX1XdXd3ckSaNaMQCq\n6lngvUvUX2SJUTxV9dfAh5fZ1ieAT4zfTUnSavNOYEnqlAEgSZ0yACSpUwaAJHXKAJCkThkAktQp\nA0CSOmUASFKnDABJ6pQBIEmdMgAkqVMGgCR1ygCQpE6N8jhoad3aduDzk+6CdNkyAKQJuJjg+sb9\nH1zFnqhnngKSpE4ZAJLUKQNAkjplAEhSp1YMgCRbk3wxyfNJnkvy0Va/OslMkpPtfWOrJ8kDSeaS\nPJvkxqFtTbf2J5NMX7rdkiStZJQjgDeAX6iq64FbgP1JrgcOAMerajtwvM0D3AZsb699wIMwCAzg\nIHAzgx+TP7gYGpKktbdiAFTVy1X1lTb9F8AJYDOwBzjSmh0B7mjTe4CHa+AJYEOSa4FbgZmqWqiq\nc8AMsHtV90aSNLKxrgEk2Qa8F3gS2FRVL7dFrwCb2vRm4NTQaqdbbbm6JGkCRg6AJD8G/A7wsar6\n9vCyqiqgVqNDSfYlmU0yOz8/vxqblCQtYaQASPI2Bv/4f7qqfreVX22ndmjvZ1v9DLB1aPUtrbZc\n/U2q6lBV7aiqHVNTU+PsiyRpDKOMAgrwEHCiqn5laNExYHEkzzTw6FD97jYa6BbgtXaq6HFgV5KN\n7eLvrlaTJE3AKM8Cej/w88AfJ3mm1f4TcD9wNMle4CXgzrbsMeB2YA54HbgHoKoWktwHPNXa3VtV\nC6uyF5Kksa0YAFX1f4Ess3jnEu0L2L/Mtg4Dh8fpoCTp0vBOYEnqlAEgSZ0yACSpUwaAJHXKAJCk\nTvmTkJo4f9dXmgwDQLrM+HvCWi2eApKkThkAktQpA0CSOmUASFKnDABJ6pQBIEmdMgAkqVMGgCR1\nyhvBpI5c7F3X3kj21uIRgCR1ygCQpE4ZAJLUKa8BaFX4RE/p8rPiEUCSw0nOJvnaUO3qJDNJTrb3\nja2eJA8kmUvybJIbh9aZbu1PJpm+NLsjSRrVKKeAfgPYfV7tAHC8qrYDx9s8wG3A9vbaBzwIg8AA\nDgI3AzcBBxdDQ5I0GSsGQFV9CVg4r7wHONKmjwB3DNUfroEngA1JrgVuBWaqaqGqzgEz/N1QkSSt\noQu9CLypql5u068Am9r0ZuDUULvTrbZc/e9Isi/JbJLZ+fn5C+yeJGklFz0KqKoKqFXoy+L2DlXV\njqraMTU1tVqblSSd50ID4NV2aof2frbVzwBbh9ptabXl6pKkCbnQADgGLI7kmQYeHarf3UYD3QK8\n1k4VPQ7sSrKxXfzd1WqSpAlZ8T6AJJ8B/hlwTZLTDEbz3A8cTbIXeAm4szV/DLgdmANeB+4BqKqF\nJPcBT7V291bV+ReWJUlraMUAqKqPLLNo5xJtC9i/zHYOA4fH6p0k6ZLxURCS1CkDQJI6ZQBIUqcM\nAEnqlAEgSZ0yACSpU/4egKSRXczvPvh7wuuPAaDv8UddpL54CkiSOmUASFKnDABJ6pQBIEmdMgAk\nqVOOApK0JhxCuv54BCBJnfII4C3EcfySxuERgCR1yiMASeue1w8uDQNgnfE0jqS1suangJLsTvJC\nkrkkB9b68yVJA2saAEmuAP47cBtwPfCRJNevZR8kSQNrfQRwEzBXVS9W1d8AjwB71rgPkiTW/hrA\nZuDU0Pxp4OY17sNIPBcvvTV4AXl56+4icJJ9wL42+5dJXphkf4ZcA/z5pDuxTvhdDPg9DLxlv4d8\ncqzm6+l7+IejNFrrADgDbB2a39Jq31NVh4BDa9mpUSSZraodk+7HeuB3MeD3MOD3MHA5fg9rfQ3g\nKWB7kuuSXAXcBRxb4z5IkljjI4CqeiPJvwceB64ADlfVc2vZB0nSwJpfA6iqx4DH1vpzV8G6Oy01\nQX4XA34PA34PA5fd95CqmnQfJEkT4MPgJKlTBsAYkvyXJF9P8mySzyXZMOk+TUKSDyd5LsnfJrms\nRj2sBh9nMpDkcJKzSb426b5MUpKtSb6Y5Pn29+Kjk+7TqAyA8cwA/6iqfhr4E+DjE+7PpHwN+FfA\nlybdkbXm40ze5DeA3ZPuxDrwBvALVXU9cAuw/3L5M2EAjKGq/qCq3mizTzC4j6E7VXWiqtbLDXpr\nzceZNFX1JWBh0v2YtKp6uaq+0qb/AjjB4KkH654BcOH+LfC/J90JrbmlHmdyWfxl16WXZBvwXuDJ\nyfZkNOvuURCTluT/AP9giUW/VFWPtja/xOCw79Nr2be1NMr3IOn7kvwY8DvAx6rq25PuzygMgPNU\n1b/4QcuT/BvgZ4Gd9RYeQ7vS99CxFR9nov4keRuDf/w/XVW/O+n+jMpTQGNIshv4ReBDVfX6pPuj\nifBxJnqTJAEeAk5U1a9Muj/jMADG89+AHwdmkjyT5H9MukOTkORfJjkNvA/4fJLHJ92ntdIGASw+\nzuQEcLTXx5kk+Qzw/4B3JzmdZO+k+zQh7wd+HvhA+3fhmSS3T7pTo/BOYEnqlEcAktQpA0CSOmUA\nSFKnDABJ6pQBIEmdMgAkqVMGgCR1ygCQpE79f86r/47cZ7PtAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYAAAAD8CAYAAAB+UHOxAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAEqtJREFUeJzt3X+sX/V93/HnK+ZHqnUrptx5nu3M\nrPU0kWo10Z1h6rRmoQEDU02lLAOpiRshuZNAarVqqtNMoiVFcre1rJFSNHd4MVVbQmkjrOCWOQ5T\nF2n8MKnrxFDKLXGGPYNvY0KTsbGZvvfH/ZB+49zr+733fu/32nyeD+mre877fM65nw/I39f3nPP5\nnpuqQpLUn3esdAckSSvDAJCkThkAktQpA0CSOmUASFKnDABJ6pQBIEmdMgAkqVMGgCR16qKV7sC5\nXHHFFbVx48aV7oYkXVCeeeaZP6+qifnandcBsHHjRg4dOrTS3ZCkC0qSrw7TzktAktQpA0CSOmUA\nSFKnDABJ6pQBIEmdmjcAkrwzyVNJ/jjJ0SS/0OqfSvKVJIfba3OrJ8knkkwlOZLkPQPH2p7khfba\nvnzDkiTNZ5hpoG8A76uqbya5GPhCkt9v2/51VT18VvsbgU3tdQ1wH3BNksuBu4BJoIBnkuyrqldH\nMRBJ0sLMewZQM77ZVi9ur3P9HcltwANtvyeAy5KsBW4ADlTV6famfwDYurTuS5IWa6h7AElWJTkM\nnGLmTfzJtumedpnn3iSXtto64KWB3Y+32lx1SdIKGOqbwFX1JrA5yWXAZ5L8APBR4GXgEmA38LPA\n3UvtUJIdwA6Ad73rXUs9nLRsNu58dNH7Htt18wh7Ii3OgmYBVdXXgceBrVV1sl3meQP4z8CW1uwE\nsGFgt/WtNlf97N+xu6omq2pyYmLeR1lIkhZpmFlAE+2TP0m+C3g/8Cftuj5JAtwCfLntsg/4cJsN\ndC3wWlWdBB4Drk+yOslq4PpWkyStgGEuAa0F9iZZxUxgPFRVn03y+SQTQIDDwL9s7fcDNwFTwOvA\nRwCq6nSSjwNPt3Z3V9Xp0Q1FkrQQ8wZAVR0Brp6l/r452hdwxxzb9gB7FthHSdIy8JvAktQpA0CS\nOmUASFKnDABJ6pQBIEmdMgAkqVMGgCR1ygCQpE4ZAJLUKQNAkjplAEhSpwwASeqUASBJnTIAJKlT\nBoAkdcoAkKROGQCS1CkDQJI6ZQBIUqcMAEnqlAEgSZ2aNwCSvDPJU0n+OMnRJL/Q6lcmeTLJVJJP\nJ7mk1S9t61Nt+8aBY3201Z9PcsNyDUqSNL9hzgDeAN5XVT8IbAa2JrkW+CXg3qr6fuBV4PbW/nbg\n1Va/t7UjyVXArcC7ga3AryVZNcrBSJKGN28A1IxvttWL26uA9wEPt/pe4Ja2vK2t07ZflySt/mBV\nvVFVXwGmgC0jGYUkacGGugeQZFWSw8Ap4ADwZ8DXq+pMa3IcWNeW1wEvAbTtrwHfO1ifZR9J0pgN\nFQBV9WZVbQbWM/Op/e8vV4eS7EhyKMmh6enp5fo1ktS9Bc0CqqqvA48D/wi4LMlFbdN64ERbPgFs\nAGjbvwf42mB9ln0Gf8fuqpqsqsmJiYmFdE+StADDzAKaSHJZW/4u4P3Ac8wEwQdas+3AI215X1un\nbf98VVWr39pmCV0JbAKeGtVAJEkLc9H8TVgL7G0zdt4BPFRVn03yLPBgkl8E/gi4v7W/H/iNJFPA\naWZm/lBVR5M8BDwLnAHuqKo3RzscSdKw5g2AqjoCXD1L/UVmmcVTVf8H+OdzHOse4J6Fd1OSNGrD\nnAFIGrGNOx9d9L7Hdt08wp6oZz4KQpI65RmALmhL+SQNfppW3zwDkKROGQCS1CkDQJI6ZQBIUqcM\nAEnqlAEgSZ0yACSpUwaAJHXKAJCkThkAktQpA0CSOmUASFKnDABJ6pQBIEmdMgAkqVMGgCR1ygCQ\npE7NGwBJNiR5PMmzSY4m+alW//kkJ5Icbq+bBvb5aJKpJM8nuWGgvrXVppLsXJ4hSZKGMcyfhDwD\n/ExVfTHJXweeSXKgbbu3qv79YOMkVwG3Au8G/jbwuSR/r23+JPB+4DjwdJJ9VfXsKAYiSVqYeQOg\nqk4CJ9vyN5I8B6w7xy7bgAer6g3gK0mmgC1t21RVvQiQ5MHW1gCQpBWwoHsASTYCVwNPttKdSY4k\n2ZNkdautA14a2O14q81VlyStgKEDIMl3A78L/HRV/QVwH/B9wGZmzhB+eRQdSrIjyaEkh6anp0dx\nSEnSLIYKgCQXM/Pm/5tV9XsAVfVKVb1ZVX8J/Dp/dZnnBLBhYPf1rTZX/dtU1e6qmqyqyYmJiYWO\nR5I0pGFmAQW4H3iuqn5loL52oNmPAV9uy/uAW5NcmuRKYBPwFPA0sCnJlUkuYeZG8b7RDEOStFDD\nzAL6IeBDwJeSHG61nwNuS7IZKOAY8JMAVXU0yUPM3Nw9A9xRVW8CJLkTeAxYBeypqqMjHIskaQGG\nmQX0BSCzbNp/jn3uAe6Zpb7/XPtJksbHbwJLUqcMAEnqlAEgSZ0yACSpUwaAJHXKAJCkThkAktQp\nA0CSOmUASFKnhnkUhPS2tXHnoyvdBWnFeAYgSZ0yACSpUwaAJHXKAJCkThkAktQpA0CSOmUASFKn\nDABJ6pQBIEmdMgAkqVMGgCR1at4ASLIhyeNJnk1yNMlPtfrlSQ4keaH9XN3qSfKJJFNJjiR5z8Cx\ntrf2LyTZvnzDkiTNZ5gzgDPAz1TVVcC1wB1JrgJ2AgerahNwsK0D3Ahsaq8dwH0wExjAXcA1wBbg\nrrdCQ5I0fvMGQFWdrKovtuVvAM8B64BtwN7WbC9wS1veBjxQM54ALkuyFrgBOFBVp6vqVeAAsHWk\no5EkDW1B9wCSbASuBp4E1lTVybbpZWBNW14HvDSw2/FWm6suSVoBQwdAku8Gfhf46ar6i8FtVVVA\njaJDSXYkOZTk0PT09CgOKUmaxVABkORiZt78f7Oqfq+VX2mXdmg/T7X6CWDDwO7rW22u+repqt1V\nNVlVkxMTEwsZiyRpAYaZBRTgfuC5qvqVgU37gLdm8mwHHhmof7jNBroWeK1dKnoMuD7J6nbz9/pW\nkyStgGH+JOQPAR8CvpTkcKv9HLALeCjJ7cBXgQ+2bfuBm4Ap4HXgIwBVdTrJx4GnW7u7q+r0SEYh\nSVqweQOgqr4AZI7N183SvoA75jjWHmDPQjooSVoefhNYkjplAEhSpwwASeqUASBJnTIAJKlTBoAk\ndcoAkKROGQCS1CkDQJI6ZQBIUqcMAEnqlAEgSZ0yACSpUwaAJHXKAJCkThkAktQpA0CSOmUASFKn\nDABJ6pQBIEmdMgAkqVPzBkCSPUlOJfnyQO3nk5xIcri9bhrY9tEkU0meT3LDQH1rq00l2Tn6oUiS\nFmKYM4BPAVtnqd9bVZvbaz9AkquAW4F3t31+LcmqJKuATwI3AlcBt7W2kqQVctF8DarqD5NsHPJ4\n24AHq+oN4CtJpoAtbdtUVb0IkOTB1vbZBfdYkjQSS7kHcGeSI+0S0epWWwe8NNDmeKvNVf8OSXYk\nOZTk0PT09BK6J0k6l8UGwH3A9wGbgZPAL4+qQ1W1u6omq2pyYmJiVIeVJJ1l3ktAs6mqV95aTvLr\nwGfb6glgw0DT9a3GOeqSpBWwqDOAJGsHVn8MeGuG0D7g1iSXJrkS2AQ8BTwNbEpyZZJLmLlRvG/x\n3ZYkLdW8ZwBJfht4L3BFkuPAXcB7k2wGCjgG/CRAVR1N8hAzN3fPAHdU1ZvtOHcCjwGrgD1VdXTk\no5EkDW2YWUC3zVK+/xzt7wHumaW+H9i/oN5JkpaN3wSWpE4ZAJLUKQNAkjq1qGmg0tk27nx00fse\n23XzCHsiaVieAUhSpwwASeqUASBJnTIAJKlTBoAkdcpZQFJnnLGlt3gGIEmdMgAkqVMGgCR1ygCQ\npE4ZAJLUKQNAkjplAEhSpwwASeqUASBJnfKbwFpxS/lmao/876VRmTcAkuwB/hlwqqp+oNUuBz4N\nbASOAR+sqleTBPhV4CbgdeAnquqLbZ/twL9ph/3Fqto72qFoqXxjkfoyzCWgTwFbz6rtBA5W1Sbg\nYFsHuBHY1F47gPvgW4FxF3ANsAW4K8nqpXZekrR48wZAVf0hcPqs8jbgrU/we4FbBuoP1IwngMuS\nrAVuAA5U1emqehU4wHeGiiRpjBZ7E3hNVZ1syy8Da9ryOuClgXbHW22uuiRphSx5FlBVFVAj6AsA\nSXYkOZTk0PT09KgOK0k6y2ID4JV2aYf281SrnwA2DLRb32pz1b9DVe2uqsmqmpyYmFhk9yRJ81ls\nAOwDtrfl7cAjA/UPZ8a1wGvtUtFjwPVJVrebv9e3miRphQwzDfS3gfcCVyQ5zsxsnl3AQ0luB74K\nfLA138/MFNApZqaBfgSgqk4n+TjwdGt3d1WdfWNZkjRG8wZAVd02x6brZmlbwB1zHGcPsGdBvZMk\nLRsfBSFJnTIAJKlTPgtI0lgs5VEjx3bdPMKe6C2eAUhSpzwDOM8s9YFsflKSNCzPACSpUwaAJHXK\nAJCkTnkPQNJ5zxlEy8MzAEnqlAEgSZ0yACSpUwaAJHXKAJCkThkAktQpp4FKGtpSH1Wi84tnAJLU\nKQNAkjrlJaC3GU/RJQ3LMwBJ6pQBIEmdWlIAJDmW5EtJDic51GqXJzmQ5IX2c3WrJ8knkkwlOZLk\nPaMYgCRpcUZxBvBPq2pzVU229Z3AwaraBBxs6wA3Apvaawdw3wh+tyRpkZbjJvA24L1teS/wX4Gf\nbfUHqqqAJ5JclmRtVZ1chj4smY+flfR2t9QzgAL+S5JnkuxotTUDb+ovA2va8jrgpYF9j7fat0my\nI8mhJIemp6eX2D1J0lyWegbwj6vqRJK/CRxI8ieDG6uqktRCDlhVu4HdAJOTkwvaV5JG7e18NWBJ\nZwBVdaL9PAV8BtgCvJJkLUD7eao1PwFsGNh9fatJklbAos8Akvw14B1V9Y22fD1wN7AP2A7saj8f\nabvsA+5M8iBwDfDa+Xr9f6n8MpZ0/vDf49yWcgloDfCZJG8d57eq6g+SPA08lOR24KvAB1v7/cBN\nwBTwOvCRJfxuSdISLToAqupF4AdnqX8NuG6WegF3LPb3SZJGy28CS1KnDABJ6pQBIEmdMgAkqVMG\ngCR1ygCQpE4ZAJLUKQNAkjplAEhSpwwASeqUASBJnTIAJKlTBoAkdcoAkKROGQCS1CkDQJI6ZQBI\nUqcMAEnq1FL+JvB5zz8GLUlz8wxAkjo19jOAJFuBXwVWAf+pqnaNuw+SNA5LuQpxbNfNI+zJ7MZ6\nBpBkFfBJ4EbgKuC2JFeNsw+SpBnjvgS0BZiqqher6v8CDwLbxtwHSRLjD4B1wEsD68dbTZI0Zufd\nLKAkO4AdbfWbSZ5fyf6M0BXAn690J8agl3FCP2PtZZxwHo01v7Sk3f/OMI3GHQAngA0D6+tb7Vuq\najewe5ydGockh6pqcqX7sdx6GSf0M9Zexgl9jRXGfwnoaWBTkiuTXALcCuwbcx8kSYz5DKCqziS5\nE3iMmWmge6rq6Dj7IEmaMfZ7AFW1H9g/7t97HnjbXdaaQy/jhH7G2ss4oa+xkqpa6T5IklaAj4KQ\npE4ZAMskyeVJDiR5of1cPUubzUn+e5KjSY4k+Rcr0delGGacrd0fJPl6ks+Ou49LlWRrkueTTCXZ\nOcv2S5N8um1/MsnG8fdy6YYY5z9J8sUkZ5J8YCX6OCpDjPVfJXm2/bs8mGSoaZUXGgNg+ewEDlbV\nJuBgWz/b68CHq+rdwFbgPyS5bIx9HIVhxgnw74APja1XIzLk40tuB16tqu8H7gWWNoN7BQw5zv8B\n/ATwW+Pt3WgNOdY/Aiar6h8ADwP/dry9HA8DYPlsA/a25b3ALWc3qKo/raoX2vL/BE4BE2Pr4WjM\nO06AqjoIfGNcnRqhYR5fMvjf4GHguiQZYx9HYd5xVtWxqjoC/OVKdHCEhhnr41X1elt9gpnvLL3t\nGADLZ01VnWzLLwNrztU4yRbgEuDPlrtjI7agcV6Ahnl8ybfaVNUZ4DXge8fSu9Hp6TEtCx3r7cDv\nL2uPVsh59yiIC0mSzwF/a5ZNHxtcqapKMud0qyRrgd8AtlfVeffpalTjlC40SX4cmAR+eKX7shwM\ngCWoqh+Za1uSV5KsraqT7Q3+1Bzt/gbwKPCxqnpimbq6JKMY5wVs3seXDLQ5nuQi4HuAr42neyMz\nzDjfLoYaa5IfYeZDzg9X1Rtj6ttYeQlo+ewDtrfl7cAjZzdoj8P4DPBAVT08xr6N0rzjvMAN8/iS\nwf8GHwA+XxfeF2x6ekzLvGNNcjXwH4Efraq324eav1JVvpbhxcw14IPAC8DngMtbfZKZv4QG8OPA\n/wMOD7w2r3TfRz3Otv7fgGngfzNzzfWGle77AsZ4E/CnzNyf+Vir3c3MmwPAO4HfAaaAp4C/u9J9\nXqZx/sP2/+5/MXOGc3Sl+7yMY/0c8MrAv8t9K93n5Xj5TWBJ6pSXgCSpUwaAJHXKAJCkThkAktQp\nA0CSOmUASFKnDABJ6pQBIEmd+v80dTjsgAo0oAAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYAAAAD8CAYAAAB+UHOxAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAFBpJREFUeJzt3X+s3fV93/HnKy6QKMkKlDtEbWdG\nrdeOVKsTeYYp1ZTBAgaqmkhNBKoaN0NyI4FGpG6NSaXRJLVEtDW0mRI0OryYisWxQiIscEtcIMry\nBz8McQiGMO7AyLYIvo2BBKEymbz3x/k4OzP3+p5z7/E99/r7fEhX9/t9fz/fc95fAffF93eqCklS\n97xt3A1IksbDAJCkjjIAJKmjDABJ6igDQJI6ygCQpI4yACSpowwASeooA0CSOuoXxt3AiZxzzjm1\natWqcbchSUvKY4899vdVNTHbuEUdAKtWrWLPnj3jbkOSlpQkLwwyzkNAktRRBoAkdZQBIEkdZQBI\nUkcZAJLUUQaAJHWUASBJHWUASFJHGQCS1FGL+k5g6WRbtfneOa+7/+YrR9iJtPDcA5Ckjho4AJIs\nS/K9JPe0+fOTPJxkMsnXkpze6me0+cm2fFXfZ9zY6s8kuWzUGyNJGtwwewA3AE/3zX8euKWqfhV4\nGbi21a8FXm71W9o4klwAXA28F1gPfDnJsvm1L0maq4ECIMkK4Ergv7X5ABcDX29DtgFXtekNbZ62\n/JI2fgOwvareqKrngUlg3Sg2QpI0vEH3AP4C+GPgZ23+l4BXqupomz8ILG/Ty4EDAG35q238z+vT\nrCNJWmCzBkCS3wYOV9VjC9APSTYl2ZNkz9TU1EJ8pSR10iB7AB8AfifJfmA7vUM/fwmcmeTYZaQr\ngENt+hCwEqAt/0Xgx/31adb5uaq6rarWVtXaiYlZX2gjSZqjWQOgqm6sqhVVtYreSdwHqur3gAeB\n323DNgJ3t+mdbZ62/IGqqla/ul0ldD6wGnhkZFsiSRrKfG4E+xSwPcmfAd8Dbm/124G/TjIJHKEX\nGlTVviQ7gKeAo8B1VfXmPL5fkjQPQwVAVX0b+Habfo5pruKpqn8APjLD+luALcM2KUkaPe8ElqSO\nMgAkqaMMAEnqKANAkjrKAJCkjjIAJKmjDABJ6igDQJI6ygCQpI4yACSpowwASeooA0CSOsoAkKSO\nMgAkqaMMAEnqKANAkjpqkJfCvz3JI0m+n2Rfks+0+leSPJ9kb/tZ0+pJ8sUkk0meSPL+vs/amOTZ\n9rNxpu+UJJ18g7wR7A3g4qp6LclpwHeT/E1b9h+q6uvHjb+c3vt+VwMXArcCFyY5G7gJWAsU8FiS\nnVX18ig2RJI0nEFeCl9V9VqbPa391AlW2QDc0dZ7CDgzyXnAZcDuqjrS/ujvBtbPr31J0lwNdA4g\nybIke4HD9P6IP9wWbWmHeW5JckarLQcO9K1+sNVmqkuSxmCgAKiqN6tqDbACWJfkN4AbgV8H/gVw\nNvCpUTSUZFOSPUn2TE1NjeIjJUnTGOoqoKp6BXgQWF9VL7bDPG8A/x1Y14YdAlb2rbai1WaqH/8d\nt1XV2qpaOzExMUx7kqQhDHIV0ESSM9v0O4APAT9sx/VJEuAq4Mm2yk7gY+1qoIuAV6vqReA+4NIk\nZyU5C7i01SRJYzDIVUDnAduSLKMXGDuq6p4kDySZAALsBT7Rxu8CrgAmgdeBjwNU1ZEknwMebeM+\nW1VHRrcpkqRhzBoAVfUE8L5p6hfPML6A62ZYthXYOmSPkqSTwDuBJamjDABJ6igDQJI6ygCQpI4y\nACSpowwASeooA0CSOsoAkKSOMgAkqaMMAEnqKANAkjrKAJCkjjIAJKmjBnkctLRordp875L87v03\nXznCTqS5cQ9AkjrKAJCkjhrklZBvT/JIku8n2ZfkM61+fpKHk0wm+VqS01v9jDY/2Zav6vusG1v9\nmSSXnayNkiTNbpA9gDeAi6vqN4E1wPr2rt/PA7dU1a8CLwPXtvHXAi+3+i1tHEkuAK4G3gusB77c\nXjMpSRqDWQOgel5rs6e1nwIuBr7e6tvovRgeYEObpy2/pL04fgOwvareqKrn6b0zeN1ItkKSNLSB\nzgEkWZZkL3AY2A38b+CVqjrahhwElrfp5cABgLb8VeCX+uvTrCNJWmADBUBVvVlVa4AV9P6v/ddP\nVkNJNiXZk2TP1NTUyfoaSeq8oa4CqqpXgAeBfwmcmeTYfQQrgENt+hCwEqAt/0Xgx/31adbp/47b\nqmptVa2dmJgYpj1J0hAGuQpoIsmZbfodwIeAp+kFwe+2YRuBu9v0zjZPW/5AVVWrX92uEjofWA08\nMqoNkSQNZ5A7gc8DtrUrdt4G7Kiqe5I8BWxP8mfA94Db2/jbgb9OMgkcoXflD1W1L8kO4CngKHBd\nVb052s2RJA1q1gCoqieA901Tf45pruKpqn8APjLDZ20BtgzfpiRp1LwTWJI6ygCQpI4yACSpowwA\nSeooA0CSOsoAkKSOMgAkqaMMAEnqKANAkjrKAJCkjjIAJKmjDABJ6igDQJI6ygCQpI4yACSpowwA\nSeooA0CSOmqQdwKvTPJgkqeS7EtyQ6v/aZJDSfa2nyv61rkxyWSSZ5Jc1ldf32qTSTafnE2SJA1i\nkHcCHwX+qKoeT/Ju4LEku9uyW6rqP/cPTnIBvfcAvxf4ZeDvkvzTtvhL9F4qfxB4NMnOqnpqFBsi\nSRrOIO8EfhF4sU3/NMnTwPITrLIB2F5VbwDPt5fDH3t38GR7lzBJtrexBoAkjcFQ5wCSrKL3gviH\nW+n6JE8k2ZrkrFZbDhzoW+1gq81UP/47NiXZk2TP1NTUMO1JkoYwcAAkeRdwF/DJqvoJcCvwK8Aa\nensIfz6KhqrqtqpaW1VrJyYmRvGRkqRpDHIOgCSn0fvjf2dVfQOgql7qW/5XwD1t9hCwsm/1Fa3G\nCeqSpAU2yFVAAW4Hnq6qL/TVz+sb9mHgyTa9E7g6yRlJzgdWA48AjwKrk5yf5HR6J4p3jmYzJEnD\nGmQP4APA7wM/SLK31T4NXJNkDVDAfuAPAapqX5Id9E7uHgWuq6o3AZJcD9wHLAO2VtW+EW6LJGkI\ng1wF9F0g0yzadYJ1tgBbpqnvOtF6kqSF453AktRRBoAkdZQBIEkdZQBIUkcZAJLUUQPdCCadTKs2\n3zvuFhbcfLZ5/81XjrATdZl7AJLUUQaAJHWUASBJHWUASFJHGQCS1FEGgCR1lAEgSR3lfQAaiS5e\nyy8tde4BSFJHGQCS1FGDvBJyZZIHkzyVZF+SG1r97CS7kzzbfp/V6knyxSSTSZ5I8v6+z9rYxj+b\nZOPJ2yxJ0mwG2QM4CvxRVV0AXARcl+QCYDNwf1WtBu5v8wCX03sP8GpgE3Ar9AIDuAm4EFgH3HQs\nNCRJC2/WAKiqF6vq8Tb9U+BpYDmwAdjWhm0DrmrTG4A7quch4Mz2AvnLgN1VdaSqXgZ2A+tHujWS\npIENdQ4gySrgfcDDwLlV9WJb9CPg3Da9HDjQt9rBVpupfvx3bEqyJ8meqampYdqTJA1h4ABI8i7g\nLuCTVfWT/mVVVUCNoqGquq2q1lbV2omJiVF8pCRpGgMFQJLT6P3xv7OqvtHKL7VDO7Tfh1v9ELCy\nb/UVrTZTXZI0BoNcBRTgduDpqvpC36KdwLEreTYCd/fVP9auBroIeLUdKroPuDTJWe3k76WtJkka\ng0HuBP4A8PvAD5LsbbVPAzcDO5JcC7wAfLQt2wVcAUwCrwMfB6iqI0k+Bzzaxn22qo6MZCskSUOb\nNQCq6rtAZlh8yTTjC7huhs/aCmwdpkFJ0snhncCS1FEGgCR1lAEgSR1lAEhSRxkAktRRBoAkdZQB\nIEkdZQBIUkcZAJLUUQaAJHWUASBJHWUASFJHGQCS1FGDPA5a0iKyavO981p//81XjqgTLXXuAUhS\nRxkAktRRg7wScmuSw0me7Kv9aZJDSfa2nyv6lt2YZDLJM0ku66uvb7XJJJtHvymSpGEMsgfwFWD9\nNPVbqmpN+9kFkOQC4GrgvW2dLydZlmQZ8CXgcuAC4Jo2VpI0JoO8EvI7SVYN+HkbgO1V9QbwfJJJ\nYF1bNllVzwEk2d7GPjV0x5KkkZjPOYDrkzzRDhGd1WrLgQN9Yw622kz1t0iyKcmeJHumpqbm0Z4k\n6UTmGgC3Ar8CrAFeBP58VA1V1W1Vtbaq1k5MTIzqYyVJx5nTfQBV9dKx6SR/BdzTZg8BK/uGrmg1\nTlCXJI3BnPYAkpzXN/th4NgVQjuBq5OckeR8YDXwCPAosDrJ+UlOp3eieOfc25YkzdesewBJvgp8\nEDgnyUHgJuCDSdYABewH/hCgqvYl2UHv5O5R4LqqerN9zvXAfcAyYGtV7Rv51kiSBjbIVUDXTFO+\n/QTjtwBbpqnvAnYN1Z0k6aTxTmBJ6igDQJI6ygCQpI4yACSpo3wfgNQx83mfgO8SOLW4ByBJHWUA\nSFJHGQCS1FEGgCR1lAEgSR1lAEhSRxkAktRRBoAkdZQBIEkdZQBIUkcZAJLUUbMGQJKtSQ4nebKv\ndnaS3Umebb/PavUk+WKSySRPJHl/3zob2/hnk2w8OZsjSRrUIHsAXwHWH1fbDNxfVauB+9s8wOX0\n3gO8GtgE3Aq9wKD3KskLgXXATcdCQ5I0HrMGQFV9BzhyXHkDsK1NbwOu6qvfUT0PAWe2F8hfBuyu\nqiNV9TKwm7eGiiRpAc31cdDnVtWLbfpHwLltejlwoG/cwVabqa5FZD6PCZa09Mz7JHBVFVAj6AWA\nJJuS7EmyZ2pqalQfK0k6zlwD4KV2aIf2+3CrHwJW9o1b0Woz1d+iqm6rqrVVtXZiYmKO7UmSZjPX\nANgJHLuSZyNwd1/9Y+1qoIuAV9uhovuAS5Oc1U7+XtpqkqQxmfUcQJKvAh8EzklykN7VPDcDO5Jc\nC7wAfLQN3wVcAUwCrwMfB6iqI0k+Bzzaxn22qo4/sSxJWkCzBkBVXTPDokumGVvAdTN8zlZg61Dd\nSZJOGu8ElqSOMgAkqaMMAEnqKANAkjrKAJCkjjIAJKmjDABJ6igDQJI6ygCQpI4yACSpowwASeoo\nA0CSOsoAkKSOMgAkqaMMAEnqKANAkjpq1hfCnEiS/cBPgTeBo1W1NsnZwNeAVcB+4KNV9XKSAH9J\n741hrwN/UFWPz+f79VarNt877hYkLRGj2AP411W1pqrWtvnNwP1VtRq4v80DXA6sbj+bgFtH8N2S\npDma1x7ADDbQe4cwwDbg28CnWv2O9trIh5KcmeS89tJ4SUvAfPYw99985Qg70SjMdw+ggG8leSzJ\nplY7t++P+o+Ac9v0cuBA37oHW02SNAbz3QP4rao6lOQfA7uT/LB/YVVVkhrmA1uQbAJ4z3veM8/2\nJEkzmdceQFUdar8PA98E1gEvJTkPoP0+3IYfAlb2rb6i1Y7/zNuqam1VrZ2YmJhPe5KkE5hzACR5\nZ5J3H5sGLgWeBHYCG9uwjcDdbXon8LH0XAS86vF/SRqf+RwCOhf4Zu/qTn4B+B9V9bdJHgV2JLkW\neAH4aBu/i94loJP0LgP9+Dy+W5I0T3MOgKp6DvjNaeo/Bi6Zpl7AdXP9PknSaHknsCR1lAEgSR1l\nAEhSRxkAktRRJ+NREJoHH+YmaaEYAJIWhM8RWnw8BCRJHWUASFJHGQCS1FEGgCR1lAEgSR1lAEhS\nR3kZ6EngtfySlgL3ACSpo9wDkLToeRPZyeEegCR11IIHQJL1SZ5JMplk80J/vySpZ0EPASVZBnwJ\n+BBwEHg0yc6qemoh+xiEJ3KlU8N8/1s+lQ8hLfQewDpgsqqeq6r/A2wHNixwD5IkFv4k8HLgQN/8\nQeDCk/Vl/l+8pPk6lU9AL7qrgJJsAja12deSPDOPjzsH+Pv5dzU29j9+S30b7H+M8nlgPNvwTwYZ\ntNABcAhY2Te/otV+rqpuA24bxZcl2VNVa0fxWeNg/+O31LfB/sdvMW/DQp8DeBRYneT8JKcDVwM7\nF7gHSRILvAdQVUeTXA/cBywDtlbVvoXsQZLUs+DnAKpqF7Brgb5uJIeSxsj+x2+pb4P9j9+i3YZU\n1bh7kCSNgY+CkKSOOqUDIMnnkjyRZG+SbyX55XH3NKwk/ynJD9t2fDPJmePuaRhJPpJkX5KfJVmU\nV0JMZ6k/siTJ1iSHkzw57l7mIsnKJA8mear9+3PDuHsaRpK3J3kkyfdb/58Zd0/TOaUPASX5R1X1\nkzb974ALquoTY25rKEkuBR5oJ9A/D1BVnxpzWwNL8s+AnwH/Ffj3VbVnzC3Nqj2y5H/R98gS4JrF\n+MiSmST5V8BrwB1V9Rvj7mdYSc4Dzquqx5O8G3gMuGqp/DNIEuCdVfVaktOA7wI3VNVDY27t/3NK\n7wEc++PfvBNYcmlXVd+qqqNt9iF6904sGVX1dFXN52a+cVjyjyypqu8AR8bdx1xV1YtV9Xib/inw\nNL0nCSwJ1fNamz2t/Sy6vz+ndAAAJNmS5ADwe8B/HHc/8/Rvgb8ZdxMdMN0jS5bMH59TTZJVwPuA\nh8fbyXCSLEuyFzgM7K6qRdf/kg+AJH+X5MlpfjYAVNWfVNVK4E7g+vF2O73ZtqGN+RPgKL3tWFQG\n6V+aiyTvAu4CPnncHv2iV1VvVtUaenvt65IsukNxi+5ZQMOqqn8z4NA76d1/cNNJbGdOZtuGJH8A\n/DZwSS3CkzZD/DNYKmZ9ZIlOvnbs/C7gzqr6xrj7mauqeiXJg8B6YFGdlF/yewAnkmR13+wG4Ifj\n6mWukqwH/hj4nap6fdz9dISPLBmzdhL1duDpqvrCuPsZVpKJY1fsJXkHvQsKFt3fn1P9KqC7gF+j\ndxXKC8AnqmpJ/Z9ckkngDODHrfTQUrqSKcmHgf8CTACvAHur6rLxdjW7JFcAf8H/e2TJljG3NJQk\nXwU+SO9JlC8BN1XV7WNtaghJfgv4n8AP6P33C/Dp9iSBRS/JPwe20fv3523Ajqr67Hi7eqtTOgAk\nSTM7pQ8BSZJmZgBIUkcZAJLUUQaAJHWUASBJHWUASFJHGQCS1FEGgCR11P8FlPP52bZVaeIAAAAA\nSUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "all_states = []\n",
    "for _ in range(1000):\n",
    "    all_states.append(env.reset())\n",
    "    done = False\n",
    "    while not done:\n",
    "        s, r, done, _ = env.step(env.action_space.sample())\n",
    "        all_states.append(s)\n",
    "        if done: break\n",
    "            \n",
    "all_states = np.array(all_states)\n",
    "\n",
    "for obs_i in range(env.observation_space.shape[0]):\n",
    "    \n",
    "    plt.hist(all_states[:,obs_i],bins=20)\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Learn binarized policy\n",
    "\n",
    "Now let's train a policy that uses binarized state space.\n",
    "\n",
    "__Tips:__ \n",
    "* If your binarization is too coarse, your agent may fail to find optimal policy. In that case, change binarization. \n",
    "* If your binarization is too fine-grained, your agent will take much longer than 1000 steps to converge. You can either increase number of iterations and decrease epsilon decay or change binarization.\n",
    "* Having 10^3 ~ 10^4 distinct states is recommended (`len(QLearningAgent._qvalues)`), but not required.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 180,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "agent = QLearningAgent(alpha=0.5, epsilon=0.25, discount=0.99,\n",
    "                       get_legal_actions = lambda s: range(n_actions))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 181,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "eps = 0.25 mean reward = 147.4\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXoAAAD8CAYAAAB5Pm/hAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAIABJREFUeJzt3XmcFNW5//HPMzMMy7DDMKzDAAIK\nLiwjoLKKuMXEJYlRiaLXyHXhxu2nwfjLcvO6udebuNwYEw2JRs0PjSaKGmMSEbd4jRhQg4ggoBhB\nhFEDYhQVeH5/dDX09HRP79Mz1d/36zWv6T5V3fXUVM/Tp06dc8rcHRERCa+yYgcgIiKFpUQvIhJy\nSvQiIiGnRC8iEnJK9CIiIadELyISckr0IiIhp0QvIhJySvQiIiFXUewAAHr37u11dXXFDkNEpE1Z\nvnz5u+5enWq9VpHo6+rqWLZsWbHDEBFpU8zszXTWU9ONiEjIKdGLiIScEr2ISMgp0YuIhJwSvYhI\nyKVM9GY2yMyeMLNVZvaKmV0clPc0s8Vmtjb43SMoNzO70czWmdkKMxtX6J0QEZHk0qnR7wIud/dR\nwCTgIjMbBcwHlrj7cGBJ8BzgOGB48DMXuDnvUYuISNpS9qN3983A5uDxDjN7FRgAnAhMD1a7A3gS\n+EZQfqdH7lH4nJl1N7N+wfu0eU+s2cqVv13B1ccfwDsf7OTtbR9z3/KNLDxvEts++pROlRVc++ga\nnn/jfQBmT6xl4dK/87t5kxncuxOPrdrCohc3cdMZ4+jWsR13/mUD337wFS6bNYIyg2sffQ2A3p0r\nOXp0Xw4f1ou6XlW89f5HXLDwBS6YPoyHXnqbUf27snjVFqoqy/nnp7s54eB+PLwi9Z94XG13Pti5\ni43/+Iidn+1Ja59H1nRhzZYdXDxzOD9asjbl+v26dWDz9p1pvbdIqfveiaM567C6gm7DMrlnrJnV\nAU8DBwJ/d/fuQbkB/3D37mb2MHCNuz8TLFsCfMPdl8W911wiNX5qa2vHv/lmWv3+i65u/u+zfu2s\nUTUsXrVl7/MN13wup/cTkXDYcM3nsnqdmS139/pU66V9MdbMOgP3AZe4+wexy4Lae0Z3GXf3Be5e\n7+711dUpR/CGwubtHxc7BBEpQWklejNrRyTJL3T3+4PiLWbWL1jeD9galG8CBsW8fGBQJiIiRZBO\nrxsDbgVedffrYxY9BMwJHs8BHowpPyvofTMJ2B6W9nkRkbYonUnNjgDOBF42s5eCsm8C1wD3mtm5\nwJvAqcGyR4DjgXXAR8A5eY04RF56a1uxQxCREpBOr5tnAEuyeGaC9R24KMe4SsJJP/nfYocgIiVA\nI2NFREJOiV5EJOSU6FvQex9+WuwQRKQEKdG3II0WFZFiUKIXEQk5JXoRkZBTohcRCTklehGRkFOi\nFxEJOSV6EZGQU6IXEQk5Jfo0/eovG3STEBFpk5To03Td4teKHYKISFaU6EVEQk6JXkQk5JToRURC\nLp1bCd5mZlvNbGVM2T1m9lLwsyF65ykzqzOzj2OW3VLI4PPtPx5exfVqixeRkEnnVoK3AzcBd0YL\n3P0r0cdmdh2wPWb99e4+Jl8BtqRfPPMGAJfNGlHkSERE8iedWwk+bWZ1iZYFNw4/FTgyv2G1Pts+\n+qzYIYiIZCXXNvopwBZ3XxtTNsTMXjSzp8xsSo7vLyIiOUqn6aY5pwN3xzzfDNS6+3tmNh54wMxG\nu/sH8S80s7nAXIDa2tocwxARkWSyrtGbWQVwCnBPtMzdP3H394LHy4H1QMIGb3df4O717l5fXV2d\nbRgF9dqWHbo4KyJtXi5NN0cBq919Y7TAzKrNrDx4PBQYDryeW4jF8+Vb/sKNS9ayY6fa50Wk7Uqn\ne+XdwF+AkWa20czODRadRuNmG4CpwIqgu+VvgfPd/f18BtySdu3eA0DkmrOISNuUTq+b05OUn52g\n7D7gvtzDEhGRfNHIWBGRkFOiFxEJOSV6EZGQU6IXEQk5JXoRkZBTohcRCTklehGRkFOiT4O7FzsE\nEZGsKdE3QyNiRSQMlOhFREJOiV5EJOSU6EVEQk6JXkQk5JToRURCToleRCTklOhFREJOiV5EJOTS\nuZXgbWa21cxWxpR918w2mdlLwc/xMcuuMrN1ZrbGzI4pVOAiIpKelLcSBG4HbgLujCu/wd2vjS0w\ns1FE7iU7GugPPGZmI9x9dx5ibTGX3fsS97+wic7t0/nziIi0bilr9O7+NJDuDb5PBH7t7p+4+xvA\nOmBCDvEVxf0vbCp2CCIieZNLG/08M1sRNO30CMoGAG/FrLMxKBMRkSLJNtHfDAwDxgCbgesyfQMz\nm2tmy8xsWUNDQ5ZhiIhIKlklenff4u673X0P8HP2Nc9sAgbFrDowKEv0Hgvcvd7d66urq7MJo8WM\n+d7iYocgIpK1rBK9mfWLeXoyEO2R8xBwmpm1N7MhwHDg+dxCLL7dezQfvYi0XSm7lZjZ3cB0oLeZ\nbQS+A0w3szGAAxuAfwVw91fM7F5gFbALuKit9bgREQmblIne3U9PUHxrM+t/H/h+LkGJiEj+aGSs\niEjIKdE3QzcSFJEwUKIXEQk5JXoRkZBTohcRCTklehGRkFOiFxEJOSV6EZGQU6IXEQk5JfpmaIYb\nEQkDJXoRkZBTom+GRsaKSBgo0Tdjxye7ih2CiEjOlOhFREJOiV5EJOSU6EVEQk6JXkQk5FImejO7\nzcy2mtnKmLIfmtlqM1thZovMrHtQXmdmH5vZS8HPLYUMXkREUkunRn87cGxc2WLgQHc/GHgNuCpm\n2Xp3HxP8nJ+fMEVEJFspE727Pw28H1f2qLtH+x4+BwwsQGwiIpIH+Wij/xfgDzHPh5jZi2b2lJlN\nSfYiM5trZsvMbFlDQ0MewhARkURySvRmdjWwC1gYFG0Gat19LHAZcJeZdU30Wndf4O717l5fXV2d\nSxgiItKMrBO9mZ0NnADMdncHcPdP3P294PFyYD0wIg9xiohIlrJK9GZ2LHAl8AV3/yimvNrMyoPH\nQ4HhwOv5CFRERLJTkWoFM7sbmA70NrONwHeI9LJpDyw2M4Dngh42U4HvmdlnwB7gfHd/P+Ebi4hI\ni0iZ6N399ATFtyZZ9z7gvlyDEhGR/NHIWBGRkFOiFxEJOSV6EZGQU6IXEQk5JXoRkZBTohcRCTkl\nehGRkFOiFxEJOSV6EZGQU6IXEQk5JXoRkZBTohcRCTklehGRkFOiFxEJOSV6EZGQU6IXEQk5JXoR\nkZBLK9Gb2W1mttXMVsaU9TSzxWa2NvjdIyg3M7vRzNaZ2QozG1eo4EVEJLV0a/S3A8fGlc0Hlrj7\ncGBJ8BzgOCI3BR8OzAVuzj3Mwlu49M1ihyAiUhBpJXp3fxqIv8n3icAdweM7gJNiyu/0iOeA7mbW\nLx/BFsqmbR9z9aKVqVcUEWmDcmmjr3H3zcHjd4Ca4PEA4K2Y9TYGZY2Y2VwzW2ZmyxoaGnIII3e7\ndu8p6vZFRAopLxdj3d0Bz/A1C9y93t3rq6ur8xFGVh55eTM7du4q2vZFRAqtIofXbjGzfu6+OWia\n2RqUbwIGxaw3MChrddZt/ZALF75Q7DBERAoqlxr9Q8Cc4PEc4MGY8rOC3jeTgO0xTTytysef7i52\nCCIiBZdWjd7M7gamA73NbCPwHeAa4F4zOxd4Ezg1WP0R4HhgHfARcE6eYxYRkQyklejd/fQki2Ym\nWNeBi3IJSkRE8qdkR8bu2r2Hv23cVuwwREQKLpeLsW3adYtf4+Yn1xc7DBGRgivZGv3KTduLHYKI\nSIso2UQvIlIqSirRv/fhJ3y6S6NgRaS0lFSiH/8fjzHvLg2QEpHSUlKJHuDRVVuKHYKISIsquUQv\nIlJqlOhFREJOiV5EJOSU6EVEQk6JXkQk5JToRURCrmQSfWRSTRGR0lMyiV5EpFSVTKJXhV5ESlXW\n0xSb2UjgnpiiocC3ge7AeUBDUP5Nd38k6whFRCQnWSd6d18DjAEws3IiNwBfROTWgTe4+7V5iTBP\nVKEXkVKVr6abmcB6d38zT+8nIiJ5kq9Efxpwd8zzeWa2wsxuM7MeedpGTtTrRkRKVc6J3swqgS8A\nvwmKbgaGEWnW2Qxcl+R1c81smZkta2hoSLSKiIjkQT5q9McBL7j7FgB33+Luu919D/BzYEKiF7n7\nAnevd/f66urqPITRPNXnRaRU5SPRn05Ms42Z9YtZdjKwMg/bEBGRLGXd6wbAzKqAWcC/xhT/wMzG\nEKlEb4hbVjRqoheRUpVTonf3fwK94srOzCmiAvlst+4VKyKlqWRGxt7/4qZihyAiUhQlk+h/99Lb\nxQ5BRKQoSibRP7/h/WKHICJSFCWT6OO9vGl7sUMQEWkRJZvot330WbFDEBFpESWb6EVESkVJJvof\nL1lb7BBERFpMSSb66xa/VuwQRERaTOgT/csbt/Puh58UOwwRkaLJaWRsW/D5m56hpmv7YochIlI0\noa/RA2z5QDV6ESldJZHoRURKmRK9iEjIhbaN/rPde2jYoSYbEZHQJvrvPPQKdy39e7HDEBEputA2\n3TyxemuxQxARaRVCm+hFRCQi56YbM9sA7AB2A7vcvd7MegL3AHVEbid4qrv/I9dtiYhI5vJVo5/h\n7mPcvT54Ph9Y4u7DgSXBcxERKYJCNd2cCNwRPL4DOKlA2xERkRTykegdeNTMlpvZ3KCsxt03B4/f\nAWrysJ20zbr+KTZv39mSmxQRabXy0b1ysrtvMrM+wGIzWx270N3dzDz+RcGXwlyA2traPISxz9qt\nH+b1/URE2rKca/Tuvin4vRVYBEwAtphZP4Dgd5O+ju6+wN3r3b2+uro61zBERCSJnBK9mVWZWZfo\nY+BoYCXwEDAnWG0O8GAu2xERkezl2nRTAywys+h73eXufzSzvwL3mtm5wJvAqTluR0QklB67bFrB\nt5FTonf314FDEpS/B8zM5b1FRPLt8lkjWt0d5vbr07ng29DIWBHJm1+efWixQ2hWp/ahnd6rWUr0\nIpI3M/bvU+wQ8mZodVWxQ8gbJXoRkQTOmJDfbt/FpEQvIhJyoUv0T67R9MQikrugN2EohC7Rn/3L\nvxY7BBEJgYlDehY7hLwJXaIXEcmHsmZq9AN7dGzBSHKnRC8irVr/bh2KHUITlxw1otghZESJXkRa\ntRPHDsjbe7k3mV+xKM46bHCLbk+JXqSNmDK8d7FDKIp85uYDB3TLy/vMGpXbzOtH7Neyx1KJXqSN\n+NW5E4sdQpvXoV152us21+mmW8d2OcXR0v15lOilkS+NH1jsEKRE1Q/ukbB8apHOZPLdynP7OYdy\n1AHFGTmsRC+N9KqqLHYI0gp8cVzmX/j79+1SgEigZ+fMP5PnTRmSsDy+jb6qsvka/vzj9k+5rRu+\n0mRex4Ri++W3dB99JXpppF25PhJh8u9fGN1i2/rjJVNbbFvZ+D9HN+0p8+C8I5p9zfnThnHaoYMy\n2s4JB/dLuqxY14L1Xy0SMl067JuhsS0N7swl1s8dlDy5Rs07cnj2G8jAxTNTb0dt9CKSk6kjwnVr\nzgHdMxucVJbnLJrqC8gySNuzJ0UmSjtoYH56/6QrNIl+646dLH39vWKH0eY5xe1nvOGaz6Vc54df\nOrgFIimM0ydk1gyQja4dEs+5/q0TRqX194XWcyZw+awRdOmQuodL7Of29f9Kbx/TlU17ev3gHgzu\n1alJ+ZH717Dhms9R07VlB4FlnejNbJCZPWFmq8zsFTO7OCj/rpltMrOXgp/j8xduYjs/282E7y/h\nKwueK/SmRHLSMm20xlcn7ZtiN3ozkOkj06vpX3HMyCZlQ6urKLPs+/JfMH1YVq+Lz7HDkswRv2dP\neu+3J4O/f//ukWSc6RmCGfz2gsN56ooZmb2wgHK53cou4HJ3fyG4QfhyM1scLLvB3a/NPbz0fOuB\nlS21KcnQ2YfXcfuzG/L6nq1jbGN2WvpinBG5GUi6NflkHr98+t7HdfN/n1tQOVhy+XRmXvck6xv+\n2ag83TPRPWkcgGNH9+WWM8fvfd7cnDdtRdY1enff7O4vBI93AK8C+RurnIE1W3YUY7MScj2z6Gpa\nnmH1r3un5pslEp3+t4RU+TBZzTrfsu2GmOx1n+1Ks+ofI51E//U0LsAWU17a6M2sDhgLLA2K5pnZ\nCjO7zcwSjoIws7lmtszMljU0NOQjDJG8OiWLOVa+cEj/Zpenqnn2i5vA64C+XTOOIVep8tra7x/H\nL+a07nvDJpvT5qNPd+99fPs5h9Krqn3K90qV6Pfv16VR804rmU6nkZwTvZl1Bu4DLnH3D4CbgWHA\nGGAzcF2i17n7Anevd/f66upw9RJoy4r5IU134EmhXDSjcTvyN48/gEvzPEthqjbiivKmSSVRr5NX\nv3dsvkJqItVnoF15GT07tczAunQr9Ol+bmtjzpAqysroUVXJK/9+TLOvSXWStn8RvowzlVOiN7N2\nRJL8Qne/H8Ddt7j7bnffA/wcmJB7mJKpdFsQbp49Lqv3P2K/Xlm9rjnpDtYaUZN8BGbn9hX87dtH\n5yWesjKjU4qRk+lYENPemyohJVq+5PJpTZJRxxRx5fqFnSrBduvUjuouqWvDhXDUAaknFEvWdDOi\npgsH9GucmKvaN3+psizf/TVjTG6hyc1y6XVjwK3Aq+5+fUx57MiFk4GCXind+dlu3tm+s5CbCK0j\n9uvFcWkMNEnk5LH5nxMn3f7IYwZ1T7rM3emWot07+WuTL0u3L3d8k8HSb87k6NF903rti9+albC8\nQ7vylMkIoG/QZa/R92WKjP3ruZPSii2R7hlM7HVwlrNGJvpMXHls02kJMvle61mV2ecj08sEydYf\n0L1xs9yL35rFrWfXZ/bmWcqlRn8EcCZwZFxXyh+Y2ctmtgKYAVyaj0CTOeZ/nmbrjk8KuYlWa1DP\n5MlnxsjUkyf16RL54D1/9cyMt13Ifgi53MJtWppdCKMuOar5i2jRNvWq9unV7PvHfSHE95eOb6Of\nOnxfvD0yvPj7t28fzd++s+/sJXphN5NeIr07V2Y8IClTz101k+MO6sefr2y+u+GE4LjHXuhNtCuJ\nLnhHv19PnxDpVprL7JJjaxtXJGL/nmUGy//vURm/55+vnMF+fRqfifaoqqR9Re5njOnIpdfNM+5u\n7n6wu48Jfh5x9zPd/aCg/AvuvjmfAcd6e9vHvPneR4V6+1bp8cun8ZvzDwOge8dKenduevo8trY7\nP8mgSSaa8DNRVsChdnf8y4S9fbkn1CVO+jVdG+/3fRccxp8umcr1p47JaFuxibm5WmG6Zxsj+3Zh\nyeXTki4f2rtxb5Uffrnx4K+JQyJNYqm+gCDSfBKb0KLJLlWi/0ajGrFxYdy1ibTbxdNbjb7BBeZB\nPZvvQXTZrJE8fvk06npl36OnT9CcFO0xNbxP5ybrRM8I+3RN3PR03pShjZ7Hfq9UVVbQK8H/3PA+\nzU/olmrfC61Nj4x9e9vHxQ6hxQ2t7kxlzLn5ksumcWHcYJS7z5uU0bzb2ahIkun/5yuRRFvZTHv7\njBS17g7tyvfek7MmyW3kHr10WqMa4vjBPRnZt0vB9zsdw6ojySVRzXPqiGoe+fqUvc/ja3T/ecqB\nPHbZVEY2cx0imWgf8VRdPM+fFp/IWkc/8fIyY2h148ScKrJ9o4Abf+0M6hFJrKP6N71Qetmskfzx\nkimNrvX0jpkhM75Nfr8EXxbxPtfMRGatQZtO9K3k81lU3Tq1Y0DcjYoTJbt0m2dy7XRz0tgBLLrw\ncH58xlgAhvRuWjsb2CNx7SZRDStZN7luHdtlXUs6Zdy+bpOx3RljNxX9x8/2oubTV8xg6TcT/80T\nJZ+o9hXlTU7x07U7CLbM2Jsw49uFIXKhMnpmkex/KJ3klo2hGfa/T/U/fuyBkesf0TtHRccdTB7e\nmwcuOoJzjmg6XXF5mTXpKbP40mnc9bWJPHdV02N20piYbrYx8RwSd62opcYWZCOXkbFF19JzOicz\nY2Q1T6xpubEA0dyTye736dKB3p0reffDTwsSE8Bd50XugDS2NjJ04p65k5gwpCffeeiVRuslGp34\nm/MP49CYZpr4Y9uzqpL3/5mf2IdVd+bZ+UfyzLp3mTK8mqevmEFFuXHHXzbsXefRSxs3v2T6UatN\nMtAp3S+O3kETxJAMksfeppsy45zD6ziwf1cmDs2ud9TcKUO55g+rm11nZE0X1m39MKP3jf8znjSm\nPw+89HajsmF9OrNk9da03u97Jx7IaRNqGTuoO5OG9mp0fWfMoO787a1tANSmqBT0qKrk8CQ9YBp9\nFmOO36/OncBb7+9rOj6gX1fWN/yTTpX70mrfrh1454PidxZp2zX6YgdQZNH9r07QZhgrWTPAoB7Z\nX4RLdIOSw4c1/keZOLRXwi/j+DMQoFGSj/fwv03mT5dMpWMWzTKPfH0Kx4xu3B2vpmsH+nfvyKn1\nkQnGant1anIRNZtRsfl0aF1PFn5tIpfNatyP/9FLp/Ls/CMTvqZHzMXYsjJrNsk3933Tt2uHtLoU\n/ufJB6Vcp12CcQGxEvX6uuKYkXu7UKaa/KtDu3LG1fbAzJiU5PMGqUcgZ6Nrh3aM7r+vN9EPvnQw\n98yd1Oiz9PuvT+Z38ybnfduZatuJvpXU6Fs6jvjmjFmjarjmlOT/dBOHNk2iw6qrsh62fdvZ9Y1q\nP4cN7cV9FxyWdP3fzZvcKDn969RhfPuEUc1uI5q0+nfvyIEDulHdpT1PXjGdBy5q/kYR8Ub179ro\nYvMPv3QwXxyXZMRrguwXvdgd22c80xigabv5s/OPTJkAjtivd5OxBSNqujT5UoqaHvS0Ks/g82js\n+1I7+/A65k4dyslpjgju1qldwu6Zj102lT9eMiXBK5o6OsFNttuVl7HgzPH8/Kz6lCONU4l2S01V\no8+HTpUVTb5ce3Vu3+JTEifSpptuWotEg2r+37kT+eqtSxOs3dTBA7uxYuP2tLcX/fBGa8ZmxmkT\napl//8uN1jt57AAWvbiJ4UGNvrZnp71NNzNG9qEiy7tJHbl/43/OX8ypb7afd/wHvbzMOP6gfnzv\n4VVJXzN5v97cPHscM2MGx9R07ZDz9K5frk8+TXCiWu4p4wbQoV05NV3b8+e173Jq/UAOaeYft2uS\nbn2j+3dtdIz7d++YNGFnK9o9M9OBTEePquGns8cxa1RNxncYmzS0Fw9cdMTeL2aA/fp04ZNdkakG\n4nsrDerZae+EZIcM7Ja0klRWZsxK8CWQqf36dOYXZ9Vz2LD8D/BrS9p0on9547ZihwBE2uYeXtG4\nF+nkuOlcZ42qYfGqLQlfP31E9d4kcP2phzCwRyequ7RnxrVPJlx/RE0Xfjp7XMopY6OjJ6P/Sr+Y\ncyjfuG8Fi1dtadJne9GFh9O5fQVPrEmvbTRWOoN5INKMsjuYA6BvzEXQRRce3mRdM8t6MFd0W+nM\nVBirR4Jh/Wa2t0fFjaePZVYzozJ/dNoYpie56UfkC2p7QXoFPXXFdDZv38mhdT2pqiznxDGpa+Sx\nZ4VmkS/eZB5McQaTaABbNMEPjGum+9FpY/nL+nepKCtr1F89m5PiPml+oR2Vhy+MqETNjm1Bm070\nmcwtXUjxH9KfJujDfstXx7P0jfeo7tyeWTc83WhZt5gEc0rMTZkf/rfJnPDjZxJus7l/zKj4PNez\nqpKffXU89y57iy+ObzyyNXoBdUjvKrp2aNfo7OCmM8Zy/aOv8fq7jaeGzVSy3ibRbWerS4IbbcRu\nK9o+++XxzY/m/dqUIby2ZQdzpw5NuDy2GeFnZ47nhsWvsfqdfTOnNpdgr/3yITyxeisjE9xA+/HL\np7Eph67Cg3tVMTjoe35Khjf1TqfZMb53SToqK8q45avjGBd3bLt1bMexBzb+7P7ynEMZ1juzXj4L\nzhzfok0iT10xnVueWs+ls/I791FLadOJPtORhKlEmzqOOqCGx15NXPuGSNe5qxatYNmGf/DJrj1U\nVVbwu3mT+fxNkaQcra3+/Kx6zrtzGRBproherIx9/8WXTmVodWceXvE2N542ttF2Dsxy2HhU146R\nwxvbtFRWFmnmSaaivKxJM9AJB/fnt8s35pzo433rhFHs/Gx36hWb8dPZ4zgoxd9p3pH70btze86c\nNLjZ9dqVl3HDV9IbcHXM6L5MqOvJ4lVb6N+9Y8qLfd06tuOkJG3fQ6s7N+k/Xmg9qyrZ8N5HVDRz\n0fWheUfw4Se7st5GfEJPJp1R3PHSnVYiXwb3quK/Tmm7dzZr04k+WxVlxq6Y04E/XzmD5994n5PH\nDmB0/67MnjiYA779x6Svr+3ViYVfm8Qnu3Zz57NvMntiLRXlZUwZ3ps/r32XdsFgomRtjLFJIdp+\nvujC5k+P+3Rpn7KXw82zxzEspv/zpUeNoHdV+7RO5eNFu4X94eLIRbXzpgzlyTUNaU0ola5zJzft\n45ypdM5s2leUM+fwupy3Fa9HVSWnHlr4WwMWwi1njuexVVubHYtw8MDMa/LSOlmyASktqb6+3pct\nW5bx6x55eTMXLnwh49c9f/VMNm/byYk/+V8g8X1Kk91F5/ZzDt3buyHeux9+wqIXNvG1KUP2nhIv\neXULXTu2a9R98IOdn/HFnz7Lj88Ym3KK0ydWb2Xh0jf57y8enHDodaG89f5HPL22gdkTI7Vgd+eW\np17njAm1eycNW7FxG2+9/3GrHxUobd/jq7dQlaBXS6kzs+XunnJmtDad6HfvcYZ98xEOHNCVlZs+\nACKTGg3o3oHHV2/l6NF99w76qOnanotnjmDc4O57k+sfV75D1w4VCQdKXHTXC/w+5gLrdz8/imkj\n+yQc6SkiUgwlkehjPbv+XbZ8sLPR9Lnuzo+WrOXU+kEZd2Xbvce57tE19Kyq5MAB3ZikmoSItDIl\nl+hFREpNuom+TY+MFRGR1JToRURCrmCJ3syONbM1ZrbOzOYXajsiItK8giR6MysHfgIcB4wCTjez\n5mexEhGRgihUjX4CsM7dX3f3T4FfAycWaFsiItKMQiX6AcBbMc83BmUiItLCinYx1szmmtkyM1vW\n0NByd2cSESk1hUr0m4DYSUAGBmV7ufsCd6939/rq6uZvFi0iItkryIApM6sAXgNmEknwfwXOcPdX\nkqzfALyZwyZ7A+/m8Pq2qBQYHCwzAAAEuUlEQVT3GUpzv7XPpSPT/R7s7ilrygWZvdLdd5nZPOBP\nQDlwW7IkH6yfU5XezJalMzosTEpxn6E091v7XDoKtd8Fm6bY3R8BHinU+4uISHo0MlZEJOTCkugX\nFDuAIijFfYbS3G/tc+koyH63itkrRUSkcMJSoxcRkSTadKIP08RpZjbIzJ4ws1Vm9oqZXRyU9zSz\nxWa2NvjdIyg3M7sx2PcVZjYu5r3mBOuvNbM5xdqndJlZuZm9aGYPB8+HmNnSYN/uMbPKoLx98Hxd\nsLwu5j2uCsrXmNkxxdmT9JlZdzP7rZmtNrNXzeywEjnWlwaf75VmdreZdQjb8Taz28xsq5mtjCnL\n27E1s/Fm9nLwmhvNLPkd3qPcvU3+EOm2uR4YClQCfwNGFTuuHPanHzAueNyFyDiEUcAPgPlB+Xzg\nv4PHxwN/AAyYBCwNynsCrwe/ewSPexR7/1Ls+2XAXcDDwfN7gdOCx7cAFwSPLwRuCR6fBtwTPB4V\nHP/2wJDgc1Fe7P1Ksc93AF8LHlcC3cN+rIlMg/IG0DHmOJ8dtuMNTAXGAStjyvJ2bIHng3UteO1x\nKWMq9h8lhz/mYcCfYp5fBVxV7LjyuH8PArOANUC/oKwfsCZ4/DPg9Jj11wTLTwd+FlPeaL3W9kNk\n1PQS4Ejg4eDD+y5QEX+ciYzLOCx4XBGsZ/HHPna91vgDdAsSnsWVh/1YR+fA6hkcv4eBY8J4vIG6\nuESfl2MbLFsdU95ovWQ/bbnpJrQTpwWnqGOBpUCNu0fvUv4OUBM8Trb/be3v8j/AlcCe4HkvYJu7\n7wqex8a/d9+C5duD9dvaPg8BGoBfBk1WvzCzKkJ+rN19E3At8HdgM5Hjt5zwH2/I37EdEDyOL29W\nW070oWRmnYH7gEvc/YPYZR75Cg9NNykzOwHY6u7Lix1LC6sgcmp/s7uPBf5J5HR+r7Ada4CgXfpE\nIl90/YEq4NiiBlUExTi2bTnRp5w4ra0xs3ZEkvxCd78/KN5iZv2C5f2ArUF5sv1vS3+XI4AvmNkG\nIvcsOBL4EdDdIvMlQeP49+5bsLwb8B5ta58hUgvb6O5Lg+e/JZL4w3ysAY4C3nD3Bnf/DLifyGcg\n7Mcb8ndsNwWP48ub1ZYT/V+B4cEV+0oiF2seKnJMWQuunN8KvOru18csegiIXnGfQ6TtPlp+VnDV\nfhKwPTg1/BNwtJn1CGpQRwdlrY67X+XuA929jsjxe9zdZwNPAF8KVovf5+jf4kvB+h6Unxb00hgC\nDCdywapVcvd3gLfMbGRQNBNYRYiPdeDvwCQz6xR83qP7HerjHcjLsQ2WfWBmk4K/4Vkx75VcsS9a\n5HjB43givVPWA1cXO54c92UykdO5FcBLwc/xRNoklwBrgceAnsH6RuR2jeuBl4H6mPf6F2Bd8HNO\nsfctzf2fzr5eN0OJ/OOuA34DtA/KOwTP1wXLh8a8/urgb7GGNHohFPsHGAMsC473A0R6VoT+WAP/\nDqwGVgK/ItJzJlTHG7ibyDWIz4icvZ2bz2ML1Ad/v/XATcRd1E/0o5GxIiIh15abbkREJA1K9CIi\nIadELyISckr0IiIhp0QvIhJySvQiIiGnRC8iEnJK9CIiIff/Afx5jD5czmksAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "rewards = []\n",
    "for i in range(10000):\n",
    "    rewards.append(play_and_train(env,agent))   \n",
    "    \n",
    "    #OPTIONAL YOUR CODE: adjust epsilon\n",
    "    if i %100 ==0:\n",
    "        clear_output(True)\n",
    "        print('eps =', agent.epsilon, 'mean reward =', np.mean(rewards[-10:]))\n",
    "        plt.plot(rewards)\n",
    "        plt.show()\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 182,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "13059"
      ]
     },
     "execution_count": 182,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(agent._qvalues)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "### Submit to Coursera II: Submission"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 183,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "submit_rewards2 = rewards.copy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 184,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Submitted to Coursera platform. See results on assignment page!\n"
     ]
    }
   ],
   "source": [
    "from submit import submit_qlearning\n",
    "submit_qlearning(submit_rewards1, submit_rewards2, 'moniquelouise@gmail.com', '8SoH7nyljTfg7x8j')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}