Navigation_DQN.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Solving Banana Collector Environment with Deep Q-Network using Prioritized Experience Replay, Fixed Q-Targets and Double DQN"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Imports"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import random\n",
    "from collections import namedtuple, deque, OrderedDict\n",
    "import heapq\n",
    "import itertools\n",
    "import math\n",
    "\n",
    "from unityagents import UnityEnvironment\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "import torch.optim as optim"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Setting global variables, device and random seed."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Device: cuda:0\n",
      "Seed: 5\n"
     ]
    }
   ],
   "source": [
    "device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\n",
    "print('Device:', device)\n",
    "\n",
    "seed = 5\n",
    "random.seed(seed)\n",
    "torch.manual_seed(seed)\n",
    "print('Seed:', seed)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Setting up the environment"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:unityagents:\n",
      "'Academy' started successfully!\n",
      "Unity Academy name: Academy\n",
      "        Number of Brains: 1\n",
      "        Number of External Brains : 1\n",
      "        Lesson number : 0\n",
      "        Reset Parameters :\n",
      "\t\t\n",
      "Unity brain name: BananaBrain\n",
      "        Number of Visual Observations (per agent): 0\n",
      "        Vector Observation space type: continuous\n",
      "        Vector Observation space size (per agent): 37\n",
      "        Number of stacked Vector Observation: 1\n",
      "        Vector Action space type: discrete\n",
      "        Vector Action space size (per agent): 4\n",
      "        Vector Action descriptions: , , , \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Brain name: BananaBrain\n",
      "Number of agents: 1\n",
      "Number of actions: 4\n",
      "State example: [1.         0.         0.         0.         0.84408134 0.\n",
      " 0.         1.         0.         0.0748472  0.         1.\n",
      " 0.         0.         0.25755    1.         0.         0.\n",
      " 0.         0.74177343 0.         1.         0.         0.\n",
      " 0.25854847 0.         0.         1.         0.         0.09355672\n",
      " 0.         1.         0.         0.         0.31969345 0.\n",
      " 0.        ]\n",
      "State size: 37\n"
     ]
    }
   ],
   "source": [
    "env = UnityEnvironment(file_name='Banana_Linux/Banana.x86_64')\n",
    "\n",
    "brain_name = env.brain_names[0]\n",
    "brain = env.brains[brain_name]\n",
    "print('Brain name:', brain_name)\n",
    "\n",
    "env_info = env.reset(train_mode=True)[brain_name]\n",
    "print('Number of agents:', len(env_info.agents))\n",
    "\n",
    "action_size = brain.vector_action_space_size\n",
    "print('Number of actions:', action_size)\n",
    "\n",
    "state = env_info.vector_observations[0]\n",
    "print('State example:', state)\n",
    "\n",
    "state_size = len(state)\n",
    "print('State size:', state_size)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Prioritized Experience Replay"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Generation of segments for importance sampling based on ranking, higher `priority_exp` skews segments more."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th>priority exponent</th>\n",
       "      <th>0.0</th>\n",
       "      <th>0.25</th>\n",
       "      <th>0.5</th>\n",
       "      <th>0.75</th>\n",
       "      <th>1.0</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>segment boundaries</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.2</td>\n",
       "      <td>0.134</td>\n",
       "      <td>0.089</td>\n",
       "      <td>0.060</td>\n",
       "      <td>0.04</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.4</td>\n",
       "      <td>0.318</td>\n",
       "      <td>0.253</td>\n",
       "      <td>0.201</td>\n",
       "      <td>0.16</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.6</td>\n",
       "      <td>0.528</td>\n",
       "      <td>0.465</td>\n",
       "      <td>0.409</td>\n",
       "      <td>0.36</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.8</td>\n",
       "      <td>0.757</td>\n",
       "      <td>0.716</td>\n",
       "      <td>0.677</td>\n",
       "      <td>0.64</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>1.0</td>\n",
       "      <td>1.000</td>\n",
       "      <td>1.000</td>\n",
       "      <td>1.000</td>\n",
       "      <td>1.00</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "priority exponent   0.00   0.25   0.50   0.75  1.00\n",
       "segment boundaries                                 \n",
       "0                    0.0  0.000  0.000  0.000  0.00\n",
       "1                    0.2  0.134  0.089  0.060  0.04\n",
       "2                    0.4  0.318  0.253  0.201  0.16\n",
       "3                    0.6  0.528  0.465  0.409  0.36\n",
       "4                    0.8  0.757  0.716  0.677  0.64\n",
       "5                    1.0  1.000  1.000  1.000  1.00"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "def compute_priority_segments(num_segments, priority_exp):\n",
    "    return [(i / num_segments) ** (priority_exp + 1) for i in range(num_segments + 1)]\n",
    " \n",
    "def demo_compute_priority_segments():\n",
    "    priority_exps = [0, 0.25, 0.5, 0.75, 1]\n",
    "    segments = {e: compute_priority_segments(5, e) for e in priority_exps}\n",
    "    \n",
    "    df = pd.DataFrame(segments)\n",
    "    df.columns.name = 'priority exponent'\n",
    "    df.index.name = 'segment boundaries'\n",
    "    pd.set_option('precision', 3)\n",
    "    display(df)\n",
    "        \n",
    "demo_compute_priority_segments()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Computes importance sampling probabilities. Higher ranked items have higher sampling probability. Probabilities of all ranks should sum up to 1."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th>priority exponent</th>\n",
       "      <th>0.0</th>\n",
       "      <th>0.25</th>\n",
       "      <th>0.5</th>\n",
       "      <th>0.75</th>\n",
       "      <th>0.99</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>rank</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.01</td>\n",
       "      <td>0.024</td>\n",
       "      <td>0.056</td>\n",
       "      <td>0.116</td>\n",
       "      <td>0.212</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>0.01</td>\n",
       "      <td>0.012</td>\n",
       "      <td>0.012</td>\n",
       "      <td>0.012</td>\n",
       "      <td>0.011</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>40</th>\n",
       "      <td>0.01</td>\n",
       "      <td>0.010</td>\n",
       "      <td>0.009</td>\n",
       "      <td>0.007</td>\n",
       "      <td>0.006</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>60</th>\n",
       "      <td>0.01</td>\n",
       "      <td>0.009</td>\n",
       "      <td>0.007</td>\n",
       "      <td>0.005</td>\n",
       "      <td>0.004</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>80</th>\n",
       "      <td>0.01</td>\n",
       "      <td>0.008</td>\n",
       "      <td>0.006</td>\n",
       "      <td>0.004</td>\n",
       "      <td>0.003</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>100</th>\n",
       "      <td>0.01</td>\n",
       "      <td>0.008</td>\n",
       "      <td>0.006</td>\n",
       "      <td>0.004</td>\n",
       "      <td>0.002</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "priority exponent  0.00   0.25   0.50   0.75   0.99\n",
       "rank                                               \n",
       "1                  0.01  0.024  0.056  0.116  0.212\n",
       "20                 0.01  0.012  0.012  0.012  0.011\n",
       "40                 0.01  0.010  0.009  0.007  0.006\n",
       "60                 0.01  0.009  0.007  0.005  0.004\n",
       "80                 0.01  0.008  0.006  0.004  0.003\n",
       "100                0.01  0.008  0.006  0.004  0.002"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "def compute_rank_probs(ranks, memory_size, priority_exp):\n",
    "    return (1 - priority_exp) / ranks**priority_exp / (memory_size**(1 - priority_exp) - 1)\n",
    "\n",
    "def demo_compute_rank_probs():\n",
    "    ranks = np.array([1, 20, 40, 60, 80, 100])\n",
    "    priority_exps = [0, 0.25, 0.5, 0.75, 0.99]\n",
    "    rank_probs = {e: compute_rank_probs(ranks, 100, e) for e in priority_exps}\n",
    "    \n",
    "    df = pd.DataFrame(rank_probs, index=ranks)\n",
    "    df.columns.name = 'priority exponent'\n",
    "    df.index.name = 'rank'\n",
    "    pd.set_option('precision', 3)\n",
    "    display(df)\n",
    "\n",
    "demo_compute_rank_probs()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Compute parameter update weights based on importance sampling probabilities. Higher probability should be assigned lower weight to avoid skewed distribution learned by the qnet."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th>sample exponent</th>\n",
       "      <th>0.0</th>\n",
       "      <th>0.25</th>\n",
       "      <th>0.5</th>\n",
       "      <th>0.75</th>\n",
       "      <th>1.0</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>rank</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1.0</td>\n",
       "      <td>0.562</td>\n",
       "      <td>0.316</td>\n",
       "      <td>0.178</td>\n",
       "      <td>0.100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>1.0</td>\n",
       "      <td>0.818</td>\n",
       "      <td>0.669</td>\n",
       "      <td>0.547</td>\n",
       "      <td>0.447</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>40</th>\n",
       "      <td>1.0</td>\n",
       "      <td>0.892</td>\n",
       "      <td>0.795</td>\n",
       "      <td>0.709</td>\n",
       "      <td>0.632</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>60</th>\n",
       "      <td>1.0</td>\n",
       "      <td>0.938</td>\n",
       "      <td>0.880</td>\n",
       "      <td>0.826</td>\n",
       "      <td>0.775</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>80</th>\n",
       "      <td>1.0</td>\n",
       "      <td>0.972</td>\n",
       "      <td>0.946</td>\n",
       "      <td>0.920</td>\n",
       "      <td>0.894</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>100</th>\n",
       "      <td>1.0</td>\n",
       "      <td>1.000</td>\n",
       "      <td>1.000</td>\n",
       "      <td>1.000</td>\n",
       "      <td>1.000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "sample exponent  0.00   0.25   0.50   0.75   1.00\n",
       "rank                                             \n",
       "1                 1.0  0.562  0.316  0.178  0.100\n",
       "20                1.0  0.818  0.669  0.547  0.447\n",
       "40                1.0  0.892  0.795  0.709  0.632\n",
       "60                1.0  0.938  0.880  0.826  0.775\n",
       "80                1.0  0.972  0.946  0.920  0.894\n",
       "100               1.0  1.000  1.000  1.000  1.000"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "def compute_sample_weights(sample_probs, memory_size, priority_exp, sample_exp):\n",
    "    min_sample_prob = compute_rank_probs(memory_size, memory_size, priority_exp)\n",
    "    max_sample_weight = (memory_size * min_sample_prob)**(-sample_exp)\n",
    "    sample_weights = (memory_size * sample_probs)**(-sample_exp) / max_sample_weight\n",
    "    return sample_weights\n",
    "    \n",
    "def demo_compute_sample_weights():\n",
    "    memory_size = 100\n",
    "    priority_exp = 0.5\n",
    "    ranks = np.array([1, 20, 40, 60, 80, 100])\n",
    "    sample_exps = [0, 0.25, 0.5, 0.75, 1]\n",
    "    sample_probs = compute_rank_probs(ranks, memory_size, priority_exp)\n",
    "    sample_weights = {e: compute_sample_weights(sample_probs, memory_size, priority_exp, e) for e in sample_exps}\n",
    "\n",
    "    df = pd.DataFrame(sample_weights, index=ranks)\n",
    "    df.index.name = 'rank'\n",
    "    df.columns.name = 'sample exponent'\n",
    "    pd.set_option('precision', 3)\n",
    "    display(df)\n",
    "\n",
    "demo_compute_sample_weights()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Implementation of priority replay memory, which uses binary heap to keep experiences in approximately sorted order by TD-error. Importance sampling is implemented using unevenly sized segments, i.e. segment size decreases from top to bottom of the heap. Number of segments is equal to the batch size. One experience is sampled per segment."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>error</th>\n",
       "      <th>weight</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>997</td>\n",
       "      <td>0.416</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>988</td>\n",
       "      <td>0.514</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>978</td>\n",
       "      <td>0.562</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>908</td>\n",
       "      <td>0.707</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>866</td>\n",
       "      <td>0.740</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>696</td>\n",
       "      <td>0.828</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>686</td>\n",
       "      <td>0.832</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>39</td>\n",
       "      <td>0.884</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>27</td>\n",
       "      <td>0.924</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>9</td>\n",
       "      <td>0.977</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   error  weight\n",
       "0    997   0.416\n",
       "1    988   0.514\n",
       "2    978   0.562\n",
       "3    908   0.707\n",
       "4    866   0.740\n",
       "5    696   0.828\n",
       "6    686   0.832\n",
       "7     39   0.884\n",
       "8     27   0.924\n",
       "9      9   0.977"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "class PriorityReplayMemory:\n",
    "    def __init__(self, \n",
    "                 memory_size, # (int) max size of memory\n",
    "                 batch_size, # (int) number of items to sample in one batch\n",
    "                 priority_exp, # (float) priority exponent controls the uniformness of sampling\n",
    "                 sort_freq # (int) number of items added before the heap is resorted\n",
    "                ):\n",
    "        self.memory_size = memory_size\n",
    "        self.batch_size = batch_size\n",
    "        self.priority_exp = priority_exp\n",
    "        self.sort_freq = sort_freq\n",
    "\n",
    "        self.heap = []  # min-heap to store experiences\n",
    "        self.step = 0\n",
    "\n",
    "        self.segments = compute_priority_segments(batch_size, priority_exp)\n",
    "        self.counter = itertools.count()\n",
    "\n",
    "    def add(self, experience, error=None):\n",
    "        if error is None:  # new expereinces get maximal priority\n",
    "            if len(self.heap) > 0:\n",
    "                priority = self.heap[0][0]\n",
    "            else:\n",
    "                priority = -10  # some default value\n",
    "        else:\n",
    "            priority = -error  # min heap stores min priority first\n",
    "\n",
    "        count = next(self.counter)\n",
    "        heapq.heappush(self.heap, (priority, count, experience))\n",
    "\n",
    "        # Remove last experience\n",
    "        if len(self.heap) > self.memory_size:\n",
    "            self.heap.pop()\n",
    "\n",
    "        # Resort the heap every sort_freq times: O(n*log(n))\n",
    "        self.step = (self.step + 1) % self.sort_freq\n",
    "\n",
    "        if self.step == 0:\n",
    "            self.heap.sort(key=lambda x: x[0])\n",
    "\n",
    "    def sample(self, sample_exp):\n",
    "        size = len(self.heap)\n",
    "\n",
    "        # Scale segements to the number of experiences\n",
    "        segments = [(math.ceil(start * (size - 1)), math.floor(end * (size - 1)))\n",
    "                    for start, end in zip(self.segments[:-1], self.segments[1:])]\n",
    "\n",
    "        # Check if the first (smallest) segment contains more than one experience\n",
    "        if segments[0][0] < segments[0][1]:\n",
    "            indexes = [random.randint(start, end) for start, end in segments]\n",
    "        else:  # Ignore segments, do uniform sampling\n",
    "            indexes = random.sample(range(size), self.batch_size)\n",
    "\n",
    "        # Select experiences\n",
    "        experiences = [self.heap[i][2] for i in indexes]\n",
    "\n",
    "        # Compute importance-sampling weights\n",
    "        ranks = np.array(indexes) + 1\n",
    "        probs = compute_rank_probs(ranks, self.memory_size, self.priority_exp)\n",
    "        weights = compute_sample_weights(probs, self.memory_size, self.priority_exp, sample_exp)\n",
    "\n",
    "        # Remove sampled experiences\n",
    "        for index in sorted(indexes, reverse=True):\n",
    "            del self.heap[index]\n",
    "\n",
    "        return experiences, weights\n",
    "\n",
    "    def __len__(self):\n",
    "        return len(self.heap)\n",
    "\n",
    "\n",
    "def demo_priority_replay_memory():\n",
    "    memory = PriorityReplayMemory(memory_size=100, batch_size=10, priority_exp=0.5, sort_freq=10)\n",
    "\n",
    "    for i in range(1000):\n",
    "        memory.add(i, i)\n",
    "\n",
    "    exps, probs = memory.sample(0.5)\n",
    "    df = pd.DataFrame({'error': exps, 'weight': probs})\n",
    "    pd.set_option('precision', 3)\n",
    "    return df\n",
    "\n",
    "\n",
    "demo_priority_replay_memory()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Q-network\n",
    "\n",
    "Q-network with parameterized number of hidden layers."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Sequential(\n",
       "  (fc0): Linear(in_features=37, out_features=64, bias=True)\n",
       "  (bc0): BatchNorm1d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "  (relu0): ReLU()\n",
       "  (fc1): Linear(in_features=64, out_features=64, bias=True)\n",
       "  (bc1): BatchNorm1d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "  (relu1): ReLU()\n",
       "  (fc2): Linear(in_features=64, out_features=4, bias=True)\n",
       ")"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def make_qnet(state_size, action_size, hidden_sizes):\n",
    "    layers = OrderedDict()\n",
    "    prev_size = state_size\n",
    "\n",
    "    for i, size in enumerate(hidden_sizes):\n",
    "        layers['fc{}'.format(i)] = nn.Linear(prev_size, size)\n",
    "        layers['bc{}'.format(i)] = nn.BatchNorm1d(size)\n",
    "        layers['relu{}'.format(i)] = nn.ReLU()\n",
    "        prev_size = size\n",
    "    \n",
    "    layers['fc{}'.format(len(hidden_sizes))] = nn.Linear(prev_size, action_size)\n",
    "    return nn.Sequential(layers)\n",
    "    \n",
    "make_qnet(state_size, action_size, [64, 64])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## DQN Agent\n",
    "\n",
    "DQN agent that implements prioritized experience replay, fixed Q-targets and Double DQN enhancements."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "Experience = namedtuple('Experience', 'state action reward next_state done')\n",
    "\n",
    "def get_fields_from_experiences(experiences):\n",
    "    states = torch.from_numpy(np.vstack([e.state for e in experiences])).float().to(device)\n",
    "    actions = torch.from_numpy(np.vstack([e.action for e in experiences])).long().to(device)\n",
    "    rewards = torch.from_numpy(np.vstack([e.reward for e in experiences])).float().to(device)\n",
    "    next_states = torch.from_numpy(np.vstack([e.next_state for e in experiences])).float().to(device)\n",
    "    dones = torch.from_numpy(np.vstack([e.done for e in experiences]).astype(np.uint8)).float().to(device)\n",
    "    return states, actions, rewards, next_states, dones\n",
    "\n",
    "class PriorityDoubleDQNAgent:\n",
    "    def __init__(self,\n",
    "                 state_size, # (int)\n",
    "                 action_size, # (int)\n",
    "                 replay_memory_size, # (int) max size of memory\n",
    "                 replay_batch_size, # (int) number of items to sample in one batch\n",
    "                 replay_priority_exp, # (float) priority exponent controls the uniformness of sampling\n",
    "                 replay_sort_freq, # (int) number of items added before the heap is resorted\n",
    "                 replay_start, # (int) number of experiences to collect before learning starts\n",
    "                 qnet_hidden_sizes, # (list[int]) sizes of hidden layers in Q-networks\n",
    "                 learn_freq, # (int) steps between learning\n",
    "                 learn_rate, # (float) how much to learn in each parameter update of online Q-network\n",
    "                 discount_factor, # (float) discounts the reward from consequent states\n",
    "                 target_update_freq, # (int) number of parameter updates before updated the target Q-network\n",
    "                 soft_update_factor # (float) how much to update the target from online Q-network\n",
    "                ):\n",
    "        self.action_size = action_size\n",
    "        self.replay_start = replay_start\n",
    "        self.learn_freq = learn_freq\n",
    "        self.discount_factor = discount_factor\n",
    "        self.target_update_freq = target_update_freq\n",
    "        self.soft_update_factor = soft_update_factor\n",
    "        \n",
    "        self.replay_memory = PriorityReplayMemory(\n",
    "            replay_memory_size, \n",
    "            replay_batch_size, \n",
    "            replay_priority_exp, \n",
    "            replay_sort_freq\n",
    "        )\n",
    "        \n",
    "        self.online_qnet = make_qnet(state_size, action_size, qnet_hidden_sizes).to(device)\n",
    "        self.target_qnet = make_qnet(state_size, action_size, qnet_hidden_sizes).to(device)\n",
    "        self.optimizer = optim.Adam(self.online_qnet.parameters(), lr=learn_rate)\n",
    "\n",
    "        self.learn_step = 0\n",
    "        self.target_update_step = 0\n",
    "\n",
    "    def perceive(self, state, action, reward, next_state, done, sample_exp):\n",
    "        experience = Experience(state, action, reward, next_state, done)\n",
    "        self.replay_memory.add(experience)\n",
    "\n",
    "        self.learn_step = (self.learn_step + 1) % self.learn_freq\n",
    "\n",
    "        if self.learn_step == 0 and len(self.replay_memory) >= self.replay_start:\n",
    "            self.learn(sample_exp)\n",
    "\n",
    "    def act(self, state, epsilon):\n",
    "        state = torch.from_numpy(state).float().unsqueeze(0).to(device)\n",
    "\n",
    "        self.online_qnet.eval()\n",
    "\n",
    "        with torch.no_grad():\n",
    "            action_values = self.online_qnet(state)\n",
    "\n",
    "        self.online_qnet.train()\n",
    "\n",
    "        # Epsilon-greedy action selection\n",
    "        if random.random() > epsilon:\n",
    "            return np.argmax(action_values.cpu().data.numpy())\n",
    "        else:\n",
    "            return random.choice(np.arange(self.action_size))\n",
    "\n",
    "    def learn(self, sample_exp):\n",
    "        experiences, sample_weights = self.replay_memory.sample(sample_exp)\n",
    "        states, actions, rewards, next_states, dones = get_fields_from_experiences(experiences)\n",
    "\n",
    "        # Get actions for next states from online Q-network (Double DQN)\n",
    "        expected_next_actions = self.online_qnet(next_states).detach().max(1)[1].unsqueeze(1)\n",
    "        # Get Q values for next actions from target Q-network\n",
    "        Q_next_targets = self.target_qnet(next_states).detach().gather(1, expected_next_actions)\n",
    "        # Compute Q targets for current states\n",
    "        Q_targets = rewards + self.discount_factor * Q_next_targets * (1 - dones)\n",
    "        # Get expected Q values from online model\n",
    "        Q_expected = self.online_qnet(states).gather(1, actions)\n",
    "        # Compute TD errors\n",
    "        td_errors = Q_targets - Q_expected\n",
    "\n",
    "        # Use absolute td_error as priorities\n",
    "        priorities = td_errors.detach().cpu().abs().squeeze().numpy()\n",
    "        \n",
    "        # Readd sampled experiences with new priorities\n",
    "        for experience, priority in zip(experiences, priorities):\n",
    "            self.replay_memory.add(experience, priority)\n",
    "\n",
    "        # Compute loss with importance-sampling weights\n",
    "        sample_weights = torch.from_numpy(sample_weights).float().unsqueeze(1).to(device)\n",
    "        loss = (sample_weights * td_errors ** 2).mean()\n",
    "\n",
    "        # Minimize the loss\n",
    "        self.optimizer.zero_grad()\n",
    "        loss.backward()\n",
    "        self.optimizer.step()\n",
    "\n",
    "        # Soft update of the target model every target_update_freq times\n",
    "        self.target_update_step = (self.target_update_step + 1) % self.target_update_freq\n",
    "\n",
    "        if self.target_update_step == 0:\n",
    "            for target_param, online_param in zip(self.target_qnet.parameters(), self.online_qnet.parameters()):\n",
    "                target_param.data.copy_(\n",
    "                    self.soft_update_factor * online_param.data + (1.0 - self.soft_update_factor) * target_param.data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 100\tAverage Score: 0.42\n",
      "Episode 200\tAverage Score: 3.75\n",
      "Episode 300\tAverage Score: 7.52\n",
      "Episode 400\tAverage Score: 11.15\n",
      "Episode 500\tAverage Score: 12.65\n",
      "Episode 516\tAverage Score: 13.01\n",
      "Environment solved in 416 episodes!\tAverage Score: 13.01\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEKCAYAAAAfGVI8AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvIxREBQAAIABJREFUeJztnXecHMW173/VPTObJO0qI4FAEkkgI5LIwYBtoo1zwPa1r42Nc3oOF7CxzbV9DU48+9pcjMMDhws4YmNABJEzAiQQQgJJKKAcVtLuandS1/uju7qrq6vD5J2Z8/189NmZjtUzmnPqxGKccxAEQRDti9HoARAEQRCNhRQBQRBEm0OKgCAIos0hRUAQBNHmkCIgCIJoc0gREARBtDmkCAiCINocUgQEQRBtDikCgiCINifV6AEkYdKkSXzmzJmNHgZBEERT8cwzz2znnE+OO64pFMHMmTOxaNGiRg+DIAiiqWCMrU1yHLmGCIIg2hxSBARBEG0OKQKCIIg2hxQBQRBEm0OKgCAIos0hRUAQBNHmkCIgCIJoc0gREARBjBKWrN+FpRt21/2+TVFQRhAE0Q689RePAgDWXHVBXe9LFgFBEESbQ4qAIAiizSFFQBAE0eaQIiAIgmhzaqYIGGMzGGP3M8aWMcZeZIx9wdk+gTF2D2PsFefv+FqNgSAIgoinlhZBAcCXOeeHAzgRwGcYY4cDuBTAQs75wQAWOu8JgiCIBlEzRcA538Q5f9Z5PQDgJQD7AngrgBudw24E8LZajYEgiOqyfTCL3cP5RMe+un0InPMaj6hyBkby2DowUvF1OOdYvW2wCiOqP3WJETDGZgI4GsCTAKZyzjc5uzYDmBpyziWMsUWMsUXbtm2rxzAJgohh/nfvxQn/dW/scc+s7ceZP3oAf3gi0booDeUNP34Qx39vYcXX+cMTa3HWjx/EM2v7qzCq+lJzRcAYGwPgrwC+yDnfI+/j9nRBO2XgnF/POZ/POZ8/eXLsSmsEQdSJkbwVe4yYGT+3fleth1MxWweyVbnOs+vsZ12zfagq16snNVUEjLE0bCXwR87535zNWxhj05z90wBsreUYCIKoP8IjZDDW2IHUEct5aKMJczFrmTXEAPwGwEuc859Iu/4J4MPO6w8D+EetxkAQRGPgjqHfPmqguZVfLXsNnQLg3wC8wBhb7Gy7HMBVAP7EGLsYwFoA76nhGAiCaABCKDahTCwbqwkC42HUTBFwzh9B+ITgDbW6L0EQjUeIRNZGNoF45ma0CJrQm0UQxGinHS0CkSrbjM9MioAgiKrjxgiaUCiWi6v8mtAKIkVAEETV8dzlDC9t2oOZl96O59bVNr9++Wb7Ps/W+D5hWCVYBAMjecy89Hb86en1NR5VMkgREARRMpYVHRj1/OXAAyvsgtAFL26u6ZjuX27f564a3ycML2so/tjNu+1K5usfXl3DESWHFAFBECWTLcQUlelmxzVOqnHz+Bvkj/IeL/7+YoijJdOIFAFBECUzki9G7pfFW73ksrBSkszIdVTaF8k7P8l1mHNORbesGqQICIIomZFCjCLQFFfVWuZZFRZ0xXi7YhHPXIzvwCGdMzo0ASkCgiBKJq7fkJtKCc9RUmuhV6lrqFihJhD3LyZ4TnGvSpVPtSBFQBBEyYxG1xCvUBFU6q8XQj0ukA4Aecds4DW3k5JBioAgiETIAm44ThG4BWX1C9wWeWUxgkotAnF2kuu4FkEJbqRaQoqAIIhEyC6POItAzK5f2rQH/3XH8orvfcvT6/CL+1dGj88RqkaZmiDOpXPf8i343u3LQvfzElxDBUkDZAtFfPx3i7Byq39Rm1/cvxK3PL0u9lrVoJZN5wiCaCHkmW6+mGz2/OSrO93XlXhe/uOvLwAAPnPmQaHHVOwaipnJf/SGRQCAr19weMj9k10HAArO58c5x6I1/bhn2RYMjhR8x/zwrhUAgPcet3/s9SqFLAKCIBIh+9BjC8o0u2ufNVQf11A2JGOqlGBxQQoWj4aaAlIEBEEkQhaUhRihmUQYVhsxJLNGriHB7r36NZtLsQhEsNji3O1NVGmMohJIERAEkQg5sFmMiXLqhFqtdYPX6ye5IuA+Kyf62M60LS53DesVgWsRlBIsliyCRihPASkCgiASIQuquKKpRsxuS+n1I5CHGSeIe7vSAIBdMRZBkvCJF2PhbkyDLAKCIEY9ftdQtCaIcx3VAjG+UoLFeUmjxbl0xnXaiqB/b067X9QEJHENibFy7imuQsIAfC0gRUAQRCIsn0UQEyPQKIpaF0+VEyyWnyPumcY5FkFYjMByLYLk6aMW5xQsJgjC4+UtA7hn2ZbIYxat2YknV+9IdL31O/fiH4s3VGNoAIJC877lW7Bs456QY4PbhJzjnOO6B1fhD0+srdrYgPLaNciz8JFCEb955FUUQvxe4zrtbHvZIsgVLPzmkVdx7QMr8cxaex2EV7cN4V/Pb9Re48+L1uPmp9ZhyfrdAEQmla0JGmFFCaiOgCBGCWdf8xAAYM1VF4Qe867rHo89RvD2ax/F9sEc3nrUvlUZn6oIRF69bixRweStA1lcdaddZPae+TOQSUXPR5P2KLKkAGxS8tI4H1u5A9/51zLM268Xx82cEDi2K2MCAAazXr7/rx9ZjR8sWOE77pZF63HLovV487zpvu3bB7P46l+e922TXUMUIyAIoupsH7Rnrkl81kkoJX00av/enJeHv2tY72+XySVs5ylcK6W4WORnEh1V5fH5rm8Fx7NnuKA9VsduTbaRxT2HWVzcpZaQIiCIFqdaaYnydeKEbZTykdtThPnb/ccnVQT231Jm1nKwOOcsthPWPkM8c77gXb+UeMTASFBpcO5ZPI3sO0SKgCBanGq5HGThHpfhEmURyII2LCc/7PgohKAuRe/Jn028IrD/ysojKkNJVYa7NNlGFufudckiIAiiZlQrG6VYUtZQ+H65c2n/ULxrqFRFUMrzyj2ThIAPu5+YucuKICpTVVWGOtcQ557CkD+zei9YQ4qAIFqcalkEvmBxjKDSWQRCuGUlV08yi6BE11CZMQLPItDfT1w351MEERYBVy0CjSIAd69bSiprtSFFQBAtTrU8Dv4WE6XHCMSW0mMEtXMN+WIEMRaBeCShMIDoZepVZagrRLO4N95CCcH4akOKgCDKZNtAFht3DVf9utV2C9QiWKzGCCyLY+mG3d7+EEHGOcfTa/rd93e9uDnyni9vGUhkNYgxyH+TUPBZBPZrYRFsHRjBpt3DWL1tEAMj+ZJdQ8VivEVQtLi2R1HSTKlqQXUEBFEmx33vXgDJcvpLoWhxpMzqrexVD9fQrx5eje/fuRx/+eRJmD9zQug9//7cBvz20Vfd94vW9mPl1gEcNGVs4Ng9I3mcfc1DbiFXHF6MINHhAPz1DkL4ihjGFbcuxVC2iEdWbsecfcZiQk8GgD+uEBUsVj+jPSN6RSBcZb703Dq3myCLgCBGGdV2C1QrWOxvMeGfsT7vWAMbd484+3UxAmDtjr3u+6vfeQQAYOuerPZ+I04+/x5N2qUOMYkuxQLKFWSLwL6fcA3t2pt3hffyzQNe+mhR7xr60ydOchvTAcEsoDDlKFxG/oV/6msRkCIgiFFGtQOF1VIsYQVllsVdt4nJotsljJVm94fuMw4A0B8WJyjRKBKCtxTXmixwxUxfLDyTL1q+Z9bFCORlMfu60+iQqqTV7zFOERRIERAEIah2X/pqVRaH1RFkC57ANB2JEtZ0bkyHpwimjusAkKy6OAlej/9SLAJdQZn9N1e0fAJZFyOQSZuGb1EcVfCHjWuHJoWWXEME0eaoQcaKr1cti4DLbhRPGI7ki4EW0LpH4BzoTJvu+/Hdts89rL9/qQgBXcpkOqqyOF/gyErPKZ5RjhHIAjuTMnwxA/VzDxPuOweDiqBlLALG2G8ZY1sZY0ulbd9mjG1gjC12/p1fq/sTRLNS7RhB1bKGQlwXI4ViYJnIsKZz8jU60ya60qa20KqS8ZXiGspp0kdFsDhftHw1DzrXkPycaZP5LAL1ewyzCHRppTmNJVJLamkR3ADgXM32azjnRzn/7qjh/QmiKal2X/qquYZCLQLPNSR85mGzXzWA2tedDq0uLrX+QczUy60sVi2CXNFyG9HJ1/XFFaTPNqO4htTPXbUQ+rrtwHKca6gexWU1UwSc84cA7KzV9QmiVamWRVDpWrjqTFT2VmSLeteQCBaHCeO8oiB6u9KhdQK6cUfNjr3FXvT7h7KFQLGYvqDMcvfJFsEeZ5zybF1euyBtGr4mdOr3qOrG8d0ZGAzYqVEE8rjqsZZxI2IEn2WMPe+4jsY34P4E4WPmpbfj4huersu9dgxmMfPS2yMXZSklRmBZHDMvvR1XL1gefr0yFMvm3SOYddkd+Ptzr2mvk5csguF80RX8X7/1Bcy89PaAwAfsymLdrFhUF6/cOoiZl96Oe5dtweL1u3DKVfcFrhH1KGIWrXveXXtzmPutu3Dq1ff7tvuzhpQYQZG7GUSAN3MXx137wEr86mGvJkINFr8iPQ8QtBDSJkNvVzpEEbSIRRDC/wA4EMBRADYB+HHYgYyxSxhjixhji7Zt21av8RFtysLlW+tyn/X9diXynxetDz2mlBmgmHVe/9Dq0GPKaTGxvt/O9//DE+u868iuIVmASllD63fazycHWQWce0L0L588CYAdJxDCdvH6XQCAO5ZucoVn4FkiPpuoGIEQ4tsHs779uqwhMfZ8wfIpHrFfCOnfPrLGd4+0yXzB4iecleQWONXTqkA3DQOZlKH9rGQXWj3aTdRVEXDOt3DOi5xzC8CvABwfcez1nPP5nPP5kydPrt8gCaKGuIIsqiK1BMkd1XFT3KEc14LI95cDuboGbYAtGNX7Z0P69YhrHD7driFImwZyqvXAgQFNFS4QowgiKov9gW69QhMCeThXDOwDPIEsrCH1K2TMHyweclYy63IypdTvIWUwmIxpl8aULZVqxXiiqKsiYIxNk96+HcDSsGMJohVx9UDEMaXMAL1ZcPwxpZByBJpOERgs6FJR76Fv3MbdZxMCM5My3Gsx96jwauKo53TbOWsOUrOc3O1SZbEQ8COFIjjnAUUgnjGqD5BsEQhF0Jk2fOMTmAaDYTCt4pIrnuthEdSs1xBj7CYAZwCYxBh7DcC3AJzBGDsK9ne9BsAnanV/ghid2D/qyGZlpSiCCMnIGAN4cLaeBDEEnSLIpAy/S6VoBQKh8rq+MkIgpw1bOGZMSRFIn8mekAByEotA5xryLVKfL2JcZ9o3HsALgIvgd9itckUrNGgtj2/QVQR6i8A0/BaEb7ySVViPGEHNFAHn/CLN5t/U6n4E0QwksQhK+eEncRuUI0jEOH05887GjOn3a+eLVmAcOkXAuT0Wxrw007TJfPewj+PaBm1AdLBYyE6dZ02eVcuZQPqCMity1i+eQ6cL5M9afAbCStBZBGbIjMCXNdRqriGCaHfETzpqQZNyXENRlONj1s28LdciMGNdQ2F++nyRu9YAYMcIVIuAI3xR+CTBYp2VJPvh5RXSwgR+mEUj0GVFqePrH7KVmXBF6WIERohF0OpZQwTR1iSxCEoR3JGuoQTHhKGd7boWAVOyhpK5nyxuB8LlFttp03MzMelTCQsW84g4uhcsDo5FFqZy/CJftJDWtPwOU0SCMAUi32f7oN1V1Wsz7T82yiKQXVktlzVEEI3Gsjje/6sncP+K6HTRW5/bgE/8flHV7y98y1F97DftHsF5P30YGxIseqO6QR58eRs++OsnfcqknBlltEXgjxFkNRaBjkdWbsOvHn7VpwTtYLE/bsJ5eLBYHteyjXvwlv9+BDc8+io+d9Nz7vgeX7UD7//VEz4rIO9TBBb+vGg9vnjzc8gVLHSmvP5HgjBFJMgVLG2cJyVZO8J9dsNja3Df8i2hwWIdl//9Bfc1WQQEUWWGcgU8tmoHPvvHZyOP++Iti3HXi/pc9kpwf9MRJsGfFq3HS5v24HePrQk9RigUdbb/8d8twiMrt2NvPtgaoVLkYLFv0feClcjq2OKsOzCU88aWNlmgwRpH+IxbfparFizHCxt249u3LcNtSza6Y9i0ewSPrdqBTc7aCIDfNTSSL+Krf3kety7eiHyRY5y0hoBgIGYNBFU4//z9RwMAvnz2IThi397A8R+9YZE+fTSBBCZFQBBVRszSOtLBWWA94CJrKOIYkUmiKzQSCNmgzjJ96+mKFhNlFJT5F6ERSsd+L7tzAH2wOClp00DB4onPlw+Te//L4xQM5TxhXghxDeWKFjrTRmAVtDBFICbwqlB/87zpAICz5+6Dn7znSP3YlfEZLNw1JKP2Z6oFpAiItsJVBKkG/dd3ZEGUayiVQBG4wjlEgOoEeSnIck5U/oa5hvLFZBaBjozzPeQtyw2gc869qHpgXN4O9TtUBa0szH3po/LYCxbSpoHxzjKUgrCspZTp1QSEPXJYSmjAIjDDXUMyddADpAiI9kJUvDZKEXhZQ+HHCCWhplXKWCGuIfc+lhd8rTRGIJqwyemj/vbNvGxhlXEEq/ysUaOVHyWjWgTKZyF3NZVn1SM5f7A4kzLQp7iHwuoY0kb8ZxqqCDQtJsgiIIgGIISaKkTqRZKJs8iqicpld1tLRFkEFXQflc8QrhQ5RiCjSx9NStpRBPki97nLeIg6KEZaBP5j5a6mfotAVgQcadNAb7ffIghzDaWdexY5Dy0qC1MEgWAxQyKLgGIEBFFlhBDo0GSK1AMh4KJcQ6aTeRLWrwfwZsZhQl6e0Zfjv5eFnMi7d11DSoQzabBYh6cILG/MkUVj3k51HOoYdu/V90lSYwRpk2G8szaAcMuFZQ2JrCDL4iFtNPyZQ1HjMw3DvV8UpAgIosoIt0CjXEPiNx3tGrL/RlkE8TECqY6gLNeQ99q1CHi4RVDuKloihz8ndTANswYAv0WlBvzV55TXQs7LriGpsjjrxAiEa2iMEzQOtQhMsRQn98UaZEL0QGB8qYgWE1Hn1QJSBETL8cTqHVj4kj7107UI0o1yDcX/qMUP/4EV20JrCW5+ah1e2TLgc4foFloHgjPRpRt24x+LN/i27RnJ41v/WIp/LtnonO/tE4JTzMZTykz8xsfXYrtm3d0kuMFixb0U9jFZEa4hlf69etfQgqWb3dcjuSIykmtIdAr923P+z0eQlmIaYQI6zCJQi9QMg0Vahu7YSREQROm87/oncPGN+mIwIdSEa6ge68HKeMFivwCQxyELrU//4Rntdb5/53K86ZqHfEJ+b06uHfCsDtU19LvH1+A7/3rJt+2Ztf248fG1+NIti53zvXOykkVgGiyROwMAPvn6A3HkjL7IY+QYgbhnZIdRaafqGlIZHNGnjy7btMd9PVIoIiW5hrpi0opF/Eb+rC87b47vmLBZfq5oYebEbu9aCSyCjGm4n1EtIUVAtBUjStZQnfWAqwnUn3/owvD56IwRee0CWZlYERbBcN4KZKKIrB1dW2u3V45lL0WZZBYLAB87bRZu+PfjIo+RYwRJZr5RWUMqvsVdnM+0J+MX9Nm8hZRpuOsHj+SL+I9z/YLdN14Rv3E+ryvefDg+8foDfcfIivLtR++La97r1RWcNWcqPn/WQQCcyuKYz/LZb74JJx04MfKYakCKgGgrPIvACfrV3SJwCsqU378srOXYQE9H9AxVDiPIQjIqWDySLwYWlw9U92rSRy3OYRhIbBF0pc1YQSd87tmCvygt7FuRxxU3m9b19O/p8BeOjRSKSBsMfV22a2hgpKDtPSRISTENefwy8rgMxnxWhml4mUIGi68s7qxTLIsUAdFWCIsg4yqC+t4/rOmcPEGXhbQquFRkS0IW/nb2qD7nfSRfDFgEujYPArFiV9HiMBmDGSEoZTrTJliMhNHFCDgPd9nJjxKnw+VnEp/pGKWCWLUIBrKFSEtDWDCiyE4XD5AVgWn4g9pyXICxeGWmxmNqBSkCoq0YzvvTR+MsgmrHEMTl1JmybBHIAqw7E20RhM38/a4h/znZvBWwCNTiNV9BWUFSBBEdM2XSJkvk+shIriExJB6RN+RTdpGlZ4oisOwmcT0ZRREUikgZDH1SHUGUT16MV3QU1VlH8udjMOZramdKy1kyRKcR1xNSBERbIQKf4sceJ+erbTEIQRZwDckxAqtciwD616pFUCiiYPkLouR1g9X2CW5lsVAECVxDQvjFHeqrI0gUI9A/rw5ZuRUsex2ETiVbzOLwBYvlMelISa4s+b2MYTD3+zUM5runWaJFUC9IERBthcj9FrPJuFll1S0C95VfAMhCUO6UGZciKbt4LCVY7DadU55BrRQGvPV6AVtR+FtMlJ41JNwh8TECkY7JpaUmwxU09ym7EiyCogXTYNpCwpRhYGynrAjCx6y6hsIEecqNA3hLVdrvPSXBEjadqwekCIimZsn6XbG942WEUBMyJG5WKe9etW0Qm6XWxmG8smUAty3ZiIGRPJ5Zu9N/PXc9Av85Ya4hud2zTintCqmelY9du2Mv1u4Yct8L95icpaNmKsl3yuaL2D6YxUub9jgBzgQWgTMLjpNzmRRz7y/GHyXgl23cg+2DWTz16s7IXkyAbeU8s3YnlqzfhS17skiZLGQNAf8zRaWlpgPBYv2x4npmIFjMvDhRwhYT9aBmaxYTRK3JFop46y8exUmzJ+KmS05Mdo7j5hCCMm5WKe9/w48fBACsueoC3zGcc19dwMd+twhrd+zFmI4UBrMFPP31N2Ly2A7nevYxqkCyfEJZL6B1SktWBP7Zsmdz3PTUOtz01Dp33MLVE64Iij5Fki1YOOtHD2DPSAHTejt9QnNsRwoDmmUduyIsgpOldEi5QEt8BlFppF/76/Oh+1TWbB/CO//ncfe97P6RkYPfpxw0Mdo1pKSPhllHYqZvu4YUReC2IieLgCAqRnhFnl3Xn/icgjvrtN9HLX0IJKszUI/Z0G9XA4t1b8Py+2V0FkHKYKHVwuqx6rWjFJywimQXlBwjsBWBfA/urhhmML9r6F+fPxVHaYrGOkMUwdXvPAJ/uPgE973YX+Sea6haLRWGlV5AcgbO8bMmuK9FbcDK752H33/0hMisITV9VBcjADyLwGD+GIHBmO+zHS0WASkCommRfcqJz7H8MYJq1BGoV4jyiwshF8ga0szOuzOmLxdeJx8Lll74W1Z4Ln421iKwfPeS4xDq8oop09AKbiH8VDk3sadDOd9LcRUWgZrKWi3S0n1l60CMIWUaMAyWLGsoIn1UXAuwP69OpY5AYLDkNRm1hhQB0bQkaVIWOEdYAm6MILlrKOkxAbePJvc96BryXovUzkzKDJ3xC2Q/uWoR6AR00eJuwZqcQpqLCBbLCkMNFpuMabtwCuGnttLoU9wzYuZctDyLIK/mu1YJ2QUkZ2OpwljELXQkyRoC4MsMkgP+tkXgZY5R1hBBVIiYQZZjEbiuoZjjy3ENBRVBcNbOlKwhedYthH9HyoidHfvaKChN23SKI1vwL8qiez2S87uGZBeSwbw22YDdaVPu7y8Ia/Pdp/T9Fz7yosXdKum4IHC5pKVxy/UEatFWdPqofyGdUItAChbLylAOFgNUR0AQFSNmkKW4d7xZcLJzk1xZvYb649bmvkcoCzEj7kgbsRZB2Ky+yLnWlTQsNUsL7W9U8AeL5XuYyoLrJmMYzgUFt5qvL1AtAiFIC5bXdK4SRRAlV32ZQdIsXbUIkrmGksUI1Bm/aTD3e0nSYqJejJJhEETpuBZBKecI5eHImjgdUk4dQVT7CCs0RuC99iwC01/opRlKPkQRhLmGRnxFVlKwWIpFqOmjahaPbBGkDMNnZQjCunj2KktCCneNZXnjjVqHIY502GIA8M/8ZeEfpwjGatxIXowgWhEIa0AOHgsYKFhMEAE451i/c6/PFSGTL1q+mWuSYLF6TkGJK6iFU6rgT5LAYnHuu4/qF7c4d1e08lxDfmShLcbYkTIwOJJHUTlXfT6Brw21Tgnki9i1V17H128RiGHvGc4H2jMIOPcLP8Pwgs8ynSGKQBWywjU0lCu433s1XEO6ojB5m6kEvGXUOoJxXenAsXGuIfFZiucT95bTR0EFZQQR5J9LNuK0H9yPb9y6VLv/2O/cg2O+c4/7Pkma4bHfuQfH/GfwHF1B2ZwrFuA3j7zqv0DCGMFx37sXJ/zXQgAai4BzXL1gOeZcsSCQ0qiOS6YjZWDVtiGc99OHMOeKBfj1Q6sDx8hC+srbXnRf6wKuc65YgAt+9oh3rlKvMM6prr30by/gCzfb6xKkTeY7zu5A6heoxx4wPnCvsJx9FSGQ/++9r+DGx9cCqMwiEN1ae7sygX2mwTB3ei8AYOq4Tnd7wCJQgsUnzpbrHvzB4rAqZMO1BMR5ThYR81xDDNHBYt3nWiuooIwYNWwbyAJA6Kpce5TlA9XFylUsiwfO8apXvWNkbluyER87bbb7PklGksW5v8I3YFVw3Pz0egDAUFbk4/uvsUdTHS3aNLy8ZRAAcOvijYFjZJfOYFZeiMX+cM573T546tWd2DEUXEFMtQgmjslg97B/HGnT8Aeh4RecadPA9R86Fis2D+Bd19nFWz981zycPXefwP3u+/LrA9t0rpVKLILLzj8M3RkTr2wZxE8XvuLblzYMfOXsQ3D23Kk+KzIqWPy3T5+MWRN78NdnX/PtE4ogTJCLrcI6HNORwsBIwXc8Y4ARmDYAf/3UyehMG9h/QndgX60gi4AYNXgN2ZKZy3ELpuv2q66h4Bii3yc5R/Wpy6uFiX3qM8qKRKBONqNcQ/v2dfmsAHGfefv14aw5U7Tjll1w2YLl67cjSJuGUtSmuFUMhrGdaRw+fZy77bBp4wKxAACYPXlMYJvOR15JHcG03k68ed50rUViGgwp08Ax+48PPIOMHEg+Zv/xSMuB5YBFEC1CxX2EtWUazDVHGfTtOo49YDzmTu/Vfh+1ghQBMWqQTeYkxLmGwvLogfA6AvV9omCxqggUQWZx7roKhJtFfcZ+x3cvt51WVyfTDUXM/DMpw9c4TtzHiMhVV1tZdGiEWto0Aq4h2a8tFJocBC01JVI9vBKLQIwto2suZ+qFv5r5o8YI5I/PXaEsH910zu0+6vyXOypbAAAgAElEQVQd66yDYEhZQ1RHQBAaiu5sOdnx5RSDeYrA7yISqKckyRmSs2YsK5i2aVmeQAgLKAuXzIQez7etxhP0LSbsbRnT8PnWhTJSK4Fl/OmjXNtaIWMy7M15LifOw1MmBRGJO1rUx6pksXYhWPXBYiNwHBCfNSQrNrfpnGgDEvJZqApSKAKTyb2GmrCOgDF2KmPsI87ryYyxWTHH/5YxtpUxtlTaNoExdg9j7BXnb/2iIcSoR/jr62ERWAktgiQ1CjtDMnH81/C7FNTf/669OXSlTV+hk1qxqxuJUCwZpfhMdkGFZabklSI2rfBMGb6mcpbTilqlEougmoix6ZRamPBXM3/U55OfR8QTRKZUWLqqOENcS7h5csWir7q8qeoIGGPfAvAfAC5zNqUB/CHmtBsAnKtsuxTAQs75wQAWOu8JAoDcmTNhjCBGEeiCyV6MwCbW9ZNgcrpzSFYEwZvariH7tZhJBl1DefR1p31CSE4HFddREW6btMl8logYhxnhfigqNQg6f3faNDA44rcIdIpFvkUjFYHhWgS6Z2GB44AEFo60Wy0oC1u203MNOTGCLlvBD4wU3P9SjMWv4FYvkuqjtwO4EMAQAHDONwIYG3UC5/whADuVzW8FcKPz+kYAb0s8UqLliQv+qsTN1nXXc5vOhbiGAjGCBOPoH/ICvXqLwBMMuVCLII++7kyMIgjeW7YIZISCiFouUm1rkda6hgxfNpK4pgrzWQTa29UFL29fZxGEFZQlC/gCXmqpcAeGWwSOa0ixCPYM5/0ZS9ICNo0kqSLIcfuXwwGAMdZT5v2mcs43Oa83A5ha5nWIJmQoW8Dx37sXj6/a4W57YvUOHP+9ezGYLUi9g5IphEpcQ+v7h3H0f96NV7cP+fart07iGuqXXUOa/P23/eJRbNljp8YKRfCnRa/5ahZ2D+fQ1+W3CMQs0htbeBaUGhz1uYZCfuW+quSihbTBMGmMP/8+nTL8hWqc65dnHCUWQWSMQKl/EMRZBLKSC6xHEGMRCMU01VmPgjGGMU6tQ0/GDG1FUW+SKoI/McZ+CaCPMfZxAPcC+FUlN5YViw7G2CWMsUWMsUXbtm2r5FbEKGH55j3YOpDFD+9a7m770V0rsHXAXv1KCN2kdkGsRRChCJas34X+vXnc/NS6yGsm0UmyL1/nGpKRM2K+869l7mvhGpJnqr/98HEAvFljmEWga2csVm0b05FKFCy2LNv//ZdPnuxLN80ogo5zvaBnNY4RyM/30VPCw5NqcNZ3jbDKYs3n8/P3H407Pn9aYLuwvHIxC9N447H/fuDEA/C1cw/FxafOwsdOm42vnXsoPnDiAQGLoVEkUgSc8x8B+AuAvwI4FMA3Oef/Xcb9tjDGpgGA83drxD2v55zP55zPnzx5chm3IkYbQljIk2YhMyypDXHSpJG4dPOoOgJBXIpmkqH42kPEtFAOy5EXriFZaM+Y0I2j9++TFtPRZw3pFpQXBWS93enwYLHS0M5gwMxJPXjvcTPc7aqLxeI81pVSatZQEmQh/s23HI7Dpo3THic+B7my+MIjpzv7QrKGNCbTm+dN99VGCDpShjfbN1hoPMvNGpJiFp8+4yB0pk10pk18+oyDkDYN97tJWoldK2IrixljJoB7OednArgn7vgY/gngwwCucv7+o8LrEU2EmK3JLg7xg+HwhHC1XEO6fjvqNrVhmnqGbixFy585IyucOEWga5/AOceuvbmARWAwf2BWW0dQtOwulopA6ncUwXgl7qA+h/zabYvgS5dUFUG8G6M2FoEBwHKDteHN3uy/cpdTnbsoFRIviCPjCO9CSPaUQOyJ+yyEopjQk8GqbUORx9aSWN3NOS8CsBhjvaVcmDF2E4DHARzKGHuNMXYxbAXwJsbYKwDe6Lwn2gTxu5FntuJnInfLTBozLidGoFoE2YJqEcS7hlT3T9FX0Rttpqj3A4ChXBEFiwdiBIz5awDCKotNgwWyV3Y6AWz1mr5zfaubeUIprF2zTbQABGqjCMQtxXjiFoTp6woqAnncsrKLixHIpFOGN8uPUgSu1RB9PcO1CIK9kepJ0l5DgwBeYIzdAydzCAA4558PO4FzflHIrjckHx7RSoj/9LKs9KwEOcc/oUVQVtZQjCJQjtfPwjmkzsR+iyBGOemqZkVH0PHdGXd2qlamagcHxzWkrCMMADuHsu41w1xDRWU9ZHEJ+XC1yjaRRVAD15BaHxDX/lnXctpXUBZiHcQhu3N0LiWB+H8dpxRF0aBcSNgIkiqCvzn/CKJsxG9ClmfiN8i5NxtPahHoXD9x+1XloArmYPqo3qqQLYdSYgR6RWDP3nulOgKve2W0RVCwLBiaGEH/3jwYs4OmYcFiWWkVpTYYvnTJQLCYJwiQ1i4DxnMN6YWw7t7i+cPWICjFNZQ2vc866jy1jiCMPcPed99IEikCzvmNjLEMgEOcTSs458EuWQSRAFlAi3xri/OSVxyLdQ3pLIKiahEoufqKnNYuGF+0fMrKv5ZATNaQJkYgFIHszzc0gkQ/FidYrAicnUM59HaltUpCIKePWmXECMJkXC0UgVCgca4h3bO6M3hf/KU811DGNNzvJsl5cdbTgFOsN64ZsoYYY2cAeAXALwBcC+BlxtjpNRwX0YLoGr3JVkLSdYQFsa4hSXJu2TOCHyxYHhDEOp+9jJj5X/fgKt915TurPXuiUC2Cqxcsx6ptdptpO1hs/yR1zdx0axnknGCxKpR2D+ddv3OYa+hnC1/xWWHeguuSIkgFs4Zcn3vIdWuREi++p46EriHdeMJXKCvNNSRWeYtSeN73F3090X58nCbdtZ4kvfuPAZzNOV8BAIyxQwDcBODYWg2MaD2iYgDcWcVLvE5CvGvIe73wpa249oFVgWPU1bV0lcUj+SKuutOrfchL6+sCwewbAHj3sfvh5S0DWPLabt/1VEXwP9KYeqScfyFA4maUe3NF9IYEhMd0eB0vZb5+/mH46cJXMJgtYNtAFlPGdTquoeA91RgBuFwNG6IIlPtdcvpsTOvt1B4rc/KBE3HWnCn47u0vBfbtP7Eb+/Z14ctnHwog3D8vWzD/+da52DaQdRv6yc9lGAznH7EP9gwXMHFMcv98JmW43+HsyeF1tUmzhj59xoFYtXUQFx65L55dtwtnH96YGtukiiAtlAAAcM5fZow11pYhmg5LEwNwA8gxWUNJ/P0qsptGbeAmCKSPcr8i4pwHu2MqrqGCL0Zg3/Ndx+6Hno4U3vzfj/jOjVp9K22wgJCNm1EWLR4aEBbrBqsejI+fPhsTejL48p+XuHUU8spj8j3VGIHvuJCJtCr8Lj//sOiHcPjy2Yfi2APGaxUB58DvLz7BfR9mEXSmvUF96KSZAIBv3PoCgOCzXPuB0uexsqL5xgWHhx7HEiry/cZ345ZPnAQAuOa9R5U8nmqRVBEsYoz9Gl6juQ8AWFSbIRGtiq5yWPxOCkWvfbO2fXSCDCAVefY9ollg3b5XcIzyNs6D4ylEWARCKaRMvW8+H+GKkgvDks4oATjN6oJSucMRirpxiDWFxediSa4heUavxgg4PCFcbddQ2LKPQNCSCrMIOjXrELjnVCGdSbaQolI+S/n+RgNJFcGnAHwGgEgXfRh2rIAgEiPkpSw43WpjSbjqxLtO6McFlWV//UhOrwhUuHJdjqASUjOD/Omj3qLmOiGQjbAIUlJqoi6DJ4zerrQ2cCmEvS5rSMych53PRU4flQW8WkcgxwhCXUNlCr+oZ1VjOWEWge5ZxddTSlA4DHk9476ITB+1sni0k1QRpAD8lHP+E8CtNu6o2aiIlkQXIxC/k7zFpf3h5/q3ea8554Fy/1zRE/4jCVe94pKLSoxVzTQqWErWkFxQ5uv6Gbx+1OpbKakwLGn6IWDPTHXHdbquoQiLwHGZyZXF/oVY6pc1FLXso+rCKyXlU3w7SdubRyGPUXyGOsSdwqym0UZSW2khgC7pfRfsxnMEkRhLGwMQFoHlKYgEPYLk64Xt97mGQmIEgWsqriDO9RaBzzWkKShLm0bJgidlSjECI/mMUm1NIeh0ZvN6i0C4hkSMAFrfvy5YHNcxs9xJcJSCSeoa0uEuBFPesPz3Tfhw2oLAUUzST7OTcz4o3jivu2szJKJVcdNDdRaBJFwTWwQxPX5ysmsooSLguhiBcu+CZUWkj3rtiUttLSy7k7yc/vjzwrKGujJRFoH90x/JF93vQ5c1pAsWu732QxRduS2VRW//sR1BR4UaZC/FIhA2QTUm50mVu/iMqmGF1IOkimCIMXaMeMMYmw9guDZDIkrBsjjufnFz4pTLWvL0mp3YMZgN3a/rLip+J4Uid9M91Wre+1dsxZCyOArgF8C3v7ApILAffWW793rlDiTB4sDLWwak9xx3vbjZd8yOwRweXeldW5c+mgpxDYVhL1soZw3B+ZvMNRQVEI7at3LrIBav3+W7l881lAoGi4XCjuu8WSrCh69ryaz+9y7F3+9ZBHUUys0h/12Sxgi+CODPjLGNzvtpAN5bmyERpfDHp9bhiluX4ofvmod3z58Rf0INefd1j2P2pB7c95UztPuFACn6LALPNeQqCmnyt3rbID7y/57GibMnhF4PAL7y5yXIFSy8/4T93W23LFrvvt6wK9m8hXOOd1z7mPt+wdLN+Pn9K33HXPL7Z3zvZUWQlSpg49pNyIjZsNcDJ7lraExnSi/sNa6hj5wy097nKIIf3uVmhWuDwB1KFs5nzjgQvU5Dt0tOnxU7tlIQSvBTZxyIb9y6FCfMmoAnX1UXOfQfO747jf690U0OXEVQJeG8z7hO7D8h2iEibqVrUTIaiVQEjLHjAKznnD/NGJsD4BMA3gFgAYBX6zA+IobXdu4F4PWfbxTCIlm9PbyVrq5gTAidfNErKLM0Pncxa5VR3UWbdldupKo/W3UFMx1yvYJoF9HXlfEtah+HWq3rWQTx52ZMQ+sq6VCCxSfNnohvvWUuAE9JyOj82l1SQPTV75/vzvbXXHVB/MBKRPj9P3jiAfjgiQcAAF7r34tTr74/9NhZk3pw34ePw9HfCe+QL4RxtSbpT1we3zeTeZqgKYhzDf0SgPjffBKAy2G3megHcH0Nx0UkRMxAozIu6kFc100A+oIx5u3TpYMKAakuICNfr5qoQxAtAKKQx7Frbw4dKQNdGbMk15BwdagFZUn87WHxCM81FDxHl/Giu2dXxju51v7uqGdQSUufU5z7rNoWQRKEG6pJ9ECsa8jknAvb7L0Aruec/xXAXxlji2s7NCIJ7uLlVciRroQkQtmtI9CsR5C3LHflMismCCxSRctZVjIONdYi2hNEIT/7zqGcm19eShqlW6SlFD0luUbKMEKEqOG7hnwpvSII3jOqQKva6ArKukIUgficDMZi21676aN1dNy7PbSaRBPETSNNxphQFm8AcJ+0r7FdkggAniJotEUQtgSjjJsVpCkoKxS5Nr1U183TK0zzb9c1ZSsVVZ+J7pBRyNZQ/94c+pxlEkuZgQpXh/gaLTeTJ5lFoHMNdUUEi02DBVJDdZXFHRG58tVGV/kbZhGkpHqLOKuJe5qgbnjNFJtDE8QJ85sAPMgY2w47S+hhAGCMHQRgd9SJRH0Q1bONVgRJAqOua0jaJn6bctuGOIvA4hwmGIqKkujfm6s4e0r94SaxCOTx7pAsglKKidKKRSCumMg1ZDBti4morCF3u6Q7dW2o5d49tUanzMLGnirFNVTlGEESXNdQc+iBaEXAOf8eY2wh7Cyhu7n3KzMAfK7WgyPiEfnVwSUF60uSGIGu6ZxwExWKllZRaAvJRPaRYhHs3ptPvPB9+Bj97/ckUASysto5lMOcfcYCKM01ZCoxAmEdJXcNBbd3Kr2G1Eup/Zd0Aeow10wtKKUdg7CgGEv+Odczp1+3CNNoJta9wzl/QrPt5doMhyiVXILe6NVmz0geHSkDQ9kixnenMZwvYkAKqmYLRWQLTp98g6EzbWJgJO8WdVmcYzBbwMZdw67AK1pel0/OgY27hmEw5nbzlBGGgFrxu2s4X3k9hXJ6EgXXL2UH7RzyXEOlfCcifVQIQ881FH+uHSzWWASKf1/1kasflZs+asgWQf0UQSmIeIJtEcQc3ABpLJRO0kWWGg35+Zsc4ZuPa8lcTeZ9+25MGtOB7YNZfO3cQ/GDBSt8+993/RN4bp2d7rn/hG489LUzccS373b3Fy2Oj934NJ5YvdNdaCRf9FYo25sr4OSr7HDUjR89PnB/OdZgMG8W3783V7FFELfCmI6Xt7hF9xgYKbjLDpbS7FJd/lA8RxLXUDokRiD8+2H/Nfbt6/LVVzCta6j2iuDI/XoD6zaoTFTW9JWtHPH6tIMnac993b69+NtzG3DAxPo1Qzhm/z489PK2ROswjAZIETQ5QhHELdJSbbY7FcT3LNsS2CeUAACsc+ocZDj3hKdIfy1Ynmtob9ZzWQxruobKhWmmwfDo187Ed//1Ep5YvUOqeg0KwBevPAfrdu7Fj+5agYXLt2qfy+LA1HEd2LInvEI6DuFOUS2COfuMxXUfPBZn/vgBTaWsrTVErMd1DTlC7pCpY7B1IOvWKciYhhHZYVSgGig3X3IiXt4ygItvXOQbr1HnGMH/fvxE7Iyog3ns0rPQk/GLqrSUNcQYwwNfOQNTx+mF7kdOmYmTD5qIOfuMK3uMT339DSVZeJ8762Ccf8Q0HDJ1bNn3rCeNdSwTFZMveK6VRlCOIcIRrBmQF4SX3TFq10nAcw2JdXan9XZh5qRuxzVk75s73f+jnzGhCz0dKRw2bVysr3goW8SMCV2Rx0QhevyogmOf3k7MnNSDyWOCjXuFq0P8zVvC5Wfv7+1K48DJY7T3SxnRWUNhX9GMCd04VZpFuzECSSrUI320pyOFGRGVutP7ugKLu6trO8+c1ON+7iqMsYqUAABMGduJSZrvLQzTYE2jBABSBE1Pts6uIdUHX45P3uKaRm5Fr8WEr2WDppDMko4TAqGvK4Oixd0CMDWLShbKcd6WoVzBnXGWg6jaVfVNlLIWzyGC/uJY4aZJm/paASC+oCzqO5JTNoVVoS7pOBpJmUHrhSgfcg01OWLFq3q5htQFQvYmXPBFxtIt/2h5TedkP73WIpBdQ44gECmbwsWg5sjLfu842cF5ZYuYhKVtRi28IxSPUGAiLVhYL7brSB+/SBlG5JoDbhq95hh5iDrX0GglbeqVLVEepAianHoHi9UZen8J/XQE2uUfQ9pQ61pLuEtaWt76uX3OsoFiPIF0Wia/TJaSWS6dITECt7uq5qtyLQJVgfmCyGELtjOpWZ08DiVGoDmXaSylZlAEXrB49I+1GSBF0OTUO1isVu9GBfmiUIdbtLjWdRJrERh6iyDSNZRAxketnxuH19rBvz1KWQsLRG37HLcIjNinX5jG8Zkn/K+h+t1HM176aIMH0iKQImhyhAuhXsFidYGXcm+rWgT5oqXNuVZdUfK5RcsT8OMdRdDvKgK/hDBKtQgqqNQWaZvqbDVKWQtBro5bXMJkDCxkSGlT7xoS1hJPuDCLmz7aBNJV7jVEVA4Fi5ucnOsaij+2ULQw54o78SepT7/MqVffh5/f90rkNdRq1CS8/dpHA9tUAW+3mNDcT9M/SHYNCXktXENX/ONFANEWQRLZUdoKWH7CqnE9iyD4oELxhMU2TIOFpkeaRvRqaF1pe74Xl/UiLtEM7ha5+yhROaQImhzuzo7jC6EGswWM5C1891/LtPtf6x/Gj+6OLhrX+ezjkOsKQsc2UghxDekqi+3j8pblCvxJYzowe3KPe4waI2A+RZCkSEv/0/jE6bPxb06v/DDCirCirDYvfTTcNXT5+Yfhu297XeDclMEwpjPcuD9x9gR8/x1H4MoL50aOO0yo/vmTJ+HWz5wSeW69SVGwuKqQImhyxCQzQfNPLyWxgtlu0rV/oxjfHVyK0K4K1iiCiPTRfJH7ZtAfPMET0MHOmvrXYYRlDc3brw+Xn39Y5LlhRVja9RgcTCVrSCC7a7oyprtgi3qMsIh0MMZw0fH7o0ezFrBMmCI4buYEHDWjL/LcekPB4upCiqBFSNLTJEoRJK0HqIYi0M2Ydw3ntT50fbDY/psvWD7BKQdaI11DCcYYljXEWHyDv7AiLF1TPYFwdWRS/tEJfRTnAunrCirXUmmC0IALBYurCymCJkcIlSTB4nxER8t8wvV1q6EIOjSCdPfevDarRucaEs+aL1pIS4JTXpxHFda+YHEi15D+GIPFW1RhrqEoZW0a0a6huJhFd0hVbSk0k79dt74yUT4NyRpijK0BMAC7G3qBcz6/EeNoBbimGjcMUXymE2RJFpYByosRqKgLogN20HsoG1QyOsUjnjlXVCyCkNeAGiPQj8s0mPs5hmUNJVEi8a6h8GBxIDYhXEMx6azVcJE0Q7aQgArKqksj00fP5Jxvb+D9WwIhXJK4hkSGkW4WldPMvHVUahEwBt8sXmYwG1wNTJ8+av/Nqa4h6bW6dGeS9NGutOmOIR0iFJPInXCLIPwcYYGoloxwl1WSxZSUZhKq6trORGWQa6jJ8ZZtTKAIxNoFmm89uUVQmSIwNStKRS1+ki1YgcCvFyz278skjBGEydS01LMnLFicRPDoXF9Asl5D6rgLEe68atNMQjXl1hE0eCAtQqMUAQdwN2PsGcbYJQ0aQ0sg+vIUOcevH16NR1eGG1lC2MuB0F8/vBoPv7LNtRYA4KlXd+K6B1dprzGS0HIIQ7QNlpnQE57xki0UA0L5mntexuL1u5Avcp8vP0wpiPsKwuSdwZirlMJcQ0mqksPcNJHB4pA6AqsKmV5JaSbXkLtmcV0XoGxdGuUaOpVzvoExNgXAPYyx5Zzzh+QDHAVxCQDsv//+jRhjU+C6hiyO797+EgBgzVUXaI8VAWH59y7Ouf8rZ7jb3vPLxwEAn3z9gYFr6NI5S8HiPPDTHdeV9i2QIjOSt2AaDJ898yAMjORx4+NrcfeyLbh72RbM2WdsqBUQjBFIr0OEh2kwdKYNDGajXEPlCZ6Ljp+BfztxJgAvffSNh01FymDYMZTF6w+Z7Izbf30RQE/iGrr2A8dg3c69KFo8tGV1FJUaBFe944jElmWluK4h8mlUhYYoAs75BufvVsbY3wEcD+Ah5ZjrAVwPAPPnz2+O9d4agFAESX5/4kdaSbC40h96kfOAOR+VjpktFJE2DXzlnENx3/ItuPHxte6+YLDYu7DasydJr6GOtOE2hgsPFocONZLvv2NeYNv7jpuBNx4+1bdN/W7E95ukHfT5R0wrb3AOlbqG3nd8/SZsXkEZWQTVoO76lDHWwxgbK14DOBvA0nqPoxWwpLYM9QoWV6oIOA/+eMN86oBtgYjZn3qeGiOQhb8aLPafqhceHSnTzfgJm4FXw48usoZ0Ckl9RnVdglrSVDECqiOoKo2wCKYC+LvzHz4F4H855wsaMI6mR86712XXqFQjfTRXBdNfvX2UIhjJFzHGqYhVhWG+wJVMocqCxR0RwWaBuEzGNMr+LKLWB1Ap1DVGUPNbVA3KGqoudVcEnPPVAI6s931bETkLZW8umHqpImIEekUQtCjkfv/ecZUrAtXPHmkRFCz0des7TaoFZbLwVl07vhhBiOzIpLwunnFZQx2p8hWBO44Ex4geUvVQBM3kZklR99Gq0kRzAEJFVgRDmhx8lXyEa0iXFprXNLLLF4I+/lJRb68rMBNkC5Y0+/O2GywYI5BjDeoY/S0mwlxDhlsDEOYaci2CmDYTSUgixISuqYciaCahWo+6inaCFEETU/Apgvj8/pziGpIVyfbBbPD6GishV7RCC6aSElAEIZW4AjdVUDoxZRjIKTUGcrBYFWpJms5lpBhBqGsInkWgv0aCnxTXj1GHVULWUKXUIw5RLQyDwWDNpbxGM6QImhgrxDV025KNAQuBc45bnHUIxA9eXht424BGETjX3z6YxbUPrMTyzXuQK1qBArCwlgphRLmGdPJOdOb0L6ruuIZCYgSRFkGI8MiYhruoTGgdgXNqR4gyVOsAokgiwwoR7rxq02wyNWUaFCyuEqQImhifRSAtIv+5m57DZX97wXfsHS9sxjNr+wF4QkWe8estAltR3LZkI36wYAV+fPfLyBeCFsGM8d2BGXLUDFbNlslE+PYBYEyHWAPYv93iUbUDDGM6UrjwyOkAgA+e5LVvlgO+Mh1pw+0cGpo1ZERbBOr2WZN6cME8f1qnFyzWXgKzJ/Xgzc45bznS/nvGoVMCxx00ZYxbf1ANmm12ffSMPhyyz9hGD6MloKUqm5ioGMHKrYO+90OSxeAqAun8gZFgjEFcXzSayxUsGCxoAUwd14m7vng6bl28Af/nT0sA2O6cQkhLBVXgdEqdMztSRiCV9b/efgQA/0xeGDO+gjLFslh65TkAgJ9ddLTveq57J+0P+HakDHRl4lxD3rEAcNl5c/D9O5e7+1XXkFyoJ3DTR0ME733SOUfvPz60QPCfnz0F3Znq/YSbrTjrlk+c1OghtAxN9tUTMnL6qKoI1OCvPIsX8qcgCUGdIhBtq70gs51d1KW0PE6bDIbBfEIwrJ+/fX9FEaT8ikBFjF12j4hn9xWRRcQIZFz3jhKk7kiZnkUQkjXE3Kyh4JjsMZTgGkp8pJ5qz+CbKUZAVBdSBE1Msah3DQHBugLZ1SEEiGxR7BnJh15fKAKLQxsjEMLPn74ZLlTUPbKS0mUQif2yzBVj91kEkvKJkmlin2rZJMka8mIEhm8cglKyiZJUC9eTZkofJaoLKYImRg72qgJJtQjk9273zhjXkEgfFa6aosWR12QNCZeM7HOPml2qu+SZvN4iCM8Zl5WPoVF2+vsz57qqRWDEZw0554pnVd1fSYLF4oxK9UC1LAJxmVGml4g6QoqgiYlqKzEcUASe0hDnya4hnUUggsnCj54vWsgVrICwzmgsgqjZpSrAZFlY2t8AAB01SURBVPeKbkYtBLbuknEVwFH7As8hWQRRK5QBnkWgptiWVl9QmeStluAWl2mm7qNEdSFF0MSEBWOBaItAWA+6YLEsQIXFIVxDBcciUIWvEJqy8IwUxMp7OVNIl5Yp7qcTVGECO0oRiWCxziLocOMRYemj/hhBQSm6S6IIhP6uvDCvOoJbPFOzZQ0R1YMUQQVs3TMSuVDLuh17I88vWhyv9dvH5IsWNuwaxsBIHjuHconuryv4ElgcGJbiBrKFYFn22OTzB0IsgqFsAZt2jdjvLe6sAaAqAsc1JAnBKJGiCjDZH98R4VrRCaowV0yUkOWOcyYYIzCl9QiihaKwJlRlHNUuQ71/pYK3ahYB8/8l2g9SBBVw/H8txL/95kntvjte2ITTf3g/HlixNfT8n9yzAqdefT827BrGlbe9iFOuug9HXnk3jvnOPYnuH9dx9Gf3veK+zkqK4Kk1O3H6D+/HvS9tcbcJ15F8yYLFceHPH8HC5fYzFBzXkKoI+rrTAFTXUPi4Tjpwou+9TxFEFKeJ5nMyYa6hKCErFGCnEpieu+84TB3XAYMB47u9xXIOnuL19hfXPX7WBADAQUrf/1MPmhR6X5VKBW+1LAJhIZFF0L5QHUGFPL2mP2T7TgB2Pr+uGAgAHn7FXk1s+0AW971kC9sEK066RLmGAGDz7hH3tVhZbO70cXhx4x4AwPLNA9HXL1pYtW3IfS+CxRmp0dufP3kSXje9F4BqEQSFyh2fPw3T+zrR25XG2YdPxdnXPIThvH8Fsqhg6/S+Ltz5hdNwzT0v4+5lthILa3cRJdJEbEQ+9+GvnYkZE7rBOceDXz3Tp5D+8dlTcNSV9yBXtNxc+3ccsx+OmznBN/aHv3Ym9hvfFXFnG881NDoErxiG7Hpb8q2z9UupES0JWQRlwmNm42KGHdaKQKWc2V3UGriHTRuHXXs9F9NIvoi+7jQmj+1wt8VluKiKJl+0AjGC42ZOcOsK4nLo+7rT6OvOgDGGGRO6pT46cowg+hqHTRuHWZN73Peqe8dzc0RYBM5zCTeOadjjEefNmNDtU2TdmRQmjrEtBHn7jAndvuyoGRO6S/oeR4ke0LqGervS6HUsPaL1IUVQJnGLxQtXTDKfcWVjUIOoKYNh0pgM+vd6fv+RfBGdKdMnuOKErqoIwmIEAn9BV3C/mpsvdKk/fTReccoKTLUIervSofcXCNdQh6Y+QaBuYyHbK6kFGC3r7ZJriCBFUCa6/v0yIwVbEcRZDpXgFlUpgrkzbaKvO4Pdw7IisNCVMX2CK25sBaXffqHItTECQSYmfVRVWCJoKmfoJFGc6QhFMK4zHXp/gWoR6I5Vt4n36vZyqnHdOoJR8usTXwtVFrcvo+S/YvMRtyiJcA2NJFjsvdyfX0FTXQs4iqAr7XMNDeeL6JAWXgH8WUVR1/feW8gVrdAUyTjXUNh6vL4YQcmKwH+8sAiiZJpI+RSKQDepDztdvW5ZFsGoixGQRdDukCIok7iVukRaaVR6qcDivCzLQbShVgVwZ9rA+O40dg/n3WNG8kV0pk2fMN4bpwgUq0coDnU9YIG6YLyKqgiEnvFlDSVQBLKyCFgEXSnfWHWI7y7jKoLg84QJxahiuFIZLWKXBV4Q7QYpgjIJUwSWZQv1YVcRRB1nv9bFG6IUA+ccRYt7FoEimDvTJnq7M7C4VyiWzVvoTBs+wSXGqEvLBILFUuJZksQIdHI0rBGdvD2JRSA/r9r3SFgEsltMxY0RRCiCMKGoyv3yXEP2/UdLbx93GJQl1LaQIigTuVXyzEtvx8OvbMOGXcOYc8UCvPu6x10BLGIFOwazmHnp7bjrxc3YPpjFnG8uwAsbdgPQxxseX7UDMy+9HWt3DOGQb9yJH97ltTq+5PfP4MDL78DHf7cIAGA6glHMrDvTBvocgbhrOOeOQ7UIxKw5TBF84ebF/mcu+l0qKnKMYNaknsD+MJ94qcHiqBiByOuPclN5MYLw1hXqNjezRtEQ5fj5K60sHhvyfZXLbOczYyQN2haqIygT1SL43eNr8YnTZyNXtLBobT9mTLDzyYWwXerk7v/hibWY3tvlUyQ6i+CPT60DYNca5AoWfnH/Knz1nDkAgHuWbfEd+9Vz5mBD/zDWbB/CLYvWozNlosdZzEW4f3YP5zFrUo9v9is6lk7oyWDznhGM6UhhMMHax285cjq+fduywHZ5hvvzi47B46u347n1u/DLB1cDCE9XNUNcQzdfciKm9XYGjh/b6aU1qq6Zz73hYMya3INz5k4NHb8aW0niGgqbvFcSYC3XIrjrS6fj1e1D8Qcm5Lf/fhyeW9fvBtqJ9oPmAGWSKwS7fcoz++GcLeizjkUgFEJHynStBIF+kXh7m64rqMrc6ePwqTMOdGsEujKmmxop3D+79ubR15WGLItF/GKSc94+GqGrcvT+fZg4piP2uN7uNM593TS8+9j93G1hgk+evcuK4MTZE3HAxKBl0ReR3542Dbz96P2is4YCMYLgMWFnc8V/UkmMoNxTp/d14ZQSKpjjmNCTwRsOC1ecROtDiqBMVItgJF/0+dTF0o/Cry4UQkfaCAQyi0UeEFzi+juH/EtI6qwHMSsVQkleYGUkX0TR4tgzkkdvd0YJFttKZpJTLDV1XLyATxLMlUmSieKzCBIU4EUpgiSIGIGwUHRjDLiGHNWghm7KmdV7bahHR4yAIEgRlElQEVjalg9i1j2Utf92pc1AJlHBsgLBYeGPX+s0rhOyUhcEFYJUjhGItMps3sKe4Tw4B8Z3p33CRyikyc4MvyfBsoelrMBljylJOqg3pqi1jgV9Uh+gchAKOxNRR5DUNUQQrQApgjJR6whG8kVtN1Ah9EXQtjNtuH1/BDoFIlxCa3YMOefZM2W5NkAg8vANVxGY7vEj+SJ2Ocqjrzvtc2wIt5Fon5AkYyeJoJYxY7p4Av6CsiSzZBEIL5dgjCD5udVIrHHXLKb+/8QogRRBmaiZPsP5IooaX79wDe1W2j3I6BTItgHbJbTGsQhcRaCzCFjQIhBplSOFIvod5dHXlXFjD/IziCBhvmjFZqSkSrQIkgRTZeWSZObdnUnWvykM8dxibMlcQzbVqBSv1gplBFEtSBGUSb4QdA3p0kDlYK04LqsoAl1NglAEIruoK8IiEK4h8bdLsgiGc5arhPq609p79TjCfyRvYUxntCIIWwgmDDHZ74kQ3r7lJhNogkrz70WwWIxNHyzWt5ioJqOl1xBBtHT66C8fXIVHV+3A7z56PCyL44p/LMXOoRwuPHI6pvZ24voHVyNlMlz9znmuMASA/7xtGdbsGEL/3hxOPnAivnrOHBQtjsv+9jwuPnU2Dt1nbMA1tH0wi0ecttIyYvYvZuUj+WKgyOyuFzdjo9QyGtCsheu4Mb71zxcD91Cbz9muIfv4y//+Ar56zqEAbN+6rjXGOMfVYnGOsZ0pbNodOMQlbOWuMMTYxveE+/XNEi2CSvHaLtvPorNyAoVjzoZq1lyRRUCMFlpaEWzcNYwl63cBAHYM5fDHJ+3c/DuXbkbaZO4M/vSDJ+M9x80AYAuu3z76qnuNzbtH8NVz5uDV7YP406LX8Mzafiz88hnamfVdyzYHtonjhpwMnZF8MbCe8L0v6Rev6cmYOOaA8W4tQbZQxPqdw76xz50+zk0bTWliBADww7tWALCDxWra637ju3DKgRPx7yfPxMdOmxUoIgPslNEj9+vDDY+tce9x3QeP0aa2/uBd8zBOsir2GdeJj582C+87fn/tMwK2lfHHj52AFZsHEs+8f/CueWUXVl33wWNx89PrcdpBk/CJ02e7372MOo5ffWg+/vfJtZitKZS7/Pw5OGrG+MT3F96l0VJZTBAt7RpKmYY7I1UXZ5fTMOXccNV/H9ZKQi4ISxkM03s7XfePTMHt9eM1oUvSfwgA3j1/Bn5/8Ql4/wn7I1soui6eb75lrnvMt94y1xUoYobbkTICaZ6M2YVY6vN844LDkDINfPvCudhvfLdbZSysCAD43tuOwJx9xrrPCsCuEZgfFKDvmT8D575umnRfhq9fcDgOVFbykjENhlMOmoSPnjorsbPkPfNn4LwjpsUfqGH25DG4/PzDYBgMl51/mHZs6jhmTerB1y84XCu8Lzn9QHfFslIgPUCMFlpbERjMFXw6IS2Q43+qkBYCP7hIi/c+bRqhKY0iECxSNXWuoTCEO6grbWI452X/jJfy6OVMH+Hh6EybAYE1rjMNU/o8BOq4xzqzeTkg29eddp8/bi3fcig1RlAP6jGM0fKsBNHaisBk7sx/93D4gvByWwXZbWMLTv+MXoh/WaCaBtMWOaVN5uasi2rikUIxUFkchgjMipTT/iEv+0c9xh6HpzhUhPIIKgL/uEX7BtlvP74747WMrkET/VKzhupBPdw2FCMgRgstrQhMw0DB6QbaPxRuEfT7lnT0BOWUsR2u4IxyGZkG8y12LuhKm65FkC3DNSRmyp0pE0WLY/ugowhki0CaTcsxApVeZ3xqZpOsVADPIhAFcPb1jJpaBLLSaSfhSFlDxGihIYqAMXYuY2wFY2wlY+zSWt0n7UiVgsW1+feCXSE5/lPGdqBgcVgWd7eLn25OUQS69V27MykpRuC5hrIJXUOuInAE++Y9dmaRrAh8bhWpjkBFFGHlCjEWgRMjGJBiKowxN+Wy1IKyJPhn3+0jHKnbJzFaqPt/RcaYCeAXAM4DcDiAixhjh9fiXqKqtWhx7JZm/SmD+dIAZSWRldw2k5zWC7miFfDr56XsG4MxbbVrd8Z0BaisCNSsoTAyriKw/27ePQzA79eXF4OJsgjCXEPqscIiULuQehZBbf/LtJNFQDECYrTQiPTR4wGs5JyvBgDG2M0A3gog2Ne4QtKOPztftAIWgRwgXrxuF75/50vgHHhmbb+7vduZHeeLlk9BPLeuH4+u9GoGTAN611DGxFCuiKsXLHdbPm8fzOHFjbuRSRmB2Xlg/I4iE43YHnx5G1IG8xVn+WMEERaB6xqKvucYJ0agpobma2gRyLSTcGyfJyVGO40wTvcFsF56/5qzzQdj7BLG2CLG2KJt27aVdaOUZBHsiXANbdg1jF8+uBo3PrbGpwhEymS+yH0dQ99+7WN4as1OAPaiLucfMc2dSQP+Cl8A+J8HVgEADppipylu2ZMN7Zcjy1kx2z906lj0daexfucwTpg9wedKkWMEh0wdi0OmjsFMp3XzW46c7u4TK3d99Zw56EgZ+MAJ+2tTHk8/eBIYAz544v5442FT8J75dhvpN8+zr/XWo6YHzimXr55zqLtug2A06YHJYzvwjQsOq9n120npEaObUVtQxjm/HsD1ADB//vyyCjrF7DVf5BjJWzh06ljM3Xcc/vbsBgDAtR84Brct2Yg7l27GhJ4M7vnS6Tj2u/cCAO7+0ulYtKbfOd8L8MpCeN++Ljx66VkAgH8s3uBuNxlDERxdSluF9x03A398ch1e3T6EyWM7sHXAazF94ZHT8bOLjsZIvog5VywA4Pn/j5zRh8XfPFv7jHL66KxJPbj7S6933//3RUfj5c0DWLFlwHUNXTBvGi6YF55/P2VcJ179/gUAgF9/2FMUB00ZgzVXXRB6Xjl85syD8JkzD/JtE8JRZ9XUm6e//saaXp/0ADFaaMSvbQMAuRJpP2db1RH+7KLFnaUaDd8Muq877frIO1OGO2u235uu2yVXsNyOoXLTsfE93vHyEosiw1JN4+xIm+49poz19/4XQl8uBEvS+z+uLbQQqJW2bq4bjnBsh9WySBEQo4VGKIKnARzMGJvFGMsAeB+Af9biRqZrEVgYzhXRkTZ9grOvK+MKys606QuEdqYNd7YtWwSyadIt9e+XZ7Ciq6XaJbMzZbh9fSYriiCTss+RLY4kvf/jfPYivlDpYi71QlgEvRW2mm4GyDVEjBbq7hrinBcYY58FcBcAE8BvOefBTmpVQMzoCxbHSMFCb1faJ1zH96Tdmby6MpasNIRrCQAGpSCq/DOWs2/cGIGy0EtXxoSI7U4Z618WUif0kyiCuMKnTlcRNIdFIJ6GFAFB1I+GxAg453cAuKPW9xGVtkXLbv3cObbD51O3LQJbUHYpPukunyLwLALZry+jVQSqcpHcR6pFoFcElQsK8VyVLuZSL8Tn3A6KgNQAMVpofESuhqR9weIiOtMmMo5wzaQM35KO6upcaZN5MYKivhpYdhPJriEjxDUkL1wzQWnLrFMEmSrk7AsFpUtvHY2IpTjbQhGQJiBGCS2tCITP/77lW7G+f9g3yx/fnQZjTNuXB7BdLkIQ3/3iFnfJyDDk64gKX7Udg1xIpiqejGb2ny5xoXgdnSnT6Tw6ahPEfOxxXG/j2kIRkCYgRgetrQgcgfzDu1agaHF0pg1XuIoeO2pl7UmzJ7qvhbC+7sFVeGL1zsD1zz58qvtavs7bnFx7dVZ70OSxeMNh9jmipkBQaozgiH17Q/fJTO/rwgETuptmfdzDnNqN1x8yucEjqR0XHlm9WgyCqAbNMU0sE3VG3ilZBKI3kJqvfuNHj3e7g8qC+LbPnor9xndh0+4R9HSYGNuZ9rWD7pT8/5eedxg+e+bBuG/FFgD2bPyBr5yBiWM68Lp9x+G81+2DiWM6sORbZ+NDv3kSS17brZ39R8UI/vKpk2IrkwHgU2cciI+cOjP2uNHCyQdNwrNXvCngOmslfvKeI/Gdt72u0cMgCJeWVgSmMgvukGIE411F4LcIMikvbVRWBEfsZ8/Aw5Zc7JDTR50mdKJls8EYJjp9i5j0urcr7bpAdPGAqDqCjpTpCz6HIT9Ps9DKSgCwXZa9Xc31nRCtTUv/b1RdK51pw90mXENCmHJN7bLI7U+CTmiLGX2UV6bHSTHVWwQt/fUQBDFKaGlJE7AIUqY7O+7r0buGZEoRxLrAn0hfjQoKdnfYiiit0RakCAiCqActLWnSympalsUDFoFwyehkdaWCWMQoouwKYRFkNf5+UgQEQdSDlpY0arB4JF/0FIEI9EZI6YoVgRFsG6HS06Hv/w9Up46AIAgijpaWNGofnuF80fX7uxk/EX1NhSBWXUzJ7y+CxeHHiLUF9uY8RSCOT5cQoyAIgiiX1lYEyox6ytgOt1GcWH1MpJHOnuzP6wc8QXzM/n1l3d8LFocL9AMm2WsHTOzxWk4csZ99v1osFE8QBKHS0umjskXwqw/Nx1lzpoABuP7fjsWxB4wHAMyd3osbPnIcTpQKyQTdmRT+9+MnYO70ZMVbC754mq/C2HRdQ+HnvGXeNHSlTZw1Z4q77caPHIflmweaLu2TIIjmpLUVgRQjeJNUBXz23H18x51x6BSEcfKBkxLfb84+43zvRYwhyiJgjPnGBtidQnWKiSAIoha09JSzXN9+q9yfIAgiCS2tCNT00brf34x3DREEQTSallYEavpovREuIVqAhCCI0UxrK4JRknVDHiKCIEYzo0NS1ohGWwSiRIEsAoIgRjOtrQicqXijKnSF+Bf9hAiCIEYjLZ0+yhjD188/DKc3aJGTWZN68KU3HoJ3HrtvQ+5PEASRhJZWBADw8dNnN+zejDF84Y0HN+z+BEEQSWhp1xBBEAQRDykCgiCINocUAUEQRJtDioAgCKLNIUVAEATR5pAiIAiCaHNIERAEQbQ5pAgIgiDaHMZ5xKK9owTG2DYAa8s8fRKA7VUczmiGnrU1aadnBdrreWv9rAdwzmNbKzSFIqgExtgizvn8Ro+jHtCztibt9KxAez3vaHlWcg0RBEG0OaQICIIg2px2UATXN3oAdYSetTVpp2cF2ut5R8WztnyMgCAIgoimHSwCgiAIIoKWVgSMsXMZYysYYysZY5c2ejyVwhj7LWNsK2NsqbRtAmPsHsbYK87f8c52xhj7mfPszzPGjmncyEuHMTaDMXY/Y2wZY+xFxtgXnO0t97yMsU7G2FOMsSXOs17pbJ/FGHvSeaZbGGMZZ3uH836ls39mI8dfDowxkzH2HGPsX877lnxWxtgaxtgLjLHFjLFFzrZR93+4ZRUBY8wE8AsA5wE4HMBFjLHDGzuqirkBwLnKtksBLOScHwxgofMesJ/7YOffJQD+p05jrBYFAF/mnB8O4EQAn3G+v1Z83iyAszjnRwI4CsC5jLETAVwN4BrO+UEA+gFc7Bx/MYB+Z/s1znHNxhcAvCS9b+VnPZNzfpSUJjr6/g9zzlvyH4CTANwlvb8MwGWNHlcVnmsmgKXS+xUApjmvpwFY4bz+JYCLdMc14z8A/wDwplZ/XgDdAJ4FcALsQqOUs939/wzgLgAnOa9TznGs0WMv4Rn3gy0AzwLwL9jLe7fqs64BMEnZNur+D7esRQBgXwDrpfevOdtajamc803O680ApjqvW+b5HXfA0QCeRIs+r+MqWQxgK4B7AKwCsItzXnAOkZ/HfVZn/24AE+s74or4vwC+BsBy3k9E6z4rB3A3Y+wZxtglzrZR93+45dcsbic455wx1lJpYIyxMQD+CuCLnPM9jDF3Xys9L+e8COAoxlgfgL8DmNPgIdUExtibAWzlnD/DGDuj0eOpA6dyzjcwxqYAuIcxtlzeOVr+D7eyRbABwAzp/X7OtlZjC2NsGgA4f7c625v++RljadhK4I+c8785m1v2eQGAc74LwP2w3SN9jDExWZOfx31WZ38vgB11Hmq5nALgQsbYGgA3w3YP/RSt+azgnG9w/m6FreCPxyj8P9zKiuBpAAc72QgZAO8D8M8Gj6kW/BPAh53XH4btSxfbP+RkIpwIYLdkjo56mD31/w2AlzjnP5F2tdzzMsYmO5YAGGNdsGMhL8FWCO9yDlOfVXwG7wJwH3ecyqMdzvllnPP9OOczYf8m7+OcfwAt+KyMsR7G2FjxGsDZAJZiNP4fbnQwpcaBmvMBvAzb3/r1Ro+nCs9zE4BNAPKw/YcXw/aXLgTwCoB7AUxwjmWws6ZWAXgBwPxGj7/EZz0Vtn/1eQCLnX/nt+LzApgH4DnnWZcC+KazfTaApwCsBPBnAB3O9k7n/Upn/+xGP0OZz30GgH+16rM6z7TE+feikEGj8f8wVRYTBEG0Oa3sGiIIgiASQIqAIAiizSFFQBAE0eaQIiAIgmhzSBEQBEG0OaQIiJaGMVZ0Oj+Kf5FdaBljn2SMfagK913DGJtUxnnnMMaudDpU3lnpOAgiCdRigmh1hjnnRyU9mHN+XS0Hk4DTYBdXnQbgkQaPhWgTyCIg2hJnxv4Dp1f8U4yxg5zt32aMfcV5/Xlmr4fwPGPsZmfbBMbYrc62Jxhj85ztExljdzN7PYFfwy4OEvf6oHOPxYyxXzot0tXxvNdpOvd52E3ZfgXgI4yxVqyGJ0YZpAiIVqdLcQ29V9q3m3N+BICfwxa+KpcCOJpzPg/AJ51tVwJ4ztl2OYDfOdu/BeARzvlc2D1l9gcAxthhAN4L4BTHMikC+IB6I875LbA7rC51xvSCc+8LK3l4gkgCuYaIVifKNXST9Pcazf7nAfyRMXYrgFudbacCeCcAcM7vcyyBcQBOB/AOZ/vtjLF+5/g3ADgWwNNO59QueE3GVA4BsNp53cM5H0jwfARRMaQIiHaGh7wWXABbwL8FwNcZY0eUcQ8G4EbO+WWRB9nLGE4CkGKMLQMwzXEVfY5z/nAZ9yWIxJBriGhn3iv9fVzewRgzAMzgnN8P4D9gtz8eA+BhOK4dp5/+ds75HgAPAXi/s/08AOOdSy0E8C6nH72IMRygDoTbyxjeDuCtAH4Au0HZUaQEiHpAFgHR6nQ5M2vBAs65SCEdzxh7HvaawRcp55kA/sAY64U9q/8Z53wXY+zbAH7rnLcXXjvhKwHcxBh7EcBjANYBAOd8GWPsG7BXqTJgd479DIC1mrEeAztY/GkAP9HsJ4iaQN1HibbEWRhlPud8e6PHQhCNhlxDBEEQbQ5ZBARBEG0OWQQEQRBtDikCgiCINocUAUEQRJtDioAgCKLNIUVAEATR5pAiIAiCaHP+P3hsVkKhUAheAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "def train(\n",
    "    agent,\n",
    "    max_episodes, # (int) maximum number of training episodes\n",
    "    max_steps, # (int) maximum number of steps per episode\n",
    "    goal_score, # (int) average score to achive over 100 episodes \n",
    "    epsilon_start, # (float) start value of epsilon, for epsilon-greedy action selection\n",
    "    epsilon_end, # (float) minimum value of epsilon\n",
    "    epsilon_decay, # (float) multiplicative factor (per episode) for decreasing epsilon\n",
    "    sample_exp_start, # (float) start value fo importance-sampling exponent\n",
    "    sample_exp_end,  # (float) end value fo importance-sampling exponent\n",
    "    model_path # (str) file path to save the Q-network parameters\n",
    "):\n",
    "    scores = [] # list containing scores from each episode\n",
    "    scores_window = deque(maxlen=100) # last 100 scores\n",
    "    epsilon = epsilon_start # initialize epsilon\n",
    "    \n",
    "    sample_exp = sample_exp_start\n",
    "    sample_exp_step = (sample_exp_end - sample_exp_start) / max_episodes\n",
    "    \n",
    "    for episode in range(1, max_episodes + 1):\n",
    "        env_info = env.reset(train_mode=True)[brain_name] # reset the environment\n",
    "        state = env_info.vector_observations[0] # get the current state\n",
    "        score = 0 # initialize the score\n",
    "        \n",
    "        for step in range(max_steps):\n",
    "            action = agent.act(state, epsilon)\n",
    "            \n",
    "            env_info = env.step(action)[brain_name] # send the action to the environment\n",
    "            next_state = env_info.vector_observations[0] # get the next state\n",
    "            reward = env_info.rewards[0] # get the reward\n",
    "            done = env_info.local_done[0] # see if episode has finished\n",
    "                \n",
    "            agent.perceive(state, action, reward, next_state, done, sample_exp)\n",
    "            \n",
    "            state = next_state\n",
    "            score += reward\n",
    "            \n",
    "            if done:\n",
    "                break\n",
    "        \n",
    "        scores_window.append(score) # save most recent score\n",
    "        scores.append(score) # save most recent score\n",
    "        epsilon = max(epsilon_end, epsilon_decay * epsilon) # decrease epsilon\n",
    "        sample_exp += sample_exp_step # increase important-sampling exponent\n",
    "        \n",
    "        print('\\rEpisode {}\\tAverage Score: {:.2f}'.format(episode, np.mean(scores_window)), end=\"\")\n",
    "        \n",
    "        if episode % 100 == 0:\n",
    "            print('\\rEpisode {}\\tAverage Score: {:.2f}'.format(episode, np.mean(scores_window)))\n",
    "        \n",
    "        if np.mean(scores_window) >= goal_score:\n",
    "            print('\\nEnvironment solved in {:d} episodes!\\tAverage Score: {:.2f}'.format(episode - 100, np.mean(scores_window)))\n",
    "            torch.save(agent.online_qnet.state_dict(), model_path)\n",
    "            break\n",
    "    \n",
    "    return scores\n",
    "\n",
    "def plot_scores(scores):\n",
    "    fig = plt.figure()\n",
    "    ax = fig.add_subplot(111)\n",
    "    plt.plot(np.arange(len(scores)), scores)\n",
    "    plt.ylabel('Score')\n",
    "    plt.xlabel('Episode #')\n",
    "    plt.show()\n",
    "\n",
    "agent = PriorityDoubleDQNAgent(\n",
    "    state_size=state_size, \n",
    "    action_size=action_size,\n",
    "    replay_memory_size=int(1e5),\n",
    "    replay_batch_size=32,\n",
    "    replay_priority_exp=0.5,\n",
    "    replay_sort_freq=int(1e3),\n",
    "    replay_start=500,\n",
    "    qnet_hidden_sizes=[64, 64],\n",
    "    learn_freq=4,\n",
    "    learn_rate=5e-4,\n",
    "    discount_factor=0.99,\n",
    "    target_update_freq=1,\n",
    "    soft_update_factor=1e-3\n",
    ")\n",
    "\n",
    "scores = train(\n",
    "    agent=agent,  \n",
    "    max_episodes=1000, \n",
    "    max_steps=1000, \n",
    "    goal_score=13, \n",
    "    epsilon_start=1.0, \n",
    "    epsilon_end=0.01,\n",
    "    epsilon_decay=0.995,\n",
    "    sample_exp_start=0,\n",
    "    sample_exp_end=0.5,\n",
    "    model_path='dqn.pth'\n",
    ")\n",
    "\n",
    "plot_scores(scores)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "See the agent in action"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Score: 17.0\n"
     ]
    }
   ],
   "source": [
    "def test(agent, epsilon=0.01):\n",
    "    env_info = env.reset(train_mode=False)[brain_name] # reset the environment\n",
    "    state = env_info.vector_observations[0] # get the current state\n",
    "    score = 0 # initialize the score\n",
    "\n",
    "    while True:\n",
    "        action = agent.act(state, epsilon) # select an action\n",
    "        env_info = env.step(action)[brain_name] # send the action to the environment\n",
    "        next_state = env_info.vector_observations[0] # get the next state\n",
    "        reward = env_info.rewards[0] # get the reward\n",
    "        done = env_info.local_done[0] # see if episode has finished\n",
    "        score += reward # update the score\n",
    "        state = next_state # roll over the state to next time step\n",
    "        \n",
    "        if done: # exit loop if episode finished\n",
    "            break\n",
    "    \n",
    "    print(\"Score: {}\".format(score))\n",
    "    \n",
    "test(agent)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "env.close()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "drlnd",
   "language": "python",
   "name": "drlnd"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}