SARSA.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pprint\n",
    "from operator import itemgetter\n",
    "from mdp_matrix import GridWorld\n",
    "from sarsa import sarsa\n",
    "from expected_sarsa import expected_sarsa\n",
    "from double_sarsa import double_sarsa\n",
    "from double_expected_sarsa import double_expected_sarsa"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "\n",
    "class MDP:\n",
    "    def __init__(self, T, S, R, A, act_list, terminal_states):\n",
    "        # State space\n",
    "        # Integer number of states\n",
    "        self.S = S\n",
    "\n",
    "        # Transition probabilities\n",
    "        # Form: np ndarray of shape (start_state, action, end_state)\n",
    "        self.T = np.array(T)\n",
    "\n",
    "        # Reward space\n",
    "        # Form: vector, rewards for each state\n",
    "        self.R = np.array(R)\n",
    "\n",
    "        # Action space\n",
    "        # integer, number of possible actions\n",
    "        self.A = A\n",
    "\n",
    "        # Possible actions in the MDP\n",
    "        self.actions = act_list\n",
    "\n",
    "        self.terminal_states = terminal_states\n",
    "\n",
    "    def is_terminal(self, s):\n",
    "        return s in self.terminal_states\n",
    "    \n",
    "class StochasticGridWorld(MDP):\n",
    "    def __init__(self, grid_size, reward_pos, terminal_states, p_success = 0.7):\n",
    "        S = grid_size*grid_size\n",
    "\n",
    "        R = np.zeros((grid_size, grid_size))\n",
    "\n",
    "        # Each row of reward_pos is a tuple: x, y, reward\n",
    "        for row in reward_pos:\n",
    "            R[row[0], row[1]] = row[2]\n",
    "        R = R.flatten()\n",
    "\n",
    "        A = 4\n",
    "        act_list = ['S', 'E', 'N', 'W']\n",
    "\n",
    "        T = np.zeros((S, A, S))\n",
    "        for start_state in range(S):\n",
    "            state_i = start_state/grid_size\n",
    "            state_j = (start_state)%grid_size\n",
    "\n",
    "            # Actions indexed as: 0:S, 1:E, 2:N, 3:W\n",
    "            for act in range(A):\n",
    "                feas_grid = np.zeros((grid_size, grid_size))\n",
    "                if(act == 0 ): # Going South\n",
    "                    if state_i+1 < grid_size:\n",
    "                        feas_grid[state_i+1, state_j] = p_success\n",
    "                    else:\n",
    "                        feas_grid[state_i, state_j] = p_success\n",
    "\n",
    "                    if state_j+1 < grid_size:\n",
    "                        feas_grid[state_i, state_j+1] = (1.0-p_success)/3.0\n",
    "                    else:\n",
    "                        feas_grid[state_i, state_j ] += (1.0-p_success)/3.0\n",
    "                        \n",
    "                    if state_i-1 >= 0:\n",
    "                        feas_grid[state_i-1, state_j] = (1.0-p_success)/3.0\n",
    "                    else:\n",
    "                        feas_grid[state_i, state_j ] += (1.0-p_success)/3.0\n",
    "                        \n",
    "                    if state_j-1 >= 0:\n",
    "                        feas_grid[state_i, state_j-1] = (1.0-p_success)/3.0\n",
    "                    else:\n",
    "                        feas_grid[state_i, state_j ] += (1.0-p_success)/3.0\n",
    "\n",
    "                elif(act == 1): # Going East\n",
    "                    if state_j+1 < grid_size:\n",
    "                        feas_grid[state_i, state_j+1] = p_success\n",
    "                    else:\n",
    "                        feas_grid[state_i, state_j] = p_success\n",
    "\n",
    "                    if state_i+1 < grid_size:\n",
    "                        feas_grid[state_i+1, state_j] = (1.0-p_success)/3.0\n",
    "                    else:\n",
    "                        feas_grid[state_i, state_j ] += (1.0-p_success)/3.0\n",
    "                        \n",
    "                    if state_i-1 >= 0:\n",
    "                        feas_grid[state_i-1, state_j] = (1.0-p_success)/3.0\n",
    "                    else:\n",
    "                        feas_grid[state_i, state_j ] += (1.0-p_success)/3.0\n",
    "                        \n",
    "                    if state_j-1 >= 0:\n",
    "                        feas_grid[state_i, state_j-1] = (1.0-p_success)/3.0\n",
    "                    else:\n",
    "                        feas_grid[state_i, state_j ] += (1.0-p_success)/3.0\n",
    "\n",
    "                elif(act == 2): # Going North\n",
    "                    if state_i-1 >= 0:\n",
    "                        feas_grid[state_i-1, state_j] = p_success\n",
    "                    else:\n",
    "                        feas_grid[state_i, state_j] = p_success\n",
    "\n",
    "                    if state_j+1 < grid_size:\n",
    "                        feas_grid[state_i, state_j+1] = (1.0-p_success)/3.0\n",
    "                    else:\n",
    "                        feas_grid[state_i, state_j ] += (1.0-p_success)/3.0\n",
    "                        \n",
    "                    if state_i+1 < grid_size:\n",
    "                        feas_grid[state_i+1, state_j] = (1.0-p_success)/3.0\n",
    "                    else:\n",
    "                        feas_grid[state_i, state_j ] += (1.0-p_success)/3.0\n",
    "                        \n",
    "                    if state_j-1 >= 0:\n",
    "                        feas_grid[state_i, state_j-1] = (1.0-p_success)/3.0\n",
    "                    else:\n",
    "                        feas_grid[state_i, state_j ] += (1.0-p_success)/3.0\n",
    "\n",
    "                elif(act == 3): # Going West\n",
    "                    if state_j-1 >= 0:\n",
    "                        feas_grid[state_i, state_j-1] = p_success\n",
    "                    else:\n",
    "                        feas_grid[state_i, state_j] = p_success\n",
    "\n",
    "                    if state_j+1 < grid_size:\n",
    "                        feas_grid[state_i, state_j+1] = (1.0-p_success)/3.0\n",
    "                    else:\n",
    "                        feas_grid[state_i, state_j ] += (1.0-p_success)/3.0\n",
    "                        \n",
    "                    if state_i+1 < grid_size:\n",
    "                        feas_grid[state_i+1, state_j] = (1.0-p_success)/3.0\n",
    "                    else:\n",
    "                        feas_grid[state_i, state_j ] += (1.0-p_success)/3.0\n",
    "                        \n",
    "                    if state_i-1 >= 0:\n",
    "                        feas_grid[state_i-1, state_j] = (1.0-p_success)/3.0\n",
    "                    else:\n",
    "                        feas_grid[state_i, state_j ] += (1.0-p_success)/3.0\n",
    "\n",
    "                # Flatten the feasibility grid and assign to transition matrix\n",
    "                T[start_state, act, :] = feas_grid.flatten()\n",
    "        MDP.__init__(self, T, S, R, A, act_list, terminal_states)\n",
    "\n",
    "class WindyGridCliffMazeWorld(MDP):\n",
    "    def __init__(self, grid_size, reward_pos, terminal_states, traps, initial_state = [0,0], obstacles = [[i, j, 0] for i in range(10) for j in range(10)]\n",
    ", p_success = 0.7):\n",
    "        S = grid_size*grid_size\n",
    "        \n",
    "        R = np.zeros((grid_size, grid_size))\n",
    "        \n",
    "        # Each row of reward_pos is a tuple: x, y, reward\n",
    "        for row in reward_pos:\n",
    "            R[row[0], row[1]] = row[2]\n",
    "        R = R.flatten()\n",
    "        \n",
    "        for i in range(len(traps)):\n",
    "            if traps[i] == 1:\n",
    "                R[i] = -25\n",
    "                \n",
    "        # Define actions\n",
    "        A = 4\n",
    "        act_list = ['S', 'E', 'N', 'W']\n",
    "        \n",
    "        # Set start state value\n",
    "        self.initial_state = initial_state[0]*grid_size + initial_state[1]\n",
    "        \n",
    "        T = np.zeros((S, A, S))\n",
    "        for start_state in range(S):\n",
    "            state_i = start_state/grid_size\n",
    "            state_j = (start_state)%grid_size\n",
    "\n",
    "            # Actions indexed as: 0:S, 1:E, 2:N, 3:W\n",
    "            for act in range(A):\n",
    "                feas_grid = np.zeros((grid_size, grid_size))\n",
    "                if traps[(state_i)*grid_size + state_j] == 1:\n",
    "                    feas_grid[initial_state[0], initial_state[1]] = 1\n",
    "                else:\n",
    "                    if(act == 0 ): # Going South\n",
    "                        if state_i+1 < grid_size and obstacles[(state_i+1)*grid_size+state_j][2] != 1:\n",
    "                            feas_grid[state_i+1, state_j] = p_success\n",
    "                        else:\n",
    "                            feas_grid[state_i, state_j] = p_success\n",
    "\n",
    "                        if state_j+1 < grid_size and obstacles[(state_i)*grid_size+state_j+1][2] != 1:\n",
    "                            feas_grid[state_i, state_j+1] = (1.0-p_success)/3.0\n",
    "                        else:\n",
    "                            feas_grid[state_i, state_j ] += (1.0-p_success)/3.0\n",
    "                        if state_i-1 >= 0 and obstacles[(state_i-1)*grid_size+state_j][2] != 1:\n",
    "                            feas_grid[state_i-1, state_j] = (1.0-p_success)/3.0\n",
    "                        else:\n",
    "                            feas_grid[state_i, state_j ] += (1.0-p_success)/3.0\n",
    "                        if state_j-1 >= 0 and obstacles[(state_i)*grid_size+state_j-1][2] != 1:\n",
    "                            feas_grid[state_i, state_j-1] = (1.0-p_success)/3.0\n",
    "                        else:\n",
    "                            feas_grid[state_i, state_j ] += (1.0-p_success)/3.0\n",
    "\n",
    "                    elif(act == 1): # Going East\n",
    "                        if state_j+1 < grid_size and obstacles[state_i*grid_size+state_j+1][2] != 1:\n",
    "                            feas_grid[state_i, state_j+1] = p_success\n",
    "                        else:\n",
    "                            feas_grid[state_i, state_j] = p_success\n",
    "\n",
    "                        if state_i+1 < grid_size and obstacles[(state_i+1)*grid_size+state_j][2] != 1:\n",
    "                            feas_grid[state_i+1, state_j] = (1.0-p_success)/3.0\n",
    "                        else:\n",
    "                            feas_grid[state_i, state_j ] += (1.0-p_success)/3.0\n",
    "                        if state_i-1 >= 0 and obstacles[(state_i-1)*grid_size+state_j][2] != 1:\n",
    "                            feas_grid[state_i-1, state_j] = (1.0-p_success)/3.0\n",
    "                        else:\n",
    "                            feas_grid[state_i, state_j ] += (1.0-p_success)/3.0\n",
    "                        if state_j-1 >= 0 and obstacles[(state_i)*grid_size+state_j-1][2] != 1:\n",
    "                            feas_grid[state_i, state_j-1] = (1.0-p_success)/3.0\n",
    "                        else:\n",
    "                            feas_grid[state_i, state_j ] += (1.0-p_success)/3.0\n",
    "\n",
    "                    elif(act == 2): # Going North\n",
    "                        if state_i-1 >= 0 and obstacles[(state_i-1)*grid_size+state_j][2] != 1:\n",
    "                            feas_grid[state_i-1, state_j] = p_success\n",
    "                        else:\n",
    "                            feas_grid[state_i, state_j] = p_success\n",
    "\n",
    "                        if state_j+1 < grid_size and obstacles[(state_i)*grid_size+state_j+1][2] != 1:\n",
    "                            feas_grid[state_i, state_j+1] = (1.0-p_success)/3.0\n",
    "                        else:\n",
    "                            feas_grid[state_i, state_j ] += (1.0-p_success)/3.0\n",
    "                        if state_i+1 < grid_size and obstacles[(state_i+1)*grid_size+state_j][2] != 1:\n",
    "                            feas_grid[state_i+1, state_j] = (1.0-p_success)/3.0\n",
    "                        else:\n",
    "                            feas_grid[state_i, state_j ] += (1.0-p_success)/3.0\n",
    "                        if state_j-1 >= 0 and obstacles[(state_i)*grid_size+state_j-1][2] != 1:\n",
    "                            feas_grid[state_i, state_j-1] = (1.0-p_success)/3.0\n",
    "                        else:\n",
    "                            feas_grid[state_i, state_j ] += (1.0-p_success)/3.0\n",
    "\n",
    "                    elif(act == 3): # Going West\n",
    "                        if state_j-1 >= 0 and obstacles[(state_i)*grid_size+state_j-1][2] != 1:\n",
    "                            feas_grid[state_i, state_j-1] = p_success\n",
    "                        else:\n",
    "                            feas_grid[state_i, state_j] = p_success\n",
    "\n",
    "                        if state_j+1 < grid_size and obstacles[(state_i)*grid_size+state_j+1][2] != 1:\n",
    "                            feas_grid[state_i, state_j+1] = (1.0-p_success)/3.0\n",
    "                        else:\n",
    "                            feas_grid[state_i, state_j ] += (1.0-p_success)/3.0\n",
    "                        if state_i+1 < grid_size and obstacles[(state_i+1)*grid_size+state_j][2] != 1:\n",
    "                            feas_grid[state_i+1, state_j] = (1.0-p_success)/3.0\n",
    "                        else:\n",
    "                            feas_grid[state_i, state_j ] += (1.0-p_success)/3.0\n",
    "                        if state_i-1 >= 0 and obstacles[(state_i-1)*grid_size+state_j][2] != 1:\n",
    "                            feas_grid[state_i-1, state_j] = (1.0-p_success)/3.0\n",
    "                        else:\n",
    "                            feas_grid[state_i, state_j ] += (1.0-p_success)/3.0\n",
    "\n",
    "                # Flatten the feasibility grid and assign to transition matrix\n",
    "                T[start_state, act, :] = feas_grid.flatten()\n",
    "        MDP.__init__(self, T, S, R, A, act_list, terminal_states)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The autoreload extension is already loaded. To reload it, use:\n",
      "  %reload_ext autoreload\n",
      "[[ -1.  -1.  -1.  -1. -25.  -1.  -1.  -1.  -1. -25.]\n",
      " [ -1.  -1.  -1. -25.  -1.  -1.  -1.  -1.  -1.  -1.]\n",
      " [ -1.  -1.  -1.  -1.  -1.  -1.  -1.  -1.  -1.  -1.]\n",
      " [ -1.  -1.  -1. -25.  -1.  -1.  -1.  -1.  -1.  -1.]\n",
      " [ -1.  -1.  -1. -25.  -1.  -1.  -1. -25.  -1.  -1.]\n",
      " [ -1.  -1.  -1.  -1.  -1.  -1.  -1.  -1.  -1.  50.]\n",
      " [ -1.  -1.  -1.  -1.  -1.  -1.  -1. -25.  -1.  -1.]\n",
      " [ -1.  -1. -25.  -1.  -1.  -1.  -1.  -1.  -1.  -1.]\n",
      " [ -1.  -1.  -1.  -1.  -1.  -1.  -1.  -1.  -1.  -1.]\n",
      " [ -1.  -1.  -1.  -1.  -1.  -1. -25.  -1.  -1.  -1.]]\n"
     ]
    }
   ],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2\n",
    "rewards = [[i, j, -1.0] for i in range(10) for j in range(10)]\n",
    "rewards[59] = [5, 9, 50]\n",
    "\n",
    "terminal_states = [59]\n",
    "\n",
    "obstacles = [[i, j, 0] for i in range(10) for j in range(10)]\n",
    "obstacles[0*10+3] = [0, 3, 1] \n",
    "obstacles[0*10+8] = [0, 8, 1] \n",
    "obstacles[1*10+1] = [1, 1, 1] \n",
    "obstacles[1*10+4] = [1, 4, 1] \n",
    "obstacles[1*10+5] = [1, 5, 1] \n",
    "obstacles[1*10+6] = [1, 6, 1] \n",
    "obstacles[2*10+1] = [2, 1, 1] \n",
    "obstacles[2*10+8] = [2, 8, 1] \n",
    "obstacles[3*10+1] = [3, 1, 1] \n",
    "obstacles[3*10+4] = [3, 4, 1] \n",
    "obstacles[3*10+5] = [3, 5, 1] \n",
    "obstacles[3*10+6] = [3, 6, 1] \n",
    "obstacles[5*10+0] = [5, 0, 1] \n",
    "obstacles[5*10+5] = [5, 5, 1] \n",
    "obstacles[6*10+3] = [6, 3, 1] \n",
    "obstacles[6*10+4] = [6, 4, 1] \n",
    "obstacles[6*10+5] = [6, 5, 1] \n",
    "obstacles[6*10+6] = [6, 6, 1] \n",
    "obstacles[6*10+8] = [6, 8, 1] \n",
    "obstacles[7*10+8] = [7, 8, 1] \n",
    "obstacles[9*10+4] = [9, 4, 1] \n",
    "\n",
    "start_state = [0, 0]\n",
    "\n",
    "traps = [0]*100\n",
    "traps[4] = 1\n",
    "traps[9] = 1\n",
    "traps[13] = 1\n",
    "traps[33] = 1\n",
    "traps[43] = 1\n",
    "traps[47] = 1\n",
    "traps[67] = 1\n",
    "traps[72] = 1\n",
    "traps[96] = 1\n",
    "\n",
    "gw = WindyGridCliffMazeWorld(10, rewards, terminal_states, traps, start_state, obstacles)\n",
    "print np.reshape(gw.R, (10,10))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "100\n"
     ]
    }
   ],
   "source": [
    "print len(obstacles)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[ 0.   0.   0.   0.   0.   0.   0.   0.   0.   0. ]\n",
      " [ 0.   0.   0.   0.   0.   0.   0.   0.   0.   0. ]\n",
      " [ 0.   0.   0.   0.   0.   0.   0.   0.   0.   0. ]\n",
      " [ 0.1  0.   0.   0.   0.   0.   0.   0.   0.   0. ]\n",
      " [ 0.2  0.7  0.   0.   0.   0.   0.   0.   0.   0. ]\n",
      " [ 0.   0.   0.   0.   0.   0.   0.   0.   0.   0. ]\n",
      " [ 0.   0.   0.   0.   0.   0.   0.   0.   0.   0. ]\n",
      " [ 0.   0.   0.   0.   0.   0.   0.   0.   0.   0. ]\n",
      " [ 0.   0.   0.   0.   0.   0.   0.   0.   0.   0. ]\n",
      " [ 0.   0.   0.   0.   0.   0.   0.   0.   0.   0. ]]\n"
     ]
    }
   ],
   "source": [
    "print np.reshape(gw.T[40, 1, :], (10,10))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The autoreload extension is already loaded. To reload it, use:\n",
      "  %reload_ext autoreload\n"
     ]
    }
   ],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2\n",
    "gw = GridWorld(10, test_rewards, terminal_states)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-3718.0851\n",
      "34.0\n"
     ]
    }
   ],
   "source": [
    "[sarsa_Q, sarsa_avg_reward, sarsa_max_reward, sarsa_reward_per_episode] = sarsa(gw, 10000)\n",
    "print sarsa_avg_reward\n",
    "print sarsa_max_reward\n",
    "# print sarsa_reward_per_episode"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "20.0\n",
      "[-2516.0, -936.0, -933.0, -2355.0, -2853.0, -6133.0, -471.0, -305.0, -7439.0, -793.0, -453.0, -953.0, -1465.0, -48.0, -881.0, -428.0, -1226.0, -115.0, -458.0, -742.0, -2963.0, -1582.0, -1615.0, -5769.0, -3048.0, -4986.0, -14554.0, -43194.0, -7148.0, -3665.0, -628.0, -782.0, -106.0, -902.0, -11.0, -272.0, -1492.0, -2825.0, -5695.0, -3614.0, -9096.0, -118.0, -310.0, -11754.0, -2113.0, -7624.0, -19383.0, -20471.0, -1626.0, -1355.0, -598.0, -2167.0, -295.0, -6943.0, -5110.0, -1408.0, -18897.0, -9127.0, -9507.0, -15645.0, -16774.0, -8571.0, -7847.0, -91519.0, -73276.0, -83134.0, -187798.0, -16510.0, -144424.0, -43508.0, -9551.0, -446.0, -2978.0, -13842.0, -3760.0, -13862.0, -655.0, -3742.0, -2402.0, -19484.0, -415.0, -5222.0, -4321.0, -209.0, -153.0, -503.0, -60.0, -714.0, -4288.0, -167.0, -831.0, -1162.0, -301.0, -814.0, -8465.0, -112.0, -311.0, -8940.0, -1994.0, -1624.0, -2800.0, -4183.0, -426.0, -8303.0, -124.0, -67.0, -3785.0, -371.0, -203.0, -82.0, -1005.0, -495.0, -2301.0, -774.0, -1156.0, -217.0, -18439.0, -2870.0, -30948.0, -2336.0, -32518.0, -1031.0, -42969.0, -16801.0, -2985.0, -20396.0, -524.0, -10937.0, -19516.0, -17803.0, -22299.0, -483.0, -563.0, -2654.0, -3834.0, -2577.0, -7750.0, -9557.0, -6591.0, -28101.0, -16570.0, -2112.0, -2746.0, -41845.0, -20637.0, -28498.0, -25399.0, -1007.0, -50030.0, -1377.0, -2750.0, -5312.0, -7304.0, -5238.0, -195.0, -5610.0, -1862.0, -1885.0, -4204.0, -3243.0, -18394.0, -946.0, -3067.0, -1313.0, -550.0, -748.0, -18.0, -3509.0, -2041.0, -6325.0, -5870.0, -2283.0, -12678.0, -684.0, -4825.0, -8570.0, -1824.0, -673.0, -2748.0, -9851.0, -5753.0, -331.0, -860.0, -26.0, -1001.0, -218.0, -1907.0, -3790.0, -567.0, -1835.0, -16695.0, -978.0, -3842.0, -1386.0, -9656.0, -97.0, -2623.0, -10281.0, -34585.0, -3276.0, -9860.0, -1330.0, -2265.0, -5847.0, -183.0, -2667.0, -1394.0, -12328.0, -458.0, -914.0, -2408.0, -4535.0, -2177.0, -504.0, -1431.0, -3232.0, -6870.0, -7635.0, -879.0, -1112.0, -3588.0, -262.0, -3857.0, -2354.0, -404.0, -909.0, -471.0, -311.0, -43.0, -902.0, -311.0, -43.0, -50.0, -1528.0, -4576.0, -1412.0, -5469.0, -1138.0, -640.0, -6.0, -9212.0, -248.0, -4.0, -2100.0, -1095.0, -1185.0, -2227.0, -1045.0, -4605.0, -90.0, -1641.0, -3738.0, -124.0, -15528.0, -21956.0, -7891.0, -21667.0, -19099.0, -14815.0, -5734.0, -15881.0, -4719.0, -3888.0, -8365.0, -4721.0, -17068.0, -4890.0, -3102.0, -5119.0, -504.0, -4677.0, -1156.0, -45929.0, -29055.0, -10371.0, -1775.0, -1169.0, -19586.0, -22520.0, -2298.0, -5864.0, -192.0, -2022.0, -38567.0, -7675.0, -27594.0, -63.0, -2541.0, -12854.0, -1762.0, -21205.0, -67969.0, -5704.0, -830.0, -416.0, -7675.0, -1548.0, -1478.0, -3638.0, -3018.0, -975.0, -775.0, -6277.0, -2599.0, -127.0, -16391.0, -14133.0, -717.0, -3471.0, -2969.0, -18983.0, -4813.0, -2511.0, -863.0, -600.0, -5635.0, -104.0, -2217.0, -10460.0, -11160.0, -28134.0, -4343.0, -24996.0, -5195.0, -7581.0, -727.0, -1022.0, -2774.0, -1765.0, -2850.0, -1195.0, -2973.0, -201.0, -384.0, -965.0, -397.0, -579.0, -662.0, -73.0, -892.0, -901.0, -114.0, -207.0, -1181.0, -361.0, -40.0, -1550.0, -996.0, -2790.0, -3948.0, -5386.0, -3038.0, -177.0, -401.0, -322.0, -3161.0, -1356.0, -1983.0, -1216.0, -918.0, -1097.0, -1007.0, -560.0, -2298.0, -191.0, -40.0, -839.0, -335.0, -722.0, -847.0, -2162.0, -2866.0, -1174.0, -693.0, -1174.0, -191.0, -32.0, -327.0, -2479.0, -8385.0, -1223.0, -48.0, -1078.0, -116.0, -3703.0, -13539.0, -1123.0, -23.0, -3761.0, -3565.0, -365.0, -3607.0, -29180.0, -20921.0, -5078.0, -3451.0, -104.0, -39.0, -1574.0, -4240.0, -7402.0, -3845.0, -34706.0, -8649.0, -12192.0, -4126.0, -1937.0, -3364.0, -1244.0, -475.0, -396.0, -350.0, -2917.0, -592.0, -3233.0, -296.0, -1906.0, -4125.0, -63.0, -950.0, -1887.0, -1208.0, -3178.0, -2035.0, -4594.0, -3470.0, -641.0, -3772.0, -313.0, -1047.0, -8417.0, -335.0, -4061.0, -2620.0, -7312.0, -3414.0, -736.0, -898.0, -12453.0, -10073.0, -667.0, -3096.0, -135.0, -10810.0, -2532.0, -6560.0, -2635.0, -3214.0, -316.0, -1945.0, -1528.0, -537.0, -6572.0, -2279.0, -1811.0, -3087.0, -643.0, -1192.0, -1011.0, -2137.0, -6565.0, -1775.0, -12322.0, -594.0, -76.0, -1287.0, -992.0, -324.0, -344.0, -45.0, -1197.0, -1343.0, -470.0, -404.0, -3108.0, -1914.0, -4396.0, -4368.0, -9659.0, -414.0, -258.0, -5246.0, -1138.0, -8536.0, -311.0, -4618.0, -820.0, -2849.0, -1409.0, -1588.0, -10830.0, -7040.0, -1260.0, -2095.0, -238.0, -181.0, -1835.0, -1125.0, -323.0, -746.0, -6358.0, -166.0, -169.0, -760.0, -1198.0, -479.0, -10713.0, -12199.0, -103.0, -6825.0, -528.0, -2143.0, -4042.0, -1140.0, -3011.0, -309.0, -6920.0, -1245.0, -745.0, -960.0, -161.0, -127.0, -2132.0, -5901.0, -6490.0, -5702.0, -818.0, -487.0, -560.0, -895.0, -299.0, -480.0, -636.0, -1322.0, -1748.0, -404.0, -509.0, -4246.0, -1724.0, -1461.0, -4493.0, -6175.0, -1490.0, -4175.0, -1602.0, -2290.0, -2496.0, -609.0, -112.0, -2833.0, -3889.0, -3922.0, -396.0, -6260.0, -1801.0, -1498.0, -423.0, -4324.0, -119.0, -573.0, -47.0, -1076.0, -340.0, -1425.0, -1957.0, -3692.0, -7174.0, -5569.0, -7043.0, -80.0, -1943.0, -35.0, -219.0, -79.0, -140.0, -417.0, -951.0, -386.0, -1833.0, -273.0, -2307.0, -177.0, -413.0, -1710.0, -345.0, -90.0, -1515.0, -2515.0, -882.0, -167.0, -7827.0, -5730.0, -6180.0, -1887.0, -1254.0, -2681.0, -61.0, -2311.0, -1800.0, -476.0, -2094.0, -206.0, -1579.0, -2288.0, -588.0, -1125.0, -1202.0, -3163.0, -3081.0, -2701.0, -2321.0, -2266.0, -746.0, 20.0, -802.0, 17.0, -17.0, -192.0, -50.0, -704.0, -23.0, -166.0, -1075.0, -2684.0, -966.0, -60.0, 11.0, -1241.0, -253.0, -1233.0, -249.0, -3143.0, -100.0, -2051.0, -7.0, -1868.0, -722.0, -422.0, -577.0, -1302.0, -319.0, -314.0, -804.0, -1435.0, -1860.0, -750.0, -70.0, -1018.0, -1082.0, -639.0, -2095.0, -564.0, -91.0, -200.0, -240.0, -333.0, -815.0, -5214.0, -4.0, -190.0, -60.0, -2798.0, -759.0, -2496.0, -1401.0, -1542.0, -49.0, -1364.0, -789.0, -486.0, -283.0, -216.0, -4919.0, -2421.0, -2993.0, -1041.0, -1127.0, -3588.0, -8240.0, -124.0, -251.0, -858.0, -750.0, -954.0, -482.0, -280.0, -2243.0, -1805.0, -448.0, 1.0, -67.0, -958.0, -226.0, -2824.0, -466.0, -220.0, -314.0, -2003.0, -273.0, -1867.0, -22.0, -22.0, -1334.0, -1774.0, -330.0, -130.0, -253.0, -226.0, -1550.0, -2116.0, 5.0, -21.0, -364.0, -2458.0, -878.0, -1260.0, -554.0, -1888.0, -2483.0, -2919.0, -768.0, -3118.0, -6565.0, -2790.0, -5778.0, -1364.0, -15894.0, -10031.0, -7917.0, -3193.0, -35.0, -10928.0, -31.0, -286.0, -1.0, -9608.0, -4033.0, -390.0, -399.0, -740.0, -7775.0, -1117.0, -6006.0, -210.0, -388.0, -244.0, -3279.0, -261.0, -746.0, -2636.0, -1412.0, -33401.0, -49125.0, -5336.0, -3972.0, -5273.0, -533.0, -417.0, -65.0, -151.0, -769.0, -48.0, -5410.0, -4213.0, -311.0, -1793.0, -523.0, -2641.0, -1244.0, -64.0, -103.0, -1062.0, -3625.0, -573.0, -926.0, -39.0, -56.0, -1133.0, -1517.0, -259.0, -161.0, -4943.0, -3342.0, -131.0, -281.0, -186.0, -1293.0, -4278.0, -135.0, -1924.0, -5944.0, -135.0, -80.0, -949.0, -7246.0, -1071.0, -102.0, -4750.0, -8941.0, -3445.0, -5252.0, -4105.0, -379.0, -1319.0, -192.0, -2268.0, -6503.0, -2700.0, -3294.0, -201.0, -1358.0, -1504.0, -96.0, -280.0, -700.0, -2890.0, -3736.0, -5481.0, -7906.0, -524.0, -301.0, -479.0, -3033.0, -87.0, -466.0, -1998.0, -31.0, -19.0, -3855.0, -3342.0, -2931.0, -3295.0, -511.0, -7047.0, -3315.0, -614.0, -13950.0, -11701.0, -8425.0, -4473.0, -858.0, -4757.0, -5136.0, -1797.0, -7995.0, -4441.0, -184.0, -4341.0, -2592.0, -4055.0, -998.0, -1900.0, -181.0, -6928.0, -197.0, -6283.0, -4443.0, -2090.0, -7613.0, -869.0, -3944.0, -308.0, -4563.0, -60.0, 19.0, -636.0, -341.0, -148.0, -44.0, -14.0, -766.0, -345.0, -454.0, 0.0, -8164.0, -7983.0, -1305.0, -1296.0, -3754.0, -3490.0, -14870.0, -7367.0, -11381.0, -992.0, -380.0, -665.0, -3091.0, -5887.0, -76.0, -1673.0, -213.0, -3026.0, -17858.0, -3060.0, -55.0, -10715.0, -2156.0, -8301.0, -528.0, -3899.0, -2755.0, -809.0, -106.0, -2646.0, -287.0, -7632.0, -4843.0, -11619.0, -4027.0, -5829.0, -3441.0, -3533.0, -3342.0, -1234.0, -1685.0, -621.0, -6905.0, -6641.0, -375.0, -645.0, -1841.0, -8079.0, -3660.0, -10207.0, -1067.0, -5011.0, -2454.0, -122.0, -274.0, -591.0, -3450.0, -4419.0, -614.0, -1096.0, -203.0, -1341.0, -417.0, -1811.0, -6271.0, -2518.0, -7052.0, -4.0, -360.0, -314.0, -4456.0, -17866.0, -1380.0, -14977.0, -6513.0, -12835.0, -2710.0, -1781.0, -1492.0, -224.0, -9208.0, -9093.0, -1076.0, -20452.0, -10304.0, -4226.0, -318.0, -105.0, -683.0, -1844.0, -964.0, -432.0, -366.0, -1407.0, -653.0, -78.0, -2224.0, -432.0, 18.0, -4154.0, -1976.0, -5774.0, -34.0, -17800.0, -22959.0, -1981.0, -832.0, -1191.0, -640.0, -3127.0, -338.0, -187.0, -3185.0, -1858.0, -147.0, -524.0, -798.0, -4597.0, -1143.0, -824.0, -2838.0, -2308.0, -8208.0, -2177.0, -535.0, -14.0]\n"
     ]
    }
   ],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[0 3 3 0 0 1 1 0 0 3]\n",
      " [1 0 2 2 0 0 0 1 1 0]\n",
      " [2 0 2 3 1 1 1 2 0 0]\n",
      " [2 0 2 3 0 0 0 1 1 0]\n",
      " [2 0 0 1 1 1 0 2 1 0]\n",
      " [0 0 3 1 0 0 1 1 1 0]\n",
      " [0 3 2 0 0 0 0 2 0 2]\n",
      " [1 0 3 1 1 1 1 0 0 2]\n",
      " [0 0 0 1 1 2 3 1 1 2]\n",
      " [1 1 1 2 0 2 3 2 1 2]]\n"
     ]
    }
   ],
   "source": [
    "print np.reshape(np.argmax(sarsa_Q, 1), (10,10))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "[expected_sarsa_Q, expected_sarsa_avg]  = expected_sarsa(gw, 1000)\n",
    "print np.reshape(np.argmax(expected_sarsa_Q, 1), (5,5))\n",
    "print expected_sarsa_avg"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "[double_sarsa_Q, double_sarsa_avg]  = double_sarsa(gw, 100000)\n",
    "print double_sarsa_avg"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "[double_expected_sarsa_Q, double_expected_sarsa_avg]  = double_expected_sarsa(gw, 100000)\n",
    "print double_expected_sarsa_avg"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<img src=\"images/maze.png\" width=\"300\" class=\"unconfined\"/>"
      ],
      "text/plain": [
       "<IPython.core.display.Image object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "display(Image(url=\"images/maze.png\", width=300, unconfined=True))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epsilon = .1\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<img src=\"images/stochastic_gw_rwd_vs_alpha.png\" width=\"500\" class=\"unconfined\"/>"
      ],
      "text/plain": [
       "<IPython.core.display.Image object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from IPython.core.display import Image, display\n",
    "print(\"Epsilon = .1\")\n",
    "display(Image(url=\"images/stochastic_gw_rwd_vs_alpha.png\", width=500, unconfined=True))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "As expected in "
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}