DRL_06_07_Cross_Entropy.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "DRL_06_07_Cross_Entropy.ipynb",
      "provenance": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "jW8FRr2fkfBg",
        "colab_type": "text"
      },
      "source": [
        "DEEP REINFORCEMENT LEARNING EXPLAINED - 06\n",
        "# **Solving Frozen-Lake Environment With Cross-Entropy Method**\n",
        "## Agent Creation Using Deep Neural Networks"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "YlShRBzEUUED",
        "colab_type": "text"
      },
      "source": [
        " \n",
        "\n",
        "## The Environment"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "FL7yuQ7zQtq9",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import numpy as np\n",
        "\n",
        "import torch\n",
        "import torch.nn as nn"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "lQBiNI-5UVs2",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import gym\n",
        "import gym.spaces\n",
        "\n",
        "env = gym.make('FrozenLake-v0', is_slippery=False)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "HMjMGEmK0MKk",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "class OneHotWrapper(gym.ObservationWrapper):\n",
        "    def __init__(self, env):\n",
        "        super(OneHotWrapper, self).__init__(env)\n",
        "        self.observation_space = gym.spaces.Box(0.0, 1.0, (env.observation_space.n, ), dtype=np.float32)\n",
        "\n",
        "    def observation(self, observation):\n",
        "        r = np.copy(self.observation_space.low)\n",
        "        r[observation] = 1.0\n",
        "        return r\n",
        "\n",
        "env = OneHotWrapper(env)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "U34ilBKUYsvj",
        "colab_type": "text"
      },
      "source": [
        "## The Agent\n",
        " ### The Model"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Cqs2t3PBVWAQ",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "obs_size = env.observation_space.shape[0] # 16\n",
        "n_actions = env.action_space.n  # 4\n",
        "HIDDEN_SIZE = 32\n",
        "\n",
        "\n",
        "net= nn.Sequential(\n",
        "            nn.Linear(obs_size, HIDDEN_SIZE),\n",
        "            nn.Sigmoid(),\n",
        "            nn.Linear(HIDDEN_SIZE, n_actions)\n",
        "        )"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "RnpWQBCXMk5o",
        "colab_type": "text"
      },
      "source": [
        "### Get an Action"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "yj-HW0qLFXVo",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "sm = nn.Softmax(dim=1)\n",
        "\n",
        "def select_action(state):\n",
        "        state_t = torch.FloatTensor([state])\n",
        "        act_probs_t = sm(net(state_t))\n",
        "        act_probs = act_probs_t.data.numpy()[0]\n",
        "        action = np.random.choice(len(act_probs), p=act_probs)\n",
        "        return action"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "KwS-YPsHY9T4",
        "colab_type": "text"
      },
      "source": [
        "### Optimizer and Loss function"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "w2H-mc7XQgSH",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import torch.optim as optim\n",
        "\n",
        "objective = nn.CrossEntropyLoss()\n",
        "optimizer = optim.Adam(params=net.parameters(), lr=0.001)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Yi0PU4s0ZGLx",
        "colab_type": "text"
      },
      "source": [
        "## Training the Agent"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "XOZZwy5Ic7wu",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "BATCH_SIZE = 100\n",
        "\n",
        "GAMMA = 0.9\n",
        "\n",
        "PERCENTILE = 30\n",
        "REWARD_GOAL = 0.8\n",
        "\n",
        "from collections import namedtuple\n",
        "\n",
        "Episode = namedtuple('Episode', field_names=['reward', 'steps'])\n",
        "EpisodeStep = namedtuple('EpisodeStep', field_names=['observation', 'action'])\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "gGPorG5Tc-51",
        "colab_type": "code",
        "outputId": "d354ed38-eb1c-4db6-beed-22936a30fff8",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        }
      },
      "source": [
        "iter_no = 0\n",
        "reward_mean = 0\n",
        "full_batch = []\n",
        "batch = []\n",
        "episode_steps = []\n",
        "episode_reward = 0.0\n",
        "state = env.reset()\n",
        "    \n",
        "while reward_mean < REWARD_GOAL:\n",
        "        action = select_action(state)\n",
        "        next_state, reward, episode_is_done, _ = env.step(action)\n",
        "\n",
        "        episode_steps.append(EpisodeStep(observation=state, action=action))\n",
        "        episode_reward += reward\n",
        "        \n",
        "        if episode_is_done: # Episode finished            \n",
        "            batch.append(Episode(reward=episode_reward, steps=episode_steps))\n",
        "            next_state = env.reset()\n",
        "            episode_steps = []\n",
        "            episode_reward = 0.0\n",
        "             \n",
        "            if len(batch) == BATCH_SIZE: # New set of batches ready --> select \"elite\"\n",
        "                reward_mean = float(np.mean(list(map(lambda s: s.reward, batch))))\n",
        "                elite_candidates= batch \n",
        "                returnG = list(map(lambda s: s.reward * (GAMMA ** len(s.steps)), elite_candidates))\n",
        "                reward_bound = np.percentile(returnG, PERCENTILE)\n",
        "\n",
        "                train_obs = []\n",
        "                train_act = []\n",
        "                elite_batch = []\n",
        "                for example, discounted_reward in zip(elite_candidates, returnG):\n",
        "                        if discounted_reward > reward_bound:\n",
        "                              train_obs.extend(map(lambda step: step.observation, example.steps))\n",
        "                              train_act.extend(map(lambda step: step.action, example.steps))\n",
        "                              elite_batch.append(example)\n",
        "                full_batch=elite_batch\n",
        "                state=train_obs\n",
        "                acts=train_act\n",
        "\n",
        "                \n",
        "                if len(full_batch) != 0 : # just in case empty during an iteration\n",
        "                 state_t = torch.FloatTensor(state)\n",
        "                 acts_t = torch.LongTensor(acts)\n",
        "                 optimizer.zero_grad()\n",
        "                 action_scores_t = net(state_t)\n",
        "                 loss_t = objective(action_scores_t, acts_t)\n",
        "                 loss_t.backward()\n",
        "                 optimizer.step()\n",
        "                 print(\"%d: loss=%.3f, reward_mean=%.3f\" % (iter_no, loss_t.item(), reward_mean))\n",
        "                 iter_no += 1\n",
        "                batch = []\n",
        "        state = next_state\n"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "0: loss=1.386, reward_mean=0.020\n",
            "1: loss=1.362, reward_mean=0.020\n",
            "2: loss=1.355, reward_mean=0.050\n",
            "3: loss=1.393, reward_mean=0.020\n",
            "4: loss=1.385, reward_mean=0.010\n",
            "5: loss=1.338, reward_mean=0.020\n",
            "6: loss=1.336, reward_mean=0.020\n",
            "7: loss=1.239, reward_mean=0.010\n",
            "8: loss=1.353, reward_mean=0.030\n",
            "9: loss=1.387, reward_mean=0.010\n",
            "10: loss=1.334, reward_mean=0.030\n",
            "11: loss=1.353, reward_mean=0.030\n",
            "12: loss=1.254, reward_mean=0.030\n",
            "13: loss=1.321, reward_mean=0.020\n",
            "14: loss=1.239, reward_mean=0.010\n",
            "15: loss=1.311, reward_mean=0.050\n",
            "16: loss=1.275, reward_mean=0.050\n",
            "17: loss=1.304, reward_mean=0.060\n",
            "18: loss=1.340, reward_mean=0.020\n",
            "19: loss=1.366, reward_mean=0.050\n",
            "20: loss=1.215, reward_mean=0.060\n",
            "21: loss=1.216, reward_mean=0.020\n",
            "22: loss=1.278, reward_mean=0.050\n",
            "23: loss=1.244, reward_mean=0.060\n",
            "24: loss=1.227, reward_mean=0.040\n",
            "25: loss=1.258, reward_mean=0.040\n",
            "26: loss=1.215, reward_mean=0.030\n",
            "27: loss=1.286, reward_mean=0.030\n",
            "28: loss=1.225, reward_mean=0.050\n",
            "29: loss=1.181, reward_mean=0.040\n",
            "30: loss=1.268, reward_mean=0.050\n",
            "31: loss=1.130, reward_mean=0.060\n",
            "32: loss=1.220, reward_mean=0.040\n",
            "33: loss=1.215, reward_mean=0.050\n",
            "34: loss=1.143, reward_mean=0.070\n",
            "35: loss=1.180, reward_mean=0.050\n",
            "36: loss=1.193, reward_mean=0.040\n",
            "37: loss=1.162, reward_mean=0.050\n",
            "38: loss=1.216, reward_mean=0.070\n",
            "39: loss=1.183, reward_mean=0.050\n",
            "40: loss=1.101, reward_mean=0.030\n",
            "41: loss=1.146, reward_mean=0.060\n",
            "42: loss=1.206, reward_mean=0.030\n",
            "43: loss=1.135, reward_mean=0.100\n",
            "44: loss=1.189, reward_mean=0.040\n",
            "45: loss=1.159, reward_mean=0.080\n",
            "46: loss=1.100, reward_mean=0.080\n",
            "47: loss=1.224, reward_mean=0.050\n",
            "48: loss=1.088, reward_mean=0.060\n",
            "49: loss=1.249, reward_mean=0.060\n",
            "50: loss=1.054, reward_mean=0.050\n",
            "51: loss=1.108, reward_mean=0.130\n",
            "52: loss=1.124, reward_mean=0.080\n",
            "53: loss=1.114, reward_mean=0.100\n",
            "54: loss=1.124, reward_mean=0.060\n",
            "55: loss=1.024, reward_mean=0.050\n",
            "56: loss=1.081, reward_mean=0.040\n",
            "57: loss=1.103, reward_mean=0.070\n",
            "58: loss=1.071, reward_mean=0.100\n",
            "59: loss=1.120, reward_mean=0.060\n",
            "60: loss=1.060, reward_mean=0.080\n",
            "61: loss=1.093, reward_mean=0.080\n",
            "62: loss=1.025, reward_mean=0.110\n",
            "63: loss=1.140, reward_mean=0.090\n",
            "64: loss=1.181, reward_mean=0.080\n",
            "65: loss=1.089, reward_mean=0.070\n",
            "66: loss=1.071, reward_mean=0.050\n",
            "67: loss=1.091, reward_mean=0.120\n",
            "68: loss=1.203, reward_mean=0.110\n",
            "69: loss=1.154, reward_mean=0.080\n",
            "70: loss=1.035, reward_mean=0.160\n",
            "71: loss=0.980, reward_mean=0.110\n",
            "72: loss=1.047, reward_mean=0.070\n",
            "73: loss=1.050, reward_mean=0.100\n",
            "74: loss=0.984, reward_mean=0.080\n",
            "75: loss=1.007, reward_mean=0.140\n",
            "76: loss=1.061, reward_mean=0.100\n",
            "77: loss=1.017, reward_mean=0.160\n",
            "78: loss=1.027, reward_mean=0.120\n",
            "79: loss=1.072, reward_mean=0.140\n",
            "80: loss=1.030, reward_mean=0.060\n",
            "81: loss=1.099, reward_mean=0.070\n",
            "82: loss=0.987, reward_mean=0.160\n",
            "83: loss=0.952, reward_mean=0.090\n",
            "84: loss=1.014, reward_mean=0.130\n",
            "85: loss=1.042, reward_mean=0.110\n",
            "86: loss=1.013, reward_mean=0.120\n",
            "87: loss=1.014, reward_mean=0.090\n",
            "88: loss=0.873, reward_mean=0.120\n",
            "89: loss=0.997, reward_mean=0.090\n",
            "90: loss=0.957, reward_mean=0.120\n",
            "91: loss=0.949, reward_mean=0.150\n",
            "92: loss=0.947, reward_mean=0.080\n",
            "93: loss=0.925, reward_mean=0.100\n",
            "94: loss=0.994, reward_mean=0.150\n",
            "95: loss=0.993, reward_mean=0.150\n",
            "96: loss=0.984, reward_mean=0.140\n",
            "97: loss=0.857, reward_mean=0.040\n",
            "98: loss=0.992, reward_mean=0.100\n",
            "99: loss=1.008, reward_mean=0.130\n",
            "100: loss=1.099, reward_mean=0.110\n",
            "101: loss=0.928, reward_mean=0.170\n",
            "102: loss=0.973, reward_mean=0.130\n",
            "103: loss=0.998, reward_mean=0.050\n",
            "104: loss=0.899, reward_mean=0.180\n",
            "105: loss=1.006, reward_mean=0.150\n",
            "106: loss=0.982, reward_mean=0.140\n",
            "107: loss=0.915, reward_mean=0.120\n",
            "108: loss=0.881, reward_mean=0.080\n",
            "109: loss=1.122, reward_mean=0.090\n",
            "110: loss=0.951, reward_mean=0.110\n",
            "111: loss=0.954, reward_mean=0.080\n",
            "112: loss=0.942, reward_mean=0.140\n",
            "113: loss=0.997, reward_mean=0.150\n",
            "114: loss=0.974, reward_mean=0.140\n",
            "115: loss=0.943, reward_mean=0.110\n",
            "116: loss=0.927, reward_mean=0.140\n",
            "117: loss=0.885, reward_mean=0.160\n",
            "118: loss=0.970, reward_mean=0.150\n",
            "119: loss=0.921, reward_mean=0.180\n",
            "120: loss=0.908, reward_mean=0.150\n",
            "121: loss=0.960, reward_mean=0.160\n",
            "122: loss=1.040, reward_mean=0.140\n",
            "123: loss=0.959, reward_mean=0.170\n",
            "124: loss=1.085, reward_mean=0.150\n",
            "125: loss=0.873, reward_mean=0.160\n",
            "126: loss=0.995, reward_mean=0.160\n",
            "127: loss=0.921, reward_mean=0.150\n",
            "128: loss=0.944, reward_mean=0.160\n",
            "129: loss=0.949, reward_mean=0.170\n",
            "130: loss=0.879, reward_mean=0.190\n",
            "131: loss=0.877, reward_mean=0.150\n",
            "132: loss=0.913, reward_mean=0.150\n",
            "133: loss=0.946, reward_mean=0.140\n",
            "134: loss=0.886, reward_mean=0.130\n",
            "135: loss=1.049, reward_mean=0.080\n",
            "136: loss=0.886, reward_mean=0.200\n",
            "137: loss=0.793, reward_mean=0.130\n",
            "138: loss=0.850, reward_mean=0.120\n",
            "139: loss=0.936, reward_mean=0.130\n",
            "140: loss=0.961, reward_mean=0.140\n",
            "141: loss=0.892, reward_mean=0.140\n",
            "142: loss=0.921, reward_mean=0.120\n",
            "143: loss=0.965, reward_mean=0.150\n",
            "144: loss=0.899, reward_mean=0.160\n",
            "145: loss=0.866, reward_mean=0.110\n",
            "146: loss=0.914, reward_mean=0.150\n",
            "147: loss=0.782, reward_mean=0.140\n",
            "148: loss=0.959, reward_mean=0.120\n",
            "149: loss=0.768, reward_mean=0.110\n",
            "150: loss=0.866, reward_mean=0.170\n",
            "151: loss=0.861, reward_mean=0.180\n",
            "152: loss=0.853, reward_mean=0.120\n",
            "153: loss=0.800, reward_mean=0.100\n",
            "154: loss=0.867, reward_mean=0.180\n",
            "155: loss=0.860, reward_mean=0.210\n",
            "156: loss=0.827, reward_mean=0.220\n",
            "157: loss=0.798, reward_mean=0.180\n",
            "158: loss=0.855, reward_mean=0.180\n",
            "159: loss=0.859, reward_mean=0.090\n",
            "160: loss=0.862, reward_mean=0.120\n",
            "161: loss=0.828, reward_mean=0.220\n",
            "162: loss=0.766, reward_mean=0.130\n",
            "163: loss=0.891, reward_mean=0.180\n",
            "164: loss=0.831, reward_mean=0.180\n",
            "165: loss=0.789, reward_mean=0.170\n",
            "166: loss=0.838, reward_mean=0.160\n",
            "167: loss=0.878, reward_mean=0.180\n",
            "168: loss=0.926, reward_mean=0.240\n",
            "169: loss=0.828, reward_mean=0.180\n",
            "170: loss=0.814, reward_mean=0.150\n",
            "171: loss=0.910, reward_mean=0.120\n",
            "172: loss=0.921, reward_mean=0.120\n",
            "173: loss=0.810, reward_mean=0.200\n",
            "174: loss=0.821, reward_mean=0.200\n",
            "175: loss=0.787, reward_mean=0.120\n",
            "176: loss=0.772, reward_mean=0.130\n",
            "177: loss=0.795, reward_mean=0.220\n",
            "178: loss=0.894, reward_mean=0.190\n",
            "179: loss=0.789, reward_mean=0.180\n",
            "180: loss=0.934, reward_mean=0.100\n",
            "181: loss=0.812, reward_mean=0.170\n",
            "182: loss=0.809, reward_mean=0.140\n",
            "183: loss=0.803, reward_mean=0.130\n",
            "184: loss=0.830, reward_mean=0.160\n",
            "185: loss=0.848, reward_mean=0.180\n",
            "186: loss=0.867, reward_mean=0.140\n",
            "187: loss=0.924, reward_mean=0.160\n",
            "188: loss=0.846, reward_mean=0.120\n",
            "189: loss=0.897, reward_mean=0.100\n",
            "190: loss=0.768, reward_mean=0.200\n",
            "191: loss=0.834, reward_mean=0.190\n",
            "192: loss=0.831, reward_mean=0.140\n",
            "193: loss=0.833, reward_mean=0.130\n",
            "194: loss=0.747, reward_mean=0.150\n",
            "195: loss=0.826, reward_mean=0.150\n",
            "196: loss=0.885, reward_mean=0.230\n",
            "197: loss=0.778, reward_mean=0.120\n",
            "198: loss=0.820, reward_mean=0.190\n",
            "199: loss=0.743, reward_mean=0.190\n",
            "200: loss=0.846, reward_mean=0.170\n",
            "201: loss=0.855, reward_mean=0.100\n",
            "202: loss=0.872, reward_mean=0.140\n",
            "203: loss=0.739, reward_mean=0.210\n",
            "204: loss=0.777, reward_mean=0.240\n",
            "205: loss=0.791, reward_mean=0.170\n",
            "206: loss=0.817, reward_mean=0.210\n",
            "207: loss=0.800, reward_mean=0.210\n",
            "208: loss=0.885, reward_mean=0.150\n",
            "209: loss=0.746, reward_mean=0.180\n",
            "210: loss=0.789, reward_mean=0.210\n",
            "211: loss=0.799, reward_mean=0.220\n",
            "212: loss=0.729, reward_mean=0.180\n",
            "213: loss=0.861, reward_mean=0.190\n",
            "214: loss=0.783, reward_mean=0.240\n",
            "215: loss=0.804, reward_mean=0.210\n",
            "216: loss=0.791, reward_mean=0.150\n",
            "217: loss=0.787, reward_mean=0.150\n",
            "218: loss=0.737, reward_mean=0.160\n",
            "219: loss=0.800, reward_mean=0.240\n",
            "220: loss=0.832, reward_mean=0.220\n",
            "221: loss=0.769, reward_mean=0.190\n",
            "222: loss=0.817, reward_mean=0.250\n",
            "223: loss=0.689, reward_mean=0.170\n",
            "224: loss=0.835, reward_mean=0.190\n",
            "225: loss=0.836, reward_mean=0.170\n",
            "226: loss=0.811, reward_mean=0.170\n",
            "227: loss=0.756, reward_mean=0.200\n",
            "228: loss=0.808, reward_mean=0.150\n",
            "229: loss=0.752, reward_mean=0.290\n",
            "230: loss=0.767, reward_mean=0.170\n",
            "231: loss=0.837, reward_mean=0.130\n",
            "232: loss=0.868, reward_mean=0.170\n",
            "233: loss=0.778, reward_mean=0.230\n",
            "234: loss=0.801, reward_mean=0.180\n",
            "235: loss=0.752, reward_mean=0.300\n",
            "236: loss=0.770, reward_mean=0.270\n",
            "237: loss=0.746, reward_mean=0.120\n",
            "238: loss=0.786, reward_mean=0.160\n",
            "239: loss=0.833, reward_mean=0.160\n",
            "240: loss=0.796, reward_mean=0.250\n",
            "241: loss=0.815, reward_mean=0.130\n",
            "242: loss=0.706, reward_mean=0.270\n",
            "243: loss=0.790, reward_mean=0.210\n",
            "244: loss=0.794, reward_mean=0.260\n",
            "245: loss=0.680, reward_mean=0.140\n",
            "246: loss=0.874, reward_mean=0.130\n",
            "247: loss=0.762, reward_mean=0.150\n",
            "248: loss=0.721, reward_mean=0.210\n",
            "249: loss=0.727, reward_mean=0.180\n",
            "250: loss=0.807, reward_mean=0.240\n",
            "251: loss=0.785, reward_mean=0.220\n",
            "252: loss=0.729, reward_mean=0.180\n",
            "253: loss=0.776, reward_mean=0.240\n",
            "254: loss=0.783, reward_mean=0.180\n",
            "255: loss=0.719, reward_mean=0.190\n",
            "256: loss=0.724, reward_mean=0.250\n",
            "257: loss=0.702, reward_mean=0.260\n",
            "258: loss=0.796, reward_mean=0.180\n",
            "259: loss=0.781, reward_mean=0.230\n",
            "260: loss=0.709, reward_mean=0.190\n",
            "261: loss=0.800, reward_mean=0.230\n",
            "262: loss=0.698, reward_mean=0.220\n",
            "263: loss=0.813, reward_mean=0.210\n",
            "264: loss=0.806, reward_mean=0.190\n",
            "265: loss=0.804, reward_mean=0.240\n",
            "266: loss=0.660, reward_mean=0.220\n",
            "267: loss=0.684, reward_mean=0.180\n",
            "268: loss=0.724, reward_mean=0.210\n",
            "269: loss=0.695, reward_mean=0.130\n",
            "270: loss=0.781, reward_mean=0.210\n",
            "271: loss=0.780, reward_mean=0.190\n",
            "272: loss=0.751, reward_mean=0.260\n",
            "273: loss=0.713, reward_mean=0.240\n",
            "274: loss=0.732, reward_mean=0.150\n",
            "275: loss=0.727, reward_mean=0.190\n",
            "276: loss=0.702, reward_mean=0.250\n",
            "277: loss=0.758, reward_mean=0.250\n",
            "278: loss=0.661, reward_mean=0.150\n",
            "279: loss=0.684, reward_mean=0.160\n",
            "280: loss=0.703, reward_mean=0.170\n",
            "281: loss=0.714, reward_mean=0.220\n",
            "282: loss=0.681, reward_mean=0.240\n",
            "283: loss=0.789, reward_mean=0.270\n",
            "284: loss=0.736, reward_mean=0.210\n",
            "285: loss=0.774, reward_mean=0.210\n",
            "286: loss=0.785, reward_mean=0.240\n",
            "287: loss=0.693, reward_mean=0.180\n",
            "288: loss=0.708, reward_mean=0.270\n",
            "289: loss=0.684, reward_mean=0.220\n",
            "290: loss=0.745, reward_mean=0.270\n",
            "291: loss=0.746, reward_mean=0.180\n",
            "292: loss=0.739, reward_mean=0.240\n",
            "293: loss=0.738, reward_mean=0.340\n",
            "294: loss=0.692, reward_mean=0.210\n",
            "295: loss=0.658, reward_mean=0.230\n",
            "296: loss=0.700, reward_mean=0.230\n",
            "297: loss=0.667, reward_mean=0.250\n",
            "298: loss=0.715, reward_mean=0.240\n",
            "299: loss=0.693, reward_mean=0.220\n",
            "300: loss=0.714, reward_mean=0.260\n",
            "301: loss=0.723, reward_mean=0.160\n",
            "302: loss=0.777, reward_mean=0.240\n",
            "303: loss=0.738, reward_mean=0.290\n",
            "304: loss=0.668, reward_mean=0.230\n",
            "305: loss=0.770, reward_mean=0.160\n",
            "306: loss=0.735, reward_mean=0.300\n",
            "307: loss=0.758, reward_mean=0.210\n",
            "308: loss=0.718, reward_mean=0.300\n",
            "309: loss=0.702, reward_mean=0.150\n",
            "310: loss=0.759, reward_mean=0.270\n",
            "311: loss=0.706, reward_mean=0.260\n",
            "312: loss=0.680, reward_mean=0.170\n",
            "313: loss=0.739, reward_mean=0.250\n",
            "314: loss=0.661, reward_mean=0.260\n",
            "315: loss=0.716, reward_mean=0.190\n",
            "316: loss=0.711, reward_mean=0.200\n",
            "317: loss=0.704, reward_mean=0.240\n",
            "318: loss=0.707, reward_mean=0.280\n",
            "319: loss=0.737, reward_mean=0.260\n",
            "320: loss=0.654, reward_mean=0.280\n",
            "321: loss=0.738, reward_mean=0.280\n",
            "322: loss=0.775, reward_mean=0.230\n",
            "323: loss=0.766, reward_mean=0.290\n",
            "324: loss=0.766, reward_mean=0.300\n",
            "325: loss=0.769, reward_mean=0.260\n",
            "326: loss=0.674, reward_mean=0.360\n",
            "327: loss=0.785, reward_mean=0.190\n",
            "328: loss=0.710, reward_mean=0.350\n",
            "329: loss=0.817, reward_mean=0.270\n",
            "330: loss=0.633, reward_mean=0.240\n",
            "331: loss=0.684, reward_mean=0.260\n",
            "332: loss=0.662, reward_mean=0.270\n",
            "333: loss=0.689, reward_mean=0.190\n",
            "334: loss=0.704, reward_mean=0.280\n",
            "335: loss=0.646, reward_mean=0.190\n",
            "336: loss=0.662, reward_mean=0.220\n",
            "337: loss=0.664, reward_mean=0.290\n",
            "338: loss=0.630, reward_mean=0.200\n",
            "339: loss=0.663, reward_mean=0.280\n",
            "340: loss=0.689, reward_mean=0.310\n",
            "341: loss=0.690, reward_mean=0.230\n",
            "342: loss=0.644, reward_mean=0.220\n",
            "343: loss=0.756, reward_mean=0.280\n",
            "344: loss=0.718, reward_mean=0.240\n",
            "345: loss=0.743, reward_mean=0.280\n",
            "346: loss=0.709, reward_mean=0.250\n",
            "347: loss=0.711, reward_mean=0.320\n",
            "348: loss=0.666, reward_mean=0.250\n",
            "349: loss=0.714, reward_mean=0.180\n",
            "350: loss=0.675, reward_mean=0.260\n",
            "351: loss=0.692, reward_mean=0.240\n",
            "352: loss=0.714, reward_mean=0.280\n",
            "353: loss=0.702, reward_mean=0.240\n",
            "354: loss=0.663, reward_mean=0.240\n",
            "355: loss=0.710, reward_mean=0.220\n",
            "356: loss=0.675, reward_mean=0.280\n",
            "357: loss=0.684, reward_mean=0.290\n",
            "358: loss=0.721, reward_mean=0.310\n",
            "359: loss=0.651, reward_mean=0.210\n",
            "360: loss=0.663, reward_mean=0.300\n",
            "361: loss=0.670, reward_mean=0.280\n",
            "362: loss=0.647, reward_mean=0.340\n",
            "363: loss=0.671, reward_mean=0.210\n",
            "364: loss=0.644, reward_mean=0.340\n",
            "365: loss=0.676, reward_mean=0.330\n",
            "366: loss=0.643, reward_mean=0.320\n",
            "367: loss=0.658, reward_mean=0.320\n",
            "368: loss=0.644, reward_mean=0.270\n",
            "369: loss=0.648, reward_mean=0.220\n",
            "370: loss=0.594, reward_mean=0.280\n",
            "371: loss=0.639, reward_mean=0.370\n",
            "372: loss=0.722, reward_mean=0.260\n",
            "373: loss=0.712, reward_mean=0.280\n",
            "374: loss=0.649, reward_mean=0.240\n",
            "375: loss=0.703, reward_mean=0.380\n",
            "376: loss=0.670, reward_mean=0.330\n",
            "377: loss=0.614, reward_mean=0.350\n",
            "378: loss=0.685, reward_mean=0.290\n",
            "379: loss=0.678, reward_mean=0.200\n",
            "380: loss=0.639, reward_mean=0.270\n",
            "381: loss=0.661, reward_mean=0.290\n",
            "382: loss=0.661, reward_mean=0.370\n",
            "383: loss=0.674, reward_mean=0.250\n",
            "384: loss=0.630, reward_mean=0.330\n",
            "385: loss=0.660, reward_mean=0.280\n",
            "386: loss=0.598, reward_mean=0.250\n",
            "387: loss=0.672, reward_mean=0.340\n",
            "388: loss=0.622, reward_mean=0.370\n",
            "389: loss=0.635, reward_mean=0.280\n",
            "390: loss=0.658, reward_mean=0.350\n",
            "391: loss=0.699, reward_mean=0.310\n",
            "392: loss=0.665, reward_mean=0.400\n",
            "393: loss=0.665, reward_mean=0.340\n",
            "394: loss=0.652, reward_mean=0.340\n",
            "395: loss=0.614, reward_mean=0.330\n",
            "396: loss=0.731, reward_mean=0.420\n",
            "397: loss=0.654, reward_mean=0.430\n",
            "398: loss=0.682, reward_mean=0.350\n",
            "399: loss=0.610, reward_mean=0.340\n",
            "400: loss=0.667, reward_mean=0.290\n",
            "401: loss=0.653, reward_mean=0.360\n",
            "402: loss=0.691, reward_mean=0.270\n",
            "403: loss=0.630, reward_mean=0.260\n",
            "404: loss=0.663, reward_mean=0.300\n",
            "405: loss=0.641, reward_mean=0.290\n",
            "406: loss=0.629, reward_mean=0.410\n",
            "407: loss=0.728, reward_mean=0.240\n",
            "408: loss=0.679, reward_mean=0.370\n",
            "409: loss=0.630, reward_mean=0.400\n",
            "410: loss=0.716, reward_mean=0.290\n",
            "411: loss=0.649, reward_mean=0.350\n",
            "412: loss=0.688, reward_mean=0.320\n",
            "413: loss=0.631, reward_mean=0.390\n",
            "414: loss=0.647, reward_mean=0.260\n",
            "415: loss=0.623, reward_mean=0.410\n",
            "416: loss=0.632, reward_mean=0.400\n",
            "417: loss=0.638, reward_mean=0.320\n",
            "418: loss=0.646, reward_mean=0.320\n",
            "419: loss=0.658, reward_mean=0.470\n",
            "420: loss=0.621, reward_mean=0.350\n",
            "421: loss=0.646, reward_mean=0.370\n",
            "422: loss=0.664, reward_mean=0.370\n",
            "423: loss=0.605, reward_mean=0.370\n",
            "424: loss=0.607, reward_mean=0.360\n",
            "425: loss=0.660, reward_mean=0.400\n",
            "426: loss=0.610, reward_mean=0.370\n",
            "427: loss=0.622, reward_mean=0.440\n",
            "428: loss=0.662, reward_mean=0.330\n",
            "429: loss=0.593, reward_mean=0.390\n",
            "430: loss=0.627, reward_mean=0.300\n",
            "431: loss=0.627, reward_mean=0.260\n",
            "432: loss=0.649, reward_mean=0.370\n",
            "433: loss=0.612, reward_mean=0.480\n",
            "434: loss=0.612, reward_mean=0.400\n",
            "435: loss=0.607, reward_mean=0.330\n",
            "436: loss=0.650, reward_mean=0.380\n",
            "437: loss=0.645, reward_mean=0.410\n",
            "438: loss=0.646, reward_mean=0.390\n",
            "439: loss=0.630, reward_mean=0.360\n",
            "440: loss=0.681, reward_mean=0.380\n",
            "441: loss=0.665, reward_mean=0.500\n",
            "442: loss=0.657, reward_mean=0.380\n",
            "443: loss=0.627, reward_mean=0.410\n",
            "444: loss=0.656, reward_mean=0.370\n",
            "445: loss=0.597, reward_mean=0.350\n",
            "446: loss=0.620, reward_mean=0.410\n",
            "447: loss=0.571, reward_mean=0.410\n",
            "448: loss=0.648, reward_mean=0.360\n",
            "449: loss=0.657, reward_mean=0.440\n",
            "450: loss=0.627, reward_mean=0.360\n",
            "451: loss=0.640, reward_mean=0.460\n",
            "452: loss=0.641, reward_mean=0.400\n",
            "453: loss=0.602, reward_mean=0.480\n",
            "454: loss=0.593, reward_mean=0.470\n",
            "455: loss=0.599, reward_mean=0.360\n",
            "456: loss=0.630, reward_mean=0.410\n",
            "457: loss=0.663, reward_mean=0.510\n",
            "458: loss=0.581, reward_mean=0.330\n",
            "459: loss=0.618, reward_mean=0.370\n",
            "460: loss=0.594, reward_mean=0.340\n",
            "461: loss=0.606, reward_mean=0.450\n",
            "462: loss=0.621, reward_mean=0.430\n",
            "463: loss=0.636, reward_mean=0.400\n",
            "464: loss=0.641, reward_mean=0.390\n",
            "465: loss=0.576, reward_mean=0.410\n",
            "466: loss=0.729, reward_mean=0.340\n",
            "467: loss=0.589, reward_mean=0.430\n",
            "468: loss=0.598, reward_mean=0.440\n",
            "469: loss=0.630, reward_mean=0.480\n",
            "470: loss=0.631, reward_mean=0.480\n",
            "471: loss=0.644, reward_mean=0.440\n",
            "472: loss=0.604, reward_mean=0.360\n",
            "473: loss=0.617, reward_mean=0.500\n",
            "474: loss=0.575, reward_mean=0.370\n",
            "475: loss=0.628, reward_mean=0.360\n",
            "476: loss=0.579, reward_mean=0.440\n",
            "477: loss=0.618, reward_mean=0.340\n",
            "478: loss=0.607, reward_mean=0.390\n",
            "479: loss=0.567, reward_mean=0.370\n",
            "480: loss=0.570, reward_mean=0.370\n",
            "481: loss=0.562, reward_mean=0.510\n",
            "482: loss=0.608, reward_mean=0.380\n",
            "483: loss=0.640, reward_mean=0.470\n",
            "484: loss=0.604, reward_mean=0.510\n",
            "485: loss=0.587, reward_mean=0.430\n",
            "486: loss=0.594, reward_mean=0.470\n",
            "487: loss=0.588, reward_mean=0.470\n",
            "488: loss=0.645, reward_mean=0.380\n",
            "489: loss=0.621, reward_mean=0.490\n",
            "490: loss=0.630, reward_mean=0.460\n",
            "491: loss=0.589, reward_mean=0.400\n",
            "492: loss=0.587, reward_mean=0.440\n",
            "493: loss=0.582, reward_mean=0.430\n",
            "494: loss=0.564, reward_mean=0.450\n",
            "495: loss=0.552, reward_mean=0.440\n",
            "496: loss=0.579, reward_mean=0.470\n",
            "497: loss=0.620, reward_mean=0.430\n",
            "498: loss=0.591, reward_mean=0.420\n",
            "499: loss=0.597, reward_mean=0.530\n",
            "500: loss=0.579, reward_mean=0.480\n",
            "501: loss=0.601, reward_mean=0.420\n",
            "502: loss=0.602, reward_mean=0.440\n",
            "503: loss=0.584, reward_mean=0.500\n",
            "504: loss=0.617, reward_mean=0.500\n",
            "505: loss=0.565, reward_mean=0.460\n",
            "506: loss=0.594, reward_mean=0.490\n",
            "507: loss=0.581, reward_mean=0.510\n",
            "508: loss=0.639, reward_mean=0.550\n",
            "509: loss=0.641, reward_mean=0.440\n",
            "510: loss=0.621, reward_mean=0.470\n",
            "511: loss=0.602, reward_mean=0.410\n",
            "512: loss=0.590, reward_mean=0.420\n",
            "513: loss=0.630, reward_mean=0.470\n",
            "514: loss=0.624, reward_mean=0.430\n",
            "515: loss=0.564, reward_mean=0.440\n",
            "516: loss=0.559, reward_mean=0.590\n",
            "517: loss=0.603, reward_mean=0.490\n",
            "518: loss=0.585, reward_mean=0.470\n",
            "519: loss=0.569, reward_mean=0.530\n",
            "520: loss=0.584, reward_mean=0.440\n",
            "521: loss=0.574, reward_mean=0.510\n",
            "522: loss=0.608, reward_mean=0.480\n",
            "523: loss=0.594, reward_mean=0.530\n",
            "524: loss=0.550, reward_mean=0.500\n",
            "525: loss=0.589, reward_mean=0.480\n",
            "526: loss=0.598, reward_mean=0.430\n",
            "527: loss=0.549, reward_mean=0.440\n",
            "528: loss=0.609, reward_mean=0.470\n",
            "529: loss=0.550, reward_mean=0.480\n",
            "530: loss=0.576, reward_mean=0.500\n",
            "531: loss=0.547, reward_mean=0.430\n",
            "532: loss=0.572, reward_mean=0.500\n",
            "533: loss=0.560, reward_mean=0.500\n",
            "534: loss=0.598, reward_mean=0.560\n",
            "535: loss=0.544, reward_mean=0.480\n",
            "536: loss=0.603, reward_mean=0.530\n",
            "537: loss=0.582, reward_mean=0.590\n",
            "538: loss=0.592, reward_mean=0.570\n",
            "539: loss=0.559, reward_mean=0.560\n",
            "540: loss=0.560, reward_mean=0.640\n",
            "541: loss=0.613, reward_mean=0.560\n",
            "542: loss=0.569, reward_mean=0.610\n",
            "543: loss=0.551, reward_mean=0.440\n",
            "544: loss=0.574, reward_mean=0.520\n",
            "545: loss=0.552, reward_mean=0.510\n",
            "546: loss=0.551, reward_mean=0.470\n",
            "547: loss=0.605, reward_mean=0.510\n",
            "548: loss=0.570, reward_mean=0.510\n",
            "549: loss=0.566, reward_mean=0.620\n",
            "550: loss=0.573, reward_mean=0.670\n",
            "551: loss=0.557, reward_mean=0.560\n",
            "552: loss=0.576, reward_mean=0.500\n",
            "553: loss=0.557, reward_mean=0.520\n",
            "554: loss=0.591, reward_mean=0.550\n",
            "555: loss=0.573, reward_mean=0.540\n",
            "556: loss=0.545, reward_mean=0.560\n",
            "557: loss=0.565, reward_mean=0.600\n",
            "558: loss=0.550, reward_mean=0.500\n",
            "559: loss=0.577, reward_mean=0.500\n",
            "560: loss=0.584, reward_mean=0.590\n",
            "561: loss=0.577, reward_mean=0.610\n",
            "562: loss=0.598, reward_mean=0.480\n",
            "563: loss=0.622, reward_mean=0.530\n",
            "564: loss=0.587, reward_mean=0.570\n",
            "565: loss=0.557, reward_mean=0.570\n",
            "566: loss=0.540, reward_mean=0.470\n",
            "567: loss=0.519, reward_mean=0.660\n",
            "568: loss=0.535, reward_mean=0.560\n",
            "569: loss=0.568, reward_mean=0.570\n",
            "570: loss=0.563, reward_mean=0.530\n",
            "571: loss=0.555, reward_mean=0.570\n",
            "572: loss=0.545, reward_mean=0.600\n",
            "573: loss=0.537, reward_mean=0.600\n",
            "574: loss=0.558, reward_mean=0.600\n",
            "575: loss=0.568, reward_mean=0.530\n",
            "576: loss=0.535, reward_mean=0.650\n",
            "577: loss=0.565, reward_mean=0.610\n",
            "578: loss=0.550, reward_mean=0.560\n",
            "579: loss=0.524, reward_mean=0.580\n",
            "580: loss=0.538, reward_mean=0.560\n",
            "581: loss=0.572, reward_mean=0.610\n",
            "582: loss=0.595, reward_mean=0.620\n",
            "583: loss=0.571, reward_mean=0.600\n",
            "584: loss=0.561, reward_mean=0.540\n",
            "585: loss=0.548, reward_mean=0.630\n",
            "586: loss=0.519, reward_mean=0.590\n",
            "587: loss=0.623, reward_mean=0.650\n",
            "588: loss=0.577, reward_mean=0.590\n",
            "589: loss=0.518, reward_mean=0.590\n",
            "590: loss=0.543, reward_mean=0.610\n",
            "591: loss=0.543, reward_mean=0.680\n",
            "592: loss=0.533, reward_mean=0.620\n",
            "593: loss=0.547, reward_mean=0.570\n",
            "594: loss=0.565, reward_mean=0.660\n",
            "595: loss=0.570, reward_mean=0.630\n",
            "596: loss=0.599, reward_mean=0.540\n",
            "597: loss=0.571, reward_mean=0.640\n",
            "598: loss=0.570, reward_mean=0.650\n",
            "599: loss=0.518, reward_mean=0.630\n",
            "600: loss=0.566, reward_mean=0.540\n",
            "601: loss=0.580, reward_mean=0.610\n",
            "602: loss=0.547, reward_mean=0.670\n",
            "603: loss=0.569, reward_mean=0.590\n",
            "604: loss=0.520, reward_mean=0.690\n",
            "605: loss=0.517, reward_mean=0.650\n",
            "606: loss=0.556, reward_mean=0.520\n",
            "607: loss=0.536, reward_mean=0.610\n",
            "608: loss=0.550, reward_mean=0.630\n",
            "609: loss=0.542, reward_mean=0.590\n",
            "610: loss=0.560, reward_mean=0.660\n",
            "611: loss=0.561, reward_mean=0.700\n",
            "612: loss=0.556, reward_mean=0.670\n",
            "613: loss=0.526, reward_mean=0.560\n",
            "614: loss=0.533, reward_mean=0.700\n",
            "615: loss=0.522, reward_mean=0.700\n",
            "616: loss=0.547, reward_mean=0.630\n",
            "617: loss=0.517, reward_mean=0.640\n",
            "618: loss=0.541, reward_mean=0.720\n",
            "619: loss=0.524, reward_mean=0.580\n",
            "620: loss=0.535, reward_mean=0.700\n",
            "621: loss=0.575, reward_mean=0.640\n",
            "622: loss=0.508, reward_mean=0.620\n",
            "623: loss=0.502, reward_mean=0.630\n",
            "624: loss=0.539, reward_mean=0.590\n",
            "625: loss=0.540, reward_mean=0.700\n",
            "626: loss=0.550, reward_mean=0.710\n",
            "627: loss=0.583, reward_mean=0.640\n",
            "628: loss=0.521, reward_mean=0.610\n",
            "629: loss=0.568, reward_mean=0.670\n",
            "630: loss=0.566, reward_mean=0.670\n",
            "631: loss=0.533, reward_mean=0.670\n",
            "632: loss=0.507, reward_mean=0.610\n",
            "633: loss=0.533, reward_mean=0.640\n",
            "634: loss=0.519, reward_mean=0.650\n",
            "635: loss=0.513, reward_mean=0.680\n",
            "636: loss=0.522, reward_mean=0.570\n",
            "637: loss=0.521, reward_mean=0.670\n",
            "638: loss=0.545, reward_mean=0.700\n",
            "639: loss=0.527, reward_mean=0.680\n",
            "640: loss=0.565, reward_mean=0.660\n",
            "641: loss=0.532, reward_mean=0.590\n",
            "642: loss=0.524, reward_mean=0.690\n",
            "643: loss=0.539, reward_mean=0.730\n",
            "644: loss=0.552, reward_mean=0.640\n",
            "645: loss=0.556, reward_mean=0.730\n",
            "646: loss=0.519, reward_mean=0.660\n",
            "647: loss=0.539, reward_mean=0.700\n",
            "648: loss=0.561, reward_mean=0.690\n",
            "649: loss=0.545, reward_mean=0.670\n",
            "650: loss=0.480, reward_mean=0.760\n",
            "651: loss=0.530, reward_mean=0.680\n",
            "652: loss=0.520, reward_mean=0.680\n",
            "653: loss=0.509, reward_mean=0.650\n",
            "654: loss=0.500, reward_mean=0.700\n",
            "655: loss=0.493, reward_mean=0.740\n",
            "656: loss=0.521, reward_mean=0.680\n",
            "657: loss=0.502, reward_mean=0.710\n",
            "658: loss=0.510, reward_mean=0.720\n",
            "659: loss=0.490, reward_mean=0.630\n",
            "660: loss=0.570, reward_mean=0.620\n",
            "661: loss=0.548, reward_mean=0.690\n",
            "662: loss=0.525, reward_mean=0.700\n",
            "663: loss=0.496, reward_mean=0.760\n",
            "664: loss=0.503, reward_mean=0.740\n",
            "665: loss=0.497, reward_mean=0.670\n",
            "666: loss=0.551, reward_mean=0.690\n",
            "667: loss=0.528, reward_mean=0.660\n",
            "668: loss=0.509, reward_mean=0.710\n",
            "669: loss=0.540, reward_mean=0.750\n",
            "670: loss=0.515, reward_mean=0.740\n",
            "671: loss=0.515, reward_mean=0.710\n",
            "672: loss=0.553, reward_mean=0.720\n",
            "673: loss=0.481, reward_mean=0.720\n",
            "674: loss=0.498, reward_mean=0.680\n",
            "675: loss=0.563, reward_mean=0.660\n",
            "676: loss=0.520, reward_mean=0.680\n",
            "677: loss=0.522, reward_mean=0.650\n",
            "678: loss=0.531, reward_mean=0.700\n",
            "679: loss=0.502, reward_mean=0.790\n",
            "680: loss=0.546, reward_mean=0.650\n",
            "681: loss=0.502, reward_mean=0.760\n",
            "682: loss=0.490, reward_mean=0.720\n",
            "683: loss=0.509, reward_mean=0.700\n",
            "684: loss=0.512, reward_mean=0.720\n",
            "685: loss=0.474, reward_mean=0.770\n",
            "686: loss=0.501, reward_mean=0.740\n",
            "687: loss=0.529, reward_mean=0.670\n",
            "688: loss=0.480, reward_mean=0.820\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "IDFUFrGPEaE0",
        "colab_type": "text"
      },
      "source": [
        "## Test the Agent"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "bklWhHswvoxR",
        "colab_type": "code",
        "outputId": "91399164-056d-4c7a-b0d4-5b2048f1767e",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        }
      },
      "source": [
        "test_env = OneHotWrapper(gym.make('FrozenLake-v0', is_slippery=False))\n",
        "state= test_env.reset()\n",
        "test_env.render()\n",
        "\n",
        "is_done = False\n",
        "\n",
        "while not is_done:\n",
        "    action = select_action(state)\n",
        "    new_state, reward, is_done, _ = test_env.step(action)\n",
        "    test_env.render()\n",
        "    state = new_state\n",
        "\n",
        "print(\"reward = \", reward)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "\n",
            "\u001b[41mS\u001b[0mFFF\n",
            "FHFH\n",
            "FFFH\n",
            "HFFG\n",
            "  (Down)\n",
            "SFFF\n",
            "\u001b[41mF\u001b[0mHFH\n",
            "FFFH\n",
            "HFFG\n",
            "  (Down)\n",
            "SFFF\n",
            "FHFH\n",
            "\u001b[41mF\u001b[0mFFH\n",
            "HFFG\n",
            "  (Right)\n",
            "SFFF\n",
            "FHFH\n",
            "F\u001b[41mF\u001b[0mFH\n",
            "HFFG\n",
            "  (Down)\n",
            "SFFF\n",
            "FHFH\n",
            "FFFH\n",
            "H\u001b[41mF\u001b[0mFG\n",
            "  (Down)\n",
            "SFFF\n",
            "FHFH\n",
            "FFFH\n",
            "H\u001b[41mF\u001b[0mFG\n",
            "  (Down)\n",
            "SFFF\n",
            "FHFH\n",
            "FFFH\n",
            "H\u001b[41mF\u001b[0mFG\n",
            "  (Down)\n",
            "SFFF\n",
            "FHFH\n",
            "FFFH\n",
            "H\u001b[41mF\u001b[0mFG\n",
            "  (Down)\n",
            "SFFF\n",
            "FHFH\n",
            "FFFH\n",
            "H\u001b[41mF\u001b[0mFG\n",
            "  (Right)\n",
            "SFFF\n",
            "FHFH\n",
            "FFFH\n",
            "HF\u001b[41mF\u001b[0mG\n",
            "  (Down)\n",
            "SFFF\n",
            "FHFH\n",
            "FFFH\n",
            "HF\u001b[41mF\u001b[0mG\n",
            "  (Right)\n",
            "SFFF\n",
            "FHFH\n",
            "FFFH\n",
            "HFF\u001b[41mG\u001b[0m\n",
            "reward =  1.0\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "T5DAW482QQxX",
        "colab_type": "text"
      },
      "source": [
        "----\n",
        "\n",
        "DEEP REINFORCEMENT LEARNING EXPLAINED - 07\n",
        "# **Cross-Entropy Method Performance Analysis**\n",
        "## Implementation of the Cross-Entropy Training Loop"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "i4zeAp9mLowE",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "%load_ext tensorboard"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "JpOMZfj6SZkU",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from torch.utils.tensorboard import SummaryWriter\n",
        "\n",
        "def train_loop():\n",
        "   writer = SummaryWriter(comment=\"-Frozen-Lake-nonslippery\")\n",
        "\n",
        "   iter_no = 0\n",
        "   reward_mean = 0\n",
        "   full_batch = []\n",
        "   batch = []\n",
        "   episode_steps = []\n",
        "   episode_reward = 0.0\n",
        "   state = env.reset()\n",
        "    \n",
        "   while reward_mean < REWARD_GOAL:\n",
        "        action = select_action(state)\n",
        "        next_state, reward, episode_is_done, _ = env.step(action)\n",
        "\n",
        "        episode_steps.append(EpisodeStep(observation=state, action=action))\n",
        "        episode_reward += reward\n",
        "        \n",
        "        if episode_is_done: # Episode finished            \n",
        "            batch.append(Episode(reward=episode_reward, steps=episode_steps))\n",
        "            next_state = env.reset()\n",
        "            episode_steps = []\n",
        "            episode_reward = 0.0\n",
        "             \n",
        "            if len(batch) == BATCH_SIZE: # New set of batches ready --> select \"elite\"\n",
        "                reward_mean = float(np.mean(list(map(lambda s: s.reward, batch))))\n",
        "                #elite_candidates= full_batch + batch \n",
        "                elite_candidates= batch \n",
        "                returnG = list(map(lambda s: s.reward * (GAMMA ** len(s.steps)), elite_candidates))\n",
        "                reward_bound = np.percentile(returnG, PERCENTILE)\n",
        "\n",
        "                train_obs = []\n",
        "                train_act = []\n",
        "                elite_batch = []\n",
        "                for example, discounted_reward in zip(elite_candidates, returnG):\n",
        "                        if discounted_reward > reward_bound:\n",
        "                              train_obs.extend(map(lambda step: step.observation, example.steps))\n",
        "                              train_act.extend(map(lambda step: step.action, example.steps))\n",
        "                              elite_batch.append(example)\n",
        "                full_batch=elite_batch\n",
        "                state=train_obs\n",
        "                acts=train_act\n",
        "\n",
        "                \n",
        "                if len(full_batch) != 0 : # just in case empty during an iteration\n",
        "                       state_t = torch.FloatTensor(state)\n",
        "                       acts_t = torch.LongTensor(acts)\n",
        "\n",
        "                       optimizer.zero_grad()\n",
        "                       action_scores_t = net(state_t)\n",
        "                       loss_t = objective(action_scores_t, acts_t)\n",
        "                       loss_t.backward()\n",
        "                       optimizer.step()\n",
        "                       writer.add_scalar(\"loss\", loss_t.item(), iter_no)\n",
        "                       writer.add_scalar(\"reward_mean\", reward_mean, iter_no)\n",
        "                       iter_no += 1\n",
        "                batch = []\n",
        "        state = next_state\n",
        "\n",
        "   writer.close()\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "BT7PWgkeJpop",
        "colab_type": "text"
      },
      "source": [
        "### Base line"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "iBhfKxo6JugR",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "HIDDEN_SIZE = 32\n",
        "net= nn.Sequential(\n",
        "            nn.Linear(obs_size, HIDDEN_SIZE),\n",
        "            nn.Sigmoid(),\n",
        "            nn.Linear(HIDDEN_SIZE, n_actions)\n",
        "        )\n",
        "objective = nn.CrossEntropyLoss()\n",
        "optimizer = optim.Adam(params=net.parameters(), lr=0.001)\n",
        "\n",
        "train_loop()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "cLeA9WrDLmyc",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "tensorboard  --logdir=runs"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "cC_sHpCzIc3-",
        "colab_type": "text"
      },
      "source": [
        "### More complex Neural Network\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ivNpM-ZdGvJB",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "HIDDEN_SIZE = 128\n",
        "net= nn.Sequential(\n",
        "            nn.Linear(obs_size, HIDDEN_SIZE),\n",
        "            nn.Sigmoid(),\n",
        "            nn.Linear(HIDDEN_SIZE, n_actions)\n",
        "        )\n",
        "objective = nn.CrossEntropyLoss()\n",
        "optimizer = optim.Adam(params=net.parameters(), lr=0.001)\n",
        "\n",
        "train_loop()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "VGIKl6U6QutK",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "tensorboard  --logdir=runs"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "wwfblK-FI--6",
        "colab_type": "text"
      },
      "source": [
        "### ReLU activation function"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "hHyy53c4I1sd",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "HIDDEN_SIZE = 128\n",
        "net= nn.Sequential(\n",
        "            nn.Linear(obs_size, HIDDEN_SIZE),\n",
        "            nn.ReLU(),\n",
        "            nn.Linear(HIDDEN_SIZE, n_actions)\n",
        "        )\n",
        "objective = nn.CrossEntropyLoss()\n",
        "optimizer = optim.Adam(params=net.parameters(), lr=0.001)\n",
        "\n",
        "train_loop()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "FXadVHxTJOmu",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "tensorboard  --logdir=runs"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "MSYJt3xfJ1xd",
        "colab_type": "text"
      },
      "source": [
        "### Improving Cross-Entropy Algorithm"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "eViQ7PVxJ-3h",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def improved_train_loop():\n",
        "   writer = SummaryWriter(comment=\"-Frozen-Lake-nonslippery\")\n",
        "\n",
        "   iter_no = 0\n",
        "   reward_mean = 0\n",
        "   full_batch = []\n",
        "   batch = []\n",
        "   episode_steps = []\n",
        "   episode_reward = 0.0\n",
        "   state = env.reset()\n",
        "    \n",
        "   while reward_mean < REWARD_GOAL:\n",
        "        action = select_action(state)\n",
        "        next_state, reward, episode_is_done, _ = env.step(action)\n",
        "\n",
        "        episode_steps.append(EpisodeStep(observation=state, action=action))\n",
        "        episode_reward += reward\n",
        "        \n",
        "        if episode_is_done: # Episode finished            \n",
        "            batch.append(Episode(reward=episode_reward, steps=episode_steps))\n",
        "            next_state = env.reset()\n",
        "            episode_steps = []\n",
        "            episode_reward = 0.0\n",
        "             \n",
        "            if len(batch) == BATCH_SIZE: # New set of batches ready --> select \"elite\"\n",
        "                reward_mean = float(np.mean(list(map(lambda s: s.reward, batch))))\n",
        "\n",
        "                elite_candidates= full_batch + batch \n",
        "                #elite_candidates= batch \n",
        "\n",
        "                returnG = list(map(lambda s: s.reward * (GAMMA ** len(s.steps)), elite_candidates))\n",
        "                reward_bound = np.percentile(returnG, PERCENTILE)\n",
        "\n",
        "                train_obs = []\n",
        "                train_act = []\n",
        "                elite_batch = []\n",
        "                for example, discounted_reward in zip(elite_candidates, returnG):\n",
        "                        if discounted_reward > reward_bound:\n",
        "                              train_obs.extend(map(lambda step: step.observation, example.steps))\n",
        "                              train_act.extend(map(lambda step: step.action, example.steps))\n",
        "                              elite_batch.append(example)\n",
        "                full_batch=elite_batch\n",
        "                state=train_obs\n",
        "                acts=train_act\n",
        "\n",
        "                \n",
        "                if len(full_batch) != 0 : # just in case empty during an iteration\n",
        "                       state_t = torch.FloatTensor(state)\n",
        "                       acts_t = torch.LongTensor(acts)\n",
        "\n",
        "                       optimizer.zero_grad()\n",
        "                       action_scores_t = net(state_t)\n",
        "                       loss_t = objective(action_scores_t, acts_t)\n",
        "                       loss_t.backward()\n",
        "                       optimizer.step()\n",
        "                       writer.add_scalar(\"loss\", loss_t.item(), iter_no)\n",
        "                       writer.add_scalar(\"reward_mean\", reward_mean, iter_no)\n",
        "                       iter_no += 1\n",
        "                batch = []\n",
        "        state = next_state\n",
        "\n",
        "   writer.close()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Kh0oCNq4KShA",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "HIDDEN_SIZE = 128\n",
        "net= nn.Sequential(\n",
        "            nn.Linear(obs_size, HIDDEN_SIZE),\n",
        "            nn.ReLU(),\n",
        "            nn.Linear(HIDDEN_SIZE, n_actions)\n",
        "        )\n",
        "objective = nn.CrossEntropyLoss()\n",
        "optimizer = optim.Adam(params=net.parameters(), lr=0.001)\n",
        "\n",
        "improved_train_loop()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Ak6_fgHeKYjP",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "tensorboard  --logdir=runs"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "6yYHjUH3Kji4",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "slippedy_env = gym.make('FrozenLake-v0', is_slippery=True)\n",
        "\n",
        "class OneHotWrapper(gym.ObservationWrapper):\n",
        "    def __init__(self, env):\n",
        "        super(OneHotWrapper, self).__init__(env)\n",
        "        self.observation_space = gym.spaces.Box(0.0, 1.0, (env.observation_space.n, ), dtype=np.float32)\n",
        "\n",
        "    def observation(self, observation):\n",
        "        r = np.copy(self.observation_space.low)\n",
        "        r[observation] = 1.0\n",
        "        return r\n",
        "\n",
        "env = OneHotWrapper(slippedy_env)\n",
        "\n",
        "HIDDEN_SIZE = 128\n",
        "net= nn.Sequential(\n",
        "            nn.Linear(obs_size, HIDDEN_SIZE),\n",
        "            nn.ReLU(),\n",
        "            nn.Linear(HIDDEN_SIZE, n_actions)\n",
        "        )\n",
        "objective = nn.CrossEntropyLoss()\n",
        "optimizer = optim.Adam(params=net.parameters(), lr=0.001)\n",
        "\n",
        "improved_train_loop()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8vSl0u9mSuru",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "tensorboard  --logdir=runs"
      ],
      "execution_count": 0,
      "outputs": []
    }
  ]
}