dueling_ddqn_main.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "dueling_ddqn_main.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "authorship_tag": "ABX9TyMA1VZ1t25Ef7SqJ3Imtrjq",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/RoyElkabetz/DQN_with_PyTorch_and_Gym/blob/main/dueling_ddqn_main.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "wdE_N0jDH5sv"
      },
      "source": [
        "# Implementing DQN Algorithms with PyTorch and Gym\n",
        "\n",
        "## Clone the DQN repository"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "5gbAv24suwRn",
        "outputId": "f5f0e590-da5c-44c7-b931-9a383b8ec2ff"
      },
      "source": [
        "## uncomment only if running from google.colab\n",
        "# clone the git reposetory\n",
        "!git clone https://github.com/RoyElkabetz/DQN_with_PyTorch_and_Gym.git\n",
        "\n",
        "# add path to .py files for import\n",
        "import sys\n",
        "sys.path.insert(1, \"/content/DQN_with_PyTorch_and_Gym\")"
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Cloning into 'DQN_with_PyTorch_and_Gym'...\n",
            "remote: Enumerating objects: 439, done.\u001b[K\n",
            "remote: Counting objects: 100% (439/439), done.\u001b[K\n",
            "remote: Compressing objects: 100% (366/366), done.\u001b[K\n",
            "remote: Total 439 (delta 235), reused 221 (delta 69), pack-reused 0\u001b[K\n",
            "Receiving objects: 100% (439/439), 61.13 MiB | 6.63 MiB/s, done.\n",
            "Resolving deltas: 100% (235/235), done.\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "JtyjPQyzIRjS"
      },
      "source": [
        "## Mount your google drive to save results and checkpoints"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "D_4kp2PoD2DX",
        "outputId": "1c375ea0-e37f-464f-f610-82f66d10a8d2"
      },
      "source": [
        "## uncomment to mount google drive\n",
        "from google.colab import drive\n",
        "drive.mount('/content/gdrive')"
      ],
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Mounted at /content/gdrive\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "-I6QVKrYIYcf"
      },
      "source": [
        "## Check GPU performance"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "47kFZQDDD4mt",
        "outputId": "5811cdb8-9709-4b13-d9f5-8998487cbb3f"
      },
      "source": [
        "# check GPU parameters\n",
        "!nvidia-smi"
      ],
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Sun Oct 24 19:33:49 2021       \n",
            "+-----------------------------------------------------------------------------+\n",
            "| NVIDIA-SMI 470.74       Driver Version: 460.32.03    CUDA Version: 11.2     |\n",
            "|-------------------------------+----------------------+----------------------+\n",
            "| GPU  Name        Persistence-M| Bus-Id        Disp.A | Volatile Uncorr. ECC |\n",
            "| Fan  Temp  Perf  Pwr:Usage/Cap|         Memory-Usage | GPU-Util  Compute M. |\n",
            "|                               |                      |               MIG M. |\n",
            "|===============================+======================+======================|\n",
            "|   0  Tesla K80           Off  | 00000000:00:04.0 Off |                    0 |\n",
            "| N/A   36C    P8    28W / 149W |      0MiB / 11441MiB |      0%      Default |\n",
            "|                               |                      |                  N/A |\n",
            "+-------------------------------+----------------------+----------------------+\n",
            "                                                                               \n",
            "+-----------------------------------------------------------------------------+\n",
            "| Processes:                                                                  |\n",
            "|  GPU   GI   CI        PID   Type   Process name                  GPU Memory |\n",
            "|        ID   ID                                                   Usage      |\n",
            "|=============================================================================|\n",
            "|  No running processes found                                                 |\n",
            "+-----------------------------------------------------------------------------+\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "dwh03dC0IgLk"
      },
      "source": [
        "## Install Gym and atari libraries"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "o_5-RtyevBFf"
      },
      "source": [
        "!pip install --quiet gym\n",
        "!pip install --quiet atari_py"
      ],
      "execution_count": 4,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "eKsP6i26Is1w"
      },
      "source": [
        "## Fetch and install ROMs for Atari library"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "UOVRF-H6vtdv",
        "outputId": "6f2960a3-73c5-49d6-ae01-f134b8e3cf08"
      },
      "source": [
        "! wget http://www.atarimania.com/roms/Roms.rar\n",
        "! mkdir /content/ROM/\n",
        "! unrar e /content/Roms.rar /content/ROM/\n",
        "! python -m atari_py.import_roms /content/ROM/"
      ],
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "--2021-10-24 19:34:04--  http://www.atarimania.com/roms/Roms.rar\n",
            "Resolving www.atarimania.com (www.atarimania.com)... 195.154.81.199\n",
            "Connecting to www.atarimania.com (www.atarimania.com)|195.154.81.199|:80... connected.\n",
            "HTTP request sent, awaiting response... 200 OK\n",
            "Length: 11128004 (11M) [application/x-rar-compressed]\n",
            "Saving to: ‘Roms.rar’\n",
            "\n",
            "Roms.rar            100%[===================>]  10.61M   263KB/s    in 45s     \n",
            "\n",
            "2021-10-24 19:34:50 (241 KB/s) - ‘Roms.rar’ saved [11128004/11128004]\n",
            "\n",
            "\n",
            "UNRAR 5.50 freeware      Copyright (c) 1993-2017 Alexander Roshal\n",
            "\n",
            "\n",
            "Extracting from /content/Roms.rar\n",
            "\n",
            "Extracting  /content/ROM/HC ROMS.zip                                     \b\b\b\b 36%\b\b\b\b\b  OK \n",
            "Extracting  /content/ROM/ROMS.zip                                        \b\b\b\b 74%\b\b\b\b 99%\b\b\b\b\b  OK \n",
            "All OK\n",
            "copying adventure.bin from ROMS/Adventure (1980) (Atari, Warren Robinett) (CX2613, CX2613P) (PAL).bin to /usr/local/lib/python3.7/dist-packages/atari_py/atari_roms/adventure.bin\n",
            "copying air_raid.bin from ROMS/Air Raid (Men-A-Vision) (PAL) ~.bin to /usr/local/lib/python3.7/dist-packages/atari_py/atari_roms/air_raid.bin\n",
            "copying alien.bin from ROMS/Alien (1982) (20th Century Fox Video Games, Douglas 'Dallas North' Neubauer) (11006) ~.bin to /usr/local/lib/python3.7/dist-packages/atari_py/atari_roms/alien.bin\n",
            "copying amidar.bin from ROMS/Amidar (1982) (Parker Brothers, Ed Temple) (PB5310) ~.bin to /usr/local/lib/python3.7/dist-packages/atari_py/atari_roms/amidar.bin\n",
            "copying assault.bin from ROMS/Assault (AKA Sky Alien) (1983) (Bomb - Onbase) (CA281).bin to /usr/local/lib/python3.7/dist-packages/atari_py/atari_roms/assault.bin\n",
            "copying asterix.bin from ROMS/Asterix (AKA Taz) (07-27-1983) (Atari, Jerome Domurat, Steve Woita) (CX2696) (Prototype).bin to /usr/local/lib/python3.7/dist-packages/atari_py/atari_roms/asterix.bin\n",
            "copying asteroids.bin from ROMS/Asteroids (1981) (Atari, Brad Stewart - Sears) (CX2649 - 49-75163) [no copyright] ~.bin to /usr/local/lib/python3.7/dist-packages/atari_py/atari_roms/asteroids.bin\n",
            "copying atlantis.bin from ROMS/Atlantis (Lost City of Atlantis) (1982) (Imagic, Dennis Koble) (720103-1A, 720103-1B, IA3203, IX-010-04) ~.bin to /usr/local/lib/python3.7/dist-packages/atari_py/atari_roms/atlantis.bin\n",
            "copying bank_heist.bin from ROMS/Bank Heist (Bonnie & Clyde, Cops 'n' Robbers, Hold-Up, Roaring 20's) (1983) (20th Century Fox Video Games, Bill Aspromonte) (11012) ~.bin to /usr/local/lib/python3.7/dist-packages/atari_py/atari_roms/bank_heist.bin\n",
            "copying battle_zone.bin from ROMS/Battlezone (1983) (Atari - GCC, Mike Feinstein, Brad Rice) (CX2681) ~.bin to /usr/local/lib/python3.7/dist-packages/atari_py/atari_roms/battle_zone.bin\n",
            "copying beam_rider.bin from ROMS/Beamrider (1984) (Activision - Cheshire Engineering, David Rolfe, Larry Zwick) (AZ-037-04) ~.bin to /usr/local/lib/python3.7/dist-packages/atari_py/atari_roms/beam_rider.bin\n",
            "copying berzerk.bin from ROMS/Berzerk (1982) (Atari, Dan Hitchens - Sears) (CX2650 - 49-75168) ~.bin to /usr/local/lib/python3.7/dist-packages/atari_py/atari_roms/berzerk.bin\n",
            "copying bowling.bin from ROMS/Bowling (1979) (Atari, Larry Kaplan - Sears) (CX2628 - 6-99842, 49-75117) ~.bin to /usr/local/lib/python3.7/dist-packages/atari_py/atari_roms/bowling.bin\n",
            "copying boxing.bin from ROMS/Boxing - La Boxe (1980) (Activision, Bob Whitehead) (AG-002, CAG-002, AG-002-04) ~.bin to /usr/local/lib/python3.7/dist-packages/atari_py/atari_roms/boxing.bin\n",
            "copying breakout.bin from ROMS/Breakout - Breakaway IV (Paddle) (1978) (Atari, Brad Stewart - Sears) (CX2622 - 6-99813, 49-75107) ~.bin to /usr/local/lib/python3.7/dist-packages/atari_py/atari_roms/breakout.bin\n",
            "copying carnival.bin from ROMS/Carnival (1982) (Coleco - Woodside Design Associates, Steve 'Jessica Stevens' Kitchen) (2468) ~.bin to /usr/local/lib/python3.7/dist-packages/atari_py/atari_roms/carnival.bin\n",
            "copying centipede.bin from ROMS/Centipede (1983) (Atari - GCC) (CX2676) ~.bin to /usr/local/lib/python3.7/dist-packages/atari_py/atari_roms/centipede.bin\n",
            "copying chopper_command.bin from ROMS/Chopper Command (1982) (Activision, Bob Whitehead) (AX-015, AX-015-04) ~.bin to /usr/local/lib/python3.7/dist-packages/atari_py/atari_roms/chopper_command.bin\n",
            "copying crazy_climber.bin from ROMS/Crazy Climber (1983) (Atari - Roklan, Joe Gaucher, Alex Leavens) (CX2683) ~.bin to /usr/local/lib/python3.7/dist-packages/atari_py/atari_roms/crazy_climber.bin\n",
            "copying defender.bin from ROMS/Defender (1982) (Atari, Robert C. Polaro, Alan J. Murphy - Sears) (CX2609 - 49-75186) ~.bin to /usr/local/lib/python3.7/dist-packages/atari_py/atari_roms/defender.bin\n",
            "copying demon_attack.bin from ROMS/Demon Attack (Death from Above) (1982) (Imagic, Rob Fulop) (720000-200, 720101-1B, 720101-1C, IA3200, IA3200C, IX-006-04) ~.bin to /usr/local/lib/python3.7/dist-packages/atari_py/atari_roms/demon_attack.bin\n",
            "copying donkey_kong.bin from ROMS/Donkey Kong (1982) (Coleco - Woodside Design Associates - Imaginative Systems Software, Garry Kitchen) (2451) ~.bin to /usr/local/lib/python3.7/dist-packages/atari_py/atari_roms/donkey_kong.bin\n",
            "copying double_dunk.bin from ROMS/Double Dunk (Super Basketball) (1989) (Atari, Matthew L. Hubbard) (CX26159) ~.bin to /usr/local/lib/python3.7/dist-packages/atari_py/atari_roms/double_dunk.bin\n",
            "copying elevator_action.bin from ROMS/Elevator Action (1983) (Atari, Dan Hitchens) (CX26126) (Prototype) ~.bin to /usr/local/lib/python3.7/dist-packages/atari_py/atari_roms/elevator_action.bin\n",
            "copying enduro.bin from ROMS/Enduro (1983) (Activision, Larry Miller) (AX-026, AX-026-04) ~.bin to /usr/local/lib/python3.7/dist-packages/atari_py/atari_roms/enduro.bin\n",
            "copying fishing_derby.bin from ROMS/Fishing Derby (1980) (Activision, David Crane) (AG-004) ~.bin to /usr/local/lib/python3.7/dist-packages/atari_py/atari_roms/fishing_derby.bin\n",
            "copying freeway.bin from ROMS/Freeway (1981) (Activision, David Crane) (AG-009, AG-009-04) ~.bin to /usr/local/lib/python3.7/dist-packages/atari_py/atari_roms/freeway.bin\n",
            "copying frogger.bin from ROMS/Frogger (1982) (Parker Brothers, Ed English, David Lamkins) (PB5300) ~.bin to /usr/local/lib/python3.7/dist-packages/atari_py/atari_roms/frogger.bin\n",
            "copying frostbite.bin from ROMS/Frostbite (1983) (Activision, Steve Cartwright) (AX-031) ~.bin to /usr/local/lib/python3.7/dist-packages/atari_py/atari_roms/frostbite.bin\n",
            "copying galaxian.bin from ROMS/Galaxian (1983) (Atari - GCC, Mark Ackerman, Tom Calderwood, Glenn Parker) (CX2684) ~.bin to /usr/local/lib/python3.7/dist-packages/atari_py/atari_roms/galaxian.bin\n",
            "copying gopher.bin from ROMS/Gopher (Gopher Attack) (1982) (U.S. Games Corporation - JWDA, Sylvia Day, Todd Marshall, Robin McDaniel, Henry Will IV) (VC2001) ~.bin to /usr/local/lib/python3.7/dist-packages/atari_py/atari_roms/gopher.bin\n",
            "copying gravitar.bin from ROMS/Gravitar (1983) (Atari, Dan Hitchens, Mimi Nyden) (CX2685) ~.bin to /usr/local/lib/python3.7/dist-packages/atari_py/atari_roms/gravitar.bin\n",
            "copying hero.bin from ROMS/H.E.R.O. (1984) (Activision, John Van Ryzin) (AZ-036-04) ~.bin to /usr/local/lib/python3.7/dist-packages/atari_py/atari_roms/hero.bin\n",
            "copying ice_hockey.bin from ROMS/Ice Hockey - Le Hockey Sur Glace (1981) (Activision, Alan Miller) (AX-012, CAX-012, AX-012-04) ~.bin to /usr/local/lib/python3.7/dist-packages/atari_py/atari_roms/ice_hockey.bin\n",
            "copying jamesbond.bin from ROMS/James Bond 007 (James Bond Agent 007) (1984) (Parker Brothers - On-Time Software, Joe Gaucher, Louis Marbel) (PB5110) ~.bin to /usr/local/lib/python3.7/dist-packages/atari_py/atari_roms/jamesbond.bin\n",
            "copying journey_escape.bin from ROMS/Journey Escape (1983) (Data Age, J. Ray Dettling) (112-006) ~.bin to /usr/local/lib/python3.7/dist-packages/atari_py/atari_roms/journey_escape.bin\n",
            "copying kaboom.bin from ROMS/Kaboom! (Paddle) (1981) (Activision, Larry Kaplan, David Crane) (AG-010, AG-010-04) ~.bin to /usr/local/lib/python3.7/dist-packages/atari_py/atari_roms/kaboom.bin\n",
            "copying kangaroo.bin from ROMS/Kangaroo (1983) (Atari - GCC, Kevin Osborn) (CX2689) ~.bin to /usr/local/lib/python3.7/dist-packages/atari_py/atari_roms/kangaroo.bin\n",
            "copying keystone_kapers.bin from ROMS/Keystone Kapers - Raueber und Gendarm (1983) (Activision, Garry Kitchen - Ariola) (EAX-025, EAX-025-04I - 711 025-725) (PAL).bin to /usr/local/lib/python3.7/dist-packages/atari_py/atari_roms/keystone_kapers.bin\n",
            "copying king_kong.bin from ROMS/King Kong (1982) (Tigervision - Software Electronics Corporation, Karl T. Olinger - Teldec) (7-001 - 3.60001 VE) (PAL).bin to /usr/local/lib/python3.7/dist-packages/atari_py/atari_roms/king_kong.bin\n",
            "copying koolaid.bin from ROMS/Kool-Aid Man (Kool Aid Pitcher Man) (1983) (M Network, Stephen Tatsumi, Jane Terjung - Kool Aid) (MT4648) ~.bin to /usr/local/lib/python3.7/dist-packages/atari_py/atari_roms/koolaid.bin\n",
            "copying krull.bin from ROMS/Krull (1983) (Atari, Jerome Domurat, Dave Staugas) (CX2682) ~.bin to /usr/local/lib/python3.7/dist-packages/atari_py/atari_roms/krull.bin\n",
            "copying kung_fu_master.bin from ROMS/Kung-Fu Master (1987) (Activision - Imagineering, Dan Kitchen, Garry Kitchen) (AG-039-04) ~.bin to /usr/local/lib/python3.7/dist-packages/atari_py/atari_roms/kung_fu_master.bin\n",
            "copying laser_gates.bin from ROMS/Laser Gates (AKA Innerspace) (1983) (Imagic, Dan Oliver) (720118-2A, 13208, EIX-007-04I) (PAL).bin to /usr/local/lib/python3.7/dist-packages/atari_py/atari_roms/laser_gates.bin\n",
            "copying lost_luggage.bin from ROMS/Lost Luggage (Airport Mayhem) (1982) (Apollo - Games by Apollo, Larry Minor, Ernie Runyon, Ed Salvo) (AP-2004) [no opening scene] ~.bin to /usr/local/lib/python3.7/dist-packages/atari_py/atari_roms/lost_luggage.bin\n",
            "copying montezuma_revenge.bin from ROMS/Montezuma's Revenge - Featuring Panama Joe (1984) (Parker Brothers - JWDA, Henry Will IV) (PB5760) ~.bin to /usr/local/lib/python3.7/dist-packages/atari_py/atari_roms/montezuma_revenge.bin\n",
            "copying mr_do.bin from ROMS/Mr. Do! (1983) (CBS Electronics, Ed English) (4L4478) (PAL).bin to /usr/local/lib/python3.7/dist-packages/atari_py/atari_roms/mr_do.bin\n",
            "copying ms_pacman.bin from ROMS/Ms. Pac-Man (1983) (Atari - GCC, Mark Ackerman, Glenn Parker) (CX2675) ~.bin to /usr/local/lib/python3.7/dist-packages/atari_py/atari_roms/ms_pacman.bin\n",
            "copying name_this_game.bin from ROMS/Name This Game (Guardians of Treasure) (1983) (U.S. Games Corporation - JWDA, Roger Booth, Sylvia Day, Ron Dubren, Todd Marshall, Robin McDaniel, Wes Trager, Henry Will IV) (VC1007) ~.bin to /usr/local/lib/python3.7/dist-packages/atari_py/atari_roms/name_this_game.bin\n",
            "copying pacman.bin from ROMS/Pac-Man (1982) (Atari, Tod Frye) (CX2646) (PAL).bin to /usr/local/lib/python3.7/dist-packages/atari_py/atari_roms/pacman.bin\n",
            "copying phoenix.bin from ROMS/Phoenix (1983) (Atari - GCC, Mike Feinstein, John Mracek) (CX2673) ~.bin to /usr/local/lib/python3.7/dist-packages/atari_py/atari_roms/phoenix.bin\n",
            "copying video_pinball.bin from ROMS/Pinball (AKA Video Pinball) (Zellers).bin to /usr/local/lib/python3.7/dist-packages/atari_py/atari_roms/video_pinball.bin\n",
            "copying pitfall.bin from ROMS/Pitfall! - Pitfall Harry's Jungle Adventure (Jungle Runner) (1982) (Activision, David Crane) (AX-018, AX-018-04) ~.bin to /usr/local/lib/python3.7/dist-packages/atari_py/atari_roms/pitfall.bin\n",
            "copying pooyan.bin from ROMS/Pooyan (1983) (Konami) (RC 100-X 02) ~.bin to /usr/local/lib/python3.7/dist-packages/atari_py/atari_roms/pooyan.bin\n",
            "copying private_eye.bin from ROMS/Private Eye (1984) (Activision, Bob Whitehead) (AG-034-04) ~.bin to /usr/local/lib/python3.7/dist-packages/atari_py/atari_roms/private_eye.bin\n",
            "copying qbert.bin from ROMS/Q-bert (1983) (Parker Brothers - Western Technologies, Dave Hampton, Tom Sloper) (PB5360) ~.bin to /usr/local/lib/python3.7/dist-packages/atari_py/atari_roms/qbert.bin\n",
            "copying riverraid.bin from ROMS/River Raid (1982) (Activision, Carol Shaw) (AX-020, AX-020-04) ~.bin to /usr/local/lib/python3.7/dist-packages/atari_py/atari_roms/riverraid.bin\n",
            "copying road_runner.bin from patched version of ROMS/Road Runner (1989) (Atari - Bobco, Robert C. Polaro) (CX2663) ~.bin to /usr/local/lib/python3.7/dist-packages/atari_py/atari_roms/road_runner.bin\n",
            "copying robotank.bin from ROMS/Robot Tank (Robotank) (1983) (Activision, Alan Miller) (AZ-028, AG-028-04) ~.bin to /usr/local/lib/python3.7/dist-packages/atari_py/atari_roms/robotank.bin\n",
            "copying seaquest.bin from ROMS/Seaquest (1983) (Activision, Steve Cartwright) (AX-022) ~.bin to /usr/local/lib/python3.7/dist-packages/atari_py/atari_roms/seaquest.bin\n",
            "copying sir_lancelot.bin from ROMS/Sir Lancelot (1983) (Xonox - K-Tel Software - Product Guild, Anthony R. Henderson) (99006, 6220) (PAL).bin to /usr/local/lib/python3.7/dist-packages/atari_py/atari_roms/sir_lancelot.bin\n",
            "copying skiing.bin from ROMS/Skiing - Le Ski (1980) (Activision, Bob Whitehead) (AG-005, CAG-005, AG-005-04) ~.bin to /usr/local/lib/python3.7/dist-packages/atari_py/atari_roms/skiing.bin\n",
            "copying solaris.bin from ROMS/Solaris (The Last Starfighter, Star Raiders II, Universe) (1986) (Atari, Douglas Neubauer, Mimi Nyden) (CX26136) ~.bin to /usr/local/lib/python3.7/dist-packages/atari_py/atari_roms/solaris.bin\n",
            "copying space_invaders.bin from ROMS/Space Invaders (1980) (Atari, Richard Maurer - Sears) (CX2632 - 49-75153) ~.bin to /usr/local/lib/python3.7/dist-packages/atari_py/atari_roms/space_invaders.bin\n",
            "copying star_gunner.bin from ROMS/Stargunner (1983) (Telesys, Alex Leavens) (1005) ~.bin to /usr/local/lib/python3.7/dist-packages/atari_py/atari_roms/star_gunner.bin\n",
            "copying surround.bin from ROMS/Surround (32 in 1) (Bit Corporation) (R320).bin to /usr/local/lib/python3.7/dist-packages/atari_py/atari_roms/surround.bin\n",
            "copying tennis.bin from ROMS/Tennis - Le Tennis (1981) (Activision, Alan Miller) (AG-007, CAG-007) ~.bin to /usr/local/lib/python3.7/dist-packages/atari_py/atari_roms/tennis.bin\n",
            "copying time_pilot.bin from ROMS/Time Pilot (1983) (Coleco - Woodside Design Associates, Harley H. Puthuff Jr.) (2663) ~.bin to /usr/local/lib/python3.7/dist-packages/atari_py/atari_roms/time_pilot.bin\n",
            "copying trondead.bin from ROMS/TRON - Deadly Discs (TRON Joystick) (1983) (M Network - INTV - APh Technological Consulting, Jeff Ronne, Brett Stutz) (MT5662) ~.bin to /usr/local/lib/python3.7/dist-packages/atari_py/atari_roms/trondead.bin\n",
            "copying tutankham.bin from ROMS/Tutankham (1983) (Parker Brothers, Dave Engman, Dawn Stockbridge) (PB5340) ~.bin to /usr/local/lib/python3.7/dist-packages/atari_py/atari_roms/tutankham.bin\n",
            "copying up_n_down.bin from ROMS/Up 'n Down (1984) (SEGA - Beck-Tech, Steve Beck, Phat Ho) (009-01) ~.bin to /usr/local/lib/python3.7/dist-packages/atari_py/atari_roms/up_n_down.bin\n",
            "copying venture.bin from ROMS/Venture (1982) (Coleco, Joseph Biel) (2457) ~.bin to /usr/local/lib/python3.7/dist-packages/atari_py/atari_roms/venture.bin\n",
            "copying pong.bin from ROMS/Video Olympics - Pong Sports (Paddle) (1977) (Atari, Joe Decuir - Sears) (CX2621 - 99806, 6-99806, 49-75104) ~.bin to /usr/local/lib/python3.7/dist-packages/atari_py/atari_roms/pong.bin\n",
            "copying wizard_of_wor.bin from ROMS/Wizard of Wor (1982) (CBS Electronics - Roklan, Joe Hellesen, Joe Wagner) (M8774, M8794) ~.bin to /usr/local/lib/python3.7/dist-packages/atari_py/atari_roms/wizard_of_wor.bin\n",
            "copying yars_revenge.bin from ROMS/Yars' Revenge (Time Freeze) (1982) (Atari, Howard Scott Warshaw - Sears) (CX2655 - 49-75167) ~.bin to /usr/local/lib/python3.7/dist-packages/atari_py/atari_roms/yars_revenge.bin\n",
            "copying zaxxon.bin from ROMS/Zaxxon (1983) (Coleco) (2454) ~.bin to /usr/local/lib/python3.7/dist-packages/atari_py/atari_roms/zaxxon.bin\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "VGhD3L7_I2DC"
      },
      "source": [
        "## Train a DQN based agent in playing an atari game"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "0UfIdVZmvKNw",
        "outputId": "15539037-1dfc-4f03-e230-14858fe86157"
      },
      "source": [
        "import numpy as np\n",
        "import gym\n",
        "from agents import DuelingDDQNAgent\n",
        "from utils import plot_learning_curve, make_env\n",
        "\n",
        "record = False\n",
        "load_checkpoint = False\n",
        "env_name = 'SpaceInvadersNoFrameskip-v4'\n",
        "algo = 'DuelingDDQNAgent'\n",
        "\n",
        "env = make_env(env_name)\n",
        "if record:\n",
        "    env = gym.wrappers.Monitor(env, \"recording\", force=True)\n",
        "best_score = -np.inf\n",
        "n_games = 1000\n",
        "agent = DuelingDDQNAgent(gamma=0.99, epsilon=1.0, lr=1e-4,\n",
        "                          input_dims=(env.observation_space.shape),\n",
        "                          n_actions=env.action_space.n,\n",
        "                          mem_size=20000,\n",
        "                          eps_min=0.1,\n",
        "                          batch_size=32,\n",
        "                          replace=5000,\n",
        "                          eps_dec=1e-5,\n",
        "                          chkpt_dir='gdrive/MyDrive/Checkpoints/',\n",
        "                          algo=algo,\n",
        "                          env_name=env_name)\n",
        "if load_checkpoint:\n",
        "    agent.load_models()\n",
        "\n",
        "fname = agent.algo + '_' + agent.env_name + '_lr_' + str(agent.lr) + '_' + str(n_games) + '_games'\n",
        "figure_file = 'gdrive/MyDrive/Checkpoints/' + fname + '.png'\n",
        "\n",
        "n_steps = 0\n",
        "scores, eps_history, steps_array = [], [], []\n",
        "\n",
        "for i in range(n_games):\n",
        "    done = False\n",
        "    score = 0\n",
        "    observation = env.reset()\n",
        "\n",
        "    while not done:\n",
        "        action = agent.choose_action(observation)\n",
        "        observation_, reward, done, info = env.step(action)\n",
        "        score += reward\n",
        "\n",
        "        if not load_checkpoint:\n",
        "            agent.store_transition(observation, action, reward, observation_, int(done))\n",
        "            agent.learn()\n",
        "\n",
        "        observation = observation_\n",
        "        n_steps += 1\n",
        "    scores.append(score)\n",
        "    steps_array.append(n_steps)\n",
        "\n",
        "    avg_score = np.mean(scores[-100:])\n",
        "    print('episode ', i, 'score: ', score, 'average score %.1f best score %.1f epsilon %.2f' %\n",
        "          (avg_score, best_score, agent.epsilon), 'steps ', n_steps)\n",
        "\n",
        "    if avg_score > best_score:\n",
        "        if not load_checkpoint:\n",
        "            agent.save_models()\n",
        "        best_score = avg_score\n",
        "\n",
        "    eps_history.append(agent.epsilon)\n",
        "env.close()"
      ],
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "<string>:6: VisibleDeprecationWarning: Creating an ndarray from ragged nested sequences (which is a list-or-tuple of lists-or-tuples-or ndarrays with different lengths or shapes) is deprecated. If you meant to do this, you must specify 'dtype=object' when creating the ndarray\n",
            "/content/DQN_with_PyTorch_and_Gym/agents.py:229: UserWarning: indexing with dtype torch.uint8 is now deprecated, please use a dtype torch.bool instead. (Triggered internally at  /pytorch/aten/src/ATen/native/IndexingUtils.h:30.)\n",
            "  q_next[dones] = 0.0\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "episode  0 score:  120.0 average score 120.0 best score -inf epsilon 1.00 steps  506\n",
            "... saving checkpoint ...\n",
            "... saving checkpoint ...\n",
            "episode  1 score:  120.0 average score 120.0 best score 120.0 epsilon 0.99 steps  982\n",
            "episode  2 score:  185.0 average score 141.7 best score 120.0 epsilon 0.98 steps  1591\n",
            "... saving checkpoint ...\n",
            "... saving checkpoint ...\n",
            "episode  3 score:  110.0 average score 133.8 best score 141.7 epsilon 0.98 steps  2128\n",
            "episode  4 score:  55.0 average score 118.0 best score 141.7 epsilon 0.98 steps  2410\n",
            "episode  5 score:  105.0 average score 115.8 best score 141.7 epsilon 0.97 steps  2887\n",
            "episode  6 score:  75.0 average score 110.0 best score 141.7 epsilon 0.97 steps  3264\n",
            "episode  7 score:  135.0 average score 113.1 best score 141.7 epsilon 0.96 steps  3774\n",
            "episode  8 score:  110.0 average score 112.8 best score 141.7 epsilon 0.96 steps  4178\n",
            "episode  9 score:  110.0 average score 112.5 best score 141.7 epsilon 0.95 steps  4538\n",
            "episode  10 score:  180.0 average score 118.6 best score 141.7 epsilon 0.95 steps  5027\n",
            "episode  11 score:  305.0 average score 134.2 best score 141.7 epsilon 0.94 steps  5542\n",
            "episode  12 score:  210.0 average score 140.0 best score 141.7 epsilon 0.94 steps  6194\n",
            "episode  13 score:  215.0 average score 145.4 best score 141.7 epsilon 0.93 steps  6867\n",
            "... saving checkpoint ...\n",
            "... saving checkpoint ...\n",
            "episode  14 score:  105.0 average score 142.7 best score 145.4 epsilon 0.93 steps  7344\n",
            "episode  15 score:  105.0 average score 140.3 best score 145.4 epsilon 0.92 steps  7855\n",
            "episode  16 score:  160.0 average score 141.5 best score 145.4 epsilon 0.92 steps  8386\n",
            "episode  17 score:  120.0 average score 140.3 best score 145.4 epsilon 0.91 steps  8958\n",
            "episode  18 score:  120.0 average score 139.2 best score 145.4 epsilon 0.91 steps  9461\n",
            "episode  19 score:  105.0 average score 137.5 best score 145.4 epsilon 0.90 steps  9962\n",
            "episode  20 score:  60.0 average score 133.8 best score 145.4 epsilon 0.90 steps  10253\n",
            "episode  21 score:  120.0 average score 133.2 best score 145.4 epsilon 0.89 steps  10776\n",
            "episode  22 score:  170.0 average score 134.8 best score 145.4 epsilon 0.89 steps  11185\n",
            "episode  23 score:  185.0 average score 136.9 best score 145.4 epsilon 0.88 steps  11875\n",
            "episode  24 score:  65.0 average score 134.0 best score 145.4 epsilon 0.88 steps  12278\n",
            "episode  25 score:  150.0 average score 134.6 best score 145.4 epsilon 0.87 steps  13021\n",
            "episode  26 score:  65.0 average score 132.0 best score 145.4 epsilon 0.87 steps  13470\n",
            "episode  27 score:  405.0 average score 141.8 best score 145.4 epsilon 0.86 steps  14275\n",
            "episode  28 score:  470.0 average score 153.1 best score 145.4 epsilon 0.85 steps  15273\n",
            "... saving checkpoint ...\n",
            "... saving checkpoint ...\n",
            "episode  29 score:  155.0 average score 153.2 best score 153.1 epsilon 0.84 steps  16119\n",
            "... saving checkpoint ...\n",
            "... saving checkpoint ...\n",
            "episode  30 score:  120.0 average score 152.1 best score 153.2 epsilon 0.83 steps  16665\n",
            "episode  31 score:  150.0 average score 152.0 best score 153.2 epsilon 0.83 steps  17139\n",
            "episode  32 score:  210.0 average score 153.8 best score 153.2 epsilon 0.82 steps  17614\n",
            "... saving checkpoint ...\n",
            "... saving checkpoint ...\n",
            "episode  33 score:  120.0 average score 152.8 best score 153.8 epsilon 0.82 steps  18083\n",
            "episode  34 score:  65.0 average score 150.3 best score 153.8 epsilon 0.82 steps  18435\n",
            "episode  35 score:  80.0 average score 148.3 best score 153.8 epsilon 0.81 steps  18903\n",
            "episode  36 score:  30.0 average score 145.1 best score 153.8 epsilon 0.81 steps  19176\n",
            "episode  37 score:  35.0 average score 142.2 best score 153.8 epsilon 0.80 steps  19589\n",
            "episode  38 score:  135.0 average score 142.1 best score 153.8 epsilon 0.80 steps  20179\n",
            "episode  39 score:  100.0 average score 141.0 best score 153.8 epsilon 0.79 steps  20564\n",
            "episode  40 score:  210.0 average score 142.7 best score 153.8 epsilon 0.79 steps  21166\n",
            "episode  41 score:  245.0 average score 145.1 best score 153.8 epsilon 0.78 steps  21939\n",
            "episode  42 score:  40.0 average score 142.7 best score 153.8 epsilon 0.78 steps  22368\n",
            "episode  43 score:  110.0 average score 141.9 best score 153.8 epsilon 0.77 steps  22854\n",
            "episode  44 score:  90.0 average score 140.8 best score 153.8 epsilon 0.77 steps  23241\n",
            "episode  45 score:  50.0 average score 138.8 best score 153.8 epsilon 0.77 steps  23532\n",
            "episode  46 score:  155.0 average score 139.1 best score 153.8 epsilon 0.76 steps  24041\n",
            "episode  47 score:  110.0 average score 138.5 best score 153.8 epsilon 0.76 steps  24461\n",
            "episode  48 score:  170.0 average score 139.2 best score 153.8 epsilon 0.75 steps  25099\n",
            "episode  49 score:  355.0 average score 143.5 best score 153.8 epsilon 0.74 steps  25674\n",
            "episode  50 score:  245.0 average score 145.5 best score 153.8 epsilon 0.73 steps  26624\n",
            "episode  51 score:  210.0 average score 146.7 best score 153.8 epsilon 0.73 steps  27075\n",
            "episode  52 score:  180.0 average score 147.4 best score 153.8 epsilon 0.72 steps  27546\n",
            "episode  53 score:  115.0 average score 146.8 best score 153.8 epsilon 0.72 steps  28062\n",
            "episode  54 score:  410.0 average score 151.5 best score 153.8 epsilon 0.71 steps  28670\n",
            "episode  55 score:  60.0 average score 149.9 best score 153.8 epsilon 0.71 steps  28929\n",
            "episode  56 score:  20.0 average score 147.6 best score 153.8 epsilon 0.71 steps  29189\n",
            "episode  57 score:  75.0 average score 146.4 best score 153.8 epsilon 0.70 steps  29647\n",
            "episode  58 score:  45.0 average score 144.7 best score 153.8 epsilon 0.70 steps  29982\n",
            "episode  59 score:  35.0 average score 142.8 best score 153.8 epsilon 0.70 steps  30237\n",
            "episode  60 score:  120.0 average score 142.5 best score 153.8 epsilon 0.69 steps  30724\n",
            "episode  61 score:  105.0 average score 141.9 best score 153.8 epsilon 0.69 steps  31078\n",
            "episode  62 score:  105.0 average score 141.3 best score 153.8 epsilon 0.69 steps  31470\n",
            "episode  63 score:  160.0 average score 141.6 best score 153.8 epsilon 0.68 steps  31978\n",
            "episode  64 score:  405.0 average score 145.6 best score 153.8 epsilon 0.67 steps  32985\n",
            "episode  65 score:  260.0 average score 147.3 best score 153.8 epsilon 0.66 steps  33640\n",
            "episode  66 score:  190.0 average score 148.0 best score 153.8 epsilon 0.66 steps  34343\n",
            "episode  67 score:  65.0 average score 146.8 best score 153.8 epsilon 0.65 steps  34675\n",
            "episode  68 score:  35.0 average score 145.1 best score 153.8 epsilon 0.65 steps  34978\n",
            "episode  69 score:  135.0 average score 145.0 best score 153.8 epsilon 0.64 steps  35542\n",
            "episode  70 score:  120.0 average score 144.6 best score 153.8 epsilon 0.64 steps  36034\n",
            "episode  71 score:  40.0 average score 143.2 best score 153.8 epsilon 0.64 steps  36304\n",
            "episode  72 score:  190.0 average score 143.8 best score 153.8 epsilon 0.63 steps  36917\n",
            "episode  73 score:  180.0 average score 144.3 best score 153.8 epsilon 0.63 steps  37524\n",
            "episode  74 score:  35.0 average score 142.9 best score 153.8 epsilon 0.62 steps  37784\n",
            "episode  75 score:  35.0 average score 141.4 best score 153.8 epsilon 0.62 steps  38039\n",
            "episode  76 score:  50.0 average score 140.3 best score 153.8 epsilon 0.62 steps  38290\n",
            "episode  77 score:  75.0 average score 139.4 best score 153.8 epsilon 0.61 steps  38744\n",
            "episode  78 score:  490.0 average score 143.9 best score 153.8 epsilon 0.60 steps  39563\n",
            "episode  79 score:  90.0 average score 143.2 best score 153.8 epsilon 0.60 steps  39925\n",
            "episode  80 score:  180.0 average score 143.6 best score 153.8 epsilon 0.59 steps  40596\n",
            "episode  81 score:  35.0 average score 142.3 best score 153.8 epsilon 0.59 steps  40867\n",
            "episode  82 score:  120.0 average score 142.0 best score 153.8 epsilon 0.59 steps  41241\n",
            "episode  83 score:  155.0 average score 142.2 best score 153.8 epsilon 0.58 steps  41815\n",
            "episode  84 score:  90.0 average score 141.6 best score 153.8 epsilon 0.58 steps  42202\n",
            "episode  85 score:  160.0 average score 141.8 best score 153.8 epsilon 0.57 steps  42611\n",
            "episode  86 score:  30.0 average score 140.5 best score 153.8 epsilon 0.57 steps  42957\n",
            "episode  87 score:  80.0 average score 139.8 best score 153.8 epsilon 0.57 steps  43410\n",
            "episode  88 score:  95.0 average score 139.3 best score 153.8 epsilon 0.56 steps  43802\n",
            "episode  89 score:  155.0 average score 139.5 best score 153.8 epsilon 0.56 steps  44270\n",
            "episode  90 score:  65.0 average score 138.7 best score 153.8 epsilon 0.55 steps  44648\n",
            "episode  91 score:  100.0 average score 138.3 best score 153.8 epsilon 0.55 steps  44993\n",
            "episode  92 score:  90.0 average score 137.7 best score 153.8 epsilon 0.55 steps  45396\n",
            "episode  93 score:  185.0 average score 138.2 best score 153.8 epsilon 0.54 steps  45887\n",
            "episode  94 score:  70.0 average score 137.5 best score 153.8 epsilon 0.54 steps  46234\n",
            "episode  95 score:  180.0 average score 138.0 best score 153.8 epsilon 0.53 steps  46876\n",
            "episode  96 score:  60.0 average score 137.2 best score 153.8 epsilon 0.53 steps  47176\n",
            "episode  97 score:  145.0 average score 137.2 best score 153.8 epsilon 0.52 steps  47678\n",
            "episode  98 score:  155.0 average score 137.4 best score 153.8 epsilon 0.52 steps  48190\n",
            "episode  99 score:  120.0 average score 137.2 best score 153.8 epsilon 0.51 steps  48582\n",
            "episode  100 score:  270.0 average score 138.8 best score 153.8 epsilon 0.51 steps  49453\n",
            "episode  101 score:  65.0 average score 138.2 best score 153.8 epsilon 0.50 steps  49834\n",
            "episode  102 score:  90.0 average score 137.2 best score 153.8 epsilon 0.50 steps  50307\n",
            "episode  103 score:  270.0 average score 138.8 best score 153.8 epsilon 0.49 steps  51137\n",
            "episode  104 score:  30.0 average score 138.6 best score 153.8 epsilon 0.49 steps  51413\n",
            "episode  105 score:  140.0 average score 138.9 best score 153.8 epsilon 0.48 steps  51926\n",
            "episode  106 score:  125.0 average score 139.4 best score 153.8 epsilon 0.48 steps  52409\n",
            "episode  107 score:  50.0 average score 138.6 best score 153.8 epsilon 0.47 steps  52695\n",
            "episode  108 score:  540.0 average score 142.9 best score 153.8 epsilon 0.46 steps  53575\n",
            "episode  109 score:  160.0 average score 143.4 best score 153.8 epsilon 0.46 steps  54168\n",
            "episode  110 score:  130.0 average score 142.9 best score 153.8 epsilon 0.45 steps  54611\n",
            "episode  111 score:  80.0 average score 140.7 best score 153.8 epsilon 0.45 steps  54903\n",
            "episode  112 score:  145.0 average score 140.0 best score 153.8 epsilon 0.45 steps  55279\n",
            "episode  113 score:  120.0 average score 139.1 best score 153.8 epsilon 0.44 steps  55761\n",
            "episode  114 score:  95.0 average score 138.9 best score 153.8 epsilon 0.44 steps  56278\n",
            "episode  115 score:  80.0 average score 138.7 best score 153.8 epsilon 0.43 steps  56697\n",
            "episode  116 score:  85.0 average score 137.9 best score 153.8 epsilon 0.43 steps  57169\n",
            "episode  117 score:  50.0 average score 137.2 best score 153.8 epsilon 0.43 steps  57453\n",
            "episode  118 score:  220.0 average score 138.2 best score 153.8 epsilon 0.42 steps  58319\n",
            "episode  119 score:  300.0 average score 140.2 best score 153.8 epsilon 0.41 steps  59100\n",
            "episode  120 score:  220.0 average score 141.8 best score 153.8 epsilon 0.40 steps  59727\n",
            "episode  121 score:  45.0 average score 141.1 best score 153.8 epsilon 0.40 steps  60081\n",
            "episode  122 score:  440.0 average score 143.8 best score 153.8 epsilon 0.39 steps  61006\n",
            "episode  123 score:  80.0 average score 142.7 best score 153.8 epsilon 0.39 steps  61394\n",
            "episode  124 score:  460.0 average score 146.7 best score 153.8 epsilon 0.38 steps  62152\n",
            "episode  125 score:  135.0 average score 146.5 best score 153.8 epsilon 0.37 steps  62655\n",
            "episode  126 score:  65.0 average score 146.5 best score 153.8 epsilon 0.37 steps  63193\n",
            "episode  127 score:  65.0 average score 143.1 best score 153.8 epsilon 0.36 steps  63668\n",
            "episode  128 score:  430.0 average score 142.7 best score 153.8 epsilon 0.36 steps  64291\n",
            "episode  129 score:  125.0 average score 142.4 best score 153.8 epsilon 0.35 steps  64610\n",
            "episode  130 score:  240.0 average score 143.6 best score 153.8 epsilon 0.35 steps  65418\n",
            "episode  131 score:  275.0 average score 144.8 best score 153.8 epsilon 0.34 steps  66007\n",
            "episode  132 score:  230.0 average score 145.1 best score 153.8 epsilon 0.33 steps  66688\n",
            "episode  133 score:  120.0 average score 145.1 best score 153.8 epsilon 0.33 steps  67194\n",
            "episode  134 score:  295.0 average score 147.3 best score 153.8 epsilon 0.32 steps  68087\n",
            "episode  135 score:  50.0 average score 147.1 best score 153.8 epsilon 0.32 steps  68353\n",
            "episode  136 score:  325.0 average score 150.0 best score 153.8 epsilon 0.31 steps  68964\n",
            "episode  137 score:  110.0 average score 150.8 best score 153.8 epsilon 0.31 steps  69434\n",
            "episode  138 score:  75.0 average score 150.2 best score 153.8 epsilon 0.30 steps  69697\n",
            "episode  139 score:  320.0 average score 152.3 best score 153.8 epsilon 0.30 steps  70230\n",
            "episode  140 score:  50.0 average score 150.8 best score 153.8 epsilon 0.30 steps  70488\n",
            "episode  141 score:  205.0 average score 150.3 best score 153.8 epsilon 0.29 steps  71203\n",
            "episode  142 score:  130.0 average score 151.2 best score 153.8 epsilon 0.28 steps  71703\n",
            "episode  143 score:  110.0 average score 151.2 best score 153.8 epsilon 0.28 steps  71996\n",
            "episode  144 score:  380.0 average score 154.2 best score 153.8 epsilon 0.27 steps  72957\n",
            "... saving checkpoint ...\n",
            "... saving checkpoint ...\n",
            "episode  145 score:  205.0 average score 155.7 best score 154.2 epsilon 0.26 steps  73676\n",
            "... saving checkpoint ...\n",
            "... saving checkpoint ...\n",
            "episode  146 score:  80.0 average score 154.9 best score 155.7 epsilon 0.26 steps  74146\n",
            "episode  147 score:  265.0 average score 156.5 best score 155.7 epsilon 0.25 steps  74920\n",
            "... saving checkpoint ...\n",
            "... saving checkpoint ...\n",
            "episode  148 score:  105.0 average score 155.8 best score 156.5 epsilon 0.25 steps  75399\n",
            "episode  149 score:  410.0 average score 156.4 best score 156.5 epsilon 0.24 steps  76305\n",
            "episode  150 score:  185.0 average score 155.8 best score 156.5 epsilon 0.23 steps  76850\n",
            "episode  151 score:  140.0 average score 155.1 best score 156.5 epsilon 0.23 steps  77347\n",
            "episode  152 score:  130.0 average score 154.6 best score 156.5 epsilon 0.22 steps  77866\n",
            "episode  153 score:  80.0 average score 154.2 best score 156.5 epsilon 0.22 steps  78140\n",
            "episode  154 score:  315.0 average score 153.3 best score 156.5 epsilon 0.21 steps  78872\n",
            "episode  155 score:  105.0 average score 153.8 best score 156.5 epsilon 0.21 steps  79354\n",
            "episode  156 score:  110.0 average score 154.7 best score 156.5 epsilon 0.20 steps  79695\n",
            "episode  157 score:  300.0 average score 156.9 best score 156.5 epsilon 0.20 steps  80430\n",
            "... saving checkpoint ...\n",
            "... saving checkpoint ...\n",
            "episode  158 score:  200.0 average score 158.4 best score 156.9 epsilon 0.19 steps  81170\n",
            "... saving checkpoint ...\n",
            "... saving checkpoint ...\n",
            "episode  159 score:  130.0 average score 159.4 best score 158.4 epsilon 0.18 steps  81887\n",
            "... saving checkpoint ...\n",
            "... saving checkpoint ...\n",
            "episode  160 score:  145.0 average score 159.7 best score 159.4 epsilon 0.17 steps  82706\n",
            "... saving checkpoint ...\n",
            "... saving checkpoint ...\n",
            "episode  161 score:  110.0 average score 159.7 best score 159.7 epsilon 0.17 steps  83300\n",
            "... saving checkpoint ...\n",
            "... saving checkpoint ...\n",
            "episode  162 score:  220.0 average score 160.8 best score 159.7 epsilon 0.16 steps  83954\n",
            "... saving checkpoint ...\n",
            "... saving checkpoint ...\n",
            "episode  163 score:  75.0 average score 160.0 best score 160.8 epsilon 0.16 steps  84228\n",
            "episode  164 score:  335.0 average score 159.3 best score 160.8 epsilon 0.15 steps  85095\n",
            "episode  165 score:  65.0 average score 157.3 best score 160.8 epsilon 0.15 steps  85517\n",
            "episode  166 score:  115.0 average score 156.6 best score 160.8 epsilon 0.14 steps  86071\n",
            "episode  167 score:  110.0 average score 157.1 best score 160.8 epsilon 0.13 steps  86612\n",
            "episode  168 score:  45.0 average score 157.2 best score 160.8 epsilon 0.13 steps  87074\n",
            "episode  169 score:  240.0 average score 158.2 best score 160.8 epsilon 0.12 steps  87695\n",
            "episode  170 score:  105.0 average score 158.1 best score 160.8 epsilon 0.12 steps  87960\n",
            "episode  171 score:  50.0 average score 158.2 best score 160.8 epsilon 0.12 steps  88221\n",
            "episode  172 score:  95.0 average score 157.2 best score 160.8 epsilon 0.12 steps  88524\n",
            "episode  173 score:  50.0 average score 155.9 best score 160.8 epsilon 0.11 steps  88818\n",
            "episode  174 score:  80.0 average score 156.3 best score 160.8 epsilon 0.11 steps  89108\n",
            "episode  175 score:  105.0 average score 157.1 best score 160.8 epsilon 0.11 steps  89416\n",
            "episode  176 score:  360.0 average score 160.2 best score 160.8 epsilon 0.10 steps  90023\n",
            "episode  177 score:  110.0 average score 160.5 best score 160.8 epsilon 0.10 steps  90506\n",
            "episode  178 score:  440.0 average score 160.0 best score 160.8 epsilon 0.10 steps  91224\n",
            "episode  179 score:  110.0 average score 160.2 best score 160.8 epsilon 0.10 steps  91696\n",
            "episode  180 score:  105.0 average score 159.4 best score 160.8 epsilon 0.10 steps  92076\n",
            "episode  181 score:  115.0 average score 160.2 best score 160.8 epsilon 0.10 steps  92480\n",
            "episode  182 score:  125.0 average score 160.3 best score 160.8 epsilon 0.10 steps  93070\n",
            "episode  183 score:  75.0 average score 159.5 best score 160.8 epsilon 0.10 steps  93349\n",
            "episode  184 score:  125.0 average score 159.8 best score 160.8 epsilon 0.10 steps  93831\n",
            "episode  185 score:  80.0 average score 159.1 best score 160.8 epsilon 0.10 steps  94219\n",
            "episode  186 score:  110.0 average score 159.8 best score 160.8 epsilon 0.10 steps  94712\n",
            "episode  187 score:  75.0 average score 159.8 best score 160.8 epsilon 0.10 steps  94978\n",
            "episode  188 score:  110.0 average score 159.9 best score 160.8 epsilon 0.10 steps  95524\n",
            "episode  189 score:  145.0 average score 159.8 best score 160.8 epsilon 0.10 steps  95930\n",
            "episode  190 score:  165.0 average score 160.8 best score 160.8 epsilon 0.10 steps  96770\n",
            "episode  191 score:  105.0 average score 160.9 best score 160.8 epsilon 0.10 steps  97146\n",
            "... saving checkpoint ...\n",
            "... saving checkpoint ...\n",
            "episode  192 score:  145.0 average score 161.4 best score 160.9 epsilon 0.10 steps  97640\n",
            "... saving checkpoint ...\n",
            "... saving checkpoint ...\n",
            "episode  193 score:  120.0 average score 160.8 best score 161.4 epsilon 0.10 steps  97912\n",
            "episode  194 score:  35.0 average score 160.4 best score 161.4 epsilon 0.10 steps  98204\n",
            "episode  195 score:  85.0 average score 159.5 best score 161.4 epsilon 0.10 steps  98675\n",
            "episode  196 score:  125.0 average score 160.2 best score 161.4 epsilon 0.10 steps  99170\n",
            "episode  197 score:  185.0 average score 160.6 best score 161.4 epsilon 0.10 steps  99684\n",
            "episode  198 score:  60.0 average score 159.6 best score 161.4 epsilon 0.10 steps  99927\n",
            "episode  199 score:  180.0 average score 160.2 best score 161.4 epsilon 0.10 steps  100583\n",
            "episode  200 score:  120.0 average score 158.7 best score 161.4 epsilon 0.10 steps  101080\n",
            "episode  201 score:  25.0 average score 158.3 best score 161.4 epsilon 0.10 steps  101362\n",
            "episode  202 score:  175.0 average score 159.2 best score 161.4 epsilon 0.10 steps  102184\n",
            "episode  203 score:  410.0 average score 160.6 best score 161.4 epsilon 0.10 steps  102805\n",
            "episode  204 score:  225.0 average score 162.5 best score 161.4 epsilon 0.10 steps  103531\n",
            "... saving checkpoint ...\n",
            "... saving checkpoint ...\n",
            "episode  205 score:  140.0 average score 162.5 best score 162.5 epsilon 0.10 steps  103918\n",
            "episode  206 score:  340.0 average score 164.7 best score 162.5 epsilon 0.10 steps  104540\n",
            "... saving checkpoint ...\n",
            "... saving checkpoint ...\n",
            "episode  207 score:  70.0 average score 164.8 best score 164.7 epsilon 0.10 steps  105054\n",
            "... saving checkpoint ...\n",
            "... saving checkpoint ...\n",
            "episode  208 score:  95.0 average score 160.4 best score 164.8 epsilon 0.10 steps  105579\n",
            "episode  209 score:  140.0 average score 160.2 best score 164.8 epsilon 0.10 steps  106116\n",
            "episode  210 score:  165.0 average score 160.6 best score 164.8 epsilon 0.10 steps  106679\n",
            "episode  211 score:  75.0 average score 160.5 best score 164.8 epsilon 0.10 steps  106944\n",
            "episode  212 score:  105.0 average score 160.1 best score 164.8 epsilon 0.10 steps  107232\n",
            "episode  213 score:  55.0 average score 159.4 best score 164.8 epsilon 0.10 steps  107515\n",
            "episode  214 score:  50.0 average score 159.0 best score 164.8 epsilon 0.10 steps  107779\n",
            "episode  215 score:  170.0 average score 159.9 best score 164.8 epsilon 0.10 steps  108321\n",
            "episode  216 score:  80.0 average score 159.8 best score 164.8 epsilon 0.10 steps  108783\n",
            "episode  217 score:  200.0 average score 161.3 best score 164.8 epsilon 0.10 steps  109526\n",
            "episode  218 score:  225.0 average score 161.4 best score 164.8 epsilon 0.10 steps  110212\n",
            "episode  219 score:  175.0 average score 160.2 best score 164.8 epsilon 0.10 steps  110761\n",
            "episode  220 score:  115.0 average score 159.1 best score 164.8 epsilon 0.10 steps  111665\n",
            "episode  221 score:  85.0 average score 159.5 best score 164.8 epsilon 0.10 steps  112341\n",
            "episode  222 score:  320.0 average score 158.3 best score 164.8 epsilon 0.10 steps  113334\n",
            "episode  223 score:  80.0 average score 158.3 best score 164.8 epsilon 0.10 steps  113807\n",
            "episode  224 score:  35.0 average score 154.1 best score 164.8 epsilon 0.10 steps  114086\n",
            "episode  225 score:  25.0 average score 152.9 best score 164.8 epsilon 0.10 steps  114368\n",
            "episode  226 score:  110.0 average score 153.4 best score 164.8 epsilon 0.10 steps  114859\n",
            "episode  227 score:  55.0 average score 153.3 best score 164.8 epsilon 0.10 steps  115102\n",
            "episode  228 score:  160.0 average score 150.6 best score 164.8 epsilon 0.10 steps  115625\n",
            "episode  229 score:  385.0 average score 153.2 best score 164.8 epsilon 0.10 steps  116533\n",
            "episode  230 score:  370.0 average score 154.5 best score 164.8 epsilon 0.10 steps  117126\n",
            "episode  231 score:  120.0 average score 152.9 best score 164.8 epsilon 0.10 steps  117601\n",
            "episode  232 score:  50.0 average score 151.2 best score 164.8 epsilon 0.10 steps  117864\n",
            "episode  233 score:  75.0 average score 150.7 best score 164.8 epsilon 0.10 steps  118385\n",
            "episode  234 score:  50.0 average score 148.2 best score 164.8 epsilon 0.10 steps  118771\n",
            "episode  235 score:  360.0 average score 151.3 best score 164.8 epsilon 0.10 steps  119489\n",
            "episode  236 score:  60.0 average score 148.7 best score 164.8 epsilon 0.10 steps  119860\n",
            "episode  237 score:  115.0 average score 148.8 best score 164.8 epsilon 0.10 steps  120331\n",
            "episode  238 score:  45.0 average score 148.4 best score 164.8 epsilon 0.10 steps  120698\n",
            "episode  239 score:  65.0 average score 145.9 best score 164.8 epsilon 0.10 steps  121082\n",
            "episode  240 score:  605.0 average score 151.4 best score 164.8 epsilon 0.10 steps  122072\n",
            "episode  241 score:  115.0 average score 150.6 best score 164.8 epsilon 0.10 steps  122347\n",
            "episode  242 score:  105.0 average score 150.3 best score 164.8 epsilon 0.10 steps  122621\n",
            "episode  243 score:  110.0 average score 150.3 best score 164.8 epsilon 0.10 steps  123062\n",
            "episode  244 score:  110.0 average score 147.6 best score 164.8 epsilon 0.10 steps  123330\n",
            "episode  245 score:  30.0 average score 145.8 best score 164.8 epsilon 0.10 steps  123626\n",
            "episode  246 score:  215.0 average score 147.2 best score 164.8 epsilon 0.10 steps  124238\n",
            "episode  247 score:  35.0 average score 144.9 best score 164.8 epsilon 0.10 steps  124628\n",
            "episode  248 score:  110.0 average score 144.9 best score 164.8 epsilon 0.10 steps  125230\n",
            "episode  249 score:  235.0 average score 143.2 best score 164.8 epsilon 0.10 steps  126028\n",
            "episode  250 score:  140.0 average score 142.8 best score 164.8 epsilon 0.10 steps  126464\n",
            "episode  251 score:  265.0 average score 144.0 best score 164.8 epsilon 0.10 steps  127404\n",
            "episode  252 score:  85.0 average score 143.6 best score 164.8 epsilon 0.10 steps  127954\n",
            "episode  253 score:  120.0 average score 143.9 best score 164.8 epsilon 0.10 steps  128515\n",
            "episode  254 score:  185.0 average score 142.7 best score 164.8 epsilon 0.10 steps  129033\n",
            "episode  255 score:  65.0 average score 142.2 best score 164.8 epsilon 0.10 steps  129422\n",
            "episode  256 score:  330.0 average score 144.4 best score 164.8 epsilon 0.10 steps  130204\n",
            "episode  257 score:  65.0 average score 142.1 best score 164.8 epsilon 0.10 steps  130492\n",
            "episode  258 score:  465.0 average score 144.8 best score 164.8 epsilon 0.10 steps  131642\n",
            "episode  259 score:  90.0 average score 144.3 best score 164.8 epsilon 0.10 steps  132112\n",
            "episode  260 score:  125.0 average score 144.2 best score 164.8 epsilon 0.10 steps  132517\n",
            "episode  261 score:  75.0 average score 143.8 best score 164.8 epsilon 0.10 steps  132825\n",
            "episode  262 score:  135.0 average score 142.9 best score 164.8 epsilon 0.10 steps  133221\n",
            "episode  263 score:  130.0 average score 143.5 best score 164.8 epsilon 0.10 steps  133690\n",
            "episode  264 score:  240.0 average score 142.6 best score 164.8 epsilon 0.10 steps  134394\n",
            "episode  265 score:  210.0 average score 144.0 best score 164.8 epsilon 0.10 steps  134991\n",
            "episode  266 score:  125.0 average score 144.1 best score 164.8 epsilon 0.10 steps  135533\n",
            "episode  267 score:  55.0 average score 143.6 best score 164.8 epsilon 0.10 steps  135812\n",
            "episode  268 score:  180.0 average score 144.9 best score 164.8 epsilon 0.10 steps  136469\n",
            "episode  269 score:  180.0 average score 144.3 best score 164.8 epsilon 0.10 steps  136984\n",
            "episode  270 score:  30.0 average score 143.6 best score 164.8 epsilon 0.10 steps  137280\n",
            "episode  271 score:  415.0 average score 147.2 best score 164.8 epsilon 0.10 steps  137996\n",
            "episode  272 score:  120.0 average score 147.4 best score 164.8 epsilon 0.10 steps  138477\n",
            "episode  273 score:  30.0 average score 147.2 best score 164.8 epsilon 0.10 steps  138765\n",
            "episode  274 score:  270.0 average score 149.2 best score 164.8 epsilon 0.10 steps  139475\n",
            "episode  275 score:  75.0 average score 148.8 best score 164.8 epsilon 0.10 steps  139777\n",
            "episode  276 score:  180.0 average score 147.1 best score 164.8 epsilon 0.10 steps  140328\n",
            "episode  277 score:  105.0 average score 147.0 best score 164.8 epsilon 0.10 steps  140708\n",
            "episode  278 score:  105.0 average score 143.7 best score 164.8 epsilon 0.10 steps  140978\n",
            "episode  279 score:  75.0 average score 143.3 best score 164.8 epsilon 0.10 steps  141247\n",
            "episode  280 score:  305.0 average score 145.3 best score 164.8 epsilon 0.10 steps  141910\n",
            "episode  281 score:  155.0 average score 145.7 best score 164.8 epsilon 0.10 steps  142453\n",
            "episode  282 score:  210.0 average score 146.6 best score 164.8 epsilon 0.10 steps  143204\n",
            "episode  283 score:  155.0 average score 147.3 best score 164.8 epsilon 0.10 steps  143834\n",
            "episode  284 score:  280.0 average score 148.9 best score 164.8 epsilon 0.10 steps  144693\n",
            "episode  285 score:  80.0 average score 148.9 best score 164.8 epsilon 0.10 steps  144971\n",
            "episode  286 score:  285.0 average score 150.7 best score 164.8 epsilon 0.10 steps  145732\n",
            "episode  287 score:  310.0 average score 153.0 best score 164.8 epsilon 0.10 steps  146689\n",
            "episode  288 score:  140.0 average score 153.3 best score 164.8 epsilon 0.10 steps  147507\n",
            "episode  289 score:  385.0 average score 155.7 best score 164.8 epsilon 0.10 steps  148242\n",
            "episode  290 score:  200.0 average score 156.1 best score 164.8 epsilon 0.10 steps  149082\n",
            "episode  291 score:  175.0 average score 156.8 best score 164.8 epsilon 0.10 steps  149821\n",
            "episode  292 score:  65.0 average score 155.9 best score 164.8 epsilon 0.10 steps  150211\n",
            "episode  293 score:  130.0 average score 156.1 best score 164.8 epsilon 0.10 steps  150648\n",
            "episode  294 score:  75.0 average score 156.4 best score 164.8 epsilon 0.10 steps  150955\n",
            "episode  295 score:  170.0 average score 157.3 best score 164.8 epsilon 0.10 steps  151512\n",
            "episode  296 score:  75.0 average score 156.8 best score 164.8 epsilon 0.10 steps  151778\n",
            "episode  297 score:  75.0 average score 155.7 best score 164.8 epsilon 0.10 steps  152174\n",
            "episode  298 score:  170.0 average score 156.8 best score 164.8 epsilon 0.10 steps  152683\n",
            "episode  299 score:  90.0 average score 155.9 best score 164.8 epsilon 0.10 steps  153246\n",
            "episode  300 score:  250.0 average score 157.2 best score 164.8 epsilon 0.10 steps  153890\n",
            "episode  301 score:  80.0 average score 157.8 best score 164.8 epsilon 0.10 steps  154153\n",
            "episode  302 score:  125.0 average score 157.2 best score 164.8 epsilon 0.10 steps  154612\n",
            "episode  303 score:  35.0 average score 153.5 best score 164.8 epsilon 0.10 steps  154846\n",
            "episode  304 score:  70.0 average score 151.9 best score 164.8 epsilon 0.10 steps  155228\n",
            "episode  305 score:  35.0 average score 150.9 best score 164.8 epsilon 0.10 steps  155502\n",
            "episode  306 score:  210.0 average score 149.6 best score 164.8 epsilon 0.10 steps  156197\n",
            "episode  307 score:  80.0 average score 149.7 best score 164.8 epsilon 0.10 steps  156604\n",
            "episode  308 score:  155.0 average score 150.3 best score 164.8 epsilon 0.10 steps  157207\n",
            "episode  309 score:  115.0 average score 150.1 best score 164.8 epsilon 0.10 steps  157620\n",
            "episode  310 score:  95.0 average score 149.3 best score 164.8 epsilon 0.10 steps  158005\n",
            "episode  311 score:  105.0 average score 149.7 best score 164.8 epsilon 0.10 steps  158443\n",
            "episode  312 score:  460.0 average score 153.2 best score 164.8 epsilon 0.10 steps  159470\n",
            "episode  313 score:  55.0 average score 153.2 best score 164.8 epsilon 0.10 steps  159734\n",
            "episode  314 score:  240.0 average score 155.1 best score 164.8 epsilon 0.10 steps  160682\n",
            "episode  315 score:  120.0 average score 154.6 best score 164.8 epsilon 0.10 steps  161080\n",
            "episode  316 score:  145.0 average score 155.2 best score 164.8 epsilon 0.10 steps  161575\n",
            "episode  317 score:  120.0 average score 154.4 best score 164.8 epsilon 0.10 steps  161983\n",
            "episode  318 score:  125.0 average score 153.4 best score 164.8 epsilon 0.10 steps  162462\n",
            "episode  319 score:  210.0 average score 153.8 best score 164.8 epsilon 0.10 steps  162990\n",
            "episode  320 score:  130.0 average score 153.9 best score 164.8 epsilon 0.10 steps  163291\n",
            "episode  321 score:  105.0 average score 154.2 best score 164.8 epsilon 0.10 steps  163663\n",
            "episode  322 score:  355.0 average score 154.5 best score 164.8 epsilon 0.10 steps  164749\n",
            "episode  323 score:  210.0 average score 155.8 best score 164.8 epsilon 0.10 steps  165340\n",
            "episode  324 score:  460.0 average score 160.1 best score 164.8 epsilon 0.10 steps  165963\n",
            "episode  325 score:  420.0 average score 164.0 best score 164.8 epsilon 0.10 steps  166560\n",
            "episode  326 score:  270.0 average score 165.6 best score 164.8 epsilon 0.10 steps  167392\n",
            "... saving checkpoint ...\n",
            "... saving checkpoint ...\n",
            "episode  327 score:  300.0 average score 168.1 best score 165.6 epsilon 0.10 steps  168072\n",
            "... saving checkpoint ...\n",
            "... saving checkpoint ...\n",
            "episode  328 score:  235.0 average score 168.8 best score 168.1 epsilon 0.10 steps  168785\n",
            "... saving checkpoint ...\n",
            "... saving checkpoint ...\n",
            "episode  329 score:  50.0 average score 165.4 best score 168.8 epsilon 0.10 steps  169043\n",
            "episode  330 score:  110.0 average score 162.8 best score 168.8 epsilon 0.10 steps  169504\n",
            "episode  331 score:  145.0 average score 163.1 best score 168.8 epsilon 0.10 steps  170070\n",
            "episode  332 score:  285.0 average score 165.4 best score 168.8 epsilon 0.10 steps  170696\n",
            "episode  333 score:  145.0 average score 166.2 best score 168.8 epsilon 0.10 steps  171346\n",
            "episode  334 score:  65.0 average score 166.3 best score 168.8 epsilon 0.10 steps  171624\n",
            "episode  335 score:  190.0 average score 164.6 best score 168.8 epsilon 0.10 steps  172194\n",
            "episode  336 score:  95.0 average score 164.9 best score 168.8 epsilon 0.10 steps  172485\n",
            "episode  337 score:  135.0 average score 165.2 best score 168.8 epsilon 0.10 steps  172993\n",
            "episode  338 score:  395.0 average score 168.7 best score 168.8 epsilon 0.10 steps  173771\n",
            "episode  339 score:  120.0 average score 169.2 best score 168.8 epsilon 0.10 steps  174144\n",
            "... saving checkpoint ...\n",
            "... saving checkpoint ...\n",
            "episode  340 score:  215.0 average score 165.3 best score 169.2 epsilon 0.10 steps  174765\n",
            "episode  341 score:  195.0 average score 166.1 best score 169.2 epsilon 0.10 steps  175551\n",
            "episode  342 score:  145.0 average score 166.5 best score 169.2 epsilon 0.10 steps  176038\n",
            "episode  343 score:  65.0 average score 166.1 best score 169.2 epsilon 0.10 steps  176294\n",
            "episode  344 score:  75.0 average score 165.7 best score 169.2 epsilon 0.10 steps  176553\n",
            "episode  345 score:  110.0 average score 166.5 best score 169.2 epsilon 0.10 steps  176965\n",
            "episode  346 score:  65.0 average score 165.0 best score 169.2 epsilon 0.10 steps  177312\n",
            "episode  347 score:  160.0 average score 166.2 best score 169.2 epsilon 0.10 steps  177916\n",
            "episode  348 score:  135.0 average score 166.5 best score 169.2 epsilon 0.10 steps  178395\n",
            "episode  349 score:  85.0 average score 165.0 best score 169.2 epsilon 0.10 steps  178877\n",
            "episode  350 score:  190.0 average score 165.5 best score 169.2 epsilon 0.10 steps  179451\n",
            "episode  351 score:  180.0 average score 164.7 best score 169.2 epsilon 0.10 steps  179960\n",
            "episode  352 score:  290.0 average score 166.7 best score 169.2 epsilon 0.10 steps  180612\n",
            "episode  353 score:  90.0 average score 166.4 best score 169.2 epsilon 0.10 steps  180966\n",
            "episode  354 score:  235.0 average score 166.9 best score 169.2 epsilon 0.10 steps  181452\n",
            "episode  355 score:  85.0 average score 167.1 best score 169.2 epsilon 0.10 steps  181718\n",
            "episode  356 score:  50.0 average score 164.3 best score 169.2 epsilon 0.10 steps  181981\n",
            "episode  357 score:  160.0 average score 165.2 best score 169.2 epsilon 0.10 steps  182581\n",
            "episode  358 score:  155.0 average score 162.2 best score 169.2 epsilon 0.10 steps  183051\n",
            "episode  359 score:  180.0 average score 163.1 best score 169.2 epsilon 0.10 steps  183608\n",
            "episode  360 score:  130.0 average score 163.1 best score 169.2 epsilon 0.10 steps  183995\n",
            "episode  361 score:  200.0 average score 164.3 best score 169.2 epsilon 0.10 steps  184483\n",
            "episode  362 score:  435.0 average score 167.3 best score 169.2 epsilon 0.10 steps  185454\n",
            "episode  363 score:  465.0 average score 170.7 best score 169.2 epsilon 0.10 steps  186407\n",
            "... saving checkpoint ...\n",
            "... saving checkpoint ...\n",
            "episode  364 score:  60.0 average score 168.9 best score 170.7 epsilon 0.10 steps  186718\n",
            "episode  365 score:  155.0 average score 168.3 best score 170.7 epsilon 0.10 steps  187238\n",
            "episode  366 score:  75.0 average score 167.8 best score 170.7 epsilon 0.10 steps  187612\n",
            "episode  367 score:  105.0 average score 168.3 best score 170.7 epsilon 0.10 steps  188020\n",
            "episode  368 score:  205.0 average score 168.6 best score 170.7 epsilon 0.10 steps  188815\n",
            "episode  369 score:  270.0 average score 169.5 best score 170.7 epsilon 0.10 steps  189477\n",
            "episode  370 score:  375.0 average score 172.9 best score 170.7 epsilon 0.10 steps  190132\n",
            "... saving checkpoint ...\n",
            "... saving checkpoint ...\n",
            "episode  371 score:  50.0 average score 169.3 best score 172.9 epsilon 0.10 steps  190426\n",
            "episode  372 score:  110.0 average score 169.2 best score 172.9 epsilon 0.10 steps  190806\n",
            "episode  373 score:  100.0 average score 169.9 best score 172.9 epsilon 0.10 steps  191194\n",
            "episode  374 score:  145.0 average score 168.7 best score 172.9 epsilon 0.10 steps  191681\n",
            "episode  375 score:  610.0 average score 174.0 best score 172.9 epsilon 0.10 steps  192632\n",
            "... saving checkpoint ...\n",
            "... saving checkpoint ...\n",
            "episode  376 score:  190.0 average score 174.1 best score 174.0 epsilon 0.10 steps  193321\n",
            "... saving checkpoint ...\n",
            "... saving checkpoint ...\n",
            "episode  377 score:  105.0 average score 174.1 best score 174.1 epsilon 0.10 steps  193650\n",
            "episode  378 score:  175.0 average score 174.8 best score 174.1 epsilon 0.10 steps  194243\n",
            "... saving checkpoint ...\n",
            "... saving checkpoint ...\n",
            "episode  379 score:  310.0 average score 177.2 best score 174.8 epsilon 0.10 steps  195106\n",
            "... saving checkpoint ...\n",
            "... saving checkpoint ...\n",
            "episode  380 score:  115.0 average score 175.2 best score 177.2 epsilon 0.10 steps  195541\n",
            "episode  381 score:  165.0 average score 175.3 best score 177.2 epsilon 0.10 steps  196009\n",
            "episode  382 score:  150.0 average score 174.8 best score 177.2 epsilon 0.10 steps  196566\n",
            "episode  383 score:  325.0 average score 176.4 best score 177.2 epsilon 0.10 steps  197437\n",
            "episode  384 score:  360.0 average score 177.2 best score 177.2 epsilon 0.10 steps  198347\n",
            "... saving checkpoint ...\n",
            "... saving checkpoint ...\n",
            "episode  385 score:  245.0 average score 178.9 best score 177.2 epsilon 0.10 steps  198990\n",
            "... saving checkpoint ...\n",
            "... saving checkpoint ...\n",
            "episode  386 score:  105.0 average score 177.1 best score 178.9 epsilon 0.10 steps  199351\n",
            "episode  387 score:  165.0 average score 175.7 best score 178.9 epsilon 0.10 steps  199995\n",
            "episode  388 score:  280.0 average score 177.1 best score 178.9 epsilon 0.10 steps  200625\n",
            "episode  389 score:  110.0 average score 174.3 best score 178.9 epsilon 0.10 steps  201039\n",
            "episode  390 score:  305.0 average score 175.3 best score 178.9 epsilon 0.10 steps  201873\n",
            "episode  391 score:  260.0 average score 176.2 best score 178.9 epsilon 0.10 steps  202631\n",
            "episode  392 score:  110.0 average score 176.7 best score 178.9 epsilon 0.10 steps  202941\n",
            "episode  393 score:  260.0 average score 177.9 best score 178.9 epsilon 0.10 steps  203582\n",
            "episode  394 score:  155.0 average score 178.8 best score 178.9 epsilon 0.10 steps  204299\n",
            "episode  395 score:  75.0 average score 177.8 best score 178.9 epsilon 0.10 steps  204556\n",
            "episode  396 score:  280.0 average score 179.8 best score 178.9 epsilon 0.10 steps  205349\n",
            "... saving checkpoint ...\n",
            "... saving checkpoint ...\n",
            "episode  397 score:  185.0 average score 180.9 best score 179.8 epsilon 0.10 steps  205760\n",
            "... saving checkpoint ...\n",
            "... saving checkpoint ...\n",
            "episode  398 score:  115.0 average score 180.4 best score 180.9 epsilon 0.10 steps  206154\n",
            "episode  399 score:  165.0 average score 181.2 best score 180.9 epsilon 0.10 steps  206594\n",
            "... saving checkpoint ...\n",
            "... saving checkpoint ...\n",
            "episode  400 score:  75.0 average score 179.4 best score 181.2 epsilon 0.10 steps  206868\n",
            "episode  401 score:  295.0 average score 181.6 best score 181.2 epsilon 0.10 steps  207704\n",
            "... saving checkpoint ...\n",
            "... saving checkpoint ...\n",
            "episode  402 score:  125.0 average score 181.6 best score 181.6 epsilon 0.10 steps  208162\n",
            "episode  403 score:  210.0 average score 183.3 best score 181.6 epsilon 0.10 steps  208852\n",
            "... saving checkpoint ...\n",
            "... saving checkpoint ...\n",
            "episode  404 score:  140.0 average score 184.0 best score 183.3 epsilon 0.10 steps  209670\n",
            "... saving checkpoint ...\n",
            "... saving checkpoint ...\n",
            "episode  405 score:  110.0 average score 184.8 best score 184.0 epsilon 0.10 steps  210081\n",
            "... saving checkpoint ...\n",
            "... saving checkpoint ...\n",
            "episode  406 score:  270.0 average score 185.3 best score 184.8 epsilon 0.10 steps  210827\n",
            "... saving checkpoint ...\n",
            "... saving checkpoint ...\n",
            "episode  407 score:  165.0 average score 186.2 best score 185.3 epsilon 0.10 steps  211234\n",
            "... saving checkpoint ...\n",
            "... saving checkpoint ...\n",
            "episode  408 score:  135.0 average score 186.0 best score 186.2 epsilon 0.10 steps  211534\n",
            "episode  409 score:  150.0 average score 186.3 best score 186.2 epsilon 0.10 steps  211965\n",
            "... saving checkpoint ...\n",
            "... saving checkpoint ...\n",
            "episode  410 score:  150.0 average score 186.9 best score 186.3 epsilon 0.10 steps  212326\n",
            "... saving checkpoint ...\n",
            "... saving checkpoint ...\n",
            "episode  411 score:  150.0 average score 187.3 best score 186.9 epsilon 0.10 steps  212822\n",
            "... saving checkpoint ...\n",
            "... saving checkpoint ...\n",
            "episode  412 score:  105.0 average score 183.8 best score 187.3 epsilon 0.10 steps  213191\n",
            "episode  413 score:  130.0 average score 184.6 best score 187.3 epsilon 0.10 steps  213558\n",
            "episode  414 score:  380.0 average score 185.9 best score 187.3 epsilon 0.10 steps  214671\n",
            "episode  415 score:  105.0 average score 185.8 best score 187.3 epsilon 0.10 steps  215130\n",
            "episode  416 score:  160.0 average score 185.9 best score 187.3 epsilon 0.10 steps  215627\n",
            "episode  417 score:  135.0 average score 186.1 best score 187.3 epsilon 0.10 steps  216001\n",
            "episode  418 score:  170.0 average score 186.6 best score 187.3 epsilon 0.10 steps  216526\n",
            "episode  419 score:  170.0 average score 186.2 best score 187.3 epsilon 0.10 steps  217133\n",
            "episode  420 score:  285.0 average score 187.7 best score 187.3 epsilon 0.10 steps  217790\n",
            "... saving checkpoint ...\n",
            "... saving checkpoint ...\n",
            "episode  421 score:  240.0 average score 189.1 best score 187.7 epsilon 0.10 steps  218390\n",
            "... saving checkpoint ...\n",
            "... saving checkpoint ...\n",
            "episode  422 score:  155.0 average score 187.1 best score 189.1 epsilon 0.10 steps  218799\n",
            "episode  423 score:  115.0 average score 186.1 best score 189.1 epsilon 0.10 steps  219268\n",
            "episode  424 score:  105.0 average score 182.6 best score 189.1 epsilon 0.10 steps  219649\n",
            "episode  425 score:  180.0 average score 180.2 best score 189.1 epsilon 0.10 steps  220125\n",
            "episode  426 score:  105.0 average score 178.5 best score 189.1 epsilon 0.10 steps  220425\n",
            "episode  427 score:  175.0 average score 177.2 best score 189.1 epsilon 0.10 steps  220927\n",
            "episode  428 score:  320.0 average score 178.1 best score 189.1 epsilon 0.10 steps  221662\n",
            "episode  429 score:  160.0 average score 179.2 best score 189.1 epsilon 0.10 steps  222187\n",
            "episode  430 score:  260.0 average score 180.7 best score 189.1 epsilon 0.10 steps  222876\n",
            "episode  431 score:  190.0 average score 181.2 best score 189.1 epsilon 0.10 steps  223394\n",
            "episode  432 score:  120.0 average score 179.5 best score 189.1 epsilon 0.10 steps  223664\n",
            "episode  433 score:  205.0 average score 180.1 best score 189.1 epsilon 0.10 steps  224347\n",
            "episode  434 score:  205.0 average score 181.5 best score 189.1 epsilon 0.10 steps  224851\n",
            "episode  435 score:  105.0 average score 180.7 best score 189.1 epsilon 0.10 steps  225150\n",
            "episode  436 score:  185.0 average score 181.6 best score 189.1 epsilon 0.10 steps  226126\n",
            "episode  437 score:  155.0 average score 181.8 best score 189.1 epsilon 0.10 steps  226516\n",
            "episode  438 score:  135.0 average score 179.2 best score 189.1 epsilon 0.10 steps  227010\n",
            "episode  439 score:  140.0 average score 179.3 best score 189.1 epsilon 0.10 steps  227417\n",
            "episode  440 score:  75.0 average score 177.9 best score 189.1 epsilon 0.10 steps  227685\n",
            "episode  441 score:  135.0 average score 177.3 best score 189.1 epsilon 0.10 steps  228115\n",
            "episode  442 score:  75.0 average score 176.7 best score 189.1 epsilon 0.10 steps  228511\n",
            "episode  443 score:  150.0 average score 177.5 best score 189.1 epsilon 0.10 steps  228993\n",
            "episode  444 score:  95.0 average score 177.7 best score 189.1 epsilon 0.10 steps  229328\n",
            "episode  445 score:  190.0 average score 178.5 best score 189.1 epsilon 0.10 steps  229723\n",
            "episode  446 score:  260.0 average score 180.4 best score 189.1 epsilon 0.10 steps  230707\n",
            "episode  447 score:  75.0 average score 179.6 best score 189.1 epsilon 0.10 steps  231010\n",
            "episode  448 score:  205.0 average score 180.3 best score 189.1 epsilon 0.10 steps  231532\n",
            "episode  449 score:  160.0 average score 181.1 best score 189.1 epsilon 0.10 steps  232030\n",
            "episode  450 score:  110.0 average score 180.2 best score 189.1 epsilon 0.10 steps  232396\n",
            "episode  451 score:  105.0 average score 179.5 best score 189.1 epsilon 0.10 steps  232808\n",
            "episode  452 score:  180.0 average score 178.4 best score 189.1 epsilon 0.10 steps  233208\n",
            "episode  453 score:  115.0 average score 178.7 best score 189.1 epsilon 0.10 steps  233723\n",
            "episode  454 score:  105.0 average score 177.3 best score 189.1 epsilon 0.10 steps  234139\n",
            "episode  455 score:  345.0 average score 179.9 best score 189.1 epsilon 0.10 steps  234976\n",
            "episode  456 score:  315.0 average score 182.6 best score 189.1 epsilon 0.10 steps  235758\n",
            "episode  457 score:  55.0 average score 181.6 best score 189.1 epsilon 0.10 steps  236226\n",
            "episode  458 score:  130.0 average score 181.3 best score 189.1 epsilon 0.10 steps  236528\n",
            "episode  459 score:  105.0 average score 180.6 best score 189.1 epsilon 0.10 steps  236885\n",
            "episode  460 score:  85.0 average score 180.1 best score 189.1 epsilon 0.10 steps  237162\n",
            "episode  461 score:  175.0 average score 179.8 best score 189.1 epsilon 0.10 steps  237657\n",
            "episode  462 score:  55.0 average score 176.1 best score 189.1 epsilon 0.10 steps  238044\n",
            "episode  463 score:  215.0 average score 173.6 best score 189.1 epsilon 0.10 steps  238595\n",
            "episode  464 score:  130.0 average score 174.2 best score 189.1 epsilon 0.10 steps  239107\n",
            "episode  465 score:  200.0 average score 174.7 best score 189.1 epsilon 0.10 steps  239601\n",
            "episode  466 score:  80.0 average score 174.8 best score 189.1 epsilon 0.10 steps  239922\n",
            "episode  467 score:  155.0 average score 175.2 best score 189.1 epsilon 0.10 steps  240350\n",
            "episode  468 score:  210.0 average score 175.3 best score 189.1 epsilon 0.10 steps  240937\n",
            "episode  469 score:  110.0 average score 173.7 best score 189.1 epsilon 0.10 steps  241278\n",
            "episode  470 score:  135.0 average score 171.3 best score 189.1 epsilon 0.10 steps  241804\n",
            "episode  471 score:  180.0 average score 172.6 best score 189.1 epsilon 0.10 steps  242290\n",
            "episode  472 score:  105.0 average score 172.6 best score 189.1 epsilon 0.10 steps  242676\n",
            "episode  473 score:  105.0 average score 172.6 best score 189.1 epsilon 0.10 steps  243024\n",
            "episode  474 score:  140.0 average score 172.6 best score 189.1 epsilon 0.10 steps  243464\n",
            "episode  475 score:  180.0 average score 168.2 best score 189.1 epsilon 0.10 steps  243852\n",
            "episode  476 score:  125.0 average score 167.6 best score 189.1 epsilon 0.10 steps  244419\n",
            "episode  477 score:  105.0 average score 167.6 best score 189.1 epsilon 0.10 steps  244805\n",
            "episode  478 score:  110.0 average score 166.9 best score 189.1 epsilon 0.10 steps  245133\n",
            "episode  479 score:  170.0 average score 165.6 best score 189.1 epsilon 0.10 steps  245639\n",
            "episode  480 score:  190.0 average score 166.3 best score 189.1 epsilon 0.10 steps  246254\n",
            "episode  481 score:  120.0 average score 165.8 best score 189.1 epsilon 0.10 steps  246698\n",
            "episode  482 score:  110.0 average score 165.4 best score 189.1 epsilon 0.10 steps  247114\n",
            "episode  483 score:  135.0 average score 163.6 best score 189.1 epsilon 0.10 steps  247544\n",
            "episode  484 score:  105.0 average score 161.0 best score 189.1 epsilon 0.10 steps  247940\n",
            "episode  485 score:  185.0 average score 160.4 best score 189.1 epsilon 0.10 steps  248463\n",
            "episode  486 score:  215.0 average score 161.5 best score 189.1 epsilon 0.10 steps  248985\n",
            "episode  487 score:  255.0 average score 162.4 best score 189.1 epsilon 0.10 steps  249779\n",
            "episode  488 score:  155.0 average score 161.2 best score 189.1 epsilon 0.10 steps  250290\n",
            "episode  489 score:  110.0 average score 161.2 best score 189.1 epsilon 0.10 steps  250706\n",
            "episode  490 score:  265.0 average score 160.8 best score 189.1 epsilon 0.10 steps  251373\n",
            "episode  491 score:  235.0 average score 160.5 best score 189.1 epsilon 0.10 steps  251930\n",
            "episode  492 score:  135.0 average score 160.8 best score 189.1 epsilon 0.10 steps  252416\n",
            "episode  493 score:  260.0 average score 160.8 best score 189.1 epsilon 0.10 steps  253329\n",
            "episode  494 score:  390.0 average score 163.1 best score 189.1 epsilon 0.10 steps  254303\n",
            "episode  495 score:  125.0 average score 163.6 best score 189.1 epsilon 0.10 steps  254708\n",
            "episode  496 score:  210.0 average score 162.9 best score 189.1 epsilon 0.10 steps  255237\n",
            "episode  497 score:  210.0 average score 163.2 best score 189.1 epsilon 0.10 steps  255779\n",
            "episode  498 score:  120.0 average score 163.2 best score 189.1 epsilon 0.10 steps  256144\n",
            "episode  499 score:  135.0 average score 162.9 best score 189.1 epsilon 0.10 steps  256563\n",
            "episode  500 score:  135.0 average score 163.5 best score 189.1 epsilon 0.10 steps  256958\n",
            "episode  501 score:  135.0 average score 161.9 best score 189.1 epsilon 0.10 steps  257395\n",
            "episode  502 score:  110.0 average score 161.8 best score 189.1 epsilon 0.10 steps  257951\n",
            "episode  503 score:  155.0 average score 161.2 best score 189.1 epsilon 0.10 steps  258367\n",
            "episode  504 score:  105.0 average score 160.8 best score 189.1 epsilon 0.10 steps  258664\n",
            "episode  505 score:  170.0 average score 161.4 best score 189.1 epsilon 0.10 steps  259151\n",
            "episode  506 score:  180.0 average score 160.6 best score 189.1 epsilon 0.10 steps  259759\n",
            "episode  507 score:  145.0 average score 160.3 best score 189.1 epsilon 0.10 steps  260236\n",
            "episode  508 score:  125.0 average score 160.2 best score 189.1 epsilon 0.10 steps  260603\n",
            "episode  509 score:  355.0 average score 162.3 best score 189.1 epsilon 0.10 steps  261499\n",
            "episode  510 score:  105.0 average score 161.8 best score 189.1 epsilon 0.10 steps  261782\n",
            "episode  511 score:  315.0 average score 163.5 best score 189.1 epsilon 0.10 steps  262601\n",
            "episode  512 score:  180.0 average score 164.2 best score 189.1 epsilon 0.10 steps  263106\n",
            "episode  513 score:  545.0 average score 168.4 best score 189.1 epsilon 0.10 steps  263866\n",
            "episode  514 score:  210.0 average score 166.7 best score 189.1 epsilon 0.10 steps  264500\n",
            "episode  515 score:  670.0 average score 172.3 best score 189.1 epsilon 0.10 steps  265683\n",
            "episode  516 score:  180.0 average score 172.6 best score 189.1 epsilon 0.10 steps  266233\n",
            "episode  517 score:  135.0 average score 172.6 best score 189.1 epsilon 0.10 steps  266577\n",
            "episode  518 score:  75.0 average score 171.6 best score 189.1 epsilon 0.10 steps  266863\n",
            "episode  519 score:  285.0 average score 172.8 best score 189.1 epsilon 0.10 steps  267510\n",
            "episode  520 score:  195.0 average score 171.8 best score 189.1 epsilon 0.10 steps  268041\n",
            "episode  521 score:  125.0 average score 170.7 best score 189.1 epsilon 0.10 steps  268424\n",
            "episode  522 score:  345.0 average score 172.6 best score 189.1 epsilon 0.10 steps  269344\n",
            "episode  523 score:  135.0 average score 172.8 best score 189.1 epsilon 0.10 steps  269893\n",
            "episode  524 score:  150.0 average score 173.2 best score 189.1 epsilon 0.10 steps  270342\n",
            "episode  525 score:  140.0 average score 172.8 best score 189.1 epsilon 0.10 steps  270715\n",
            "episode  526 score:  145.0 average score 173.2 best score 189.1 epsilon 0.10 steps  271284\n",
            "episode  527 score:  70.0 average score 172.2 best score 189.1 epsilon 0.10 steps  271672\n",
            "episode  528 score:  35.0 average score 169.3 best score 189.1 epsilon 0.10 steps  272050\n",
            "episode  529 score:  185.0 average score 169.6 best score 189.1 epsilon 0.10 steps  272565\n",
            "episode  530 score:  235.0 average score 169.3 best score 189.1 epsilon 0.10 steps  273178\n",
            "episode  531 score:  180.0 average score 169.2 best score 189.1 epsilon 0.10 steps  273582\n",
            "episode  532 score:  180.0 average score 169.8 best score 189.1 epsilon 0.10 steps  273913\n",
            "episode  533 score:  320.0 average score 171.0 best score 189.1 epsilon 0.10 steps  274458\n",
            "episode  534 score:  80.0 average score 169.8 best score 189.1 epsilon 0.10 steps  274740\n",
            "episode  535 score:  195.0 average score 170.7 best score 189.1 epsilon 0.10 steps  275386\n",
            "episode  536 score:  125.0 average score 170.1 best score 189.1 epsilon 0.10 steps  275798\n",
            "episode  537 score:  165.0 average score 170.2 best score 189.1 epsilon 0.10 steps  276334\n",
            "episode  538 score:  210.0 average score 170.9 best score 189.1 epsilon 0.10 steps  277122\n",
            "episode  539 score:  370.0 average score 173.2 best score 189.1 epsilon 0.10 steps  277776\n",
            "episode  540 score:  105.0 average score 173.5 best score 189.1 epsilon 0.10 steps  278156\n",
            "episode  541 score:  185.0 average score 174.0 best score 189.1 epsilon 0.10 steps  278682\n",
            "episode  542 score:  110.0 average score 174.3 best score 189.1 epsilon 0.10 steps  279171\n",
            "episode  543 score:  50.0 average score 173.3 best score 189.1 epsilon 0.10 steps  279500\n",
            "episode  544 score:  85.0 average score 173.2 best score 189.1 epsilon 0.10 steps  279870\n",
            "episode  545 score:  245.0 average score 173.8 best score 189.1 epsilon 0.10 steps  280431\n",
            "episode  546 score:  120.0 average score 172.4 best score 189.1 epsilon 0.10 steps  281159\n",
            "episode  547 score:  385.0 average score 175.5 best score 189.1 epsilon 0.10 steps  281769\n",
            "episode  548 score:  220.0 average score 175.7 best score 189.1 epsilon 0.10 steps  282582\n",
            "episode  549 score:  110.0 average score 175.2 best score 189.1 epsilon 0.10 steps  282904\n",
            "episode  550 score:  80.0 average score 174.8 best score 189.1 epsilon 0.10 steps  283297\n",
            "episode  551 score:  110.0 average score 174.9 best score 189.1 epsilon 0.10 steps  283737\n",
            "episode  552 score:  110.0 average score 174.2 best score 189.1 epsilon 0.10 steps  284068\n",
            "episode  553 score:  90.0 average score 173.9 best score 189.1 epsilon 0.10 steps  284491\n",
            "episode  554 score:  75.0 average score 173.7 best score 189.1 epsilon 0.10 steps  284791\n",
            "episode  555 score:  80.0 average score 171.0 best score 189.1 epsilon 0.10 steps  285190\n",
            "episode  556 score:  160.0 average score 169.4 best score 189.1 epsilon 0.10 steps  285563\n",
            "episode  557 score:  165.0 average score 170.6 best score 189.1 epsilon 0.10 steps  286061\n",
            "episode  558 score:  150.0 average score 170.8 best score 189.1 epsilon 0.10 steps  286525\n",
            "episode  559 score:  240.0 average score 172.1 best score 189.1 epsilon 0.10 steps  287142\n",
            "episode  560 score:  125.0 average score 172.5 best score 189.1 epsilon 0.10 steps  287557\n",
            "episode  561 score:  200.0 average score 172.8 best score 189.1 epsilon 0.10 steps  288061\n",
            "episode  562 score:  125.0 average score 173.4 best score 189.1 epsilon 0.10 steps  288558\n",
            "episode  563 score:  120.0 average score 172.5 best score 189.1 epsilon 0.10 steps  288951\n",
            "episode  564 score:  210.0 average score 173.3 best score 189.1 epsilon 0.10 steps  289818\n",
            "episode  565 score:  115.0 average score 172.4 best score 189.1 epsilon 0.10 steps  290314\n",
            "episode  566 score:  315.0 average score 174.8 best score 189.1 epsilon 0.10 steps  291050\n",
            "episode  567 score:  80.0 average score 174.1 best score 189.1 epsilon 0.10 steps  291394\n",
            "episode  568 score:  180.0 average score 173.8 best score 189.1 epsilon 0.10 steps  291852\n",
            "episode  569 score:  210.0 average score 174.8 best score 189.1 epsilon 0.10 steps  292347\n",
            "episode  570 score:  110.0 average score 174.5 best score 189.1 epsilon 0.10 steps  292765\n",
            "episode  571 score:  180.0 average score 174.5 best score 189.1 epsilon 0.10 steps  293220\n",
            "episode  572 score:  155.0 average score 175.0 best score 189.1 epsilon 0.10 steps  293703\n",
            "episode  573 score:  255.0 average score 176.5 best score 189.1 epsilon 0.10 steps  294261\n",
            "episode  574 score:  270.0 average score 177.8 best score 189.1 epsilon 0.10 steps  294872\n",
            "episode  575 score:  240.0 average score 178.4 best score 189.1 epsilon 0.10 steps  295464\n",
            "episode  576 score:  515.0 average score 182.3 best score 189.1 epsilon 0.10 steps  296231\n",
            "episode  577 score:  115.0 average score 182.4 best score 189.1 epsilon 0.10 steps  296730\n",
            "episode  578 score:  125.0 average score 182.6 best score 189.1 epsilon 0.10 steps  297160\n",
            "episode  579 score:  250.0 average score 183.3 best score 189.1 epsilon 0.10 steps  297838\n",
            "episode  580 score:  105.0 average score 182.5 best score 189.1 epsilon 0.10 steps  298137\n",
            "episode  581 score:  160.0 average score 182.9 best score 189.1 epsilon 0.10 steps  298589\n",
            "episode  582 score:  195.0 average score 183.8 best score 189.1 epsilon 0.10 steps  299457\n",
            "episode  583 score:  470.0 average score 187.1 best score 189.1 epsilon 0.10 steps  300341\n",
            "episode  584 score:  110.0 average score 187.2 best score 189.1 epsilon 0.10 steps  300698\n",
            "episode  585 score:  215.0 average score 187.4 best score 189.1 epsilon 0.10 steps  301252\n",
            "episode  586 score:  105.0 average score 186.3 best score 189.1 epsilon 0.10 steps  301615\n",
            "episode  587 score:  210.0 average score 185.9 best score 189.1 epsilon 0.10 steps  302070\n",
            "episode  588 score:  110.0 average score 185.4 best score 189.1 epsilon 0.10 steps  302454\n",
            "episode  589 score:  185.0 average score 186.2 best score 189.1 epsilon 0.10 steps  302948\n",
            "episode  590 score:  210.0 average score 185.7 best score 189.1 epsilon 0.10 steps  303465\n",
            "episode  591 score:  110.0 average score 184.4 best score 189.1 epsilon 0.10 steps  303788\n",
            "episode  592 score:  170.0 average score 184.8 best score 189.1 epsilon 0.10 steps  304217\n",
            "episode  593 score:  55.0 average score 182.7 best score 189.1 epsilon 0.10 steps  304579\n",
            "episode  594 score:  105.0 average score 179.8 best score 189.1 epsilon 0.10 steps  304868\n",
            "episode  595 score:  175.0 average score 180.3 best score 189.1 epsilon 0.10 steps  305460\n",
            "episode  596 score:  125.0 average score 179.5 best score 189.1 epsilon 0.10 steps  305837\n",
            "episode  597 score:  125.0 average score 178.7 best score 189.1 epsilon 0.10 steps  306423\n",
            "episode  598 score:  280.0 average score 180.2 best score 189.1 epsilon 0.10 steps  307001\n",
            "episode  599 score:  50.0 average score 179.4 best score 189.1 epsilon 0.10 steps  307262\n",
            "episode  600 score:  215.0 average score 180.2 best score 189.1 epsilon 0.10 steps  307719\n",
            "episode  601 score:  80.0 average score 179.7 best score 189.1 epsilon 0.10 steps  308068\n",
            "episode  602 score:  105.0 average score 179.6 best score 189.1 epsilon 0.10 steps  308460\n",
            "episode  603 score:  325.0 average score 181.3 best score 189.1 epsilon 0.10 steps  309061\n",
            "episode  604 score:  135.0 average score 181.6 best score 189.1 epsilon 0.10 steps  309419\n",
            "episode  605 score:  120.0 average score 181.1 best score 189.1 epsilon 0.10 steps  309798\n",
            "episode  606 score:  155.0 average score 180.8 best score 189.1 epsilon 0.10 steps  310163\n",
            "episode  607 score:  160.0 average score 181.0 best score 189.1 epsilon 0.10 steps  310789\n",
            "episode  608 score:  105.0 average score 180.8 best score 189.1 epsilon 0.10 steps  311095\n",
            "episode  609 score:  230.0 average score 179.6 best score 189.1 epsilon 0.10 steps  311640\n",
            "episode  610 score:  105.0 average score 179.6 best score 189.1 epsilon 0.10 steps  311957\n",
            "episode  611 score:  190.0 average score 178.3 best score 189.1 epsilon 0.10 steps  312329\n",
            "episode  612 score:  80.0 average score 177.3 best score 189.1 epsilon 0.10 steps  312610\n",
            "episode  613 score:  165.0 average score 173.5 best score 189.1 epsilon 0.10 steps  312972\n",
            "episode  614 score:  150.0 average score 172.9 best score 189.1 epsilon 0.10 steps  313416\n",
            "episode  615 score:  225.0 average score 168.4 best score 189.1 epsilon 0.10 steps  314046\n",
            "episode  616 score:  150.0 average score 168.2 best score 189.1 epsilon 0.10 steps  314524\n",
            "episode  617 score:  295.0 average score 169.8 best score 189.1 epsilon 0.10 steps  315200\n",
            "episode  618 score:  135.0 average score 170.3 best score 189.1 epsilon 0.10 steps  315621\n",
            "episode  619 score:  190.0 average score 169.4 best score 189.1 epsilon 0.10 steps  316034\n",
            "episode  620 score:  130.0 average score 168.8 best score 189.1 epsilon 0.10 steps  316613\n",
            "episode  621 score:  265.0 average score 170.2 best score 189.1 epsilon 0.10 steps  317411\n",
            "episode  622 score:  160.0 average score 168.3 best score 189.1 epsilon 0.10 steps  317902\n",
            "episode  623 score:  290.0 average score 169.8 best score 189.1 epsilon 0.10 steps  318531\n",
            "episode  624 score:  210.0 average score 170.4 best score 189.1 epsilon 0.10 steps  319045\n",
            "episode  625 score:  185.0 average score 170.9 best score 189.1 epsilon 0.10 steps  319519\n",
            "episode  626 score:  120.0 average score 170.7 best score 189.1 epsilon 0.10 steps  319823\n",
            "episode  627 score:  100.0 average score 170.9 best score 189.1 epsilon 0.10 steps  320325\n",
            "episode  628 score:  155.0 average score 172.2 best score 189.1 epsilon 0.10 steps  320930\n",
            "episode  629 score:  110.0 average score 171.4 best score 189.1 epsilon 0.10 steps  321354\n",
            "episode  630 score:  105.0 average score 170.1 best score 189.1 epsilon 0.10 steps  321667\n",
            "episode  631 score:  580.0 average score 174.1 best score 189.1 epsilon 0.10 steps  322453\n",
            "episode  632 score:  225.0 average score 174.6 best score 189.1 epsilon 0.10 steps  323216\n",
            "episode  633 score:  235.0 average score 173.7 best score 189.1 epsilon 0.10 steps  323787\n",
            "episode  634 score:  110.0 average score 174.0 best score 189.1 epsilon 0.10 steps  324169\n",
            "episode  635 score:  120.0 average score 173.2 best score 189.1 epsilon 0.10 steps  324709\n",
            "episode  636 score:  230.0 average score 174.3 best score 189.1 epsilon 0.10 steps  325323\n",
            "episode  637 score:  365.0 average score 176.3 best score 189.1 epsilon 0.10 steps  326011\n",
            "episode  638 score:  195.0 average score 176.2 best score 189.1 epsilon 0.10 steps  326507\n",
            "episode  639 score:  190.0 average score 174.3 best score 189.1 epsilon 0.10 steps  327067\n",
            "episode  640 score:  240.0 average score 175.7 best score 189.1 epsilon 0.10 steps  327606\n",
            "episode  641 score:  315.0 average score 177.0 best score 189.1 epsilon 0.10 steps  328344\n",
            "episode  642 score:  215.0 average score 178.1 best score 189.1 epsilon 0.10 steps  328851\n",
            "episode  643 score:  155.0 average score 179.1 best score 189.1 epsilon 0.10 steps  329229\n",
            "episode  644 score:  165.0 average score 179.9 best score 189.1 epsilon 0.10 steps  329851\n",
            "episode  645 score:  105.0 average score 178.5 best score 189.1 epsilon 0.10 steps  330321\n",
            "episode  646 score:  160.0 average score 178.9 best score 189.1 epsilon 0.10 steps  330760\n",
            "episode  647 score:  155.0 average score 176.6 best score 189.1 epsilon 0.10 steps  331217\n",
            "episode  648 score:  210.0 average score 176.5 best score 189.1 epsilon 0.10 steps  331783\n",
            "episode  649 score:  270.0 average score 178.1 best score 189.1 epsilon 0.10 steps  332666\n",
            "episode  650 score:  255.0 average score 179.8 best score 189.1 epsilon 0.10 steps  333272\n",
            "episode  651 score:  160.0 average score 180.3 best score 189.1 epsilon 0.10 steps  333719\n",
            "episode  652 score:  160.0 average score 180.8 best score 189.1 epsilon 0.10 steps  334189\n",
            "episode  653 score:  110.0 average score 181.1 best score 189.1 epsilon 0.10 steps  334570\n",
            "episode  654 score:  115.0 average score 181.4 best score 189.1 epsilon 0.10 steps  334960\n",
            "episode  655 score:  165.0 average score 182.3 best score 189.1 epsilon 0.10 steps  335568\n",
            "episode  656 score:  290.0 average score 183.6 best score 189.1 epsilon 0.10 steps  336164\n",
            "episode  657 score:  320.0 average score 185.2 best score 189.1 epsilon 0.10 steps  337143\n",
            "episode  658 score:  75.0 average score 184.4 best score 189.1 epsilon 0.10 steps  337453\n",
            "episode  659 score:  135.0 average score 183.3 best score 189.1 epsilon 0.10 steps  337912\n",
            "episode  660 score:  130.0 average score 183.4 best score 189.1 epsilon 0.10 steps  338396\n",
            "episode  661 score:  160.0 average score 183.0 best score 189.1 epsilon 0.10 steps  338907\n",
            "episode  662 score:  270.0 average score 184.4 best score 189.1 epsilon 0.10 steps  339706\n",
            "episode  663 score:  125.0 average score 184.5 best score 189.1 epsilon 0.10 steps  340196\n",
            "episode  664 score:  215.0 average score 184.6 best score 189.1 epsilon 0.10 steps  340771\n",
            "episode  665 score:  100.0 average score 184.4 best score 189.1 epsilon 0.10 steps  341163\n",
            "episode  666 score:  215.0 average score 183.4 best score 189.1 epsilon 0.10 steps  341676\n",
            "episode  667 score:  250.0 average score 185.1 best score 189.1 epsilon 0.10 steps  342392\n",
            "episode  668 score:  135.0 average score 184.7 best score 189.1 epsilon 0.10 steps  342885\n",
            "episode  669 score:  175.0 average score 184.3 best score 189.1 epsilon 0.10 steps  343255\n",
            "episode  670 score:  110.0 average score 184.3 best score 189.1 epsilon 0.10 steps  343787\n",
            "episode  671 score:  105.0 average score 183.6 best score 189.1 epsilon 0.10 steps  344207\n",
            "episode  672 score:  395.0 average score 185.9 best score 189.1 epsilon 0.10 steps  344929\n",
            "episode  673 score:  415.0 average score 187.6 best score 189.1 epsilon 0.10 steps  346266\n",
            "episode  674 score:  170.0 average score 186.6 best score 189.1 epsilon 0.10 steps  346974\n",
            "episode  675 score:  110.0 average score 185.2 best score 189.1 epsilon 0.10 steps  347401\n",
            "episode  676 score:  105.0 average score 181.2 best score 189.1 epsilon 0.10 steps  347858\n",
            "episode  677 score:  180.0 average score 181.8 best score 189.1 epsilon 0.10 steps  348483\n",
            "episode  678 score:  410.0 average score 184.7 best score 189.1 epsilon 0.10 steps  349457\n",
            "episode  679 score:  130.0 average score 183.4 best score 189.1 epsilon 0.10 steps  349947\n",
            "episode  680 score:  225.0 average score 184.7 best score 189.1 epsilon 0.10 steps  350566\n",
            "episode  681 score:  375.0 average score 186.8 best score 189.1 epsilon 0.10 steps  351449\n",
            "episode  682 score:  260.0 average score 187.4 best score 189.1 epsilon 0.10 steps  352157\n",
            "episode  683 score:  165.0 average score 184.4 best score 189.1 epsilon 0.10 steps  352579\n",
            "episode  684 score:  100.0 average score 184.3 best score 189.1 epsilon 0.10 steps  352924\n",
            "episode  685 score:  340.0 average score 185.6 best score 189.1 epsilon 0.10 steps  353619\n",
            "episode  686 score:  125.0 average score 185.8 best score 189.1 epsilon 0.10 steps  354066\n",
            "episode  687 score:  180.0 average score 185.4 best score 189.1 epsilon 0.10 steps  354554\n",
            "episode  688 score:  125.0 average score 185.6 best score 189.1 epsilon 0.10 steps  355005\n",
            "episode  689 score:  250.0 average score 186.2 best score 189.1 epsilon 0.10 steps  355591\n",
            "episode  690 score:  180.0 average score 185.9 best score 189.1 epsilon 0.10 steps  356020\n",
            "episode  691 score:  140.0 average score 186.2 best score 189.1 epsilon 0.10 steps  356434\n",
            "episode  692 score:  50.0 average score 185.1 best score 189.1 epsilon 0.10 steps  356692\n",
            "episode  693 score:  215.0 average score 186.7 best score 189.1 epsilon 0.10 steps  357390\n",
            "episode  694 score:  225.0 average score 187.8 best score 189.1 epsilon 0.10 steps  357881\n",
            "episode  695 score:  350.0 average score 189.6 best score 189.1 epsilon 0.10 steps  359006\n",
            "... saving checkpoint ...\n",
            "... saving checkpoint ...\n",
            "episode  696 score:  170.0 average score 190.1 best score 189.6 epsilon 0.10 steps  359544\n",
            "... saving checkpoint ...\n",
            "... saving checkpoint ...\n",
            "episode  697 score:  170.0 average score 190.5 best score 190.1 epsilon 0.10 steps  360082\n",
            "... saving checkpoint ...\n",
            "... saving checkpoint ...\n",
            "episode  698 score:  80.0 average score 188.5 best score 190.5 epsilon 0.10 steps  360416\n",
            "episode  699 score:  155.0 average score 189.6 best score 190.5 epsilon 0.10 steps  360887\n",
            "episode  700 score:  185.0 average score 189.2 best score 190.5 epsilon 0.10 steps  361547\n",
            "episode  701 score:  190.0 average score 190.3 best score 190.5 epsilon 0.10 steps  362067\n",
            "episode  702 score:  210.0 average score 191.4 best score 190.5 epsilon 0.10 steps  362553\n",
            "... saving checkpoint ...\n",
            "... saving checkpoint ...\n",
            "episode  703 score:  380.0 average score 191.9 best score 191.4 epsilon 0.10 steps  363375\n",
            "... saving checkpoint ...\n",
            "... saving checkpoint ...\n",
            "episode  704 score:  210.0 average score 192.7 best score 191.9 epsilon 0.10 steps  363882\n",
            "... saving checkpoint ...\n",
            "... saving checkpoint ...\n",
            "episode  705 score:  110.0 average score 192.6 best score 192.7 epsilon 0.10 steps  364245\n",
            "episode  706 score:  210.0 average score 193.2 best score 192.7 epsilon 0.10 steps  364889\n",
            "... saving checkpoint ...\n",
            "... saving checkpoint ...\n",
            "episode  707 score:  300.0 average score 194.6 best score 193.2 epsilon 0.10 steps  365681\n",
            "... saving checkpoint ...\n",
            "... saving checkpoint ...\n",
            "episode  708 score:  225.0 average score 195.8 best score 194.6 epsilon 0.10 steps  366268\n",
            "... saving checkpoint ...\n",
            "... saving checkpoint ...\n",
            "episode  709 score:  185.0 average score 195.3 best score 195.8 epsilon 0.10 steps  366781\n",
            "episode  710 score:  450.0 average score 198.8 best score 195.8 epsilon 0.10 steps  367740\n",
            "... saving checkpoint ...\n",
            "... saving checkpoint ...\n",
            "episode  711 score:  130.0 average score 198.2 best score 198.8 epsilon 0.10 steps  368222\n",
            "episode  712 score:  135.0 average score 198.7 best score 198.8 epsilon 0.10 steps  368608\n",
            "episode  713 score:  515.0 average score 202.2 best score 198.8 epsilon 0.10 steps  369710\n",
            "... saving checkpoint ...\n",
            "... saving checkpoint ...\n",
            "episode  714 score:  135.0 average score 202.1 best score 202.2 epsilon 0.10 steps  370017\n",
            "episode  715 score:  120.0 average score 201.0 best score 202.2 epsilon 0.10 steps  370365\n",
            "episode  716 score:  180.0 average score 201.3 best score 202.2 epsilon 0.10 steps  370779\n",
            "episode  717 score:  135.0 average score 199.7 best score 202.2 epsilon 0.10 steps  371254\n",
            "episode  718 score:  260.0 average score 200.9 best score 202.2 epsilon 0.10 steps  371752\n",
            "episode  719 score:  215.0 average score 201.2 best score 202.2 epsilon 0.10 steps  372412\n",
            "episode  720 score:  260.0 average score 202.5 best score 202.2 epsilon 0.10 steps  372937\n",
            "... saving checkpoint ...\n",
            "... saving checkpoint ...\n",
            "episode  721 score:  255.0 average score 202.4 best score 202.5 epsilon 0.10 steps  373561\n",
            "episode  722 score:  155.0 average score 202.3 best score 202.5 epsilon 0.10 steps  373974\n",
            "episode  723 score:  210.0 average score 201.6 best score 202.5 epsilon 0.10 steps  374432\n",
            "episode  724 score:  135.0 average score 200.8 best score 202.5 epsilon 0.10 steps  374770\n",
            "episode  725 score:  135.0 average score 200.3 best score 202.5 epsilon 0.10 steps  375176\n",
            "episode  726 score:  150.0 average score 200.6 best score 202.5 epsilon 0.10 steps  375602\n",
            "episode  727 score:  185.0 average score 201.4 best score 202.5 epsilon 0.10 steps  376113\n",
            "episode  728 score:  420.0 average score 204.1 best score 202.5 epsilon 0.10 steps  376869\n",
            "... saving checkpoint ...\n",
            "... saving checkpoint ...\n",
            "episode  729 score:  145.0 average score 204.4 best score 204.1 epsilon 0.10 steps  377413\n",
            "... saving checkpoint ...\n",
            "... saving checkpoint ...\n",
            "episode  730 score:  105.0 average score 204.4 best score 204.4 epsilon 0.10 steps  377765\n",
            "episode  731 score:  170.0 average score 200.3 best score 204.4 epsilon 0.10 steps  378185\n",
            "episode  732 score:  160.0 average score 199.7 best score 204.4 epsilon 0.10 steps  378613\n",
            "episode  733 score:  165.0 average score 199.0 best score 204.4 epsilon 0.10 steps  379099\n",
            "episode  734 score:  205.0 average score 199.9 best score 204.4 epsilon 0.10 steps  379672\n",
            "episode  735 score:  135.0 average score 200.1 best score 204.4 epsilon 0.10 steps  380114\n",
            "episode  736 score:  175.0 average score 199.6 best score 204.4 epsilon 0.10 steps  380736\n",
            "episode  737 score:  190.0 average score 197.8 best score 204.4 epsilon 0.10 steps  381278\n",
            "episode  738 score:  190.0 average score 197.8 best score 204.4 epsilon 0.10 steps  381886\n",
            "episode  739 score:  445.0 average score 200.3 best score 204.4 epsilon 0.10 steps  382467\n",
            "episode  740 score:  225.0 average score 200.2 best score 204.4 epsilon 0.10 steps  382951\n",
            "episode  741 score:  110.0 average score 198.1 best score 204.4 epsilon 0.10 steps  383368\n",
            "episode  742 score:  880.0 average score 204.8 best score 204.4 epsilon 0.10 steps  384376\n",
            "... saving checkpoint ...\n",
            "... saving checkpoint ...\n",
            "episode  743 score:  305.0 average score 206.2 best score 204.8 epsilon 0.10 steps  385079\n",
            "... saving checkpoint ...\n",
            "... saving checkpoint ...\n",
            "episode  744 score:  385.0 average score 208.4 best score 206.2 epsilon 0.10 steps  385930\n",
            "... saving checkpoint ...\n",
            "... saving checkpoint ...\n",
            "episode  745 score:  465.0 average score 212.1 best score 208.4 epsilon 0.10 steps  386828\n",
            "... saving checkpoint ...\n",
            "... saving checkpoint ...\n",
            "episode  746 score:  230.0 average score 212.8 best score 212.1 epsilon 0.10 steps  387348\n",
            "... saving checkpoint ...\n",
            "... saving checkpoint ...\n",
            "episode  747 score:  105.0 average score 212.2 best score 212.8 epsilon 0.10 steps  387782\n",
            "episode  748 score:  110.0 average score 211.2 best score 212.8 epsilon 0.10 steps  388190\n",
            "episode  749 score:  270.0 average score 211.2 best score 212.8 epsilon 0.10 steps  389259\n",
            "episode  750 score:  220.0 average score 210.9 best score 212.8 epsilon 0.10 steps  389742\n",
            "episode  751 score:  400.0 average score 213.3 best score 212.8 epsilon 0.10 steps  390338\n",
            "... saving checkpoint ...\n",
            "... saving checkpoint ...\n",
            "episode  752 score:  195.0 average score 213.7 best score 213.3 epsilon 0.10 steps  391003\n",
            "... saving checkpoint ...\n",
            "... saving checkpoint ...\n",
            "episode  753 score:  135.0 average score 213.9 best score 213.7 epsilon 0.10 steps  391465\n",
            "... saving checkpoint ...\n",
            "... saving checkpoint ...\n",
            "episode  754 score:  410.0 average score 216.8 best score 213.9 epsilon 0.10 steps  392255\n",
            "... saving checkpoint ...\n",
            "... saving checkpoint ...\n",
            "episode  755 score:  210.0 average score 217.3 best score 216.8 epsilon 0.10 steps  392642\n",
            "... saving checkpoint ...\n",
            "... saving checkpoint ...\n",
            "episode  756 score:  210.0 average score 216.5 best score 217.3 epsilon 0.10 steps  393155\n",
            "episode  757 score:  135.0 average score 214.7 best score 217.3 epsilon 0.10 steps  393534\n",
            "episode  758 score:  205.0 average score 215.9 best score 217.3 epsilon 0.10 steps  394250\n",
            "episode  759 score:  175.0 average score 216.3 best score 217.3 epsilon 0.10 steps  394836\n",
            "episode  760 score:  220.0 average score 217.2 best score 217.3 epsilon 0.10 steps  395463\n",
            "episode  761 score:  300.0 average score 218.7 best score 217.3 epsilon 0.10 steps  396215\n",
            "... saving checkpoint ...\n",
            "... saving checkpoint ...\n",
            "episode  762 score:  110.0 average score 217.1 best score 218.7 epsilon 0.10 steps  396674\n",
            "episode  763 score:  530.0 average score 221.1 best score 218.7 epsilon 0.10 steps  397419\n",
            "... saving checkpoint ...\n",
            "... saving checkpoint ...\n",
            "episode  764 score:  600.0 average score 224.9 best score 221.1 epsilon 0.10 steps  398781\n",
            "... saving checkpoint ...\n",
            "... saving checkpoint ...\n",
            "episode  765 score:  205.0 average score 226.0 best score 224.9 epsilon 0.10 steps  399399\n",
            "... saving checkpoint ...\n",
            "... saving checkpoint ...\n",
            "episode  766 score:  120.0 average score 225.1 best score 226.0 epsilon 0.10 steps  399817\n",
            "episode  767 score:  120.0 average score 223.8 best score 226.0 epsilon 0.10 steps  400241\n",
            "episode  768 score:  645.0 average score 228.8 best score 226.0 epsilon 0.10 steps  401283\n",
            "... saving checkpoint ...\n",
            "... saving checkpoint ...\n",
            "episode  769 score:  135.0 average score 228.4 best score 228.8 epsilon 0.10 steps  401790\n",
            "episode  770 score:  330.0 average score 230.7 best score 228.8 epsilon 0.10 steps  402608\n",
            "... saving checkpoint ...\n",
            "... saving checkpoint ...\n",
            "episode  771 score:  135.0 average score 230.9 best score 230.7 epsilon 0.10 steps  403027\n",
            "... saving checkpoint ...\n",
            "... saving checkpoint ...\n",
            "episode  772 score:  260.0 average score 229.6 best score 230.9 epsilon 0.10 steps  403649\n",
            "episode  773 score:  285.0 average score 228.3 best score 230.9 epsilon 0.10 steps  404502\n",
            "episode  774 score:  300.0 average score 229.6 best score 230.9 epsilon 0.10 steps  405183\n",
            "episode  775 score:  215.0 average score 230.7 best score 230.9 epsilon 0.10 steps  405737\n",
            "episode  776 score:  280.0 average score 232.4 best score 230.9 epsilon 0.10 steps  406396\n",
            "... saving checkpoint ...\n",
            "... saving checkpoint ...\n",
            "episode  777 score:  135.0 average score 231.9 best score 232.4 epsilon 0.10 steps  406774\n",
            "episode  778 score:  250.0 average score 230.3 best score 232.4 epsilon 0.10 steps  407343\n",
            "episode  779 score:  185.0 average score 230.9 best score 232.4 epsilon 0.10 steps  407820\n",
            "episode  780 score:  220.0 average score 230.8 best score 232.4 epsilon 0.10 steps  408634\n",
            "episode  781 score:  175.0 average score 228.8 best score 232.4 epsilon 0.10 steps  409279\n",
            "episode  782 score:  120.0 average score 227.4 best score 232.4 epsilon 0.10 steps  409627\n",
            "episode  783 score:  110.0 average score 226.9 best score 232.4 epsilon 0.10 steps  409981\n",
            "episode  784 score:  180.0 average score 227.7 best score 232.4 epsilon 0.10 steps  410404\n",
            "episode  785 score:  170.0 average score 226.0 best score 232.4 epsilon 0.10 steps  411021\n",
            "episode  786 score:  195.0 average score 226.7 best score 232.4 epsilon 0.10 steps  411497\n",
            "episode  787 score:  345.0 average score 228.3 best score 232.4 epsilon 0.10 steps  412149\n",
            "episode  788 score:  75.0 average score 227.8 best score 232.4 epsilon 0.10 steps  412417\n",
            "episode  789 score:  125.0 average score 226.6 best score 232.4 epsilon 0.10 steps  412905\n",
            "episode  790 score:  60.0 average score 225.4 best score 232.4 epsilon 0.10 steps  413209\n",
            "episode  791 score:  390.0 average score 227.9 best score 232.4 epsilon 0.10 steps  414020\n",
            "episode  792 score:  200.0 average score 229.4 best score 232.4 epsilon 0.10 steps  414553\n",
            "episode  793 score:  105.0 average score 228.3 best score 232.4 epsilon 0.10 steps  414899\n",
            "episode  794 score:  140.0 average score 227.4 best score 232.4 epsilon 0.10 steps  415240\n",
            "episode  795 score:  160.0 average score 225.6 best score 232.4 epsilon 0.10 steps  415721\n",
            "episode  796 score:  140.0 average score 225.2 best score 232.4 epsilon 0.10 steps  416122\n",
            "episode  797 score:  135.0 average score 224.9 best score 232.4 epsilon 0.10 steps  416614\n",
            "episode  798 score:  275.0 average score 226.8 best score 232.4 epsilon 0.10 steps  417359\n",
            "episode  799 score:  225.0 average score 227.6 best score 232.4 epsilon 0.10 steps  417856\n",
            "episode  800 score:  315.0 average score 228.8 best score 232.4 epsilon 0.10 steps  418451\n",
            "episode  801 score:  80.0 average score 227.8 best score 232.4 epsilon 0.10 steps  418840\n",
            "episode  802 score:  190.0 average score 227.6 best score 232.4 epsilon 0.10 steps  419416\n",
            "episode  803 score:  105.0 average score 224.8 best score 232.4 epsilon 0.10 steps  419835\n",
            "episode  804 score:  110.0 average score 223.8 best score 232.4 epsilon 0.10 steps  420238\n",
            "episode  805 score:  120.0 average score 223.9 best score 232.4 epsilon 0.10 steps  420614\n",
            "episode  806 score:  110.0 average score 222.9 best score 232.4 epsilon 0.10 steps  420948\n",
            "episode  807 score:  105.0 average score 220.9 best score 232.4 epsilon 0.10 steps  421297\n",
            "episode  808 score:  135.0 average score 220.1 best score 232.4 epsilon 0.10 steps  421647\n",
            "episode  809 score:  155.0 average score 219.8 best score 232.4 epsilon 0.10 steps  422019\n",
            "episode  810 score:  120.0 average score 216.4 best score 232.4 epsilon 0.10 steps  422370\n",
            "episode  811 score:  50.0 average score 215.7 best score 232.4 epsilon 0.10 steps  422644\n",
            "episode  812 score:  350.0 average score 217.8 best score 232.4 epsilon 0.10 steps  423468\n",
            "episode  813 score:  180.0 average score 214.4 best score 232.4 epsilon 0.10 steps  423852\n",
            "episode  814 score:  335.0 average score 216.4 best score 232.4 epsilon 0.10 steps  424463\n",
            "episode  815 score:  110.0 average score 216.3 best score 232.4 epsilon 0.10 steps  424789\n",
            "episode  816 score:  130.0 average score 215.8 best score 232.4 epsilon 0.10 steps  425188\n",
            "episode  817 score:  260.0 average score 217.1 best score 232.4 epsilon 0.10 steps  425787\n",
            "episode  818 score:  180.0 average score 216.3 best score 232.4 epsilon 0.10 steps  426470\n",
            "episode  819 score:  105.0 average score 215.2 best score 232.4 epsilon 0.10 steps  426797\n",
            "episode  820 score:  235.0 average score 214.9 best score 232.4 epsilon 0.10 steps  427369\n",
            "episode  821 score:  105.0 average score 213.4 best score 232.4 epsilon 0.10 steps  427818\n",
            "episode  822 score:  240.0 average score 214.3 best score 232.4 epsilon 0.10 steps  428502\n",
            "episode  823 score:  415.0 average score 216.3 best score 232.4 epsilon 0.10 steps  429296\n",
            "episode  824 score:  245.0 average score 217.4 best score 232.4 epsilon 0.10 steps  429940\n",
            "episode  825 score:  240.0 average score 218.5 best score 232.4 epsilon 0.10 steps  430563\n",
            "episode  826 score:  315.0 average score 220.2 best score 232.4 epsilon 0.10 steps  431340\n",
            "episode  827 score:  105.0 average score 219.3 best score 232.4 epsilon 0.10 steps  431639\n",
            "episode  828 score:  310.0 average score 218.2 best score 232.4 epsilon 0.10 steps  432409\n",
            "episode  829 score:  75.0 average score 217.6 best score 232.4 epsilon 0.10 steps  432793\n",
            "episode  830 score:  390.0 average score 220.4 best score 232.4 epsilon 0.10 steps  433461\n",
            "episode  831 score:  225.0 average score 220.9 best score 232.4 epsilon 0.10 steps  434127\n",
            "episode  832 score:  85.0 average score 220.2 best score 232.4 epsilon 0.10 steps  434465\n",
            "episode  833 score:  105.0 average score 219.6 best score 232.4 epsilon 0.10 steps  434805\n",
            "episode  834 score:  290.0 average score 220.4 best score 232.4 epsilon 0.10 steps  435365\n",
            "episode  835 score:  215.0 average score 221.2 best score 232.4 epsilon 0.10 steps  436096\n",
            "episode  836 score:  110.0 average score 220.6 best score 232.4 epsilon 0.10 steps  436474\n",
            "episode  837 score:  350.0 average score 222.2 best score 232.4 epsilon 0.10 steps  437187\n",
            "episode  838 score:  255.0 average score 222.8 best score 232.4 epsilon 0.10 steps  437737\n",
            "episode  839 score:  150.0 average score 219.9 best score 232.4 epsilon 0.10 steps  438233\n",
            "episode  840 score:  225.0 average score 219.9 best score 232.4 epsilon 0.10 steps  438930\n",
            "episode  841 score:  265.0 average score 221.4 best score 232.4 epsilon 0.10 steps  439664\n",
            "episode  842 score:  385.0 average score 216.5 best score 232.4 epsilon 0.10 steps  440355\n",
            "episode  843 score:  185.0 average score 215.3 best score 232.4 epsilon 0.10 steps  440840\n",
            "episode  844 score:  155.0 average score 213.0 best score 232.4 epsilon 0.10 steps  441250\n",
            "episode  845 score:  250.0 average score 210.8 best score 232.4 epsilon 0.10 steps  441868\n",
            "episode  846 score:  105.0 average score 209.6 best score 232.4 epsilon 0.10 steps  442183\n",
            "episode  847 score:  275.0 average score 211.3 best score 232.4 epsilon 0.10 steps  442759\n",
            "episode  848 score:  260.0 average score 212.8 best score 232.4 epsilon 0.10 steps  443289\n",
            "episode  849 score:  160.0 average score 211.7 best score 232.4 epsilon 0.10 steps  443736\n",
            "episode  850 score:  160.0 average score 211.1 best score 232.4 epsilon 0.10 steps  444248\n",
            "episode  851 score:  140.0 average score 208.5 best score 232.4 epsilon 0.10 steps  444638\n",
            "episode  852 score:  260.0 average score 209.2 best score 232.4 epsilon 0.10 steps  445452\n",
            "episode  853 score:  120.0 average score 209.0 best score 232.4 epsilon 0.10 steps  445931\n",
            "episode  854 score:  285.0 average score 207.8 best score 232.4 epsilon 0.10 steps  446597\n",
            "episode  855 score:  155.0 average score 207.2 best score 232.4 epsilon 0.10 steps  447032\n",
            "episode  856 score:  105.0 average score 206.2 best score 232.4 epsilon 0.10 steps  447504\n",
            "episode  857 score:  260.0 average score 207.4 best score 232.4 epsilon 0.10 steps  448013\n",
            "episode  858 score:  170.0 average score 207.1 best score 232.4 epsilon 0.10 steps  448499\n",
            "episode  859 score:  140.0 average score 206.7 best score 232.4 epsilon 0.10 steps  448880\n",
            "episode  860 score:  235.0 average score 206.8 best score 232.4 epsilon 0.10 steps  449433\n",
            "episode  861 score:  325.0 average score 207.1 best score 232.4 epsilon 0.10 steps  450096\n",
            "episode  862 score:  185.0 average score 207.8 best score 232.4 epsilon 0.10 steps  450700\n",
            "episode  863 score:  185.0 average score 204.4 best score 232.4 epsilon 0.10 steps  451091\n",
            "episode  864 score:  135.0 average score 199.8 best score 232.4 epsilon 0.10 steps  451487\n",
            "episode  865 score:  220.0 average score 199.9 best score 232.4 epsilon 0.10 steps  452157\n",
            "episode  866 score:  270.0 average score 201.4 best score 232.4 epsilon 0.10 steps  452751\n",
            "episode  867 score:  95.0 average score 201.2 best score 232.4 epsilon 0.10 steps  453101\n",
            "episode  868 score:  215.0 average score 196.8 best score 232.4 epsilon 0.10 steps  453565\n",
            "episode  869 score:  180.0 average score 197.3 best score 232.4 epsilon 0.10 steps  454029\n",
            "episode  870 score:  230.0 average score 196.3 best score 232.4 epsilon 0.10 steps  454536\n",
            "episode  871 score:  235.0 average score 197.3 best score 232.4 epsilon 0.10 steps  455071\n",
            "episode  872 score:  370.0 average score 198.4 best score 232.4 epsilon 0.10 steps  455837\n",
            "episode  873 score:  235.0 average score 197.9 best score 232.4 epsilon 0.10 steps  456335\n",
            "episode  874 score:  180.0 average score 196.7 best score 232.4 epsilon 0.10 steps  456822\n",
            "episode  875 score:  120.0 average score 195.8 best score 232.4 epsilon 0.10 steps  457162\n",
            "episode  876 score:  355.0 average score 196.5 best score 232.4 epsilon 0.10 steps  457917\n",
            "episode  877 score:  160.0 average score 196.8 best score 232.4 epsilon 0.10 steps  458600\n",
            "episode  878 score:  250.0 average score 196.8 best score 232.4 epsilon 0.10 steps  459182\n",
            "episode  879 score:  260.0 average score 197.5 best score 232.4 epsilon 0.10 steps  460013\n",
            "episode  880 score:  505.0 average score 200.3 best score 232.4 epsilon 0.10 steps  460725\n",
            "episode  881 score:  245.0 average score 201.1 best score 232.4 epsilon 0.10 steps  461668\n",
            "episode  882 score:  295.0 average score 202.8 best score 232.4 epsilon 0.10 steps  462165\n",
            "episode  883 score:  225.0 average score 203.9 best score 232.4 epsilon 0.10 steps  462648\n",
            "episode  884 score:  170.0 average score 203.8 best score 232.4 epsilon 0.10 steps  463097\n",
            "episode  885 score:  150.0 average score 203.7 best score 232.4 epsilon 0.10 steps  463442\n",
            "episode  886 score:  210.0 average score 203.8 best score 232.4 epsilon 0.10 steps  463971\n",
            "episode  887 score:  180.0 average score 202.2 best score 232.4 epsilon 0.10 steps  464352\n",
            "episode  888 score:  110.0 average score 202.5 best score 232.4 epsilon 0.10 steps  464680\n",
            "episode  889 score:  130.0 average score 202.6 best score 232.4 epsilon 0.10 steps  465079\n",
            "episode  890 score:  240.0 average score 204.3 best score 232.4 epsilon 0.10 steps  465600\n",
            "episode  891 score:  365.0 average score 204.1 best score 232.4 epsilon 0.10 steps  466391\n",
            "episode  892 score:  210.0 average score 204.2 best score 232.4 epsilon 0.10 steps  467002\n",
            "episode  893 score:  580.0 average score 208.9 best score 232.4 epsilon 0.10 steps  467684\n",
            "episode  894 score:  255.0 average score 210.1 best score 232.4 epsilon 0.10 steps  468175\n",
            "episode  895 score:  525.0 average score 213.8 best score 232.4 epsilon 0.10 steps  468861\n",
            "episode  896 score:  200.0 average score 214.3 best score 232.4 epsilon 0.10 steps  469402\n",
            "episode  897 score:  155.0 average score 214.6 best score 232.4 epsilon 0.10 steps  469800\n",
            "episode  898 score:  210.0 average score 213.9 best score 232.4 epsilon 0.10 steps  470601\n",
            "episode  899 score:  80.0 average score 212.4 best score 232.4 epsilon 0.10 steps  470943\n",
            "episode  900 score:  150.0 average score 210.8 best score 232.4 epsilon 0.10 steps  471324\n",
            "episode  901 score:  350.0 average score 213.5 best score 232.4 epsilon 0.10 steps  472001\n",
            "episode  902 score:  185.0 average score 213.4 best score 232.4 epsilon 0.10 steps  472538\n",
            "episode  903 score:  205.0 average score 214.4 best score 232.4 epsilon 0.10 steps  473167\n",
            "episode  904 score:  255.0 average score 215.9 best score 232.4 epsilon 0.10 steps  473606\n",
            "episode  905 score:  85.0 average score 215.6 best score 232.4 epsilon 0.10 steps  473860\n",
            "episode  906 score:  120.0 average score 215.7 best score 232.4 epsilon 0.10 steps  474207\n",
            "episode  907 score:  285.0 average score 217.4 best score 232.4 epsilon 0.10 steps  474867\n",
            "episode  908 score:  335.0 average score 219.4 best score 232.4 epsilon 0.10 steps  475690\n",
            "episode  909 score:  360.0 average score 221.5 best score 232.4 epsilon 0.10 steps  476197\n",
            "episode  910 score:  255.0 average score 222.8 best score 232.4 epsilon 0.10 steps  476673\n",
            "episode  911 score:  450.0 average score 226.8 best score 232.4 epsilon 0.10 steps  477546\n",
            "episode  912 score:  105.0 average score 224.4 best score 232.4 epsilon 0.10 steps  477901\n",
            "episode  913 score:  165.0 average score 224.2 best score 232.4 epsilon 0.10 steps  478320\n",
            "episode  914 score:  105.0 average score 221.9 best score 232.4 epsilon 0.10 steps  478729\n",
            "episode  915 score:  185.0 average score 222.7 best score 232.4 epsilon 0.10 steps  479194\n",
            "episode  916 score:  325.0 average score 224.7 best score 232.4 epsilon 0.10 steps  480050\n",
            "episode  917 score:  270.0 average score 224.8 best score 232.4 epsilon 0.10 steps  480700\n",
            "episode  918 score:  210.0 average score 225.1 best score 232.4 epsilon 0.10 steps  481120\n",
            "episode  919 score:  150.0 average score 225.5 best score 232.4 epsilon 0.10 steps  481510\n",
            "episode  920 score:  155.0 average score 224.7 best score 232.4 epsilon 0.10 steps  481981\n",
            "episode  921 score:  575.0 average score 229.4 best score 232.4 epsilon 0.10 steps  482873\n",
            "episode  922 score:  120.0 average score 228.2 best score 232.4 epsilon 0.10 steps  483297\n",
            "episode  923 score:  300.0 average score 227.1 best score 232.4 epsilon 0.10 steps  484017\n",
            "episode  924 score:  360.0 average score 228.2 best score 232.4 epsilon 0.10 steps  484774\n",
            "episode  925 score:  110.0 average score 226.9 best score 232.4 epsilon 0.10 steps  485216\n",
            "episode  926 score:  270.0 average score 226.4 best score 232.4 epsilon 0.10 steps  485866\n",
            "episode  927 score:  160.0 average score 227.0 best score 232.4 epsilon 0.10 steps  486233\n",
            "episode  928 score:  415.0 average score 228.1 best score 232.4 epsilon 0.10 steps  486769\n",
            "episode  929 score:  315.0 average score 230.4 best score 232.4 epsilon 0.10 steps  487454\n",
            "episode  930 score:  380.0 average score 230.3 best score 232.4 epsilon 0.10 steps  488322\n",
            "episode  931 score:  605.0 average score 234.2 best score 232.4 epsilon 0.10 steps  489091\n",
            "... saving checkpoint ...\n",
            "... saving checkpoint ...\n",
            "episode  932 score:  210.0 average score 235.4 best score 234.2 epsilon 0.10 steps  489521\n",
            "... saving checkpoint ...\n",
            "... saving checkpoint ...\n",
            "episode  933 score:  185.0 average score 236.2 best score 235.4 epsilon 0.10 steps  489986\n",
            "... saving checkpoint ...\n",
            "... saving checkpoint ...\n",
            "episode  934 score:  70.0 average score 234.0 best score 236.2 epsilon 0.10 steps  490254\n",
            "episode  935 score:  95.0 average score 232.8 best score 236.2 epsilon 0.10 steps  490528\n",
            "episode  936 score:  260.0 average score 234.3 best score 236.2 epsilon 0.10 steps  491200\n",
            "episode  937 score:  135.0 average score 232.2 best score 236.2 epsilon 0.10 steps  491688\n",
            "episode  938 score:  395.0 average score 233.6 best score 236.2 epsilon 0.10 steps  492594\n",
            "episode  939 score:  235.0 average score 234.4 best score 236.2 epsilon 0.10 steps  493283\n",
            "episode  940 score:  135.0 average score 233.5 best score 236.2 epsilon 0.10 steps  493831\n",
            "episode  941 score:  185.0 average score 232.7 best score 236.2 epsilon 0.10 steps  494360\n",
            "episode  942 score:  165.0 average score 230.5 best score 236.2 epsilon 0.10 steps  494727\n",
            "episode  943 score:  210.0 average score 230.8 best score 236.2 epsilon 0.10 steps  495178\n",
            "episode  944 score:  310.0 average score 232.3 best score 236.2 epsilon 0.10 steps  495878\n",
            "episode  945 score:  350.0 average score 233.3 best score 236.2 epsilon 0.10 steps  496607\n",
            "episode  946 score:  260.0 average score 234.8 best score 236.2 epsilon 0.10 steps  497595\n",
            "episode  947 score:  140.0 average score 233.5 best score 236.2 epsilon 0.10 steps  498108\n",
            "episode  948 score:  350.0 average score 234.4 best score 236.2 epsilon 0.10 steps  498907\n",
            "episode  949 score:  255.0 average score 235.3 best score 236.2 epsilon 0.10 steps  499620\n",
            "episode  950 score:  90.0 average score 234.7 best score 236.2 epsilon 0.10 steps  500091\n",
            "episode  951 score:  185.0 average score 235.1 best score 236.2 epsilon 0.10 steps  500568\n",
            "episode  952 score:  245.0 average score 234.9 best score 236.2 epsilon 0.10 steps  501152\n",
            "episode  953 score:  285.0 average score 236.6 best score 236.2 epsilon 0.10 steps  501806\n",
            "... saving checkpoint ...\n",
            "... saving checkpoint ...\n",
            "episode  954 score:  165.0 average score 235.4 best score 236.6 epsilon 0.10 steps  502443\n",
            "episode  955 score:  320.0 average score 237.1 best score 236.6 epsilon 0.10 steps  503261\n",
            "... saving checkpoint ...\n",
            "... saving checkpoint ...\n",
            "episode  956 score:  255.0 average score 238.6 best score 237.1 epsilon 0.10 steps  503942\n",
            "... saving checkpoint ...\n",
            "... saving checkpoint ...\n",
            "episode  957 score:  460.0 average score 240.6 best score 238.6 epsilon 0.10 steps  504773\n",
            "... saving checkpoint ...\n",
            "... saving checkpoint ...\n",
            "episode  958 score:  155.0 average score 240.4 best score 240.6 epsilon 0.10 steps  505170\n",
            "episode  959 score:  590.0 average score 244.9 best score 240.6 epsilon 0.10 steps  505972\n",
            "... saving checkpoint ...\n",
            "... saving checkpoint ...\n",
            "episode  960 score:  190.0 average score 244.4 best score 244.9 epsilon 0.10 steps  506599\n",
            "episode  961 score:  115.0 average score 242.3 best score 244.9 epsilon 0.10 steps  507047\n",
            "episode  962 score:  180.0 average score 242.3 best score 244.9 epsilon 0.10 steps  507481\n",
            "episode  963 score:  150.0 average score 241.9 best score 244.9 epsilon 0.10 steps  508001\n",
            "episode  964 score:  220.0 average score 242.8 best score 244.9 epsilon 0.10 steps  508474\n",
            "episode  965 score:  105.0 average score 241.7 best score 244.9 epsilon 0.10 steps  508831\n",
            "episode  966 score:  180.0 average score 240.8 best score 244.9 epsilon 0.10 steps  509315\n",
            "episode  967 score:  105.0 average score 240.8 best score 244.9 epsilon 0.10 steps  509589\n",
            "episode  968 score:  390.0 average score 242.6 best score 244.9 epsilon 0.10 steps  510468\n",
            "episode  969 score:  475.0 average score 245.6 best score 244.9 epsilon 0.10 steps  511262\n",
            "... saving checkpoint ...\n",
            "... saving checkpoint ...\n",
            "episode  970 score:  75.0 average score 244.0 best score 245.6 epsilon 0.10 steps  511689\n",
            "episode  971 score:  135.0 average score 243.0 best score 245.6 epsilon 0.10 steps  512012\n",
            "episode  972 score:  150.0 average score 240.8 best score 245.6 epsilon 0.10 steps  512474\n",
            "episode  973 score:  290.0 average score 241.3 best score 245.6 epsilon 0.10 steps  512936\n",
            "episode  974 score:  325.0 average score 242.8 best score 245.6 epsilon 0.10 steps  513786\n",
            "episode  975 score:  110.0 average score 242.7 best score 245.6 epsilon 0.10 steps  514137\n",
            "episode  976 score:  210.0 average score 241.2 best score 245.6 epsilon 0.10 steps  514717\n",
            "episode  977 score:  125.0 average score 240.9 best score 245.6 epsilon 0.10 steps  515091\n",
            "episode  978 score:  145.0 average score 239.8 best score 245.6 epsilon 0.10 steps  515479\n",
            "episode  979 score:  240.0 average score 239.7 best score 245.6 epsilon 0.10 steps  516000\n",
            "episode  980 score:  150.0 average score 236.1 best score 245.6 epsilon 0.10 steps  516374\n",
            "episode  981 score:  630.0 average score 239.9 best score 245.6 epsilon 0.10 steps  517220\n",
            "episode  982 score:  75.0 average score 237.8 best score 245.6 epsilon 0.10 steps  517526\n",
            "episode  983 score:  155.0 average score 237.1 best score 245.6 epsilon 0.10 steps  517898\n",
            "episode  984 score:  155.0 average score 236.9 best score 245.6 epsilon 0.10 steps  518293\n",
            "episode  985 score:  90.0 average score 236.3 best score 245.6 epsilon 0.10 steps  518602\n",
            "episode  986 score:  180.0 average score 236.0 best score 245.6 epsilon 0.10 steps  518944\n",
            "episode  987 score:  55.0 average score 234.8 best score 245.6 epsilon 0.10 steps  519195\n",
            "episode  988 score:  115.0 average score 234.8 best score 245.6 epsilon 0.10 steps  519567\n",
            "episode  989 score:  100.0 average score 234.5 best score 245.6 epsilon 0.10 steps  519935\n",
            "episode  990 score:  195.0 average score 234.1 best score 245.6 epsilon 0.10 steps  520311\n",
            "episode  991 score:  210.0 average score 232.5 best score 245.6 epsilon 0.10 steps  520943\n",
            "episode  992 score:  65.0 average score 231.1 best score 245.6 epsilon 0.10 steps  521241\n",
            "episode  993 score:  170.0 average score 226.9 best score 245.6 epsilon 0.10 steps  521771\n",
            "episode  994 score:  255.0 average score 226.9 best score 245.6 epsilon 0.10 steps  522416\n",
            "episode  995 score:  130.0 average score 223.0 best score 245.6 epsilon 0.10 steps  522738\n",
            "episode  996 score:  280.0 average score 223.8 best score 245.6 epsilon 0.10 steps  523375\n",
            "episode  997 score:  360.0 average score 225.8 best score 245.6 epsilon 0.10 steps  524110\n",
            "episode  998 score:  200.0 average score 225.8 best score 245.6 epsilon 0.10 steps  524700\n",
            "episode  999 score:  95.0 average score 225.9 best score 245.6 epsilon 0.10 steps  525002\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "O7LWS39OJFeq"
      },
      "source": [
        "## Plot and save learning curve"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Wg3r121bvV7D",
        "outputId": "c90f21ad-0d2f-4fa2-d816-0d351a41fc96",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 279
        }
      },
      "source": [
        "plot_learning_curve(steps_array, scores, eps_history, figure_file)"
      ],
      "execution_count": 7,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAaoAAAEGCAYAAAA0UdFjAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3de3xU5Z348c83IYFwcSAZLwjohAS1aFOF1Ath17Z2rToo1d0fq3tRW3+ltrjW4m6N6++Fs+y6O7aKLVut4mq121abrmiRaKulVVdAFG8pqGiAUcALBiQqIoHk+f1xzsAQ5sycuZy5ne/79ZpXZs6cM/MMxvnmeZ7v833EGINSSilVqqqK3QCllFIqFQ1USimlSpoGKqWUUiVNA5VSSqmSpoFKKaVUSRtS7AZkKhgMmlAoVOxmKKVUWXn++ed7jDGHFrsd2Si7QBUKhVi9enWxm6GUUmVFRN4sdhuypUN/SimlSpoGKqWUUiVNA5VSSqmSpoFKKaVUSfMsUInI3SKyVUTWODwvIrJQRLpFpEtEpnjVFqWUUuXLyx7VPcBZKZ4/G5hk32YDP/GwLUopVbq6OuCWEyAy2vrZ1VHsFpUUz9LTjTFPiUgoxSkzgZ8Zq3z7MyIyWkTGGmPe8apNSilVMro6YNl86N0ECGDvZNG7CR6+0rrfMqtYrSspxVxHNQ7YlPB4s33soEAVau+cjdXrompnX1Zv9vybH/D7V9/je185FhHJ6jWUUiovujrgN3OgP/59Nmi7pT27rCCmgQookwW/sWh4EbAIoPX312e1gdbat3v5yRPrufDzEzi6YURe26eUUhl59JqEIOWgd1Pq532kmFl/W4AJCY/H28c80dYcBGB59zav3kIppdzZtd3deTpXBRQ3UC0BLraz/04Fer2cn5oYHMERhwxj+foer95CKaXy6zdzNFjhbXr6fcBK4FgR2Swil4nI5SJyuX3KI8AGoBu4E/i2V22x28O05gZWdPcwMJDV6KFSShVWf581V+VzXmb9XZTmeQPM8er9k5neHGTxC1t49d0POf7IQCHfWimlLJn2kAo0VyUiE4CfAYdjZXcsMsb8KOH5q4GbgEONMT1iZaX9CDgH+AS41Bjzghdt81Vlivg81Qqdp1JKFUs2PaTCDP/tBa42xkwGTgXmiMhk2BfEzgTeSji/YGthfRWoDj9kGE2HjtB5KqVU8WTTQyrA8J8x5p14j8gY8xHwKtaSIYBbgO9xYB79vrWwxphngNEiMtaLtpVFeno+tTUH+fXqzfTtHaB2iK/itFKqFEg1mP7MrsnD8N/Vp9UGiQQSN/NbRKR3UbJz7WINJwGrRGQmsMUY8/KgNaiu18LmypeB6mcr3+SlTTs4ubG+2M1RSvlNpkEKsCpX5ObmlX09N63Y3Zr2nURGAg8AV2ENB/4z1rBf0fiuS3HqxAaqBJZ36/CfUqrAujrILugUJlNZRGqwgtQvjDGLgSagEXhZRGJY611fEJEjKOBaWN8FqkBdDZ8dF9BApZQqvGXzyTroeJxQYWfx3QW8aoxZAGCM+ZMx5jBjTMgYE8Ia3ptijHmXAq6F9V2gAmv476VNO9i5e2+xm6KU8pOUc01Cyt7Wo9fkuzWDtQF/D3xJRF6yb+ekOL9ga2F9N0cFVqC67Yn1PLtxO1887rBiN0cpVUmWzoXn77HmoqQapl4KMxbAj09JfV1kh9VrWvyN5M+7LbuUJWPM06QZl7R7VfH7BVsL68se1dSjx1A7pEqH/5RS+bV0Lqy+a3/ChOm3Ht90HPS8lv56rZaelC8D1bCaalqPHsPTGqiUUvm0+u7kxz9OM3Uj1e5ef+nczNpTIXwZqMAa/nvt3Y/o+Xh3sZuilKoYWSZKTL3U3XlOgbDC+TpQAaxcr+WUlCopftuWPXicNYflij8Lavs2UH12XIBRw4awQsspKVUaujrgxkYrmaB3E2Csnw9dXh7BKts2XrHqwMeBCcnP8zHfBqrqKuG0iQ06T6VUKejqgIevTJ7ZNtBv7ctU6rKpx9d62cHHzpiXe1sqjG8DFVjDf5u272LT9k+K3RSl/G3ZfNizy/n5/r7STyTo3ZzZ+Y2nJx/ya5llDQcmI1Xl0bvMM58HqgZAyykpVXRuvuRX31XaX9KB8Zmdf8kS5+f27Ex+3Az4ciNFXweqpkNHctiooTr8p1Sx1Q53d96j15RmosXSuZlVOE83D5UqcGfac6sAvqxMESciTG8O8uTr7zMwYKiqyr1CsVIqC30OPYjBdm3fP4/Vu8ma14LiLpT98SnuFvMmSjcPFRjvHPgy7blVAF/3qACmNQfZtrOPde99VOymKOVPucw97dlV3KGwpXMzD1J19ekD6xnzoKrm4ONS7ctkC98HKp2nUqrInr8nt+vzsKlg1jJte00dnH1j+vNaZsGUiw8+XuWygkWF8X2gGhuoY2JwhAYqpYolq40EByn0XFV8UbKbtks1INa81LkL3Q9TvvHYwcf6+3yZTOHrOaq4tuYgi1/YzJ7+AWqqfR+7lSosqbKy2XKxbH7h5qm6Oqx1Xf197s4///bs2ubUUyxmD7JI9FsZa/hvZ18/L2/aUeymKOUvS+fmHqSgsJlwj17jPkhB9gE0VaHae8/L7jXLlAYqrO3pRdA0daUKqavDWhuVD4XMhPN4X6h9Ug0rbnyydFLzC0ADFTB6eC2fHRdgRbcWqFWqYPI51zLpTOfnilnkNpe6femu9dFclQYq27SmIC9u+oBP+nR7eqUKIu1wXQZfT8kSD5yK3D58ZW7Bqq7e3Xk1dbmlkqe71kcLfzVQ2dqaG9jTb3h2Y4G69Ur5XbrhugvugJoR7l5rcIJBVwc8eHnyYbpc1165SS+X6swy/JJpmQVVtc7P+2jhrwYqW+vR9dRW6/b0ShVMqh5DfFHsdW8nrzCezsNXpZ7jyaU38tYz6c/JNtNvsIE9zs/5aOGvBipbXW01U48ew3Kdp1KqMFpmJQ9C1bUH9lpmLIAL7nQ35BYf7nMq6hqXbW+kqyP9Lrutl+UvVd6pnW6qW1QQDVQJ2pobeOWdD9m+M4PUU6VU9o469cAAVFcPM289+Eu4ZRZcszH1a3V1wEPfTp+VV12bfW9k2XzS7rLrerdeF86YZ811JXJb3SJDIjJBRP4oIq+IyFoR+Y59/Aci8pqIdInIgyIyOuGaa0WkW0TWichX8t4omwaqBNPs7el111+lCiDZZol7U+xJBVCbYs7qN1ekHiqLq6rJvjeSbsgw37vztsyy5roCE8iqukVm9gJXG2MmA6cCc0RkMvA4cIIxpgV4HbgWwH7uQuB44CzgNpFUi7+yp4EqQcu4AKOGDtHhP6UKIdlmiekSHWb80Pm5/t3u3jfdsGAqqYYMc+mppdIyC767BiI7rJ8eDfkZY94xxrxg3/8IeBUYZ4x5zBgTT4d+Boj/I8wE7jfG7DbGbAS6gZO9aJsGqgRDqqs4ZWKD9qiUKgTHEkEpei3FnpdxWq9VPTT5kGUJufq02iCRwOqE22ync0UkBJwErBr01NeBR+3744DE/4ib7WN5p7X+BmlrbuD3r77Hpu2fMKHe5WZuSin3ls5NXZHC67Rrt+ugkkm2Xgtg5GElHaQAbl7Z13PTit2t6c4TkZHAA8BVxpgPE45fhzU8+AvvWpmc9qgGadN5KqW8ky5Igfdp18efn/21Tr29Cll8KyI1WEHqF8aYxQnHLwVmAH9rjIlnk2wBEiflxtvH8s7TQCUiZ9nZIN0i0p7k+aPsLJMX7YySc7xsjxuTDhvJoaOG6jyVUl5ws39Tup5JLj0igJd/mX1liroxmR0vIyIiwF3Aq8aYBQnHzwK+B5xnjPkk4ZIlwIUiMlREGoFJwLNetM2zQGVnf9wKnA1MBi6ys0QS/T+gwxhzElb2yG1etcctEaGtyZqn2v+Hg1IqL9Lt3+QmaSzX1OxcKlPsdZmwUZ7agL8HviQiL9m3c4AfA6OAx+1jtwMYY9YCHcArwG+BOcbkY3Oxg3k5R3Uy0G2M2QAgIvdjZYm8knCOAQ6x7weAtz1sj2vTmoM89NLbrHvvI4474pD0Fyil8mPqpYV5n0z3dOrqsLb3cMoY3PVB7m0qMmPM04AkeeqRFNfcANzgWaNsXgaqZBkhpww6JwI8JiL/AIwAvpzshULtnbOB2QBVBViMG5+nWt69TQOVUoXkZrFsPqqGZ7Lcx828mo/q7hVDsZMpLgLuMcaMB84B/ltEDmpTLBpeFIuGW2PRcGv9iBRFGvNk3Og6GoMjWKF1/5QqoGR/zCfhNnGheqjzc25HqNzumeWjunvF4GWgcpMRchnWGCfGmJXAMCDoYZtcm9bUwKqN29nTn4fdR5Xyu64OmH9ompNczgm77b2MPCxFz8llUHz0GnfnlXhqernzMlA9B0wSkUYRqcVKllgy6Jy3gDMAROQzWIHqfQ/b5Fpbc5CPd++la7NuT69UTuI1+AbSDNu7LT+UrP5dMr2bU/ScjLvMv0Lt5qtS8ixQ2SU3rgB+h1WKo8MYs1ZE5ovIefZpVwPfEJGXgfuAS02JpNqdZm9Pr2nqSuXo4avS1+DLZJPBA+rfpRAYn/qch69y937puN0zS2XN0zkqY8wjxphjjDFNdnYIxph5xpgl9v1XjDFtxpjPGWNONMY4LPsuvDEjajn+yEN0fyqlctHV4a62XqaFVuP17y640yoyO1i87l6q4LdnZ/pelZs1W+emqD+o8qLYyRQlra0pyAtv6fb0SmXNTYZe4+nZz/G0zIKv3ua8VUi6103Xq3KzZkvnpzyntf5SaGsOcsdTG3gu9gGnH5NuIlgpdRA3GXqXDJ66zpCbgOQk3qtKeX0V4JBUlWuVDOWK9qhS+HzI2p5e09SVylK6DL1stpnPtwcvdx4CfPQaHINUVY0nGxiqg2mgSqGutpqTjhrNci1Qq1R2nLbFAGvIL5+74WbL9MPi2dbC3kRdHamz/r56mw77FYgGqjTamoOsfftDPtDt6ZXKnNO2GHX1uQ/5ueVqeM5YC3sTe1ap5tcCEzRIFZAGqjTamoMYAys3aJq6UhlzmqMqZG28TIbnEhf4pppf00oUBaWBKo3PjQ8wcugQTVNXKhtO218UsjZeyyz3c2GJQ31Oba8Zob2pAtNAlcaQ6ipOaaxnxXrtUSmVka4O+LT34OPxNU6FNGMB1OZpYe6QFDUElSc0ULkwrTnIxp6dbNmxq9hNUaq0dXXALSdAZDQ8+M3kJYyqaorTI5nhYmFu4nyW0/BkBWzpUW40ULkwfd+2Hzr8p5Sjrg4rOPVuAgwYh7RuN5UqvJA2OMr++awfn4JjkdwK2M233GigcuGYw0cSHDlU11Mp5WTpXFj8DefgVCpSzVVV26WYbjoOel4rTHuUK1qZwgURYVpTA8vXb8MYg4jLLQKUqnRdHfDQnPSV0RMVs5pDfN1Wsj2m+vusrL90FdN16K/gtEfl0vTmIO9/tJs3tn5c7KYoVRq6OqxeVCZBqhSqOcxYgON+VG629dDdfAtOA5VL05obAJ2nUmqfTLfJqKsvnWoOuQQbXUNVcBqoXBo/ZjhHNwzX/amUiss0KaKvhEYjcgk2pRBoPSAiE0TkjyLyioisFZHv2MfrReRxEXnD/jnGPi4islBEukWkS0SmeNU2DVQZmNYUZNWGbezV7emVylx8DkiVqr3A1caYycCpwBwRmQy0A8uMMZOAZfZjgLOBSfZtNvATrxqmgSoD05uDfLR7L11bkixiVEqlVypbu7vZJyuZCt7WwxjzjjHmBfv+R1g7s48DZgL32qfdC3zVvj8T+JmxPAOMFpGxXrRNs/4ycFqTNU+1oruHKUfpWgrlY+l2xi11bvbJGqwUEkFycPVptUEigdUJhxYR6V2U7FwRCQEnAauAw40x79hPvQscbt8fB2xKuGyzfewd8kwDVQbqR9QyeewhPN3dwxVfmlTs5ihVHF0d8NC3i92K3ATG2wuTXaqrt4JUGc9P3byyr+emFbtb050nIiOBB4CrjDEfJi7HMcYYEXFYCe0dHfrL0PRJQV54cwe7+pKUhlHKD5bNh4E92V0rJfKVc8Y8q4fkRutlcM3Gsg5SbolIDVaQ+oUxZrF9+L34kJ79c6t9fAswIeHy8faxvCuR35ryMa2pgb7+AVa/WSJj7UoVWjbDZnGlUrmiZRZMudjduU57alUYsbpOdwGvGmMSd7RcAlxi378E+E3C8Yvt7L9Tgd6EIcK80kCVoZMb66mpFk1TV/6VrtZdqjJFgQnOzxWa2wCUS2AuL23A3wNfEpGX7Ns5QBT4CxF5A/iy/RjgEWAD0A3cCXg2HqxzVBkaXjuEkyaM0YW/SiVTV29VfjjqVHj4StiTsONATV1pLZZ1G4B8UonCGPM0jiU7OCPJ+QaY42mjbNqjykJbc5A1b/ey4xPdnl75UKoU83hWXMssOHeh3YMS6+e5C0trnsdtACql4OpTGqiy0NbcgDHwjG5Pr/xm6dwUT8qBgahlFnx3DUR2WD9LKUiBu4SKuvrSa7cPaaDKwucmjGZEbTVP6/Cf8pOuDlh9d4oTCp61nJuWWVbtwRqHnX+ra8t63VQl0UCVhZrqKk6Z2MAKTahQfrJsPimDUSklSrjVMguuexsuuPPAqhN19TDzVu1NlQhNpsjStKYG/vDaVt7p3cXYQF2xm6OU99ItkC3nuZyWWRqUSpj2qLLUtm97eu1VKQXoF73yjAaqLB17+CgaRtRqmrqqfF0dcMORqc8px2E/VTZ06C9LVVXCtOYgy7t7dHt6Vbnidf3SlUwq52E/VfK0R5WDtqYGtn60m/Xvl9CGcOpAS+fCv9RDJGD9TJle7WNdHXDLCRAZbf2MV0d3U9ev8XQd9lOe0kCVg/g81dNv6PBfSVo6F1bfBcYuIGz6rcflFKycAki+3+PhK+1kCWP9XDzbCu5uKoxfsiT/bVIqgQaqHEyoH86E+jqWr9eEipL0/E8djt9T0GZkLVkAefjK/AerZfMPLHUElN2aKFXRNFDlaHpzkGd0e/rS0dUBNzZavQGnSt2mTLZoWXrVwQFkz678b+eeyb5Mg1XwjreqdHgaqETkLBFZJyLdItLucM4sEXlFRNaKyC+9bI8XpjUF+ejTvax5+8NiN0UtnQuLv+Fiu/MySHxZOhf6diZ/btf2/PaqctkjSis3qAJwlfUXau+8ALgROAzr/3IBTCwaPsTpGhGpBm4F/gJri+LnRGSJMeaVhHMmAdcCbcaYD0TksKw/SZFMs7enX97dw4kTRhe5NT7W1WHNP7lRO9zbtuRDuuHJR6/JPoGhq8Ma7sulJwXWdh6aRKEKwG16+veBc2PR8KsZvPbJQLcxZgOAiNwPzAReSTjnG8CtxpgPAIwxWw96lRLXMHIonxl7CMu7e5jzxeZiN8e/ls13f65TT6WUpBueTNtrdOA23Tyd+HYeShWA2z7/exkGKYBxQOKfbJvtY4mOAY4RkeUi8oyInJXhe5SEtqYGVr/5AZ/uKZO5j0qUyeZ2Uu1dO/LFTRuzGf57+KrcgxTokJ8qKLc9qtWh9s5fAQ8Bu+MHY9Hw4jy8/yTgC8B44CkR+awxZkfiSaH2ztnAbICqnaW3B1Rbc5D/enojq2MfMH1SsNjN8afAePdDWeWQTBGaDhufTH1OpsN/XR2wJ0+9SR3yU9mIBOqAo4j0rsvkMreB6hDgE+DMhGMGSBWotgCJdVXG28cSbQZWGWP2ABtF5HWswPVc4kmxaHgRsAig9ffXl1ze7MmN9QypEpav79FAVSz1E90HqlLPVOvqSB+kIPPhv0yGR1PJJflC+VckcC5wE1ALNBIJnAjMJ9J7XrpLXQWqWDT8tSya9RwwSUQasQLUhcDfDDrnIeAi4KciEsQaCtyQxXsV1YihQzjpqNGs0Lp/xRP7X/fn7tpupa+DFbTOvhHeesZKYDD91rDb1EuLNwezeLb7cyOjrd7kGfPS93IyGR5NZWo2XwdKEcHKXXjCetT7EpFAo5sL3Wb9jQf+E2izD/0v8J1YNOz4m2+M2SsiVwC/A6qBu40xa0VkPrDaGLPEfu5MEXkF6Af+yRhTlqtnpzUF+c8/vEHvJ3sIDE+za6jKr64O5zVT6ezabgeGhI56vIIFFD5Y3XsemS22TVgIDKmDVd2Y7JMw4lov0yQKla09RHp79/2RaHH1y+62D/9TYAlwpH172D6WkjHmEWPMMcaYJmPMDfaxeXaQwljmGmMmG2M+a4y532V7Sk5bc5ABAyt1e/rC6uqw1k7lxOH/Fbfp7vnkZsgvmT270g/tfdqb3WsD1NRZmwtqkKpYInK3iGwVkTUJx060E91eEpHVInKyfVxEZKG9RrZLRKa4eIu1RAJ/A1QTCUwiEvhPYIWbtrkNVIfGouGfxqLhvfbtHuBQl9f6wokTRjO8tpoV63X4r2DyEqRcvEe5SDVHd+952SeRBCbAuQs1gaLy3QMMzrz+PvAvxpgTgXn2Y4CzsfIJJmEluv3Exev/A3A8VkLeL4Fe4Co3DXObTLEt1N75d8B99uOLAO06JKgdUsXJjfW6P1UhuUkOqBmRW6ZbLgtrCy5JxY2uDislPdW/gVSlGDoV+O4ah+dUJTHGPCUiocGHsZLpAALA2/b9mcDPjDEGeEZERovIWGPMO0lfPBKoBjqJ9H4RuC7TtrntUX0dmAW8C7wD/BWgM6qDtDUFWf/+Tt7t/bTYTalcS+daCQRuK3sPzzHDL9c5nUwFj8vh4kFDmF0dsPib6QP1+Xc4b3wYGJ9De1Qpufq02iCRwOqEm5usnauAH4jIJqyMvWvt427Wye4X6e0HBogEAo7npOA26+9NIG0Kod9Na95fTukvp+r/4HkX37bDrbr63MsEFVJXB3yQx6TXZfMBF0km8R7jw1ceWAS3pk43RKwgN6/s67lpxe7WDC/7FvBdY8wDIjILuAv4cpZN+Bj4E5HA48D+v54ivVemuzBloAq1d/4nKbIyYtFw2jfwk88ccQj1I2pZvl4DVd5lUssv7uwb4cHLy2OBL1jDjP05Lmjv6rACT1eHuyBdVWv9jAerZfOtNHa3Ke+q0l0CfMe+/2vgv+z7btbJDraY1GtvHaXrUa3O5kX9qqpKOK2pgRXd23R7+nzq6oDfzMnsmgvutL5kc0228GJx8L6isIMCQj6GGePzdm7/vb566/77LbM0MKnB3gZOx1r79CXgDfv4EuAKu4brKUCv4/xUXKT3XiKBWqz1sgDriPS6queVMlDFouF73byI2q+tKUhn1zusf38nzYeNLHZzKkM2PY34F25gQm7Df/279/dS8iG+GWJ8iC1xDVQqweOg57X05/VutvaxcvvvpYFJ2UTkPqxydkER2Qxcj1U4/EciMgT4FLuUHfAIcA7QjVW1KH3OQiTwBeBeIIaV+TOBSOASIr1Ppbs03dDfD2PR8FWh9s6HSTIEGIuGdd5qkOn29vQr1vdooMqXTHsaiQVdz5iXW7Xwvp3W9ZCfL/Vku+nG10BV1yYPMDUj4IpV7ubo8rGoV/mSMeYih6emJjnXABkOc3AzcOa+On+RwDFYmeQHvf5g6bL+/tv+eZP9JoNvapCjGoYzfkydpqkX09RL999vmQVfvS23IbyBPfnbVdepd9e7ybkXNGSo9XPGAoj0WsOaVUmqn1TXZraFSanXPFSVpuaAYrSR3tcBV2V80g39PW//3LdcPtTeOQaYEIuGu7Jqqg+0NQV5dM079A8Yqqt0nion2Sy4HVw9YfDcSySLDNl89FJ+fEp21+364MDH8c/y0BwYSAhuYya6Gx4EK9DpVh2qsFYTCfwX8HP78d/iMg/Cba2/J7DS04cAzwNbQ+2dy2PR8NzM21r5pjU38KvVm1izpZfP6a6/ucm0J+Oml9B6WXblkdLNVXV1WO1NFtSkBkyWw4/J1jK99cyBQQrcB6nABM3oU8XwLazhwvik7P8Ct7m50O2C30AsGv4QuAD4WSwaPoXsc+kr3rQma55quZZTyl0mPZmqane9hBkLrGAVn8uSautxOg9f6dzDi++c69TebIMUJF/LlG6reieBCValCQ1SqvCGAD8i0nsBkd4LgIVYBcvTchuohoTaO8diVadYml0b/ePQUUM57ohRrOjWKlM5WZpBh72uHr56u/sv4BkL4Prt1pzP9dvdFVtNVfh12fz87JybTLLPlNXaMNEFvKqYlgF1CY/rgN+7udBtrb/5WFtyLI9Fw8+F2jsnsj+fXiUxrSnIL1a9yad7+hlWUwZbn5earg5YfXfqc/K95URdffoenNOeTl5VwGg8Pflxqc48WLV+XXtSqpiGEen9eN+jSO/HRALD3VzoqkcVi4Z/HYuGW2LR8Lfsxxti0fBfZtVUn2hrbmD33gFeePOD9CerA3V1WBUlUm1VUzMi/1tOnH0jaf+XcKp9Jx78MdJ4OlyyJPlziZmNbl9Lt+hQxbWTSGD/diCRQCuwy/n0/dwmU0wEfgScivXtsRL4biwaLrvdeAvllIkNVNvb009r1u3pD+JUnSE+15Out3DuD/PfpnhvI1U1C6ehs3yXaUoVpMAKOl33u0tHrxmR+rWUKoyrgF8TCcQrsI8F/trNhW7nqH4JdNgvfCRWzaf7Ul7hcyOHDuHECaNZrvNU+3V1wC0nWOnhi2fbw2UJO9TGs+bSzfU0nu7dEFbLrP317w5S7fy++V6T5CawuF0zteeT3NqiVC4igc8TCRxBpPc54DjgV8Ae4LfARjcv4TZQDY9Fw/+dsHHiz4FhWTXaR9qaGujavIPeXR5NspeTeE9p31zOoGG9eKKCmyy/LR6XoEysf5eo9dLkx7s6cts996D3cZGBmAndqkMV1x1AfC3FacA/A7cCHwCL3LyA22SKR0Ptne3A/VjfMH8NPBJq76wHiEXDWrMlibbmIAv/0M2qDds48/gjit2c4nLTU3KbkJBJ9YVstMyy1ikNXmsVfzx4rufRa/I79JfvuSTN9FPFVU2kNx4j/hpYRKT3AeABIoGX3LyA2x7VLOCbwB+xquh+C7gQa/GvVlh3cNJRY6irqWbFeh3+K7v6cy/fn/z46rsOXkuVzWercvgb0WkDw2y1XqaZfqrYqokE4r/wZwB/SHjOVWfJ7caJjRk2TGFtT74U4VAAABuqSURBVP/5xnqe1rp/eVaAslSpdsVdNj+3L/+RY+HM+d5vVJjv9H2lsnMf8CSRQA9Wlt//AhAJNAOuxsxT9qhC7Z3fS7j/fwY99+8ZNtaX2poa6N76Me99qNvT503r14v7/oOHKJ0SKWpGWLc4qbKCxz++ZgW6cxfaPSixfp67MLMA6NT7kiqrcK0GKVUKIr03AFcD9wDTifTGJ6irgH9w8xLpelQXAt+371+Lle0XdxbWpJhKoS1h24/zTyrgpLZT+ncxZFNY1kmheglSBSbFNu6Jdf/OvvHgrUSqaqwU+lT/5rluVHjGvOS9skwDnlJei/Q+k+TY624vTzdHJQ73kz1WSUweewhjhtcUNk29qwMWf/PA9O/F38xvwMiEU9mhTBVyKGtqmn3gHrwcIqOtdHuAKRcfWDtwysXeB4t89MqUKgPpelTG4X6yxyqJ+Pb0y7t78rs9faoe08NXAYN7AwPW8WJ8iTmVHXJLqqzAUcihrBkLUldYj2f59W46eIGw6YeXfwlHnVqYYKWBSVW4dIHqc6H2zg+xek919n3sx7qOyqW25iCP/OldNvbsZOKhedj1N9V25i2znBMBUiUIeCkwPrdaeOffUZwv49oR2afCx9eFaRBRKmfpNk7Uaqp50LZv249t2QeqxB6UVB28bidVZe9E/1Jv1YkrZO9k0pnZ7f8Ut7RIPcEZP0xdTimdXHuSSinA/ToqlYOjG4YzbnQdy9/IMk093oOKzzk5LS5NNgw1mOm3gsa/H1m4Oas3Hsvteq8X+DrJNThqRQil8kIDVQGICNOaGli5YRv9A1lM7S2bf2BmVz707Uy9EWA+uelZXHCn9+3IRmJ6eaa0IoQqIyJyt4hsFZE1g47/g4i8JiJrReT7CcevFZFuEVknIl/xsm0aqApk+qQgvbv28MrbH6Y/eTCv9jpyO1yYq3Q9i7p6u/filGhSxATTvTn8gaDzU6q83IO17GgfEfkiMBP4nDHmeOAm+/hkrOVLx9vX3CbixV43Fg1UBXJaUwNQgtvTF2IeZdKZOAabmrr928c7LeQt5gLfVGupUsl3NXWlPGaMeQoYXA/sW0DUGLPbPmerfXwmcL8xZrcxZiPQDZzsVds0UBXIYaOGcczhI1nuppzSvu0wRsMNR3rbMK/nUbo6rFTtZKsZBq/7mbHAWiuVuB6p2GWAsv0jMR58lSpvxwB/JiKrRORJEfm8fXwckDjUs9k+5gm31dNVHrQ1B7nv2bfYvbefoUMcvgAHp557nVKez3mUZGu7nObXAhPgu2sOPj5jQWmV/pl6aXYZizrsp0rM1afVBokEEouILyLSm26bjSFAPdamuZ8HOkRkoldtTNUIVSBtTUF+ujzGC2/u2DcUeBAvEicKYelcWH03+3pO8bVdTp+lXFK3ZyyAbd2w8Un31+iwnypBN6/s67lpxe7WDC/bDCw2xhjgWREZAILAFiCx2OR4+5gndOivgE6ZWG9tT59q+M+rxAknj16T2/VL51o79q6+i6SbIToNnZVT6vYlS6ysxMQAVFdv7TQ8WHWtDvupSvIQ8EUAETkGqAV6gCXAhSIyVEQagUnAs141QntUBTRqWA0t4wMsX9/DP3LswSd0dWAlHRSwOtWu7QcWWM3Eveel72k4rfmadGbm71dMTqWKSqn4r1I5EJH7gC8AQRHZDFwP3A3cbaes9wGX2L2rtSLSAbwC7AXmGJPP3UMP5GmgEpGzgB8B1cB/GWOiDuf9JfA/wOeNMRW9EeP05iC3PbGejz7dw6hhNQc+uWw+RSmhmFh+ya2ujsyGwwbLdRFwqdBae6pCGGMucnjq7xzOvwG4wbsW7efZ0J+dU38rcDYwGbjIzr0ffN4o4DvAKq/aUkqmNQXpHzCs2pBkV9hch/3q6pMMtVWlnzPJZj1VruuvCj3EqZQqW17OUZ0MdBtjNhhj+oD7sXLvB/tX4Eag+DsLJqaF33KCJ1Ubphw9mmE1Vd7s+nvNRjj/9gO3fbjgDuv4BXda8ydOMg0cOVdE1zKSSil3vBz6S5Znf0riCSIyBZhgjOkUkX9yeqFQe+dsYDZA1c4+D5rKwfMtvZuszfAgr0M7Q4dU8/lQPSsGL/zNV1B0GopqmQUv/tx5uC7TwJFrRXTvhrOVUhWmaFl/IlIFLMDaojilWDS8KBYNt8ai4db6ESl6BdlaOjf5F/jAntyz4pJoaw7y+nsfs/WjhE5krkNp6Yb3ujpg41POz2caOOpzXErhtI26UkoN4mWgSpdnPwo4AXhCRGJYC8qWiEimef65e/4e5+d22XNJeRwWjG/7sXJ9wq6/ufROqmrSp0SnTdTIoJ5euqDnhhZsVUq55GWgeg6YJCKNIlKLVcBwSfxJY0yvMSZojAkZY0LAM8B5Rcn6S9ebGLzNRnwxa5bBavKRhxCoq+Hp+LYf956X2QtccOeB81BfvS398GTaOSXjPgDnIztRM+WUUi55NkdljNkrIlcAv8NKT7/bGLNWROYDq40xS1K/QgFJdepglaxaRA47uFZXWdt+rFi/zdqePtOqB9mkRLuZU4rvZ/XWM6nLGOWaSNF6WW7XK6V8xdM5KmPMI8aYY4wxTXbOPcaYecmClDHmC0VbQzX10tTPO33B5zBcN605yJYdu+hZ8XP3F7kZ4nOSyVDb6rtS96zqxmTXBih+kVmlVNnREkpgfXEGj8v8uhxSrNvsWn+jnvh/bt7I/RCfk5ZZyUv+OBmc3JE4R7cryRqwdKqqrSFLDVJKqQxpoIq7Iov1xqY/68SKxuAIxgaGMXTPjvQnR3ZYlcZznde5ZIn7YbfE4b2uDnjo8v1zdNmoHqbzUkqprGigSuSY4p0iI653E/xmTsbBSkRoaw7m0KYsue3RJBaNXXoVDOS47snr7UqUUhVLA5UraXoR/X2Zrbeyh9F+sPb01C+dy5xUKjUj0p+TOKfVp0FGKVU8GqgS7fogh2tdztt0dVg9sN5NCAZx7KxJbnNSqZz7w/TnvPWM9TOTnmKq3p/u0aSUypI/A5XT4t1C7JH06DVWDyydmuHezem4ed3Vd1n7TC2e7e416+rt3l+SX6mqat2jSSmVNf8FqlSLdwuxR5LbnpfXczri9j+9y+SJs2+0AuAFdxy8weBXb9dECqVU1vy3cWKqxbu5+tfDYOaPy+NLeerX7F158yT+mXV/JqVUnvkvUDlVVci12gJA/+6DKzt0dVjDfZmuPfJ6TifevnwGK6WU8oD/ApVTKaH4/FQ+NvRbfRccdap1/8HLs9vSohBzOvkKVpoooZTykP/mqJzmoXZtz++us49eY92yCVI1Iwo3fDZjARlVTk9GEyWUUh7yT4+qq8Oah3IKRvleK5RNmaE4N+njeZVDJfRCBlWllC/5o0d1QKZfiYtXRy+kXDYx3PNJ/tqhlFJJ+CNQJcv0K0EDUJxhtDPmWVUwslGItWdKKc+JyN0islVE1iR57moRMSIStB+LiCwUkW4R6RKRKV62zR+BKh8ZfR4zwH/v/TLvN84s/Ju3zLKqYLheW2WrqdOdepWqHPcAZw0+KCITgDOBtxIOnw1Msm+zgZ942TB/BKoy+Kv/rdN/xPV7v86K9T3FaUDLLDj/Div4HMBOtAhMsCqvJ+4sfO5CnZ9SqkIYY54Ckk2u3wJ8jwMns2cCPzOWZ4DRIjLWq7b5I5nijHnWHJWb4b+aEYWv9C3VjD/9Eg558jFWdG9j5onjCvv+cfGgs2y+1QsNjLf+7TQYKVX2rj6tNkgkkLg57SIivYtSXSMiM4EtxpiX5cDCpOOAxEn/zfaxd/LV3kT+CFQHfAGnSai47m1YOrewC2GnXkp1lXBaUwNPd/dY29M7V6v1llaWUKoi3byyr+emFbtb3Z4vIsOBf8Ya9isqfwz9gfXl+9011i6zqSydm37PpsCE/C1yrard935t9vb0b23XTDqlVNE1AY3AyyISA8YDL4jIEcAWIDFdeLx9zBP+CVRxLbNS73K7+m4rnT1VYsEZ86zsvOra3NszsGff3fhGisu7t+X+ukoplQNjzJ+MMYcZY0LGmBDW8N4UY8y7wBLgYjv771Sg1xjjybAf+DFQQZoek7GGCKd+LfnTjafvHx6beev+5IJsJSR6TAyO4IhDhrG8WAkVSinfEpH7gJXAsSKyWURS/EXPI8AGoBu4E/i2l23zxxxVMoEJzvNVvZv2B7Pn77HKIEk1TL30wCCXOJ+T7bxWQnq3iDCtuYE/vraVgQFDVVWR5qmUUr5jjLkozfOhhPsGmON1m+L8E6gSSyhJdZoafGKdP2NB+vmquBkLILYcel5z36bWyw5KXGhrCrL4hS28+u6HHH9kwP1rKaVUhfLH0N/gEkppC8Wa7PanumJV8vmvxtOtJI7ENUgX3Jk0CMbnqVboPJVSSgF+6VFlU0Ip22oWqXphLtK+jwgMo+nQESxf38M3/nxidm1QSqkK4o8eVTZBp4jVLNqag6zasJ2+vQNFa4NSSpUKfwSqbIKO075VBTCtKciuPf28tGlH0dqglFKlwh+B6ox5SWrYpfHGY960xYXTJjZQJbC8W9PUlVLKH4GqZZZVQDW+75JUp7+miBXXA8Nr+Oy4gAYqpZTCL8kUkLyGXVcHPHh58izAIldcn9Yc5M6nNrBz915GDPXPfyallBrMHz0qJy2z4PzbDx4WLIF9lqY3B9k7YHh2Yw5b2iulVAXwd6CCQcOCpbPP0tSjx1A7pEqH/5RSvqdjSlCSW1sMq6mm9egxPK2BSinlc9qjKmFtzUFee/cjej7eXeymKKVU0WigKmHxckor12s5JaWUf3kaqETkLBFZJyLdItKe5Pm5IvKKiHSJyDIROdrL9pSbz44LMGrYEFboth9KKR/zLFCJSDVwK3A2MBm4SEQmDzrtRaDVGNMC/A/wfa/aU46qq4RTJzboPJVSyte87FGdDHQbYzYYY/qA+4GZiScYY/5ojInvu/4M1nbGKsH05iCbtu9ik25Pr5TyKS+z/sYBiTsTbgZOSXH+ZcCjyZ4ItXfOBmYDVO3sy1f7ykJbcwNglVO68OSjitwapZQqvJJITxeRvwNagdOTPR+LhhcBiwBaf3+9KWDTiq7p0JEcNmooT2ugUkr5lJdDf1uACQmPx9vHDiAiXwauA84zxmge9iAiQltzkJXrtzEw4KsYrZRSgLeB6jlgkog0ikgtcCGwJPEEETkJuAMrSG31sC1lra05yLadfax776NiN0UppQrOs0BljNkLXAH8DngV6DDGrBWR+SJynn3aD4CRwK9F5CURWeLwcrnp6oAbGyESsG43NlrHykTiPJVSSnlBRO4Wka0isibh2A9E5DV7CdGDIjI64blr7aVH60TkK162zdN1VMaYR4wxxxhjmowxN9jH5hljltj3v2yMOdwYc6J9Oy/1K2ahqwMe+jbsSijuums7/GZO2QSrsYE6JgZHaKBSSnnpHuCsQcceB06wlxC9DlwLYC81uhA43r7mNntJkicqvzLFsvkwsOfg4/191hYfZRKs2pqDPLtxO3v6dXt6pVT+GWOeArYPOvaYPToGBy4hmgncb4zZbYzZCHRjLUnyROUHqlQbIJp+ePjKsghWbc0N7Ozr52Xdnl4plYWrT6sNEgmsTrjNzvAlvs7+JUTJlh+Ny0c7kymJ9HRPBcZD7ybn5/fssnpdJVY9fbBTJzYgAk9399Aaqi92c5RSZebmlX09N63Y3ZrNtSJyHbAX+EV+W+VO5feo3GyAWMRt590aPbyWE44MsKJbC9QqpQpHRC4FZgB/a4yJr5FxtfwoXyo/ULnpKRV523m32pqDvLjpAz7p25v+ZKWUypGInAV8D2sJUWIdtyXAhSIyVEQagUnAs161o/IDFdi79zqori36tvNutTU3sKdft6dXSuWfiNwHrASOFZHNInIZ8GNgFPC4vYTodgBjzFqgA3gF+C0wxxjT71XbKn+OCqxAtHg2kKSyQ+3Ikp+fims9up7aamt7+i8ce1ixm6OUqiDGmIuSHL4rxfk3ADd416L9/NGjaplF0iAFsOuDgjYlF3W11Uw5ejTLdZ5KKeUj/ghU4Dz8VybzU3HTm4O88s6HbPdZFXmllH/5J1BNOjP58fqJhW1HjqbZ29Prrr9KKb/wT6Ba+2Dy4xufLIsFv3Et4wKMGjpEh/+UUr7hn0C1K0Wm3LL5hWtHjoZUV3HKxAbtUSmlfMM/gSqVMljwm6ituYE3t32i29MrpXzBP4GqLkXZoTJLqGjTeSqllI/4J1Adf37y41XVZbPgN27SYSM5dNRQnadSSvmCPxb8ArzxWPLjQwNls+A3TkRoa2rgkT+9y5dufoIqEaTYjVJKee7KMyZx7ueOLHYzCs4/gcppHqqMFvwm+r9/ZqXV7xkw7K8TqZSqZIG6mmI3oSj8E6ictvsos/mpuBPGBfjhhScVuxlKKeU5/8xRnTEPauoOPFZTV3bzU0op5Tf+CVQts+DchXYpJbF+nruw7OanlFLKb/wz9AdWUNLApJRSZcU/PSqllFJlSQOVUkqpkqaBSimlVEnTQKWUUqqkaaBSSilV0qTcqhqIyPvAm9lcWzV8dHDgkx0VX8lVP2dl0c9ZWYr4OY82xhxahPfNnTHGN7ejr1m6utht0M+pn1M/p37OYreh3G469KeUUqqkaaBSSilV0vwWqBYVuwEFop+zsujnrCx++Zx5U3bJFEoppfzFbz0qpZRSZUYDlVJKqZLmi0AlImeJyDoR6RaR9mK3x4mI3C0iW0VkTcKxehF5XETesH+OsY+LiCy0P1OXiExJuOYS+/w3ROSShONTReRP9jULRURSvYeHn3OCiPxRRF4RkbUi8p1K/KwiMkxEnhWRl+3P+S/28UYRWWW37VciUmsfH2o/7rafDyW81rX28XUi8pWE40l/t53ew0siUi0iL4rI0kr9nCISs3+vXhKR1faxivq9LUnFzo/3+gZUA+uBiUAt8DIwudjtcmjrnwNTgDUJx74PtNv324Eb7fvnAI8CApwKrLKP1wMb7J9j7Ptj7Oeetc8V+9qzU72Hh59zLDDFvj8KeB2YXGmf1X7vkfb9GmCV3aYO4EL7+O3At+z73wZut+9fCPzKvj/Z/r0dCjTav8/VqX63nd7D4/+uc4FfAktTtaGcPycQA4KDjlXU720p3oreAM8/IJwG/C7h8bXAtcVuV4r2hjgwUK0Dxtr3xwLr7Pt3ABcNPg+4CLgj4fgd9rGxwGsJx/ed5/QeBfzMvwH+opI/KzAceAE4BegBhgz+/QR+B5xm3x9inyeDf2fj5zn9btvXJH0PDz/feGAZ8CVgaao2lPnnjHFwoKrY39tSuflh6G8csCnh8Wb7WLk43Bjzjn3/XeBw+77T50p1fHOS46new3P2sM9JWL2Nivus9nDYS8BW4HGsnsEOY8zeJG3b93ns53uBBjL//A0p3sMrPwS+BwzYj1O1oZw/pwEeE5HnRWS2fazifm9Ljb92+C1zxhgjIp6uJyjEe8SJyEjgAeAqY8yH9nB8wdpRoPfoB04UkdHAg8BxXr5fMYjIDGCrMeZ5EflCsdvjsenGmC0ichjwuIi8lvhkpfzelho/9Ki2ABMSHo+3j5WL90RkLID9c6t93OlzpTo+PsnxVO/hGRGpwQpSvzDGLE7TjrL+rADGmB3AH7GGp0aLSPyPxMS27fs89vMBYBuZf/5tKd7DC23AeSISA+7HGv77UYo2lOvnxBizxf65FesPj5Op4N/bUuGHQPUcMMnODqrFmrxdUuQ2ZWIJEM8KugRrPid+/GI7s+hUoNceGvgdcKaIjLEzg87EGrd/B/hQRE61M4kuHvRayd7DE/b73wW8aoxZkPBURX1WETnU7kkhInVY83CvYgWsv3L4nPG2/RXwB2NNSiwBLrSz5RqBSViT7kl/t+1rnN4j74wx1xpjxhtjQnYb/mCM+dtK+5wiMkJERsXvY/2+raHCfm9LUrEnyQpxw8q+eR1rfuC6YrcnRTvvA94B9mCNT1+GNQ6/DHgD+D1Qb58rwK32Z/oT0JrwOl8Huu3b1xKOt2L9j7Ue+DH7K5MkfQ8PP+d0rLH+LuAl+3ZOpX1WoAV40f6ca4B59vGJWF/A3cCvgaH28WH24277+YkJr3Wd/VnWYWeCpfrddnqPAvwOf4H9WX8V9Tnt93rZvq2Nt6PSfm9L8aYllJRSSpU0Pwz9KaWUKmMaqJRSSpU0DVRKKaVKmgYqpZRSJU0DlVJKqZKmlSlUWQu1d8bTdgGOAPqB9+3HJ8ei4b4U17YCF8ei4SvTvMeKWDQ8LQ9tHQ7ciZW2LsAO4Cys/w//JhYN35breyhViTQ9XVWMUHtnBPg4Fg3flHBsSCwa3ut8VeGE2juvBQ6NRcNz7cfHYhU5HQssjUXDJxSxeUqVLO1RqYoTau+8B/gUq9jt8lB75/1YJX2GAbuAr8Wi4XWh9s4vAP8Yi4Zn2EHuKKxFnUcBP4xFwwvt1/s4Fg2PtM+PYFXsPgF4Hvi7WDRsQu2d5wALgJ3AcmBiLBqeMahpY4E34w9i0fA6+/WjQFOovfMl4PFYNPxPofbOfwJmYW158WAsGr4+1N4ZAn5rv+8UrEWnF8ei4U/s1zgP2As8FouG/zH3f0mlSoPOUalKNR6YZvdeXgP+LBYNnwTMA/7d4ZrjgK9g1W+7PtTeWZPknJOAq7D2TpoItIXaO4dhbdVwdiwangoc6vD6dwPXhNo7V4baO/8t1N45yT7eDqyPRcMn2kHqTKzyQScDJwJTQ+2df26feyxwWywa/gzwIfBte/jzfOD4WDTcAvxb+n8epcqHBipVqX4di4b77fsB4Neh9s41wC3A8Q7XdMai4d2xaLgHq+hnsq0Uno1Fw5tj0fAAVumnEFaA2xCLhjfa59yX7MVj0fBLWMHtB1ib5j0Xau/8TJJTz7RvL2LtYXUcVuAC2BSLhpfb93+OVY6qF6sHeVeovfMC4BOHz6dUWdKhP1Wpdibc/1fgj7Fo+Hx7+OwJh2t2J9zvJ/n/H27OcRSLhj8GFgOLQ+2dA1g17B4YdJoA/xGLhu9IPGi3ffCksolFw3tD7Z0nA2dgFWi9AquCuVIVQXtUyg8C7N8u4VIPXn8dMNEOJAB/neykUHtnW6i9c4x9vxZr+PBN4CNgVMKpvwO+HmrvHGmfOy7U3nmY/dxRofbO0+z7fwM8bZ8XiEXDjwDfBT6Xt0+mVAnQQKX84PvAf4TaO1/Eg1GEWDS8C/g28NtQe+fzWIGnN8mpTcCTofbOP2EN660GHohFw9uwkj7WhNo7fxCLhh8DfgmstM/9H/YHsnXAnFB756vAGOAn9nNLQ+2dXcDTwNx8f0aliknT05XKg1B758hYNPxxqL0zvrXDG7Fo+JY8v0cITWNXPqQ9KqXy4xt2evlarKHGO9Kcr5RySXtUSimlSpr2qJRSSpU0DVRKKaVKmgYqpZRSJU0DlVJKqZKmgUoppVRJ+/+15pmiiLOebAAAAABJRU5ErkJggg==\n",
            "text/plain": [
              "<Figure size 432x288 with 2 Axes>"
            ]
          },
          "metadata": {
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "mQ70iR9DCq_n"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}