car_racing_dqn.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "a7961838-ad31-40a8-96b4-7a0f9d2b13f0",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2024-01-17 15:06:47.282166: I tensorflow/core/util/port.cc:113] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`.\n",
      "2024-01-17 15:06:47.305413: E external/local_xla/xla/stream_executor/cuda/cuda_dnn.cc:9261] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered\n",
      "2024-01-17 15:06:47.305442: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:607] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered\n",
      "2024-01-17 15:06:47.306078: E external/local_xla/xla/stream_executor/cuda/cuda_blas.cc:1515] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered\n",
      "2024-01-17 15:06:47.310471: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations.\n",
      "To enable the following instructions: AVX2 AVX_VNNI FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags.\n"
     ]
    }
   ],
   "source": [
    "import gymnasium as gym\n",
    "from stable_baselines3.common.env_util import make_vec_env\n",
    "from stable_baselines3.common.vec_env import DummyVecEnv, SubprocVecEnv\n",
    "from stable_baselines3.common.utils import set_random_seed\n",
    "from stable_baselines3 import DQN\n",
    "from stable_baselines3.common.evaluation import evaluate_policy\n",
    "import matplotlib.pyplot as plt\n",
    "from gymnasium.wrappers import RecordVideo\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "b11cb87f-120b-45b7-9fcd-904c523de8d7",
   "metadata": {},
   "outputs": [],
   "source": [
    "def make_env(env_id):\n",
    "    def _init():\n",
    "        env = gym.make(env_id, max_episode_steps=800, continuous=False)\n",
    "        env.reset()\n",
    "        return env\n",
    "    return _init"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "e76fb163-4520-414b-a1b3-c78163eb9b80",
   "metadata": {},
   "outputs": [],
   "source": [
    "n_procs = 16\n",
    "env_id = \"CarRacing-v2\"\n",
    "TRAIN_STEPS = 2000000\n",
    "EVAL_EPS = 20"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "138e8483-72d7-49ed-b25b-ae7506f3a753",
   "metadata": {},
   "outputs": [],
   "source": [
    "train_env = SubprocVecEnv([make_env(env_id) for _ in range(n_procs)],start_method=\"fork\")\n",
    "eval_env = gym.make(env_id, max_episode_steps=3200, continuous=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "10d69dad-58f7-40c0-be57-a11f4054c02f",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Using cuda device\n",
      "Wrapping the env in a VecTransposeImage.\n",
      "Logging to tensorboard_logs/first_run_1\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.939    |\n",
      "| time/               |          |\n",
      "|    episodes         | 4        |\n",
      "|    fps              | 949      |\n",
      "|    time_elapsed     | 13       |\n",
      "|    total_timesteps  | 12800    |\n",
      "----------------------------------\n",
      "------------------------------\n",
      "| time/              |       |\n",
      "|    episodes        | 8     |\n",
      "|    fps             | 949   |\n",
      "|    time_elapsed    | 13    |\n",
      "|    total_timesteps | 12800 |\n",
      "------------------------------\n",
      "------------------------------\n",
      "| time/              |       |\n",
      "|    episodes        | 12    |\n",
      "|    fps             | 949   |\n",
      "|    time_elapsed    | 13    |\n",
      "|    total_timesteps | 12800 |\n",
      "------------------------------\n",
      "------------------------------\n",
      "| time/              |       |\n",
      "|    episodes        | 16    |\n",
      "|    fps             | 949   |\n",
      "|    time_elapsed    | 13    |\n",
      "|    total_timesteps | 12800 |\n",
      "------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.878    |\n",
      "| time/               |          |\n",
      "|    episodes         | 20       |\n",
      "|    fps              | 945      |\n",
      "|    time_elapsed     | 27       |\n",
      "|    total_timesteps  | 25600    |\n",
      "----------------------------------\n",
      "------------------------------\n",
      "| time/              |       |\n",
      "|    episodes        | 24    |\n",
      "|    fps             | 944   |\n",
      "|    time_elapsed    | 27    |\n",
      "|    total_timesteps | 25600 |\n",
      "------------------------------\n",
      "------------------------------\n",
      "| time/              |       |\n",
      "|    episodes        | 28    |\n",
      "|    fps             | 944   |\n",
      "|    time_elapsed    | 27    |\n",
      "|    total_timesteps | 25600 |\n",
      "------------------------------\n",
      "------------------------------\n",
      "| time/              |       |\n",
      "|    episodes        | 32    |\n",
      "|    fps             | 944   |\n",
      "|    time_elapsed    | 27    |\n",
      "|    total_timesteps | 25600 |\n",
      "------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.818    |\n",
      "| time/               |          |\n",
      "|    episodes         | 36       |\n",
      "|    fps              | 941      |\n",
      "|    time_elapsed     | 40       |\n",
      "|    total_timesteps  | 38400    |\n",
      "----------------------------------\n",
      "------------------------------\n",
      "| time/              |       |\n",
      "|    episodes        | 40    |\n",
      "|    fps             | 941   |\n",
      "|    time_elapsed    | 40    |\n",
      "|    total_timesteps | 38400 |\n",
      "------------------------------\n",
      "------------------------------\n",
      "| time/              |       |\n",
      "|    episodes        | 44    |\n",
      "|    fps             | 941   |\n",
      "|    time_elapsed    | 40    |\n",
      "|    total_timesteps | 38400 |\n",
      "------------------------------\n",
      "------------------------------\n",
      "| time/              |       |\n",
      "|    episodes        | 48    |\n",
      "|    fps             | 941   |\n",
      "|    time_elapsed    | 40    |\n",
      "|    total_timesteps | 38400 |\n",
      "------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.757    |\n",
      "| time/               |          |\n",
      "|    episodes         | 52       |\n",
      "|    fps              | 917      |\n",
      "|    time_elapsed     | 55       |\n",
      "|    total_timesteps  | 51200    |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.000256 |\n",
      "|    n_updates        | 18       |\n",
      "----------------------------------\n",
      "------------------------------\n",
      "| time/              |       |\n",
      "|    episodes        | 56    |\n",
      "|    fps             | 917   |\n",
      "|    time_elapsed    | 55    |\n",
      "|    total_timesteps | 51200 |\n",
      "------------------------------\n",
      "------------------------------\n",
      "| time/              |       |\n",
      "|    episodes        | 60    |\n",
      "|    fps             | 917   |\n",
      "|    time_elapsed    | 55    |\n",
      "|    total_timesteps | 51200 |\n",
      "------------------------------\n",
      "------------------------------\n",
      "| time/              |       |\n",
      "|    episodes        | 64    |\n",
      "|    fps             | 917   |\n",
      "|    time_elapsed    | 55    |\n",
      "|    total_timesteps | 51200 |\n",
      "------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.696    |\n",
      "| time/               |          |\n",
      "|    episodes         | 68       |\n",
      "|    fps              | 880      |\n",
      "|    time_elapsed     | 72       |\n",
      "|    total_timesteps  | 64000    |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.000261 |\n",
      "|    n_updates        | 218      |\n",
      "----------------------------------\n",
      "------------------------------\n",
      "| time/              |       |\n",
      "|    episodes        | 72    |\n",
      "|    fps             | 880   |\n",
      "|    time_elapsed    | 72    |\n",
      "|    total_timesteps | 64000 |\n",
      "------------------------------\n",
      "------------------------------\n",
      "| time/              |       |\n",
      "|    episodes        | 76    |\n",
      "|    fps             | 880   |\n",
      "|    time_elapsed    | 72    |\n",
      "|    total_timesteps | 64000 |\n",
      "------------------------------\n",
      "------------------------------\n",
      "| time/              |       |\n",
      "|    episodes        | 80    |\n",
      "|    fps             | 880   |\n",
      "|    time_elapsed    | 72    |\n",
      "|    total_timesteps | 64000 |\n",
      "------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.635    |\n",
      "| time/               |          |\n",
      "|    episodes         | 84       |\n",
      "|    fps              | 850      |\n",
      "|    time_elapsed     | 90       |\n",
      "|    total_timesteps  | 76800    |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.103    |\n",
      "|    n_updates        | 418      |\n",
      "----------------------------------\n",
      "------------------------------\n",
      "| time/              |       |\n",
      "|    episodes        | 88    |\n",
      "|    fps             | 850   |\n",
      "|    time_elapsed    | 90    |\n",
      "|    total_timesteps | 76800 |\n",
      "------------------------------\n",
      "------------------------------\n",
      "| time/              |       |\n",
      "|    episodes        | 92    |\n",
      "|    fps             | 850   |\n",
      "|    time_elapsed    | 90    |\n",
      "|    total_timesteps | 76800 |\n",
      "------------------------------\n",
      "------------------------------\n",
      "| time/              |       |\n",
      "|    episodes        | 96    |\n",
      "|    fps             | 850   |\n",
      "|    time_elapsed    | 90    |\n",
      "|    total_timesteps | 76800 |\n",
      "------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.574    |\n",
      "| time/               |          |\n",
      "|    episodes         | 100      |\n",
      "|    fps              | 834      |\n",
      "|    time_elapsed     | 107      |\n",
      "|    total_timesteps  | 89600    |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.000203 |\n",
      "|    n_updates        | 618      |\n",
      "----------------------------------\n",
      "------------------------------\n",
      "| time/              |       |\n",
      "|    episodes        | 104   |\n",
      "|    fps             | 834   |\n",
      "|    time_elapsed    | 107   |\n",
      "|    total_timesteps | 89600 |\n",
      "------------------------------\n",
      "------------------------------\n",
      "| time/              |       |\n",
      "|    episodes        | 108   |\n",
      "|    fps             | 834   |\n",
      "|    time_elapsed    | 107   |\n",
      "|    total_timesteps | 89600 |\n",
      "------------------------------\n",
      "------------------------------\n",
      "| time/              |       |\n",
      "|    episodes        | 112   |\n",
      "|    fps             | 834   |\n",
      "|    time_elapsed    | 107   |\n",
      "|    total_timesteps | 89600 |\n",
      "------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.514    |\n",
      "| time/               |          |\n",
      "|    episodes         | 116      |\n",
      "|    fps              | 819      |\n",
      "|    time_elapsed     | 124      |\n",
      "|    total_timesteps  | 102400   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.000338 |\n",
      "|    n_updates        | 818      |\n",
      "----------------------------------\n",
      "-------------------------------\n",
      "| time/              |        |\n",
      "|    episodes        | 120    |\n",
      "|    fps             | 819    |\n",
      "|    time_elapsed    | 124    |\n",
      "|    total_timesteps | 102400 |\n",
      "-------------------------------\n",
      "-------------------------------\n",
      "| time/              |        |\n",
      "|    episodes        | 124    |\n",
      "|    fps             | 819    |\n",
      "|    time_elapsed    | 124    |\n",
      "|    total_timesteps | 102400 |\n",
      "-------------------------------\n",
      "-------------------------------\n",
      "| time/              |        |\n",
      "|    episodes        | 128    |\n",
      "|    fps             | 819    |\n",
      "|    time_elapsed    | 124    |\n",
      "|    total_timesteps | 102400 |\n",
      "-------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.453    |\n",
      "| time/               |          |\n",
      "|    episodes         | 132      |\n",
      "|    fps              | 806      |\n",
      "|    time_elapsed     | 142      |\n",
      "|    total_timesteps  | 115200   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.189    |\n",
      "|    n_updates        | 1018     |\n",
      "----------------------------------\n",
      "-------------------------------\n",
      "| time/              |        |\n",
      "|    episodes        | 136    |\n",
      "|    fps             | 806    |\n",
      "|    time_elapsed    | 142    |\n",
      "|    total_timesteps | 115200 |\n",
      "-------------------------------\n",
      "-------------------------------\n",
      "| time/              |        |\n",
      "|    episodes        | 140    |\n",
      "|    fps             | 806    |\n",
      "|    time_elapsed    | 142    |\n",
      "|    total_timesteps | 115200 |\n",
      "-------------------------------\n",
      "-------------------------------\n",
      "| time/              |        |\n",
      "|    episodes        | 144    |\n",
      "|    fps             | 806    |\n",
      "|    time_elapsed    | 142    |\n",
      "|    total_timesteps | 115200 |\n",
      "-------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.392    |\n",
      "| time/               |          |\n",
      "|    episodes         | 148      |\n",
      "|    fps              | 797      |\n",
      "|    time_elapsed     | 160      |\n",
      "|    total_timesteps  | 128000   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.245    |\n",
      "|    n_updates        | 1218     |\n",
      "----------------------------------\n",
      "-------------------------------\n",
      "| time/              |        |\n",
      "|    episodes        | 152    |\n",
      "|    fps             | 797    |\n",
      "|    time_elapsed    | 160    |\n",
      "|    total_timesteps | 128000 |\n",
      "-------------------------------\n",
      "-------------------------------\n",
      "| time/              |        |\n",
      "|    episodes        | 156    |\n",
      "|    fps             | 797    |\n",
      "|    time_elapsed    | 160    |\n",
      "|    total_timesteps | 128000 |\n",
      "-------------------------------\n",
      "-------------------------------\n",
      "| time/              |        |\n",
      "|    episodes        | 160    |\n",
      "|    fps             | 797    |\n",
      "|    time_elapsed    | 160    |\n",
      "|    total_timesteps | 128000 |\n",
      "-------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.331    |\n",
      "| time/               |          |\n",
      "|    episodes         | 164      |\n",
      "|    fps              | 788      |\n",
      "|    time_elapsed     | 178      |\n",
      "|    total_timesteps  | 140800   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.001    |\n",
      "|    n_updates        | 1418     |\n",
      "----------------------------------\n",
      "-------------------------------\n",
      "| time/              |        |\n",
      "|    episodes        | 168    |\n",
      "|    fps             | 788    |\n",
      "|    time_elapsed    | 178    |\n",
      "|    total_timesteps | 140800 |\n",
      "-------------------------------\n",
      "-------------------------------\n",
      "| time/              |        |\n",
      "|    episodes        | 172    |\n",
      "|    fps             | 788    |\n",
      "|    time_elapsed    | 178    |\n",
      "|    total_timesteps | 140800 |\n",
      "-------------------------------\n",
      "-------------------------------\n",
      "| time/              |        |\n",
      "|    episodes        | 176    |\n",
      "|    fps             | 788    |\n",
      "|    time_elapsed    | 178    |\n",
      "|    total_timesteps | 140800 |\n",
      "-------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.27     |\n",
      "| time/               |          |\n",
      "|    episodes         | 180      |\n",
      "|    fps              | 780      |\n",
      "|    time_elapsed     | 196      |\n",
      "|    total_timesteps  | 153600   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.00144  |\n",
      "|    n_updates        | 1618     |\n",
      "----------------------------------\n",
      "-------------------------------\n",
      "| time/              |        |\n",
      "|    episodes        | 184    |\n",
      "|    fps             | 780    |\n",
      "|    time_elapsed    | 196    |\n",
      "|    total_timesteps | 153600 |\n",
      "-------------------------------\n",
      "-------------------------------\n",
      "| time/              |        |\n",
      "|    episodes        | 188    |\n",
      "|    fps             | 780    |\n",
      "|    time_elapsed    | 196    |\n",
      "|    total_timesteps | 153600 |\n",
      "-------------------------------\n",
      "-------------------------------\n",
      "| time/              |        |\n",
      "|    episodes        | 192    |\n",
      "|    fps             | 780    |\n",
      "|    time_elapsed    | 196    |\n",
      "|    total_timesteps | 153600 |\n",
      "-------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.21     |\n",
      "| time/               |          |\n",
      "|    episodes         | 196      |\n",
      "|    fps              | 774      |\n",
      "|    time_elapsed     | 214      |\n",
      "|    total_timesteps  | 166400   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.000609 |\n",
      "|    n_updates        | 1818     |\n",
      "----------------------------------\n",
      "-------------------------------\n",
      "| time/              |        |\n",
      "|    episodes        | 200    |\n",
      "|    fps             | 774    |\n",
      "|    time_elapsed    | 214    |\n",
      "|    total_timesteps | 166400 |\n",
      "-------------------------------\n",
      "-------------------------------\n",
      "| time/              |        |\n",
      "|    episodes        | 204    |\n",
      "|    fps             | 774    |\n",
      "|    time_elapsed    | 214    |\n",
      "|    total_timesteps | 166400 |\n",
      "-------------------------------\n",
      "-------------------------------\n",
      "| time/              |        |\n",
      "|    episodes        | 208    |\n",
      "|    fps             | 774    |\n",
      "|    time_elapsed    | 214    |\n",
      "|    total_timesteps | 166400 |\n",
      "-------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.149    |\n",
      "| time/               |          |\n",
      "|    episodes         | 212      |\n",
      "|    fps              | 764      |\n",
      "|    time_elapsed     | 234      |\n",
      "|    total_timesteps  | 179200   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.00254  |\n",
      "|    n_updates        | 2018     |\n",
      "----------------------------------\n",
      "-------------------------------\n",
      "| time/              |        |\n",
      "|    episodes        | 216    |\n",
      "|    fps             | 764    |\n",
      "|    time_elapsed    | 234    |\n",
      "|    total_timesteps | 179200 |\n",
      "-------------------------------\n",
      "-------------------------------\n",
      "| time/              |        |\n",
      "|    episodes        | 220    |\n",
      "|    fps             | 764    |\n",
      "|    time_elapsed    | 234    |\n",
      "|    total_timesteps | 179200 |\n",
      "-------------------------------\n",
      "-------------------------------\n",
      "| time/              |        |\n",
      "|    episodes        | 224    |\n",
      "|    fps             | 764    |\n",
      "|    time_elapsed    | 234    |\n",
      "|    total_timesteps | 179200 |\n",
      "-------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.088    |\n",
      "| time/               |          |\n",
      "|    episodes         | 228      |\n",
      "|    fps              | 758      |\n",
      "|    time_elapsed     | 253      |\n",
      "|    total_timesteps  | 192000   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.0988   |\n",
      "|    n_updates        | 2218     |\n",
      "----------------------------------\n",
      "-------------------------------\n",
      "| time/              |        |\n",
      "|    episodes        | 232    |\n",
      "|    fps             | 758    |\n",
      "|    time_elapsed    | 253    |\n",
      "|    total_timesteps | 192000 |\n",
      "-------------------------------\n",
      "-------------------------------\n",
      "| time/              |        |\n",
      "|    episodes        | 236    |\n",
      "|    fps             | 758    |\n",
      "|    time_elapsed    | 253    |\n",
      "|    total_timesteps | 192000 |\n",
      "-------------------------------\n",
      "-------------------------------\n",
      "| time/              |        |\n",
      "|    episodes        | 240    |\n",
      "|    fps             | 758    |\n",
      "|    time_elapsed    | 253    |\n",
      "|    total_timesteps | 192000 |\n",
      "-------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 244      |\n",
      "|    fps              | 761      |\n",
      "|    time_elapsed     | 268      |\n",
      "|    total_timesteps  | 204800   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.000244 |\n",
      "|    n_updates        | 2418     |\n",
      "----------------------------------\n",
      "-------------------------------\n",
      "| time/              |        |\n",
      "|    episodes        | 248    |\n",
      "|    fps             | 761    |\n",
      "|    time_elapsed    | 268    |\n",
      "|    total_timesteps | 204800 |\n",
      "-------------------------------\n",
      "-------------------------------\n",
      "| time/              |        |\n",
      "|    episodes        | 252    |\n",
      "|    fps             | 761    |\n",
      "|    time_elapsed    | 268    |\n",
      "|    total_timesteps | 204800 |\n",
      "-------------------------------\n",
      "-------------------------------\n",
      "| time/              |        |\n",
      "|    episodes        | 256    |\n",
      "|    fps             | 761    |\n",
      "|    time_elapsed    | 268    |\n",
      "|    total_timesteps | 204800 |\n",
      "-------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 260      |\n",
      "|    fps              | 757      |\n",
      "|    time_elapsed     | 280      |\n",
      "|    total_timesteps  | 212672   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.0966   |\n",
      "|    n_updates        | 2541     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 264      |\n",
      "|    fps              | 755      |\n",
      "|    time_elapsed     | 287      |\n",
      "|    total_timesteps  | 217600   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.107    |\n",
      "|    n_updates        | 2618     |\n",
      "----------------------------------\n",
      "-------------------------------\n",
      "| time/              |        |\n",
      "|    episodes        | 268    |\n",
      "|    fps             | 755    |\n",
      "|    time_elapsed    | 287    |\n",
      "|    total_timesteps | 217600 |\n",
      "-------------------------------\n",
      "-------------------------------\n",
      "| time/              |        |\n",
      "|    episodes        | 272    |\n",
      "|    fps             | 755    |\n",
      "|    time_elapsed    | 287    |\n",
      "|    total_timesteps | 217600 |\n",
      "-------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 276      |\n",
      "|    fps              | 750      |\n",
      "|    time_elapsed     | 300      |\n",
      "|    total_timesteps  | 225472   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.00184  |\n",
      "|    n_updates        | 2741     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 280      |\n",
      "|    fps              | 747      |\n",
      "|    time_elapsed     | 308      |\n",
      "|    total_timesteps  | 230400   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.102    |\n",
      "|    n_updates        | 2818     |\n",
      "----------------------------------\n",
      "-------------------------------\n",
      "| time/              |        |\n",
      "|    episodes        | 284    |\n",
      "|    fps             | 747    |\n",
      "|    time_elapsed    | 308    |\n",
      "|    total_timesteps | 230400 |\n",
      "-------------------------------\n",
      "-------------------------------\n",
      "| time/              |        |\n",
      "|    episodes        | 288    |\n",
      "|    fps             | 747    |\n",
      "|    time_elapsed    | 308    |\n",
      "|    total_timesteps | 230400 |\n",
      "-------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 292      |\n",
      "|    fps              | 745      |\n",
      "|    time_elapsed     | 319      |\n",
      "|    total_timesteps  | 238272   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.0936   |\n",
      "|    n_updates        | 2941     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 296      |\n",
      "|    fps              | 744      |\n",
      "|    time_elapsed     | 326      |\n",
      "|    total_timesteps  | 243200   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.078    |\n",
      "|    n_updates        | 3018     |\n",
      "----------------------------------\n",
      "-------------------------------\n",
      "| time/              |        |\n",
      "|    episodes        | 300    |\n",
      "|    fps             | 744    |\n",
      "|    time_elapsed    | 326    |\n",
      "|    total_timesteps | 243200 |\n",
      "-------------------------------\n",
      "-------------------------------\n",
      "| time/              |        |\n",
      "|    episodes        | 304    |\n",
      "|    fps             | 744    |\n",
      "|    time_elapsed    | 326    |\n",
      "|    total_timesteps | 243200 |\n",
      "-------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 308      |\n",
      "|    fps              | 741      |\n",
      "|    time_elapsed     | 338      |\n",
      "|    total_timesteps  | 251072   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.000413 |\n",
      "|    n_updates        | 3141     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 312      |\n",
      "|    fps              | 739      |\n",
      "|    time_elapsed     | 346      |\n",
      "|    total_timesteps  | 256000   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.00613  |\n",
      "|    n_updates        | 3218     |\n",
      "----------------------------------\n",
      "-------------------------------\n",
      "| time/              |        |\n",
      "|    episodes        | 316    |\n",
      "|    fps             | 739    |\n",
      "|    time_elapsed    | 346    |\n",
      "|    total_timesteps | 256000 |\n",
      "-------------------------------\n",
      "-------------------------------\n",
      "| time/              |        |\n",
      "|    episodes        | 320    |\n",
      "|    fps             | 739    |\n",
      "|    time_elapsed    | 346    |\n",
      "|    total_timesteps | 256000 |\n",
      "-------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 324      |\n",
      "|    fps              | 737      |\n",
      "|    time_elapsed     | 357      |\n",
      "|    total_timesteps  | 263840   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.00292  |\n",
      "|    n_updates        | 3341     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 328      |\n",
      "|    fps              | 737      |\n",
      "|    time_elapsed     | 358      |\n",
      "|    total_timesteps  | 264112   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.099    |\n",
      "|    n_updates        | 3345     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 332      |\n",
      "|    fps              | 735      |\n",
      "|    time_elapsed     | 365      |\n",
      "|    total_timesteps  | 268800   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.0726   |\n",
      "|    n_updates        | 3418     |\n",
      "----------------------------------\n",
      "-------------------------------\n",
      "| time/              |        |\n",
      "|    episodes        | 336    |\n",
      "|    fps             | 735    |\n",
      "|    time_elapsed    | 365    |\n",
      "|    total_timesteps | 268800 |\n",
      "-------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 340      |\n",
      "|    fps              | 732      |\n",
      "|    time_elapsed     | 377      |\n",
      "|    total_timesteps  | 276640   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.105    |\n",
      "|    n_updates        | 3541     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 344      |\n",
      "|    fps              | 732      |\n",
      "|    time_elapsed     | 377      |\n",
      "|    total_timesteps  | 276768   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.00414  |\n",
      "|    n_updates        | 3543     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 348      |\n",
      "|    fps              | 731      |\n",
      "|    time_elapsed     | 385      |\n",
      "|    total_timesteps  | 281600   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.00779  |\n",
      "|    n_updates        | 3618     |\n",
      "----------------------------------\n",
      "-------------------------------\n",
      "| time/              |        |\n",
      "|    episodes        | 352    |\n",
      "|    fps             | 731    |\n",
      "|    time_elapsed    | 385    |\n",
      "|    total_timesteps | 281600 |\n",
      "-------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 356      |\n",
      "|    fps              | 728      |\n",
      "|    time_elapsed     | 397      |\n",
      "|    total_timesteps  | 289440   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.00211  |\n",
      "|    n_updates        | 3741     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 360      |\n",
      "|    fps              | 728      |\n",
      "|    time_elapsed     | 397      |\n",
      "|    total_timesteps  | 289568   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.00327  |\n",
      "|    n_updates        | 3743     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 364      |\n",
      "|    fps              | 726      |\n",
      "|    time_elapsed     | 405      |\n",
      "|    total_timesteps  | 294400   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.366    |\n",
      "|    n_updates        | 3818     |\n",
      "----------------------------------\n",
      "-------------------------------\n",
      "| time/              |        |\n",
      "|    episodes        | 368    |\n",
      "|    fps             | 726    |\n",
      "|    time_elapsed    | 405    |\n",
      "|    total_timesteps | 294400 |\n",
      "-------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 372      |\n",
      "|    fps              | 724      |\n",
      "|    time_elapsed     | 417      |\n",
      "|    total_timesteps  | 302160   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.00923  |\n",
      "|    n_updates        | 3940     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 376      |\n",
      "|    fps              | 724      |\n",
      "|    time_elapsed     | 417      |\n",
      "|    total_timesteps  | 302368   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.00563  |\n",
      "|    n_updates        | 3943     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 380      |\n",
      "|    fps              | 722      |\n",
      "|    time_elapsed     | 425      |\n",
      "|    total_timesteps  | 307120   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.0026   |\n",
      "|    n_updates        | 4017     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 384      |\n",
      "|    fps              | 722      |\n",
      "|    time_elapsed     | 425      |\n",
      "|    total_timesteps  | 307200   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.0742   |\n",
      "|    n_updates        | 4018     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 388      |\n",
      "|    fps              | 722      |\n",
      "|    time_elapsed     | 427      |\n",
      "|    total_timesteps  | 308688   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.0243   |\n",
      "|    n_updates        | 4042     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 392      |\n",
      "|    fps              | 720      |\n",
      "|    time_elapsed     | 437      |\n",
      "|    total_timesteps  | 315072   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.102    |\n",
      "|    n_updates        | 4141     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 396      |\n",
      "|    fps              | 719      |\n",
      "|    time_elapsed     | 444      |\n",
      "|    total_timesteps  | 319632   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.274    |\n",
      "|    n_updates        | 4213     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 400      |\n",
      "|    fps              | 719      |\n",
      "|    time_elapsed     | 444      |\n",
      "|    total_timesteps  | 320000   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.173    |\n",
      "|    n_updates        | 4218     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 404      |\n",
      "|    fps              | 719      |\n",
      "|    time_elapsed     | 444      |\n",
      "|    total_timesteps  | 320112   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.0896   |\n",
      "|    n_updates        | 4220     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 408      |\n",
      "|    fps              | 717      |\n",
      "|    time_elapsed     | 456      |\n",
      "|    total_timesteps  | 327760   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.00414  |\n",
      "|    n_updates        | 4340     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 412      |\n",
      "|    fps              | 717      |\n",
      "|    time_elapsed     | 457      |\n",
      "|    total_timesteps  | 328112   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.336    |\n",
      "|    n_updates        | 4345     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 416      |\n",
      "|    fps              | 716      |\n",
      "|    time_elapsed     | 464      |\n",
      "|    total_timesteps  | 332800   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.0906   |\n",
      "|    n_updates        | 4418     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 420      |\n",
      "|    fps              | 716      |\n",
      "|    time_elapsed     | 464      |\n",
      "|    total_timesteps  | 332912   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.00818  |\n",
      "|    n_updates        | 4420     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 424      |\n",
      "|    fps              | 714      |\n",
      "|    time_elapsed     | 476      |\n",
      "|    total_timesteps  | 340560   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.192    |\n",
      "|    n_updates        | 4540     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 428      |\n",
      "|    fps              | 714      |\n",
      "|    time_elapsed     | 476      |\n",
      "|    total_timesteps  | 340912   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.103    |\n",
      "|    n_updates        | 4545     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 432      |\n",
      "|    fps              | 714      |\n",
      "|    time_elapsed     | 484      |\n",
      "|    total_timesteps  | 345600   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.00231  |\n",
      "|    n_updates        | 4618     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 436      |\n",
      "|    fps              | 713      |\n",
      "|    time_elapsed     | 484      |\n",
      "|    total_timesteps  | 345712   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.00187  |\n",
      "|    n_updates        | 4620     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 440      |\n",
      "|    fps              | 713      |\n",
      "|    time_elapsed     | 489      |\n",
      "|    total_timesteps  | 348816   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.00302  |\n",
      "|    n_updates        | 4669     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 444      |\n",
      "|    fps              | 712      |\n",
      "|    time_elapsed     | 496      |\n",
      "|    total_timesteps  | 353472   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.116    |\n",
      "|    n_updates        | 4741     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 448      |\n",
      "|    fps              | 711      |\n",
      "|    time_elapsed     | 503      |\n",
      "|    total_timesteps  | 358400   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.0827   |\n",
      "|    n_updates        | 4818     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 452      |\n",
      "|    fps              | 711      |\n",
      "|    time_elapsed     | 503      |\n",
      "|    total_timesteps  | 358512   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.0943   |\n",
      "|    n_updates        | 4820     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 456      |\n",
      "|    fps              | 710      |\n",
      "|    time_elapsed     | 508      |\n",
      "|    total_timesteps  | 361616   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.276    |\n",
      "|    n_updates        | 4869     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 460      |\n",
      "|    fps              | 710      |\n",
      "|    time_elapsed     | 515      |\n",
      "|    total_timesteps  | 366272   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.268    |\n",
      "|    n_updates        | 4941     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 464      |\n",
      "|    fps              | 709      |\n",
      "|    time_elapsed     | 519      |\n",
      "|    total_timesteps  | 368320   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.092    |\n",
      "|    n_updates        | 4973     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 468      |\n",
      "|    fps              | 709      |\n",
      "|    time_elapsed     | 523      |\n",
      "|    total_timesteps  | 371200   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.101    |\n",
      "|    n_updates        | 5018     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 472      |\n",
      "|    fps              | 707      |\n",
      "|    time_elapsed     | 528      |\n",
      "|    total_timesteps  | 374416   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.00179  |\n",
      "|    n_updates        | 5069     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 476      |\n",
      "|    fps              | 706      |\n",
      "|    time_elapsed     | 536      |\n",
      "|    total_timesteps  | 379072   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.0148   |\n",
      "|    n_updates        | 5141     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 480      |\n",
      "|    fps              | 706      |\n",
      "|    time_elapsed     | 539      |\n",
      "|    total_timesteps  | 381120   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.0877   |\n",
      "|    n_updates        | 5173     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 484      |\n",
      "|    fps              | 705      |\n",
      "|    time_elapsed     | 544      |\n",
      "|    total_timesteps  | 384000   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.111    |\n",
      "|    n_updates        | 5218     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 488      |\n",
      "|    fps              | 705      |\n",
      "|    time_elapsed     | 548      |\n",
      "|    total_timesteps  | 387216   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.178    |\n",
      "|    n_updates        | 5269     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 492      |\n",
      "|    fps              | 704      |\n",
      "|    time_elapsed     | 555      |\n",
      "|    total_timesteps  | 391760   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.00707  |\n",
      "|    n_updates        | 5340     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 496      |\n",
      "|    fps              | 704      |\n",
      "|    time_elapsed     | 558      |\n",
      "|    total_timesteps  | 393888   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.101    |\n",
      "|    n_updates        | 5373     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 500      |\n",
      "|    fps              | 704      |\n",
      "|    time_elapsed     | 563      |\n",
      "|    total_timesteps  | 396800   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.0828   |\n",
      "|    n_updates        | 5418     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 504      |\n",
      "|    fps              | 704      |\n",
      "|    time_elapsed     | 568      |\n",
      "|    total_timesteps  | 400016   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.268    |\n",
      "|    n_updates        | 5469     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 508      |\n",
      "|    fps              | 703      |\n",
      "|    time_elapsed     | 573      |\n",
      "|    total_timesteps  | 403840   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.0912   |\n",
      "|    n_updates        | 5528     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 512      |\n",
      "|    fps              | 703      |\n",
      "|    time_elapsed     | 575      |\n",
      "|    total_timesteps  | 404912   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.00843  |\n",
      "|    n_updates        | 5545     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 516      |\n",
      "|    fps              | 703      |\n",
      "|    time_elapsed     | 582      |\n",
      "|    total_timesteps  | 409232   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.00463  |\n",
      "|    n_updates        | 5613     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 520      |\n",
      "|    fps              | 702      |\n",
      "|    time_elapsed     | 587      |\n",
      "|    total_timesteps  | 412784   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.098    |\n",
      "|    n_updates        | 5668     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 524      |\n",
      "|    fps              | 702      |\n",
      "|    time_elapsed     | 593      |\n",
      "|    total_timesteps  | 416640   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.103    |\n",
      "|    n_updates        | 5728     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 528      |\n",
      "|    fps              | 702      |\n",
      "|    time_elapsed     | 594      |\n",
      "|    total_timesteps  | 417712   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.000678 |\n",
      "|    n_updates        | 5745     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 532      |\n",
      "|    fps              | 701      |\n",
      "|    time_elapsed     | 601      |\n",
      "|    total_timesteps  | 422032   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.323    |\n",
      "|    n_updates        | 5813     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 536      |\n",
      "|    fps              | 701      |\n",
      "|    time_elapsed     | 606      |\n",
      "|    total_timesteps  | 425584   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.102    |\n",
      "|    n_updates        | 5868     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 540      |\n",
      "|    fps              | 701      |\n",
      "|    time_elapsed     | 612      |\n",
      "|    total_timesteps  | 429440   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.172    |\n",
      "|    n_updates        | 5928     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 544      |\n",
      "|    fps              | 700      |\n",
      "|    time_elapsed     | 614      |\n",
      "|    total_timesteps  | 430512   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.0805   |\n",
      "|    n_updates        | 5945     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 548      |\n",
      "|    fps              | 700      |\n",
      "|    time_elapsed     | 620      |\n",
      "|    total_timesteps  | 434832   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.0779   |\n",
      "|    n_updates        | 6013     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 552      |\n",
      "|    fps              | 700      |\n",
      "|    time_elapsed     | 626      |\n",
      "|    total_timesteps  | 438384   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.00302  |\n",
      "|    n_updates        | 6068     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 556      |\n",
      "|    fps              | 699      |\n",
      "|    time_elapsed     | 632      |\n",
      "|    total_timesteps  | 442240   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.176    |\n",
      "|    n_updates        | 6128     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 560      |\n",
      "|    fps              | 699      |\n",
      "|    time_elapsed     | 633      |\n",
      "|    total_timesteps  | 443312   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.18     |\n",
      "|    n_updates        | 6145     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 564      |\n",
      "|    fps              | 698      |\n",
      "|    time_elapsed     | 640      |\n",
      "|    total_timesteps  | 447632   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.0095   |\n",
      "|    n_updates        | 6213     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 568      |\n",
      "|    fps              | 698      |\n",
      "|    time_elapsed     | 646      |\n",
      "|    total_timesteps  | 451184   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.0792   |\n",
      "|    n_updates        | 6268     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 572      |\n",
      "|    fps              | 697      |\n",
      "|    time_elapsed     | 651      |\n",
      "|    total_timesteps  | 455040   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.162    |\n",
      "|    n_updates        | 6328     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 576      |\n",
      "|    fps              | 697      |\n",
      "|    time_elapsed     | 653      |\n",
      "|    total_timesteps  | 456112   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.00312  |\n",
      "|    n_updates        | 6345     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 580      |\n",
      "|    fps              | 697      |\n",
      "|    time_elapsed     | 659      |\n",
      "|    total_timesteps  | 460432   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.0112   |\n",
      "|    n_updates        | 6413     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 584      |\n",
      "|    fps              | 697      |\n",
      "|    time_elapsed     | 665      |\n",
      "|    total_timesteps  | 463984   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.159    |\n",
      "|    n_updates        | 6468     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 588      |\n",
      "|    fps              | 696      |\n",
      "|    time_elapsed     | 671      |\n",
      "|    total_timesteps  | 467840   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.178    |\n",
      "|    n_updates        | 6528     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 592      |\n",
      "|    fps              | 696      |\n",
      "|    time_elapsed     | 673      |\n",
      "|    total_timesteps  | 468912   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.254    |\n",
      "|    n_updates        | 6545     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 596      |\n",
      "|    fps              | 696      |\n",
      "|    time_elapsed     | 679      |\n",
      "|    total_timesteps  | 473232   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.178    |\n",
      "|    n_updates        | 6613     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 600      |\n",
      "|    fps              | 695      |\n",
      "|    time_elapsed     | 685      |\n",
      "|    total_timesteps  | 476784   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.0143   |\n",
      "|    n_updates        | 6668     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 604      |\n",
      "|    fps              | 694      |\n",
      "|    time_elapsed     | 691      |\n",
      "|    total_timesteps  | 480640   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.00969  |\n",
      "|    n_updates        | 6728     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 608      |\n",
      "|    fps              | 694      |\n",
      "|    time_elapsed     | 693      |\n",
      "|    total_timesteps  | 481712   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.246    |\n",
      "|    n_updates        | 6745     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 612      |\n",
      "|    fps              | 694      |\n",
      "|    time_elapsed     | 700      |\n",
      "|    total_timesteps  | 486032   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.19     |\n",
      "|    n_updates        | 6813     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 616      |\n",
      "|    fps              | 694      |\n",
      "|    time_elapsed     | 705      |\n",
      "|    total_timesteps  | 489584   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.0102   |\n",
      "|    n_updates        | 6868     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 620      |\n",
      "|    fps              | 694      |\n",
      "|    time_elapsed     | 710      |\n",
      "|    total_timesteps  | 493440   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.28     |\n",
      "|    n_updates        | 6928     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 624      |\n",
      "|    fps              | 694      |\n",
      "|    time_elapsed     | 711      |\n",
      "|    total_timesteps  | 494272   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.175    |\n",
      "|    n_updates        | 6941     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 628      |\n",
      "|    fps              | 693      |\n",
      "|    time_elapsed     | 716      |\n",
      "|    total_timesteps  | 497136   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.355    |\n",
      "|    n_updates        | 6986     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 632      |\n",
      "|    fps              | 693      |\n",
      "|    time_elapsed     | 724      |\n",
      "|    total_timesteps  | 502384   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.343    |\n",
      "|    n_updates        | 7068     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 636      |\n",
      "|    fps              | 692      |\n",
      "|    time_elapsed     | 730      |\n",
      "|    total_timesteps  | 506096   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.279    |\n",
      "|    n_updates        | 7126     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 640      |\n",
      "|    fps              | 692      |\n",
      "|    time_elapsed     | 731      |\n",
      "|    total_timesteps  | 507072   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.0905   |\n",
      "|    n_updates        | 7141     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 644      |\n",
      "|    fps              | 692      |\n",
      "|    time_elapsed     | 736      |\n",
      "|    total_timesteps  | 509936   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.116    |\n",
      "|    n_updates        | 7186     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 648      |\n",
      "|    fps              | 691      |\n",
      "|    time_elapsed     | 744      |\n",
      "|    total_timesteps  | 515184   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.0105   |\n",
      "|    n_updates        | 7268     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 652      |\n",
      "|    fps              | 691      |\n",
      "|    time_elapsed     | 750      |\n",
      "|    total_timesteps  | 518896   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.00577  |\n",
      "|    n_updates        | 7326     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 656      |\n",
      "|    fps              | 691      |\n",
      "|    time_elapsed     | 752      |\n",
      "|    total_timesteps  | 519872   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.205    |\n",
      "|    n_updates        | 7341     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 660      |\n",
      "|    fps              | 690      |\n",
      "|    time_elapsed     | 756      |\n",
      "|    total_timesteps  | 522320   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.291    |\n",
      "|    n_updates        | 7380     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 664      |\n",
      "|    fps              | 690      |\n",
      "|    time_elapsed     | 759      |\n",
      "|    total_timesteps  | 524800   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.255    |\n",
      "|    n_updates        | 7418     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 668      |\n",
      "|    fps              | 689      |\n",
      "|    time_elapsed     | 770      |\n",
      "|    total_timesteps  | 531696   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.449    |\n",
      "|    n_updates        | 7526     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 672      |\n",
      "|    fps              | 689      |\n",
      "|    time_elapsed     | 772      |\n",
      "|    total_timesteps  | 532672   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.294    |\n",
      "|    n_updates        | 7541     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 676      |\n",
      "|    fps              | 689      |\n",
      "|    time_elapsed     | 775      |\n",
      "|    total_timesteps  | 535120   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.289    |\n",
      "|    n_updates        | 7580     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 680      |\n",
      "|    fps              | 689      |\n",
      "|    time_elapsed     | 779      |\n",
      "|    total_timesteps  | 537600   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.203    |\n",
      "|    n_updates        | 7618     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 684      |\n",
      "|    fps              | 688      |\n",
      "|    time_elapsed     | 790      |\n",
      "|    total_timesteps  | 544496   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.0918   |\n",
      "|    n_updates        | 7726     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 688      |\n",
      "|    fps              | 688      |\n",
      "|    time_elapsed     | 791      |\n",
      "|    total_timesteps  | 545472   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.114    |\n",
      "|    n_updates        | 7741     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 692      |\n",
      "|    fps              | 688      |\n",
      "|    time_elapsed     | 795      |\n",
      "|    total_timesteps  | 547920   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.223    |\n",
      "|    n_updates        | 7780     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 696      |\n",
      "|    fps              | 689      |\n",
      "|    time_elapsed     | 798      |\n",
      "|    total_timesteps  | 550400   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.195    |\n",
      "|    n_updates        | 7818     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 700      |\n",
      "|    fps              | 688      |\n",
      "|    time_elapsed     | 809      |\n",
      "|    total_timesteps  | 557296   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.172    |\n",
      "|    n_updates        | 7926     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 704      |\n",
      "|    fps              | 688      |\n",
      "|    time_elapsed     | 810      |\n",
      "|    total_timesteps  | 558272   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.256    |\n",
      "|    n_updates        | 7941     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 708      |\n",
      "|    fps              | 688      |\n",
      "|    time_elapsed     | 814      |\n",
      "|    total_timesteps  | 560720   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.369    |\n",
      "|    n_updates        | 7980     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 712      |\n",
      "|    fps              | 687      |\n",
      "|    time_elapsed     | 818      |\n",
      "|    total_timesteps  | 563200   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.222    |\n",
      "|    n_updates        | 8018     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 716      |\n",
      "|    fps              | 687      |\n",
      "|    time_elapsed     | 829      |\n",
      "|    total_timesteps  | 570096   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.0118   |\n",
      "|    n_updates        | 8126     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 720      |\n",
      "|    fps              | 687      |\n",
      "|    time_elapsed     | 831      |\n",
      "|    total_timesteps  | 571072   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.164    |\n",
      "|    n_updates        | 8141     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 724      |\n",
      "|    fps              | 686      |\n",
      "|    time_elapsed     | 835      |\n",
      "|    total_timesteps  | 573520   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.258    |\n",
      "|    n_updates        | 8180     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 728      |\n",
      "|    fps              | 686      |\n",
      "|    time_elapsed     | 839      |\n",
      "|    total_timesteps  | 576000   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.111    |\n",
      "|    n_updates        | 8218     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 732      |\n",
      "|    fps              | 685      |\n",
      "|    time_elapsed     | 850      |\n",
      "|    total_timesteps  | 582896   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.274    |\n",
      "|    n_updates        | 8326     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 736      |\n",
      "|    fps              | 685      |\n",
      "|    time_elapsed     | 851      |\n",
      "|    total_timesteps  | 583872   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.485    |\n",
      "|    n_updates        | 8341     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 740      |\n",
      "|    fps              | 685      |\n",
      "|    time_elapsed     | 855      |\n",
      "|    total_timesteps  | 586320   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.204    |\n",
      "|    n_updates        | 8380     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 744      |\n",
      "|    fps              | 685      |\n",
      "|    time_elapsed     | 859      |\n",
      "|    total_timesteps  | 588800   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.31     |\n",
      "|    n_updates        | 8418     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 748      |\n",
      "|    fps              | 686      |\n",
      "|    time_elapsed     | 867      |\n",
      "|    total_timesteps  | 595024   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.365    |\n",
      "|    n_updates        | 8516     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 752      |\n",
      "|    fps              | 686      |\n",
      "|    time_elapsed     | 868      |\n",
      "|    total_timesteps  | 596144   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.362    |\n",
      "|    n_updates        | 8533     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 756      |\n",
      "|    fps              | 685      |\n",
      "|    time_elapsed     | 873      |\n",
      "|    total_timesteps  | 599120   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.581    |\n",
      "|    n_updates        | 8580     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 760      |\n",
      "|    fps              | 685      |\n",
      "|    time_elapsed     | 877      |\n",
      "|    total_timesteps  | 601600   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.554    |\n",
      "|    n_updates        | 8618     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 764      |\n",
      "|    fps              | 685      |\n",
      "|    time_elapsed     | 887      |\n",
      "|    total_timesteps  | 607824   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.276    |\n",
      "|    n_updates        | 8716     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 768      |\n",
      "|    fps              | 685      |\n",
      "|    time_elapsed     | 888      |\n",
      "|    total_timesteps  | 608944   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.177    |\n",
      "|    n_updates        | 8733     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 772      |\n",
      "|    fps              | 684      |\n",
      "|    time_elapsed     | 893      |\n",
      "|    total_timesteps  | 611920   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.189    |\n",
      "|    n_updates        | 8780     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 776      |\n",
      "|    fps              | 684      |\n",
      "|    time_elapsed     | 897      |\n",
      "|    total_timesteps  | 614400   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.018    |\n",
      "|    n_updates        | 8818     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 780      |\n",
      "|    fps              | 683      |\n",
      "|    time_elapsed     | 906      |\n",
      "|    total_timesteps  | 619728   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.445    |\n",
      "|    n_updates        | 8902     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 784      |\n",
      "|    fps              | 683      |\n",
      "|    time_elapsed     | 909      |\n",
      "|    total_timesteps  | 621744   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.0909   |\n",
      "|    n_updates        | 8933     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 788      |\n",
      "|    fps              | 683      |\n",
      "|    time_elapsed     | 911      |\n",
      "|    total_timesteps  | 623152   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.174    |\n",
      "|    n_updates        | 8955     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 792      |\n",
      "|    fps              | 683      |\n",
      "|    time_elapsed     | 917      |\n",
      "|    total_timesteps  | 627200   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.269    |\n",
      "|    n_updates        | 9018     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 796      |\n",
      "|    fps              | 683      |\n",
      "|    time_elapsed     | 926      |\n",
      "|    total_timesteps  | 632528   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.275    |\n",
      "|    n_updates        | 9102     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 800      |\n",
      "|    fps              | 682      |\n",
      "|    time_elapsed     | 929      |\n",
      "|    total_timesteps  | 634544   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.537    |\n",
      "|    n_updates        | 9133     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 804      |\n",
      "|    fps              | 683      |\n",
      "|    time_elapsed     | 931      |\n",
      "|    total_timesteps  | 635952   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.154    |\n",
      "|    n_updates        | 9155     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 808      |\n",
      "|    fps              | 683      |\n",
      "|    time_elapsed     | 936      |\n",
      "|    total_timesteps  | 640000   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.0772   |\n",
      "|    n_updates        | 9218     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 812      |\n",
      "|    fps              | 683      |\n",
      "|    time_elapsed     | 943      |\n",
      "|    total_timesteps  | 645328   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.335    |\n",
      "|    n_updates        | 9302     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 816      |\n",
      "|    fps              | 683      |\n",
      "|    time_elapsed     | 946      |\n",
      "|    total_timesteps  | 647344   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.173    |\n",
      "|    n_updates        | 9333     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 820      |\n",
      "|    fps              | 683      |\n",
      "|    time_elapsed     | 948      |\n",
      "|    total_timesteps  | 648752   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.27     |\n",
      "|    n_updates        | 9355     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 824      |\n",
      "|    fps              | 684      |\n",
      "|    time_elapsed     | 954      |\n",
      "|    total_timesteps  | 652800   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.257    |\n",
      "|    n_updates        | 9418     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 828      |\n",
      "|    fps              | 684      |\n",
      "|    time_elapsed     | 961      |\n",
      "|    total_timesteps  | 658128   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.0883   |\n",
      "|    n_updates        | 9502     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 832      |\n",
      "|    fps              | 684      |\n",
      "|    time_elapsed     | 964      |\n",
      "|    total_timesteps  | 660144   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.174    |\n",
      "|    n_updates        | 9533     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 836      |\n",
      "|    fps              | 684      |\n",
      "|    time_elapsed     | 966      |\n",
      "|    total_timesteps  | 661552   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.281    |\n",
      "|    n_updates        | 9555     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 840      |\n",
      "|    fps              | 684      |\n",
      "|    time_elapsed     | 972      |\n",
      "|    total_timesteps  | 665600   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.342    |\n",
      "|    n_updates        | 9618     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 844      |\n",
      "|    fps              | 684      |\n",
      "|    time_elapsed     | 977      |\n",
      "|    total_timesteps  | 668784   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.339    |\n",
      "|    n_updates        | 9668     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 848      |\n",
      "|    fps              | 683      |\n",
      "|    time_elapsed     | 984      |\n",
      "|    total_timesteps  | 672944   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.111    |\n",
      "|    n_updates        | 9733     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 852      |\n",
      "|    fps              | 683      |\n",
      "|    time_elapsed     | 986      |\n",
      "|    total_timesteps  | 674352   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.248    |\n",
      "|    n_updates        | 9755     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 856      |\n",
      "|    fps              | 683      |\n",
      "|    time_elapsed     | 992      |\n",
      "|    total_timesteps  | 678240   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.098    |\n",
      "|    n_updates        | 9816     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 860      |\n",
      "|    fps              | 682      |\n",
      "|    time_elapsed     | 997      |\n",
      "|    total_timesteps  | 681392   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.621    |\n",
      "|    n_updates        | 9865     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 864      |\n",
      "|    fps              | 682      |\n",
      "|    time_elapsed     | 1004     |\n",
      "|    total_timesteps  | 685440   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.287    |\n",
      "|    n_updates        | 9928     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 868      |\n",
      "|    fps              | 682      |\n",
      "|    time_elapsed     | 1006     |\n",
      "|    total_timesteps  | 686560   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.115    |\n",
      "|    n_updates        | 9946     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 872      |\n",
      "|    fps              | 682      |\n",
      "|    time_elapsed     | 1009     |\n",
      "|    total_timesteps  | 688720   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.527    |\n",
      "|    n_updates        | 9980     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 876      |\n",
      "|    fps              | 682      |\n",
      "|    time_elapsed     | 1013     |\n",
      "|    total_timesteps  | 691200   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.16     |\n",
      "|    n_updates        | 10018    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 880      |\n",
      "|    fps              | 682      |\n",
      "|    time_elapsed     | 1017     |\n",
      "|    total_timesteps  | 694384   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.335    |\n",
      "|    n_updates        | 10068    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 884      |\n",
      "|    fps              | 682      |\n",
      "|    time_elapsed     | 1024     |\n",
      "|    total_timesteps  | 699136   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.193    |\n",
      "|    n_updates        | 10142    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 888      |\n",
      "|    fps              | 682      |\n",
      "|    time_elapsed     | 1027     |\n",
      "|    total_timesteps  | 700960   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.456    |\n",
      "|    n_updates        | 10171    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 892      |\n",
      "|    fps              | 681      |\n",
      "|    time_elapsed     | 1032     |\n",
      "|    total_timesteps  | 704000   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.18     |\n",
      "|    n_updates        | 10218    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 896      |\n",
      "|    fps              | 681      |\n",
      "|    time_elapsed     | 1037     |\n",
      "|    total_timesteps  | 707184   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.297    |\n",
      "|    n_updates        | 10268    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 900      |\n",
      "|    fps              | 681      |\n",
      "|    time_elapsed     | 1041     |\n",
      "|    total_timesteps  | 709792   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.206    |\n",
      "|    n_updates        | 10309    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 904      |\n",
      "|    fps              | 681      |\n",
      "|    time_elapsed     | 1048     |\n",
      "|    total_timesteps  | 713760   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.195    |\n",
      "|    n_updates        | 10371    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 908      |\n",
      "|    fps              | 681      |\n",
      "|    time_elapsed     | 1052     |\n",
      "|    total_timesteps  | 716640   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.282    |\n",
      "|    n_updates        | 10416    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 912      |\n",
      "|    fps              | 681      |\n",
      "|    time_elapsed     | 1057     |\n",
      "|    total_timesteps  | 719984   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.492    |\n",
      "|    n_updates        | 10468    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 916      |\n",
      "|    fps              | 680      |\n",
      "|    time_elapsed     | 1061     |\n",
      "|    total_timesteps  | 722592   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.261    |\n",
      "|    n_updates        | 10509    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 920      |\n",
      "|    fps              | 680      |\n",
      "|    time_elapsed     | 1067     |\n",
      "|    total_timesteps  | 726560   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.129    |\n",
      "|    n_updates        | 10571    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 924      |\n",
      "|    fps              | 680      |\n",
      "|    time_elapsed     | 1072     |\n",
      "|    total_timesteps  | 729440   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.577    |\n",
      "|    n_updates        | 10616    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 928      |\n",
      "|    fps              | 680      |\n",
      "|    time_elapsed     | 1076     |\n",
      "|    total_timesteps  | 732784   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.244    |\n",
      "|    n_updates        | 10668    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 932      |\n",
      "|    fps              | 680      |\n",
      "|    time_elapsed     | 1081     |\n",
      "|    total_timesteps  | 735312   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.5      |\n",
      "|    n_updates        | 10708    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 936      |\n",
      "|    fps              | 679      |\n",
      "|    time_elapsed     | 1087     |\n",
      "|    total_timesteps  | 739360   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.28     |\n",
      "|    n_updates        | 10771    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 940      |\n",
      "|    fps              | 679      |\n",
      "|    time_elapsed     | 1092     |\n",
      "|    total_timesteps  | 742240   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.396    |\n",
      "|    n_updates        | 10816    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 944      |\n",
      "|    fps              | 679      |\n",
      "|    time_elapsed     | 1097     |\n",
      "|    total_timesteps  | 745584   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.019    |\n",
      "|    n_updates        | 10868    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 948      |\n",
      "|    fps              | 678      |\n",
      "|    time_elapsed     | 1101     |\n",
      "|    total_timesteps  | 748112   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.646    |\n",
      "|    n_updates        | 10908    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 952      |\n",
      "|    fps              | 678      |\n",
      "|    time_elapsed     | 1108     |\n",
      "|    total_timesteps  | 752160   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.325    |\n",
      "|    n_updates        | 10971    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 956      |\n",
      "|    fps              | 678      |\n",
      "|    time_elapsed     | 1112     |\n",
      "|    total_timesteps  | 755040   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.286    |\n",
      "|    n_updates        | 11016    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 960      |\n",
      "|    fps              | 678      |\n",
      "|    time_elapsed     | 1117     |\n",
      "|    total_timesteps  | 758384   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.222    |\n",
      "|    n_updates        | 11068    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 964      |\n",
      "|    fps              | 678      |\n",
      "|    time_elapsed     | 1121     |\n",
      "|    total_timesteps  | 760912   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.177    |\n",
      "|    n_updates        | 11108    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 968      |\n",
      "|    fps              | 678      |\n",
      "|    time_elapsed     | 1127     |\n",
      "|    total_timesteps  | 764960   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.335    |\n",
      "|    n_updates        | 11171    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 972      |\n",
      "|    fps              | 678      |\n",
      "|    time_elapsed     | 1132     |\n",
      "|    total_timesteps  | 767840   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.337    |\n",
      "|    n_updates        | 11216    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 976      |\n",
      "|    fps              | 678      |\n",
      "|    time_elapsed     | 1137     |\n",
      "|    total_timesteps  | 771184   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.132    |\n",
      "|    n_updates        | 11268    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 980      |\n",
      "|    fps              | 677      |\n",
      "|    time_elapsed     | 1141     |\n",
      "|    total_timesteps  | 773712   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.219    |\n",
      "|    n_updates        | 11308    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 984      |\n",
      "|    fps              | 677      |\n",
      "|    time_elapsed     | 1147     |\n",
      "|    total_timesteps  | 777760   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.264    |\n",
      "|    n_updates        | 11371    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 988      |\n",
      "|    fps              | 677      |\n",
      "|    time_elapsed     | 1152     |\n",
      "|    total_timesteps  | 780640   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.405    |\n",
      "|    n_updates        | 11416    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 992      |\n",
      "|    fps              | 677      |\n",
      "|    time_elapsed     | 1157     |\n",
      "|    total_timesteps  | 783984   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.312    |\n",
      "|    n_updates        | 11468    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 996      |\n",
      "|    fps              | 677      |\n",
      "|    time_elapsed     | 1161     |\n",
      "|    total_timesteps  | 786512   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.553    |\n",
      "|    n_updates        | 11508    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1000     |\n",
      "|    fps              | 676      |\n",
      "|    time_elapsed     | 1168     |\n",
      "|    total_timesteps  | 790560   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.295    |\n",
      "|    n_updates        | 11571    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1004     |\n",
      "|    fps              | 676      |\n",
      "|    time_elapsed     | 1172     |\n",
      "|    total_timesteps  | 793440   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.288    |\n",
      "|    n_updates        | 11616    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1008     |\n",
      "|    fps              | 676      |\n",
      "|    time_elapsed     | 1177     |\n",
      "|    total_timesteps  | 796784   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.225    |\n",
      "|    n_updates        | 11668    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1012     |\n",
      "|    fps              | 676      |\n",
      "|    time_elapsed     | 1181     |\n",
      "|    total_timesteps  | 799312   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.428    |\n",
      "|    n_updates        | 11708    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1016     |\n",
      "|    fps              | 676      |\n",
      "|    time_elapsed     | 1188     |\n",
      "|    total_timesteps  | 803360   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.247    |\n",
      "|    n_updates        | 11771    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1020     |\n",
      "|    fps              | 676      |\n",
      "|    time_elapsed     | 1192     |\n",
      "|    total_timesteps  | 806240   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.454    |\n",
      "|    n_updates        | 11816    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1024     |\n",
      "|    fps              | 675      |\n",
      "|    time_elapsed     | 1197     |\n",
      "|    total_timesteps  | 809584   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.413    |\n",
      "|    n_updates        | 11868    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1028     |\n",
      "|    fps              | 675      |\n",
      "|    time_elapsed     | 1201     |\n",
      "|    total_timesteps  | 812112   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.489    |\n",
      "|    n_updates        | 11908    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1032     |\n",
      "|    fps              | 675      |\n",
      "|    time_elapsed     | 1208     |\n",
      "|    total_timesteps  | 816160   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.215    |\n",
      "|    n_updates        | 11971    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1036     |\n",
      "|    fps              | 675      |\n",
      "|    time_elapsed     | 1212     |\n",
      "|    total_timesteps  | 819040   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.278    |\n",
      "|    n_updates        | 12016    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1040     |\n",
      "|    fps              | 675      |\n",
      "|    time_elapsed     | 1217     |\n",
      "|    total_timesteps  | 822384   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.722    |\n",
      "|    n_updates        | 12068    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1044     |\n",
      "|    fps              | 675      |\n",
      "|    time_elapsed     | 1221     |\n",
      "|    total_timesteps  | 824912   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.356    |\n",
      "|    n_updates        | 12108    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1048     |\n",
      "|    fps              | 674      |\n",
      "|    time_elapsed     | 1228     |\n",
      "|    total_timesteps  | 828960   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.722    |\n",
      "|    n_updates        | 12171    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1052     |\n",
      "|    fps              | 674      |\n",
      "|    time_elapsed     | 1232     |\n",
      "|    total_timesteps  | 831840   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.185    |\n",
      "|    n_updates        | 12216    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1056     |\n",
      "|    fps              | 674      |\n",
      "|    time_elapsed     | 1237     |\n",
      "|    total_timesteps  | 835184   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.229    |\n",
      "|    n_updates        | 12268    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1060     |\n",
      "|    fps              | 674      |\n",
      "|    time_elapsed     | 1241     |\n",
      "|    total_timesteps  | 837712   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.378    |\n",
      "|    n_updates        | 12308    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1064     |\n",
      "|    fps              | 674      |\n",
      "|    time_elapsed     | 1248     |\n",
      "|    total_timesteps  | 841760   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.267    |\n",
      "|    n_updates        | 12371    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1068     |\n",
      "|    fps              | 674      |\n",
      "|    time_elapsed     | 1252     |\n",
      "|    total_timesteps  | 844640   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.386    |\n",
      "|    n_updates        | 12416    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1072     |\n",
      "|    fps              | 674      |\n",
      "|    time_elapsed     | 1258     |\n",
      "|    total_timesteps  | 847984   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.391    |\n",
      "|    n_updates        | 12468    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1076     |\n",
      "|    fps              | 673      |\n",
      "|    time_elapsed     | 1261     |\n",
      "|    total_timesteps  | 850512   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.0417   |\n",
      "|    n_updates        | 12508    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1080     |\n",
      "|    fps              | 673      |\n",
      "|    time_elapsed     | 1268     |\n",
      "|    total_timesteps  | 854560   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.592    |\n",
      "|    n_updates        | 12571    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1084     |\n",
      "|    fps              | 673      |\n",
      "|    time_elapsed     | 1272     |\n",
      "|    total_timesteps  | 857440   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.235    |\n",
      "|    n_updates        | 12616    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1088     |\n",
      "|    fps              | 673      |\n",
      "|    time_elapsed     | 1277     |\n",
      "|    total_timesteps  | 860784   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.506    |\n",
      "|    n_updates        | 12668    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1092     |\n",
      "|    fps              | 673      |\n",
      "|    time_elapsed     | 1281     |\n",
      "|    total_timesteps  | 863312   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.153    |\n",
      "|    n_updates        | 12708    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1096     |\n",
      "|    fps              | 673      |\n",
      "|    time_elapsed     | 1288     |\n",
      "|    total_timesteps  | 867360   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.337    |\n",
      "|    n_updates        | 12771    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1100     |\n",
      "|    fps              | 673      |\n",
      "|    time_elapsed     | 1292     |\n",
      "|    total_timesteps  | 870240   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.137    |\n",
      "|    n_updates        | 12816    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1104     |\n",
      "|    fps              | 673      |\n",
      "|    time_elapsed     | 1297     |\n",
      "|    total_timesteps  | 873584   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.207    |\n",
      "|    n_updates        | 12868    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1108     |\n",
      "|    fps              | 673      |\n",
      "|    time_elapsed     | 1301     |\n",
      "|    total_timesteps  | 876112   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.133    |\n",
      "|    n_updates        | 12908    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1112     |\n",
      "|    fps              | 673      |\n",
      "|    time_elapsed     | 1307     |\n",
      "|    total_timesteps  | 880160   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.266    |\n",
      "|    n_updates        | 12971    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1116     |\n",
      "|    fps              | 672      |\n",
      "|    time_elapsed     | 1312     |\n",
      "|    total_timesteps  | 883040   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.227    |\n",
      "|    n_updates        | 13016    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1120     |\n",
      "|    fps              | 672      |\n",
      "|    time_elapsed     | 1317     |\n",
      "|    total_timesteps  | 886384   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.297    |\n",
      "|    n_updates        | 13068    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1124     |\n",
      "|    fps              | 672      |\n",
      "|    time_elapsed     | 1321     |\n",
      "|    total_timesteps  | 888912   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.609    |\n",
      "|    n_updates        | 13108    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1128     |\n",
      "|    fps              | 672      |\n",
      "|    time_elapsed     | 1327     |\n",
      "|    total_timesteps  | 892960   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.431    |\n",
      "|    n_updates        | 13171    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1132     |\n",
      "|    fps              | 672      |\n",
      "|    time_elapsed     | 1332     |\n",
      "|    total_timesteps  | 895840   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.166    |\n",
      "|    n_updates        | 13216    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1136     |\n",
      "|    fps              | 672      |\n",
      "|    time_elapsed     | 1337     |\n",
      "|    total_timesteps  | 899184   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.13     |\n",
      "|    n_updates        | 13268    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1140     |\n",
      "|    fps              | 672      |\n",
      "|    time_elapsed     | 1341     |\n",
      "|    total_timesteps  | 901712   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.451    |\n",
      "|    n_updates        | 13308    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1144     |\n",
      "|    fps              | 672      |\n",
      "|    time_elapsed     | 1347     |\n",
      "|    total_timesteps  | 905760   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.196    |\n",
      "|    n_updates        | 13371    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1148     |\n",
      "|    fps              | 672      |\n",
      "|    time_elapsed     | 1351     |\n",
      "|    total_timesteps  | 908640   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.512    |\n",
      "|    n_updates        | 13416    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1152     |\n",
      "|    fps              | 671      |\n",
      "|    time_elapsed     | 1357     |\n",
      "|    total_timesteps  | 911984   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.272    |\n",
      "|    n_updates        | 13468    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1156     |\n",
      "|    fps              | 671      |\n",
      "|    time_elapsed     | 1361     |\n",
      "|    total_timesteps  | 914512   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.184    |\n",
      "|    n_updates        | 13508    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1160     |\n",
      "|    fps              | 671      |\n",
      "|    time_elapsed     | 1367     |\n",
      "|    total_timesteps  | 918560   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.216    |\n",
      "|    n_updates        | 13571    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1164     |\n",
      "|    fps              | 671      |\n",
      "|    time_elapsed     | 1371     |\n",
      "|    total_timesteps  | 921440   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.313    |\n",
      "|    n_updates        | 13616    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1168     |\n",
      "|    fps              | 671      |\n",
      "|    time_elapsed     | 1376     |\n",
      "|    total_timesteps  | 924304   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.219    |\n",
      "|    n_updates        | 13661    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1172     |\n",
      "|    fps              | 671      |\n",
      "|    time_elapsed     | 1381     |\n",
      "|    total_timesteps  | 927312   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.189    |\n",
      "|    n_updates        | 13708    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1176     |\n",
      "|    fps              | 671      |\n",
      "|    time_elapsed     | 1387     |\n",
      "|    total_timesteps  | 931360   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.425    |\n",
      "|    n_updates        | 13771    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1180     |\n",
      "|    fps              | 671      |\n",
      "|    time_elapsed     | 1392     |\n",
      "|    total_timesteps  | 934240   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.312    |\n",
      "|    n_updates        | 13816    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1184     |\n",
      "|    fps              | 670      |\n",
      "|    time_elapsed     | 1396     |\n",
      "|    total_timesteps  | 937104   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.201    |\n",
      "|    n_updates        | 13861    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1188     |\n",
      "|    fps              | 670      |\n",
      "|    time_elapsed     | 1401     |\n",
      "|    total_timesteps  | 940112   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.291    |\n",
      "|    n_updates        | 13908    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1192     |\n",
      "|    fps              | 670      |\n",
      "|    time_elapsed     | 1407     |\n",
      "|    total_timesteps  | 944160   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.384    |\n",
      "|    n_updates        | 13971    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1196     |\n",
      "|    fps              | 670      |\n",
      "|    time_elapsed     | 1412     |\n",
      "|    total_timesteps  | 947040   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.548    |\n",
      "|    n_updates        | 14016    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1200     |\n",
      "|    fps              | 670      |\n",
      "|    time_elapsed     | 1416     |\n",
      "|    total_timesteps  | 949904   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.234    |\n",
      "|    n_updates        | 14061    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1204     |\n",
      "|    fps              | 670      |\n",
      "|    time_elapsed     | 1421     |\n",
      "|    total_timesteps  | 952912   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.284    |\n",
      "|    n_updates        | 14108    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1208     |\n",
      "|    fps              | 670      |\n",
      "|    time_elapsed     | 1428     |\n",
      "|    total_timesteps  | 956960   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.652    |\n",
      "|    n_updates        | 14171    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1212     |\n",
      "|    fps              | 670      |\n",
      "|    time_elapsed     | 1432     |\n",
      "|    total_timesteps  | 959840   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.262    |\n",
      "|    n_updates        | 14216    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1216     |\n",
      "|    fps              | 670      |\n",
      "|    time_elapsed     | 1436     |\n",
      "|    total_timesteps  | 962704   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.215    |\n",
      "|    n_updates        | 14261    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1220     |\n",
      "|    fps              | 669      |\n",
      "|    time_elapsed     | 1441     |\n",
      "|    total_timesteps  | 965712   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.218    |\n",
      "|    n_updates        | 14308    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1224     |\n",
      "|    fps              | 669      |\n",
      "|    time_elapsed     | 1447     |\n",
      "|    total_timesteps  | 969760   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.546    |\n",
      "|    n_updates        | 14371    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1228     |\n",
      "|    fps              | 669      |\n",
      "|    time_elapsed     | 1452     |\n",
      "|    total_timesteps  | 972640   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.793    |\n",
      "|    n_updates        | 14416    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1232     |\n",
      "|    fps              | 669      |\n",
      "|    time_elapsed     | 1456     |\n",
      "|    total_timesteps  | 975504   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.4      |\n",
      "|    n_updates        | 14461    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1236     |\n",
      "|    fps              | 669      |\n",
      "|    time_elapsed     | 1461     |\n",
      "|    total_timesteps  | 978512   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.554    |\n",
      "|    n_updates        | 14508    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1240     |\n",
      "|    fps              | 669      |\n",
      "|    time_elapsed     | 1467     |\n",
      "|    total_timesteps  | 982560   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.76     |\n",
      "|    n_updates        | 14571    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1244     |\n",
      "|    fps              | 669      |\n",
      "|    time_elapsed     | 1472     |\n",
      "|    total_timesteps  | 985440   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.428    |\n",
      "|    n_updates        | 14616    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1248     |\n",
      "|    fps              | 669      |\n",
      "|    time_elapsed     | 1476     |\n",
      "|    total_timesteps  | 988304   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.715    |\n",
      "|    n_updates        | 14661    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1252     |\n",
      "|    fps              | 669      |\n",
      "|    time_elapsed     | 1481     |\n",
      "|    total_timesteps  | 991312   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.366    |\n",
      "|    n_updates        | 14708    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1256     |\n",
      "|    fps              | 669      |\n",
      "|    time_elapsed     | 1487     |\n",
      "|    total_timesteps  | 995360   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.535    |\n",
      "|    n_updates        | 14771    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1260     |\n",
      "|    fps              | 669      |\n",
      "|    time_elapsed     | 1491     |\n",
      "|    total_timesteps  | 998240   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.51     |\n",
      "|    n_updates        | 14816    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1264     |\n",
      "|    fps              | 669      |\n",
      "|    time_elapsed     | 1496     |\n",
      "|    total_timesteps  | 1001104  |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.424    |\n",
      "|    n_updates        | 14861    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1268     |\n",
      "|    fps              | 669      |\n",
      "|    time_elapsed     | 1500     |\n",
      "|    total_timesteps  | 1004112  |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.535    |\n",
      "|    n_updates        | 14908    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1272     |\n",
      "|    fps              | 668      |\n",
      "|    time_elapsed     | 1506     |\n",
      "|    total_timesteps  | 1008160  |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.492    |\n",
      "|    n_updates        | 14971    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1276     |\n",
      "|    fps              | 668      |\n",
      "|    time_elapsed     | 1511     |\n",
      "|    total_timesteps  | 1011040  |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.369    |\n",
      "|    n_updates        | 15016    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1280     |\n",
      "|    fps              | 668      |\n",
      "|    time_elapsed     | 1515     |\n",
      "|    total_timesteps  | 1013904  |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.692    |\n",
      "|    n_updates        | 15061    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1284     |\n",
      "|    fps              | 668      |\n",
      "|    time_elapsed     | 1520     |\n",
      "|    total_timesteps  | 1016912  |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.38     |\n",
      "|    n_updates        | 15108    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1288     |\n",
      "|    fps              | 668      |\n",
      "|    time_elapsed     | 1526     |\n",
      "|    total_timesteps  | 1020960  |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.511    |\n",
      "|    n_updates        | 15171    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1292     |\n",
      "|    fps              | 668      |\n",
      "|    time_elapsed     | 1530     |\n",
      "|    total_timesteps  | 1023840  |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.702    |\n",
      "|    n_updates        | 15216    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1296     |\n",
      "|    fps              | 668      |\n",
      "|    time_elapsed     | 1535     |\n",
      "|    total_timesteps  | 1026704  |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.631    |\n",
      "|    n_updates        | 15261    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1300     |\n",
      "|    fps              | 668      |\n",
      "|    time_elapsed     | 1539     |\n",
      "|    total_timesteps  | 1029712  |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.616    |\n",
      "|    n_updates        | 15308    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1304     |\n",
      "|    fps              | 668      |\n",
      "|    time_elapsed     | 1545     |\n",
      "|    total_timesteps  | 1033760  |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.992    |\n",
      "|    n_updates        | 15371    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1308     |\n",
      "|    fps              | 668      |\n",
      "|    time_elapsed     | 1550     |\n",
      "|    total_timesteps  | 1036640  |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.56     |\n",
      "|    n_updates        | 15416    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1312     |\n",
      "|    fps              | 668      |\n",
      "|    time_elapsed     | 1554     |\n",
      "|    total_timesteps  | 1039504  |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.277    |\n",
      "|    n_updates        | 15461    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1316     |\n",
      "|    fps              | 668      |\n",
      "|    time_elapsed     | 1558     |\n",
      "|    total_timesteps  | 1042512  |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.544    |\n",
      "|    n_updates        | 15508    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1320     |\n",
      "|    fps              | 669      |\n",
      "|    time_elapsed     | 1563     |\n",
      "|    total_timesteps  | 1046560  |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.554    |\n",
      "|    n_updates        | 15571    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1324     |\n",
      "|    fps              | 669      |\n",
      "|    time_elapsed     | 1567     |\n",
      "|    total_timesteps  | 1049440  |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.437    |\n",
      "|    n_updates        | 15616    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1328     |\n",
      "|    fps              | 669      |\n",
      "|    time_elapsed     | 1571     |\n",
      "|    total_timesteps  | 1052304  |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.491    |\n",
      "|    n_updates        | 15661    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1332     |\n",
      "|    fps              | 669      |\n",
      "|    time_elapsed     | 1576     |\n",
      "|    total_timesteps  | 1055312  |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.387    |\n",
      "|    n_updates        | 15708    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1336     |\n",
      "|    fps              | 669      |\n",
      "|    time_elapsed     | 1582     |\n",
      "|    total_timesteps  | 1059360  |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.626    |\n",
      "|    n_updates        | 15771    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1340     |\n",
      "|    fps              | 669      |\n",
      "|    time_elapsed     | 1587     |\n",
      "|    total_timesteps  | 1062240  |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.432    |\n",
      "|    n_updates        | 15816    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1344     |\n",
      "|    fps              | 669      |\n",
      "|    time_elapsed     | 1591     |\n",
      "|    total_timesteps  | 1065104  |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.697    |\n",
      "|    n_updates        | 15861    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1348     |\n",
      "|    fps              | 669      |\n",
      "|    time_elapsed     | 1596     |\n",
      "|    total_timesteps  | 1068112  |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.386    |\n",
      "|    n_updates        | 15908    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1352     |\n",
      "|    fps              | 669      |\n",
      "|    time_elapsed     | 1602     |\n",
      "|    total_timesteps  | 1072160  |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.349    |\n",
      "|    n_updates        | 15971    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1356     |\n",
      "|    fps              | 669      |\n",
      "|    time_elapsed     | 1606     |\n",
      "|    total_timesteps  | 1075040  |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.528    |\n",
      "|    n_updates        | 16016    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1360     |\n",
      "|    fps              | 669      |\n",
      "|    time_elapsed     | 1610     |\n",
      "|    total_timesteps  | 1077904  |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.508    |\n",
      "|    n_updates        | 16061    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1364     |\n",
      "|    fps              | 669      |\n",
      "|    time_elapsed     | 1614     |\n",
      "|    total_timesteps  | 1080912  |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.237    |\n",
      "|    n_updates        | 16108    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1368     |\n",
      "|    fps              | 669      |\n",
      "|    time_elapsed     | 1620     |\n",
      "|    total_timesteps  | 1084960  |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.604    |\n",
      "|    n_updates        | 16171    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1372     |\n",
      "|    fps              | 669      |\n",
      "|    time_elapsed     | 1625     |\n",
      "|    total_timesteps  | 1087840  |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.78     |\n",
      "|    n_updates        | 16216    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1376     |\n",
      "|    fps              | 669      |\n",
      "|    time_elapsed     | 1629     |\n",
      "|    total_timesteps  | 1090704  |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.423    |\n",
      "|    n_updates        | 16261    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1380     |\n",
      "|    fps              | 669      |\n",
      "|    time_elapsed     | 1634     |\n",
      "|    total_timesteps  | 1093712  |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.497    |\n",
      "|    n_updates        | 16308    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1384     |\n",
      "|    fps              | 669      |\n",
      "|    time_elapsed     | 1640     |\n",
      "|    total_timesteps  | 1097760  |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.344    |\n",
      "|    n_updates        | 16371    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1388     |\n",
      "|    fps              | 669      |\n",
      "|    time_elapsed     | 1645     |\n",
      "|    total_timesteps  | 1100640  |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.722    |\n",
      "|    n_updates        | 16416    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1392     |\n",
      "|    fps              | 668      |\n",
      "|    time_elapsed     | 1649     |\n",
      "|    total_timesteps  | 1103504  |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.387    |\n",
      "|    n_updates        | 16461    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1396     |\n",
      "|    fps              | 668      |\n",
      "|    time_elapsed     | 1654     |\n",
      "|    total_timesteps  | 1106512  |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.339    |\n",
      "|    n_updates        | 16508    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1400     |\n",
      "|    fps              | 668      |\n",
      "|    time_elapsed     | 1660     |\n",
      "|    total_timesteps  | 1110560  |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.666    |\n",
      "|    n_updates        | 16571    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1404     |\n",
      "|    fps              | 668      |\n",
      "|    time_elapsed     | 1664     |\n",
      "|    total_timesteps  | 1113440  |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.516    |\n",
      "|    n_updates        | 16616    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1408     |\n",
      "|    fps              | 669      |\n",
      "|    time_elapsed     | 1668     |\n",
      "|    total_timesteps  | 1116304  |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.529    |\n",
      "|    n_updates        | 16661    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1412     |\n",
      "|    fps              | 669      |\n",
      "|    time_elapsed     | 1672     |\n",
      "|    total_timesteps  | 1119312  |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.552    |\n",
      "|    n_updates        | 16708    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1416     |\n",
      "|    fps              | 669      |\n",
      "|    time_elapsed     | 1678     |\n",
      "|    total_timesteps  | 1123360  |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.743    |\n",
      "|    n_updates        | 16771    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1420     |\n",
      "|    fps              | 669      |\n",
      "|    time_elapsed     | 1683     |\n",
      "|    total_timesteps  | 1126240  |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.607    |\n",
      "|    n_updates        | 16816    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1424     |\n",
      "|    fps              | 669      |\n",
      "|    time_elapsed     | 1687     |\n",
      "|    total_timesteps  | 1129104  |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.135    |\n",
      "|    n_updates        | 16861    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1428     |\n",
      "|    fps              | 669      |\n",
      "|    time_elapsed     | 1691     |\n",
      "|    total_timesteps  | 1132112  |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.3      |\n",
      "|    n_updates        | 16908    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1432     |\n",
      "|    fps              | 669      |\n",
      "|    time_elapsed     | 1697     |\n",
      "|    total_timesteps  | 1136160  |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.395    |\n",
      "|    n_updates        | 16971    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1436     |\n",
      "|    fps              | 669      |\n",
      "|    time_elapsed     | 1701     |\n",
      "|    total_timesteps  | 1139040  |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.455    |\n",
      "|    n_updates        | 17016    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1440     |\n",
      "|    fps              | 669      |\n",
      "|    time_elapsed     | 1706     |\n",
      "|    total_timesteps  | 1141904  |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.416    |\n",
      "|    n_updates        | 17061    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1444     |\n",
      "|    fps              | 669      |\n",
      "|    time_elapsed     | 1710     |\n",
      "|    total_timesteps  | 1144912  |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.557    |\n",
      "|    n_updates        | 17108    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1448     |\n",
      "|    fps              | 669      |\n",
      "|    time_elapsed     | 1717     |\n",
      "|    total_timesteps  | 1148960  |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.426    |\n",
      "|    n_updates        | 17171    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1452     |\n",
      "|    fps              | 668      |\n",
      "|    time_elapsed     | 1722     |\n",
      "|    total_timesteps  | 1151840  |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.216    |\n",
      "|    n_updates        | 17216    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1456     |\n",
      "|    fps              | 668      |\n",
      "|    time_elapsed     | 1726     |\n",
      "|    total_timesteps  | 1154704  |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.687    |\n",
      "|    n_updates        | 17261    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1460     |\n",
      "|    fps              | 668      |\n",
      "|    time_elapsed     | 1731     |\n",
      "|    total_timesteps  | 1157712  |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.424    |\n",
      "|    n_updates        | 17308    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1464     |\n",
      "|    fps              | 668      |\n",
      "|    time_elapsed     | 1737     |\n",
      "|    total_timesteps  | 1161760  |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.711    |\n",
      "|    n_updates        | 17371    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1468     |\n",
      "|    fps              | 668      |\n",
      "|    time_elapsed     | 1741     |\n",
      "|    total_timesteps  | 1164640  |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.308    |\n",
      "|    n_updates        | 17416    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1472     |\n",
      "|    fps              | 668      |\n",
      "|    time_elapsed     | 1746     |\n",
      "|    total_timesteps  | 1167504  |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.617    |\n",
      "|    n_updates        | 17461    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1476     |\n",
      "|    fps              | 668      |\n",
      "|    time_elapsed     | 1751     |\n",
      "|    total_timesteps  | 1170512  |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.395    |\n",
      "|    n_updates        | 17508    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1480     |\n",
      "|    fps              | 668      |\n",
      "|    time_elapsed     | 1757     |\n",
      "|    total_timesteps  | 1174560  |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.623    |\n",
      "|    n_updates        | 17571    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1484     |\n",
      "|    fps              | 668      |\n",
      "|    time_elapsed     | 1762     |\n",
      "|    total_timesteps  | 1177440  |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.797    |\n",
      "|    n_updates        | 17616    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1488     |\n",
      "|    fps              | 668      |\n",
      "|    time_elapsed     | 1766     |\n",
      "|    total_timesteps  | 1180304  |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.38     |\n",
      "|    n_updates        | 17661    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1492     |\n",
      "|    fps              | 668      |\n",
      "|    time_elapsed     | 1771     |\n",
      "|    total_timesteps  | 1183312  |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.559    |\n",
      "|    n_updates        | 17708    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1496     |\n",
      "|    fps              | 667      |\n",
      "|    time_elapsed     | 1777     |\n",
      "|    total_timesteps  | 1187360  |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.61     |\n",
      "|    n_updates        | 17771    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1500     |\n",
      "|    fps              | 667      |\n",
      "|    time_elapsed     | 1781     |\n",
      "|    total_timesteps  | 1190240  |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.539    |\n",
      "|    n_updates        | 17816    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1504     |\n",
      "|    fps              | 667      |\n",
      "|    time_elapsed     | 1786     |\n",
      "|    total_timesteps  | 1193104  |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.757    |\n",
      "|    n_updates        | 17861    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1508     |\n",
      "|    fps              | 667      |\n",
      "|    time_elapsed     | 1791     |\n",
      "|    total_timesteps  | 1196112  |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.371    |\n",
      "|    n_updates        | 17908    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1512     |\n",
      "|    fps              | 667      |\n",
      "|    time_elapsed     | 1797     |\n",
      "|    total_timesteps  | 1200160  |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.779    |\n",
      "|    n_updates        | 17971    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1516     |\n",
      "|    fps              | 667      |\n",
      "|    time_elapsed     | 1801     |\n",
      "|    total_timesteps  | 1203040  |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.737    |\n",
      "|    n_updates        | 18016    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1520     |\n",
      "|    fps              | 667      |\n",
      "|    time_elapsed     | 1806     |\n",
      "|    total_timesteps  | 1205904  |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.376    |\n",
      "|    n_updates        | 18061    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1524     |\n",
      "|    fps              | 667      |\n",
      "|    time_elapsed     | 1811     |\n",
      "|    total_timesteps  | 1208912  |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.47     |\n",
      "|    n_updates        | 18108    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1528     |\n",
      "|    fps              | 667      |\n",
      "|    time_elapsed     | 1817     |\n",
      "|    total_timesteps  | 1212960  |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.587    |\n",
      "|    n_updates        | 18171    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1532     |\n",
      "|    fps              | 667      |\n",
      "|    time_elapsed     | 1822     |\n",
      "|    total_timesteps  | 1215840  |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.724    |\n",
      "|    n_updates        | 18216    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1536     |\n",
      "|    fps              | 667      |\n",
      "|    time_elapsed     | 1826     |\n",
      "|    total_timesteps  | 1218704  |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.698    |\n",
      "|    n_updates        | 18261    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1540     |\n",
      "|    fps              | 666      |\n",
      "|    time_elapsed     | 1831     |\n",
      "|    total_timesteps  | 1221712  |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.588    |\n",
      "|    n_updates        | 18308    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1544     |\n",
      "|    fps              | 666      |\n",
      "|    time_elapsed     | 1838     |\n",
      "|    total_timesteps  | 1225760  |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.574    |\n",
      "|    n_updates        | 18371    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1548     |\n",
      "|    fps              | 666      |\n",
      "|    time_elapsed     | 1842     |\n",
      "|    total_timesteps  | 1228640  |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.588    |\n",
      "|    n_updates        | 18416    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1552     |\n",
      "|    fps              | 666      |\n",
      "|    time_elapsed     | 1846     |\n",
      "|    total_timesteps  | 1231504  |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.619    |\n",
      "|    n_updates        | 18461    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1556     |\n",
      "|    fps              | 666      |\n",
      "|    time_elapsed     | 1851     |\n",
      "|    total_timesteps  | 1234512  |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.359    |\n",
      "|    n_updates        | 18508    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1560     |\n",
      "|    fps              | 666      |\n",
      "|    time_elapsed     | 1857     |\n",
      "|    total_timesteps  | 1237904  |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.46     |\n",
      "|    n_updates        | 18561    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1564     |\n",
      "|    fps              | 666      |\n",
      "|    time_elapsed     | 1861     |\n",
      "|    total_timesteps  | 1240976  |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.892    |\n",
      "|    n_updates        | 18609    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1568     |\n",
      "|    fps              | 666      |\n",
      "|    time_elapsed     | 1865     |\n",
      "|    total_timesteps  | 1243504  |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.63     |\n",
      "|    n_updates        | 18648    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1572     |\n",
      "|    fps              | 666      |\n",
      "|    time_elapsed     | 1871     |\n",
      "|    total_timesteps  | 1247312  |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.592    |\n",
      "|    n_updates        | 18708    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1576     |\n",
      "|    fps              | 666      |\n",
      "|    time_elapsed     | 1877     |\n",
      "|    total_timesteps  | 1250704  |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.495    |\n",
      "|    n_updates        | 18761    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1580     |\n",
      "|    fps              | 666      |\n",
      "|    time_elapsed     | 1882     |\n",
      "|    total_timesteps  | 1253776  |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 1.05     |\n",
      "|    n_updates        | 18809    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1584     |\n",
      "|    fps              | 665      |\n",
      "|    time_elapsed     | 1886     |\n",
      "|    total_timesteps  | 1256128  |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.683    |\n",
      "|    n_updates        | 18845    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1588     |\n",
      "|    fps              | 665      |\n",
      "|    time_elapsed     | 1892     |\n",
      "|    total_timesteps  | 1260112  |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.572    |\n",
      "|    n_updates        | 18908    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1592     |\n",
      "|    fps              | 665      |\n",
      "|    time_elapsed     | 1898     |\n",
      "|    total_timesteps  | 1263376  |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.281    |\n",
      "|    n_updates        | 18959    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1596     |\n",
      "|    fps              | 665      |\n",
      "|    time_elapsed     | 1903     |\n",
      "|    total_timesteps  | 1266576  |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.454    |\n",
      "|    n_updates        | 19009    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1600     |\n",
      "|    fps              | 665      |\n",
      "|    time_elapsed     | 1907     |\n",
      "|    total_timesteps  | 1268928  |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.406    |\n",
      "|    n_updates        | 19045    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1604     |\n",
      "|    fps              | 665      |\n",
      "|    time_elapsed     | 1913     |\n",
      "|    total_timesteps  | 1272912  |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.958    |\n",
      "|    n_updates        | 19108    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1608     |\n",
      "|    fps              | 664      |\n",
      "|    time_elapsed     | 1919     |\n",
      "|    total_timesteps  | 1276176  |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.361    |\n",
      "|    n_updates        | 19159    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1612     |\n",
      "|    fps              | 664      |\n",
      "|    time_elapsed     | 1924     |\n",
      "|    total_timesteps  | 1279376  |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.794    |\n",
      "|    n_updates        | 19209    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1616     |\n",
      "|    fps              | 664      |\n",
      "|    time_elapsed     | 1928     |\n",
      "|    total_timesteps  | 1281728  |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.6      |\n",
      "|    n_updates        | 19245    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1620     |\n",
      "|    fps              | 664      |\n",
      "|    time_elapsed     | 1934     |\n",
      "|    total_timesteps  | 1285712  |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.61     |\n",
      "|    n_updates        | 19308    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1624     |\n",
      "|    fps              | 664      |\n",
      "|    time_elapsed     | 1939     |\n",
      "|    total_timesteps  | 1288976  |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.593    |\n",
      "|    n_updates        | 19359    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1628     |\n",
      "|    fps              | 664      |\n",
      "|    time_elapsed     | 1944     |\n",
      "|    total_timesteps  | 1292176  |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.769    |\n",
      "|    n_updates        | 19409    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1632     |\n",
      "|    fps              | 664      |\n",
      "|    time_elapsed     | 1948     |\n",
      "|    total_timesteps  | 1294528  |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.657    |\n",
      "|    n_updates        | 19445    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1636     |\n",
      "|    fps              | 664      |\n",
      "|    time_elapsed     | 1955     |\n",
      "|    total_timesteps  | 1298512  |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.702    |\n",
      "|    n_updates        | 19508    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1640     |\n",
      "|    fps              | 664      |\n",
      "|    time_elapsed     | 1960     |\n",
      "|    total_timesteps  | 1301776  |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.451    |\n",
      "|    n_updates        | 19559    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1644     |\n",
      "|    fps              | 663      |\n",
      "|    time_elapsed     | 1965     |\n",
      "|    total_timesteps  | 1304976  |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.548    |\n",
      "|    n_updates        | 19609    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1648     |\n",
      "|    fps              | 663      |\n",
      "|    time_elapsed     | 1969     |\n",
      "|    total_timesteps  | 1307328  |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.549    |\n",
      "|    n_updates        | 19645    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1652     |\n",
      "|    fps              | 663      |\n",
      "|    time_elapsed     | 1975     |\n",
      "|    total_timesteps  | 1311312  |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.744    |\n",
      "|    n_updates        | 19708    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1656     |\n",
      "|    fps              | 663      |\n",
      "|    time_elapsed     | 1980     |\n",
      "|    total_timesteps  | 1314576  |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.987    |\n",
      "|    n_updates        | 19759    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1660     |\n",
      "|    fps              | 663      |\n",
      "|    time_elapsed     | 1986     |\n",
      "|    total_timesteps  | 1317776  |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.495    |\n",
      "|    n_updates        | 19809    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1664     |\n",
      "|    fps              | 663      |\n",
      "|    time_elapsed     | 1989     |\n",
      "|    total_timesteps  | 1320128  |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.895    |\n",
      "|    n_updates        | 19845    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1668     |\n",
      "|    fps              | 663      |\n",
      "|    time_elapsed     | 1996     |\n",
      "|    total_timesteps  | 1324112  |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.407    |\n",
      "|    n_updates        | 19908    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1672     |\n",
      "|    fps              | 663      |\n",
      "|    time_elapsed     | 2001     |\n",
      "|    total_timesteps  | 1327376  |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.853    |\n",
      "|    n_updates        | 19959    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1676     |\n",
      "|    fps              | 663      |\n",
      "|    time_elapsed     | 2006     |\n",
      "|    total_timesteps  | 1330576  |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.502    |\n",
      "|    n_updates        | 20009    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1680     |\n",
      "|    fps              | 662      |\n",
      "|    time_elapsed     | 2010     |\n",
      "|    total_timesteps  | 1332928  |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.644    |\n",
      "|    n_updates        | 20045    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1684     |\n",
      "|    fps              | 662      |\n",
      "|    time_elapsed     | 2016     |\n",
      "|    total_timesteps  | 1336912  |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.579    |\n",
      "|    n_updates        | 20108    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1688     |\n",
      "|    fps              | 662      |\n",
      "|    time_elapsed     | 2022     |\n",
      "|    total_timesteps  | 1340176  |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.847    |\n",
      "|    n_updates        | 20159    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1692     |\n",
      "|    fps              | 662      |\n",
      "|    time_elapsed     | 2027     |\n",
      "|    total_timesteps  | 1343376  |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.676    |\n",
      "|    n_updates        | 20209    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1696     |\n",
      "|    fps              | 662      |\n",
      "|    time_elapsed     | 2031     |\n",
      "|    total_timesteps  | 1345728  |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.963    |\n",
      "|    n_updates        | 20245    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1700     |\n",
      "|    fps              | 662      |\n",
      "|    time_elapsed     | 2037     |\n",
      "|    total_timesteps  | 1349712  |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.959    |\n",
      "|    n_updates        | 20308    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1704     |\n",
      "|    fps              | 662      |\n",
      "|    time_elapsed     | 2042     |\n",
      "|    total_timesteps  | 1352976  |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.551    |\n",
      "|    n_updates        | 20359    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1708     |\n",
      "|    fps              | 662      |\n",
      "|    time_elapsed     | 2047     |\n",
      "|    total_timesteps  | 1356176  |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.683    |\n",
      "|    n_updates        | 20409    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1712     |\n",
      "|    fps              | 662      |\n",
      "|    time_elapsed     | 2051     |\n",
      "|    total_timesteps  | 1358528  |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.583    |\n",
      "|    n_updates        | 20445    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1716     |\n",
      "|    fps              | 662      |\n",
      "|    time_elapsed     | 2057     |\n",
      "|    total_timesteps  | 1362512  |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.574    |\n",
      "|    n_updates        | 20508    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1720     |\n",
      "|    fps              | 662      |\n",
      "|    time_elapsed     | 2062     |\n",
      "|    total_timesteps  | 1365776  |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.761    |\n",
      "|    n_updates        | 20559    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1724     |\n",
      "|    fps              | 661      |\n",
      "|    time_elapsed     | 2067     |\n",
      "|    total_timesteps  | 1368976  |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.92     |\n",
      "|    n_updates        | 20609    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1728     |\n",
      "|    fps              | 661      |\n",
      "|    time_elapsed     | 2071     |\n",
      "|    total_timesteps  | 1371328  |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.86     |\n",
      "|    n_updates        | 20645    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1732     |\n",
      "|    fps              | 661      |\n",
      "|    time_elapsed     | 2077     |\n",
      "|    total_timesteps  | 1375312  |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.618    |\n",
      "|    n_updates        | 20708    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1736     |\n",
      "|    fps              | 661      |\n",
      "|    time_elapsed     | 2082     |\n",
      "|    total_timesteps  | 1378576  |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.654    |\n",
      "|    n_updates        | 20759    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1740     |\n",
      "|    fps              | 661      |\n",
      "|    time_elapsed     | 2087     |\n",
      "|    total_timesteps  | 1381776  |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.727    |\n",
      "|    n_updates        | 20809    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1744     |\n",
      "|    fps              | 661      |\n",
      "|    time_elapsed     | 2091     |\n",
      "|    total_timesteps  | 1384128  |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.921    |\n",
      "|    n_updates        | 20845    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1748     |\n",
      "|    fps              | 661      |\n",
      "|    time_elapsed     | 2097     |\n",
      "|    total_timesteps  | 1388112  |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.588    |\n",
      "|    n_updates        | 20908    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1752     |\n",
      "|    fps              | 661      |\n",
      "|    time_elapsed     | 2102     |\n",
      "|    total_timesteps  | 1391376  |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.997    |\n",
      "|    n_updates        | 20959    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1756     |\n",
      "|    fps              | 661      |\n",
      "|    time_elapsed     | 2107     |\n",
      "|    total_timesteps  | 1394576  |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.451    |\n",
      "|    n_updates        | 21009    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1760     |\n",
      "|    fps              | 661      |\n",
      "|    time_elapsed     | 2111     |\n",
      "|    total_timesteps  | 1396928  |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.989    |\n",
      "|    n_updates        | 21045    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1764     |\n",
      "|    fps              | 661      |\n",
      "|    time_elapsed     | 2117     |\n",
      "|    total_timesteps  | 1400912  |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.44     |\n",
      "|    n_updates        | 21108    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1768     |\n",
      "|    fps              | 661      |\n",
      "|    time_elapsed     | 2122     |\n",
      "|    total_timesteps  | 1404176  |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.742    |\n",
      "|    n_updates        | 21159    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1772     |\n",
      "|    fps              | 661      |\n",
      "|    time_elapsed     | 2127     |\n",
      "|    total_timesteps  | 1407376  |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.644    |\n",
      "|    n_updates        | 21209    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1776     |\n",
      "|    fps              | 661      |\n",
      "|    time_elapsed     | 2131     |\n",
      "|    total_timesteps  | 1409728  |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.864    |\n",
      "|    n_updates        | 21245    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1780     |\n",
      "|    fps              | 661      |\n",
      "|    time_elapsed     | 2137     |\n",
      "|    total_timesteps  | 1413712  |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.702    |\n",
      "|    n_updates        | 21308    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1784     |\n",
      "|    fps              | 661      |\n",
      "|    time_elapsed     | 2143     |\n",
      "|    total_timesteps  | 1416976  |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.699    |\n",
      "|    n_updates        | 21359    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1788     |\n",
      "|    fps              | 661      |\n",
      "|    time_elapsed     | 2147     |\n",
      "|    total_timesteps  | 1419408  |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.752    |\n",
      "|    n_updates        | 21397    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1792     |\n",
      "|    fps              | 661      |\n",
      "|    time_elapsed     | 2150     |\n",
      "|    total_timesteps  | 1421760  |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.799    |\n",
      "|    n_updates        | 21433    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1796     |\n",
      "|    fps              | 660      |\n",
      "|    time_elapsed     | 2157     |\n",
      "|    total_timesteps  | 1426160  |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.581    |\n",
      "|    n_updates        | 21502    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1800     |\n",
      "|    fps              | 660      |\n",
      "|    time_elapsed     | 2163     |\n",
      "|    total_timesteps  | 1429776  |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.814    |\n",
      "|    n_updates        | 21559    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1804     |\n",
      "|    fps              | 660      |\n",
      "|    time_elapsed     | 2167     |\n",
      "|    total_timesteps  | 1432208  |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.482    |\n",
      "|    n_updates        | 21597    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1808     |\n",
      "|    fps              | 660      |\n",
      "|    time_elapsed     | 2170     |\n",
      "|    total_timesteps  | 1434560  |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.651    |\n",
      "|    n_updates        | 21633    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1812     |\n",
      "|    fps              | 660      |\n",
      "|    time_elapsed     | 2177     |\n",
      "|    total_timesteps  | 1438800  |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 1.12     |\n",
      "|    n_updates        | 21700    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1816     |\n",
      "|    fps              | 660      |\n",
      "|    time_elapsed     | 2183     |\n",
      "|    total_timesteps  | 1442576  |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.656    |\n",
      "|    n_updates        | 21759    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1820     |\n",
      "|    fps              | 660      |\n",
      "|    time_elapsed     | 2186     |\n",
      "|    total_timesteps  | 1445008  |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.992    |\n",
      "|    n_updates        | 21797    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1824     |\n",
      "|    fps              | 660      |\n",
      "|    time_elapsed     | 2190     |\n",
      "|    total_timesteps  | 1447360  |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.666    |\n",
      "|    n_updates        | 21833    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1828     |\n",
      "|    fps              | 660      |\n",
      "|    time_elapsed     | 2197     |\n",
      "|    total_timesteps  | 1451600  |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 1.07     |\n",
      "|    n_updates        | 21900    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1832     |\n",
      "|    fps              | 660      |\n",
      "|    time_elapsed     | 2203     |\n",
      "|    total_timesteps  | 1455344  |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.551    |\n",
      "|    n_updates        | 21958    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1836     |\n",
      "|    fps              | 660      |\n",
      "|    time_elapsed     | 2206     |\n",
      "|    total_timesteps  | 1457456  |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.94     |\n",
      "|    n_updates        | 21991    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1840     |\n",
      "|    fps              | 660      |\n",
      "|    time_elapsed     | 2208     |\n",
      "|    total_timesteps  | 1459200  |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.607    |\n",
      "|    n_updates        | 22018    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1844     |\n",
      "|    fps              | 660      |\n",
      "|    time_elapsed     | 2217     |\n",
      "|    total_timesteps  | 1464400  |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.686    |\n",
      "|    n_updates        | 22100    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1848     |\n",
      "|    fps              | 660      |\n",
      "|    time_elapsed     | 2221     |\n",
      "|    total_timesteps  | 1467360  |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.514    |\n",
      "|    n_updates        | 22146    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1852     |\n",
      "|    fps              | 660      |\n",
      "|    time_elapsed     | 2226     |\n",
      "|    total_timesteps  | 1469936  |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.703    |\n",
      "|    n_updates        | 22186    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1856     |\n",
      "|    fps              | 660      |\n",
      "|    time_elapsed     | 2229     |\n",
      "|    total_timesteps  | 1472000  |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.787    |\n",
      "|    n_updates        | 22218    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1860     |\n",
      "|    fps              | 660      |\n",
      "|    time_elapsed     | 2237     |\n",
      "|    total_timesteps  | 1477200  |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.889    |\n",
      "|    n_updates        | 22300    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1864     |\n",
      "|    fps              | 660      |\n",
      "|    time_elapsed     | 2242     |\n",
      "|    total_timesteps  | 1480160  |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.788    |\n",
      "|    n_updates        | 22346    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1868     |\n",
      "|    fps              | 659      |\n",
      "|    time_elapsed     | 2246     |\n",
      "|    total_timesteps  | 1482736  |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.668    |\n",
      "|    n_updates        | 22386    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1872     |\n",
      "|    fps              | 659      |\n",
      "|    time_elapsed     | 2250     |\n",
      "|    total_timesteps  | 1484800  |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.765    |\n",
      "|    n_updates        | 22418    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1876     |\n",
      "|    fps              | 659      |\n",
      "|    time_elapsed     | 2258     |\n",
      "|    total_timesteps  | 1490000  |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.442    |\n",
      "|    n_updates        | 22500    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1880     |\n",
      "|    fps              | 659      |\n",
      "|    time_elapsed     | 2262     |\n",
      "|    total_timesteps  | 1492960  |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.632    |\n",
      "|    n_updates        | 22546    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1884     |\n",
      "|    fps              | 659      |\n",
      "|    time_elapsed     | 2266     |\n",
      "|    total_timesteps  | 1495536  |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.497    |\n",
      "|    n_updates        | 22586    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1888     |\n",
      "|    fps              | 659      |\n",
      "|    time_elapsed     | 2270     |\n",
      "|    total_timesteps  | 1497600  |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.534    |\n",
      "|    n_updates        | 22618    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1892     |\n",
      "|    fps              | 659      |\n",
      "|    time_elapsed     | 2278     |\n",
      "|    total_timesteps  | 1502800  |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.883    |\n",
      "|    n_updates        | 22700    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1896     |\n",
      "|    fps              | 659      |\n",
      "|    time_elapsed     | 2283     |\n",
      "|    total_timesteps  | 1505760  |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.73     |\n",
      "|    n_updates        | 22746    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1900     |\n",
      "|    fps              | 659      |\n",
      "|    time_elapsed     | 2287     |\n",
      "|    total_timesteps  | 1508336  |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.383    |\n",
      "|    n_updates        | 22786    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1904     |\n",
      "|    fps              | 659      |\n",
      "|    time_elapsed     | 2290     |\n",
      "|    total_timesteps  | 1510400  |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.442    |\n",
      "|    n_updates        | 22818    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1908     |\n",
      "|    fps              | 659      |\n",
      "|    time_elapsed     | 2298     |\n",
      "|    total_timesteps  | 1515600  |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.921    |\n",
      "|    n_updates        | 22900    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1912     |\n",
      "|    fps              | 659      |\n",
      "|    time_elapsed     | 2302     |\n",
      "|    total_timesteps  | 1518560  |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.63     |\n",
      "|    n_updates        | 22946    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1916     |\n",
      "|    fps              | 659      |\n",
      "|    time_elapsed     | 2306     |\n",
      "|    total_timesteps  | 1521136  |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.669    |\n",
      "|    n_updates        | 22986    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1920     |\n",
      "|    fps              | 659      |\n",
      "|    time_elapsed     | 2309     |\n",
      "|    total_timesteps  | 1523200  |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.902    |\n",
      "|    n_updates        | 23018    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1924     |\n",
      "|    fps              | 659      |\n",
      "|    time_elapsed     | 2316     |\n",
      "|    total_timesteps  | 1528400  |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.742    |\n",
      "|    n_updates        | 23100    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1928     |\n",
      "|    fps              | 659      |\n",
      "|    time_elapsed     | 2321     |\n",
      "|    total_timesteps  | 1531360  |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.436    |\n",
      "|    n_updates        | 23146    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1932     |\n",
      "|    fps              | 659      |\n",
      "|    time_elapsed     | 2325     |\n",
      "|    total_timesteps  | 1533936  |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.548    |\n",
      "|    n_updates        | 23186    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1936     |\n",
      "|    fps              | 659      |\n",
      "|    time_elapsed     | 2328     |\n",
      "|    total_timesteps  | 1536000  |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.648    |\n",
      "|    n_updates        | 23218    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1940     |\n",
      "|    fps              | 659      |\n",
      "|    time_elapsed     | 2336     |\n",
      "|    total_timesteps  | 1541200  |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.664    |\n",
      "|    n_updates        | 23300    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1944     |\n",
      "|    fps              | 659      |\n",
      "|    time_elapsed     | 2341     |\n",
      "|    total_timesteps  | 1544160  |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.461    |\n",
      "|    n_updates        | 23346    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1948     |\n",
      "|    fps              | 659      |\n",
      "|    time_elapsed     | 2345     |\n",
      "|    total_timesteps  | 1546736  |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.283    |\n",
      "|    n_updates        | 23386    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1952     |\n",
      "|    fps              | 659      |\n",
      "|    time_elapsed     | 2348     |\n",
      "|    total_timesteps  | 1548800  |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.625    |\n",
      "|    n_updates        | 23418    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1956     |\n",
      "|    fps              | 659      |\n",
      "|    time_elapsed     | 2357     |\n",
      "|    total_timesteps  | 1554000  |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.637    |\n",
      "|    n_updates        | 23500    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1960     |\n",
      "|    fps              | 659      |\n",
      "|    time_elapsed     | 2361     |\n",
      "|    total_timesteps  | 1556960  |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.543    |\n",
      "|    n_updates        | 23546    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1964     |\n",
      "|    fps              | 659      |\n",
      "|    time_elapsed     | 2365     |\n",
      "|    total_timesteps  | 1559536  |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.546    |\n",
      "|    n_updates        | 23586    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1968     |\n",
      "|    fps              | 659      |\n",
      "|    time_elapsed     | 2369     |\n",
      "|    total_timesteps  | 1561600  |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.648    |\n",
      "|    n_updates        | 23618    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1972     |\n",
      "|    fps              | 659      |\n",
      "|    time_elapsed     | 2377     |\n",
      "|    total_timesteps  | 1566800  |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.613    |\n",
      "|    n_updates        | 23700    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1976     |\n",
      "|    fps              | 659      |\n",
      "|    time_elapsed     | 2382     |\n",
      "|    total_timesteps  | 1569760  |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.704    |\n",
      "|    n_updates        | 23746    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1980     |\n",
      "|    fps              | 658      |\n",
      "|    time_elapsed     | 2386     |\n",
      "|    total_timesteps  | 1572336  |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.461    |\n",
      "|    n_updates        | 23786    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1984     |\n",
      "|    fps              | 658      |\n",
      "|    time_elapsed     | 2389     |\n",
      "|    total_timesteps  | 1574400  |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.402    |\n",
      "|    n_updates        | 23818    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1988     |\n",
      "|    fps              | 658      |\n",
      "|    time_elapsed     | 2397     |\n",
      "|    total_timesteps  | 1579600  |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.684    |\n",
      "|    n_updates        | 23900    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1992     |\n",
      "|    fps              | 658      |\n",
      "|    time_elapsed     | 2402     |\n",
      "|    total_timesteps  | 1582560  |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.65     |\n",
      "|    n_updates        | 23946    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1996     |\n",
      "|    fps              | 658      |\n",
      "|    time_elapsed     | 2406     |\n",
      "|    total_timesteps  | 1585136  |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.888    |\n",
      "|    n_updates        | 23986    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 2000     |\n",
      "|    fps              | 658      |\n",
      "|    time_elapsed     | 2409     |\n",
      "|    total_timesteps  | 1587200  |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.735    |\n",
      "|    n_updates        | 24018    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 2004     |\n",
      "|    fps              | 658      |\n",
      "|    time_elapsed     | 2418     |\n",
      "|    total_timesteps  | 1592400  |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 1.06     |\n",
      "|    n_updates        | 24100    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 2008     |\n",
      "|    fps              | 658      |\n",
      "|    time_elapsed     | 2422     |\n",
      "|    total_timesteps  | 1595360  |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 1.24     |\n",
      "|    n_updates        | 24146    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 2012     |\n",
      "|    fps              | 658      |\n",
      "|    time_elapsed     | 2427     |\n",
      "|    total_timesteps  | 1597936  |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.617    |\n",
      "|    n_updates        | 24186    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 2016     |\n",
      "|    fps              | 658      |\n",
      "|    time_elapsed     | 2430     |\n",
      "|    total_timesteps  | 1600000  |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.828    |\n",
      "|    n_updates        | 24218    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 2020     |\n",
      "|    fps              | 658      |\n",
      "|    time_elapsed     | 2437     |\n",
      "|    total_timesteps  | 1605200  |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.325    |\n",
      "|    n_updates        | 24300    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 2024     |\n",
      "|    fps              | 658      |\n",
      "|    time_elapsed     | 2442     |\n",
      "|    total_timesteps  | 1608160  |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.477    |\n",
      "|    n_updates        | 24346    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 2028     |\n",
      "|    fps              | 658      |\n",
      "|    time_elapsed     | 2446     |\n",
      "|    total_timesteps  | 1610736  |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.803    |\n",
      "|    n_updates        | 24386    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 2032     |\n",
      "|    fps              | 658      |\n",
      "|    time_elapsed     | 2449     |\n",
      "|    total_timesteps  | 1612800  |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.62     |\n",
      "|    n_updates        | 24418    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 2036     |\n",
      "|    fps              | 658      |\n",
      "|    time_elapsed     | 2458     |\n",
      "|    total_timesteps  | 1618000  |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.729    |\n",
      "|    n_updates        | 24500    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 2040     |\n",
      "|    fps              | 658      |\n",
      "|    time_elapsed     | 2462     |\n",
      "|    total_timesteps  | 1620960  |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.754    |\n",
      "|    n_updates        | 24546    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 2044     |\n",
      "|    fps              | 658      |\n",
      "|    time_elapsed     | 2466     |\n",
      "|    total_timesteps  | 1623536  |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.566    |\n",
      "|    n_updates        | 24586    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 2048     |\n",
      "|    fps              | 658      |\n",
      "|    time_elapsed     | 2470     |\n",
      "|    total_timesteps  | 1625600  |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.198    |\n",
      "|    n_updates        | 24618    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 2052     |\n",
      "|    fps              | 657      |\n",
      "|    time_elapsed     | 2478     |\n",
      "|    total_timesteps  | 1630800  |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.765    |\n",
      "|    n_updates        | 24700    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 2056     |\n",
      "|    fps              | 657      |\n",
      "|    time_elapsed     | 2483     |\n",
      "|    total_timesteps  | 1633760  |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 1.04     |\n",
      "|    n_updates        | 24746    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 2060     |\n",
      "|    fps              | 657      |\n",
      "|    time_elapsed     | 2486     |\n",
      "|    total_timesteps  | 1636336  |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.505    |\n",
      "|    n_updates        | 24786    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 2064     |\n",
      "|    fps              | 658      |\n",
      "|    time_elapsed     | 2489     |\n",
      "|    total_timesteps  | 1638400  |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.618    |\n",
      "|    n_updates        | 24818    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 2068     |\n",
      "|    fps              | 657      |\n",
      "|    time_elapsed     | 2498     |\n",
      "|    total_timesteps  | 1643600  |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.919    |\n",
      "|    n_updates        | 24900    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 2072     |\n",
      "|    fps              | 657      |\n",
      "|    time_elapsed     | 2502     |\n",
      "|    total_timesteps  | 1646560  |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.966    |\n",
      "|    n_updates        | 24946    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 2076     |\n",
      "|    fps              | 657      |\n",
      "|    time_elapsed     | 2506     |\n",
      "|    total_timesteps  | 1649136  |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.54     |\n",
      "|    n_updates        | 24986    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 2080     |\n",
      "|    fps              | 657      |\n",
      "|    time_elapsed     | 2510     |\n",
      "|    total_timesteps  | 1651200  |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.789    |\n",
      "|    n_updates        | 25018    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 2084     |\n",
      "|    fps              | 657      |\n",
      "|    time_elapsed     | 2518     |\n",
      "|    total_timesteps  | 1656400  |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.737    |\n",
      "|    n_updates        | 25100    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 2088     |\n",
      "|    fps              | 657      |\n",
      "|    time_elapsed     | 2523     |\n",
      "|    total_timesteps  | 1659360  |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.329    |\n",
      "|    n_updates        | 25146    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 2092     |\n",
      "|    fps              | 657      |\n",
      "|    time_elapsed     | 2527     |\n",
      "|    total_timesteps  | 1661936  |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.647    |\n",
      "|    n_updates        | 25186    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 2096     |\n",
      "|    fps              | 657      |\n",
      "|    time_elapsed     | 2530     |\n",
      "|    total_timesteps  | 1664000  |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.752    |\n",
      "|    n_updates        | 25218    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 2100     |\n",
      "|    fps              | 657      |\n",
      "|    time_elapsed     | 2538     |\n",
      "|    total_timesteps  | 1669200  |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.549    |\n",
      "|    n_updates        | 25300    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 2104     |\n",
      "|    fps              | 657      |\n",
      "|    time_elapsed     | 2543     |\n",
      "|    total_timesteps  | 1672160  |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.556    |\n",
      "|    n_updates        | 25346    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 2108     |\n",
      "|    fps              | 657      |\n",
      "|    time_elapsed     | 2546     |\n",
      "|    total_timesteps  | 1673760  |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.584    |\n",
      "|    n_updates        | 25371    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 2112     |\n",
      "|    fps              | 657      |\n",
      "|    time_elapsed     | 2550     |\n",
      "|    total_timesteps  | 1676640  |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.665    |\n",
      "|    n_updates        | 25416    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 2116     |\n",
      "|    fps              | 657      |\n",
      "|    time_elapsed     | 2559     |\n",
      "|    total_timesteps  | 1682000  |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 1.09     |\n",
      "|    n_updates        | 25500    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 2120     |\n",
      "|    fps              | 657      |\n",
      "|    time_elapsed     | 2563     |\n",
      "|    total_timesteps  | 1684960  |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.621    |\n",
      "|    n_updates        | 25546    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 2124     |\n",
      "|    fps              | 657      |\n",
      "|    time_elapsed     | 2566     |\n",
      "|    total_timesteps  | 1686560  |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.955    |\n",
      "|    n_updates        | 25571    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 2128     |\n",
      "|    fps              | 657      |\n",
      "|    time_elapsed     | 2570     |\n",
      "|    total_timesteps  | 1689440  |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.664    |\n",
      "|    n_updates        | 25616    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 2132     |\n",
      "|    fps              | 657      |\n",
      "|    time_elapsed     | 2579     |\n",
      "|    total_timesteps  | 1694800  |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.614    |\n",
      "|    n_updates        | 25700    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 2136     |\n",
      "|    fps              | 657      |\n",
      "|    time_elapsed     | 2584     |\n",
      "|    total_timesteps  | 1697760  |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.823    |\n",
      "|    n_updates        | 25746    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 2140     |\n",
      "|    fps              | 657      |\n",
      "|    time_elapsed     | 2586     |\n",
      "|    total_timesteps  | 1699360  |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 1.09     |\n",
      "|    n_updates        | 25771    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 2144     |\n",
      "|    fps              | 656      |\n",
      "|    time_elapsed     | 2591     |\n",
      "|    total_timesteps  | 1702240  |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.295    |\n",
      "|    n_updates        | 25816    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 2148     |\n",
      "|    fps              | 656      |\n",
      "|    time_elapsed     | 2599     |\n",
      "|    total_timesteps  | 1707600  |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.871    |\n",
      "|    n_updates        | 25900    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 2152     |\n",
      "|    fps              | 656      |\n",
      "|    time_elapsed     | 2604     |\n",
      "|    total_timesteps  | 1710496  |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.87     |\n",
      "|    n_updates        | 25945    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 2156     |\n",
      "|    fps              | 656      |\n",
      "|    time_elapsed     | 2606     |\n",
      "|    total_timesteps  | 1712160  |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.718    |\n",
      "|    n_updates        | 25971    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 2160     |\n",
      "|    fps              | 656      |\n",
      "|    time_elapsed     | 2610     |\n",
      "|    total_timesteps  | 1715040  |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.673    |\n",
      "|    n_updates        | 26016    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 2164     |\n",
      "|    fps              | 656      |\n",
      "|    time_elapsed     | 2619     |\n",
      "|    total_timesteps  | 1720400  |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.925    |\n",
      "|    n_updates        | 26100    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 2168     |\n",
      "|    fps              | 656      |\n",
      "|    time_elapsed     | 2623     |\n",
      "|    total_timesteps  | 1723296  |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.828    |\n",
      "|    n_updates        | 26145    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 2172     |\n",
      "|    fps              | 656      |\n",
      "|    time_elapsed     | 2626     |\n",
      "|    total_timesteps  | 1724960  |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.568    |\n",
      "|    n_updates        | 26171    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 2176     |\n",
      "|    fps              | 656      |\n",
      "|    time_elapsed     | 2630     |\n",
      "|    total_timesteps  | 1727840  |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.449    |\n",
      "|    n_updates        | 26216    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 2180     |\n",
      "|    fps              | 656      |\n",
      "|    time_elapsed     | 2639     |\n",
      "|    total_timesteps  | 1733200  |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.856    |\n",
      "|    n_updates        | 26300    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 2184     |\n",
      "|    fps              | 656      |\n",
      "|    time_elapsed     | 2644     |\n",
      "|    total_timesteps  | 1736096  |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.543    |\n",
      "|    n_updates        | 26345    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 2188     |\n",
      "|    fps              | 656      |\n",
      "|    time_elapsed     | 2646     |\n",
      "|    total_timesteps  | 1737760  |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.913    |\n",
      "|    n_updates        | 26371    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 2192     |\n",
      "|    fps              | 656      |\n",
      "|    time_elapsed     | 2649     |\n",
      "|    total_timesteps  | 1739408  |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.696    |\n",
      "|    n_updates        | 26397    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 2196     |\n",
      "|    fps              | 656      |\n",
      "|    time_elapsed     | 2659     |\n",
      "|    total_timesteps  | 1746000  |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.675    |\n",
      "|    n_updates        | 26500    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 2200     |\n",
      "|    fps              | 656      |\n",
      "|    time_elapsed     | 2664     |\n",
      "|    total_timesteps  | 1748896  |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.757    |\n",
      "|    n_updates        | 26545    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 2204     |\n",
      "|    fps              | 656      |\n",
      "|    time_elapsed     | 2666     |\n",
      "|    total_timesteps  | 1750560  |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.609    |\n",
      "|    n_updates        | 26571    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 2208     |\n",
      "|    fps              | 656      |\n",
      "|    time_elapsed     | 2669     |\n",
      "|    total_timesteps  | 1752208  |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.624    |\n",
      "|    n_updates        | 26597    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 2212     |\n",
      "|    fps              | 656      |\n",
      "|    time_elapsed     | 2679     |\n",
      "|    total_timesteps  | 1758800  |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.702    |\n",
      "|    n_updates        | 26700    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 2216     |\n",
      "|    fps              | 656      |\n",
      "|    time_elapsed     | 2684     |\n",
      "|    total_timesteps  | 1761696  |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.59     |\n",
      "|    n_updates        | 26745    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 2220     |\n",
      "|    fps              | 656      |\n",
      "|    time_elapsed     | 2687     |\n",
      "|    total_timesteps  | 1763360  |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.693    |\n",
      "|    n_updates        | 26771    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 2224     |\n",
      "|    fps              | 656      |\n",
      "|    time_elapsed     | 2689     |\n",
      "|    total_timesteps  | 1765008  |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.805    |\n",
      "|    n_updates        | 26797    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 2228     |\n",
      "|    fps              | 656      |\n",
      "|    time_elapsed     | 2700     |\n",
      "|    total_timesteps  | 1771600  |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.838    |\n",
      "|    n_updates        | 26900    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 2232     |\n",
      "|    fps              | 656      |\n",
      "|    time_elapsed     | 2704     |\n",
      "|    total_timesteps  | 1774304  |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.609    |\n",
      "|    n_updates        | 26942    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 2236     |\n",
      "|    fps              | 656      |\n",
      "|    time_elapsed     | 2707     |\n",
      "|    total_timesteps  | 1775936  |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.858    |\n",
      "|    n_updates        | 26967    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 2240     |\n",
      "|    fps              | 656      |\n",
      "|    time_elapsed     | 2710     |\n",
      "|    total_timesteps  | 1777808  |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.674    |\n",
      "|    n_updates        | 26997    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 2244     |\n",
      "|    fps              | 655      |\n",
      "|    time_elapsed     | 2720     |\n",
      "|    total_timesteps  | 1784400  |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.804    |\n",
      "|    n_updates        | 27100    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 2248     |\n",
      "|    fps              | 655      |\n",
      "|    time_elapsed     | 2724     |\n",
      "|    total_timesteps  | 1787104  |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 1.24     |\n",
      "|    n_updates        | 27142    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 2252     |\n",
      "|    fps              | 655      |\n",
      "|    time_elapsed     | 2727     |\n",
      "|    total_timesteps  | 1788736  |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.815    |\n",
      "|    n_updates        | 27167    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 2256     |\n",
      "|    fps              | 655      |\n",
      "|    time_elapsed     | 2730     |\n",
      "|    total_timesteps  | 1790256  |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.747    |\n",
      "|    n_updates        | 27191    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 2260     |\n",
      "|    fps              | 655      |\n",
      "|    time_elapsed     | 2741     |\n",
      "|    total_timesteps  | 1797200  |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.896    |\n",
      "|    n_updates        | 27300    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 2264     |\n",
      "|    fps              | 655      |\n",
      "|    time_elapsed     | 2745     |\n",
      "|    total_timesteps  | 1799392  |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.735    |\n",
      "|    n_updates        | 27334    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 2268     |\n",
      "|    fps              | 655      |\n",
      "|    time_elapsed     | 2747     |\n",
      "|    total_timesteps  | 1800688  |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.836    |\n",
      "|    n_updates        | 27354    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 2272     |\n",
      "|    fps              | 655      |\n",
      "|    time_elapsed     | 2751     |\n",
      "|    total_timesteps  | 1803056  |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.546    |\n",
      "|    n_updates        | 27391    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 2276     |\n",
      "|    fps              | 655      |\n",
      "|    time_elapsed     | 2763     |\n",
      "|    total_timesteps  | 1810000  |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 1.18     |\n",
      "|    n_updates        | 27500    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 2280     |\n",
      "|    fps              | 654      |\n",
      "|    time_elapsed     | 2766     |\n",
      "|    total_timesteps  | 1812192  |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 1.04     |\n",
      "|    n_updates        | 27534    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 2284     |\n",
      "|    fps              | 654      |\n",
      "|    time_elapsed     | 2768     |\n",
      "|    total_timesteps  | 1813488  |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.631    |\n",
      "|    n_updates        | 27554    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 2288     |\n",
      "|    fps              | 654      |\n",
      "|    time_elapsed     | 2772     |\n",
      "|    total_timesteps  | 1815856  |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.619    |\n",
      "|    n_updates        | 27591    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 2292     |\n",
      "|    fps              | 654      |\n",
      "|    time_elapsed     | 2783     |\n",
      "|    total_timesteps  | 1822800  |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.665    |\n",
      "|    n_updates        | 27700    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 2296     |\n",
      "|    fps              | 654      |\n",
      "|    time_elapsed     | 2786     |\n",
      "|    total_timesteps  | 1824992  |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.775    |\n",
      "|    n_updates        | 27734    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 2300     |\n",
      "|    fps              | 654      |\n",
      "|    time_elapsed     | 2789     |\n",
      "|    total_timesteps  | 1826288  |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.688    |\n",
      "|    n_updates        | 27754    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 2304     |\n",
      "|    fps              | 654      |\n",
      "|    time_elapsed     | 2792     |\n",
      "|    total_timesteps  | 1828656  |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.959    |\n",
      "|    n_updates        | 27791    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 2308     |\n",
      "|    fps              | 654      |\n",
      "|    time_elapsed     | 2801     |\n",
      "|    total_timesteps  | 1834352  |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.724    |\n",
      "|    n_updates        | 27880    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 2312     |\n",
      "|    fps              | 654      |\n",
      "|    time_elapsed     | 2806     |\n",
      "|    total_timesteps  | 1837520  |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.813    |\n",
      "|    n_updates        | 27930    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 2316     |\n",
      "|    fps              | 654      |\n",
      "|    time_elapsed     | 2808     |\n",
      "|    total_timesteps  | 1838496  |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.573    |\n",
      "|    n_updates        | 27945    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 2320     |\n",
      "|    fps              | 654      |\n",
      "|    time_elapsed     | 2813     |\n",
      "|    total_timesteps  | 1841456  |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.396    |\n",
      "|    n_updates        | 27991    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 2324     |\n",
      "|    fps              | 654      |\n",
      "|    time_elapsed     | 2820     |\n",
      "|    total_timesteps  | 1845424  |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.542    |\n",
      "|    n_updates        | 28053    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 2328     |\n",
      "|    fps              | 654      |\n",
      "|    time_elapsed     | 2827     |\n",
      "|    total_timesteps  | 1849920  |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 1.2      |\n",
      "|    n_updates        | 28123    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 2332     |\n",
      "|    fps              | 654      |\n",
      "|    time_elapsed     | 2829     |\n",
      "|    total_timesteps  | 1851296  |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 1.01     |\n",
      "|    n_updates        | 28145    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 2336     |\n",
      "|    fps              | 654      |\n",
      "|    time_elapsed     | 2834     |\n",
      "|    total_timesteps  | 1854256  |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.708    |\n",
      "|    n_updates        | 28191    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 2340     |\n",
      "|    fps              | 653      |\n",
      "|    time_elapsed     | 2841     |\n",
      "|    total_timesteps  | 1858224  |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.504    |\n",
      "|    n_updates        | 28253    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 2344     |\n",
      "|    fps              | 654      |\n",
      "|    time_elapsed     | 2847     |\n",
      "|    total_timesteps  | 1862720  |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.581    |\n",
      "|    n_updates        | 28323    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 2348     |\n",
      "|    fps              | 654      |\n",
      "|    time_elapsed     | 2850     |\n",
      "|    total_timesteps  | 1864096  |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.882    |\n",
      "|    n_updates        | 28345    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 2352     |\n",
      "|    fps              | 653      |\n",
      "|    time_elapsed     | 2854     |\n",
      "|    total_timesteps  | 1867056  |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.794    |\n",
      "|    n_updates        | 28391    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 2356     |\n",
      "|    fps              | 653      |\n",
      "|    time_elapsed     | 2861     |\n",
      "|    total_timesteps  | 1871024  |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 1.19     |\n",
      "|    n_updates        | 28453    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 2360     |\n",
      "|    fps              | 653      |\n",
      "|    time_elapsed     | 2868     |\n",
      "|    total_timesteps  | 1875520  |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.628    |\n",
      "|    n_updates        | 28523    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 2364     |\n",
      "|    fps              | 653      |\n",
      "|    time_elapsed     | 2870     |\n",
      "|    total_timesteps  | 1876896  |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.878    |\n",
      "|    n_updates        | 28545    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 2368     |\n",
      "|    fps              | 653      |\n",
      "|    time_elapsed     | 2875     |\n",
      "|    total_timesteps  | 1879856  |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.784    |\n",
      "|    n_updates        | 28591    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 2372     |\n",
      "|    fps              | 653      |\n",
      "|    time_elapsed     | 2881     |\n",
      "|    total_timesteps  | 1883824  |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 1.02     |\n",
      "|    n_updates        | 28653    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 2376     |\n",
      "|    fps              | 653      |\n",
      "|    time_elapsed     | 2888     |\n",
      "|    total_timesteps  | 1888064  |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.708    |\n",
      "|    n_updates        | 28719    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 2380     |\n",
      "|    fps              | 653      |\n",
      "|    time_elapsed     | 2890     |\n",
      "|    total_timesteps  | 1889696  |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.954    |\n",
      "|    n_updates        | 28745    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 2384     |\n",
      "|    fps              | 653      |\n",
      "|    time_elapsed     | 2895     |\n",
      "|    total_timesteps  | 1892656  |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 1.06     |\n",
      "|    n_updates        | 28791    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 2388     |\n",
      "|    fps              | 653      |\n",
      "|    time_elapsed     | 2901     |\n",
      "|    total_timesteps  | 1896624  |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.499    |\n",
      "|    n_updates        | 28853    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 2392     |\n",
      "|    fps              | 653      |\n",
      "|    time_elapsed     | 2908     |\n",
      "|    total_timesteps  | 1900864  |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 1.15     |\n",
      "|    n_updates        | 28919    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 2396     |\n",
      "|    fps              | 653      |\n",
      "|    time_elapsed     | 2909     |\n",
      "|    total_timesteps  | 1902304  |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.844    |\n",
      "|    n_updates        | 28942    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 2400     |\n",
      "|    fps              | 653      |\n",
      "|    time_elapsed     | 2914     |\n",
      "|    total_timesteps  | 1905456  |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.938    |\n",
      "|    n_updates        | 28991    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 2404     |\n",
      "|    fps              | 653      |\n",
      "|    time_elapsed     | 2921     |\n",
      "|    total_timesteps  | 1909424  |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.896    |\n",
      "|    n_updates        | 29053    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 2408     |\n",
      "|    fps              | 653      |\n",
      "|    time_elapsed     | 2926     |\n",
      "|    total_timesteps  | 1913664  |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.948    |\n",
      "|    n_updates        | 29119    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 2412     |\n",
      "|    fps              | 653      |\n",
      "|    time_elapsed     | 2929     |\n",
      "|    total_timesteps  | 1915104  |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.6      |\n",
      "|    n_updates        | 29142    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 2416     |\n",
      "|    fps              | 653      |\n",
      "|    time_elapsed     | 2934     |\n",
      "|    total_timesteps  | 1918256  |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.603    |\n",
      "|    n_updates        | 29191    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 2420     |\n",
      "|    fps              | 653      |\n",
      "|    time_elapsed     | 2940     |\n",
      "|    total_timesteps  | 1922224  |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.73     |\n",
      "|    n_updates        | 29253    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 2424     |\n",
      "|    fps              | 653      |\n",
      "|    time_elapsed     | 2947     |\n",
      "|    total_timesteps  | 1926464  |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.61     |\n",
      "|    n_updates        | 29319    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 2428     |\n",
      "|    fps              | 653      |\n",
      "|    time_elapsed     | 2949     |\n",
      "|    total_timesteps  | 1927904  |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.63     |\n",
      "|    n_updates        | 29342    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 2432     |\n",
      "|    fps              | 653      |\n",
      "|    time_elapsed     | 2954     |\n",
      "|    total_timesteps  | 1931056  |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.641    |\n",
      "|    n_updates        | 29391    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 2436     |\n",
      "|    fps              | 653      |\n",
      "|    time_elapsed     | 2961     |\n",
      "|    total_timesteps  | 1935024  |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.683    |\n",
      "|    n_updates        | 29453    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 2440     |\n",
      "|    fps              | 653      |\n",
      "|    time_elapsed     | 2967     |\n",
      "|    total_timesteps  | 1939264  |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 1.02     |\n",
      "|    n_updates        | 29519    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 2444     |\n",
      "|    fps              | 653      |\n",
      "|    time_elapsed     | 2970     |\n",
      "|    total_timesteps  | 1940704  |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 1.05     |\n",
      "|    n_updates        | 29542    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 2448     |\n",
      "|    fps              | 653      |\n",
      "|    time_elapsed     | 2975     |\n",
      "|    total_timesteps  | 1943856  |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.999    |\n",
      "|    n_updates        | 29591    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 2452     |\n",
      "|    fps              | 653      |\n",
      "|    time_elapsed     | 2981     |\n",
      "|    total_timesteps  | 1947824  |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.935    |\n",
      "|    n_updates        | 29653    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 2456     |\n",
      "|    fps              | 653      |\n",
      "|    time_elapsed     | 2987     |\n",
      "|    total_timesteps  | 1952064  |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.78     |\n",
      "|    n_updates        | 29719    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 2460     |\n",
      "|    fps              | 653      |\n",
      "|    time_elapsed     | 2990     |\n",
      "|    total_timesteps  | 1953504  |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.826    |\n",
      "|    n_updates        | 29742    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 2464     |\n",
      "|    fps              | 653      |\n",
      "|    time_elapsed     | 2995     |\n",
      "|    total_timesteps  | 1956656  |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.601    |\n",
      "|    n_updates        | 29791    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 2468     |\n",
      "|    fps              | 653      |\n",
      "|    time_elapsed     | 3001     |\n",
      "|    total_timesteps  | 1960624  |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.557    |\n",
      "|    n_updates        | 29853    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 2472     |\n",
      "|    fps              | 653      |\n",
      "|    time_elapsed     | 3008     |\n",
      "|    total_timesteps  | 1964864  |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.859    |\n",
      "|    n_updates        | 29919    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 2476     |\n",
      "|    fps              | 653      |\n",
      "|    time_elapsed     | 3011     |\n",
      "|    total_timesteps  | 1966304  |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.617    |\n",
      "|    n_updates        | 29942    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 2480     |\n",
      "|    fps              | 653      |\n",
      "|    time_elapsed     | 3016     |\n",
      "|    total_timesteps  | 1969456  |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.573    |\n",
      "|    n_updates        | 29991    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 2484     |\n",
      "|    fps              | 652      |\n",
      "|    time_elapsed     | 3022     |\n",
      "|    total_timesteps  | 1973424  |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.902    |\n",
      "|    n_updates        | 30053    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 2488     |\n",
      "|    fps              | 652      |\n",
      "|    time_elapsed     | 3028     |\n",
      "|    total_timesteps  | 1977664  |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.646    |\n",
      "|    n_updates        | 30119    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 2492     |\n",
      "|    fps              | 653      |\n",
      "|    time_elapsed     | 3030     |\n",
      "|    total_timesteps  | 1979104  |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.709    |\n",
      "|    n_updates        | 30142    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 2496     |\n",
      "|    fps              | 652      |\n",
      "|    time_elapsed     | 3035     |\n",
      "|    total_timesteps  | 1982256  |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.771    |\n",
      "|    n_updates        | 30191    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 2500     |\n",
      "|    fps              | 652      |\n",
      "|    time_elapsed     | 3042     |\n",
      "|    total_timesteps  | 1986224  |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.877    |\n",
      "|    n_updates        | 30253    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 2504     |\n",
      "|    fps              | 652      |\n",
      "|    time_elapsed     | 3048     |\n",
      "|    total_timesteps  | 1990464  |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.788    |\n",
      "|    n_updates        | 30319    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 2508     |\n",
      "|    fps              | 652      |\n",
      "|    time_elapsed     | 3050     |\n",
      "|    total_timesteps  | 1991904  |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.887    |\n",
      "|    n_updates        | 30342    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 2512     |\n",
      "|    fps              | 652      |\n",
      "|    time_elapsed     | 3055     |\n",
      "|    total_timesteps  | 1995056  |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 1.11     |\n",
      "|    n_updates        | 30391    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 2516     |\n",
      "|    fps              | 652      |\n",
      "|    time_elapsed     | 3062     |\n",
      "|    total_timesteps  | 1999024  |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.653    |\n",
      "|    n_updates        | 30453    |\n",
      "----------------------------------\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.11/dist-packages/stable_baselines3/common/evaluation.py:67: UserWarning: Evaluation environment is not wrapped with a ``Monitor`` wrapper. This may result in reporting modified episode lengths and rewards, if other wrappers happen to modify these. Consider wrapping environment first with ``Monitor`` wrapper.\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "712.7648527156562\n"
     ]
    }
   ],
   "source": [
    "train_env.reset()\n",
    "model = DQN(\"CnnPolicy\", train_env, verbose=1, buffer_size=100000, tensorboard_log='tensorboard_logs/')\n",
    "model.learn(total_timesteps=TRAIN_STEPS, tb_log_name=\"first_run\")\n",
    "mean_reward, _ = evaluate_policy(model, eval_env, n_eval_episodes=EVAL_EPS)\n",
    "train_env.close()\n",
    "print(mean_reward)\n",
    "model.save(\"dqn_carracing\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "fd292ca7-7398-49e6-883a-cd0f810eab44",
   "metadata": {},
   "outputs": [],
   "source": [
    "model = DQN.load(\"dqn_carracing\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "478d98bd-ec43-41de-84aa-578670664f4e",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.11/dist-packages/gymnasium/wrappers/record_video.py:94: UserWarning: \u001b[33mWARN: Overwriting existing videos at /tf/video folder (try specifying a different `video_folder` for the `RecordVideo` wrapper if this is not desired)\u001b[0m\n",
      "  logger.warn(\n"
     ]
    }
   ],
   "source": [
    "eval_env = gym.make(env_id, max_episode_steps=3200, continuous=False, render_mode=\"rgb_array\")\n",
    "eval_env = RecordVideo(eval_env, 'video')\n",
    "eval_env = DummyVecEnv([lambda: eval_env])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "d0a6277c-da21-4793-ba3e-8e88fec6f1e5",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.11/dist-packages/stable_baselines3/common/evaluation.py:67: UserWarning: Evaluation environment is not wrapped with a ``Monitor`` wrapper. This may result in reporting modified episode lengths and rewards, if other wrappers happen to modify these. Consider wrapping environment first with ``Monitor`` wrapper.\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Moviepy - Building video /tf/video/rl-video-episode-0.mp4.\n",
      "Moviepy - Writing video /tf/video/rl-video-episode-0.mp4\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                                                        \r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Moviepy - Done !\n",
      "Moviepy - video ready /tf/video/rl-video-episode-0.mp4\n"
     ]
    }
   ],
   "source": [
    "mean_reward, _ = evaluate_policy(model, eval_env, n_eval_episodes=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1e0365b3-eb45-47e5-94ac-975612b85ad0",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.0rc1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}