games/connectx/ConnectX.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "source": [
    "# ConnectX Agents"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%% md\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "outputs": [],
   "source": [
    "from kaggle_environments import make\n",
    "\n",
    "from agents.AlphaBetaAgent.AlphaBetaAgent import AlphaBetaAgent\n",
    "from agents.AlphaBetaAgent.AlphaBetaBitboard import AlphaBetaBitboard\n",
    "from agents.MontyCarlo.MontyCarloHeuristic import MontyCarloHeuristic\n",
    "from agents.MontyCarlo.MontyCarloPure import MontyCarloPure\n",
    "from agents.MontyCarlo.MontyCarloTreeSearch import MontyCarloTreeSearch\n",
    "from agents.Negamax.Negamax import Negamax\n",
    "\n",
    "\n",
    "agents = {\n",
    "    'Negamax':              Negamax(),\n",
    "    'AlphaBetaAgent':       AlphaBetaAgent.agent(),\n",
    "    'AlphaBetaBitboard':    AlphaBetaBitboard.agent(),\n",
    "    'MontyCarloTreeSearch': MontyCarloTreeSearch(),\n",
    "    'MontyCarloPure': MontyCarloPure(),\n",
    "    'MontyCarloHeuristic':  MontyCarloHeuristic(),\n",
    "    'negamax':              \"negamax\",\n",
    "    'random':               \"random\",\n",
    "}\n",
    "\n",
    "%load_ext autoreload\n",
    "%autoreload 1"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Negamax vs MontyCarloHeuristic\n",
      "\n",
      "Negamax                 | depth: 1 2 3  = 3 (2.00) | 2658 iterations in 2.58s \n",
      "MontyCarloHeuristicNode: p2 action = 3 after 2722 simulations in 4.001s\n",
      "\n",
      "Negamax                 | depth: 1 2 3 4  = 3 (-0.50) | 4655 iterations in 4.04s \n",
      "MontyCarloHeuristicNode: p2 action = 3 after 2585 simulations in 4.001s\n",
      "\n",
      "Negamax                 | depth: 1 2 3 4  = 2 (-1.80) | 5119 iterations in 3.79s \n",
      "MontyCarloHeuristicNode: p2 action = 4 after 2240 simulations in 4.001s\n",
      "\n",
      "Negamax                 | depth: 1 2 3 4  = 4 (-1.70) | 4560 iterations in 3.83s \n",
      "MontyCarloHeuristicNode: p2 action = 4 after 1498 simulations in 4.000s\n",
      "\n",
      "Negamax                 | depth: 1 2 3 4  = 4 (-4.50) | 3128 iterations in 4.11s \n",
      "MontyCarloHeuristicNode: p2 action = 2 after 1220 simulations in 4.001s\n",
      "\n",
      "Negamax                 | depth: 1 2 3  = 0 (1.80) | 1545 iterations in 3.84s \n",
      "MontyCarloHeuristicNode: p2 action = 1 after 1142 simulations in 4.001s\n",
      "\n",
      "Negamax                 | depth: 1 2 3  = 3 (3.30) | 2180 iterations in 2.66s \n",
      "MontyCarloHeuristicNode: p2 action = 1 after 1724 simulations in 4.000s\n",
      "\n",
      "Negamax                 | depth: 1 2  = 0 (-7.10) | 34 iterations in 0.05s \n",
      "MontyCarloHeuristicNode: p2 action = 1 after 1596 simulations in 4.002s\n",
      "\n",
      "Negamax                 | depth: 1 2  = 1 (-1.90) | 31 iterations in 0.05s \n",
      "MontyCarloHeuristicNode: p2 action = 4 after 1524 simulations in 4.001s\n",
      "\n",
      "Negamax                 | depth: 1 2  = 2 (1.70) | 27 iterations in 0.03s \n",
      "MontyCarloHeuristicNode: p2 action = 2 after 1563 simulations in 4.003s\n",
      "\n",
      "Negamax                 | depth: 1 2 3 4  = 0 (3.60) | 3921 iterations in 4.28s \n",
      "MontyCarloHeuristicNode: p2 action = 0 after 1331 simulations in 4.002s\n",
      "\n",
      "Negamax                 | depth: 1 2 3 4  = 3 (2.00) | 3776 iterations in 3.70s \n",
      "MontyCarloHeuristicNode: p2 action = 2 after 1476 simulations in 4.001s\n",
      "\n",
      "Negamax                 | depth: 1 2 3 4 5  = 2 (2.10) | 4104 iterations in 3.80s \n",
      "MontyCarloHeuristicNode: p2 action = 4 after 1588 simulations in 4.001s\n",
      "\n",
      "Negamax                 | depth: 1 2 3 4 5 6 7 8  = 6 (-1.00) | 6539 iterations in 3.83s \n",
      "MontyCarloHeuristicNode: p2 action = 6 after 2780 simulations in 4.000s\n",
      "\n",
      "Negamax                 | depth: 1 2 3 4 5 6 7 8 -inf  = 0 (-inf) | 2594 iterations in 1.82s \n",
      "MontyCarloHeuristicNode: p2 action = 0 after 3204 simulations in 4.000s\n",
      "\n",
      "Negamax                 | depth: 1 2  = 6 (-1.50) | 14 iterations in 0.02s \n",
      "MontyCarloHeuristicNode: p2 action = 1 after 3328 simulations in 4.001s\n",
      "\n",
      "Negamax                 | depth: 1 2  = 1 (0.00) | 11 iterations in 0.01s \n",
      "MontyCarloHeuristicNode: p2 action = 6 after 3374 simulations in 4.000s\n",
      "\n",
      "Negamax                 | depth: 1 2  = 6 (0.00) | 8 iterations in 0.01s \n",
      "MontyCarloHeuristicNode: p2 action = 6 after 3125 simulations in 4.001s\n",
      "\n",
      "Negamax                 | depth: 1  = 5 (0.00) | 0 iterations in 0.00s \n",
      "MontyCarloHeuristicNode: p2 action = 5 after 3943 simulations in 4.001s\n",
      "--------------------\n",
      "Negamax vs MontyCarloHeuristic\n",
      "scores: Negamax = -1 | MontyCarloHeuristic = 1\n"
     ]
    },
    {
     "data": {
      "text/plain": "<IPython.core.display.HTML object>",
      "text/html": "<iframe srcdoc=\"<!--\n  Copyright 2020 Kaggle Inc\n\n  Licensed under the Apache License, Version 2.0 (the &quot;License&quot;);\n  you may not use this file except in compliance with the License.\n  You may obtain a copy of the License at\n\n      http://www.apache.org/licenses/LICENSE-2.0\n\n  Unless required by applicable law or agreed to in writing, software\n  distributed under the License is distributed on an &quot;AS IS&quot; BASIS,\n  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n  See the License for the specific language governing permissions and\n  limitations under the License.\n-->\n<!DOCTYPE html>\n<html lang=&quot;en&quot;>\n  <head>\n    <title>Kaggle Simulation Player</title>\n    <meta name=&quot;viewport&quot; content=&quot;width=device-width,initial-scale=1&quot; />\n    <link\n      rel=&quot;stylesheet&quot;\n      href=&quot;https://cdnjs.cloudflare.com/ajax/libs/meyer-reset/2.0/reset.css&quot;\n      crossorigin=&quot;anonymous&quot;\n    />\n    <style type=&quot;text/css&quot;>\n      html,\n      body {\n        height: 100%;\n        font-family: sans-serif;\n      }\n      canvas {\n        /* image-rendering: -moz-crisp-edges;\n        image-rendering: -webkit-crisp-edges;\n        image-rendering: pixelated;\n        image-rendering: crisp-edges; */\n      }\n    </style>\n    <script src=&quot;https://unpkg.com/preact@10.0.1/dist/preact.umd.js&quot;></script>\n    <script src=&quot;https://unpkg.com/preact@10.0.1/hooks/dist/hooks.umd.js&quot;></script>\n    <script src=&quot;https://unpkg.com/htm@2.2.1/dist/htm.umd.js&quot;></script>\n    <script>\n      // Polyfill for Styled Components\n      window.React = {\n        ...preact,\n        createElement: preact.h,\n        PropTypes: { func: {} },\n      };\n    </script>\n    <script src=&quot;https://unpkg.com/styled-components@3.5.0-0/dist/styled-components.min.js&quot;></script>\n  </head>\n  <body>\n    <script>\n      \nwindow.kaggle = {\n  &quot;debug&quot;: true,\n  &quot;autoplay&quot;: true,\n  &quot;step&quot;: 0,\n  &quot;controls&quot;: true,\n  &quot;environment&quot;: {\n    &quot;id&quot;: &quot;fe1d11e2-d655-11ea-bd55-3c07545773e3&quot;,\n    &quot;name&quot;: &quot;connectx&quot;,\n    &quot;title&quot;: &quot;ConnectX&quot;,\n    &quot;description&quot;: &quot;Classic Connect in a row but configurable.&quot;,\n    &quot;version&quot;: &quot;1.0.1&quot;,\n    &quot;configuration&quot;: {\n      &quot;episodeSteps&quot;: 1000,\n      &quot;agentExec&quot;: &quot;LOCAL&quot;,\n      &quot;agentTimeout&quot;: 16,\n      &quot;actTimeout&quot;: 8,\n      &quot;runTimeout&quot;: 1200,\n      &quot;columns&quot;: 7,\n      &quot;rows&quot;: 6,\n      &quot;inarow&quot;: 4,\n      &quot;timeout&quot;: 6\n    },\n    &quot;specification&quot;: {\n      &quot;action&quot;: {\n        &quot;description&quot;: &quot;Column to drop a checker onto the board.&quot;,\n        &quot;type&quot;: &quot;integer&quot;,\n        &quot;minimum&quot;: 0,\n        &quot;default&quot;: 0\n      },\n      &quot;agents&quot;: [\n        2\n      ],\n      &quot;configuration&quot;: {\n        &quot;episodeSteps&quot;: {\n          &quot;description&quot;: &quot;Maximum number of steps in the episode.&quot;,\n          &quot;type&quot;: &quot;integer&quot;,\n          &quot;minimum&quot;: 1,\n          &quot;default&quot;: 1000\n        },\n        &quot;agentExec&quot;: {\n          &quot;description&quot;: &quot;How the agent is executed alongside the running envionment.&quot;,\n          &quot;type&quot;: &quot;string&quot;,\n          &quot;default&quot;: &quot;LOCAL&quot;,\n          &quot;enum&quot;: [\n            &quot;LOCAL&quot;,\n            &quot;PROCESS&quot;\n          ]\n        },\n        &quot;agentTimeout&quot;: {\n          &quot;description&quot;: &quot;Maximum runtime (seconds) to initialize an agent.&quot;,\n          &quot;type&quot;: &quot;integer&quot;,\n          &quot;minimum&quot;: 1,\n          &quot;default&quot;: 16\n        },\n        &quot;actTimeout&quot;: {\n          &quot;description&quot;: &quot;Maximum runtime (seconds) to obtain an action from an agent.&quot;,\n          &quot;type&quot;: &quot;integer&quot;,\n          &quot;minimum&quot;: 1,\n          &quot;default&quot;: 8\n        },\n        &quot;runTimeout&quot;: {\n          &quot;description&quot;: &quot;Maximum runtime (seconds) of an episode (not necessarily DONE).&quot;,\n          &quot;type&quot;: &quot;integer&quot;,\n          &quot;minimum&quot;: 1,\n          &quot;default&quot;: 1200\n        },\n        &quot;columns&quot;: {\n          &quot;description&quot;: &quot;The number of columns on the board&quot;,\n          &quot;type&quot;: &quot;integer&quot;,\n          &quot;default&quot;: 7,\n          &quot;minimum&quot;: 1\n        },\n        &quot;rows&quot;: {\n          &quot;description&quot;: &quot;The number of rows on the board&quot;,\n          &quot;type&quot;: &quot;integer&quot;,\n          &quot;default&quot;: 6,\n          &quot;minimum&quot;: 1\n        },\n        &quot;inarow&quot;: {\n          &quot;description&quot;: &quot;The number of checkers in a row required to win.&quot;,\n          &quot;type&quot;: &quot;integer&quot;,\n          &quot;default&quot;: 4,\n          &quot;minimum&quot;: 1\n        },\n        &quot;timeout&quot;: {\n          &quot;description&quot;: &quot;Obsolete copy of actTimeout maintained for backwards compatibility. May be removed in the future.&quot;,\n          &quot;type&quot;: &quot;integer&quot;,\n          &quot;default&quot;: 8,\n          &quot;minimum&quot;: 0\n        }\n      },\n      &quot;info&quot;: {},\n      &quot;observation&quot;: {\n        &quot;board&quot;: {\n          &quot;description&quot;: &quot;Serialized grid (rows x columns). 0 = Empty, 1 = P1, 2 = P2&quot;,\n          &quot;type&quot;: &quot;array&quot;,\n          &quot;shared&quot;: true,\n          &quot;default&quot;: []\n        },\n        &quot;mark&quot;: {\n          &quot;defaults&quot;: [\n            1,\n            2\n          ],\n          &quot;description&quot;: &quot;Which checkers are the agents.&quot;,\n          &quot;enum&quot;: [\n            1,\n            2\n          ]\n        }\n      },\n      &quot;reward&quot;: {\n        &quot;description&quot;: &quot;-1 = Lost, 0 = Draw/Ongoing, 1 = Won&quot;,\n        &quot;enum&quot;: [\n          -1,\n          0,\n          1\n        ],\n        &quot;default&quot;: 0,\n        &quot;type&quot;: [\n          &quot;number&quot;,\n          &quot;null&quot;\n        ]\n      }\n    },\n    &quot;steps&quot;: [\n      [\n        {\n          &quot;action&quot;: 0,\n          &quot;reward&quot;: 0,\n          &quot;info&quot;: {},\n          &quot;observation&quot;: {\n            &quot;board&quot;: [\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0\n            ],\n            &quot;mark&quot;: 1\n          },\n          &quot;status&quot;: &quot;ACTIVE&quot;\n        },\n        {\n          &quot;action&quot;: 0,\n          &quot;reward&quot;: 0,\n          &quot;info&quot;: {},\n          &quot;observation&quot;: {\n            &quot;mark&quot;: 2\n          },\n          &quot;status&quot;: &quot;INACTIVE&quot;\n        }\n      ],\n      [\n        {\n          &quot;action&quot;: 3,\n          &quot;reward&quot;: 0,\n          &quot;info&quot;: {},\n          &quot;observation&quot;: {\n            &quot;board&quot;: [\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              1,\n              0,\n              0,\n              0\n            ],\n            &quot;mark&quot;: 1\n          },\n          &quot;status&quot;: &quot;INACTIVE&quot;\n        },\n        {\n          &quot;action&quot;: 0,\n          &quot;reward&quot;: 0,\n          &quot;info&quot;: {},\n          &quot;observation&quot;: {\n            &quot;mark&quot;: 2\n          },\n          &quot;status&quot;: &quot;ACTIVE&quot;\n        }\n      ],\n      [\n        {\n          &quot;action&quot;: 0,\n          &quot;reward&quot;: 0,\n          &quot;info&quot;: {},\n          &quot;observation&quot;: {\n            &quot;board&quot;: [\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              2,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              1,\n              0,\n              0,\n              0\n            ],\n            &quot;mark&quot;: 1\n          },\n          &quot;status&quot;: &quot;ACTIVE&quot;\n        },\n        {\n          &quot;action&quot;: 3,\n          &quot;reward&quot;: 0,\n          &quot;info&quot;: {},\n          &quot;observation&quot;: {\n            &quot;mark&quot;: 2\n          },\n          &quot;status&quot;: &quot;INACTIVE&quot;\n        }\n      ],\n      [\n        {\n          &quot;action&quot;: 3,\n          &quot;reward&quot;: 0,\n          &quot;info&quot;: {},\n          &quot;observation&quot;: {\n            &quot;board&quot;: [\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              1,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              2,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              1,\n              0,\n              0,\n              0\n            ],\n            &quot;mark&quot;: 1\n          },\n          &quot;status&quot;: &quot;INACTIVE&quot;\n        },\n        {\n          &quot;action&quot;: 0,\n          &quot;reward&quot;: 0,\n          &quot;info&quot;: {},\n          &quot;observation&quot;: {\n            &quot;mark&quot;: 2\n          },\n          &quot;status&quot;: &quot;ACTIVE&quot;\n        }\n      ],\n      [\n        {\n          &quot;action&quot;: 0,\n          &quot;reward&quot;: 0,\n          &quot;info&quot;: {},\n          &quot;observation&quot;: {\n            &quot;board&quot;: [\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              2,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              1,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              2,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              1,\n              0,\n              0,\n              0\n            ],\n            &quot;mark&quot;: 1\n          },\n          &quot;status&quot;: &quot;ACTIVE&quot;\n        },\n        {\n          &quot;action&quot;: 3,\n          &quot;reward&quot;: 0,\n          &quot;info&quot;: {},\n          &quot;observation&quot;: {\n            &quot;mark&quot;: 2\n          },\n          &quot;status&quot;: &quot;INACTIVE&quot;\n        }\n      ],\n      [\n        {\n          &quot;action&quot;: 2,\n          &quot;reward&quot;: 0,\n          &quot;info&quot;: {},\n          &quot;observation&quot;: {\n            &quot;board&quot;: [\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              2,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              1,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              2,\n              0,\n              0,\n              0,\n              0,\n              0,\n              1,\n              1,\n              0,\n              0,\n              0\n            ],\n            &quot;mark&quot;: 1\n          },\n          &quot;status&quot;: &quot;INACTIVE&quot;\n        },\n        {\n          &quot;action&quot;: 0,\n          &quot;reward&quot;: 0,\n          &quot;info&quot;: {},\n          &quot;observation&quot;: {\n            &quot;mark&quot;: 2\n          },\n          &quot;status&quot;: &quot;ACTIVE&quot;\n        }\n      ],\n      [\n        {\n          &quot;action&quot;: 0,\n          &quot;reward&quot;: 0,\n          &quot;info&quot;: {},\n          &quot;observation&quot;: {\n            &quot;board&quot;: [\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              2,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              1,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              2,\n              0,\n              0,\n              0,\n              0,\n              0,\n              1,\n              1,\n              2,\n              0,\n              0\n            ],\n            &quot;mark&quot;: 1\n          },\n          &quot;status&quot;: &quot;ACTIVE&quot;\n        },\n        {\n          &quot;action&quot;: 4,\n          &quot;reward&quot;: 0,\n          &quot;info&quot;: {},\n          &quot;observation&quot;: {\n            &quot;mark&quot;: 2\n          },\n          &quot;status&quot;: &quot;INACTIVE&quot;\n        }\n      ],\n      [\n        {\n          &quot;action&quot;: 4,\n          &quot;reward&quot;: 0,\n          &quot;info&quot;: {},\n          &quot;observation&quot;: {\n            &quot;board&quot;: [\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              2,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              1,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              2,\n              1,\n              0,\n              0,\n              0,\n              0,\n              1,\n              1,\n              2,\n              0,\n              0\n            ],\n            &quot;mark&quot;: 1\n          },\n          &quot;status&quot;: &quot;INACTIVE&quot;\n        },\n        {\n          &quot;action&quot;: 0,\n          &quot;reward&quot;: 0,\n          &quot;info&quot;: {},\n          &quot;observation&quot;: {\n            &quot;mark&quot;: 2\n          },\n          &quot;status&quot;: &quot;ACTIVE&quot;\n        }\n      ],\n      [\n        {\n          &quot;action&quot;: 0,\n          &quot;reward&quot;: 0,\n          &quot;info&quot;: {},\n          &quot;observation&quot;: {\n            &quot;board&quot;: [\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              2,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              1,\n              2,\n              0,\n              0,\n              0,\n              0,\n              0,\n              2,\n              1,\n              0,\n              0,\n              0,\n              0,\n              1,\n              1,\n              2,\n              0,\n              0\n            ],\n            &quot;mark&quot;: 1\n          },\n          &quot;status&quot;: &quot;ACTIVE&quot;\n        },\n        {\n          &quot;action&quot;: 4,\n          &quot;reward&quot;: 0,\n          &quot;info&quot;: {},\n          &quot;observation&quot;: {\n            &quot;mark&quot;: 2\n          },\n          &quot;status&quot;: &quot;INACTIVE&quot;\n        }\n      ],\n      [\n        {\n          &quot;action&quot;: 4,\n          &quot;reward&quot;: 0,\n          &quot;info&quot;: {},\n          &quot;observation&quot;: {\n            &quot;board&quot;: [\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              2,\n              1,\n              0,\n              0,\n              0,\n              0,\n              0,\n              1,\n              2,\n              0,\n              0,\n              0,\n              0,\n              0,\n              2,\n              1,\n              0,\n              0,\n              0,\n              0,\n              1,\n              1,\n              2,\n              0,\n              0\n            ],\n            &quot;mark&quot;: 1\n          },\n          &quot;status&quot;: &quot;INACTIVE&quot;\n        },\n        {\n          &quot;action&quot;: 0,\n          &quot;reward&quot;: 0,\n          &quot;info&quot;: {},\n          &quot;observation&quot;: {\n            &quot;mark&quot;: 2\n          },\n          &quot;status&quot;: &quot;ACTIVE&quot;\n        }\n      ],\n      [\n        {\n          &quot;action&quot;: 0,\n          &quot;reward&quot;: 0,\n          &quot;info&quot;: {},\n          &quot;observation&quot;: {\n            &quot;board&quot;: [\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              2,\n              1,\n              0,\n              0,\n              0,\n              0,\n              0,\n              1,\n              2,\n              0,\n              0,\n              0,\n              0,\n              2,\n              2,\n              1,\n              0,\n              0,\n              0,\n              0,\n              1,\n              1,\n              2,\n              0,\n              0\n            ],\n            &quot;mark&quot;: 1\n          },\n          &quot;status&quot;: &quot;ACTIVE&quot;\n        },\n        {\n          &quot;action&quot;: 2,\n          &quot;reward&quot;: 0,\n          &quot;info&quot;: {},\n          &quot;observation&quot;: {\n            &quot;mark&quot;: 2\n          },\n          &quot;status&quot;: &quot;INACTIVE&quot;\n        }\n      ],\n      [\n        {\n          &quot;action&quot;: 0,\n          &quot;reward&quot;: 0,\n          &quot;info&quot;: {},\n          &quot;observation&quot;: {\n            &quot;board&quot;: [\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              2,\n              1,\n              0,\n              0,\n              0,\n              0,\n              0,\n              1,\n              2,\n              0,\n              0,\n              0,\n              0,\n              2,\n              2,\n              1,\n              0,\n              0,\n              1,\n              0,\n              1,\n              1,\n              2,\n              0,\n              0\n            ],\n            &quot;mark&quot;: 1\n          },\n          &quot;status&quot;: &quot;INACTIVE&quot;\n        },\n        {\n          &quot;action&quot;: 0,\n          &quot;reward&quot;: 0,\n          &quot;info&quot;: {},\n          &quot;observation&quot;: {\n            &quot;mark&quot;: 2\n          },\n          &quot;status&quot;: &quot;ACTIVE&quot;\n        }\n      ],\n      [\n        {\n          &quot;action&quot;: 0,\n          &quot;reward&quot;: 0,\n          &quot;info&quot;: {},\n          &quot;observation&quot;: {\n            &quot;board&quot;: [\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              2,\n              1,\n              0,\n              0,\n              0,\n              0,\n              0,\n              1,\n              2,\n              0,\n              0,\n              0,\n              0,\n              2,\n              2,\n              1,\n              0,\n              0,\n              1,\n              2,\n              1,\n              1,\n              2,\n              0,\n              0\n            ],\n            &quot;mark&quot;: 1\n          },\n          &quot;status&quot;: &quot;ACTIVE&quot;\n        },\n        {\n          &quot;action&quot;: 1,\n          &quot;reward&quot;: 0,\n          &quot;info&quot;: {},\n          &quot;observation&quot;: {\n            &quot;mark&quot;: 2\n          },\n          &quot;status&quot;: &quot;INACTIVE&quot;\n        }\n      ],\n      [\n        {\n          &quot;action&quot;: 3,\n          &quot;reward&quot;: 0,\n          &quot;info&quot;: {},\n          &quot;observation&quot;: {\n            &quot;board&quot;: [\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              1,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              2,\n              1,\n              0,\n              0,\n              0,\n              0,\n              0,\n              1,\n              2,\n              0,\n              0,\n              0,\n              0,\n              2,\n              2,\n              1,\n              0,\n              0,\n              1,\n              2,\n              1,\n              1,\n              2,\n              0,\n              0\n            ],\n            &quot;mark&quot;: 1\n          },\n          &quot;status&quot;: &quot;INACTIVE&quot;\n        },\n        {\n          &quot;action&quot;: 0,\n          &quot;reward&quot;: 0,\n          &quot;info&quot;: {},\n          &quot;observation&quot;: {\n            &quot;mark&quot;: 2\n          },\n          &quot;status&quot;: &quot;ACTIVE&quot;\n        }\n      ],\n      [\n        {\n          &quot;action&quot;: 0,\n          &quot;reward&quot;: 0,\n          &quot;info&quot;: {},\n          &quot;observation&quot;: {\n            &quot;board&quot;: [\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              1,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              2,\n              1,\n              0,\n              0,\n              0,\n              0,\n              0,\n              1,\n              2,\n              0,\n              0,\n              0,\n              2,\n              2,\n              2,\n              1,\n              0,\n              0,\n              1,\n              2,\n              1,\n              1,\n              2,\n              0,\n              0\n            ],\n            &quot;mark&quot;: 1\n          },\n          &quot;status&quot;: &quot;ACTIVE&quot;\n        },\n        {\n          &quot;action&quot;: 1,\n          &quot;reward&quot;: 0,\n          &quot;info&quot;: {},\n          &quot;observation&quot;: {\n            &quot;mark&quot;: 2\n          },\n          &quot;status&quot;: &quot;INACTIVE&quot;\n        }\n      ],\n      [\n        {\n          &quot;action&quot;: 0,\n          &quot;reward&quot;: 0,\n          &quot;info&quot;: {},\n          &quot;observation&quot;: {\n            &quot;board&quot;: [\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              1,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              2,\n              1,\n              0,\n              0,\n              0,\n              0,\n              0,\n              1,\n              2,\n              0,\n              0,\n              1,\n              2,\n              2,\n              2,\n              1,\n              0,\n              0,\n              1,\n              2,\n              1,\n              1,\n              2,\n              0,\n              0\n            ],\n            &quot;mark&quot;: 1\n          },\n          &quot;status&quot;: &quot;INACTIVE&quot;\n        },\n        {\n          &quot;action&quot;: 0,\n          &quot;reward&quot;: 0,\n          &quot;info&quot;: {},\n          &quot;observation&quot;: {\n            &quot;mark&quot;: 2\n          },\n          &quot;status&quot;: &quot;ACTIVE&quot;\n        }\n      ],\n      [\n        {\n          &quot;action&quot;: 0,\n          &quot;reward&quot;: 0,\n          &quot;info&quot;: {},\n          &quot;observation&quot;: {\n            &quot;board&quot;: [\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              1,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              2,\n              1,\n              0,\n              0,\n              0,\n              2,\n              0,\n              1,\n              2,\n              0,\n              0,\n              1,\n              2,\n              2,\n              2,\n              1,\n              0,\n              0,\n              1,\n              2,\n              1,\n              1,\n              2,\n              0,\n              0\n            ],\n            &quot;mark&quot;: 1\n          },\n          &quot;status&quot;: &quot;ACTIVE&quot;\n        },\n        {\n          &quot;action&quot;: 1,\n          &quot;reward&quot;: 0,\n          &quot;info&quot;: {},\n          &quot;observation&quot;: {\n            &quot;mark&quot;: 2\n          },\n          &quot;status&quot;: &quot;INACTIVE&quot;\n        }\n      ],\n      [\n        {\n          &quot;action&quot;: 1,\n          &quot;reward&quot;: 0,\n          &quot;info&quot;: {},\n          &quot;observation&quot;: {\n            &quot;board&quot;: [\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              1,\n              0,\n              0,\n              0,\n              0,\n              1,\n              0,\n              2,\n              1,\n              0,\n              0,\n              0,\n              2,\n              0,\n              1,\n              2,\n              0,\n              0,\n              1,\n              2,\n              2,\n              2,\n              1,\n              0,\n              0,\n              1,\n              2,\n              1,\n              1,\n              2,\n              0,\n              0\n            ],\n            &quot;mark&quot;: 1\n          },\n          &quot;status&quot;: &quot;INACTIVE&quot;\n        },\n        {\n          &quot;action&quot;: 0,\n          &quot;reward&quot;: 0,\n          &quot;info&quot;: {},\n          &quot;observation&quot;: {\n            &quot;mark&quot;: 2\n          },\n          &quot;status&quot;: &quot;ACTIVE&quot;\n        }\n      ],\n      [\n        {\n          &quot;action&quot;: 0,\n          &quot;reward&quot;: 0,\n          &quot;info&quot;: {},\n          &quot;observation&quot;: {\n            &quot;board&quot;: [\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              1,\n              2,\n              0,\n              0,\n              0,\n              1,\n              0,\n              2,\n              1,\n              0,\n              0,\n              0,\n              2,\n              0,\n              1,\n              2,\n              0,\n              0,\n              1,\n              2,\n              2,\n              2,\n              1,\n              0,\n              0,\n              1,\n              2,\n              1,\n              1,\n              2,\n              0,\n              0\n            ],\n            &quot;mark&quot;: 1\n          },\n          &quot;status&quot;: &quot;ACTIVE&quot;\n        },\n        {\n          &quot;action&quot;: 4,\n          &quot;reward&quot;: 0,\n          &quot;info&quot;: {},\n          &quot;observation&quot;: {\n            &quot;mark&quot;: 2\n          },\n          &quot;status&quot;: &quot;INACTIVE&quot;\n        }\n      ],\n      [\n        {\n          &quot;action&quot;: 2,\n          &quot;reward&quot;: 0,\n          &quot;info&quot;: {},\n          &quot;observation&quot;: {\n            &quot;board&quot;: [\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              1,\n              2,\n              0,\n              0,\n              0,\n              1,\n              0,\n              2,\n              1,\n              0,\n              0,\n              0,\n              2,\n              1,\n              1,\n              2,\n              0,\n              0,\n              1,\n              2,\n              2,\n              2,\n              1,\n              0,\n              0,\n              1,\n              2,\n              1,\n              1,\n              2,\n              0,\n              0\n            ],\n            &quot;mark&quot;: 1\n          },\n          &quot;status&quot;: &quot;INACTIVE&quot;\n        },\n        {\n          &quot;action&quot;: 0,\n          &quot;reward&quot;: 0,\n          &quot;info&quot;: {},\n          &quot;observation&quot;: {\n            &quot;mark&quot;: 2\n          },\n          &quot;status&quot;: &quot;ACTIVE&quot;\n        }\n      ],\n      [\n        {\n          &quot;action&quot;: 0,\n          &quot;reward&quot;: 0,\n          &quot;info&quot;: {},\n          &quot;observation&quot;: {\n            &quot;board&quot;: [\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              1,\n              2,\n              0,\n              0,\n              0,\n              1,\n              2,\n              2,\n              1,\n              0,\n              0,\n              0,\n              2,\n              1,\n              1,\n              2,\n              0,\n              0,\n              1,\n              2,\n              2,\n              2,\n              1,\n              0,\n              0,\n              1,\n              2,\n              1,\n              1,\n              2,\n              0,\n              0\n            ],\n            &quot;mark&quot;: 1\n          },\n          &quot;status&quot;: &quot;ACTIVE&quot;\n        },\n        {\n          &quot;action&quot;: 2,\n          &quot;reward&quot;: 0,\n          &quot;info&quot;: {},\n          &quot;observation&quot;: {\n            &quot;mark&quot;: 2\n          },\n          &quot;status&quot;: &quot;INACTIVE&quot;\n        }\n      ],\n      [\n        {\n          &quot;action&quot;: 0,\n          &quot;reward&quot;: 0,\n          &quot;info&quot;: {},\n          &quot;observation&quot;: {\n            &quot;board&quot;: [\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              1,\n              2,\n              0,\n              0,\n              0,\n              1,\n              2,\n              2,\n              1,\n              0,\n              0,\n              1,\n              2,\n              1,\n              1,\n              2,\n              0,\n              0,\n              1,\n              2,\n              2,\n              2,\n              1,\n              0,\n              0,\n              1,\n              2,\n              1,\n              1,\n              2,\n              0,\n              0\n            ],\n            &quot;mark&quot;: 1\n          },\n          &quot;status&quot;: &quot;INACTIVE&quot;\n        },\n        {\n          &quot;action&quot;: 0,\n          &quot;reward&quot;: 0,\n          &quot;info&quot;: {},\n          &quot;observation&quot;: {\n            &quot;mark&quot;: 2\n          },\n          &quot;status&quot;: &quot;ACTIVE&quot;\n        }\n      ],\n      [\n        {\n          &quot;action&quot;: 0,\n          &quot;reward&quot;: 0,\n          &quot;info&quot;: {},\n          &quot;observation&quot;: {\n            &quot;board&quot;: [\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              1,\n              2,\n              0,\n              0,\n              2,\n              1,\n              2,\n              2,\n              1,\n              0,\n              0,\n              1,\n              2,\n              1,\n              1,\n              2,\n              0,\n              0,\n              1,\n              2,\n              2,\n              2,\n              1,\n              0,\n              0,\n              1,\n              2,\n              1,\n              1,\n              2,\n              0,\n              0\n            ],\n            &quot;mark&quot;: 1\n          },\n          &quot;status&quot;: &quot;ACTIVE&quot;\n        },\n        {\n          &quot;action&quot;: 0,\n          &quot;reward&quot;: 0,\n          &quot;info&quot;: {},\n          &quot;observation&quot;: {\n            &quot;mark&quot;: 2\n          },\n          &quot;status&quot;: &quot;INACTIVE&quot;\n        }\n      ],\n      [\n        {\n          &quot;action&quot;: 3,\n          &quot;reward&quot;: 0,\n          &quot;info&quot;: {},\n          &quot;observation&quot;: {\n            &quot;board&quot;: [\n              0,\n              0,\n              0,\n              1,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              1,\n              2,\n              0,\n              0,\n              2,\n              1,\n              2,\n              2,\n              1,\n              0,\n              0,\n              1,\n              2,\n              1,\n              1,\n              2,\n              0,\n              0,\n              1,\n              2,\n              2,\n              2,\n              1,\n              0,\n              0,\n              1,\n              2,\n              1,\n              1,\n              2,\n              0,\n              0\n            ],\n            &quot;mark&quot;: 1\n          },\n          &quot;status&quot;: &quot;INACTIVE&quot;\n        },\n        {\n          &quot;action&quot;: 0,\n          &quot;reward&quot;: 0,\n          &quot;info&quot;: {},\n          &quot;observation&quot;: {\n            &quot;mark&quot;: 2\n          },\n          &quot;status&quot;: &quot;ACTIVE&quot;\n        }\n      ],\n      [\n        {\n          &quot;action&quot;: 0,\n          &quot;reward&quot;: 0,\n          &quot;info&quot;: {},\n          &quot;observation&quot;: {\n            &quot;board&quot;: [\n              0,\n              0,\n              0,\n              1,\n              0,\n              0,\n              0,\n              0,\n              0,\n              2,\n              1,\n              2,\n              0,\n              0,\n              2,\n              1,\n              2,\n              2,\n              1,\n              0,\n              0,\n              1,\n              2,\n              1,\n              1,\n              2,\n              0,\n              0,\n              1,\n              2,\n              2,\n              2,\n              1,\n              0,\n              0,\n              1,\n              2,\n              1,\n              1,\n              2,\n              0,\n              0\n            ],\n            &quot;mark&quot;: 1\n          },\n          &quot;status&quot;: &quot;ACTIVE&quot;\n        },\n        {\n          &quot;action&quot;: 2,\n          &quot;reward&quot;: 0,\n          &quot;info&quot;: {},\n          &quot;observation&quot;: {\n            &quot;mark&quot;: 2\n          },\n          &quot;status&quot;: &quot;INACTIVE&quot;\n        }\n      ],\n      [\n        {\n          &quot;action&quot;: 2,\n          &quot;reward&quot;: 0,\n          &quot;info&quot;: {},\n          &quot;observation&quot;: {\n            &quot;board&quot;: [\n              0,\n              0,\n              1,\n              1,\n              0,\n              0,\n              0,\n              0,\n              0,\n              2,\n              1,\n              2,\n              0,\n              0,\n              2,\n              1,\n              2,\n              2,\n              1,\n              0,\n              0,\n              1,\n              2,\n              1,\n              1,\n              2,\n              0,\n              0,\n              1,\n              2,\n              2,\n              2,\n              1,\n              0,\n              0,\n              1,\n              2,\n              1,\n              1,\n              2,\n              0,\n              0\n            ],\n            &quot;mark&quot;: 1\n          },\n          &quot;status&quot;: &quot;INACTIVE&quot;\n        },\n        {\n          &quot;action&quot;: 0,\n          &quot;reward&quot;: 0,\n          &quot;info&quot;: {},\n          &quot;observation&quot;: {\n            &quot;mark&quot;: 2\n          },\n          &quot;status&quot;: &quot;ACTIVE&quot;\n        }\n      ],\n      [\n        {\n          &quot;action&quot;: 0,\n          &quot;reward&quot;: 0,\n          &quot;info&quot;: {},\n          &quot;observation&quot;: {\n            &quot;board&quot;: [\n              0,\n              0,\n              1,\n              1,\n              2,\n              0,\n              0,\n              0,\n              0,\n              2,\n              1,\n              2,\n              0,\n              0,\n              2,\n              1,\n              2,\n              2,\n              1,\n              0,\n              0,\n              1,\n              2,\n              1,\n              1,\n              2,\n              0,\n              0,\n              1,\n              2,\n              2,\n              2,\n              1,\n              0,\n              0,\n              1,\n              2,\n              1,\n              1,\n              2,\n              0,\n              0\n            ],\n            &quot;mark&quot;: 1\n          },\n          &quot;status&quot;: &quot;ACTIVE&quot;\n        },\n        {\n          &quot;action&quot;: 4,\n          &quot;reward&quot;: 0,\n          &quot;info&quot;: {},\n          &quot;observation&quot;: {\n            &quot;mark&quot;: 2\n          },\n          &quot;status&quot;: &quot;INACTIVE&quot;\n        }\n      ],\n      [\n        {\n          &quot;action&quot;: 6,\n          &quot;reward&quot;: 0,\n          &quot;info&quot;: {},\n          &quot;observation&quot;: {\n            &quot;board&quot;: [\n              0,\n              0,\n              1,\n              1,\n              2,\n              0,\n              0,\n              0,\n              0,\n              2,\n              1,\n              2,\n              0,\n              0,\n              2,\n              1,\n              2,\n              2,\n              1,\n              0,\n              0,\n              1,\n              2,\n              1,\n              1,\n              2,\n              0,\n              0,\n              1,\n              2,\n              2,\n              2,\n              1,\n              0,\n              0,\n              1,\n              2,\n              1,\n              1,\n              2,\n              0,\n              1\n            ],\n            &quot;mark&quot;: 1\n          },\n          &quot;status&quot;: &quot;INACTIVE&quot;\n        },\n        {\n          &quot;action&quot;: 0,\n          &quot;reward&quot;: 0,\n          &quot;info&quot;: {},\n          &quot;observation&quot;: {\n            &quot;mark&quot;: 2\n          },\n          &quot;status&quot;: &quot;ACTIVE&quot;\n        }\n      ],\n      [\n        {\n          &quot;action&quot;: 0,\n          &quot;reward&quot;: 0,\n          &quot;info&quot;: {},\n          &quot;observation&quot;: {\n            &quot;board&quot;: [\n              0,\n              0,\n              1,\n              1,\n              2,\n              0,\n              0,\n              0,\n              0,\n              2,\n              1,\n              2,\n              0,\n              0,\n              2,\n              1,\n              2,\n              2,\n              1,\n              0,\n              0,\n              1,\n              2,\n              1,\n              1,\n              2,\n              0,\n              0,\n              1,\n              2,\n              2,\n              2,\n              1,\n              0,\n              2,\n              1,\n              2,\n              1,\n              1,\n              2,\n              0,\n              1\n            ],\n            &quot;mark&quot;: 1\n          },\n          &quot;status&quot;: &quot;ACTIVE&quot;\n        },\n        {\n          &quot;action&quot;: 6,\n          &quot;reward&quot;: 0,\n          &quot;info&quot;: {},\n          &quot;observation&quot;: {\n            &quot;mark&quot;: 2\n          },\n          &quot;status&quot;: &quot;INACTIVE&quot;\n        }\n      ],\n      [\n        {\n          &quot;action&quot;: 0,\n          &quot;reward&quot;: 0,\n          &quot;info&quot;: {},\n          &quot;observation&quot;: {\n            &quot;board&quot;: [\n              0,\n              0,\n              1,\n              1,\n              2,\n              0,\n              0,\n              1,\n              0,\n              2,\n              1,\n              2,\n              0,\n              0,\n              2,\n              1,\n              2,\n              2,\n              1,\n              0,\n              0,\n              1,\n              2,\n              1,\n              1,\n              2,\n              0,\n              0,\n              1,\n              2,\n              2,\n              2,\n              1,\n              0,\n              2,\n              1,\n              2,\n              1,\n              1,\n              2,\n              0,\n              1\n            ],\n            &quot;mark&quot;: 1\n          },\n          &quot;status&quot;: &quot;INACTIVE&quot;\n        },\n        {\n          &quot;action&quot;: 0,\n          &quot;reward&quot;: 0,\n          &quot;info&quot;: {},\n          &quot;observation&quot;: {\n            &quot;mark&quot;: 2\n          },\n          &quot;status&quot;: &quot;ACTIVE&quot;\n        }\n      ],\n      [\n        {\n          &quot;action&quot;: 0,\n          &quot;reward&quot;: 0,\n          &quot;info&quot;: {},\n          &quot;observation&quot;: {\n            &quot;board&quot;: [\n              2,\n              0,\n              1,\n              1,\n              2,\n              0,\n              0,\n              1,\n              0,\n              2,\n              1,\n              2,\n              0,\n              0,\n              2,\n              1,\n              2,\n              2,\n              1,\n              0,\n              0,\n              1,\n              2,\n              1,\n              1,\n              2,\n              0,\n              0,\n              1,\n              2,\n              2,\n              2,\n              1,\n              0,\n              2,\n              1,\n              2,\n              1,\n              1,\n              2,\n              0,\n              1\n            ],\n            &quot;mark&quot;: 1\n          },\n          &quot;status&quot;: &quot;ACTIVE&quot;\n        },\n        {\n          &quot;action&quot;: 0,\n          &quot;reward&quot;: 0,\n          &quot;info&quot;: {},\n          &quot;observation&quot;: {\n            &quot;mark&quot;: 2\n          },\n          &quot;status&quot;: &quot;INACTIVE&quot;\n        }\n      ],\n      [\n        {\n          &quot;action&quot;: 6,\n          &quot;reward&quot;: 0,\n          &quot;info&quot;: {},\n          &quot;observation&quot;: {\n            &quot;board&quot;: [\n              2,\n              0,\n              1,\n              1,\n              2,\n              0,\n              0,\n              1,\n              0,\n              2,\n              1,\n              2,\n              0,\n              0,\n              2,\n              1,\n              2,\n              2,\n              1,\n              0,\n              0,\n              1,\n              2,\n              1,\n              1,\n              2,\n              0,\n              1,\n              1,\n              2,\n              2,\n              2,\n              1,\n              0,\n              2,\n              1,\n              2,\n              1,\n              1,\n              2,\n              0,\n              1\n            ],\n            &quot;mark&quot;: 1\n          },\n          &quot;status&quot;: &quot;INACTIVE&quot;\n        },\n        {\n          &quot;action&quot;: 0,\n          &quot;reward&quot;: 0,\n          &quot;info&quot;: {},\n          &quot;observation&quot;: {\n            &quot;mark&quot;: 2\n          },\n          &quot;status&quot;: &quot;ACTIVE&quot;\n        }\n      ],\n      [\n        {\n          &quot;action&quot;: 0,\n          &quot;reward&quot;: 0,\n          &quot;info&quot;: {},\n          &quot;observation&quot;: {\n            &quot;board&quot;: [\n              2,\n              0,\n              1,\n              1,\n              2,\n              0,\n              0,\n              1,\n              2,\n              2,\n              1,\n              2,\n              0,\n              0,\n              2,\n              1,\n              2,\n              2,\n              1,\n              0,\n              0,\n              1,\n              2,\n              1,\n              1,\n              2,\n              0,\n              1,\n              1,\n              2,\n              2,\n              2,\n              1,\n              0,\n              2,\n              1,\n              2,\n              1,\n              1,\n              2,\n              0,\n              1\n            ],\n            &quot;mark&quot;: 1\n          },\n          &quot;status&quot;: &quot;ACTIVE&quot;\n        },\n        {\n          &quot;action&quot;: 1,\n          &quot;reward&quot;: 0,\n          &quot;info&quot;: {},\n          &quot;observation&quot;: {\n            &quot;mark&quot;: 2\n          },\n          &quot;status&quot;: &quot;INACTIVE&quot;\n        }\n      ],\n      [\n        {\n          &quot;action&quot;: 1,\n          &quot;reward&quot;: 0,\n          &quot;info&quot;: {},\n          &quot;observation&quot;: {\n            &quot;board&quot;: [\n              2,\n              1,\n              1,\n              1,\n              2,\n              0,\n              0,\n              1,\n              2,\n              2,\n              1,\n              2,\n              0,\n              0,\n              2,\n              1,\n              2,\n              2,\n              1,\n              0,\n              0,\n              1,\n              2,\n              1,\n              1,\n              2,\n              0,\n              1,\n              1,\n              2,\n              2,\n              2,\n              1,\n              0,\n              2,\n              1,\n              2,\n              1,\n              1,\n              2,\n              0,\n              1\n            ],\n            &quot;mark&quot;: 1\n          },\n          &quot;status&quot;: &quot;INACTIVE&quot;\n        },\n        {\n          &quot;action&quot;: 0,\n          &quot;reward&quot;: 0,\n          &quot;info&quot;: {},\n          &quot;observation&quot;: {\n            &quot;mark&quot;: 2\n          },\n          &quot;status&quot;: &quot;ACTIVE&quot;\n        }\n      ],\n      [\n        {\n          &quot;action&quot;: 0,\n          &quot;reward&quot;: 0,\n          &quot;info&quot;: {},\n          &quot;observation&quot;: {\n            &quot;board&quot;: [\n              2,\n              1,\n              1,\n              1,\n              2,\n              0,\n              0,\n              1,\n              2,\n              2,\n              1,\n              2,\n              0,\n              0,\n              2,\n              1,\n              2,\n              2,\n              1,\n              0,\n              2,\n              1,\n              2,\n              1,\n              1,\n              2,\n              0,\n              1,\n              1,\n              2,\n              2,\n              2,\n              1,\n              0,\n              2,\n              1,\n              2,\n              1,\n              1,\n              2,\n              0,\n              1\n            ],\n            &quot;mark&quot;: 1\n          },\n          &quot;status&quot;: &quot;ACTIVE&quot;\n        },\n        {\n          &quot;action&quot;: 6,\n          &quot;reward&quot;: 0,\n          &quot;info&quot;: {},\n          &quot;observation&quot;: {\n            &quot;mark&quot;: 2\n          },\n          &quot;status&quot;: &quot;INACTIVE&quot;\n        }\n      ],\n      [\n        {\n          &quot;action&quot;: 6,\n          &quot;reward&quot;: 0,\n          &quot;info&quot;: {},\n          &quot;observation&quot;: {\n            &quot;board&quot;: [\n              2,\n              1,\n              1,\n              1,\n              2,\n              0,\n              0,\n              1,\n              2,\n              2,\n              1,\n              2,\n              0,\n              1,\n              2,\n              1,\n              2,\n              2,\n              1,\n              0,\n              2,\n              1,\n              2,\n              1,\n              1,\n              2,\n              0,\n              1,\n              1,\n              2,\n              2,\n              2,\n              1,\n              0,\n              2,\n              1,\n              2,\n              1,\n              1,\n              2,\n              0,\n              1\n            ],\n            &quot;mark&quot;: 1\n          },\n          &quot;status&quot;: &quot;INACTIVE&quot;\n        },\n        {\n          &quot;action&quot;: 0,\n          &quot;reward&quot;: 0,\n          &quot;info&quot;: {},\n          &quot;observation&quot;: {\n            &quot;mark&quot;: 2\n          },\n          &quot;status&quot;: &quot;ACTIVE&quot;\n        }\n      ],\n      [\n        {\n          &quot;action&quot;: 0,\n          &quot;reward&quot;: 0,\n          &quot;info&quot;: {},\n          &quot;observation&quot;: {\n            &quot;board&quot;: [\n              2,\n              1,\n              1,\n              1,\n              2,\n              0,\n              2,\n              1,\n              2,\n              2,\n              1,\n              2,\n              0,\n              1,\n              2,\n              1,\n              2,\n              2,\n              1,\n              0,\n              2,\n              1,\n              2,\n              1,\n              1,\n              2,\n              0,\n              1,\n              1,\n              2,\n              2,\n              2,\n              1,\n              0,\n              2,\n              1,\n              2,\n              1,\n              1,\n              2,\n              0,\n              1\n            ],\n            &quot;mark&quot;: 1\n          },\n          &quot;status&quot;: &quot;ACTIVE&quot;\n        },\n        {\n          &quot;action&quot;: 6,\n          &quot;reward&quot;: 0,\n          &quot;info&quot;: {},\n          &quot;observation&quot;: {\n            &quot;mark&quot;: 2\n          },\n          &quot;status&quot;: &quot;INACTIVE&quot;\n        }\n      ],\n      [\n        {\n          &quot;action&quot;: 5,\n          &quot;reward&quot;: 0,\n          &quot;info&quot;: {},\n          &quot;observation&quot;: {\n            &quot;board&quot;: [\n              2,\n              1,\n              1,\n              1,\n              2,\n              0,\n              2,\n              1,\n              2,\n              2,\n              1,\n              2,\n              0,\n              1,\n              2,\n              1,\n              2,\n              2,\n              1,\n              0,\n              2,\n              1,\n              2,\n              1,\n              1,\n              2,\n              0,\n              1,\n              1,\n              2,\n              2,\n              2,\n              1,\n              0,\n              2,\n              1,\n              2,\n              1,\n              1,\n              2,\n              1,\n              1\n            ],\n            &quot;mark&quot;: 1\n          },\n          &quot;status&quot;: &quot;INACTIVE&quot;\n        },\n        {\n          &quot;action&quot;: 0,\n          &quot;reward&quot;: 0,\n          &quot;info&quot;: {},\n          &quot;observation&quot;: {\n            &quot;mark&quot;: 2\n          },\n          &quot;status&quot;: &quot;ACTIVE&quot;\n        }\n      ],\n      [\n        {\n          &quot;action&quot;: 0,\n          &quot;reward&quot;: -1,\n          &quot;info&quot;: {},\n          &quot;observation&quot;: {\n            &quot;board&quot;: [\n              2,\n              1,\n              1,\n              1,\n              2,\n              0,\n              2,\n              1,\n              2,\n              2,\n              1,\n              2,\n              0,\n              1,\n              2,\n              1,\n              2,\n              2,\n              1,\n              0,\n              2,\n              1,\n              2,\n              1,\n              1,\n              2,\n              0,\n              1,\n              1,\n              2,\n              2,\n              2,\n              1,\n              2,\n              2,\n              1,\n              2,\n              1,\n              1,\n              2,\n              1,\n              1\n            ],\n            &quot;mark&quot;: 1\n          },\n          &quot;status&quot;: &quot;DONE&quot;\n        },\n        {\n          &quot;action&quot;: 5,\n          &quot;reward&quot;: 1,\n          &quot;info&quot;: {},\n          &quot;observation&quot;: {\n            &quot;mark&quot;: 2\n          },\n          &quot;status&quot;: &quot;DONE&quot;\n        }\n      ]\n    ],\n    &quot;rewards&quot;: [\n      -1,\n      1\n    ],\n    &quot;statuses&quot;: [\n      &quot;DONE&quot;,\n      &quot;DONE&quot;\n    ],\n    &quot;schema_version&quot;: 1\n  },\n  &quot;mode&quot;: &quot;ipython&quot;,\n  &quot;width&quot;: 500,\n  &quot;height&quot;: 450\n};\n\n\nwindow.kaggle.renderer = // Copyright 2020 Kaggle Inc\n//\n// Licensed under the Apache License, Version 2.0 (the &quot;License&quot;);\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//      http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an &quot;AS IS&quot; BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\nfunction renderer({\n  act,\n  agents,\n  environment,\n  frame,\n  height = 400,\n  interactive,\n  isInteractive,\n  parent,\n  step,\n  update,\n  width = 400,\n}) {\n  // Configuration.\n  const { rows, columns, inarow } = environment.configuration;\n\n  // Common Dimensions.\n  const unit = 8;\n  const minCanvasSize = Math.min(height, width);\n  const minOffset = minCanvasSize > 400 ? 30 : unit / 2;\n  const cellSize = Math.min(\n    (width - minOffset * 2) / columns,\n    (height - minOffset * 2) / rows\n  );\n  const cellInset = 0.8;\n  const pieceScale = cellSize / 100;\n  const xOffset = Math.max(0, (width - cellSize * columns) / 2);\n  const yOffset = Math.max(0, (height - cellSize * rows) / 2);\n\n  // Canvas Setup.\n  let canvas = parent.querySelector(&quot;canvas&quot;);\n  if (!canvas) {\n    canvas = document.createElement(&quot;canvas&quot;);\n    parent.appendChild(canvas);\n\n    if (interactive) {\n      canvas.addEventListener(&quot;click&quot;, evt => {\n        if (!isInteractive()) return;\n        const rect = evt.target.getBoundingClientRect();\n        const col = Math.floor((evt.clientX - rect.left - xOffset) / cellSize);\n        if (col >= 0 && col < columns) act(col);\n      });\n    }\n  }\n  canvas.style.cursor = isInteractive() ? &quot;pointer&quot; : &quot;default&quot;;\n\n  // Character Paths (based on 100x100 tiles).\n  const kPath = new Path2D(\n    `M78.3,96.5c-0.1,0.4-0.5,0.6-1.1,0.6H64.9c-0.7,0-1.4-0.3-1.9-1l-20.3-26L37,75.5v20.1 c0,0.9-0.5,1.4-1.4,1.4H26c-0.9,0-1.4-0.5-1.4-1.4V3.9c0-0.9,0.5-1.4,1.4-1.4h9.5C36.5,2.5,37,3,37,3.9v56.5l24.3-24.7 c0.6-0.6,1.3-1,1.9-1H76c0.6,0,0.9,0.2,1.1,0.7c0.2,0.6,0.1,1-0.1,1.2l-25.7,25L78,95.1C78.4,95.5,78.5,95.9,78.3,96.5z`\n  );\n  const goose1Path = new Path2D(\n    `M8.8,92.7c-4-18.5,4.7-37.2,20.7-46.2c0,0,2.7-1.4,3.4-1.9c2.2-1.6,3-2.1,3-5c0-5-2.1-7.2-2.1-7.2 c-3.9-3.3-6.3-8.2-6.3-13.7c0-10,8.1-18.1,18.1-18.1s18.1,8.1,18.1,18.1c0,6-1.5,32.7-2.3,38.8l-0.1,1`\n  );\n  const goose2Path = new Path2D(\n    `M27.4,19L8.2,27.6c0,0-7.3,2.9,2.6,5c6.1,1.3,24,5.9,24,5.9l1,0.3`\n  );\n  const goose3Path = new Path2D(\n    `M63.7,99.6C52.3,99.6,43,90.3,43,78.9s9.3-20.7,20.7-20.7c10.6,0,34.4,0.1,35.8,9`\n  );\n\n  // Canvas setup and reset.\n  let c = canvas.getContext(&quot;2d&quot;);\n  canvas.width = width;\n  canvas.height = height;\n  c.fillStyle = &quot;#000B2A&quot;;\n  c.fillRect(0, 0, canvas.width, canvas.height);\n\n  const getRowCol = cell => [Math.floor(cell / columns), cell % columns];\n\n  const getColor = (mark, opacity = 1) => {\n    if (mark === 1) return `rgba(0,255,255,${opacity})`;\n    if (mark === 2) return `rgba(255,255,255,${opacity})`;\n    return &quot;#fff&quot;;\n  };\n\n  const drawCellCircle = (cell, xFrame = 1, yFrame = 1, radiusOffset = 0) => {\n    const [row, col] = getRowCol(cell);\n    c.arc(\n      xOffset + xFrame * (col * cellSize + cellSize / 2),\n      yOffset + yFrame * (row * cellSize + cellSize / 2),\n      (cellInset * cellSize) / 2 - radiusOffset,\n      2 * Math.PI,\n      false\n    );\n  };\n\n  // Render the pieces.\n  const board = environment.steps[step][0].observation.board;\n\n  const drawPiece = mark => {\n    // Base Styles.\n    const opacity = minCanvasSize < 300 ? 0.6 - minCanvasSize / 1000 : 0.1;\n    c.fillStyle = getColor(mark, opacity);\n    c.strokeStyle = getColor(mark);\n    c.shadowColor = getColor(mark);\n    c.shadowBlur = 8 / cellInset;\n    c.lineWidth = 1 / cellInset;\n\n    // Outer circle.\n    c.save();\n    c.beginPath();\n    c.arc(50, 50, 50, 2 * Math.PI, false);\n    c.closePath();\n    c.lineWidth *= 4;\n    c.stroke();\n    c.fill();\n    c.restore();\n\n    // Inner circle.\n    c.beginPath();\n    c.arc(50, 50, 40, 2 * Math.PI, false);\n    c.closePath();\n    c.stroke();\n\n    // Kaggle &quot;K&quot;.\n    if (mark === 1) {\n      const scale = 0.54;\n      c.save();\n      c.translate(23, 23);\n      c.scale(scale, scale);\n      c.lineWidth /= scale;\n      c.shadowBlur /= scale;\n      c.stroke(kPath);\n      c.restore();\n    }\n\n    // Kaggle &quot;Goose&quot;.\n    if (mark === 2) {\n      const scale = 0.6;\n      c.save();\n      c.translate(24, 28);\n      c.scale(scale, scale);\n      c.lineWidth /= scale;\n      c.shadowBlur /= scale;\n      c.stroke(goose1Path);\n      c.stroke(goose2Path);\n      c.stroke(goose3Path);\n      c.beginPath();\n      c.arc(38.5, 18.6, 2.7, 0, Math.PI * 2, false);\n      c.closePath();\n      c.fill();\n      c.restore();\n    }\n  };\n\n  for (let i = 0; i < board.length; i++) {\n    const [row, col] = getRowCol(i);\n    if (board[i] === 0) continue;\n    // Easing In.\n    let yFrame = Math.min(\n      (columns * Math.pow(frame, 3)) / Math.floor(i / columns),\n      1\n    );\n\n    if (\n      step > 1 &&\n      environment.steps[step - 1][0].observation.board[i] === board[i]\n    ) {\n      yFrame = 1;\n    }\n\n    c.save();\n    c.translate(\n      xOffset + cellSize * col + (cellSize - cellSize * cellInset) / 2,\n      yOffset +\n        yFrame * (cellSize * row) +\n        (cellSize - cellSize * cellInset) / 2\n    );\n    c.scale(pieceScale * cellInset, pieceScale * cellInset);\n    drawPiece(board[i]);\n    c.restore();\n  }\n\n  // Background Gradient.\n  const bgRadius = (Math.min(rows, columns) * cellSize) / 2;\n  const bgStyle = c.createRadialGradient(\n    xOffset + (cellSize * columns) / 2,\n    yOffset + (cellSize * rows) / 2,\n    0,\n    xOffset + (cellSize * columns) / 2,\n    yOffset + (cellSize * rows) / 2,\n    bgRadius\n  );\n  bgStyle.addColorStop(0, &quot;#000B49&quot;);\n  bgStyle.addColorStop(1, &quot;#000B2A&quot;);\n\n  // Render the board overlay.\n  c.beginPath();\n  c.rect(0, 0, canvas.width, canvas.height);\n  c.closePath();\n  c.shadowBlur = 0;\n  for (let i = 0; i < board.length; i++) {\n    drawCellCircle(i);\n    c.closePath();\n  }\n  c.fillStyle = bgStyle;\n  c.fill(&quot;evenodd&quot;);\n\n  // Render the board overlay cell outlines.\n  for (let i = 0; i < board.length; i++) {\n    c.beginPath();\n    drawCellCircle(i);\n    c.strokeStyle = &quot;#0361B2&quot;;\n    c.lineWidth = 1;\n    c.stroke();\n    c.closePath();\n  }\n\n  const drawLine = (fromCell, toCell) => {\n    if (frame < 0.5) return;\n    const lineFrame = (frame - 0.5) / 0.5;\n    const x1 = xOffset + (fromCell % columns) * cellSize + cellSize / 2;\n    const x2 =\n      x1 +\n      lineFrame *\n        (xOffset + ((toCell % columns) * cellSize + cellSize / 2) - x1);\n    const y1 =\n      yOffset + Math.floor(fromCell / columns) * cellSize + cellSize / 2;\n    const y2 =\n      y1 +\n      lineFrame *\n        (yOffset + Math.floor(toCell / columns) * cellSize + cellSize / 2 - y1);\n    c.beginPath();\n    c.lineCap = &quot;round&quot;;\n    c.lineWidth = 4;\n    c.strokeStyle = getColor(board[fromCell]);\n    c.shadowBlur = 8;\n    c.shadowColor = getColor(board[fromCell]);\n    c.moveTo(x1, y1);\n    c.lineTo(x2, y2);\n    c.stroke();\n  };\n\n  // Generate a graph of the board.\n  const getCell = (cell, rowOffset, columnOffset) => {\n    const row = Math.floor(cell / columns) + rowOffset;\n    const col = (cell % columns) + columnOffset;\n    if (row < 0 || row >= rows || col < 0 || col >= columns) return -1;\n    return col + row * columns;\n  };\n  const makeNode = cell => {\n    const node = { cell, directions: [], value: board[cell] };\n    for (let r = -1; r <= 1; r++) {\n      for (let c = -1; c <= 1; c++) {\n        if (r === 0 && c === 0) continue;\n        node.directions.push(getCell(cell, r, c));\n      }\n    }\n    return node;\n  };\n  const graph = board.map((_, i) => makeNode(i));\n\n  // Check for any wins!\n  const getSequence = (node, direction) => {\n    const sequence = [node.cell];\n    while (sequence.length < inarow) {\n      const next = graph[node.directions[direction]];\n      if (!next || node.value !== next.value || next.value === 0) return;\n      node = next;\n      sequence.push(node.cell);\n    }\n    return sequence;\n  };\n\n  // Check all nodes.\n  for (let i = 0; i < board.length; i++) {\n    // Check all directions (not the most efficient).\n    for (let d = 0; d < 8; d++) {\n      const seq = getSequence(graph[i], d);\n      if (seq) {\n        drawLine(seq[0], seq[inarow - 1]);\n        i = board.length;\n        break;\n      }\n    }\n  }\n\n  // Upgrade the legend.\n  if (agents.length && (!agents[0].color || !agents[0].image)) {\n    const getPieceImage = mark => {\n      const pieceCanvas = document.createElement(&quot;canvas&quot;);\n      parent.appendChild(pieceCanvas);\n      pieceCanvas.style.marginLeft = &quot;10000px&quot;;\n      pieceCanvas.width = 100;\n      pieceCanvas.height = 100;\n      c = pieceCanvas.getContext(&quot;2d&quot;);\n      c.translate(10, 10);\n      c.scale(0.8, 0.8);\n      drawPiece(mark);\n      const dataUrl = pieceCanvas.toDataURL();\n      parent.removeChild(pieceCanvas);\n      return dataUrl;\n    };\n\n    agents.forEach(agent => {\n      agent.color = getColor(agent.index + 1);\n      agent.image = getPieceImage(agent.index + 1);\n    });\n    update({ agents });\n  }\n};\n\n\n    \n    </script>\n    <script>\n      const h = htm.bind(preact.h);\n      const { useContext, useEffect, useRef, useState } = preactHooks;\n      const styled = window.styled.default;\n\n      const Context = preact.createContext({});\n\n      const Loading = styled.div`\n        animation: rotate360 1.1s infinite linear;\n        border: 8px solid rgba(255, 255, 255, 0.2);\n        border-left-color: #0cb1ed;\n        border-radius: 50%;\n        height: 40px;\n        position: relative;\n        transform: translateZ(0);\n        width: 40px;\n\n        @keyframes rotate360 {\n          0% {\n            transform: rotate(0deg);\n          }\n          100% {\n            transform: rotate(360deg);\n          }\n        }\n      `;\n\n      const Logo = styled(\n        (props) => h`\n        <a href=&quot;https://kaggle.com&quot; target=&quot;_blank&quot; className=${props.className}>\n          <svg width=&quot;62px&quot; height=&quot;20px&quot; viewBox=&quot;0 0 62 24&quot; version=&quot;1.1&quot; xmlns=&quot;http://www.w3.org/2000/svg&quot;>\n            <g fill=&quot;#1EBEFF&quot; fill-rule=&quot;nonzero&quot;>\n              <path d=&quot;M10.2,17.8c0,0.1-0.1,0.1-0.2,0.1H7.7c-0.1,0-0.3-0.1-0.4-0.2l-3.8-4.9l-1.1,1v3.8 c0,0.2-0.1,0.3-0.3,0.3H0.3c-0.2,0-0.3-0.1-0.3-0.3V0.3C0.1,0.1,0.2,0,0.3,0h1.8c0.2,0,0.3,0.1,0.3,0.3V11L7,6.3 c0.1-0.1,0.2-0.2,0.4-0.2h2.4c0.1,0,0.2,0,0.2,0.1c0,0.1,0,0.2,0,0.2l-4.9,4.7l5.1,6.3C10.2,17.6,10.2,17.7,10.2,17.8z&quot;/>\n              <path d=&quot;M19.6,17.9h-1.8c-0.2,0-0.3-0.1-0.3-0.3v-0.4c-0.8,0.6-1.8,0.9-3,0.9c-1.1,0-2-0.3-2.8-1 c-0.8-0.7-1.2-1.6-1.2-2.7c0-1.7,1.1-2.9,3.2-3.5c0.8-0.2,2.1-0.5,3.8-0.6c0.1-0.6-0.1-1.2-0.5-1.7c-0.4-0.5-1-0.7-1.7-0.7 c-1,0-2,0.4-3,1C12.2,9.1,12.1,9.1,12,9l-0.9-1.3C11,7.5,11,7.4,11.1,7.3c1.3-0.9,2.7-1.4,4.2-1.4c1.1,0,2.1,0.3,2.8,0.8 c1.1,0.8,1.7,2,1.7,3.7v7.3C19.9,17.8,19.8,17.9,19.6,17.9z M17.5,12.4c-1.7,0.2-2.9,0.4-3.5,0.7c-0.9,0.4-1.2,0.9-1.1,1.6 c0.1,0.4,0.2,0.7,0.6,0.9c0.3,0.2,0.7,0.4,1.1,0.4c1.2,0.1,2.2-0.2,2.9-1V12.4z&quot;/>\n              <path d=&quot;M30.6,22.5c-0.9,1-2.3,1.5-4,1.5c-1,0-2-0.3-2.9-0.8c-0.2-0.1-0.4-0.3-0.7-0.5 c-0.3-0.2-0.6-0.5-0.9-0.7c-0.1-0.1-0.1-0.2,0-0.4l1.2-1.2c0.1-0.1,0.1-0.1,0.2-0.1c0.1,0,0.1,0,0.2,0.1c1,1,1.9,1.5,2.8,1.5 c2.1,0,3.2-1.1,3.2-3.3v-1.4c-0.8,0.7-1.9,1-3.3,1c-1.7,0-3-0.6-4-1.9c-0.8-1.1-1.3-2.5-1.3-4.2c0-1.6,0.4-3,1.2-4.1 c0.9-1.3,2.3-2,4-2c1.3,0,2.4,0.3,3.3,1V6.4c0-0.2,0.1-0.3,0.3-0.3h1.8c0.2,0,0.3,0.1,0.3,0.3v11.7C32,20,31.5,21.5,30.6,22.5z M29.7,9.9c-0.4-1.1-1.4-1.7-3-1.7c-2,0-3.1,1.3-3.1,3.8c0,1.4,0.3,2.4,1,3.1c0.5,0.5,1.2,0.8,2,0.8c1.6,0,2.7-0.6,3.1-1.7V9.9z&quot;/>\n              <path d=&quot;M42.9,22.5c-0.9,1-2.3,1.5-4,1.5c-1,0-2-0.3-2.9-0.8c-0.2-0.1-0.4-0.3-0.7-0.5 c-0.3-0.2-0.6-0.5-0.9-0.7c-0.1-0.1-0.1-0.2,0-0.4l1.2-1.2c0.1-0.1,0.1-0.1,0.2-0.1c0.1,0,0.1,0,0.2,0.1c1,1,1.9,1.5,2.8,1.5 c2.1,0,3.2-1.1,3.2-3.3v-1.4c-0.8,0.7-1.9,1-3.3,1c-1.7,0-3-0.6-4-1.9c-0.8-1.1-1.3-2.5-1.3-4.2c0-1.6,0.4-3,1.2-4.1 c0.9-1.3,2.3-2,4-2c1.3,0,2.4,0.3,3.3,1V6.4c0-0.2,0.1-0.3,0.3-0.3H44c0.2,0,0.3,0.1,0.3,0.3v11.7C44.3,20,43.8,21.5,42.9,22.5z M42,9.9c-0.4-1.1-1.4-1.7-3-1.7c-2,0-3.1,1.3-3.1,3.8c0,1.4,0.3,2.4,1,3.1c0.5,0.5,1.2,0.8,2,0.8c1.6,0,2.7-0.6,3.1-1.7L42,9.9 L42,9.9z&quot;/>\n              <path d=&quot;M48.3,17.9h-1.8c-0.2,0-0.3-0.1-0.3-0.3V0.3c0-0.2,0.1-0.3,0.3-0.3h1.8c0.2,0,0.3,0.1,0.3,0.3 v17.3C48.5,17.8,48.5,17.9,48.3,17.9z&quot;/>\n              <path d=&quot;M61.4,12.6c0,0.2-0.1,0.3-0.3,0.3h-8.5c0.1,0.9,0.5,1.6,1.1,2.2c0.7,0.6,1.6,0.9,2.7,0.9 c1,0,1.8-0.3,2.6-0.8c0.2-0.1,0.3-0.1,0.4,0l1.2,1.3c0.1,0.1,0.1,0.3,0,0.4c-1.3,0.9-2.7,1.4-4.4,1.4c-1.8,0-3.3-0.6-4.4-1.8 c-1.1-1.2-1.7-2.7-1.7-4.5c0-1.7,0.6-3.2,1.7-4.4c1-1.1,2.4-1.6,4.1-1.6c1.6,0,2.9,0.6,4,1.7c1.1,1.2,1.6,2.6,1.5,4.4L61.4,12.6 z M58,8.7c-0.6-0.5-1.3-0.8-2.1-0.8c-0.8,0-1.5,0.3-2.1,0.8c-0.6,0.5-1,1.2-1.1,2H59C59,9.9,58.6,9.3,58,8.7z&quot;/>\n            </g>\n          </svg>\n        </a>\n      `\n      )`\n        display: inline-flex;\n      `;\n\n      const Header = styled((props) => {\n        const { environment } = useContext(Context);\n\n        return h`<div className=${props.className} >\n          <${Logo} />\n          <span><b>Left / Right Arrow:</b> Increase / Decrease Step</span><span><b>0-9 Row Keys:</b> Playback Speed</span><span><b>Space:</b> Pause / Play</span>\n          ${environment.title}\n        </div>`;\n      })`\n        align-items: center;\n        border-bottom: 4px solid #212121;\n        box-sizing: border-box;\n        color: #fff;\n        display: flex;\n        flex: 0 0 36px;\n        font-size: 14px;\n        justify-content: space-between;\n        padding: 0 8px;\n        width: 100%;\n      `;\n\n      const Renderer = styled((props) => {\n        const context = useContext(Context);\n        const { animate, debug, playing, renderer, speed } = context;\n        const ref = preact.createRef();\n\n        useEffect(async () => {\n          if (!ref.current) return;\n\n          const renderFrame = async (start, step, lastFrame) => {\n            if (step !== context.step) return;\n            if (lastFrame === 1) {\n              if (!animate) return;\n              start = Date.now();\n            }\n            const frame =\n              playing || animate\n                ? Math.min((Date.now() - start) / speed, 1)\n                : 1;\n            try {\n              if (debug) console.time(&quot;render&quot;);\n              await renderer({\n                ...context,\n                frame,\n                height: ref.current.clientHeight,\n                hooks: preactHooks,\n                parent: ref.current,\n                preact,\n                styled,\n                width: ref.current.clientWidth,\n              });\n            } catch (error) {\n              if (debug) console.error(error);\n              console.log({ ...context, frame, error });\n            } finally {\n              if (debug) console.timeEnd(&quot;render&quot;);\n            }\n            window.requestAnimationFrame(() => renderFrame(start, step, frame));\n          };\n\n          await renderFrame(Date.now(), context.step);\n        }, [ref.current, context.step, context.renderer]);\n\n        return h`<div className=${props.className} ref=${ref} />`;\n      })`\n        align-items: center;\n        box-sizing: border-box;\n        display: flex;\n        height: 100%;\n        left: 0;\n        justify-content: center;\n        position: absolute;\n        top: 0;\n        width: 100%;\n      `;\n\n      const Processing = styled((props) => {\n        const { processing } = useContext(Context);\n        const text = processing === true ? &quot;Processing...&quot; : processing;\n        return h`<div className=${props.className}>${text}</div>`;\n      })`\n        bottom: 0;\n        color: #fff;\n        font-size: 12px;\n        left: 0;\n        line-height: 24px;\n        position: absolute;\n        text-align: center;\n        width: 100%;\n      `;\n\n      const Viewer = styled((props) => {\n        const { processing } = useContext(Context);\n        return h`<div className=${props.className}>\n          <${Renderer} />\n          ${processing && h`<${Processing} />`}\n        </div>`;\n      })`\n        background-color: #000b2a;\n        background-image: radial-gradient(\n          circle closest-side,\n          #000b49,\n          #000b2a\n        );\n        display: flex;\n        flex: 1;\n        overflow: hidden;\n        position: relative;\n        width: 100%;\n      `;\n\n      // Partitions the elements of arr into subarrays of max length num.\n      const groupIntoSets = (arr, num) => {\n        const sets = [];\n        arr.forEach(a => {\n          if (sets.length === 0 || sets[sets.length - 1].length === num) {\n            sets.push([]);\n          }\n          sets[sets.length - 1].push(a);\n        });\n        return sets;\n      }\n\n      // Expects `width` input prop to set proper max-width for agent name span.\n      const Legend = styled((props) => {\n        const { agents, legend } = useContext(Context);\n\n        const agentPairs = groupIntoSets(agents.sort((a, b) => a.index - b.index), 2);\n\n        return h`<div className=${props.className}>\n          ${agentPairs.map(agentList =>\n            h`<ul>\n                ${agentList.map(a =>\n                  h`<li key=${a.id} title=&quot;id: ${a.id}&quot; style=&quot;color:${a.color || &quot;#FFF&quot;}&quot;>\n                      ${a.image && h`<img src=${a.image} />`}\n                      <span>${a.name}</span>\n                    </li>`\n                )}\n              </ul>`)}\n        </div>`;\n      })`\n        background-color: #000b2a;\n        font-family: sans-serif;\n        font-size: 14px;\n        width: 100%;\n\n        ul {\n          align-items: center;\n          display: flex;\n          flex-direction: row;\n          justify-content: center;\n        }\n\n        li {\n          align-items: center;\n          display: inline-flex;\n          transition: color 1s;\n        }\n\n        span {\n          max-width: ${p => (p.width || 400) * 0.5 - 36}px;\n          overflow: hidden;\n          text-overflow: ellipsis;\n          white-space: nowrap;\n        }\n\n        img {\n          height: 24px;\n          margin-left: 4px;\n          margin-right: 4px;\n          width: 24px;\n        }\n      `;\n\n      const StepInput = styled.input.attrs({\n        type: &quot;range&quot;,\n      })`\n        appearance: none;\n        background: rgba(255, 255, 255, 0.15);\n        border-radius: 2px;\n        display: block;\n        flex: 1;\n        height: 4px;\n        opacity: 0.8;\n        outline: none;\n        transition: opacity 0.2s;\n        width: 100%;\n\n        &:hover {\n          opacity: 1;\n        }\n\n        &::-webkit-slider-thumb {\n          appearance: none;\n          background: #1ebeff;\n          border-radius: 100%;\n          cursor: pointer;\n          height: 12px;\n          margin: 0;\n          position: relative;\n          width: 12px;\n\n          &::after {\n            content: &quot;&quot;;\n            position: absolute;\n            top: 0px;\n            left: 0px;\n            width: 200px;\n            height: 8px;\n            background: green;\n          }\n        }\n      `;\n\n      const PlayButton = styled.button`\n        align-items: center;\n        background: none;\n        border: none;\n        color: white;\n        cursor: pointer;\n        display: flex;\n        flex: 0 0 56px;\n        font-size: 20px;\n        height: 40px;\n        justify-content: center;\n        opacity: 0.8;\n        outline: none;\n        transition: opacity 0.2s;\n\n        &:hover {\n          opacity: 1;\n        }\n      `;\n\n      const StepCount = styled.span`\n        align-items: center;\n        color: white;\n        display: flex;\n        font-size: 14px;\n        justify-content: center;\n        opacity: 0.8;\n        padding: 0 16px;\n        pointer-events: none;\n      `;\n\n      const Controls = styled((props) => {\n        const { environment, pause, play, playing, setStep, step } = useContext(\n          Context\n        );\n        const value = step + 1;\n        const onClick = () => (playing ? pause() : play());\n        const onInput = (e) => {\n          pause();\n          setStep(parseInt(e.target.value) - 1);\n        };\n\n        return h`\n          <div className=${props.className}>\n            <${PlayButton} onClick=${onClick}><svg xmlns=&quot;http://www.w3.org/2000/svg&quot; width=&quot;24px&quot; height=&quot;24px&quot; viewBox=&quot;0 0 24 24&quot; fill=&quot;#FFFFFF&quot;>${\n          playing\n            ? h`<path d=&quot;M6 19h4V5H6v14zm8-14v14h4V5h-4z&quot;/><path d=&quot;M0 0h24v24H0z&quot; fill=&quot;none&quot;/>`\n            : h`<path d=&quot;M8 5v14l11-7z&quot;/><path d=&quot;M0 0h24v24H0z&quot; fill=&quot;none&quot;/>`\n        }</svg><//>\n            <${StepInput} min=&quot;1&quot; max=${\n          environment.steps.length\n        } value=&quot;${value}&quot; onInput=${onInput} />\n            <${StepCount}>${value} / ${environment.steps.length}<//>\n          </div>\n        `;\n      })`\n        align-items: center;\n        border-top: 4px solid #212121;\n        display: flex;\n        flex: 0 0 44px;\n        width: 100%;\n      `;\n\n      const Info = styled((props) => {\n        const {\n          environment,\n          playing,\n          step,\n          speed,\n          animate,\n          header,\n          controls,\n          settings,\n        } = useContext(Context);\n\n        return h`\n          <div className=${props.className}>\n            info:\n            step(${step}),\n            playing(${playing ? &quot;T&quot; : &quot;F&quot;}),\n            speed(${speed}),\n            animate(${animate ? &quot;T&quot; : &quot;F&quot;})\n          </div>`;\n      })`\n        color: #888;\n        font-family: monospace;\n        font-size: 12px;\n      `;\n\n      const Settings = styled((props) => {\n        const { environment, pause, play, playing, setStep, step } = useContext(\n          Context\n        );\n\n        return h`\n          <div className=${props.className}>\n            <${Info} />\n          </div>\n        `;\n      })`\n        background: #fff;\n        border-top: 4px solid #212121;\n        box-sizing: border-box;\n        padding: 20px;\n        width: 100%;\n\n        h1 {\n          font-size: 20px;\n        }\n      `;\n\n      const Player = styled((props) => {\n        const context = useContext(Context);\n        const { agents, controls, header, legend, loading, settings, width } = context;\n        return h`\n          <div className=${props.className}>\n            ${loading && h`<${Loading} />`}\n            ${!loading && header && h`<${Header} />`}\n            ${!loading && h`<${Viewer} />`}\n            ${!loading && agents.length > 0 && legend && h`<${Legend} width=${width}/>`}\n            ${!loading && controls && h`<${Controls} />`}\n            ${!loading && settings && h`<${Settings} />`}\n          </div>`;\n      })`\n        align-items: center;\n        background: #212121;\n        border: 4px solid #212121;\n        box-sizing: border-box;\n        display: flex;\n        flex-direction: column;\n        height: 100%;\n        justify-content: center;\n        position: relative;\n        width: 100%;\n      `;\n\n      const App = () => {\n        const renderCountRef = useRef(0);\n        const [_, setRenderCount] = useState(0);\n\n        const contextRef = useRef({\n          animate: false,\n          agents: [],\n          autoplay: false,\n          controls: false,\n          debug: false,\n          environment: { steps: [] },\n          header: window.innerHeight >= 600,\n          height: window.innerHeight,\n          interactive: false,\n          legend: true,\n          loading: false,\n          playing: false,\n          processing: false,\n          renderer: () => &quot;DNE&quot;,\n          settings: false,\n          speed: 4,\n          step: 0,\n          width: window.innerWidth,\n        });\n\n        // Context helpers.\n        const rerender = (contextRef.current.rerender = () =>\n          setRenderCount((renderCountRef.current += 1)));\n        const setStep = (contextRef.current.setStep = (newStep) => {\n          contextRef.current.step = newStep;\n          rerender();\n        });\n        const setPlaying = (contextRef.current.setPlaying = (playing) => {\n          contextRef.current.playing = playing;\n          rerender();\n        });\n        const pause = (contextRef.current.pause = () => setPlaying(false));\n\n        const playNext = () => {\n          const context = contextRef.current;\n\n          if (\n            context.playing &&\n            context.step < context.environment.steps.length - 1\n          ) {\n            setStep(context.step + 1);\n            play(true);\n          } else {\n            pause();\n          }\n        };\n\n        // These are bindings to the 0-9 keys and are milliseconds of timeout per step\n        const speeds = [\n          0,\n          3000,\n          1000,\n          500,\n          333, // Default\n          200,\n          100,\n          50,\n          25,\n          10,\n        ];\n\n        const play = (contextRef.current.play = (continuing) => {\n          const context = contextRef.current;\n          if (context.playing && !continuing) return;\n          if (!context.playing) setPlaying(true);\n          if (\n            !continuing &&\n            context.step === context.environment.steps.length - 1\n          ) {\n            setStep(0);\n          }\n          setTimeout(playNext, speeds[context.speed]);\n        });\n\n        const updateContext = (o) => {\n          const context = contextRef.current;\n          Object.assign(context, o, {\n            environment: { ...context.environment, ...(o.environment || {}) },\n          });\n          rerender();\n\n          // If autoplay, toggle to playing.\n          if (context.autoplay) play();\n        };\n\n        // First time setup.\n        useEffect(() => {\n          // Timeout is used to ensure useEffect renders once.\n          setTimeout(() => {\n            // Initialize context with window.kaggle.\n            updateContext(window.kaggle || {});\n            // Listen for messages received to update the context.\n            window.addEventListener(\n              &quot;message&quot;,\n              (event) => {\n                // Ensure the environment names match before updating.\n                try {\n                  if (\n                    event.data.environment.name ==\n                    contextRef.current.environment.name\n                  ) {\n                    updateContext(event.data);\n                  }\n                } catch {}\n              },\n              false\n            );\n            // Listen for keyboard commands.\n            window.addEventListener(\n              &quot;keydown&quot;,\n              (event) => {\n                const {\n                  interactive,\n                  isInteractive,\n                  playing,\n                  step,\n                  environment,\n                } = contextRef.current;\n                const key = event.keyCode;\n                const zero_key = 48\n                const nine_key = 57\n                if (\n                  interactive ||\n                  isInteractive() ||\n                  (key !== 32 && key !== 37 && key !== 39 && !(key >= zero_key && key <= nine_key))\n                )\n                  return;\n\n                if (key === 32) {\n                  playing ? pause() : play();\n                } else if (key === 39) {\n                  contextRef.current.playing = false;\n                  if (step < environment.steps.length - 1) setStep(step + 1);\n                  rerender();\n                } else if (key === 37) {\n                  contextRef.current.playing = false;\n                  if (step > 0) setStep(step - 1);\n                  rerender();\n                } else if (key >= zero_key && key <= nine_key) {\n                  contextRef.current.speed = key - zero_key;\n                }\n                event.preventDefault();\n                return false;\n              },\n              false\n            );\n          }, 1);\n        }, []);\n\n        if (contextRef.current.debug) {\n          console.log(&quot;context&quot;, contextRef.current);\n        }\n\n        // Ability to update context.\n        contextRef.current.update = updateContext;\n\n        // Ability to communicate with ipython.\n        const execute = (contextRef.current.execute = (source) =>\n          new Promise((resolve, reject) => {\n            try {\n              window.parent.IPython.notebook.kernel.execute(source, {\n                iopub: {\n                  output: (resp) => {\n                    const type = resp.msg_type;\n                    if (type === &quot;stream&quot;) return resolve(resp.content.text);\n                    if (type === &quot;error&quot;) return reject(new Error(resp.evalue));\n                    return reject(new Error(&quot;Unknown message type: &quot; + type));\n                  },\n                },\n              });\n            } catch (e) {\n              reject(new Error(&quot;IPython Unavailable: &quot; + e));\n            }\n          }));\n\n        // Ability to return an action from an interactive session.\n        contextRef.current.act = (action) => {\n          const id = contextRef.current.environment.id;\n          updateContext({ processing: true });\n          execute(`\n            import json\n            from kaggle_environments import interactives\n            if &quot;${id}&quot; in interactives:\n                action = json.loads('${JSON.stringify(action)}')\n                env, trainer = interactives[&quot;${id}&quot;]\n                trainer.step(action)\n                print(json.dumps(env.steps))`)\n            .then((resp) => {\n              try {\n                updateContext({\n                  processing: false,\n                  environment: { steps: JSON.parse(resp) },\n                });\n                play();\n              } catch (e) {\n                updateContext({ processing: resp.split(&quot;\\n&quot;)[0] });\n                console.error(resp, e);\n              }\n            })\n            .catch((e) => console.error(e));\n        };\n\n        // Check if currently interactive.\n        contextRef.current.isInteractive = () => {\n          const context = contextRef.current;\n          const steps = context.environment.steps;\n          return (\n            context.interactive &&\n            !context.processing &&\n            context.step === steps.length - 1 &&\n            steps[context.step].some((s) => s.status === &quot;ACTIVE&quot;)\n          );\n        };\n\n        return h`\n          <${Context.Provider} value=${contextRef.current}>\n            <${Player} />\n          <//>`;\n      };\n\n      preact.render(h`<${App} />`, document.body);\n    </script>\n  </body>\n</html>\n\" width=\"500\" height=\"450\" frameborder=\"0\"></iframe> "
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "player_1 = 'Negamax'\n",
    "player_2 = 'MontyCarloHeuristic'\n",
    "\n",
    "env = make(\"connectx\", debug=True)\n",
    "env.configuration.timeout = 6\n",
    "\n",
    "print(f'{player_1} vs {player_2}')\n",
    "env.run([ agents[player_1], agents[player_2] ])\n",
    "\n",
    "print('-'*20 + f'\\n{player_1} vs {player_2}')\n",
    "env.render(mode=\"ipython\", width=500, height=450)\n",
    "print(f\"scores: {player_1} = {env.state[0].reward} | {player_2} = {env.state[1].reward}\")"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Negamax vs MontyCarloHeuristic\n",
      "MontyCarloHeuristicNode: p1 action = 3 after 3586 simulations in 4.001s\n",
      "\n",
      "Negamax                 | depth: 1 2 3 4  = 1 (-3.20) | 7841 iterations in 3.81s \n",
      "MontyCarloHeuristicNode: p1 action = 3 after 3074 simulations in 4.000s\n",
      "\n",
      "Negamax                 | depth: 1 2 3 4  = 3 (-3.30) | 6318 iterations in 3.81s \n",
      "MontyCarloHeuristicNode: p1 action = 2 after 2909 simulations in 4.000s\n",
      "\n",
      "Negamax                 | depth: 1 2 3 4  = 1 (-5.00) | 3224 iterations in 3.14s \n",
      "MontyCarloHeuristicNode: p1 action = 4 after 2245 simulations in 4.000s\n",
      "\n",
      "Negamax                 | depth: 1 2  = 5 (-4.00) | 34 iterations in 0.03s \n",
      "MontyCarloHeuristicNode: p1 action = 4 after 1531 simulations in 4.002s\n",
      "\n",
      "Negamax                 | depth: 1 2 3  = 4 (9.40) | 2060 iterations in 3.53s \n",
      "MontyCarloHeuristicNode: p1 action = 4 after 668 simulations in 4.003s\n",
      "\n",
      "Negamax                 | depth: 1 2 3  = 5 (9.30) | 2164 iterations in 4.60s \n",
      "MontyCarloHeuristicNode: p1 action = 6 after 841 simulations in 4.001s\n",
      "\n",
      "Negamax                 | depth: 1 2 3  = 1 (7.90) | 1737 iterations in 5.11s \n",
      "MontyCarloHeuristicNode: p1 action = 1 after 1765 simulations in 4.002s\n",
      "\n",
      "Negamax                 | depth: 1 2 3 4  = 3 (1.30) | 2794 iterations in 5.55s \n",
      "MontyCarloHeuristicNode: p1 action = 4 after 1144 simulations in 4.001s\n",
      "\n",
      "Negamax                 | depth: 1 2 3 4  = 2 (1.20) | 2327 iterations in 5.06s \n",
      "MontyCarloHeuristicNode: p1 action = 2 after 531 simulations in 4.002s\n",
      "\n",
      "Negamax                 | depth: 1 2 3 4  = 1 (-0.10) | 1538 iterations in 4.01s \n",
      "MontyCarloHeuristicNode: p1 action = 1 after 540 simulations in 4.001s\n",
      "\n",
      "Negamax                 | depth: 1 2  = 2 (1.40) | 27 iterations in 0.11s \n",
      "MontyCarloHeuristicNode: p1 action = 2 after 609 simulations in 4.003s\n",
      "\n",
      "Negamax                 | depth: 1 2 3 4  = 0 (-0.20) | 1769 iterations in 4.03s \n",
      "MontyCarloHeuristicNode: p1 action = 0 after 723 simulations in 4.001s\n",
      "\n",
      "Negamax                 | depth: 1 2  = 0 (-0.20) | 30 iterations in 0.08s \n",
      "MontyCarloHeuristicNode: p1 action = 0 after 1052 simulations in 4.001s\n",
      "\n",
      "Negamax                 | depth: 1 2 3 4  = 3 (-1.00) | 4067 iterations in 3.12s \n",
      "MontyCarloHeuristicNode: p1 action = 5 after 1998 simulations in 4.002s\n",
      "\n",
      "Negamax                 | depth: 1 2 3 4 5  = 2 (0.10) | 6216 iterations in 4.02s \n",
      "MontyCarloHeuristicNode: p1 action = 1 after 1759 simulations in 4.001s\n",
      "\n",
      "Negamax                 | depth: 1 2 3 4 5 6  = 6 (-1.10) | 5107 iterations in 3.60s \n",
      "MontyCarloHeuristicNode: p1 action = 4 after 1000 simulations in 4.003s\n",
      "\n",
      "Negamax                 | depth: 1 2 3 4 5 6 7 8 9 10 11 12 13 14  = 5 (-0.90) | 4633 iterations in 4.00s \n",
      "MontyCarloHeuristicNode: p1 action = 6 after 2866 simulations in 4.001s\n",
      "\n",
      "Negamax                 | depth: 1 2  = 6 (-1.00) | 11 iterations in 0.02s \n",
      "MontyCarloHeuristicNode: p1 action = 6 after 3266 simulations in 4.000s\n",
      "\n",
      "Negamax                 | depth: 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49  = 0 (0.00) | 2439 iterations in 4.02s \n",
      "MontyCarloHeuristicNode: p1 action = 5 after 3060 simulations in 4.000s\n",
      "\n",
      "Negamax                 | depth: 1 2  = 6 (0.00) | 11 iterations in 0.01s \n",
      "MontyCarloHeuristicNode: p1 action = 0 after 3561 simulations in 4.000s\n",
      "\n",
      "Negamax                 | depth: 1  = 5 (0.00) | 0 iterations in 0.00s \n",
      "--------------------\n",
      "Negamax vs MontyCarloHeuristic\n",
      "scores: Negamax = 0 | MontyCarloHeuristic = 0\n"
     ]
    },
    {
     "data": {
      "text/plain": "<IPython.core.display.HTML object>",
      "text/html": "<iframe srcdoc=\"<!--\n  Copyright 2020 Kaggle Inc\n\n  Licensed under the Apache License, Version 2.0 (the &quot;License&quot;);\n  you may not use this file except in compliance with the License.\n  You may obtain a copy of the License at\n\n      http://www.apache.org/licenses/LICENSE-2.0\n\n  Unless required by applicable law or agreed to in writing, software\n  distributed under the License is distributed on an &quot;AS IS&quot; BASIS,\n  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n  See the License for the specific language governing permissions and\n  limitations under the License.\n-->\n<!DOCTYPE html>\n<html lang=&quot;en&quot;>\n  <head>\n    <title>Kaggle Simulation Player</title>\n    <meta name=&quot;viewport&quot; content=&quot;width=device-width,initial-scale=1&quot; />\n    <link\n      rel=&quot;stylesheet&quot;\n      href=&quot;https://cdnjs.cloudflare.com/ajax/libs/meyer-reset/2.0/reset.css&quot;\n      crossorigin=&quot;anonymous&quot;\n    />\n    <style type=&quot;text/css&quot;>\n      html,\n      body {\n        height: 100%;\n        font-family: sans-serif;\n      }\n      canvas {\n        /* image-rendering: -moz-crisp-edges;\n        image-rendering: -webkit-crisp-edges;\n        image-rendering: pixelated;\n        image-rendering: crisp-edges; */\n      }\n    </style>\n    <script src=&quot;https://unpkg.com/preact@10.0.1/dist/preact.umd.js&quot;></script>\n    <script src=&quot;https://unpkg.com/preact@10.0.1/hooks/dist/hooks.umd.js&quot;></script>\n    <script src=&quot;https://unpkg.com/htm@2.2.1/dist/htm.umd.js&quot;></script>\n    <script>\n      // Polyfill for Styled Components\n      window.React = {\n        ...preact,\n        createElement: preact.h,\n        PropTypes: { func: {} },\n      };\n    </script>\n    <script src=&quot;https://unpkg.com/styled-components@3.5.0-0/dist/styled-components.min.js&quot;></script>\n  </head>\n  <body>\n    <script>\n      \nwindow.kaggle = {\n  &quot;debug&quot;: true,\n  &quot;autoplay&quot;: true,\n  &quot;step&quot;: 0,\n  &quot;controls&quot;: true,\n  &quot;environment&quot;: {\n    &quot;id&quot;: &quot;fe1d11e2-d655-11ea-bd55-3c07545773e3&quot;,\n    &quot;name&quot;: &quot;connectx&quot;,\n    &quot;title&quot;: &quot;ConnectX&quot;,\n    &quot;description&quot;: &quot;Classic Connect in a row but configurable.&quot;,\n    &quot;version&quot;: &quot;1.0.1&quot;,\n    &quot;configuration&quot;: {\n      &quot;episodeSteps&quot;: 1000,\n      &quot;agentExec&quot;: &quot;LOCAL&quot;,\n      &quot;agentTimeout&quot;: 16,\n      &quot;actTimeout&quot;: 8,\n      &quot;runTimeout&quot;: 1200,\n      &quot;columns&quot;: 7,\n      &quot;rows&quot;: 6,\n      &quot;inarow&quot;: 4,\n      &quot;timeout&quot;: 6\n    },\n    &quot;specification&quot;: {\n      &quot;action&quot;: {\n        &quot;description&quot;: &quot;Column to drop a checker onto the board.&quot;,\n        &quot;type&quot;: &quot;integer&quot;,\n        &quot;minimum&quot;: 0,\n        &quot;default&quot;: 0\n      },\n      &quot;agents&quot;: [\n        2\n      ],\n      &quot;configuration&quot;: {\n        &quot;episodeSteps&quot;: {\n          &quot;description&quot;: &quot;Maximum number of steps in the episode.&quot;,\n          &quot;type&quot;: &quot;integer&quot;,\n          &quot;minimum&quot;: 1,\n          &quot;default&quot;: 1000\n        },\n        &quot;agentExec&quot;: {\n          &quot;description&quot;: &quot;How the agent is executed alongside the running envionment.&quot;,\n          &quot;type&quot;: &quot;string&quot;,\n          &quot;default&quot;: &quot;LOCAL&quot;,\n          &quot;enum&quot;: [\n            &quot;LOCAL&quot;,\n            &quot;PROCESS&quot;\n          ]\n        },\n        &quot;agentTimeout&quot;: {\n          &quot;description&quot;: &quot;Maximum runtime (seconds) to initialize an agent.&quot;,\n          &quot;type&quot;: &quot;integer&quot;,\n          &quot;minimum&quot;: 1,\n          &quot;default&quot;: 16\n        },\n        &quot;actTimeout&quot;: {\n          &quot;description&quot;: &quot;Maximum runtime (seconds) to obtain an action from an agent.&quot;,\n          &quot;type&quot;: &quot;integer&quot;,\n          &quot;minimum&quot;: 1,\n          &quot;default&quot;: 8\n        },\n        &quot;runTimeout&quot;: {\n          &quot;description&quot;: &quot;Maximum runtime (seconds) of an episode (not necessarily DONE).&quot;,\n          &quot;type&quot;: &quot;integer&quot;,\n          &quot;minimum&quot;: 1,\n          &quot;default&quot;: 1200\n        },\n        &quot;columns&quot;: {\n          &quot;description&quot;: &quot;The number of columns on the board&quot;,\n          &quot;type&quot;: &quot;integer&quot;,\n          &quot;default&quot;: 7,\n          &quot;minimum&quot;: 1\n        },\n        &quot;rows&quot;: {\n          &quot;description&quot;: &quot;The number of rows on the board&quot;,\n          &quot;type&quot;: &quot;integer&quot;,\n          &quot;default&quot;: 6,\n          &quot;minimum&quot;: 1\n        },\n        &quot;inarow&quot;: {\n          &quot;description&quot;: &quot;The number of checkers in a row required to win.&quot;,\n          &quot;type&quot;: &quot;integer&quot;,\n          &quot;default&quot;: 4,\n          &quot;minimum&quot;: 1\n        },\n        &quot;timeout&quot;: {\n          &quot;description&quot;: &quot;Obsolete copy of actTimeout maintained for backwards compatibility. May be removed in the future.&quot;,\n          &quot;type&quot;: &quot;integer&quot;,\n          &quot;default&quot;: 8,\n          &quot;minimum&quot;: 0\n        }\n      },\n      &quot;info&quot;: {},\n      &quot;observation&quot;: {\n        &quot;board&quot;: {\n          &quot;description&quot;: &quot;Serialized grid (rows x columns). 0 = Empty, 1 = P1, 2 = P2&quot;,\n          &quot;type&quot;: &quot;array&quot;,\n          &quot;shared&quot;: true,\n          &quot;default&quot;: []\n        },\n        &quot;mark&quot;: {\n          &quot;defaults&quot;: [\n            1,\n            2\n          ],\n          &quot;description&quot;: &quot;Which checkers are the agents.&quot;,\n          &quot;enum&quot;: [\n            1,\n            2\n          ]\n        }\n      },\n      &quot;reward&quot;: {\n        &quot;description&quot;: &quot;-1 = Lost, 0 = Draw/Ongoing, 1 = Won&quot;,\n        &quot;enum&quot;: [\n          -1,\n          0,\n          1\n        ],\n        &quot;default&quot;: 0,\n        &quot;type&quot;: [\n          &quot;number&quot;,\n          &quot;null&quot;\n        ]\n      }\n    },\n    &quot;steps&quot;: [\n      [\n        {\n          &quot;action&quot;: 0,\n          &quot;reward&quot;: 0,\n          &quot;info&quot;: {},\n          &quot;observation&quot;: {\n            &quot;board&quot;: [\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0\n            ],\n            &quot;mark&quot;: 1\n          },\n          &quot;status&quot;: &quot;ACTIVE&quot;\n        },\n        {\n          &quot;action&quot;: 0,\n          &quot;reward&quot;: 0,\n          &quot;info&quot;: {},\n          &quot;observation&quot;: {\n            &quot;mark&quot;: 2\n          },\n          &quot;status&quot;: &quot;INACTIVE&quot;\n        }\n      ],\n      [\n        {\n          &quot;action&quot;: 3,\n          &quot;reward&quot;: 0,\n          &quot;info&quot;: {},\n          &quot;observation&quot;: {\n            &quot;board&quot;: [\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              1,\n              0,\n              0,\n              0\n            ],\n            &quot;mark&quot;: 1\n          },\n          &quot;status&quot;: &quot;INACTIVE&quot;\n        },\n        {\n          &quot;action&quot;: 0,\n          &quot;reward&quot;: 0,\n          &quot;info&quot;: {},\n          &quot;observation&quot;: {\n            &quot;mark&quot;: 2\n          },\n          &quot;status&quot;: &quot;ACTIVE&quot;\n        }\n      ],\n      [\n        {\n          &quot;action&quot;: 0,\n          &quot;reward&quot;: 0,\n          &quot;info&quot;: {},\n          &quot;observation&quot;: {\n            &quot;board&quot;: [\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              2,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              1,\n              0,\n              0,\n              0\n            ],\n            &quot;mark&quot;: 1\n          },\n          &quot;status&quot;: &quot;ACTIVE&quot;\n        },\n        {\n          &quot;action&quot;: 3,\n          &quot;reward&quot;: 0,\n          &quot;info&quot;: {},\n          &quot;observation&quot;: {\n            &quot;mark&quot;: 2\n          },\n          &quot;status&quot;: &quot;INACTIVE&quot;\n        }\n      ],\n      [\n        {\n          &quot;action&quot;: 3,\n          &quot;reward&quot;: 0,\n          &quot;info&quot;: {},\n          &quot;observation&quot;: {\n            &quot;board&quot;: [\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              1,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              2,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              1,\n              0,\n              0,\n              0\n            ],\n            &quot;mark&quot;: 1\n          },\n          &quot;status&quot;: &quot;INACTIVE&quot;\n        },\n        {\n          &quot;action&quot;: 0,\n          &quot;reward&quot;: 0,\n          &quot;info&quot;: {},\n          &quot;observation&quot;: {\n            &quot;mark&quot;: 2\n          },\n          &quot;status&quot;: &quot;ACTIVE&quot;\n        }\n      ],\n      [\n        {\n          &quot;action&quot;: 0,\n          &quot;reward&quot;: 0,\n          &quot;info&quot;: {},\n          &quot;observation&quot;: {\n            &quot;board&quot;: [\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              2,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              1,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              2,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              1,\n              0,\n              0,\n              0\n            ],\n            &quot;mark&quot;: 1\n          },\n          &quot;status&quot;: &quot;ACTIVE&quot;\n        },\n        {\n          &quot;action&quot;: 3,\n          &quot;reward&quot;: 0,\n          &quot;info&quot;: {},\n          &quot;observation&quot;: {\n            &quot;mark&quot;: 2\n          },\n          &quot;status&quot;: &quot;INACTIVE&quot;\n        }\n      ],\n      [\n        {\n          &quot;action&quot;: 2,\n          &quot;reward&quot;: 0,\n          &quot;info&quot;: {},\n          &quot;observation&quot;: {\n            &quot;board&quot;: [\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              2,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              1,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              2,\n              0,\n              0,\n              0,\n              0,\n              0,\n              1,\n              1,\n              0,\n              0,\n              0\n            ],\n            &quot;mark&quot;: 1\n          },\n          &quot;status&quot;: &quot;INACTIVE&quot;\n        },\n        {\n          &quot;action&quot;: 0,\n          &quot;reward&quot;: 0,\n          &quot;info&quot;: {},\n          &quot;observation&quot;: {\n            &quot;mark&quot;: 2\n          },\n          &quot;status&quot;: &quot;ACTIVE&quot;\n        }\n      ],\n      [\n        {\n          &quot;action&quot;: 0,\n          &quot;reward&quot;: 0,\n          &quot;info&quot;: {},\n          &quot;observation&quot;: {\n            &quot;board&quot;: [\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              2,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              1,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              2,\n              0,\n              0,\n              0,\n              0,\n              2,\n              1,\n              1,\n              0,\n              0,\n              0\n            ],\n            &quot;mark&quot;: 1\n          },\n          &quot;status&quot;: &quot;ACTIVE&quot;\n        },\n        {\n          &quot;action&quot;: 1,\n          &quot;reward&quot;: 0,\n          &quot;info&quot;: {},\n          &quot;observation&quot;: {\n            &quot;mark&quot;: 2\n          },\n          &quot;status&quot;: &quot;INACTIVE&quot;\n        }\n      ],\n      [\n        {\n          &quot;action&quot;: 4,\n          &quot;reward&quot;: 0,\n          &quot;info&quot;: {},\n          &quot;observation&quot;: {\n            &quot;board&quot;: [\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              2,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              1,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              2,\n              0,\n              0,\n              0,\n              0,\n              2,\n              1,\n              1,\n              1,\n              0,\n              0\n            ],\n            &quot;mark&quot;: 1\n          },\n          &quot;status&quot;: &quot;INACTIVE&quot;\n        },\n        {\n          &quot;action&quot;: 0,\n          &quot;reward&quot;: 0,\n          &quot;info&quot;: {},\n          &quot;observation&quot;: {\n            &quot;mark&quot;: 2\n          },\n          &quot;status&quot;: &quot;ACTIVE&quot;\n        }\n      ],\n      [\n        {\n          &quot;action&quot;: 0,\n          &quot;reward&quot;: 0,\n          &quot;info&quot;: {},\n          &quot;observation&quot;: {\n            &quot;board&quot;: [\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              2,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              1,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              2,\n              0,\n              0,\n              0,\n              0,\n              2,\n              1,\n              1,\n              1,\n              2,\n              0\n            ],\n            &quot;mark&quot;: 1\n          },\n          &quot;status&quot;: &quot;ACTIVE&quot;\n        },\n        {\n          &quot;action&quot;: 5,\n          &quot;reward&quot;: 0,\n          &quot;info&quot;: {},\n          &quot;observation&quot;: {\n            &quot;mark&quot;: 2\n          },\n          &quot;status&quot;: &quot;INACTIVE&quot;\n        }\n      ],\n      [\n        {\n          &quot;action&quot;: 4,\n          &quot;reward&quot;: 0,\n          &quot;info&quot;: {},\n          &quot;observation&quot;: {\n            &quot;board&quot;: [\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              2,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              1,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              2,\n              1,\n              0,\n              0,\n              0,\n              2,\n              1,\n              1,\n              1,\n              2,\n              0\n            ],\n            &quot;mark&quot;: 1\n          },\n          &quot;status&quot;: &quot;INACTIVE&quot;\n        },\n        {\n          &quot;action&quot;: 0,\n          &quot;reward&quot;: 0,\n          &quot;info&quot;: {},\n          &quot;observation&quot;: {\n            &quot;mark&quot;: 2\n          },\n          &quot;status&quot;: &quot;ACTIVE&quot;\n        }\n      ],\n      [\n        {\n          &quot;action&quot;: 0,\n          &quot;reward&quot;: 0,\n          &quot;info&quot;: {},\n          &quot;observation&quot;: {\n            &quot;board&quot;: [\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              2,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              1,\n              2,\n              0,\n              0,\n              0,\n              0,\n              0,\n              2,\n              1,\n              0,\n              0,\n              0,\n              2,\n              1,\n              1,\n              1,\n              2,\n              0\n            ],\n            &quot;mark&quot;: 1\n          },\n          &quot;status&quot;: &quot;ACTIVE&quot;\n        },\n        {\n          &quot;action&quot;: 4,\n          &quot;reward&quot;: 0,\n          &quot;info&quot;: {},\n          &quot;observation&quot;: {\n            &quot;mark&quot;: 2\n          },\n          &quot;status&quot;: &quot;INACTIVE&quot;\n        }\n      ],\n      [\n        {\n          &quot;action&quot;: 4,\n          &quot;reward&quot;: 0,\n          &quot;info&quot;: {},\n          &quot;observation&quot;: {\n            &quot;board&quot;: [\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              2,\n              1,\n              0,\n              0,\n              0,\n              0,\n              0,\n              1,\n              2,\n              0,\n              0,\n              0,\n              0,\n              0,\n              2,\n              1,\n              0,\n              0,\n              0,\n              2,\n              1,\n              1,\n              1,\n              2,\n              0\n            ],\n            &quot;mark&quot;: 1\n          },\n          &quot;status&quot;: &quot;INACTIVE&quot;\n        },\n        {\n          &quot;action&quot;: 0,\n          &quot;reward&quot;: 0,\n          &quot;info&quot;: {},\n          &quot;observation&quot;: {\n            &quot;mark&quot;: 2\n          },\n          &quot;status&quot;: &quot;ACTIVE&quot;\n        }\n      ],\n      [\n        {\n          &quot;action&quot;: 0,\n          &quot;reward&quot;: 0,\n          &quot;info&quot;: {},\n          &quot;observation&quot;: {\n            &quot;board&quot;: [\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              2,\n              1,\n              0,\n              0,\n              0,\n              0,\n              0,\n              1,\n              2,\n              0,\n              0,\n              0,\n              0,\n              0,\n              2,\n              1,\n              2,\n              0,\n              0,\n              2,\n              1,\n              1,\n              1,\n              2,\n              0\n            ],\n            &quot;mark&quot;: 1\n          },\n          &quot;status&quot;: &quot;ACTIVE&quot;\n        },\n        {\n          &quot;action&quot;: 5,\n          &quot;reward&quot;: 0,\n          &quot;info&quot;: {},\n          &quot;observation&quot;: {\n            &quot;mark&quot;: 2\n          },\n          &quot;status&quot;: &quot;INACTIVE&quot;\n        }\n      ],\n      [\n        {\n          &quot;action&quot;: 6,\n          &quot;reward&quot;: 0,\n          &quot;info&quot;: {},\n          &quot;observation&quot;: {\n            &quot;board&quot;: [\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              2,\n              1,\n              0,\n              0,\n              0,\n              0,\n              0,\n              1,\n              2,\n              0,\n              0,\n              0,\n              0,\n              0,\n              2,\n              1,\n              2,\n              0,\n              0,\n              2,\n              1,\n              1,\n              1,\n              2,\n              1\n            ],\n            &quot;mark&quot;: 1\n          },\n          &quot;status&quot;: &quot;INACTIVE&quot;\n        },\n        {\n          &quot;action&quot;: 0,\n          &quot;reward&quot;: 0,\n          &quot;info&quot;: {},\n          &quot;observation&quot;: {\n            &quot;mark&quot;: 2\n          },\n          &quot;status&quot;: &quot;ACTIVE&quot;\n        }\n      ],\n      [\n        {\n          &quot;action&quot;: 0,\n          &quot;reward&quot;: 0,\n          &quot;info&quot;: {},\n          &quot;observation&quot;: {\n            &quot;board&quot;: [\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              2,\n              1,\n              0,\n              0,\n              0,\n              0,\n              0,\n              1,\n              2,\n              0,\n              0,\n              0,\n              2,\n              0,\n              2,\n              1,\n              2,\n              0,\n              0,\n              2,\n              1,\n              1,\n              1,\n              2,\n              1\n            ],\n            &quot;mark&quot;: 1\n          },\n          &quot;status&quot;: &quot;ACTIVE&quot;\n        },\n        {\n          &quot;action&quot;: 1,\n          &quot;reward&quot;: 0,\n          &quot;info&quot;: {},\n          &quot;observation&quot;: {\n            &quot;mark&quot;: 2\n          },\n          &quot;status&quot;: &quot;INACTIVE&quot;\n        }\n      ],\n      [\n        {\n          &quot;action&quot;: 1,\n          &quot;reward&quot;: 0,\n          &quot;info&quot;: {},\n          &quot;observation&quot;: {\n            &quot;board&quot;: [\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              2,\n              1,\n              0,\n              0,\n              0,\n              1,\n              0,\n              1,\n              2,\n              0,\n              0,\n              0,\n              2,\n              0,\n              2,\n              1,\n              2,\n              0,\n              0,\n              2,\n              1,\n              1,\n              1,\n              2,\n              1\n            ],\n            &quot;mark&quot;: 1\n          },\n          &quot;status&quot;: &quot;INACTIVE&quot;\n        },\n        {\n          &quot;action&quot;: 0,\n          &quot;reward&quot;: 0,\n          &quot;info&quot;: {},\n          &quot;observation&quot;: {\n            &quot;mark&quot;: 2\n          },\n          &quot;status&quot;: &quot;ACTIVE&quot;\n        }\n      ],\n      [\n        {\n          &quot;action&quot;: 0,\n          &quot;reward&quot;: 0,\n          &quot;info&quot;: {},\n          &quot;observation&quot;: {\n            &quot;board&quot;: [\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              2,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              2,\n              1,\n              0,\n              0,\n              0,\n              1,\n              0,\n              1,\n              2,\n              0,\n              0,\n              0,\n              2,\n              0,\n              2,\n              1,\n              2,\n              0,\n              0,\n              2,\n              1,\n              1,\n              1,\n              2,\n              1\n            ],\n            &quot;mark&quot;: 1\n          },\n          &quot;status&quot;: &quot;ACTIVE&quot;\n        },\n        {\n          &quot;action&quot;: 3,\n          &quot;reward&quot;: 0,\n          &quot;info&quot;: {},\n          &quot;observation&quot;: {\n            &quot;mark&quot;: 2\n          },\n          &quot;status&quot;: &quot;INACTIVE&quot;\n        }\n      ],\n      [\n        {\n          &quot;action&quot;: 4,\n          &quot;reward&quot;: 0,\n          &quot;info&quot;: {},\n          &quot;observation&quot;: {\n            &quot;board&quot;: [\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              2,\n              1,\n              0,\n              0,\n              0,\n              0,\n              0,\n              2,\n              1,\n              0,\n              0,\n              0,\n              1,\n              0,\n              1,\n              2,\n              0,\n              0,\n              0,\n              2,\n              0,\n              2,\n              1,\n              2,\n              0,\n              0,\n              2,\n              1,\n              1,\n              1,\n              2,\n              1\n            ],\n            &quot;mark&quot;: 1\n          },\n          &quot;status&quot;: &quot;INACTIVE&quot;\n        },\n        {\n          &quot;action&quot;: 0,\n          &quot;reward&quot;: 0,\n          &quot;info&quot;: {},\n          &quot;observation&quot;: {\n            &quot;mark&quot;: 2\n          },\n          &quot;status&quot;: &quot;ACTIVE&quot;\n        }\n      ],\n      [\n        {\n          &quot;action&quot;: 0,\n          &quot;reward&quot;: 0,\n          &quot;info&quot;: {},\n          &quot;observation&quot;: {\n            &quot;board&quot;: [\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              2,\n              1,\n              0,\n              0,\n              0,\n              0,\n              0,\n              2,\n              1,\n              0,\n              0,\n              0,\n              1,\n              0,\n              1,\n              2,\n              0,\n              0,\n              0,\n              2,\n              2,\n              2,\n              1,\n              2,\n              0,\n              0,\n              2,\n              1,\n              1,\n              1,\n              2,\n              1\n            ],\n            &quot;mark&quot;: 1\n          },\n          &quot;status&quot;: &quot;ACTIVE&quot;\n        },\n        {\n          &quot;action&quot;: 2,\n          &quot;reward&quot;: 0,\n          &quot;info&quot;: {},\n          &quot;observation&quot;: {\n            &quot;mark&quot;: 2\n          },\n          &quot;status&quot;: &quot;INACTIVE&quot;\n        }\n      ],\n      [\n        {\n          &quot;action&quot;: 2,\n          &quot;reward&quot;: 0,\n          &quot;info&quot;: {},\n          &quot;observation&quot;: {\n            &quot;board&quot;: [\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              2,\n              1,\n              0,\n              0,\n              0,\n              0,\n              0,\n              2,\n              1,\n              0,\n              0,\n              0,\n              1,\n              1,\n              1,\n              2,\n              0,\n              0,\n              0,\n              2,\n              2,\n              2,\n              1,\n              2,\n              0,\n              0,\n              2,\n              1,\n              1,\n              1,\n              2,\n              1\n            ],\n            &quot;mark&quot;: 1\n          },\n          &quot;status&quot;: &quot;INACTIVE&quot;\n        },\n        {\n          &quot;action&quot;: 0,\n          &quot;reward&quot;: 0,\n          &quot;info&quot;: {},\n          &quot;observation&quot;: {\n            &quot;mark&quot;: 2\n          },\n          &quot;status&quot;: &quot;ACTIVE&quot;\n        }\n      ],\n      [\n        {\n          &quot;action&quot;: 0,\n          &quot;reward&quot;: 0,\n          &quot;info&quot;: {},\n          &quot;observation&quot;: {\n            &quot;board&quot;: [\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              2,\n              1,\n              0,\n              0,\n              0,\n              2,\n              0,\n              2,\n              1,\n              0,\n              0,\n              0,\n              1,\n              1,\n              1,\n              2,\n              0,\n              0,\n              0,\n              2,\n              2,\n              2,\n              1,\n              2,\n              0,\n              0,\n              2,\n              1,\n              1,\n              1,\n              2,\n              1\n            ],\n            &quot;mark&quot;: 1\n          },\n          &quot;status&quot;: &quot;ACTIVE&quot;\n        },\n        {\n          &quot;action&quot;: 1,\n          &quot;reward&quot;: 0,\n          &quot;info&quot;: {},\n          &quot;observation&quot;: {\n            &quot;mark&quot;: 2\n          },\n          &quot;status&quot;: &quot;INACTIVE&quot;\n        }\n      ],\n      [\n        {\n          &quot;action&quot;: 1,\n          &quot;reward&quot;: 0,\n          &quot;info&quot;: {},\n          &quot;observation&quot;: {\n            &quot;board&quot;: [\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              1,\n              0,\n              2,\n              1,\n              0,\n              0,\n              0,\n              2,\n              0,\n              2,\n              1,\n              0,\n              0,\n              0,\n              1,\n              1,\n              1,\n              2,\n              0,\n              0,\n              0,\n              2,\n              2,\n              2,\n              1,\n              2,\n              0,\n              0,\n              2,\n              1,\n              1,\n              1,\n              2,\n              1\n            ],\n            &quot;mark&quot;: 1\n          },\n          &quot;status&quot;: &quot;INACTIVE&quot;\n        },\n        {\n          &quot;action&quot;: 0,\n          &quot;reward&quot;: 0,\n          &quot;info&quot;: {},\n          &quot;observation&quot;: {\n            &quot;mark&quot;: 2\n          },\n          &quot;status&quot;: &quot;ACTIVE&quot;\n        }\n      ],\n      [\n        {\n          &quot;action&quot;: 0,\n          &quot;reward&quot;: 0,\n          &quot;info&quot;: {},\n          &quot;observation&quot;: {\n            &quot;board&quot;: [\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              1,\n              0,\n              2,\n              1,\n              0,\n              0,\n              0,\n              2,\n              2,\n              2,\n              1,\n              0,\n              0,\n              0,\n              1,\n              1,\n              1,\n              2,\n              0,\n              0,\n              0,\n              2,\n              2,\n              2,\n              1,\n              2,\n              0,\n              0,\n              2,\n              1,\n              1,\n              1,\n              2,\n              1\n            ],\n            &quot;mark&quot;: 1\n          },\n          &quot;status&quot;: &quot;ACTIVE&quot;\n        },\n        {\n          &quot;action&quot;: 2,\n          &quot;reward&quot;: 0,\n          &quot;info&quot;: {},\n          &quot;observation&quot;: {\n            &quot;mark&quot;: 2\n          },\n          &quot;status&quot;: &quot;INACTIVE&quot;\n        }\n      ],\n      [\n        {\n          &quot;action&quot;: 2,\n          &quot;reward&quot;: 0,\n          &quot;info&quot;: {},\n          &quot;observation&quot;: {\n            &quot;board&quot;: [\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              1,\n              1,\n              2,\n              1,\n              0,\n              0,\n              0,\n              2,\n              2,\n              2,\n              1,\n              0,\n              0,\n              0,\n              1,\n              1,\n              1,\n              2,\n              0,\n              0,\n              0,\n              2,\n              2,\n              2,\n              1,\n              2,\n              0,\n              0,\n              2,\n              1,\n              1,\n              1,\n              2,\n              1\n            ],\n            &quot;mark&quot;: 1\n          },\n          &quot;status&quot;: &quot;INACTIVE&quot;\n        },\n        {\n          &quot;action&quot;: 0,\n          &quot;reward&quot;: 0,\n          &quot;info&quot;: {},\n          &quot;observation&quot;: {\n            &quot;mark&quot;: 2\n          },\n          &quot;status&quot;: &quot;ACTIVE&quot;\n        }\n      ],\n      [\n        {\n          &quot;action&quot;: 0,\n          &quot;reward&quot;: 0,\n          &quot;info&quot;: {},\n          &quot;observation&quot;: {\n            &quot;board&quot;: [\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              1,\n              1,\n              2,\n              1,\n              0,\n              0,\n              0,\n              2,\n              2,\n              2,\n              1,\n              0,\n              0,\n              0,\n              1,\n              1,\n              1,\n              2,\n              0,\n              0,\n              0,\n              2,\n              2,\n              2,\n              1,\n              2,\n              0,\n              2,\n              2,\n              1,\n              1,\n              1,\n              2,\n              1\n            ],\n            &quot;mark&quot;: 1\n          },\n          &quot;status&quot;: &quot;ACTIVE&quot;\n        },\n        {\n          &quot;action&quot;: 0,\n          &quot;reward&quot;: 0,\n          &quot;info&quot;: {},\n          &quot;observation&quot;: {\n            &quot;mark&quot;: 2\n          },\n          &quot;status&quot;: &quot;INACTIVE&quot;\n        }\n      ],\n      [\n        {\n          &quot;action&quot;: 0,\n          &quot;reward&quot;: 0,\n          &quot;info&quot;: {},\n          &quot;observation&quot;: {\n            &quot;board&quot;: [\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              1,\n              1,\n              2,\n              1,\n              0,\n              0,\n              0,\n              2,\n              2,\n              2,\n              1,\n              0,\n              0,\n              0,\n              1,\n              1,\n              1,\n              2,\n              0,\n              0,\n              1,\n              2,\n              2,\n              2,\n              1,\n              2,\n              0,\n              2,\n              2,\n              1,\n              1,\n              1,\n              2,\n              1\n            ],\n            &quot;mark&quot;: 1\n          },\n          &quot;status&quot;: &quot;INACTIVE&quot;\n        },\n        {\n          &quot;action&quot;: 0,\n          &quot;reward&quot;: 0,\n          &quot;info&quot;: {},\n          &quot;observation&quot;: {\n            &quot;mark&quot;: 2\n          },\n          &quot;status&quot;: &quot;ACTIVE&quot;\n        }\n      ],\n      [\n        {\n          &quot;action&quot;: 0,\n          &quot;reward&quot;: 0,\n          &quot;info&quot;: {},\n          &quot;observation&quot;: {\n            &quot;board&quot;: [\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              1,\n              1,\n              2,\n              1,\n              0,\n              0,\n              0,\n              2,\n              2,\n              2,\n              1,\n              0,\n              0,\n              2,\n              1,\n              1,\n              1,\n              2,\n              0,\n              0,\n              1,\n              2,\n              2,\n              2,\n              1,\n              2,\n              0,\n              2,\n              2,\n              1,\n              1,\n              1,\n              2,\n              1\n            ],\n            &quot;mark&quot;: 1\n          },\n          &quot;status&quot;: &quot;ACTIVE&quot;\n        },\n        {\n          &quot;action&quot;: 0,\n          &quot;reward&quot;: 0,\n          &quot;info&quot;: {},\n          &quot;observation&quot;: {\n            &quot;mark&quot;: 2\n          },\n          &quot;status&quot;: &quot;INACTIVE&quot;\n        }\n      ],\n      [\n        {\n          &quot;action&quot;: 0,\n          &quot;reward&quot;: 0,\n          &quot;info&quot;: {},\n          &quot;observation&quot;: {\n            &quot;board&quot;: [\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              0,\n              1,\n              1,\n              2,\n              1,\n              0,\n              0,\n              1,\n              2,\n              2,\n              2,\n              1,\n              0,\n              0,\n              2,\n              1,\n              1,\n              1,\n              2,\n              0,\n              0,\n              1,\n              2,\n              2,\n              2,\n              1,\n              2,\n              0,\n              2,\n              2,\n              1,\n              1,\n              1,\n              2,\n              1\n            ],\n            &quot;mark&quot;: 1\n          },\n          &quot;status&quot;: &quot;INACTIVE&quot;\n        },\n        {\n          &quot;action&quot;: 0,\n          &quot;reward&quot;: 0,\n          &quot;info&quot;: {},\n          &quot;observation&quot;: {\n            &quot;mark&quot;: 2\n          },\n          &quot;status&quot;: &quot;ACTIVE&quot;\n        }\n      ],\n      [\n        {\n          &quot;action&quot;: 0,\n          &quot;reward&quot;: 0,\n          &quot;info&quot;: {},\n          &quot;observation&quot;: {\n            &quot;board&quot;: [\n              0,\n              0,\n              0,\n              2,\n              0,\n              0,\n              0,\n              0,\n              1,\n              1,\n              2,\n              1,\n              0,\n              0,\n              1,\n              2,\n              2,\n              2,\n              1,\n              0,\n              0,\n              2,\n              1,\n              1,\n              1,\n              2,\n              0,\n              0,\n              1,\n              2,\n              2,\n              2,\n              1,\n              2,\n              0,\n              2,\n              2,\n              1,\n              1,\n              1,\n              2,\n              1\n            ],\n            &quot;mark&quot;: 1\n          },\n          &quot;status&quot;: &quot;ACTIVE&quot;\n        },\n        {\n          &quot;action&quot;: 3,\n          &quot;reward&quot;: 0,\n          &quot;info&quot;: {},\n          &quot;observation&quot;: {\n            &quot;mark&quot;: 2\n          },\n          &quot;status&quot;: &quot;INACTIVE&quot;\n        }\n      ],\n      [\n        {\n          &quot;action&quot;: 5,\n          &quot;reward&quot;: 0,\n          &quot;info&quot;: {},\n          &quot;observation&quot;: {\n            &quot;board&quot;: [\n              0,\n              0,\n              0,\n              2,\n              0,\n              0,\n              0,\n              0,\n              1,\n              1,\n              2,\n              1,\n              0,\n              0,\n              1,\n              2,\n              2,\n              2,\n              1,\n              0,\n              0,\n              2,\n              1,\n              1,\n              1,\n              2,\n              1,\n              0,\n              1,\n              2,\n              2,\n              2,\n              1,\n              2,\n              0,\n              2,\n              2,\n              1,\n              1,\n              1,\n              2,\n              1\n            ],\n            &quot;mark&quot;: 1\n          },\n          &quot;status&quot;: &quot;INACTIVE&quot;\n        },\n        {\n          &quot;action&quot;: 0,\n          &quot;reward&quot;: 0,\n          &quot;info&quot;: {},\n          &quot;observation&quot;: {\n            &quot;mark&quot;: 2\n          },\n          &quot;status&quot;: &quot;ACTIVE&quot;\n        }\n      ],\n      [\n        {\n          &quot;action&quot;: 0,\n          &quot;reward&quot;: 0,\n          &quot;info&quot;: {},\n          &quot;observation&quot;: {\n            &quot;board&quot;: [\n              0,\n              0,\n              2,\n              2,\n              0,\n              0,\n              0,\n              0,\n              1,\n              1,\n              2,\n              1,\n              0,\n              0,\n              1,\n              2,\n              2,\n              2,\n              1,\n              0,\n              0,\n              2,\n              1,\n              1,\n              1,\n              2,\n              1,\n              0,\n              1,\n              2,\n              2,\n              2,\n              1,\n              2,\n              0,\n              2,\n              2,\n              1,\n              1,\n              1,\n              2,\n              1\n            ],\n            &quot;mark&quot;: 1\n          },\n          &quot;status&quot;: &quot;ACTIVE&quot;\n        },\n        {\n          &quot;action&quot;: 2,\n          &quot;reward&quot;: 0,\n          &quot;info&quot;: {},\n          &quot;observation&quot;: {\n            &quot;mark&quot;: 2\n          },\n          &quot;status&quot;: &quot;INACTIVE&quot;\n        }\n      ],\n      [\n        {\n          &quot;action&quot;: 1,\n          &quot;reward&quot;: 0,\n          &quot;info&quot;: {},\n          &quot;observation&quot;: {\n            &quot;board&quot;: [\n              0,\n              1,\n              2,\n              2,\n              0,\n              0,\n              0,\n              0,\n              1,\n              1,\n              2,\n              1,\n              0,\n              0,\n              1,\n              2,\n              2,\n              2,\n              1,\n              0,\n              0,\n              2,\n              1,\n              1,\n              1,\n              2,\n              1,\n              0,\n              1,\n              2,\n              2,\n              2,\n              1,\n              2,\n              0,\n              2,\n              2,\n              1,\n              1,\n              1,\n              2,\n              1\n            ],\n            &quot;mark&quot;: 1\n          },\n          &quot;status&quot;: &quot;INACTIVE&quot;\n        },\n        {\n          &quot;action&quot;: 0,\n          &quot;reward&quot;: 0,\n          &quot;info&quot;: {},\n          &quot;observation&quot;: {\n            &quot;mark&quot;: 2\n          },\n          &quot;status&quot;: &quot;ACTIVE&quot;\n        }\n      ],\n      [\n        {\n          &quot;action&quot;: 0,\n          &quot;reward&quot;: 0,\n          &quot;info&quot;: {},\n          &quot;observation&quot;: {\n            &quot;board&quot;: [\n              0,\n              1,\n              2,\n              2,\n              0,\n              0,\n              0,\n              0,\n              1,\n              1,\n              2,\n              1,\n              0,\n              0,\n              1,\n              2,\n              2,\n              2,\n              1,\n              0,\n              0,\n              2,\n              1,\n              1,\n              1,\n              2,\n              1,\n              0,\n              1,\n              2,\n              2,\n              2,\n              1,\n              2,\n              2,\n              2,\n              2,\n              1,\n              1,\n              1,\n              2,\n              1\n            ],\n            &quot;mark&quot;: 1\n          },\n          &quot;status&quot;: &quot;ACTIVE&quot;\n        },\n        {\n          &quot;action&quot;: 6,\n          &quot;reward&quot;: 0,\n          &quot;info&quot;: {},\n          &quot;observation&quot;: {\n            &quot;mark&quot;: 2\n          },\n          &quot;status&quot;: &quot;INACTIVE&quot;\n        }\n      ],\n      [\n        {\n          &quot;action&quot;: 4,\n          &quot;reward&quot;: 0,\n          &quot;info&quot;: {},\n          &quot;observation&quot;: {\n            &quot;board&quot;: [\n              0,\n              1,\n              2,\n              2,\n              1,\n              0,\n              0,\n              0,\n              1,\n              1,\n              2,\n              1,\n              0,\n              0,\n              1,\n              2,\n              2,\n              2,\n              1,\n              0,\n              0,\n              2,\n              1,\n              1,\n              1,\n              2,\n              1,\n              0,\n              1,\n              2,\n              2,\n              2,\n              1,\n              2,\n              2,\n              2,\n              2,\n              1,\n              1,\n              1,\n              2,\n              1\n            ],\n            &quot;mark&quot;: 1\n          },\n          &quot;status&quot;: &quot;INACTIVE&quot;\n        },\n        {\n          &quot;action&quot;: 0,\n          &quot;reward&quot;: 0,\n          &quot;info&quot;: {},\n          &quot;observation&quot;: {\n            &quot;mark&quot;: 2\n          },\n          &quot;status&quot;: &quot;ACTIVE&quot;\n        }\n      ],\n      [\n        {\n          &quot;action&quot;: 0,\n          &quot;reward&quot;: 0,\n          &quot;info&quot;: {},\n          &quot;observation&quot;: {\n            &quot;board&quot;: [\n              0,\n              1,\n              2,\n              2,\n              1,\n              0,\n              0,\n              0,\n              1,\n              1,\n              2,\n              1,\n              0,\n              0,\n              1,\n              2,\n              2,\n              2,\n              1,\n              2,\n              0,\n              2,\n              1,\n              1,\n              1,\n              2,\n              1,\n              0,\n              1,\n              2,\n              2,\n              2,\n              1,\n              2,\n              2,\n              2,\n              2,\n              1,\n              1,\n              1,\n              2,\n              1\n            ],\n            &quot;mark&quot;: 1\n          },\n          &quot;status&quot;: &quot;ACTIVE&quot;\n        },\n        {\n          &quot;action&quot;: 5,\n          &quot;reward&quot;: 0,\n          &quot;info&quot;: {},\n          &quot;observation&quot;: {\n            &quot;mark&quot;: 2\n          },\n          &quot;status&quot;: &quot;INACTIVE&quot;\n        }\n      ],\n      [\n        {\n          &quot;action&quot;: 6,\n          &quot;reward&quot;: 0,\n          &quot;info&quot;: {},\n          &quot;observation&quot;: {\n            &quot;board&quot;: [\n              0,\n              1,\n              2,\n              2,\n              1,\n              0,\n              0,\n              0,\n              1,\n              1,\n              2,\n              1,\n              0,\n              0,\n              1,\n              2,\n              2,\n              2,\n              1,\n              2,\n              0,\n              2,\n              1,\n              1,\n              1,\n              2,\n              1,\n              1,\n              1,\n              2,\n              2,\n              2,\n              1,\n              2,\n              2,\n              2,\n              2,\n              1,\n              1,\n              1,\n              2,\n              1\n            ],\n            &quot;mark&quot;: 1\n          },\n          &quot;status&quot;: &quot;INACTIVE&quot;\n        },\n        {\n          &quot;action&quot;: 0,\n          &quot;reward&quot;: 0,\n          &quot;info&quot;: {},\n          &quot;observation&quot;: {\n            &quot;mark&quot;: 2\n          },\n          &quot;status&quot;: &quot;ACTIVE&quot;\n        }\n      ],\n      [\n        {\n          &quot;action&quot;: 0,\n          &quot;reward&quot;: 0,\n          &quot;info&quot;: {},\n          &quot;observation&quot;: {\n            &quot;board&quot;: [\n              0,\n              1,\n              2,\n              2,\n              1,\n              0,\n              0,\n              0,\n              1,\n              1,\n              2,\n              1,\n              0,\n              0,\n              1,\n              2,\n              2,\n              2,\n              1,\n              2,\n              2,\n              2,\n              1,\n              1,\n              1,\n              2,\n              1,\n              1,\n              1,\n              2,\n              2,\n              2,\n              1,\n              2,\n              2,\n              2,\n              2,\n              1,\n              1,\n              1,\n              2,\n              1\n            ],\n            &quot;mark&quot;: 1\n          },\n          &quot;status&quot;: &quot;ACTIVE&quot;\n        },\n        {\n          &quot;action&quot;: 6,\n          &quot;reward&quot;: 0,\n          &quot;info&quot;: {},\n          &quot;observation&quot;: {\n            &quot;mark&quot;: 2\n          },\n          &quot;status&quot;: &quot;INACTIVE&quot;\n        }\n      ],\n      [\n        {\n          &quot;action&quot;: 6,\n          &quot;reward&quot;: 0,\n          &quot;info&quot;: {},\n          &quot;observation&quot;: {\n            &quot;board&quot;: [\n              0,\n              1,\n              2,\n              2,\n              1,\n              0,\n              0,\n              0,\n              1,\n              1,\n              2,\n              1,\n              0,\n              1,\n              1,\n              2,\n              2,\n              2,\n              1,\n              2,\n              2,\n              2,\n              1,\n              1,\n              1,\n              2,\n              1,\n              1,\n              1,\n              2,\n              2,\n              2,\n              1,\n              2,\n              2,\n              2,\n              2,\n              1,\n              1,\n              1,\n              2,\n              1\n            ],\n            &quot;mark&quot;: 1\n          },\n          &quot;status&quot;: &quot;INACTIVE&quot;\n        },\n        {\n          &quot;action&quot;: 0,\n          &quot;reward&quot;: 0,\n          &quot;info&quot;: {},\n          &quot;observation&quot;: {\n            &quot;mark&quot;: 2\n          },\n          &quot;status&quot;: &quot;ACTIVE&quot;\n        }\n      ],\n      [\n        {\n          &quot;action&quot;: 0,\n          &quot;reward&quot;: 0,\n          &quot;info&quot;: {},\n          &quot;observation&quot;: {\n            &quot;board&quot;: [\n              0,\n              1,\n              2,\n              2,\n              1,\n              0,\n              0,\n              2,\n              1,\n              1,\n              2,\n              1,\n              0,\n              1,\n              1,\n              2,\n              2,\n              2,\n              1,\n              2,\n              2,\n              2,\n              1,\n              1,\n              1,\n              2,\n              1,\n              1,\n              1,\n              2,\n              2,\n              2,\n              1,\n              2,\n              2,\n              2,\n              2,\n              1,\n              1,\n              1,\n              2,\n              1\n            ],\n            &quot;mark&quot;: 1\n          },\n          &quot;status&quot;: &quot;ACTIVE&quot;\n        },\n        {\n          &quot;action&quot;: 0,\n          &quot;reward&quot;: 0,\n          &quot;info&quot;: {},\n          &quot;observation&quot;: {\n            &quot;mark&quot;: 2\n          },\n          &quot;status&quot;: &quot;INACTIVE&quot;\n        }\n      ],\n      [\n        {\n          &quot;action&quot;: 5,\n          &quot;reward&quot;: 0,\n          &quot;info&quot;: {},\n          &quot;observation&quot;: {\n            &quot;board&quot;: [\n              0,\n              1,\n              2,\n              2,\n              1,\n              0,\n              0,\n              2,\n              1,\n              1,\n              2,\n              1,\n              1,\n              1,\n              1,\n              2,\n              2,\n              2,\n              1,\n              2,\n              2,\n              2,\n              1,\n              1,\n              1,\n              2,\n              1,\n              1,\n              1,\n              2,\n              2,\n              2,\n              1,\n              2,\n              2,\n              2,\n              2,\n              1,\n              1,\n              1,\n              2,\n              1\n            ],\n            &quot;mark&quot;: 1\n          },\n          &quot;status&quot;: &quot;INACTIVE&quot;\n        },\n        {\n          &quot;action&quot;: 0,\n          &quot;reward&quot;: 0,\n          &quot;info&quot;: {},\n          &quot;observation&quot;: {\n            &quot;mark&quot;: 2\n          },\n          &quot;status&quot;: &quot;ACTIVE&quot;\n        }\n      ],\n      [\n        {\n          &quot;action&quot;: 0,\n          &quot;reward&quot;: 0,\n          &quot;info&quot;: {},\n          &quot;observation&quot;: {\n            &quot;board&quot;: [\n              0,\n              1,\n              2,\n              2,\n              1,\n              0,\n              2,\n              2,\n              1,\n              1,\n              2,\n              1,\n              1,\n              1,\n              1,\n              2,\n              2,\n              2,\n              1,\n              2,\n              2,\n              2,\n              1,\n              1,\n              1,\n              2,\n              1,\n              1,\n              1,\n              2,\n              2,\n              2,\n              1,\n              2,\n              2,\n              2,\n              2,\n              1,\n              1,\n              1,\n              2,\n              1\n            ],\n            &quot;mark&quot;: 1\n          },\n          &quot;status&quot;: &quot;ACTIVE&quot;\n        },\n        {\n          &quot;action&quot;: 6,\n          &quot;reward&quot;: 0,\n          &quot;info&quot;: {},\n          &quot;observation&quot;: {\n            &quot;mark&quot;: 2\n          },\n          &quot;status&quot;: &quot;INACTIVE&quot;\n        }\n      ],\n      [\n        {\n          &quot;action&quot;: 0,\n          &quot;reward&quot;: 0,\n          &quot;info&quot;: {},\n          &quot;observation&quot;: {\n            &quot;board&quot;: [\n              1,\n              1,\n              2,\n              2,\n              1,\n              0,\n              2,\n              2,\n              1,\n              1,\n              2,\n              1,\n              1,\n              1,\n              1,\n              2,\n              2,\n              2,\n              1,\n              2,\n              2,\n              2,\n              1,\n              1,\n              1,\n              2,\n              1,\n              1,\n              1,\n              2,\n              2,\n              2,\n              1,\n              2,\n              2,\n              2,\n              2,\n              1,\n              1,\n              1,\n              2,\n              1\n            ],\n            &quot;mark&quot;: 1\n          },\n          &quot;status&quot;: &quot;INACTIVE&quot;\n        },\n        {\n          &quot;action&quot;: 0,\n          &quot;reward&quot;: 0,\n          &quot;info&quot;: {},\n          &quot;observation&quot;: {\n            &quot;mark&quot;: 2\n          },\n          &quot;status&quot;: &quot;ACTIVE&quot;\n        }\n      ],\n      [\n        {\n          &quot;action&quot;: 0,\n          &quot;reward&quot;: 0,\n          &quot;info&quot;: {},\n          &quot;observation&quot;: {\n            &quot;board&quot;: [\n              1,\n              1,\n              2,\n              2,\n              1,\n              2,\n              2,\n              2,\n              1,\n              1,\n              2,\n              1,\n              1,\n              1,\n              1,\n              2,\n              2,\n              2,\n              1,\n              2,\n              2,\n              2,\n              1,\n              1,\n              1,\n              2,\n              1,\n              1,\n              1,\n              2,\n              2,\n              2,\n              1,\n              2,\n              2,\n              2,\n              2,\n              1,\n              1,\n              1,\n              2,\n              1\n            ],\n            &quot;mark&quot;: 1\n          },\n          &quot;status&quot;: &quot;DONE&quot;\n        },\n        {\n          &quot;action&quot;: 5,\n          &quot;reward&quot;: 0,\n          &quot;info&quot;: {},\n          &quot;observation&quot;: {\n            &quot;mark&quot;: 2\n          },\n          &quot;status&quot;: &quot;DONE&quot;\n        }\n      ]\n    ],\n    &quot;rewards&quot;: [\n      0,\n      0\n    ],\n    &quot;statuses&quot;: [\n      &quot;DONE&quot;,\n      &quot;DONE&quot;\n    ],\n    &quot;schema_version&quot;: 1\n  },\n  &quot;mode&quot;: &quot;ipython&quot;,\n  &quot;width&quot;: 500,\n  &quot;height&quot;: 450\n};\n\n\nwindow.kaggle.renderer = // Copyright 2020 Kaggle Inc\n//\n// Licensed under the Apache License, Version 2.0 (the &quot;License&quot;);\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//      http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an &quot;AS IS&quot; BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\nfunction renderer({\n  act,\n  agents,\n  environment,\n  frame,\n  height = 400,\n  interactive,\n  isInteractive,\n  parent,\n  step,\n  update,\n  width = 400,\n}) {\n  // Configuration.\n  const { rows, columns, inarow } = environment.configuration;\n\n  // Common Dimensions.\n  const unit = 8;\n  const minCanvasSize = Math.min(height, width);\n  const minOffset = minCanvasSize > 400 ? 30 : unit / 2;\n  const cellSize = Math.min(\n    (width - minOffset * 2) / columns,\n    (height - minOffset * 2) / rows\n  );\n  const cellInset = 0.8;\n  const pieceScale = cellSize / 100;\n  const xOffset = Math.max(0, (width - cellSize * columns) / 2);\n  const yOffset = Math.max(0, (height - cellSize * rows) / 2);\n\n  // Canvas Setup.\n  let canvas = parent.querySelector(&quot;canvas&quot;);\n  if (!canvas) {\n    canvas = document.createElement(&quot;canvas&quot;);\n    parent.appendChild(canvas);\n\n    if (interactive) {\n      canvas.addEventListener(&quot;click&quot;, evt => {\n        if (!isInteractive()) return;\n        const rect = evt.target.getBoundingClientRect();\n        const col = Math.floor((evt.clientX - rect.left - xOffset) / cellSize);\n        if (col >= 0 && col < columns) act(col);\n      });\n    }\n  }\n  canvas.style.cursor = isInteractive() ? &quot;pointer&quot; : &quot;default&quot;;\n\n  // Character Paths (based on 100x100 tiles).\n  const kPath = new Path2D(\n    `M78.3,96.5c-0.1,0.4-0.5,0.6-1.1,0.6H64.9c-0.7,0-1.4-0.3-1.9-1l-20.3-26L37,75.5v20.1 c0,0.9-0.5,1.4-1.4,1.4H26c-0.9,0-1.4-0.5-1.4-1.4V3.9c0-0.9,0.5-1.4,1.4-1.4h9.5C36.5,2.5,37,3,37,3.9v56.5l24.3-24.7 c0.6-0.6,1.3-1,1.9-1H76c0.6,0,0.9,0.2,1.1,0.7c0.2,0.6,0.1,1-0.1,1.2l-25.7,25L78,95.1C78.4,95.5,78.5,95.9,78.3,96.5z`\n  );\n  const goose1Path = new Path2D(\n    `M8.8,92.7c-4-18.5,4.7-37.2,20.7-46.2c0,0,2.7-1.4,3.4-1.9c2.2-1.6,3-2.1,3-5c0-5-2.1-7.2-2.1-7.2 c-3.9-3.3-6.3-8.2-6.3-13.7c0-10,8.1-18.1,18.1-18.1s18.1,8.1,18.1,18.1c0,6-1.5,32.7-2.3,38.8l-0.1,1`\n  );\n  const goose2Path = new Path2D(\n    `M27.4,19L8.2,27.6c0,0-7.3,2.9,2.6,5c6.1,1.3,24,5.9,24,5.9l1,0.3`\n  );\n  const goose3Path = new Path2D(\n    `M63.7,99.6C52.3,99.6,43,90.3,43,78.9s9.3-20.7,20.7-20.7c10.6,0,34.4,0.1,35.8,9`\n  );\n\n  // Canvas setup and reset.\n  let c = canvas.getContext(&quot;2d&quot;);\n  canvas.width = width;\n  canvas.height = height;\n  c.fillStyle = &quot;#000B2A&quot;;\n  c.fillRect(0, 0, canvas.width, canvas.height);\n\n  const getRowCol = cell => [Math.floor(cell / columns), cell % columns];\n\n  const getColor = (mark, opacity = 1) => {\n    if (mark === 1) return `rgba(0,255,255,${opacity})`;\n    if (mark === 2) return `rgba(255,255,255,${opacity})`;\n    return &quot;#fff&quot;;\n  };\n\n  const drawCellCircle = (cell, xFrame = 1, yFrame = 1, radiusOffset = 0) => {\n    const [row, col] = getRowCol(cell);\n    c.arc(\n      xOffset + xFrame * (col * cellSize + cellSize / 2),\n      yOffset + yFrame * (row * cellSize + cellSize / 2),\n      (cellInset * cellSize) / 2 - radiusOffset,\n      2 * Math.PI,\n      false\n    );\n  };\n\n  // Render the pieces.\n  const board = environment.steps[step][0].observation.board;\n\n  const drawPiece = mark => {\n    // Base Styles.\n    const opacity = minCanvasSize < 300 ? 0.6 - minCanvasSize / 1000 : 0.1;\n    c.fillStyle = getColor(mark, opacity);\n    c.strokeStyle = getColor(mark);\n    c.shadowColor = getColor(mark);\n    c.shadowBlur = 8 / cellInset;\n    c.lineWidth = 1 / cellInset;\n\n    // Outer circle.\n    c.save();\n    c.beginPath();\n    c.arc(50, 50, 50, 2 * Math.PI, false);\n    c.closePath();\n    c.lineWidth *= 4;\n    c.stroke();\n    c.fill();\n    c.restore();\n\n    // Inner circle.\n    c.beginPath();\n    c.arc(50, 50, 40, 2 * Math.PI, false);\n    c.closePath();\n    c.stroke();\n\n    // Kaggle &quot;K&quot;.\n    if (mark === 1) {\n      const scale = 0.54;\n      c.save();\n      c.translate(23, 23);\n      c.scale(scale, scale);\n      c.lineWidth /= scale;\n      c.shadowBlur /= scale;\n      c.stroke(kPath);\n      c.restore();\n    }\n\n    // Kaggle &quot;Goose&quot;.\n    if (mark === 2) {\n      const scale = 0.6;\n      c.save();\n      c.translate(24, 28);\n      c.scale(scale, scale);\n      c.lineWidth /= scale;\n      c.shadowBlur /= scale;\n      c.stroke(goose1Path);\n      c.stroke(goose2Path);\n      c.stroke(goose3Path);\n      c.beginPath();\n      c.arc(38.5, 18.6, 2.7, 0, Math.PI * 2, false);\n      c.closePath();\n      c.fill();\n      c.restore();\n    }\n  };\n\n  for (let i = 0; i < board.length; i++) {\n    const [row, col] = getRowCol(i);\n    if (board[i] === 0) continue;\n    // Easing In.\n    let yFrame = Math.min(\n      (columns * Math.pow(frame, 3)) / Math.floor(i / columns),\n      1\n    );\n\n    if (\n      step > 1 &&\n      environment.steps[step - 1][0].observation.board[i] === board[i]\n    ) {\n      yFrame = 1;\n    }\n\n    c.save();\n    c.translate(\n      xOffset + cellSize * col + (cellSize - cellSize * cellInset) / 2,\n      yOffset +\n        yFrame * (cellSize * row) +\n        (cellSize - cellSize * cellInset) / 2\n    );\n    c.scale(pieceScale * cellInset, pieceScale * cellInset);\n    drawPiece(board[i]);\n    c.restore();\n  }\n\n  // Background Gradient.\n  const bgRadius = (Math.min(rows, columns) * cellSize) / 2;\n  const bgStyle = c.createRadialGradient(\n    xOffset + (cellSize * columns) / 2,\n    yOffset + (cellSize * rows) / 2,\n    0,\n    xOffset + (cellSize * columns) / 2,\n    yOffset + (cellSize * rows) / 2,\n    bgRadius\n  );\n  bgStyle.addColorStop(0, &quot;#000B49&quot;);\n  bgStyle.addColorStop(1, &quot;#000B2A&quot;);\n\n  // Render the board overlay.\n  c.beginPath();\n  c.rect(0, 0, canvas.width, canvas.height);\n  c.closePath();\n  c.shadowBlur = 0;\n  for (let i = 0; i < board.length; i++) {\n    drawCellCircle(i);\n    c.closePath();\n  }\n  c.fillStyle = bgStyle;\n  c.fill(&quot;evenodd&quot;);\n\n  // Render the board overlay cell outlines.\n  for (let i = 0; i < board.length; i++) {\n    c.beginPath();\n    drawCellCircle(i);\n    c.strokeStyle = &quot;#0361B2&quot;;\n    c.lineWidth = 1;\n    c.stroke();\n    c.closePath();\n  }\n\n  const drawLine = (fromCell, toCell) => {\n    if (frame < 0.5) return;\n    const lineFrame = (frame - 0.5) / 0.5;\n    const x1 = xOffset + (fromCell % columns) * cellSize + cellSize / 2;\n    const x2 =\n      x1 +\n      lineFrame *\n        (xOffset + ((toCell % columns) * cellSize + cellSize / 2) - x1);\n    const y1 =\n      yOffset + Math.floor(fromCell / columns) * cellSize + cellSize / 2;\n    const y2 =\n      y1 +\n      lineFrame *\n        (yOffset + Math.floor(toCell / columns) * cellSize + cellSize / 2 - y1);\n    c.beginPath();\n    c.lineCap = &quot;round&quot;;\n    c.lineWidth = 4;\n    c.strokeStyle = getColor(board[fromCell]);\n    c.shadowBlur = 8;\n    c.shadowColor = getColor(board[fromCell]);\n    c.moveTo(x1, y1);\n    c.lineTo(x2, y2);\n    c.stroke();\n  };\n\n  // Generate a graph of the board.\n  const getCell = (cell, rowOffset, columnOffset) => {\n    const row = Math.floor(cell / columns) + rowOffset;\n    const col = (cell % columns) + columnOffset;\n    if (row < 0 || row >= rows || col < 0 || col >= columns) return -1;\n    return col + row * columns;\n  };\n  const makeNode = cell => {\n    const node = { cell, directions: [], value: board[cell] };\n    for (let r = -1; r <= 1; r++) {\n      for (let c = -1; c <= 1; c++) {\n        if (r === 0 && c === 0) continue;\n        node.directions.push(getCell(cell, r, c));\n      }\n    }\n    return node;\n  };\n  const graph = board.map((_, i) => makeNode(i));\n\n  // Check for any wins!\n  const getSequence = (node, direction) => {\n    const sequence = [node.cell];\n    while (sequence.length < inarow) {\n      const next = graph[node.directions[direction]];\n      if (!next || node.value !== next.value || next.value === 0) return;\n      node = next;\n      sequence.push(node.cell);\n    }\n    return sequence;\n  };\n\n  // Check all nodes.\n  for (let i = 0; i < board.length; i++) {\n    // Check all directions (not the most efficient).\n    for (let d = 0; d < 8; d++) {\n      const seq = getSequence(graph[i], d);\n      if (seq) {\n        drawLine(seq[0], seq[inarow - 1]);\n        i = board.length;\n        break;\n      }\n    }\n  }\n\n  // Upgrade the legend.\n  if (agents.length && (!agents[0].color || !agents[0].image)) {\n    const getPieceImage = mark => {\n      const pieceCanvas = document.createElement(&quot;canvas&quot;);\n      parent.appendChild(pieceCanvas);\n      pieceCanvas.style.marginLeft = &quot;10000px&quot;;\n      pieceCanvas.width = 100;\n      pieceCanvas.height = 100;\n      c = pieceCanvas.getContext(&quot;2d&quot;);\n      c.translate(10, 10);\n      c.scale(0.8, 0.8);\n      drawPiece(mark);\n      const dataUrl = pieceCanvas.toDataURL();\n      parent.removeChild(pieceCanvas);\n      return dataUrl;\n    };\n\n    agents.forEach(agent => {\n      agent.color = getColor(agent.index + 1);\n      agent.image = getPieceImage(agent.index + 1);\n    });\n    update({ agents });\n  }\n};\n\n\n    \n    </script>\n    <script>\n      const h = htm.bind(preact.h);\n      const { useContext, useEffect, useRef, useState } = preactHooks;\n      const styled = window.styled.default;\n\n      const Context = preact.createContext({});\n\n      const Loading = styled.div`\n        animation: rotate360 1.1s infinite linear;\n        border: 8px solid rgba(255, 255, 255, 0.2);\n        border-left-color: #0cb1ed;\n        border-radius: 50%;\n        height: 40px;\n        position: relative;\n        transform: translateZ(0);\n        width: 40px;\n\n        @keyframes rotate360 {\n          0% {\n            transform: rotate(0deg);\n          }\n          100% {\n            transform: rotate(360deg);\n          }\n        }\n      `;\n\n      const Logo = styled(\n        (props) => h`\n        <a href=&quot;https://kaggle.com&quot; target=&quot;_blank&quot; className=${props.className}>\n          <svg width=&quot;62px&quot; height=&quot;20px&quot; viewBox=&quot;0 0 62 24&quot; version=&quot;1.1&quot; xmlns=&quot;http://www.w3.org/2000/svg&quot;>\n            <g fill=&quot;#1EBEFF&quot; fill-rule=&quot;nonzero&quot;>\n              <path d=&quot;M10.2,17.8c0,0.1-0.1,0.1-0.2,0.1H7.7c-0.1,0-0.3-0.1-0.4-0.2l-3.8-4.9l-1.1,1v3.8 c0,0.2-0.1,0.3-0.3,0.3H0.3c-0.2,0-0.3-0.1-0.3-0.3V0.3C0.1,0.1,0.2,0,0.3,0h1.8c0.2,0,0.3,0.1,0.3,0.3V11L7,6.3 c0.1-0.1,0.2-0.2,0.4-0.2h2.4c0.1,0,0.2,0,0.2,0.1c0,0.1,0,0.2,0,0.2l-4.9,4.7l5.1,6.3C10.2,17.6,10.2,17.7,10.2,17.8z&quot;/>\n              <path d=&quot;M19.6,17.9h-1.8c-0.2,0-0.3-0.1-0.3-0.3v-0.4c-0.8,0.6-1.8,0.9-3,0.9c-1.1,0-2-0.3-2.8-1 c-0.8-0.7-1.2-1.6-1.2-2.7c0-1.7,1.1-2.9,3.2-3.5c0.8-0.2,2.1-0.5,3.8-0.6c0.1-0.6-0.1-1.2-0.5-1.7c-0.4-0.5-1-0.7-1.7-0.7 c-1,0-2,0.4-3,1C12.2,9.1,12.1,9.1,12,9l-0.9-1.3C11,7.5,11,7.4,11.1,7.3c1.3-0.9,2.7-1.4,4.2-1.4c1.1,0,2.1,0.3,2.8,0.8 c1.1,0.8,1.7,2,1.7,3.7v7.3C19.9,17.8,19.8,17.9,19.6,17.9z M17.5,12.4c-1.7,0.2-2.9,0.4-3.5,0.7c-0.9,0.4-1.2,0.9-1.1,1.6 c0.1,0.4,0.2,0.7,0.6,0.9c0.3,0.2,0.7,0.4,1.1,0.4c1.2,0.1,2.2-0.2,2.9-1V12.4z&quot;/>\n              <path d=&quot;M30.6,22.5c-0.9,1-2.3,1.5-4,1.5c-1,0-2-0.3-2.9-0.8c-0.2-0.1-0.4-0.3-0.7-0.5 c-0.3-0.2-0.6-0.5-0.9-0.7c-0.1-0.1-0.1-0.2,0-0.4l1.2-1.2c0.1-0.1,0.1-0.1,0.2-0.1c0.1,0,0.1,0,0.2,0.1c1,1,1.9,1.5,2.8,1.5 c2.1,0,3.2-1.1,3.2-3.3v-1.4c-0.8,0.7-1.9,1-3.3,1c-1.7,0-3-0.6-4-1.9c-0.8-1.1-1.3-2.5-1.3-4.2c0-1.6,0.4-3,1.2-4.1 c0.9-1.3,2.3-2,4-2c1.3,0,2.4,0.3,3.3,1V6.4c0-0.2,0.1-0.3,0.3-0.3h1.8c0.2,0,0.3,0.1,0.3,0.3v11.7C32,20,31.5,21.5,30.6,22.5z M29.7,9.9c-0.4-1.1-1.4-1.7-3-1.7c-2,0-3.1,1.3-3.1,3.8c0,1.4,0.3,2.4,1,3.1c0.5,0.5,1.2,0.8,2,0.8c1.6,0,2.7-0.6,3.1-1.7V9.9z&quot;/>\n              <path d=&quot;M42.9,22.5c-0.9,1-2.3,1.5-4,1.5c-1,0-2-0.3-2.9-0.8c-0.2-0.1-0.4-0.3-0.7-0.5 c-0.3-0.2-0.6-0.5-0.9-0.7c-0.1-0.1-0.1-0.2,0-0.4l1.2-1.2c0.1-0.1,0.1-0.1,0.2-0.1c0.1,0,0.1,0,0.2,0.1c1,1,1.9,1.5,2.8,1.5 c2.1,0,3.2-1.1,3.2-3.3v-1.4c-0.8,0.7-1.9,1-3.3,1c-1.7,0-3-0.6-4-1.9c-0.8-1.1-1.3-2.5-1.3-4.2c0-1.6,0.4-3,1.2-4.1 c0.9-1.3,2.3-2,4-2c1.3,0,2.4,0.3,3.3,1V6.4c0-0.2,0.1-0.3,0.3-0.3H44c0.2,0,0.3,0.1,0.3,0.3v11.7C44.3,20,43.8,21.5,42.9,22.5z M42,9.9c-0.4-1.1-1.4-1.7-3-1.7c-2,0-3.1,1.3-3.1,3.8c0,1.4,0.3,2.4,1,3.1c0.5,0.5,1.2,0.8,2,0.8c1.6,0,2.7-0.6,3.1-1.7L42,9.9 L42,9.9z&quot;/>\n              <path d=&quot;M48.3,17.9h-1.8c-0.2,0-0.3-0.1-0.3-0.3V0.3c0-0.2,0.1-0.3,0.3-0.3h1.8c0.2,0,0.3,0.1,0.3,0.3 v17.3C48.5,17.8,48.5,17.9,48.3,17.9z&quot;/>\n              <path d=&quot;M61.4,12.6c0,0.2-0.1,0.3-0.3,0.3h-8.5c0.1,0.9,0.5,1.6,1.1,2.2c0.7,0.6,1.6,0.9,2.7,0.9 c1,0,1.8-0.3,2.6-0.8c0.2-0.1,0.3-0.1,0.4,0l1.2,1.3c0.1,0.1,0.1,0.3,0,0.4c-1.3,0.9-2.7,1.4-4.4,1.4c-1.8,0-3.3-0.6-4.4-1.8 c-1.1-1.2-1.7-2.7-1.7-4.5c0-1.7,0.6-3.2,1.7-4.4c1-1.1,2.4-1.6,4.1-1.6c1.6,0,2.9,0.6,4,1.7c1.1,1.2,1.6,2.6,1.5,4.4L61.4,12.6 z M58,8.7c-0.6-0.5-1.3-0.8-2.1-0.8c-0.8,0-1.5,0.3-2.1,0.8c-0.6,0.5-1,1.2-1.1,2H59C59,9.9,58.6,9.3,58,8.7z&quot;/>\n            </g>\n          </svg>\n        </a>\n      `\n      )`\n        display: inline-flex;\n      `;\n\n      const Header = styled((props) => {\n        const { environment } = useContext(Context);\n\n        return h`<div className=${props.className} >\n          <${Logo} />\n          <span><b>Left / Right Arrow:</b> Increase / Decrease Step</span><span><b>0-9 Row Keys:</b> Playback Speed</span><span><b>Space:</b> Pause / Play</span>\n          ${environment.title}\n        </div>`;\n      })`\n        align-items: center;\n        border-bottom: 4px solid #212121;\n        box-sizing: border-box;\n        color: #fff;\n        display: flex;\n        flex: 0 0 36px;\n        font-size: 14px;\n        justify-content: space-between;\n        padding: 0 8px;\n        width: 100%;\n      `;\n\n      const Renderer = styled((props) => {\n        const context = useContext(Context);\n        const { animate, debug, playing, renderer, speed } = context;\n        const ref = preact.createRef();\n\n        useEffect(async () => {\n          if (!ref.current) return;\n\n          const renderFrame = async (start, step, lastFrame) => {\n            if (step !== context.step) return;\n            if (lastFrame === 1) {\n              if (!animate) return;\n              start = Date.now();\n            }\n            const frame =\n              playing || animate\n                ? Math.min((Date.now() - start) / speed, 1)\n                : 1;\n            try {\n              if (debug) console.time(&quot;render&quot;);\n              await renderer({\n                ...context,\n                frame,\n                height: ref.current.clientHeight,\n                hooks: preactHooks,\n                parent: ref.current,\n                preact,\n                styled,\n                width: ref.current.clientWidth,\n              });\n            } catch (error) {\n              if (debug) console.error(error);\n              console.log({ ...context, frame, error });\n            } finally {\n              if (debug) console.timeEnd(&quot;render&quot;);\n            }\n            window.requestAnimationFrame(() => renderFrame(start, step, frame));\n          };\n\n          await renderFrame(Date.now(), context.step);\n        }, [ref.current, context.step, context.renderer]);\n\n        return h`<div className=${props.className} ref=${ref} />`;\n      })`\n        align-items: center;\n        box-sizing: border-box;\n        display: flex;\n        height: 100%;\n        left: 0;\n        justify-content: center;\n        position: absolute;\n        top: 0;\n        width: 100%;\n      `;\n\n      const Processing = styled((props) => {\n        const { processing } = useContext(Context);\n        const text = processing === true ? &quot;Processing...&quot; : processing;\n        return h`<div className=${props.className}>${text}</div>`;\n      })`\n        bottom: 0;\n        color: #fff;\n        font-size: 12px;\n        left: 0;\n        line-height: 24px;\n        position: absolute;\n        text-align: center;\n        width: 100%;\n      `;\n\n      const Viewer = styled((props) => {\n        const { processing } = useContext(Context);\n        return h`<div className=${props.className}>\n          <${Renderer} />\n          ${processing && h`<${Processing} />`}\n        </div>`;\n      })`\n        background-color: #000b2a;\n        background-image: radial-gradient(\n          circle closest-side,\n          #000b49,\n          #000b2a\n        );\n        display: flex;\n        flex: 1;\n        overflow: hidden;\n        position: relative;\n        width: 100%;\n      `;\n\n      // Partitions the elements of arr into subarrays of max length num.\n      const groupIntoSets = (arr, num) => {\n        const sets = [];\n        arr.forEach(a => {\n          if (sets.length === 0 || sets[sets.length - 1].length === num) {\n            sets.push([]);\n          }\n          sets[sets.length - 1].push(a);\n        });\n        return sets;\n      }\n\n      // Expects `width` input prop to set proper max-width for agent name span.\n      const Legend = styled((props) => {\n        const { agents, legend } = useContext(Context);\n\n        const agentPairs = groupIntoSets(agents.sort((a, b) => a.index - b.index), 2);\n\n        return h`<div className=${props.className}>\n          ${agentPairs.map(agentList =>\n            h`<ul>\n                ${agentList.map(a =>\n                  h`<li key=${a.id} title=&quot;id: ${a.id}&quot; style=&quot;color:${a.color || &quot;#FFF&quot;}&quot;>\n                      ${a.image && h`<img src=${a.image} />`}\n                      <span>${a.name}</span>\n                    </li>`\n                )}\n              </ul>`)}\n        </div>`;\n      })`\n        background-color: #000b2a;\n        font-family: sans-serif;\n        font-size: 14px;\n        width: 100%;\n\n        ul {\n          align-items: center;\n          display: flex;\n          flex-direction: row;\n          justify-content: center;\n        }\n\n        li {\n          align-items: center;\n          display: inline-flex;\n          transition: color 1s;\n        }\n\n        span {\n          max-width: ${p => (p.width || 400) * 0.5 - 36}px;\n          overflow: hidden;\n          text-overflow: ellipsis;\n          white-space: nowrap;\n        }\n\n        img {\n          height: 24px;\n          margin-left: 4px;\n          margin-right: 4px;\n          width: 24px;\n        }\n      `;\n\n      const StepInput = styled.input.attrs({\n        type: &quot;range&quot;,\n      })`\n        appearance: none;\n        background: rgba(255, 255, 255, 0.15);\n        border-radius: 2px;\n        display: block;\n        flex: 1;\n        height: 4px;\n        opacity: 0.8;\n        outline: none;\n        transition: opacity 0.2s;\n        width: 100%;\n\n        &:hover {\n          opacity: 1;\n        }\n\n        &::-webkit-slider-thumb {\n          appearance: none;\n          background: #1ebeff;\n          border-radius: 100%;\n          cursor: pointer;\n          height: 12px;\n          margin: 0;\n          position: relative;\n          width: 12px;\n\n          &::after {\n            content: &quot;&quot;;\n            position: absolute;\n            top: 0px;\n            left: 0px;\n            width: 200px;\n            height: 8px;\n            background: green;\n          }\n        }\n      `;\n\n      const PlayButton = styled.button`\n        align-items: center;\n        background: none;\n        border: none;\n        color: white;\n        cursor: pointer;\n        display: flex;\n        flex: 0 0 56px;\n        font-size: 20px;\n        height: 40px;\n        justify-content: center;\n        opacity: 0.8;\n        outline: none;\n        transition: opacity 0.2s;\n\n        &:hover {\n          opacity: 1;\n        }\n      `;\n\n      const StepCount = styled.span`\n        align-items: center;\n        color: white;\n        display: flex;\n        font-size: 14px;\n        justify-content: center;\n        opacity: 0.8;\n        padding: 0 16px;\n        pointer-events: none;\n      `;\n\n      const Controls = styled((props) => {\n        const { environment, pause, play, playing, setStep, step } = useContext(\n          Context\n        );\n        const value = step + 1;\n        const onClick = () => (playing ? pause() : play());\n        const onInput = (e) => {\n          pause();\n          setStep(parseInt(e.target.value) - 1);\n        };\n\n        return h`\n          <div className=${props.className}>\n            <${PlayButton} onClick=${onClick}><svg xmlns=&quot;http://www.w3.org/2000/svg&quot; width=&quot;24px&quot; height=&quot;24px&quot; viewBox=&quot;0 0 24 24&quot; fill=&quot;#FFFFFF&quot;>${\n          playing\n            ? h`<path d=&quot;M6 19h4V5H6v14zm8-14v14h4V5h-4z&quot;/><path d=&quot;M0 0h24v24H0z&quot; fill=&quot;none&quot;/>`\n            : h`<path d=&quot;M8 5v14l11-7z&quot;/><path d=&quot;M0 0h24v24H0z&quot; fill=&quot;none&quot;/>`\n        }</svg><//>\n            <${StepInput} min=&quot;1&quot; max=${\n          environment.steps.length\n        } value=&quot;${value}&quot; onInput=${onInput} />\n            <${StepCount}>${value} / ${environment.steps.length}<//>\n          </div>\n        `;\n      })`\n        align-items: center;\n        border-top: 4px solid #212121;\n        display: flex;\n        flex: 0 0 44px;\n        width: 100%;\n      `;\n\n      const Info = styled((props) => {\n        const {\n          environment,\n          playing,\n          step,\n          speed,\n          animate,\n          header,\n          controls,\n          settings,\n        } = useContext(Context);\n\n        return h`\n          <div className=${props.className}>\n            info:\n            step(${step}),\n            playing(${playing ? &quot;T&quot; : &quot;F&quot;}),\n            speed(${speed}),\n            animate(${animate ? &quot;T&quot; : &quot;F&quot;})\n          </div>`;\n      })`\n        color: #888;\n        font-family: monospace;\n        font-size: 12px;\n      `;\n\n      const Settings = styled((props) => {\n        const { environment, pause, play, playing, setStep, step } = useContext(\n          Context\n        );\n\n        return h`\n          <div className=${props.className}>\n            <${Info} />\n          </div>\n        `;\n      })`\n        background: #fff;\n        border-top: 4px solid #212121;\n        box-sizing: border-box;\n        padding: 20px;\n        width: 100%;\n\n        h1 {\n          font-size: 20px;\n        }\n      `;\n\n      const Player = styled((props) => {\n        const context = useContext(Context);\n        const { agents, controls, header, legend, loading, settings, width } = context;\n        return h`\n          <div className=${props.className}>\n            ${loading && h`<${Loading} />`}\n            ${!loading && header && h`<${Header} />`}\n            ${!loading && h`<${Viewer} />`}\n            ${!loading && agents.length > 0 && legend && h`<${Legend} width=${width}/>`}\n            ${!loading && controls && h`<${Controls} />`}\n            ${!loading && settings && h`<${Settings} />`}\n          </div>`;\n      })`\n        align-items: center;\n        background: #212121;\n        border: 4px solid #212121;\n        box-sizing: border-box;\n        display: flex;\n        flex-direction: column;\n        height: 100%;\n        justify-content: center;\n        position: relative;\n        width: 100%;\n      `;\n\n      const App = () => {\n        const renderCountRef = useRef(0);\n        const [_, setRenderCount] = useState(0);\n\n        const contextRef = useRef({\n          animate: false,\n          agents: [],\n          autoplay: false,\n          controls: false,\n          debug: false,\n          environment: { steps: [] },\n          header: window.innerHeight >= 600,\n          height: window.innerHeight,\n          interactive: false,\n          legend: true,\n          loading: false,\n          playing: false,\n          processing: false,\n          renderer: () => &quot;DNE&quot;,\n          settings: false,\n          speed: 4,\n          step: 0,\n          width: window.innerWidth,\n        });\n\n        // Context helpers.\n        const rerender = (contextRef.current.rerender = () =>\n          setRenderCount((renderCountRef.current += 1)));\n        const setStep = (contextRef.current.setStep = (newStep) => {\n          contextRef.current.step = newStep;\n          rerender();\n        });\n        const setPlaying = (contextRef.current.setPlaying = (playing) => {\n          contextRef.current.playing = playing;\n          rerender();\n        });\n        const pause = (contextRef.current.pause = () => setPlaying(false));\n\n        const playNext = () => {\n          const context = contextRef.current;\n\n          if (\n            context.playing &&\n            context.step < context.environment.steps.length - 1\n          ) {\n            setStep(context.step + 1);\n            play(true);\n          } else {\n            pause();\n          }\n        };\n\n        // These are bindings to the 0-9 keys and are milliseconds of timeout per step\n        const speeds = [\n          0,\n          3000,\n          1000,\n          500,\n          333, // Default\n          200,\n          100,\n          50,\n          25,\n          10,\n        ];\n\n        const play = (contextRef.current.play = (continuing) => {\n          const context = contextRef.current;\n          if (context.playing && !continuing) return;\n          if (!context.playing) setPlaying(true);\n          if (\n            !continuing &&\n            context.step === context.environment.steps.length - 1\n          ) {\n            setStep(0);\n          }\n          setTimeout(playNext, speeds[context.speed]);\n        });\n\n        const updateContext = (o) => {\n          const context = contextRef.current;\n          Object.assign(context, o, {\n            environment: { ...context.environment, ...(o.environment || {}) },\n          });\n          rerender();\n\n          // If autoplay, toggle to playing.\n          if (context.autoplay) play();\n        };\n\n        // First time setup.\n        useEffect(() => {\n          // Timeout is used to ensure useEffect renders once.\n          setTimeout(() => {\n            // Initialize context with window.kaggle.\n            updateContext(window.kaggle || {});\n            // Listen for messages received to update the context.\n            window.addEventListener(\n              &quot;message&quot;,\n              (event) => {\n                // Ensure the environment names match before updating.\n                try {\n                  if (\n                    event.data.environment.name ==\n                    contextRef.current.environment.name\n                  ) {\n                    updateContext(event.data);\n                  }\n                } catch {}\n              },\n              false\n            );\n            // Listen for keyboard commands.\n            window.addEventListener(\n              &quot;keydown&quot;,\n              (event) => {\n                const {\n                  interactive,\n                  isInteractive,\n                  playing,\n                  step,\n                  environment,\n                } = contextRef.current;\n                const key = event.keyCode;\n                const zero_key = 48\n                const nine_key = 57\n                if (\n                  interactive ||\n                  isInteractive() ||\n                  (key !== 32 && key !== 37 && key !== 39 && !(key >= zero_key && key <= nine_key))\n                )\n                  return;\n\n                if (key === 32) {\n                  playing ? pause() : play();\n                } else if (key === 39) {\n                  contextRef.current.playing = false;\n                  if (step < environment.steps.length - 1) setStep(step + 1);\n                  rerender();\n                } else if (key === 37) {\n                  contextRef.current.playing = false;\n                  if (step > 0) setStep(step - 1);\n                  rerender();\n                } else if (key >= zero_key && key <= nine_key) {\n                  contextRef.current.speed = key - zero_key;\n                }\n                event.preventDefault();\n                return false;\n              },\n              false\n            );\n          }, 1);\n        }, []);\n\n        if (contextRef.current.debug) {\n          console.log(&quot;context&quot;, contextRef.current);\n        }\n\n        // Ability to update context.\n        contextRef.current.update = updateContext;\n\n        // Ability to communicate with ipython.\n        const execute = (contextRef.current.execute = (source) =>\n          new Promise((resolve, reject) => {\n            try {\n              window.parent.IPython.notebook.kernel.execute(source, {\n                iopub: {\n                  output: (resp) => {\n                    const type = resp.msg_type;\n                    if (type === &quot;stream&quot;) return resolve(resp.content.text);\n                    if (type === &quot;error&quot;) return reject(new Error(resp.evalue));\n                    return reject(new Error(&quot;Unknown message type: &quot; + type));\n                  },\n                },\n              });\n            } catch (e) {\n              reject(new Error(&quot;IPython Unavailable: &quot; + e));\n            }\n          }));\n\n        // Ability to return an action from an interactive session.\n        contextRef.current.act = (action) => {\n          const id = contextRef.current.environment.id;\n          updateContext({ processing: true });\n          execute(`\n            import json\n            from kaggle_environments import interactives\n            if &quot;${id}&quot; in interactives:\n                action = json.loads('${JSON.stringify(action)}')\n                env, trainer = interactives[&quot;${id}&quot;]\n                trainer.step(action)\n                print(json.dumps(env.steps))`)\n            .then((resp) => {\n              try {\n                updateContext({\n                  processing: false,\n                  environment: { steps: JSON.parse(resp) },\n                });\n                play();\n              } catch (e) {\n                updateContext({ processing: resp.split(&quot;\\n&quot;)[0] });\n                console.error(resp, e);\n              }\n            })\n            .catch((e) => console.error(e));\n        };\n\n        // Check if currently interactive.\n        contextRef.current.isInteractive = () => {\n          const context = contextRef.current;\n          const steps = context.environment.steps;\n          return (\n            context.interactive &&\n            !context.processing &&\n            context.step === steps.length - 1 &&\n            steps[context.step].some((s) => s.status === &quot;ACTIVE&quot;)\n          );\n        };\n\n        return h`\n          <${Context.Provider} value=${contextRef.current}>\n            <${Player} />\n          <//>`;\n      };\n\n      preact.render(h`<${App} />`, document.body);\n    </script>\n  </body>\n</html>\n\" width=\"500\" height=\"450\" frameborder=\"0\"></iframe> "
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "env.reset()\n",
    "\n",
    "print(f'{player_1} vs {player_2}')\n",
    "env.run([ agents[player_2], agents[player_1] ])\n",
    "\n",
    "print('-'*20 + f'\\n{player_1} vs {player_2}')\n",
    "env.render(mode=\"ipython\", width=500, height=450)\n",
    "print(f\"scores: {player_1} = {env.state[0].reward} | {player_2} = {env.state[1].reward}\")"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "outputs": [],
   "source": [],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  }
 ],
 "metadata": {
  "kernelspec": {
   "name": "python3",
   "language": "python",
   "display_name": "Python 3"
  },
  "pycharm": {
   "stem_cell": {
    "cell_type": "raw",
    "source": [],
    "metadata": {
     "collapsed": false
    }
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}