DRL_RSMA_Radar.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "DRL-RSMA-Radar.ipynb",
      "provenance": [],
      "collapsed_sections": [
        "mpllzjbjn6lb",
        "DNgW_zzrn_3y",
        "Xr5CEJAANWY6",
        "5xUhLjdsNRlC",
        "vMKb-zkdkz1F",
        "7p0roQPkH46C"
      ]
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "source": [
        "# Initialization"
      ],
      "metadata": {
        "id": "mpllzjbjn6lb"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/drive')\n",
        "%cd /content/drive/Shareddrives/DuongLuongHoa-BKHN/DRL-RSMA-Radar\n",
        "!ls"
      ],
      "metadata": {
        "id": "j4wirn3GgNvw",
        "outputId": "0ab33891-7b5a-492e-bde5-e6177a019342",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Drive already mounted at /content/drive; to attempt to forcibly remount, call drive.mount(\"/content/drive\", force_remount=True).\n",
            "/content/drive/Shareddrives/DuongLuongHoa-BKHN/DRL-RSMA-Radar\n",
            "'Copy of DRL-RSMA-Radar.ipynb'\t DRL-RSMA-Radar.ipynb\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import sys\n",
        "IN_COLAB = \"google.colab\" in sys.modules\n",
        "\n",
        "if IN_COLAB:\n",
        "    !apt install python-opengl\n",
        "    !apt install ffmpeg\n",
        "    !apt install xvfb\n",
        "    !pip install pyvirtualdisplay\n",
        "    from pyvirtualdisplay import Display\n",
        "    \n",
        "    # Start virtual display\n",
        "    dis = Display(visible=0, size=(600, 400))\n",
        "    dis.start()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "BH0dOjduMJZE",
        "outputId": "fab61ae7-f117-4e39-e85e-ac9c0a786d4d"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Reading package lists... Done\n",
            "Building dependency tree       \n",
            "Reading state information... Done\n",
            "python-opengl is already the newest version (3.1.0+dfsg-1).\n",
            "The following package was automatically installed and is no longer required:\n",
            "  libnvidia-common-460\n",
            "Use 'apt autoremove' to remove it.\n",
            "0 upgraded, 0 newly installed, 0 to remove and 42 not upgraded.\n",
            "Reading package lists... Done\n",
            "Building dependency tree       \n",
            "Reading state information... Done\n",
            "ffmpeg is already the newest version (7:3.4.8-0ubuntu0.2).\n",
            "The following package was automatically installed and is no longer required:\n",
            "  libnvidia-common-460\n",
            "Use 'apt autoremove' to remove it.\n",
            "0 upgraded, 0 newly installed, 0 to remove and 42 not upgraded.\n",
            "Reading package lists... Done\n",
            "Building dependency tree       \n",
            "Reading state information... Done\n",
            "xvfb is already the newest version (2:1.19.6-1ubuntu4.10).\n",
            "The following package was automatically installed and is no longer required:\n",
            "  libnvidia-common-460\n",
            "Use 'apt autoremove' to remove it.\n",
            "0 upgraded, 0 newly installed, 0 to remove and 42 not upgraded.\n",
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Requirement already satisfied: pyvirtualdisplay in /usr/local/lib/python3.7/dist-packages (3.0)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Library definition"
      ],
      "metadata": {
        "id": "DNgW_zzrn_3y"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# Fundamental\n",
        "import numpy as np\n",
        "\n",
        "import math\n",
        "import copy\n",
        "import random\n",
        "from typing import Dict, List, Tuple\n",
        "\n",
        "import scipy\n",
        "from scipy import special\n",
        "from scipy.special import lambertw\n",
        "\n",
        "import matplotlib.pyplot as plt\n",
        "\n",
        "# Pytorch\n",
        "\n",
        "import torch\n",
        "import torch.nn as nn\n",
        "import torch.nn.functional as F\n",
        "import torch.optim as optim\n",
        "\n",
        "dtype = np.float32\n",
        "\n",
        "\n",
        "from IPython.display import clear_output"
      ],
      "metadata": {
        "id": "5Gf0XAdGoDms"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# GPU \n",
        "if torch.backends.cudnn.enabled:\n",
        "    torch.backends.cudnn.benchmark = False\n",
        "    torch.backends.cudnn.deterministic = True\n",
        "    print(\"GPU enabled\")\n",
        "\n",
        "seed = 777\n",
        "torch.manual_seed(seed)\n",
        "np.random.seed(seed)\n",
        "random.seed(seed)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "WizJlJScMDRK",
        "outputId": "d87bd42b-4a01-4ebf-8afd-b67fca4cbf11"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "GPU enabled\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Environment "
      ],
      "metadata": {
        "id": "bRzGOQ-xn-pP"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Definition of Channel\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "\n"
      ],
      "metadata": {
        "id": "ek73oLa8bjjE"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Code"
      ],
      "metadata": {
        "id": "fGVutbBQesxr"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "### CNUR settings\n",
        "Here is a CNUR settings\n",
        "\n",
        "| Notations            | Value        ||| Notations            | Value        | \n",
        "|---                   |---           ||| ---                  | ---          | \n",
        "|Number of CU (Q)      | 10 - 20      ||| Antenna Gain T BS    |  17dBi~50    | \n",
        "|Wave length (lambda)  | 0.1m         ||| Antenna Gain R BS    |   0dBi~1     | \n",
        "|BS to CU              | 200-300m     ||| Antanna Gain Radar   |  30dBi~1000  | \n",
        "|Radar to CU           | 1000-2000m   ||| G'^R_t               | -27dBi~0.002 | \n",
        "|Radar to UAV          | 5000-10000m  ||| G'^R_r               | -27dBi~0.002 | \n",
        "|P_BS                  | 30 dBm / 1W  ||| sigma_RCS            | 1m^2         | \n",
        "|P_radar               | 1000W        ||| sigma^2_q, sigma^2   | -150dBm/Hz   | |B                     | 10^6         ||| C^TH                 | 10^5 - 4*10^5| "
      ],
      "metadata": {
        "id": "A1Hu2XGdiQF1"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "GwXtcdjfejiX"
      },
      "outputs": [],
      "source": [
        "class Env_CNUR():\n",
        "    ### Initialization ###\n",
        "    def __init__(self, MAX_EP_STEPS, N_user_max,   \n",
        "                 BS_R_min, BS_R_max, radar_R,           # Radius of BS and Radar distance to BS\n",
        "                 h_max, h_min,                          # Height of the UAV\n",
        "                 uav_R_max, uav_R_min,                  # Radius of the UAV\n",
        "                #  Radar_R_max,                         # Radius of the Radar\n",
        "                 C_TH,                                  # Datarate threshold\n",
        "                 P_BS_max, P_radar_max,                 # Power\n",
        "                 Bandwidth, noise):                     # Data Rate, Bandwidth\n",
        "        # Visualization\n",
        "        self.visualization = \"2D\"          # 3D or else\n",
        "        # Data logging\n",
        "        self.verbose_distance = False\n",
        "        self.verbose_channelGain = False\n",
        "        self.verbose_SINR = False\n",
        "        self.verbose_DataRate = False\n",
        "\n",
        "        # Network settings\n",
        "        self.BS_R_max = BS_R_max           # Base Station maximum radius\n",
        "        self.BS_R_min = BS_R_min           # Base Station minimum radius\n",
        "        self.N_User = N_user_max           # Number of User / m\n",
        "        self.P_BS_max  = P_BS_max          # Base Station 's max power / m\n",
        "        self.noise  = noise                # Noise density: -174 dBm/Hz\n",
        "        self.B      = Bandwidth            # Bandwidth\n",
        "        self.C_TH   = C_TH                 # Lower threshold for data rate - CU\n",
        "\n",
        "        self.n_0 = noise**2 * self.B       # Noise power     \n",
        "        self.lambda_c = 0.1                #   \n",
        "        # Power setting\n",
        "        self.P_CU   = 1                    # 1W\n",
        "        self.P_R    = 1000                 # P_radar_max = 1000\n",
        "        self.P_BS_0 = np.random.uniform(self.P_BS_max/2, self.P_BS_max)\n",
        "        self.P_BS_U = (self.P_BS_max / self.N_User)* np.ones((self.N_User, 1))        \n",
        "\n",
        "        # Antenna Gain\n",
        "        self.G_R_t = 1000                               # Transmitting Radar Gain\n",
        "        self.G_R_r = 1000                               # Transmitting Radar Gain\n",
        "        self.G_R_t_s = 0.002\n",
        "        self.G_R_r_s = 0.002\n",
        "        self.G_C_t = 50                                 # Transmitting BS Gain\n",
        "        self.G_CU_list = np.ones((self.N_User,1))       # Receiving antenna gain of all CUs\n",
        "        self.sigma_RCS = 1                              # Radar coss section of target - Radar\n",
        "        self.sigma2 = -10**(-18)                        # W/Hz\n",
        "\n",
        "        # Base Station initialization\n",
        "        self.P_max = P_BS_max              # Max power        \n",
        "        self.BS_x = 0                      # BS location initialization \n",
        "        self.BS_y = 0                      # BS location initialization \n",
        "        self.BS_location = np.expand_dims(self._location_BS_Generator(), axis=0)\n",
        "        self.Common_User_Allocation = np.random.rand(self.N_User,1)\n",
        "\n",
        "        # Radar initialization\n",
        "        self.Radar_R = radar_R             # Radius\n",
        "        self.Gamma_R = 10                  # dB ??? \n",
        "        self.Radar_location = np.expand_dims(self._location_Radar_Generator(), axis=0)\n",
        "\n",
        "        # UAV initialization\n",
        "        self.uav_h_max = h_max             # Height max\n",
        "        self.uav_h_min = h_min             # Height min\n",
        "        self.uav_R_max = uav_R_max         # R UAV max \n",
        "        self.uav_R_min = uav_R_min         # R UAV min \n",
        "        self.UAV_location   = np.expand_dims(self._location_UAV_Generator()  , axis=0)\n",
        "        self.UAV_trajectory = np.expand_dims(self._trajectory_UAV_Generator(), axis=0)\n",
        "\n",
        "        # User location initialization\n",
        "        self.CU_location = self._location_CU_Generator()\n",
        "        \n",
        "        # Distance calculation\n",
        "\n",
        "        self.distance_Radar_UAV    = self._calculateDistance(self.UAV_location, self.Radar_location)\n",
        "        self.distance_BS_Radar     = self._calculateDistance(self.UAV_location, self.BS_location)\n",
        "        self.distanceList_BS_CU    = self._calculateDistance(self.BS_location , self.CU_location)\n",
        "        self.distanceList_Radar_CU = self._calculateDistance(self.CU_location , self.Radar_location)\n",
        "        if self.verbose_distance == True:\n",
        "            print(\"Distance\")\n",
        "            print(f\"radar-uav: {self.distance_Radar_UAV}\")\n",
        "            print(f\"radar-bs: {self.distance_BS_Radar}\")\n",
        "            print(f\"bs-cu: {self.distanceList_BS_CU}\")\n",
        "            print(f\"radar-cu: {self.distanceList_Radar_CU}\")\n",
        "         \n",
        "        # Channel Gain\n",
        "        self.H_R  = self._channelGain_Radar_UAV()        # Channel round-trip between Radar and UAV\n",
        "        self.H_CR = self._channelGain_BS_Radar()         # Channel from BS to Radar\n",
        "        self.H_C  = self._channelGain_BS_CU()            # Channel from BS to CU    (Interference)\n",
        "        self.H_RC = self._channelGain_Radar_CU()         # Channel from Radar to CU (Interference)\n",
        "        if self.verbose_channelGain == True:\n",
        "            print(\"Channel Gain\")\n",
        "            print(f\"radar-uav: {self.H_R}\")\n",
        "            print(f\"radar-bs: {self.H_CR}\")\n",
        "            print(f\"bs-cu: {self.H_C}\")\n",
        "            print(f\"radar-cu: {self.H_RC}\")            \n",
        "\n",
        "        # SINR calculation\n",
        "        self.SINR_Radar = self._SINR_Radar(power_radar = self.P_R, \n",
        "                                           power_bs_comm = self.P_BS_0, power_bs_priv = self.P_BS_U,\n",
        "                                           H_radar_uav = self.H_R, H_BS_radar = self.H_CR)\n",
        "        if self.verbose_SINR == True:\n",
        "            print(f\"SINR: {self.SINR_Radar}\")\n",
        "        self.commonDataRate, self.privateDataRate = self._calculateDataRate(self.H_C, self.H_RC)\n",
        "        if 0 == 1:\n",
        "            print(f\"common: {self.commonDataRate}\")\n",
        "            print(f\"private: {self.privateDataRate}\")\n",
        "\n",
        "        # Environment settings\n",
        "        \"state_size can be represented by number of user initialized each episode\"\n",
        "        self.rewardMatrix = np.array([])\n",
        "\n",
        "        self.observation_space = self._wrapState().squeeze()\n",
        "        self.action_space      = self._wrapAction().squeeze()\n",
        "        self.reward_space      = np.array(())\n",
        "\n",
        "    ### Functions ###\n",
        "    # Channel gain generation\n",
        "    def _channelGain_BS_CU(self):\n",
        "        numerator   = self.G_C_t * self.G_CU_list * (self.lambda_c**2)\n",
        "        denominator = ((4*np.pi)**3) * (self.distanceList_BS_CU**4)\n",
        "        channelGain = numerator/denominator \n",
        "        return channelGain\n",
        "\n",
        "    def _channelGain_Radar_CU(self):\n",
        "        numerator   = self.G_R_t * self.G_CU_list * (self.lambda_c**2)\n",
        "        denominator = ((4*np.pi)**3) * (self.distanceList_Radar_CU**4)\n",
        "        channelGain = numerator/denominator \n",
        "        return channelGain\n",
        "\n",
        "    def _channelGain_BS_Radar(self):\n",
        "        numerator   = self.G_C_t * self.G_R_r * (self.lambda_c**2)\n",
        "        denominator = ((4*np.pi)**3) * (self.distance_BS_Radar**4)\n",
        "        channelGain = numerator/denominator \n",
        "        return channelGain\n",
        "\n",
        "    def _channelGain_Radar_UAV(self):            # h^R\n",
        "        numerator   = self.G_R_t * self.G_R_r * self.sigma_RCS * (self.lambda_c**2)\n",
        "        denominator = ((4*np.pi)**3) * (self.distance_Radar_UAV**4)\n",
        "        channelGain = numerator/denominator \n",
        "        return channelGain\n",
        "\n",
        "    # Radar initialization\n",
        "    def _location_BS_Generator(self):\n",
        "        # r = self.BS_R_max * np.sqrt(np.random.rand())\n",
        "        # theta = np.random.uniform(-np.pi, np.pi)\n",
        "        # Radar_x = self.BS_x + r*np.cos(theta)\n",
        "        # Radar_y = self.BS_y + r*np.sin(theta)\n",
        "        BS_location = [self.BS_x, self.BS_y, 0]\n",
        "        return np.array(BS_location)\n",
        "\n",
        "    # Radar initialization\n",
        "    def _location_Radar_Generator(self):\n",
        "        r = self.Radar_R * np.sqrt(np.random.rand())\n",
        "        theta = np.random.uniform(-np.pi, np.pi)\n",
        "        Radar_x = self.BS_x + r*np.cos(theta)\n",
        "        Radar_y = self.BS_y + r*np.sin(theta)\n",
        "        Radar_location = [Radar_x, Radar_y, 0]\n",
        "        return np.array(Radar_location)\n",
        "\n",
        "    # UAV initialization\n",
        "    def _location_UAV_Generator(self):\n",
        "        # h_min < h < h_max\n",
        "        # -180 < theta < 180 (Oxy hyper-space)\n",
        "        # 0 < R < R_Max\n",
        "        r = self.uav_R_max * np.sqrt(np.random.rand())\n",
        "        theta = np.random.uniform(-np.pi, np.pi)\n",
        "        UAV_x = self.BS_x + r*np.cos(theta)\n",
        "        UAV_y = self.BS_y + r*np.sin(theta)\n",
        "        UAV_h = np.random.uniform(self.uav_h_min, self.uav_h_max)\n",
        "        UAV_location = [UAV_x, UAV_y, UAV_h]\n",
        "        return np.array(UAV_location)\n",
        "\n",
        "    def _trajectory_UAV_Generator(self):\n",
        "        theta  = 0\n",
        "        theta  = theta + np.pi/360\n",
        "        r      = np.sin(theta)\n",
        "        UAV_vx = r*np.cos(2*theta)\n",
        "        UAV_vy = r*np.sin(2*theta)\n",
        "        UAV_vh = 0\n",
        "        UAV_trajectory = [UAV_vx, UAV_vy, UAV_vh]\n",
        "        return np.array(UAV_trajectory)\n",
        "\n",
        "    # User distance initialization\n",
        "    def _location_CU_Generator(self):\n",
        "        # Generate random CU locations\n",
        "        # N_User: number of users\n",
        "        # BS_R_min, BS_R_max: circle radius\n",
        "        # [BS_x, BS_y]: center of the circle\n",
        "        userList = []\n",
        "        for i in range(self.N_User):\n",
        "            r = self.BS_R_max * np.sqrt(np.random.rand())\n",
        "            # theta =  i/self.N_User*2*np.pi + np.random.uniform(-np.pi/10, np.pi/10)\n",
        "            theta = np.random.uniform(-np.pi, np.pi)\n",
        "            xUser_temp = self.BS_x + r*np.cos(theta)\n",
        "            yUser_temp = self.BS_y + r*np.sin(theta)\n",
        "            userList.append([xUser_temp, yUser_temp, 0])\n",
        "            \n",
        "        comm_user = np.array(userList)\n",
        "        if self.visualization == \"2D\":\n",
        "            #Plot scatter figure\n",
        "            fig, ax = plt.subplots()\n",
        "            circle1 = plt.Circle((self.BS_x, self.BS_y), 1, color='r', fill=True)\n",
        "            circle2 = plt.Circle((self.BS_x, self.BS_y), self.BS_R_min, color='g', fill=False)\n",
        "            circle3 = plt.Circle((self.BS_x, self.BS_y), self.BS_R_max, color='b', fill=False)\n",
        "            ax.add_patch(circle1)\n",
        "            ax.add_patch(circle2)\n",
        "            ax.add_patch(circle3)\n",
        "            # print(comm_user.transpose()[0:2].transpose())\n",
        "            gu = comm_user.transpose()[0:2].transpose()\n",
        "            plt.scatter(gu[:,0], gu[:,1])\n",
        "            plt.show()\n",
        "        # elif self.visualization == \"3D\":\n",
        "        #     fig = plt.figure()\n",
        "        #     ax = fig.add_subplot(projection='3d')\n",
        "        #     circle1 = plt.Circle((self.BS_x, self.BS_y, 0), 1, color='r', fill=True)\n",
        "        #     circle2 = plt.Circle((self.BS_x, self.BS_y, 0), self.BS_R_min, color='g', fill=False)\n",
        "        #     circle3 = plt.Circle((self.BS_x, self.BS_y, 0), self.BS_R_max, color='b', fill=False)\n",
        "        #     n = 100\n",
        "\n",
        "        #     # For each set of style and range settings, plot n random points in the box\n",
        "        #     # defined by x in [23, 32], y in [0, 100], z in [zlow, zhigh].\n",
        "        #     for m, zlow, zhigh in [('o', -50, -25), ('^', -30, -5)]:\n",
        "        #         xs = randrange(n, 23, 32)\n",
        "        #         ys = randrange(n, 0, 100)\n",
        "        #         zs = randrange(n, zlow, zhigh)\n",
        "        #         ax.scatter(xs, ys, zs, marker=m)\n",
        "\n",
        "        #     ax.set_xlabel('X')\n",
        "        #     ax.set_ylabel('Y')\n",
        "        #     ax.set_zlabel('Height')\n",
        "\n",
        "        #     plt.show()            \n",
        "        return comm_user\n",
        "\n",
        "    # SINR Calculator\n",
        "    ### power_bs_comm = 1 value\n",
        "    ### power_bs_priv = N_User value\n",
        "    ### power_radar   = 1 value\n",
        "    ### H_radar_uav   = 1\n",
        "    ### H_userradar   = N_User value\n",
        "    def _SINR_Radar(self, power_radar, power_bs_comm, power_bs_priv,\n",
        "                    H_radar_uav, H_BS_radar):\n",
        "        numerator = (H_radar_uav**2)*power_radar\n",
        "        denominator = (H_BS_radar**2) * (power_bs_comm + np.sum(power_bs_priv)) + (self.B*(self.sigma2))\n",
        "        SINR = numerator/denominator\n",
        "        return SINR\n",
        "\n",
        "    def _calculateDistance(self, A, B):\n",
        "        return np.array([np.sqrt(np.sum((A - B)**2,axis=1))]).transpose()\n",
        "    \n",
        "    def _selfExcluded_MatrixSum(self, V):\n",
        "        # Private user exclude sum of user at this point\n",
        "        # Clone P_BS_U vector into P_BS_U matrix \n",
        "        clonedM = np.ones((len(V),1))*V\n",
        "        # Remove diagonal\n",
        "        diagonalExcludedM = clonedM -np.diag(np.diag(clonedM))\n",
        "        # Sum over self-excluded matrix\n",
        "        rowWise_SumM = np.matrix(diagonalExcludedM).sum(axis=1)\n",
        "        return rowWise_SumM\n",
        "\n",
        "    def _calculateDataRate(self, channelGain_BS_CU, channelGain_Radar_CU):\n",
        "        # Generate partial components\n",
        "        interferenceRadarUser   = ((channelGain_Radar_CU)**2)*self.P_R\n",
        "\n",
        "        sumCommonUserPower      = np.sum(self.P_BS_U)\n",
        "        sumPrivateUserPower     = self._selfExcluded_MatrixSum(self.P_BS_U)\n",
        "\n",
        "        interferenceCommonUser  = ((channelGain_BS_CU)**2)*sumCommonUserPower  \n",
        "\n",
        "        interferencePrivateUser = np.multiply(((channelGain_BS_CU)**2),np.array([sumPrivateUserPower]).transpose())\n",
        "        interferenceBandwidth   = self.B*self.sigma2\n",
        "        \n",
        "        commonNumerator       = ((channelGain_BS_CU)**2)*self.P_BS_0\n",
        "        commonDenominator     = interferenceCommonUser + interferenceCommonUser + interferenceBandwidth\n",
        "        \n",
        "        privateNumerator      = ((channelGain_BS_CU)**2)*self.P_BS_U\n",
        "        privateDenominator    = interferencePrivateUser + interferenceCommonUser + interferenceBandwidth\n",
        "        \n",
        "        commonDataRate  = self.B * np.log2(1+(commonNumerator/commonDenominator))\n",
        "        privateDataRate = self.B * np.log2(1+(privateNumerator/privateDenominator))\n",
        "        if self.verbose_DataRate == True:\n",
        "            print(f\"sumPrivateUserPower: {sumPrivateUserPower}\")\n",
        "            print(f\"interferenceCommonUser: {interferenceCommonUser}\")\n",
        "            print(f\"interferencePrivateUser: {interferencePrivateUser}\")\n",
        "            print(f\"interferenceBandwidth: {interferenceBandwidth}\")\n",
        "            print(f\"commonNumerator: {commonNumerator}\")\n",
        "            print(f\"commonDenominator: {commonDenominator}\")\n",
        "            print(f\"privateNumerator: {privateNumerator}\")\n",
        "            print(f\"privateDenominator: {privateDenominator}\")\n",
        "        return commonDataRate, privateDataRate\n",
        "\n",
        "    def _wrapState(self):\n",
        "        # Channel Gain\n",
        "        self.H_R  = self._channelGain_Radar_UAV()        # Channel round-trip between Radar and UAV\n",
        "        self.H_CR = self._channelGain_BS_Radar()         # Channel from BS to Radar\n",
        "        self.H_C  = self._channelGain_BS_CU()            # Channel from BS to CU    (Interference)  \n",
        "        self.H_RC = self._channelGain_Radar_CU()         # Channel from Radar to CU (Interference)  \n",
        "        if self.verbose_channelGain == True:\n",
        "            print(\"Channel Gain\")\n",
        "            print(f\"radar-uav: {self.H_R}\")\n",
        "            print(f\"radar-bs: {self.H_CR}\")\n",
        "            print(f\"bs-cu: {self.H_C}\")    \n",
        "            print(f\"radar-cu: {self.H_RC}\")\n",
        "\n",
        "        # SINR calculation\n",
        "        self.SINR_Radar = self._SINR_Radar(power_radar = self.P_R,                                  \n",
        "                                           power_bs_comm = self.P_BS_0, power_bs_priv = self.P_BS_U,\n",
        "                                           H_radar_uav = self.H_R, H_BS_radar = self.H_CR)          \n",
        "        if self.verbose_SINR == True:\n",
        "            print(f\"SINR: {self.SINR_Radar}\")\n",
        "        self.commonDataRate, self.privateDataRate = self._calculateDataRate(self.H_C, self.H_RC)\n",
        "\n",
        "        # print(np.shape(self.H_RC))\n",
        "        # print(np.shape(self.H_C))\n",
        "        # print(np.shape(self.H_CR))\n",
        "        # print(np.shape(self.H_R))\n",
        "        # print(np.shape(self.UAV_location))\n",
        "        # print(np.shape(self.UAV_trajectory))\n",
        "\n",
        "        state = np.concatenate((np.array(self.H_RC).reshape(1,self.N_User), np.array(self.H_C).reshape(1,self.N_User),                    \n",
        "                                np.array(self.H_CR), np.array(self.H_R),                    \n",
        "                                np.array(self.UAV_location), np.array(self.UAV_trajectory)\n",
        "                               ), axis = 1)\n",
        "\n",
        "        return state\n",
        "\n",
        "    def _wrapAction(self):\n",
        "        action = np.concatenate((np.array([[self.P_BS_0]]), np.array(self.P_BS_U).reshape(1,self.N_User),                    \n",
        "                                np.array([[self.P_R]]), np.array(self.Common_User_Allocation).reshape(1,self.N_User)                    \n",
        "                               ), axis = 1)\n",
        "        return action\n",
        "\n",
        "    def _decomposeState(self, state):\n",
        "        H_RC = state[0 : self.N_user]\n",
        "        H_C  = state[self.N_User     : 2*self.N_User]\n",
        "        H_CR = state[2*self.N_User   : 2*self.N_User+1]\n",
        "        H_R  = state[2*self.N_User+1 : 2*self.N_User+2]\n",
        "        UAV_location   = state[self.N_User+2 : 2*self.N_User+5]\n",
        "        UAV_trajectory = state[self.N_User+5 : 2*self.N_User+8]\n",
        "        return  [\n",
        "                 np.array(H_RC).reshape(self.N_User,1), np.array(H_C).reshape(self.N_User,1),                    \n",
        "                 np.array(H_CR), np.array(H_R),                    \n",
        "                 np.array(UAV_location), np.array(UAV_trajectory)\n",
        "                ]\n",
        "\n",
        "    def _decomposeAction(self, action):\n",
        "        P_BS_0 = action[0:1]\n",
        "        P_BS_U = action[1:1+self.N_User]\n",
        "        P_R    = action[1+self.N_User: 2+self.N_User]\n",
        "        Common_User_Allocation = action[2+self.N_User:2+2*self.N_User]\n",
        "        return [\n",
        "                np.array(P_BS_0), np.array(P_BS_U), \n",
        "                np.array(P_R), np.array(Common_User_Allocation)\n",
        "               ]\n",
        "\n",
        "    ###########################\n",
        "    # DRL Environment process #\n",
        "    def step(self, action):\n",
        "        # Current state decomposition\n",
        "        # State: [H_RC, H_C, H_CR, H_R, UAV_location, UAV_trajectory]\n",
        "        # self.H_RC, self.H_C, self.H_CR, self.H_R, self.UAV_location, self.UAV_trajectory = self._decomposeState()\n",
        "        # Current action decomposition\n",
        "        # Action: [P_BS_0, P_BS_U, P_R, Common_User_Allocation]\n",
        "        self.P_BS_0, self.P_BS_U, self.P_R, self.Common_User_Allocation = self._decomposeAction(action)\n",
        "\n",
        "        # Environment changed\n",
        "            # UAV moves\n",
        "        self.UAV_trajectory = np.expand_dims(self._trajectory_UAV_Generator(), axis=0)\n",
        "        self.UAV_location = self.UAV_location + self.UAV_trajectory\n",
        "            # Users move\n",
        "      \n",
        "        # state wrap [Radar-UAV, BS-Radar, BS-CU, Radar-CU]\n",
        "        state_next = self._wrapState()\n",
        "        # Re-calculate dataRate\n",
        "        self.commonDataRate, self.privateDataRate = self._calculateDataRate(self.H_C, self.H_RC)  \n",
        "        # Re-generate current Radar-SINR\n",
        "        self.SINR_Radar = self._SINR_Radar(power_radar = self.P_R,                                  \n",
        "                                           power_bs_comm = self.P_BS_0, power_bs_priv = self.P_BS_U,\n",
        "                                           H_radar_uav = self.H_R, H_BS_radar = self.H_CR)          \n",
        "        # Total dataRate\n",
        "        cRate = np.sum(self.privateDataRate) + np.min(self.commonDataRate)\n",
        "        # self.rewardMatrix = np.concatenate((self.rewardMatrix, cRate), axis = 0)\n",
        "\n",
        "        # Calculate reward (Sum-Rate)\n",
        "        reward = cRate \n",
        "\n",
        "        # If reach number of rounds / achieve the desirable reward -> done\n",
        "        done = False # not implement yet\n",
        "\n",
        "        return state_next, reward, done, {}\n",
        "\n",
        "    def reset(self):\n",
        "        # Base Station initialization\n",
        "        self.BS_location = np.expand_dims(self._location_BS_Generator(), axis=0)\n",
        "\n",
        "        # Radar initialization\n",
        "        self.Radar_location = np.expand_dims(self._location_Radar_Generator(), axis=0)\n",
        "\n",
        "        # UAV initialization\n",
        "        self.UAV_location   = np.expand_dims(self._location_UAV_Generator()  , axis=0)\n",
        "        self.UAV_trajectory = np.expand_dims(self._trajectory_UAV_Generator(), axis=0)\n",
        "\n",
        "        # User location initialization\n",
        "        self.CU_location    = self._location_CU_Generator()\n",
        "        \n",
        "        # Distance calculation\n",
        "        self.distance_Radar_UAV    = self._calculateDistance(self.UAV_location, self.Radar_location)\n",
        "        self.distance_BS_Radar     = self._calculateDistance(self.UAV_location, self.BS_location)\n",
        "        self.distanceList_BS_CU    = self._calculateDistance(self.BS_location , self.CU_location)\n",
        "        self.distanceList_Radar_CU = self._calculateDistance(self.CU_location , self.Radar_location) \n",
        "        \n",
        "        # Generate next state\n",
        "        state_next = self._wrapState()\n",
        "\n",
        "        return state_next\n",
        "\n",
        "    def close(self):\n",
        "        pass"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "env = Env_CNUR(MAX_EP_STEPS = 10000, N_user_max = 5,   \n",
        "               BS_R_min = 10, BS_R_max = 150, radar_R = 1000,     # Radius of BS and Radar distance to BS\n",
        "               h_max = 500, h_min = 10,                           # Height of the UAV\n",
        "               uav_R_max = 1000, uav_R_min = 200,                 # Radius of the UAV\n",
        "               C_TH = 50,                                         # Datarate threshold\n",
        "               P_BS_max = 100, P_radar_max = 1000, \n",
        "               Bandwidth = 100, noise = -174)                     # Power, Data Rate, Bandwidth"
      ],
      "metadata": {
        "id": "KNWzAnU1CWZp",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 264
        },
        "outputId": "243c7f90-bb9f-4544-ccb1-a30a3b459c5b"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ],
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX8AAAD4CAYAAAAEhuazAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3dd3hUdfb48fchJFQpAiJVQgdBKREBwQYIli8BK1YsK+6Krn0XdB9114aLvaHoWneFRUWIqCBF0EfNalgUEAgEkJWA9CbEEMLn98eZ/DJAKpmbOzP3vJ5nnpm5czNz7mTm3M98qjjnMMYYEyxV/A7AGGNM5bPkb4wxAWTJ3xhjAsiSvzHGBJAlf2OMCaCqfgdQVg0bNnStWrXyOwxjjIkZCxcu3Oqca1TUYzGT/Fu1akVGRobfYRhjTMwQkXXFPWbVPsYYE0CW/I0xJoAs+RtjTABZ8jfGmACy5G+MMQEUkeQvIq+LyGYRWRq27UERyRaR70OX88IeGysiWSKSKSKDIxGDMcaYsotUyf9NYEgR2592znULXT4BEJHOwAjgxNDfvCQiCRGKwxhjTBlEpJ+/c+4LEWlVxt1TgcnOuVxgrYhkAb2AbyIRizGRcPAg7NkDO3fC7t1w4ADk5+v2/PxDLyJQpQokJBReCu5XrQp160K9enDMMbqvMdHA60Fet4jINUAGcJdzbgfQDEgP22d9aNsRRGQUMAqgZcuWHodq4pFzsHkzrFsH27fDjh2a0Eu63rFDE36NGlC/PtSpo0n88MRecIEjTwgFJ4m8PH2uHTvgt9/0RFC/vp4M6tc/9Pbh18ceC61aQcOGdtIwkedl8p8APAS40PWTwPXleQLn3ERgIkBKSoqtOmOKtHs3rF176GXNGr3+6SeoWRNOOAEaNDg0uTZoAG3bFp1869XThB9JeXl6ginp5PPTT4X3t27Vk1ZuLiQn66V168LbBZfatSMbpwkGz5K/c25TwW0ReRWYEbqbDbQI27V5aJsxxdq5ExYtgpUrD03ua9dCTs6hybB1axgwoPD+Mcf4Hb1KTIRGjfRSHrt2HXpiy8qCzz4rPLnVrn3k8XfoAN266a8WY4riWfIXkSbOuY2hu8OBgp5AacC7IvIU0BRoB3zrVRwm9mzbBv/9r14WLtTrTZvgpJOgc2dNcMOHFya6Ro3iu1qkbl1N5N26HfmYc/rehJ8M09Ph9ddh8WJo3hx69ICePfW6Rw/9VWNMRJK/iEwCzgQaish64AHgTBHphlb7/ATcBOCc+1FEpgDLgAPAaOdcfiTiMLFny5bCBF9wvW0bdO+uCWvoUPjrX6F9+8L6dVNIBI4/Xi99+x762IEDsGJF4fs6bRr88AMcd5y+t+EnhAYN/Inf+EdiZQH3lJQUZ7N6xrYDByAjA+bOhe++06S0Z09hybQgGbVtq42qJvLy87XqLPxku2iRtnX07AmnnKJVZj162Mk2HojIQudcSpGPWfI3XnFO66dnz9bL/PnQsqUmlz59NMG0bh3fVTax4OBBWL1aTwbffANz5sAvv8DZZ8OgQXpJTvY7SnM0LPmbSrNtm5bsCxJ+Xl5hAhkwQKsnTPTLztaTwOzZel27duH/8ayz9JeCiX6W/I1ncnPhq68Kk/2qVdC/f2Gi6NTJSvaxzjlYurTwf/zVV/p/Lfgf9+kDSUl+R2mKYsnfRNSvv8L06TB5slbldOkCAwdqIujd2xJBvMvNha+/LjwZZGbq/3/ECLjgAh1XYaKDJX9TYb/9Bp9+qgl/5kzo1w8uv1y/7NZ1MNi2bYO0NJg0Cb79Fs4/Xz8b55xjBQG/WfI3R+XAAZg3T7/U06fDySfrl/rCC3XKAWMOt2kTvP++fmZWrNDPyogRcMYZ1nvID5b8TZkdPKg/6SdN0i9xq1aa8C+5BJoVOQOTMUVbtw6mTNHP0i+/wKWX6mepVy9rB6oslvxNqZYsgbffhn//W0eUXn45XHYZtGnjd2QmHmRmapXhpEmwf7/+GrjmGujY0e/I4ltJyd+G0gRYfj58+CGceSace67Wz37yiZ4I7r3XEr+JnA4d4IEHYPly+OADrVI86yxtF/j4Y/3FaSqXlfwDaMcOnfvlhRegSRO47Tatm01M9DsyEyS5uVot9OyzOjPrrbfCtddGz0R88cBK/gbQBribb9ZRtYsW6Rfv66+1escSv6ls1arB1VfrVB9vvAFffqltTLffriOOjbcs+ce5gwe1i+aQIdrjomFDWLYM/vlPncfFGL+JwGmnaWHk+++henUdLzJ0qI4Wj5HKiZhj1T5xas8eeOsteP55HXRz223ayFa9ut+RGVO6ffvgX//SKiGAP/4RrrrKBpCVl1X7BMivv8LDD+tEXPPnw2uv6cyN115rid/Ejpo14cYbtfPBs8/CjBlaJTR+vC7eYyrOkn+c2L9fG3DbtdNqnfR07affv7/1qTaxS0QnBExLgwUL9HPdvr0Wag4c8Du62GbJP8YdPKg/jzt21C5zn3wC776rc+IbE086ddJuou+/r5/xLl30dozUXEcdS/4xyjlN9t27a4n/9de1Ybd7d78jM8Zbp56qDcHPPQePPqojhufO9Tuq2OPZGr7GO199BWPH6oRajz6qvSKsascEiYgOEBs4UEv/v/+9tgk89hikFNm8aQ5nJf8YsmSJJvorroDrr9cFulNTLfGb4KpSRecMWrYMLr5Yvw+XXqrTSZiSWfKPAVu3arIfOFCX1svM1N47NkuiMSoxEW66SRcT6tFDpxy/+WbYudPvyKKXJf8o5pz21T/xRJ1sbdUqHf1oXTaNKVrNmjBmjBaQnIPOnXWyQmsUPpLV+UepVau0HnPnTu3B07On3xEZEzuOPRYmTNDpI0aN0kLUSy9pu4BRVvKPMvv36yCtPn10laz//McSvzFHq29fHeTYr582BD/xhI0PKGDJP4r88IN2W/vmG/3A3nEHVLXfZsZUSFKSTlGenq7doU87TaeWDrqIJH8ReV1ENovI0rBtx4rIbBFZFbquH9ouIvKciGSJyGIR6RGJGGJZXh489JA26N5+uw5lb9nS76iMiS9t2+qC89deqyPfn3hC17QIqkiV/N8Ehhy2bQww1znXDpgbug9wLtAudBkFTIhQDDFp6VKdwfDrr3Wa5Wuvta6bxnilShX4wx90ofkZM/QksHKl31H5IyLJ3zn3BbD9sM2pwFuh228Bw8K2v+1UOlBPRJpEIo5Y4hw884yuZnTzzdqo27y531EZEwytW8O8eTpmpm9fePVVvyOqfF7WKDd2zm0M3f4FaBy63Qz4OWy/9aFtGzmMiIxCfx3QMo7qQfbt0xkLly3TEkhyst8RGRM8VarALbfAoEEwfLh+F194QReZCYJKafB1umhAuXvaOucmOudSnHMpjRo18iCyyrdmjfbkqVJFp2mwxG+Mvzp00F5127fD6afD+vV+R1Q5vEz+mwqqc0LXm0Pbs4EWYfs1D22LezNnauL/3e/g7bdtYQpjosUxx+gcQcOHa4+7BQv8jsh7Xib/NGBk6PZIYHrY9mtCvX56A7vCqofiknM6Adv118N77+lC1daoa0x0EdHRwW++qfMDPftsfI8Mjkidv4hMAs4EGorIeuABYBwwRURuANYBl4Z2/wQ4D8gC9gHXRSKGaLV7t/bg2bBBF6pu1szviIwxJTnnHB0TMHy4fmcnTozPX+mR6u1zuXOuiXMu0TnX3Dn3D+fcNufcAOdcO+fcQOfc9tC+zjk32jnXxjnX1TkXtwvzrlihc483aqQ/Iy3xGxMbkpO1+zXooLC1a/2Nxws2wtcj06dr49Fdd8ErrwSnB4Ex8aJmTXjnHf3l3rs3fPaZ3xFFliV/D7zxhvbd/+gjbdw1xsQmEbjtNpgyBa65Rtvs4oXNHBNhr7yiE7PNm6ddyIwxse+MM2DWLBgyRCdfvPJKvyOqOEv+EfT88/DkkzB/PrRp43c0xphIOvlkmDNHG4T374frYryriiX/CHnySXjxRU38Nme4MfHpxBP1V/3AgToh46hRfkd09Cz5R8Cjj2rf4AULoEWLUnc3xsSwDh3g889hwAD9BXDLLX5HdHQs+VeAc/DXv+oycQsWQJPATU9nTDC1bavf+bPPhtxc7dUXayz5HyXndIGIGTO0qqdx41L/xBgTR1q1OvQEcO+9fkdUPpb8j4JzcPfdWvf3+efQsKHfERlj/NCihZ4ABgzQE8CDD8bO1C3Wz/8ojBunSX/uXEv8xgRd06b66/+993SR+FhhJf9ySkvTOb+//RaOPdbvaIwx0aBxYx3U2bcvdOqkVUHRzkr+5fDjj3DDDTB1qs3TY4w5VJs2MGkSXH45rF7tdzSls+RfRtu2wdCh8NRTOlmbMcYc7uyz4YEHNFfs3u13NCWz5F8GeXlwySVw0UVw9dV+R2OMiWZ/+IMuDH/llZCf73c0xbPkXwZ33KEz/D32mN+RlN+0RdmcNm4eyWM+5rRx85i2KBCLphnjGxGd6mXPHvjLX/yOpnjW4FuKV17RLp3p6ZCQ4Hc05TNtUTZjpy4hJ0+LH9k7cxg7dQkAw7pbo4WJbtMWZTN+ViYbdubQtF4N7hncIWY+t4mJuixkr17QpUt0TgRnJf8SLFgA99+vPXzq1PE7mvIbPyvz/yf+Ajl5+YyflelTRMaUTUHBJXtnDo7Cgkss/XJt2FDX9bj9dl0RLNpY8i/G+vUwYgS8+64O5Y5FG3bmlGu7MdEiXgouXbvCa6/pkpBbtvgdzaEs+RfBORg9Gm66SUfuxaqm9WqUa7sx0SKeCi6pqXDZZdp2GE0s+Rfhgw8gKwvGjvU7koq5Z3AHaiQe2lBRIzGBewbbKjMmusVbweVvf4OvvtIFYaKFJf/D7NgBf/wjvPpq7K+7O6x7Mx67sCvN6tVAgGb1avDYhV1jptHMBFe8FVxq1dLOI7//Pezd63c0SpxzfsdQJikpKS4jI8Pz17nxRkhK0oVZjDH+ieXePsW5+mo47jhd/KkyiMhC51xKkY9Z8i+0YAFcdZVO4xCLvXuMqSzxmJgrw5Yt2vVzxgw45RTvX6+k5G/VPiG//aal/hdesMRvTEnioRumXxo1giee0FyTl+dvLJ4nfxH5SUSWiMj3IpIR2nasiMwWkVWh6/pex1Gahx7SBZpTU/2OxJjoFi/dMP1y1VU6C+hTT/kbR2WV/M9yznUL+/kxBpjrnGsHzA3d983ixdrA+9xzfkZhTGyIp26YfhCBl1+G8eO1V6Ff/Kr2SQXeCt1+CxjmUxw4pxMxPfKIrcFrTFnEWzdMPyQna1dyPxd/r4zk74DPRGShiIwKbWvsnNsYuv0LUOQKuCIySkQyRCRji0fD4+bNg+3bdZ5+Y0zp4q0bpl/++EdYvlwXhvJDZST/fs65HsC5wGgROT38QafdjYrscuScm+icS3HOpTRq1MiT4MaNgz//GapY07cxZWLjRyIjMRHuugsef9yf16/Urp4i8iDwK3AjcKZzbqOINAHmO+dKLDZ40dVz4UIYNkxX3UlKiuhTG2NMqfbu1SqgL7+EDh78cPKtq6eI1BKRYwpuA+cAS4E0YGRot5HAdC/jKM7jj8Odd1riN8b4o1YtnUds/PjKf21PS/4i0hr4MHS3KvCuc+4REWkATAFaAuuAS51z20t6rkiX/Fet0sWW166F2rUj9rTGGFMu27ZBu3awZEnk1wYvqeTv6WIuzrk1wMlFbN8G+Dpf5vjxcPPNlviNMf5q0ABGjoSnn9YBYJUlkM2cGzboKju33up3JMYYo9XPr7+uPQ8rSyCT/zPP6ARLDRv6HYkxxkCLFjq7wEsvVd5rBi7579qlK+vceaffkRhjTKE//UkXfs+ppIHSgUv+aWnQvz+ccILfkRhjTKFOneDEE+Gzzyrn9QKZ/If5NpmEMcYULzVVc1RlCFTyz82F2bPh/PP9jsQYY440dKjO9Z+fX/q+FRWo5D9vni6kcNxxfkdijDFHSk7W6Z7T071/rUAl/+nTbb5+Y0x0S03VXOW1wCT/gwe1Ls2SvzEmmlnyj7CMDKhbF9q39zsSY4wpXs+e8OuvsGKFt68TmOQ/fbo2phhjTDQT0Vzldek/UMnfqnyMMbGgMqp+ApH8N2yAX36BU0/1OxJjjCndWWfpLJ+7dnn3GoFI/suWaRfPhITS9zXGGL9Vq6bTPGdmevcagUj+mZnerJJjjDFe6dDBkn+FWfI3xsQaS/4RYMnfGBNrOnTwtrunJX9jjIlCVvKvoJwc7enTqpXfkRhjTNm1bw9ZWd5N8hb3yX/VKmjTBqp6ulqxCSrnHM45v8Mwcah2bV1t8H//8+b54z4lrlhhVT4mMvbl7eOLdV8we/Vs5qydw5oda9iXtw+AWom1aHtsWwa1HsSgNoPo17If1atW9zliE+sKqn6SkyP/3HGf/K2+31TUjpwdPPTFQ7z239fodnw3BrUexMvnv8yJx51IzcSaAOzdv5clm5cwe/Vs7v/8fpZuXsroU0Yztv9Y6lSr4/MRmFhVkPyHDIn8c8d98s/OhpNO8jsKE6vm/zSfq6ZexfntzmfVratoXLtxkfvVrV6Xfi370a9lP/561l/J3p3NvfPu5aQJJzHpokn0adGnkiM38aBFC81hXoj75L9/P1S3X9/mKKSvT+fS9y7lnxf+k3PanFOuv21WpxlvDXuL6SumM3TyUOZeM5eTGlspxJRP9eq6AqEXfGvwFZEhIpIpIlkiMsar18nNhaQkr57dxKu9+/dy9YdXM+H8CeVO/OFSO6YyftB4rpp6FbkHPPoWm7iVlKQFWC/4kvxFJAF4ETgX6AxcLiKdvXit3FydJ8OY8pjy4xQ6NuzIRZ0vKnnHnBz47bcSdxl58kjq16jPRys/imCEJgiqVYu/kn8vIMs5t8Y5tx+YDHgy4fL+/VbyN+U3f918hrYvZQGIjz+G+vX18vnnxe4mIgxtP5T5P82PbJAm7sVdyR9oBvwcdn99aNshRGSUiGSISMaWLVuO6oWc08URjCmPAwcPkJRQSqlh0ya9PngQtm4tcddqVauRl58XoehMUIjox8sLUT3Iyzk30TmX4pxLadSo0VE9h5dnThO/ejfrzdy1c0ve6frrYdEinXj9kktK3HXu2rnW48eU2/793lVb+5X8s4EWYfebh7ZFnJd1ZiZ+jegygs9Wf8bXP39d8o6dOpW6MPTs1bPJ2JDB8I7DIxihCQIv2yz9Sv7fAe1EJFlEkoARQJoXLxTtJf9pi7I5bdw8ksd8zGnj5jFtkUedek25NKrViAnnT+DKqVeyYuvRT624eNNirpt+HW+kvkHd6nUjGKEJAi/bLH1J/s65A8AtwCxgOTDFOfejF68VzSX/aYuyGTt1Cdk7c3BA9s4cxk5dYieAKDG803Du7Xcv/d/oz9PfPM3+/LKXIn478BuPffkYA94ewOMDH2dg64EeRmrilZclf98GeTnnPgE+8fp1jjsONm70+lWOzvhZmeTkHTplX05ePuNnZTKs+xHt38YHN/a8kf4n9OfOWXfyVPpTnNv2XAa1HsTZyWfToGaDQ/bdvHczc9fMZfaa2czMmknv5r1JvyGdNse28Sl6E+s2boQmTbx57rgf4duhA8yY4XcURduwM6dc240/OjbsyCdXfsKyLcv4bPVnvPnDm9yQdgO1k2pTK6kWzjn25u0lJy+HM1qdwaDWgxjTbwztG5TcFmBMaTIz4cwzvXnuQCT/J5/0O4qiNa1Xg+wiEn3TejV8iMaUpnOjznRu1Jnbe9/O/vz9bNm7hb15exGEWkm1aFSzEYkJiX6HaeJIZiZ07OjNcwci+a9cqX1lq0RZx9Z7Bndg7NQlh1T91EhM4J7BNg1ptEtKSKJZHauaM97JzdVJ3Vq39ub5oywdRl6dOlC3rncz41XEsO7NeOzCrjSrVwMBmtWrwWMXdrX6fmMMWVlwwgmQ6NGPybgv+UPhQsgtWpS+b2Ub1r2ZJXtjzBG8Xogq7kv+4P1CyMYYE2leL0Rlyd8YY6KQl429YMnfGGOikpX8I6BjR/jxR53h0xhjol1+vtX5R0RysraYL13qdyTGGFO69HRo2RKOcjLjMglE8heB1FSYPt3vSIwxpnTTp2vO8lIgkj9Y8jfGxA5L/hHUvz+sXh2dg72MMabAihXw66/Qs6e3rxOY5J+YCOeeCx/ZGtrGmCiWlgZDh3q//Gxgkj9Y1Y8xJvpVRpUPBCz5DxkCX30Fe/b4HYkxxhxp0ybtln7WWd6/VqCSf5060LcvzJzpdyTGGHOkGTPgnHO8W70rXKCSP1jVjzEmeqWlVU6VDwQw+Q8bBp98Atu2+R2JMcYUys6GL7+E88+vnNcLXPJv0gSGD4cXX/Q7EmOMKfT00zByJNSrVzmvF7jkD3DPPfDCC7B3r9+RGGMMbN8Or78Od95Zea8ZyOTfsaMO+vrHP/yOxBhj4KWXtK6/MhecCmTyB/jzn3Vh97w8vyMxxgTZvn3w/PPwpz9V7usGNvn36gVt2sDkyX5HYowJsjfegD59oFOnyn1dz5K/iDwoItki8n3ocl7YY2NFJEtEMkVksFcxlGbMGHj8cTh40K8IjDFBlpcH48drLqpsXpf8n3bOdQtdPgEQkc7ACOBEYAjwkogkeBxHkQYNgqQk+PhjP17dGBN0U6ZAq1bQu3flv7Yf1T6pwGTnXK5zbi2QBfTyIQ5E9Iw7bpwfr26MCTLntObBj1I/eJ/8bxGRxSLyuojUD21rBvwcts/60LYjiMgoEckQkYwtW7Z4EuBFF8HWrTqs2hhjKsvkyVC1Kgz2qeK7QslfROaIyNIiLqnABKAN0A3YCDxZ3ud3zk10zqU451IaebSeWUKCdrMaPdomfDPGVI5t27RP/4QJ3k/dXJyqFflj59zAsuwnIq8CBWXrbCC8N2vz0DbfDBgAZ58Nf/kLPPusn5EYY4Lg7rvh0kvh1FP9i8HL3j5Nwu4OBwqWT08DRohINRFJBtoB33oVR1k98YQ2vqSn+x2JMSaezZkD8+bBww/7G0eFSv6l+LuIdAMc8BNwE4Bz7kcRmQIsAw4Ao51z+R7GUSYNGujcGjfeCAsXai8gY4yJpH374KabtKr5mGP8jUWcc/5GUEYpKSkuIyPD09dwDi64QOf8v+8+T1/KGBNAf/4zrFtXeYNLRWShcy6lqMe8LPnHHBFtgOnRAy6+GDp08DsiY0y8WLRIR/MuWeJ3JCqw0zsUp2VLuP9+GDXKRv4aYyLjwAH43e+0X3/jxn5Hoyz5F2H0aMjNtVk/jTGR8eyzOk//tdf6HUkhq/YpQkICvPqqdv/s0we6dPE7ImNMrPruO51F4Jtv/OvTXxQr+Reja1d45hmdY3vrVr+jMcbEog0bdOXAV1+Ftm39juZQlvxLcOWVcMklerF5/40x5ZGTo2uG/+EPeh1tLPmX4pFHoFYtuO02vyMxxsQK53TMUOvWcO+9fkdTNEv+pUhIgHffhfnztRuoMcaUZvx4WL5c1+WNpnr+cNbgWwZ16kBaGpx2mq62c+aZfkdkjIlWM2Zo7570dKhZ0+9oimcl/zJq21Z/AYwYAWvW+B2NMSYaLVsG110H779fuYuxHw1L/uUwYIBO+5CaatM/G2MOtX07DB2qk0T26eN3NKWz5F9Ot9yiS65dcQXs3+93NMaYaJCTo1PCpKbCyJF+R1M2lvzLSQRefBGqVNF/dm6u3xEZY/y0d69OCHn88fD3v/sdTdlZ8j8KSUnw3nt6PWyYnvWNMcGzZw+ce67W77/zjvYOjBWW/I9SUpJOy1q/Pvzf/+nZ3xgTHLt26fq7HTtql85YSvxgyb9CqlbVs32zZnDeedYIbExQ7NgBgwbp9O8vv6zVwLEmBkOOLgkJOkd3+/ZaCti1y++IjDFe2rpVJ33s1w+efz42Ez9Y8o+IKlXglVege3ctDezY4XdExhgvbN6siX/IEHjyyegdvVsWlvwjpEoVeOEFLQ0MGGAzgRoTbzZu1NH9F14Ijz4a24kfLPlHlIiWBgYP1tLBzz97+3rTFmVz2rh5JI/5mNPGzWPaomxvX9CYgFq9Gs44A666Ch58MPYTP1jyjzgRLRVccw306qUTwnlh2qJsxk5dQvbOHByQvTOHsVOX2AnAmAj79FPo2xfuuCN6Z+g8Gpb8PSACd98Nb7+tcwE9/bRO8RpJ42dlkpOXf8i2nLx8xs/KjOwLGRNQBw/Cww/r2rsffKDz8scTm9XTQ4MG6cx+F14IGRm6mk+kZvnbsLPokWXFbTfGlN3u3frrffNmXYaxaVO/I4o8K/l7rFUr+Oor7RLap0/kZgRtWq9GubYbY8pm+XKtsm3SRKtt4zHxQwWTv4hcIiI/ishBEUk57LGxIpIlIpkiMjhs+5DQtiwRGVOR148VNWrAW2/pz8c+fWDmzIo/5z2DO1Aj8dAhhTUSE7hncIeKP7kxAfXhh9qw+6c/6eJNSUl+R+Sdilb7LAUuBF4J3yginYERwIlAU2COiLQPPfwiMAhYD3wnImnOuWUVjCPqicCtt0K3btoOMHo0jB179L0GhnVvBmjd/4adOTStV4N7Bnf4/9uNMWWXnw/33w///Cd8/DGccorfEXmvQsnfObccQI7MYKnAZOdcLrBWRLKAXqHHspxza0J/Nzm0b9wn/wL9+8O33+qMoBkZ8OabulLY0RjWvZkle2MqaPt2naI9N1fr9487zu+IKodXdf7NgPBe7utD24rbXiQRGSUiGSKSsWXLFk8C9UOzZlqXeNxxOjfIl1/6HZExwTR7to7MP/FEvR2UxA9lKPmLyBzg+CIeus85Nz3yIRVyzk0EJgKkpKREuLOkv6pV0wmh0tK0GujSS+GRR6J7zU9j4sWePdode+ZM7YV3zjl+R1T5Si35O+cGOue6FHEpKfFnA+ErWDYPbStue2ANHQqLF2uXsm7dtGeQMcY7c+dC165az794cTATP3hX7ZMGjBCRaiKSDLQDvgW+A9qJSLKIJKGNwmkexRAzGjSAf/0Lxo3TtoC77rL1AYyJtN274eab4dprtSfPa69B3bp+R+Wfinb1HC4i64E+wMciMgvAOfcjMAVtyJ0JjHbO5TvnDgC3ALOA5cCU0L4GHQy2eDFs2gRduuiwcmNMxTgHU1CWIc0AAAtpSURBVKdC58667vaSJbr6VtCJi/S8Ax5JSUlxGRkZfodRaT77TIeTn3IKPPOMrg9qjCmfn3+GW26BzEyYOBFOP93viCqXiCx0zqUU9ZiN8I1S55yjJZTkZK2ffOUVnWvEGFO6/Hx49lntydOjB/zwQ/ASf2ks+UexmjXhscdg3jwdIdyzp/ZOiJEfa8ZUOudg2jQ46SS9/uoreOAB7V1nDmUTu8WArl31Q/zhhzqtbOPGelLo08fvyIyJHgsWwJgxsG8fjB+v9frxMO++V6zkHyNEtEF4yRKdbfCyy2DYMPjRmstNwH3/vSb6667T+v1Fi+C88yzxl8aSf4ypWhWuvx5WrtSpIs46Sz/069b5HZkxlSsrS6dlOPdcuOACWLECrrwydhdUr2z2NsWo6tV1PMCqVdC8uTZq3XEHxNEsGMYUaeNG7a/fu7d231y1SidKjOcZOL1gyT/G1a0LDz2k1T95edCpE/ztbzp83Zh4snMn3HefjoGpWVNL+n/5C9Su7XdkscmSf5w4/nh44QWdMXTlSmjTRr8YGzb4HZkxFbNunc6v364d/PKL1vE/8QQ0bOh3ZLHNkn+cad1a5yT/8kstKXXpovWi6el+R2ZM2Tmnn+GLL9YqzQMH4D//gX/8A1q0KP3vTeks+cepDh30l8CaNTpK+Ior4NRT4d13dYi7MdEoN7dwTMvvfqcdGtatg6ee0oKNiRxL/nGuXj1tCF61Cu69VyezSk6Ghx+2xmETPTZu1JW0TjgBJk3S6c2XL9eGXKvT94Yl/4BISIDUVB0t/OmnWppq3167jf7wg9/RmaD67ju46ipdTGXbNl3kaOZM7b5pXTa9ZW9vAJ10ki5gsWqVNqJdcAGceSa88471EjLe27kT3ngD+vbVRYy6d9fqyRdfhI4d/Y4uOCz5B1jDhrqI/Jo12m96yhQdM3DJJfDBB5CT43eEJl7s3Qv//reOSj/hBPjoI+3Bk5Wl41Xq1fM7wuCx5G9ITNQS2Ecfwdq1MHgwvPQSNG0KI0dqNVFent9Rmlizf79+pq64QtetfuMNGD4c/vc/nV9/2DCtjjT+sPn8TbE2btRfA5Mnawnt4ot1veH+/a0+1hQtP1/r7SdN0okIO3eGyy/Xz06QFkePFiXN52/J35TJ2rV6Epg8WRvmLrtMv9Q9e9oEWkHnnI4jmTQJ3ntPfzFefrl+RqxPvr8s+ZuIWrZMv+iTJ2u7wKBBehk40Ep3QbFhA8yZA7Nn63W9eprwR4zQXmQmOljyN55wTnsMFSSA+fOhZcvCk0H//joHi4l9v/4KX3yh/+vZszX5n3124f/aBmBFJ0v+plIcOKD9tgsSxPffQ69ehQmie3drK4gV+fmQkVH4v/zvfyElpfAXXs+e1lgbCyz5G1/s2aO/BgoSyJYthaXF3r11BtKqtpZcVMjL05lh09P1f/X559pDp+DEffrpUKuW31Ga8rLkb6LC+vVaPTR3rv5C+PlnXaKyRw8tSfbooSM9bV52b+XmwtKlsHChlugXLtR2nFatdB6oAQO0dN+kid+Rmoqy5G+i0p49WjUUnoTWrtUTQPgJoWtXW4D7aOXk6NKfCxcWvs8rVkDbtoXvcc+ecPLJVrKPR5b8TczYu1fnGgo/IWRl6SylPXtqVVFycuHFRoaqHTt0pPbatXpZtkzfv1WrCt+7gmR/0klQo4bfEZvK4FnyF5FLgAeBTkAv51xGaHsrYDmQGdo13Tn3+9BjPYE3gRrAJ8BtrgxBWPIPrpwcWLxYk9nKlZrcChJdYuKhJ4PkZO15kpys0whUr+539JGRkwM//VSY3MMT/dq12vMq/Pjbt9dE36VL/LwHpvxKSv4VbW5bClwIvFLEY6udc92K2D4BuBH4D5r8hwCfVjAOE8dq1NC1CE499dDtzumAs/Bk+MMPMG2a3v/5Z52/qHVrPRE0aAD16+uvheKua9f2ftCac1rltXOnltiLu962rTDhb9+u3WgLTmzJydpoXpDs69e3wXamfCqU/J1zywGkjJ86EWkC1HHOpYfuvw0Mw5K/OQoimtwbNtSGysPl50N2tibPdes0ge7cqfcXLSo64ebm6rrI4SeFOnW0V1JCgl6qVCm8XdDdMT+/8HLwYOHtAwdg165DX2PXLj2hHX7iCb/dpo0eU6tWmuCbNrVusiayvOxolywii4DdwF+cc18CzYD1YfusD20rkoiMAkYBtGzZ0sNQTTxKSNDScnk+Onl5mqTDTwq7dmkSPzyxF1xEjjwhFNyvWvXIxF6vnnVxNf4r9SMoInOA44t46D7n3PRi/mwj0NI5ty1Uxz9NRE4sb3DOuYnARNA6//L+vTHllZgIjRrpxZh4Vmryd84NLO+TOudygdzQ7YUishpoD2QDzcN2bR7aZowxphJ5UosoIo1EJCF0uzXQDljjnNsI7BaR3qINBdcAxf16MMYY45EKJX8RGS4i64E+wMciMiv00OnAYhH5Hngf+L1zbnvosZuB14AsYDXW2GuMMZXOBnkZY0ycKqmfv3UeM8aYALLkb4wxAWTJ3xhjAsiSvzHGBFDMNPiKyBZg3VH+eUNgawTD8VO8HEu8HAfYsUSjeDkOqNixnOCcK3LIYswk/4oQkYziWrxjTbwcS7wcB9ixRKN4OQ7w7lis2scYYwLIkr8xxgRQUJL/RL8DiKB4OZZ4OQ6wY4lG8XIc4NGxBKLO3xhjzKGCUvI3xhgTxpK/McYEUFwlfxG5RER+FJGDIpIStr2ViOSIyPehy8thj/UUkSUikiUiz0lZ16T0WHHHEnpsbCjeTBEZHLZ9SGhbloiMqfyoSyciD4pIdtj/4rywx4o8rmgVC+93SUTkp9Bn/3sRyQhtO1ZEZovIqtB1fb/jLIqIvC4im0Vkadi2ImMX9Vzo/7RYRHr4F/mRijkW778nzrm4uQCdgA7AfCAlbHsrYGkxf/Mt0BsQdHrpc/0+jlKOpTPwA1ANSEanxU4IXVYDrYGk0D6d/T6OIo7rQeDuIrYXeVx+x1vCccTE+13KMfwENDxs29+BMaHbY4DH/Y6zmNhPB3qEf6+Lix04L/TdltB3/T9+x1+GY/H8exJXJX/n3HLnXGZZ9w9fUN7pO1uwoLzvSjiWVGCycy7XObcWXRehV+iS5Zxb45zbD0wO7RsrijuuaBXr73dxUoG3QrffIkq+D4dzzn0BbD9sc3GxpwJvO5UO1At996NCMcdSnIh9T+Iq+ZciWUQWicgCEekf2lauBeWjRDPg57D7BTEXtz0a3RL6+f16WLVCLMUPsRdvURzwmYgsFJFRoW2Nna64B/AL0Nif0I5KcbHH6v/K0+9JqWv4Rhs/F5SPtKM8lqhX0nEBE4CH0MTzEPAkcH3lRWfC9HPOZYvIccBsEVkR/qBzzolITPYFj+XYQzz/nsRc8ndxtKD80RwLGl+LsPvhMRe3vVKV9bhE5FVgRuhuSccVjWIt3iM457JD15tF5EO0+mCTiDRxzm0MVY1s9jXI8iku9pj7XznnNhXc9up7EohqH4mvBeXTgBEiUk1EktFj+Rb4DmgnIskikgSMCO0bVQ6rax0OFPRwKO64olVMvN/FEZFaInJMwW3gHPR/kQaMDO02kuj/PoQrLvY04JpQr5/ewK6w6qGoVCnfE79buiPcaj4crQPLBTYBs0LbLwJ+BL4H/gv8X9jfpITe2NXAC4RGPft9Ke5YQo/dF4o3k7DeSWivhpWhx+7z+xiKOa53gCXA4tAHuUlpxxWtl1h4v0uIvTXaa+SH0HfjvtD2BsBcYBUwBzjW71iLiX8SWp2bF/qe3FBc7GgvnxdD/6clhPWei4ZLMcfi+ffEpncwxpgACkS1jzHGmENZ8jfGmACy5G+MMQFkyd8YYwLIkr8xxgSQJX9jjAkgS/7GGBNA/w9R09K5SdZvDQAAAABJRU5ErkJggg==\n"
          },
          "metadata": {
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# DDPG Agent"
      ],
      "metadata": {
        "id": "6rYoxokfLYEy"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Replay buffer\n",
        "Typically, people implement replay buffers with one of the following three data structures:\n",
        "\n",
        "- collections.deque\n",
        "- list\n",
        "- numpy.ndarray\n",
        "\n",
        "**deque** is very easy to handle once you initialize its maximum length (e.g. deque(maxlen=buffer_size)). However, the indexing operation of deque gets terribly slow as it grows up because it is [internally doubly linked list](https://wiki.python.org/moin/TimeComplexity#collections.deque). On the other hands, **list** is an array, so it is relatively faster than deque when you sample batches at every step. Its amortized cost of Get item is [O(1)](https://wiki.python.org/moin/TimeComplexity#list).\n",
        "\n",
        "Last but not least, let's see **numpy.ndarray**. numpy.ndarray is even faster than list due to the fact that it is [a homogeneous array of fixed-size items](https://docs.scipy.org/doc/numpy/reference/generated/numpy.ndarray.html#numpy.ndarray), so you can get the benefits of [locality of reference](https://en.wikipedia.org/wiki/Locality_of_reference), . Whereas list is an array of pointers to objects, even when all of them are of the same type.\n",
        "\n",
        "Here, we are going to implement a replay buffer using numpy.ndarray.\n",
        "\n",
        "Reference: \n",
        "- [OpenAI spinning-up](https://github.com/openai/spinningup/blob/master/spinup/algos/sac/sac.py#L10)\n",
        "- [rainbow-is-all-you-need](https://render.githubusercontent.com/view/ipynb?commit=032d11277cf2436853478a69ca5a4aba03202598&enc_url=68747470733a2f2f7261772e67697468756275736572636f6e74656e742e636f6d2f437572742d5061726b2f7261696e626f772d69732d616c6c2d796f752d6e6565642f303332643131323737636632343336383533343738613639636135613461626130333230323539382f30312e64716e2e6970796e62&nwo=Curt-Park%2Frainbow-is-all-you-need&path=01.dqn.ipynb&repository_id=191133946&repository_type=Repository#Replay-buffer)"
      ],
      "metadata": {
        "id": "Xr5CEJAANWY6"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "class ReplayBuffer:\n",
        "    \"\"\"A simple numpy replay buffer.\"\"\"\n",
        "\n",
        "    def __init__(self, obs_dim: int, action_dim: int, size: int, batch_size: int = 32):\n",
        "        \"\"\"Initializate.\"\"\"\n",
        "        self.obs_buf = np.zeros([size, obs_dim], dtype=np.float32)\n",
        "        self.next_obs_buf = np.zeros([size, obs_dim], dtype=np.float32)\n",
        "        self.acts_buf = np.zeros([size, action_dim], dtype=np.float32)\n",
        "        self.rews_buf = np.zeros([size], dtype=np.float32)\n",
        "        self.done_buf = np.zeros([size], dtype=np.float32)\n",
        "        self.max_size, self.batch_size = size, batch_size\n",
        "        self.ptr, self.size, = 0, 0\n",
        "\n",
        "    def store(\n",
        "        self,\n",
        "        obs: np.ndarray,\n",
        "        act: np.ndarray, \n",
        "        rew: float, \n",
        "        next_obs: np.ndarray, \n",
        "        done: bool,\n",
        "    ):\n",
        "        \"\"\"Store the transition in buffer.\"\"\"\n",
        "        self.obs_buf[self.ptr] = obs\n",
        "        self.next_obs_buf[self.ptr] = next_obs\n",
        "        self.acts_buf[self.ptr] = act\n",
        "        self.rews_buf[self.ptr] = rew\n",
        "        self.done_buf[self.ptr] = done\n",
        "        self.ptr = (self.ptr + 1) % self.max_size\n",
        "        self.size = min(self.size + 1, self.max_size)\n",
        "\n",
        "    def sample_batch(self) -> Dict[str, np.ndarray]:\n",
        "        \"\"\"Randomly sample a batch of experiences from memory.\"\"\"\n",
        "        idxs = np.random.choice(self.size, size=self.batch_size, replace=False)\n",
        "        return dict(obs=self.obs_buf[idxs],\n",
        "                    next_obs=self.next_obs_buf[idxs],\n",
        "                    acts=self.acts_buf[idxs],\n",
        "                    rews=self.rews_buf[idxs],\n",
        "                    done=self.done_buf[idxs])\n",
        "\n",
        "    def __len__(self) -> int:\n",
        "        return self.size"
      ],
      "metadata": {
        "id": "r2U6kzQgLa4L"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "## OU Noise\n",
        "**Ornstein-Uhlenbeck** process generates temporally correlated exploration, and it effectively copes with physical control problems of inertia.\n",
        "\n",
        "$$\n",
        "dx_t = \\theta(\\mu - x_t) dt + \\sigma dW_t\n",
        "$$"
      ],
      "metadata": {
        "id": "5xUhLjdsNRlC"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "class OUNoise:\n",
        "    \"\"\"Ornstein-Uhlenbeck process.\n",
        "    Taken from Udacity deep-reinforcement-learning github repository:\n",
        "    https://github.com/udacity/deep-reinforcement-learning/blob/master/\n",
        "    ddpg-pendulum/ddpg_agent.py\n",
        "    \"\"\"\n",
        "\n",
        "    def __init__(\n",
        "        self, \n",
        "        size: int, \n",
        "        mu: float = 0.0, \n",
        "        theta: float = 0.15, \n",
        "        sigma: float = 0.2,\n",
        "    ):\n",
        "        \"\"\"Initialize parameters and noise process.\"\"\"\n",
        "        self.state = np.float64(0.0)\n",
        "        self.mu = mu * np.ones(size)\n",
        "        self.theta = theta\n",
        "        self.sigma = sigma\n",
        "        self.reset()\n",
        "\n",
        "    def reset(self):\n",
        "        \"\"\"Reset the internal state (= noise) to mean (mu).\"\"\"\n",
        "        self.state = copy.copy(self.mu)\n",
        "\n",
        "    def sample(self) -> np.ndarray:\n",
        "        \"\"\"Update internal state and return it as a noise sample.\"\"\"\n",
        "        x = self.state\n",
        "        dx = self.theta * (self.mu - x) + self.sigma * np.array(\n",
        "            [random.random() for _ in range(len(x))]\n",
        "        )\n",
        "        self.state = x + dx\n",
        "        return self.state"
      ],
      "metadata": {
        "id": "jVpi-esHMevz"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Network"
      ],
      "metadata": {
        "id": "TsvTlYXDNfJK"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Old network"
      ],
      "metadata": {
        "id": "4tUssQyJkx2f"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "class Actor(nn.Module):\n",
        "    def __init__(\n",
        "        self, \n",
        "        in_dim: int, \n",
        "        out_dim: int,\n",
        "        init_w: float = 3e-3,\n",
        "    ):\n",
        "        \"\"\"Initialize.\"\"\"\n",
        "        super(Actor, self).__init__()\n",
        "        self.hidden1 = nn.Linear(in_dim, 128)\n",
        "        self.hidden2 = nn.Linear(128, 128)\n",
        "        self.out = nn.Linear(128, out_dim)        \n",
        "        self.out.weight.data.uniform_(-init_w, init_w)\n",
        "        self.out.bias.data.uniform_(-init_w, init_w)\n",
        "\n",
        "    def forward(self, state: torch.Tensor) -> torch.Tensor:\n",
        "        \"\"\"Forward method implementation.\"\"\"\n",
        "        x = F.relu(self.hidden1(state))\n",
        "        x = F.tanh(self.hidden2(x))\n",
        "        x = F.relu(self.hidden2(x))\n",
        "        x = F.tanh(self.hidden2(x))\n",
        "        x = F.relu(self.hidden2(x))\n",
        "        action = self.out(x).tanh()\n",
        "    \n",
        "        return action\n",
        "    \n",
        "    \n",
        "class Critic(nn.Module):\n",
        "    def __init__(\n",
        "        self, \n",
        "        in_dim: int, \n",
        "        init_w: float = 3e-3,\n",
        "    ):\n",
        "        \"\"\"Initialize.\"\"\"\n",
        "        super(Critic, self).__init__()\n",
        "        \n",
        "        self.hidden1 = nn.Linear(in_dim, 128)\n",
        "        self.hidden2 = nn.Linear(128, 128)\n",
        "        self.out = nn.Linear(128, 1)\n",
        "        \n",
        "        self.out.weight.data.uniform_(-init_w, init_w)\n",
        "        self.out.bias.data.uniform_(-init_w, init_w)\n",
        "\n",
        "    def forward(\n",
        "        self, state: torch.Tensor, action: torch.Tensor\n",
        "    ) -> torch.Tensor:\n",
        "        \"\"\"Forward method implementation.\"\"\"\n",
        "        x = torch.cat((state, action), dim=-1)\n",
        "        x = F.relu(self.hidden1(x))\n",
        "        x = F.tanh(self.hidden2(x))\n",
        "        x = F.relu(self.hidden2(x))\n",
        "        x = F.tanh(self.hidden2(x))\n",
        "        x = F.relu(self.hidden2(x))\n",
        "        value = self.out(x)\n",
        "        \n",
        "        return value"
      ],
      "metadata": {
        "id": "hkQ83MuwMiIs"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "### New network"
      ],
      "metadata": {
        "id": "vMKb-zkdkz1F"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# WEIGHTS_FINAL_INIT = 3e-3\n",
        "# BIAS_FINAL_INIT = 3e-4\n",
        "\n",
        "\n",
        "# def fan_in_uniform_init(tensor, fan_in=None):\n",
        "#     \"\"\"Utility function for initializing actor and critic\"\"\"\n",
        "#     if fan_in is None:\n",
        "#         fan_in = tensor.size(-1)\n",
        "\n",
        "#     w = 1. / np.sqrt(fan_in)\n",
        "#     nn.init.uniform_(tensor, -w, w)\n",
        "\n",
        "\n",
        "# class Actor(nn.Module):\n",
        "#     def __init__(self, hidden_size, \n",
        "#                  num_inputs, action_space):\n",
        "#         super(Actor, self).__init__()\n",
        "#         self.action_space = action_space\n",
        "#         num_outputs = action_space.shape[0]\n",
        "\n",
        "#         # Layer 1\n",
        "#         self.linear1 = nn.Linear(num_inputs, hidden_size[0])\n",
        "#         self.ln1 = nn.LayerNorm(hidden_size[0])\n",
        "\n",
        "#         # Layer 2\n",
        "#         self.linear2 = nn.Linear(hidden_size[0], hidden_size[1])\n",
        "#         self.ln2 = nn.LayerNorm(hidden_size[1])\n",
        "\n",
        "#         # Output Layer\n",
        "#         self.mu = nn.Linear(hidden_size[1], num_outputs)\n",
        "\n",
        "#         # Weight Init\n",
        "#         fan_in_uniform_init(self.linear1.weight)\n",
        "#         fan_in_uniform_init(self.linear1.bias)\n",
        "\n",
        "#         fan_in_uniform_init(self.linear2.weight)\n",
        "#         fan_in_uniform_init(self.linear2.bias)\n",
        "\n",
        "#         nn.init.uniform_(self.mu.weight, -WEIGHTS_FINAL_INIT, WEIGHTS_FINAL_INIT)\n",
        "#         nn.init.uniform_(self.mu.bias, -BIAS_FINAL_INIT, BIAS_FINAL_INIT)\n",
        "\n",
        "#     def forward(self, inputs):\n",
        "#         x = inputs\n",
        "\n",
        "#         # Layer 1\n",
        "#         x = self.linear1(x)\n",
        "#         x = self.ln1(x)\n",
        "#         x = F.relu(x)\n",
        "\n",
        "#         # Layer 2\n",
        "#         x = self.linear2(x)\n",
        "#         x = self.ln2(x)\n",
        "#         x = F.relu(x)\n",
        "\n",
        "#         # Output\n",
        "#         mu = torch.tanh(self.mu(x))\n",
        "#         return mu\n",
        "\n",
        "# class Critic(nn.Module):\n",
        "#     def __init__(self, hidden_size, \n",
        "#                  num_inputs, action_space):\n",
        "#         super(Critic, self).__init__()\n",
        "#         self.action_space = action_space\n",
        "#         num_outputs = action_space.shape[0]\n",
        "\n",
        "#         # Layer 1\n",
        "#         self.linear1 = nn.Linear(num_inputs, hidden_size[0])\n",
        "#         self.ln1 = nn.LayerNorm(hidden_size[0])\n",
        "\n",
        "#         # Layer 2\n",
        "#         # In the second layer the actions will be inserted also \n",
        "#         self.linear2 = nn.Linear(hidden_size[0] + num_outputs, hidden_size[1])\n",
        "#         self.ln2 = nn.LayerNorm(hidden_size[1])\n",
        "\n",
        "#         # Output layer (single value)\n",
        "#         self.V = nn.Linear(hidden_size[1], 1)\n",
        "\n",
        "#         # Weight Init\n",
        "#         fan_in_uniform_init(self.linear1.weight)\n",
        "#         fan_in_uniform_init(self.linear1.bias)\n",
        "\n",
        "#         fan_in_uniform_init(self.linear2.weight)\n",
        "#         fan_in_uniform_init(self.linear2.bias)\n",
        "\n",
        "#         nn.init.uniform_(self.V.weight, -WEIGHTS_FINAL_INIT, WEIGHTS_FINAL_INIT)\n",
        "#         nn.init.uniform_(self.V.bias, -BIAS_FINAL_INIT, BIAS_FINAL_INIT)\n",
        "\n",
        "#     def forward(self, inputs, actions):\n",
        "#         x = inputs\n",
        "\n",
        "#         # Layer 1\n",
        "#         x = self.linear1(x)\n",
        "#         x = self.ln1(x)\n",
        "#         x = F.relu(x)\n",
        "\n",
        "#         # Layer 2\n",
        "#         x = torch.cat((x, actions), 1)  # Insert the actions\n",
        "#         x = self.linear2(x)\n",
        "#         x = self.ln2(x)\n",
        "#         x = F.relu(x)\n",
        "\n",
        "#         # Output\n",
        "#         V = self.V(x)\n",
        "#         return V"
      ],
      "metadata": {
        "id": "xRUeMz45k1i3"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "## DDPG Agent\n",
        "Here is a summary of DDPGAgent class.\n",
        "\n",
        "| Method           | Note                                                 |\n",
        "|---               |---                                                  |\n",
        "|select_action     | select an action from the input state.               |\n",
        "|step              | take an action and return the response of the env.   |\n",
        "|update_model      | update the model by gradient descent.                |\n",
        "|train             | train the agent during num_frames.                   |\n",
        "|test              | test the agent (1 episode).                          |\n",
        "|\\_target_soft_update| soft update from the local model to the target model.|\n",
        "|\\_plot              | plot the training progresses.     "
      ],
      "metadata": {
        "id": "7aAkNYiZNLjL"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "class DDPGAgent:\n",
        "    \"\"\"DDPGAgent interacting with environment.\n",
        "    \n",
        "    Attribute:\n",
        "        env (gym.Env): openAI Gym environment\n",
        "        actor (nn.Module): target actor model to select actions\n",
        "        actor_target (nn.Module): actor model to predict next actions\n",
        "        actor_optimizer (Optimizer): optimizer for training actor\n",
        "        critic (nn.Module): critic model to predict state values\n",
        "        critic_target (nn.Module): target critic model to predict state values\n",
        "        critic_optimizer (Optimizer): optimizer for training critic\n",
        "        memory (ReplayBuffer): replay memory to store transitions\n",
        "        batch_size (int): batch size for sampling\n",
        "        gamma (float): discount factor\n",
        "        tau (float): parameter for soft target update\n",
        "        initial_random_steps (int): initial random action steps\n",
        "        noise (OUNoise): noise generator for exploration\n",
        "        device (torch.device): cpu / gpu\n",
        "        transition (list): temporory storage for the recent transition\n",
        "        total_step (int): total step numbers\n",
        "        is_test (bool): flag to show the current mode (train / test)\n",
        "    \"\"\"\n",
        "    def __init__(\n",
        "        self,\n",
        "        env,\n",
        "        memory_size: int,\n",
        "        batch_size: int,\n",
        "        ou_noise_theta: float,\n",
        "        ou_noise_sigma: float,\n",
        "        gamma: float = 0.99,\n",
        "        tau: float = 5e-3,\n",
        "        initial_random_steps: int = 1e4,\n",
        "    ):\n",
        "        \"\"\"Initialize.\"\"\"\n",
        "        obs_dim = env.observation_space.shape[0]\n",
        "        print(env.observation_space.shape)\n",
        "        action_dim = env.action_space.shape[0]\n",
        "        print(env.action_space.shape)\n",
        "\n",
        "        self.env = env\n",
        "        self.memory = ReplayBuffer(obs_dim, action_dim, memory_size, batch_size)\n",
        "        self.batch_size = batch_size\n",
        "        self.gamma = gamma\n",
        "        self.tau = tau\n",
        "        self.initial_random_steps = initial_random_steps\n",
        "                \n",
        "        # noise\n",
        "        self.noise = OUNoise(\n",
        "            action_dim,\n",
        "            theta=ou_noise_theta,\n",
        "            sigma=ou_noise_sigma,\n",
        "        )\n",
        "\n",
        "        # device: cpu / gpu\n",
        "        self.device = torch.device(\n",
        "            \"cuda\" if torch.cuda.is_available() else \"cpu\"\n",
        "        )\n",
        "        print(self.device)\n",
        "\n",
        "        # networks\n",
        "        self.actor = Actor(obs_dim, action_dim).to(self.device)\n",
        "        self.actor_target = Actor(obs_dim, action_dim).to(self.device)\n",
        "        self.actor_target.load_state_dict(self.actor.state_dict())\n",
        "        \n",
        "        self.critic = Critic(obs_dim + action_dim).to(self.device)\n",
        "        self.critic_target = Critic(obs_dim + action_dim).to(self.device)\n",
        "        self.critic_target.load_state_dict(self.critic.state_dict())\n",
        "\n",
        "        # optimizer\n",
        "        self.actor_optimizer = optim.Adam(self.actor.parameters(), lr=3e-4)\n",
        "        self.critic_optimizer = optim.Adam(self.critic.parameters(), lr=1e-3)\n",
        "        \n",
        "        # transition to store in memory\n",
        "        self.transition = list()\n",
        "        \n",
        "        # total steps count\n",
        "        self.total_step = 0\n",
        "\n",
        "        # mode: train / test\n",
        "        self.is_test = False\n",
        "    \n",
        "    def select_action(self, state: np.ndarray) -> np.ndarray:\n",
        "        \"\"\"Select an action from the input state.\"\"\"\n",
        "        # if initial random action should be conducted\n",
        "        if self.total_step < self.initial_random_steps and not self.is_test:\n",
        "            selected_action = self.actor(\n",
        "                torch.FloatTensor(state).to(self.device)\n",
        "            ).detach().cpu().numpy()\n",
        "        else:\n",
        "            selected_action = self.actor(\n",
        "                torch.FloatTensor(state).to(self.device)\n",
        "            ).detach().cpu().numpy()\n",
        "        \n",
        "        # add noise for exploration during training\n",
        "        if not self.is_test:\n",
        "            noise = self.noise.sample()\n",
        "            selected_action = np.clip(selected_action + noise, -1.0, 1.0)\n",
        "        \n",
        "        self.transition = [state, selected_action]\n",
        "        \n",
        "        return selected_action\n",
        "    \n",
        "    def step(self, action: np.ndarray) -> Tuple[np.ndarray, np.float64, bool]:\n",
        "        \"\"\"Take an action and return the response of the env.\"\"\"\n",
        "        next_state, reward, done, _ = self.env.step(action)\n",
        "        \n",
        "        if not self.is_test:\n",
        "            self.transition += [reward, next_state, done]\n",
        "            # print(self.transition)\n",
        "            self.memory.store(*self.transition)\n",
        "    \n",
        "        return next_state, reward, done\n",
        "    \n",
        "    def update_model(self) -> torch.Tensor:\n",
        "        \"\"\"Update the model by gradient descent.\"\"\"\n",
        "        device = self.device  # for shortening the following lines\n",
        "        \n",
        "        samples = self.memory.sample_batch()\n",
        "        state      = torch.FloatTensor(samples[\"obs\"]).to(device)\n",
        "        next_state = torch.FloatTensor(samples[\"next_obs\"]).to(device)\n",
        "        action     = torch.FloatTensor(samples[\"acts\"]).to(device)     \n",
        "        reward     = torch.FloatTensor(samples[\"rews\"].reshape(-1, 1)).to(device)\n",
        "        done       = torch.FloatTensor(samples[\"done\"].reshape(-1, 1)).to(device)\n",
        "        if 0 == True: \n",
        "            print(f\"state size: {np.shape(state)}\")\n",
        "\n",
        "        masks = 1 - done\n",
        "        next_action = self.actor_target(next_state)\n",
        "        # print(f\"next action: {np.shape(next_action)}\")\n",
        "        next_value = self.critic_target(next_state, next_action)\n",
        "        curr_return = reward + self.gamma * next_value * masks\n",
        "        \n",
        "        # train critic\n",
        "        values = self.critic(state, action)\n",
        "        critic_loss = F.mse_loss(values, curr_return)\n",
        "        \n",
        "        self.critic_optimizer.zero_grad()\n",
        "        critic_loss.backward()\n",
        "        self.critic_optimizer.step()\n",
        "                \n",
        "        # train actor\n",
        "        actor_loss = -self.critic(state, self.actor(state)).mean()\n",
        "        \n",
        "        self.actor_optimizer.zero_grad()\n",
        "        actor_loss.backward()\n",
        "        self.actor_optimizer.step()\n",
        "        \n",
        "        # target update\n",
        "        self._target_soft_update()\n",
        "        \n",
        "        return actor_loss.data, critic_loss.data\n",
        "    \n",
        "    def train(self, num_frames: int, plotting_interval: int = 20000):\n",
        "        \"\"\"Train the agent.\"\"\"\n",
        "        self.is_test = False\n",
        "        \n",
        "        state = self.env.reset().squeeze()\n",
        "        \n",
        "        actor_losses = []\n",
        "        critic_losses = []\n",
        "        scores = []\n",
        "        score = 0\n",
        "        \n",
        "        for self.total_step in range(1, num_frames + 1):\n",
        "            action = self.select_action(state)\n",
        "            next_state, reward, done = self.step(action)\n",
        "            next_state = next_state.squeeze()\n",
        "            print(f\"reward of step {self.total_step} is: {reward}\")\n",
        "            state = next_state\n",
        "\n",
        "            score = score + reward\n",
        "            # if episode ends\n",
        "            if done:         \n",
        "                state = env.reset().squeeze()\n",
        "                scores.append(score)\n",
        "                score = 0\n",
        "\n",
        "            # if training is ready\n",
        "            if (\n",
        "                len(self.memory) >= self.batch_size \n",
        "                and self.total_step > self.initial_random_steps\n",
        "            ):\n",
        "                actor_loss, critic_loss = self.update_model()\n",
        "                actor_losses.append(actor_loss)\n",
        "                critic_losses.append(critic_loss)\n",
        "            \n",
        "            # plotting\n",
        "            if self.total_step % plotting_interval == 0:\n",
        "                self._plot(\n",
        "                    self.total_step, \n",
        "                    scores, \n",
        "                    actor_losses, \n",
        "                    critic_losses,\n",
        "                )\n",
        "                pass\n",
        "        self.env.close()\n",
        "        \n",
        "    def test(self):\n",
        "        # \"\"\"Test the agent.\"\"\"\n",
        "        # self.is_test = True\n",
        "        \n",
        "        # state = self.env.reset()\n",
        "        # done = False\n",
        "        # score = 0\n",
        "        \n",
        "        # frames = []\n",
        "        # while not done:\n",
        "        #     frames.append(self.env.render(mode=\"rgb_array\"))\n",
        "        #     action = self.select_action(state)\n",
        "        #     next_state, reward, done = self.step(action)\n",
        "\n",
        "        #     state = next_state\n",
        "        #     score = self.discount*score + reward\n",
        "        \n",
        "        # print(\"score: \", score)\n",
        "        # self.env.close()\n",
        "        \n",
        "        # return frames\n",
        "        pass\n",
        "    \n",
        "    def _target_soft_update(self):\n",
        "        \"\"\"Soft-update: target = tau*local + (1-tau)*target.\"\"\"\n",
        "        tau = self.tau\n",
        "        \n",
        "        for t_param, l_param in zip(\n",
        "            self.actor_target.parameters(), self.actor.parameters()\n",
        "        ):\n",
        "            t_param.data.copy_(tau * l_param.data + (1.0 - tau) * t_param.data)\n",
        "            \n",
        "        for t_param, l_param in zip(\n",
        "            self.critic_target.parameters(), self.critic.parameters()\n",
        "        ):\n",
        "            t_param.data.copy_(tau * l_param.data + (1.0 - tau) * t_param.data)\n",
        "    \n",
        "    def _plot(\n",
        "        self, \n",
        "        frame_idx: int, \n",
        "        scores: List[float], \n",
        "        actor_losses: List[float], \n",
        "        critic_losses: List[float], \n",
        "    ):\n",
        "        \"\"\"Plot the training progresses.\"\"\"\n",
        "        def subplot(loc: int, title: str, values: List[float]):\n",
        "            plt.subplot(loc)\n",
        "            plt.title(title)\n",
        "            plt.plot(values)\n",
        "\n",
        "        subplot_params = [\n",
        "            (131, f\"frame {frame_idx}. score: {np.mean(scores[-10:])}\", scores),\n",
        "            (132, \"actor_loss\", actor_losses),\n",
        "            (133, \"critic_loss\", critic_losses),\n",
        "        ]\n",
        "        \n",
        "        clear_output(True)\n",
        "        plt.figure(figsize=(30, 5))\n",
        "        for loc, title, values in subplot_params:\n",
        "            subplot(loc, title, values)\n",
        "        plt.show()"
      ],
      "metadata": {
        "id": "GW1sIaf9Mkv6"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Algorithm Execution"
      ],
      "metadata": {
        "id": "mWavl7GIMqH5"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "num_frames = 50000\n",
        "memory_size = 100000\n",
        "batch_size = 128\n",
        "ou_noise_theta = 1.0\n",
        "ou_noise_sigma = 0.1\n",
        "initial_random_steps = 10000\n",
        "\n",
        "agent = DDPGAgent(\n",
        "    env, \n",
        "    memory_size, \n",
        "    batch_size,\n",
        "    ou_noise_theta,\n",
        "    ou_noise_sigma,\n",
        "    initial_random_steps=initial_random_steps\n",
        ")"
      ],
      "metadata": {
        "id": "3dXH12mlMt0C",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "6871d67f-dd05-4d55-c021-03c932603f3e"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "(18,)\n",
            "(12,)\n",
            "cpu\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "agent.train(num_frames)"
      ],
      "metadata": {
        "id": "Ya3or16UM0SS",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "outputId": "18e3063d-f66f-46f7-8acb-83f16aad4f70"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 2160x360 with 3 Axes>"
            ],
            "image/png": "iVBORw0KGgoAAAANSUhEUgAABr0AAAE/CAYAAAD7bQgUAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nOzdd3xUVfrH8e+T0DvSpIeiICCgIHZUxAKouLvu2vuK7q6rrvtTEXsFde26a8NewLri0qX3Jr230EuoCQnp5/fHTIaZZCaZJJNMEj7v14vX3nvPufc+mcB65z7nPMeccwIAAAAAAAAAAADKs5hoBwAAAAAAAAAAAAAUF0kvAAAAAAAAAAAAlHskvQAAAAAAAAAAAFDukfQCAAAAAAAAAABAuUfSCwAAAAAAAAAAAOUeSS8AAAAAAAAAAACUeyS9gFJgZh3MbImZJZnZfdGOBwAAAAAAAACAioakF1A6HpY0xTlX2zn3VrSD8WdmJ5vZz2aWYGYHzGy8mXXI1ecfZrbbzBLN7GMzq+rXFmdmU8wsxczWmFnfSJ0LAACAwjOzeJ6rAABARWBmN5rZhHzazzeztcW4/m1mNrOo5wMoe0h6AaWjtaSVoRrNLLYUY8mtnqRRkjpIaiJpvqSfcxrN7DJJgyVdLM/P0VbSM37nfyNpsaQGkh6T9L2ZNSruudFkHvz/IwAAKDEkpgAAAArmnPvKOXdpzr6ZOTNr79c+wznXIfjZAI5HvNQFSpiZTZZ0kaR3zOyId2bVp2b2HzMbY2bJki4yswFmttg7I2qbmT3td40473/Ub/e2HTSze8zsDDNbZmaHzOydXPe9w8xWe/uON7PWweJzzs13zg13zh1wzmVIel1SBzNr4O1yq6ThzrmVzrmDkp6TdJv3HidLOl3SU865o865HyQtl/SHCJxb0Of6iJnt8JaMXGtmF3uPx5rZEDPb6G1bZGYtvW3nmNkCMzvs/d9z/K431cxeMLNZklIktTWzjmY20TsDbq2Z/Smc2Pyu95yZzfLGMcHMGvq1f+edAXfYzKabWWe/tk/N7F0zG+09d56ZtQv33gAAoGIzs0rRjgEAAKCk8cwDoChIegElzDnXR9IMSfc652o559Z5m26Q9IKk2pJmSkqWdIs8M68GSPqLmV2d63JnSjpJ0rWS3pBndlRfSZ0l/cnMLpAkMxsoaYik30tq5L3/N2GG3FvSbufcfu9+Z0lL/dqXSmriTYp1lrTJOZeUq71zBM4NyVt+8V5JZzjnaku6TFK8t/lBSddL6i+pjqQ7JKWY2QmSRkt6S56ZZa9JGu2X3JOkmyUNkud3kiBpoqSvJTWWdJ2kf5tZJ28MN5jZsgJCvUHS7d7zq0j6P7+2sfL8LhtL+k3SV7nOvU6eWXH1JW2Q5+8KAAAog8xssN+Am1Vm9ju/tru8A5Fy2k43sy8ktZL0i3dQ1MPevleZ2UrvgKapZnaK33XivYN+lklKDvclkJlVNbM3zGyn988b5i03bWYNzex/3vsdMLMZObPdQw0wAgAAiAQza2lmP5pnuY39ZvaOeUoNzjKz181sv6Snza/8oJlN956+1PsMda2ZXWhm2/O7biHjym/A9G1mtsn7fLTZzG70Hm9vZtO85+wzs5HF/4QAFBVJLyB6fnbOzXLOZTvnUp1zU51zy737y+RJUl2Q65znvH0nyJMk+8Y5t9c5t0OexNZp3n73SBrqnFvtnMuU9KKk7hZitlcOM2sh6V15Ekc5akk67Lefs107SFtOe+0InJufLElVJXUys8rOuXjn3EZv258lPe6cW+s8lnoTeAMkrXfOfeGcy3TOfSNpjaQr/a77qXdWWqakyyXFO+c+8fZfLOkHSX+UJOfc1865rgXE+Ylzbp1z7qikbyV1z2lwzn3snEtyzqVJelpSNzOr63fuT95ZeJnyJMS6CwAAlFUbJZ0vqa48g1a+NLOmZvZHef47f4s8g3GukrTfOXezpK2SrvQOinrZPLPgv5H0gDyDlsbIkxSr4nef6+V5pqnnfUYIx2OSzpLnWaKbpF6SHve2/VPSdu/9msgzaMoVMMAIAACgWMyzzMf/JG2RFCepuaQR3uYzJW2S59kkYACwc663d7Ob9xkqILlUwHXDiSvkgGkzq+k93s/7fHSOpCXeU5+TNEGegcstJL0d7j0BRB5JLyB6tvnvmNmZZjbFOxLlsDyJq4a5ztnjt300yH4t73ZrSW96R+0eknRAksnzH/ugzLOW1gRJ//YmhHIckeclTY6c7aQgbTntObO3inNuSM65DfK8EHpa0l4zG2FmzbzNLeV58ZRbM3keevxtUeBn4v87aS3pzJzP0Ps53ijpxILi87PbbztF3t+PeUowDvOOCE/UsZdIDQs6FwAAlD3Oue+cczu9g5dGSlovT3Lpz5Jeds4t8A7G2eCcy/08kuNaSaOdcxO9Jaf/Jam6PC9UcrzlnNvmHVATrhslPesdKJUgT1LuZm9bhqSmklo75zK8a2I45T/ACAAAoLh6yfOe5iHnXLJ3gPdMb9tO59zb3gHIhXnmKei64ShowHS2pC5mVt05t8s5t9J7PEOe90jNinBPABFG0guIHpdr/2tJoyS1dM7VlfSePImqotgm6W7nXD2/P9Wdc7ODdTaz+vIkvEY553KX0Vspz6jgHN0k7fHOnlopz9pXtXO1r4zAufnyzrQ6T56HCifpJb+fPdj6Vzu9ff21krTD/7J+29skTcv1GdZyzv0lnPgKcIOkgfKUpqwrz+gjqei/bwAAEEVmdouZLfEbKNNFnsEsoQbjBBMwQMc5ly3P80ioATrhyj3wZ4v3mCS9Ik8Z5QneUj2DvffOb4ARAABAcbWUtCXEzPWiPO+Ec91whBww7ZxLlmeQ0j2SdplnHfaO3j4Py/NOZ763VPUdRbw/gAgg6QWUHbUlHXDOpZpZL3kSI0X1nqRHzayzJJlZXW95nTzMrI6k8ZJmOecGB+nyuaQ7zayTmdWTpxzOp5LkXZ9siaSnzKyad/2KrvKUASzuuSGZWQcz6+NdjyJVnllu2d7mjyQ9Z2YnmUdX86zbNUbSyeZZi6uSmV0rqZM8096D+Z+3/81mVtn75wzzW1ujGGpLSpO0X1INecpPAgCAcshbPvpDecoBNnDO1ZO0Qp4XH6EG40h5B0AFDNAxM5PnxU2oATrhyj3wp5X3mLyllv/pnGsrT+nFB3PW7spngBEAAEBxbZPUyoKvUVqU551wrhuOfAdMO+fGO+cukWem/Bp5ngHlnNvtnLvLOddM0t3yrAnfvogxACgmkl5A2fFXSc+aWZKkJ+VZA6pInHM/yfNiYoS3fN4KSf1CdP+dpDMk3e5dBDTnTyvvtcZJelnSFHnWntgi6Sm/86+T1FPSQUnDJF3jLZ1TrHPN7EYzCzXrq6q3/z55ygA2lvSot+01eT67CZISJQ2XVN07u+wKedau2C/PKJwrnHP7QnyGSZIu9ca403ufl7z3Lii+gnwuz2exQ9IqSXOLeB0AABB9NeV5OZPzDHO7PDO9JM9gnP8zsx7ewTjt/dZY3SOprd91vpU0wMwuNrPK8jyzpEkKOlO/EL6R9LiZNTKzhvI8Z37pjfUKb0wmz9qqWZKyCxhgBAAAUFzzJe2SNMzManoHQ58b5rm5n6EidV0pnwHTZtbEzAZ61/ZKk2fZjmxJMrM/mlkL7zUOyvNsyLMTECXmKdkOAAAAACgKM3tB0l/kebnxuaQekr5wzn1kZvdI+oc8ZQrjJd3snFtsZgPlWeS8jqTnnXP/8s58f8Hbd4mkv+asFWFm8ZL+7Jz7NYx4fH3NrJo8g5ByZv1/J+lhb3WBf0i6X1IjeV7QvO+ce87MusqTsDtFnjUqZksa5JzbWZzPCQAAIId3sPVbks6XJ0n0taTf5HmGOc+v323+x7zPVk/Js/bpIEl7JX3pnGsR6rrOufvyiSP39c+T9Kak9vKUgb7fOTfTzJpKGiGpu/e6Oc9qq8zsZXnWUa0rT1LuJefcB8X9jAAUDUkvAAAAAAAAAAAAlHuUNwQAAChHzKylmU0xs1XeRZLvj3ZMAAAAAAAAZQFJLwAAgPIlU9I/nXOdJJ0l6W9m1inKMQEoRWbWKtdarHnWZQUAAICHmb0X4rnpvWjHBiDyKG8IAABQjpnZz5Lecc5NjHYsAAAAAAAA0cRMLwAAgHLKzOIknSZpXnQjAQAAAAAAiL5K0Q6gKBo2bOji4uKiHQYAAMjHokWL9jnnGkU7jorKzGpJ+kHSA865xCDtgyQNkqSaNWv26NixYylHCAAACoNnp7KBd04AAJQPoZ6dymXSKy4uTgsXLox2GAAAIB9mtiXaMVRUZlZZnoTXV865H4P1cc59IOkDSerZs6fj2QkAgLKNZ6eygXdOAACUD6GenShvCAAAUI6YmUkaLmm1c+61aMcDAAAAAABQVpD0AgAAKF/OlXSzpD5mtsT7p3+0gwIAAAAAAIi2clneEAAA4HjlnJspyaIdBwAAAAAAQFnDTC8AAAAAAAAAAACUeyS9AAAAAAAAAAAAUO6R9AIAAAAAAAAAAEC5R9ILAAAAAAAAAAAA5R5JLwAAAAAAAAAAAJR7JL0AAAAAAAAAAABQ7pH0AgAAAI5TexJTtWpnYsCxTQlH5JyLUkQAAACoqLYdSNHGhCPRDgNABUfSCwAAADjOpKRnauSCrRrw1kz1f2uGZm/cp8ten64pa/eqz6vT9NnseC3ackAnPTZGq3clFnxBAAAAoADnvzxFF786LdphAKjgKkU7AAAAAADF45xTtt/krNgYU2pGliQpMTVDaRnZemH0ao1buTvo+Td8OE+SdPsnCyRJ41bu1tO/rJIk9Xtzhta/0E8mqVJs8DFz3y3cpkVbDmrYH7pKkqatS1Csmc47qWEkfjwAAAAAAMJC0gsAAAAoZzbsTVLLE2po/5F0nTNsckBbnWqVNP+xvjpr6CQdSsko0vXnbjoQsH/SY2OPtT16sapXidWW/cnq2qKe1uxO1EPfL5MkjViwTd/fc7Zu/Xi+JCl+2ADfeTsOHdXns+P1yOUdFRNjRYoLAAAAKG827D2i5vWqq3qV2GiHAhwXSHoBAAAA5UBWttPKnYe1eleiHvlhech+iamZ6vjEuBKL46yhk1SjSqxS0rOCtl/z3hzf9updiTqxTjXVr1lFD323VLM37telnU9Uj9b1JXlmqI1fuVt9T2kSchYZAAAAUF6lZWap72vTdHHHxhp+2xnRDgc4LpD0AgAAAMqQA8npWrr9kE5tXlf93pyhfUfS9Mu95+mKt2dGOzSfUAmv3Pq9OUOS9OAlJ2v2xv2SPKUQ2zeupbrVK2v8yj2658vf9NBlHfS3i9qXWLwAAABANGR5a5DnPAsDKHkkvQAAAIAyYuv+FF3/4VztOHQ04HhZSngVxWsT1/m2RyzYphELtum+i0/SW5PWS5I+nLFJLepX18DuzZWRla0nf16hb+Zv07rn+6lKJWaAAQAAoHxIy8xSdrYoZQhEEUkvAAAAoBTsPHRUMWY6sW61gOM7Dh3VucMm6489Wui7RdujFF3py0l4SdKhlAzdP2KJalWtpDs/W+g7fvLjYzV/yMVqXKdasEsAAAAAZUqff03TjkNHfWvbOhflgIDjEMMmAQAAgBL285IdOmfYZJ01dJIeGLHYd3zU0p06d9hkSSp2wqtto5ph9z0jrn6+7df0aJFn/7M7evn244cN0JvXdS9cgGHwT3jluOGjeYobPFqbEo5IkpJSM7TtQErE7w0AAAAUV+6KDQBKHzO9AAAAgBKSmZWtoxlZun/EEt+x/y7ZqRU7E1WraiUt2XaoWNdvWKuK9h1J15D+HTWodzvFDR4tSZr20IX64bcd+t1pzfXprM3q2qKeLu9yojYlJKvlCdVVvUqs0jKzlZ6ZrfenbdSuw6n637JdeuKKTqoca7rl7Dj1anOCWtSrru6t6qlGlUpyuYapDuzeXA9+u9S3TsGlnZpowqo9xfp5gtmw15Ps6vPqNK1/oZ9u+Xi+Fm895Bs9u3Z3ko6kZahH6xMifm8AAAAAQPlC0gsAAAAoIfePWKLRy3flOZ6TyAnXkP4d9eKYNZKkJnWq6qYzW6tujcq66czW+nnpDl3ZtZkk6bz2DTVzwz61blBTD15ysiTpmYFdfNc5tUVd33bVSp51Bh4b0El7E1PVtmFN3Xp2a1WK9RSD+FPPlgExmJlGDjpLe5PSfMe+vPNMXf/hXA3q3VZD+p+iiav2qH3jWkpOy/StQ/bmdd11Sacm6vTkeEnS709rrgFdmwad1VWQkx4b69teuu2QurWsp8vemC5JviQYAAAAAJQ1d32+UE3qVNXzV58a7VAqPJJeAAAAQIQ452RmkqTUjKygCa/Cuvei9hrUu53q1aiih79fprt7t9Md57Xxtf/utGOlCD++7QylZWYV+h6N61TTg5d2KLDfmW0bBOyf3a5BQLLpkk5NfNtLnrxE8ftT1L1lPUnStT1b6r9Ldui1az1lEW8+q7W+mLul0LHmGPjuLH14S88inw8AAAAApWWityoGSa+SR9ILAAAAiIAnf16hz+ds0dKnLtW/p27Q+9M2Fer835/WXL8s26mMLKfNQ/v7kmc5/tSzZZ7ZV7lVqRSjKpXKxrK99WpUUfcaVXz7L/yui54Z2Nm3/9zVXfTc1Z5ZaD8t3q5/jFyqwf06atjYNWHf467Pj80WG7lgq05rVV8nN6kdgegBAACAyMn1aA+gBJH0AgAAAIohNSNLw8au0edzPLOWuj0zIexzf/jLOXr8vyu0eleinr26i28WVEVUKTZG3oqKefzutBa6smszVYqNKVTSy98jPyyXJP3413N0eqv6edo37D2iNg1rKjaGNw4AAAAAUFGVjWGgAAAAQDmUkZWtL+Zs0aez4wt13gk1q2jVs5epR+v6Gnv/+do8tL9qVT2+x6PlrCXWrG4137Hbzokr9HVu/HCeVuw4HHBsT2Kq+r42Tc+PXlWsGAEAAADktTD+gOIGj9bOQ0ejHQrATC8AAACgKI6kZarLU+MLfd5zV3fRDb1aBcw4yl3K8Hj2+Z291Pe16ZKkp6/qrCev6KS2Q8aEff7RjCxd8fbMgGMXnNxIkjRtXULkAgUAAAAgSfp6/lZJ0uyN+3VNjxYF9I4e55zGr9ytvqc08Q26y8/8zQf0p/fn6NcHe6t9Y8qolxfM9AIAAADCtHV/igZ9vlAfz9xcYMKrRpVjtfwm/KO3b/vms1pTYi8f7RvX1oyHL9JE72cWE2PqeKLnC+bjA07x9fvqz2dq+kMXhXXNnGTXpoRkrdx5WN94v5QDAAAAJclFO4BcpqzZq7jBo7XrcGRnZMV6B/FlZWdH9LqRNmHVHt3z5W96d8rGsPr/snSnJE8y73g2Ze1efTA9vM+sLGCmFwAAABCm3q9MkeT5shTKo/06amD35jqxbjUdTc+Sk1ONKp7H7jPi8q41hbxanlAjYP+Xv5+nbOdkMj0/erUu7NBI57ZvqISktEJfe8BbnllgR9OzdEGHRmrXqFZEYgYAAAByc65spb2+mucZ/LVs+2E1rVs9YtfNGdSXVbZzXjqQnC5JEU/6VXS3f7JAkjSod7soRxIekl4AAABAAWau36ebhs8Lq+/dFxz7IlDdb7bX/CEXq071yhGP7XhQ2a/0yPgHeqt1A09SrE51z9eZxweconErdmvhloNhX/PZ/61SzGhp8ZOX6khaprYdSNFZbRtENnAAAACgFOUk2UKVTy+Nquo3D5+nCzs01p3ntSn5mxVRGctFllsrdx5WtcqxZW4gIUkvAAAAIAjnnJLTs/ItY/i3i9qp44l1dGW3Zlq7O0lZ2aG/PTWuU60kwjzudDjxWC39qpViFT9sgCTp9nPbKNs53T9isTYlJKtJnWoFruGV7aRuz0zw7c9/7GK9Mm6trj2jpXrGnVAyPwAAAACOG6WZW9mbmKpeL07Si787VTec2SrfviWV9DGTZqzfpxnr95XJpFdRc34kyYLLqaKR852srCDpBQAAAOSSlpmlDo+Py7fPp7efoQs7NPbt+ydjUPpiY0yxMv37xh6SpIysbCUezdAvS3fqvWmbtDsxtcBrrN9zRN8t2q4fftuuTUPL1hc3AAAAHLN02yFd895szRrcR41rHz+Dyw4fzVBKembQ0oTx+1MkST8t3h4y6cXKwoVTGjPjEHkxBXcBAAAAjh+bEo6o85OhZ3dJ0lNXdgpIeKHsqRwbowa1quq2c9to7P3n6/mru2jo70/N95wbP/KUsMx20rgVu0sjTAAAABTBx7M2KyPLafaG/dEOJV+RniF0wStTdPbQyUU+vyjhrN+TpOs/mKuj6VlFvm9F5JzT1LV7lZ1PtQ9ER0SSXmZ2uZmtNbMNZjY4SHtVMxvpbZ9nZnG52luZ2REz+79IxAMAAAAUVkZWtjYlHFGfV6cpM8gXl7aNamrR433141/P0e3nlr1SHQitfs0quums1rq+VyvNePgiNalTVT/+9Zx8z7nny0X6betBX8nK2Rv26R8jl5S5xcgBAABQhnkfHSM1YehQSkbBt4zw4+qz/1ulOZv2a378gUKfuycxVX3+NVXbDqTk22/34VSt2HE4ZPtns+M1csHWkO0p6ZmauX5f2HG5CBSeHLtit277ZIE+nrU5336zN+zT8u2hfzZEXrGTXmYWK+ldSf0kdZJ0vZl1ytXtTkkHnXPtJb0u6aVc7a9JGlvcWAAAAICiOJqepcten64+r04L2n5fn/aa/M8L1aBWVZ3eqn4pR4dIanlCDc0b0lent6qvfl1OzLfv7/89W//8dokk6YaP5umnxTuUnpVdGmECAAAAhRJOKb5IJd8SUzMUN3i0Pp4ZOuHjnNOZL07Spn3J+nLelnyvd95Lk3XF2zN9+xNX7dGuw0d9+0+NWqlHflge8vxHf1yum4bPU/y+5HzvU9hyhfklEHcf9pRP337waOhO8nyPuPKdmfn2QWRFYk2vXpI2OOc2SZKZjZA0UNIqvz4DJT3t3f5e0jtmZs45Z2ZXS9osKf+/kQAAAEAJOeXJ0Ot3lbVFeRE5/7mph297xPytGvxj3i/S/12yU60a1PTtz998QF2a1dWOQ0fVpXndUokTAAAA5VPOjKLk9CztTUot0fXH8kvQHEnL1PAZm5Xt61S8mU57vevlfjVvi+44r03Qe4czKy1H7kobd32+UI1qV9WCx/qGPOe/i3fogZFLNGtwH63fc0SS5+csLTkJNCpBlD2RKG/YXNI2v/3t3mNB+zjnMiUdltTAzGpJekTSMxGIAwAAAAjb2t1Jihs8WnGDRwdtf/eG07Xw8dBfslCxXNerlS7s0Cho21uT1vu2bx4+X6c9N1FXvD1T09cl8CUXAAAAYbll+HxJ0svj1uivXy0K+7zXJqzV5DV78u0zfuVu/fO7JSHbXxq7Rq//uk6/rt4b1j2TvDO5Ri3dGbQ9Oc2zvtfGhGS9NnGdL7nnP5Hqy7n5z+7Kznaatyn0mmwJSWn5nv/T4h2SpHV7kvLtF46sbKdfV+3J82yf38ywnCa+DRyzKeGIfgnxd6Y0RWRNr2J4WtLrzrkjBXU0s0FmttDMFiYkJJR8ZAAAAGWQmX1sZnvNbEW0YymPpq1LUJenxuvjmZt12RvTg/a554J2mvTPCzSga1M1rFW1lCNENH16ey9tHtpfA7s3kyQ9cUXuqu2Bbvl4vsYs310aoQEAAKCcmbp2r7o/O9G3v9s7O+rfUzeG9Qy5MP6AtuxP1luTN+iOTxcG7bPvSJqys53u/mKRth0IXWYvOT1wBlRObud/y3Zqxvq879pzrvXvKRt8x76YE+/b/m7RsTkwb01ar28Xbs9zjVcnrgvY33HoaMC6XZ/Mjte1H8wtMKFXEnKPWxs+c5P+/PlCfTN/W/ATgjBvRqw0x8C9+et6vZ7rcy1L+r42TX//ZnG0w4hI0muHpJZ++y28x4L2MbNKkupK2i/pTEkvm1m8pAckDTGze4PdxDn3gXOup3OuZ6NGwUdgAgAAHAc+lXR5tIMor279eL6OpGXq2f+tCtr+aL+OeuTyDmrXqFYpR4aywsz0xrXdNemfF+jGM1sV2P9vX/+m96ZtLIXIAAAAUJ78vKR4M16ueW+OLnhlatC2tMwsbd2fop7P/6q3Jq8P2icc9369WDcPn6/FWw9KkvYkpipu8GhNXeeZEeaf0Al3llgo5w4LXLdrY4JnHsyiLQcD+vmv5ZXbf6ZuVNzg0crOLlqmyUKsarbDuy7XkJ9CrxtWFrz+6zq9Oanov++SVsRfS8RFIum1QNJJZtbGzKpIuk7SqFx9Rkm61bt9jaTJzuN851yccy5O0huSXnTOvROBmAAAACok59x0SQeiHUd51P3ZCSHb7u7dVvHDBujuC9r5Ruzh+GVmateolqpVjtVHt/RUj9b18+0/bOyaUooMAAAApS0zKzvaIUjyzPzKce6wyer9yhRJ0qQiJqNSM7J827/792xJ0rLtnplYX8/bKunYmmQ5Vu1MVHpm8T6PPd4ZbznenRI4gOwb772D+deEtZKkrAKmV/2waLumrA39uRQ2N+Nf9vC3rQe1bPuhQl6h/MqvBGVZVeykl3eNrnsljZe0WtK3zrmVZvasmV3l7TZcnjW8Nkh6UNLg4t4XAAAACEdmVrbiBo8OuZDy//5+nh7tf0opR4Xyom+nJvrhL+corkGNfPt9Njte2dlO/xq/VsNnbg54iQAAKBlmdrmZrTWzDWYW8l2Tmf3BzJyZ9SzN+ACUf+v3JKn9Y2M1dvmuaIeia96b49vedyTdt30gOT2gXzgJHSfp8f/mrZh/JM3znSlnHGDu3FL/t2bo+g/nhpwxFY5HflgW9Nr+sYWSlZ137bBg/vndUt3+yQLtPpyqTQl5V1bKznZ65peVeRJwuQW7z+//PVtXvTPr2GdUjFW9UjOy9NGMTb6fqyxat7fAlanKnEqRuIhzboykMbmOPem3nSrpjwVc4+lIxAIAAADPeqiSBklSq1YFl2irqA4mp+u05yYGbZs35GI1qVOtlCNCeTXugd669eP5mrc5+ETLp0at1FOjVvr2Z23Yp+G39mTmIACUEDOLlfSupEskbZe0wMxGOedW5epXW9L9kgpRyXwAACAASURBVOaVfpQAyrvl3jWo/vLVbxrSv6MG9W4X5YjycvnMenLOadjYNQFraUlStnO+8oL+/jFyqaRj5f6CWbTloE5pWjtoW6gZWGkZx2aHZWU7ZWU7fTM/+IyuSK6RddbQSZKk+GEDAo7/uNizOtPmfcn69PZeIc/3D6XdkDG696L2vv2cp/yceLOynWJMhXr+f/3XdXp/2iY1qFVFvzutRch+e5NS1bg2313DFYnyhgAAAChjWA/V8wXvoe+XBm3rfXIjEl4olGqVYzXy7rO14YV+uq9P+wL7T16zV7d+soAZXwBQcnpJ2uCc2+ScS5c0QtLAIP2ek/SSpPyH8wNAAT6bvSXsvs/8slIp6Zn59knLzNLkNXuKG1aedZT8Uy4ZWU7vT9+kdXsCE1z3fr04z3XGr9yd55rr9x5RQlJanr4LNh/Mc0ySHvsp7+wxSfp0dnzAfqgZVkmpGWHNnHKSpq1LKLBfHrnyUeHOsNpx6Kiysl3gelp+ya2nfl6hdkPG6B8jlxQqnMSjnr8jKemhvzMsjD+gXi9M0qilxVsjLpRPZm3WX79alOf4vE37tWFvUoncs6SR9AIAAECFk5iaoTaPjgm52PLnd4QezQfkp1JsjB68tIM2D+2vWYP75Nt3+roEdXxinCb4vUAAAERMc0nb/Pa3e4/5mNnpklo650aXZmAASk9xSstJ0oMjl+isFydFKJpjPpkVr/Er809o3f7JAt3x6cKAtboiYeGWg1q725OsKEzRgV2Hgs/uWrM7b+IjvRjrnOU3k6v3y1MC1vgKlRSM1GywcGdlfThjc8g2J+mzOZ6E6H+XFC0xdTQ9S+cOm6zZG/cFHP92wTbd8JFnovKCEBUniuuZX1ZpzPK831eu/WCu+r42PeDDvu6DOXn6lUUkvQAAAMoRM/tG0hxJHcxsu5ndGe2YyorUjCylpGfq8znx6vr0hDztr1/bTd/fc7Ze/kPX0g8OFY6ZqXm96przaB+9c8Npev/mHnr5mq7q3KxOnr6DvlikpdsOaW9Sqg4fDb62HAAgsswsRtJrkv4ZRt9BZrbQzBYmJBRh5gCAcuvHxTu0O9eso9SMLL0+cZ3SMwMTO/mVEQxH7jWGZ2/cH/R4YQVL/F32xnSNCFE+MNoOJKdr/5H0oG0Hc30Wd3y6sERjSUrN+9lv2HtE6/Z4k4b5nJu7vGG4tu5POXZ970XW7UnSjkNHNWzsmoC+D/+wzPf3sCxUTZ+7qWQSb5EWkTW9AAAAUDqcc9dHO4ay6Is58Xri55Uh22cP7qNm9apLknrGnVBKUeF40LRudV3Rtbpv/+t5wV8uDHx3liQprkENTX3oolKJDQAquB2SWvrtt/Aey1FbUhdJU70j+U+UNMrMrnLOBbxFdc59IOkDSerZs2cEV5MBUNIs37RE0bw3baPenLReczft17VntCz4hGIqbjIjVIW+J35eoT/0CL1O1OKth4p34yJatStRV74zM+z+d32eN/EVaobfoZQMTV0bvNpHMIu3HlJyWqb2JB4r4dj3tWmS8q4Dltux31vh/rPR+5UpkqT3bjrdlzDL+Xu8bPvhUKdF3OGjGapV9Vh6aNuBFLWoXz3v7LeykG0rJGZ6AQAAoFx7f9rGoAmvz+7opZXPXKb4YQN8CS+gpGUXMNQzfn+Kth9MKaVoAKBCWyDpJDNrY2ZVJF0naVROo3PusHOuoXMuzjkXJ2mupDwJLwAV1+sT12nomNWFPu+od03WeZsPBMzi8U8GjF2+S5sSjuQ+tUBfzM27Ltidny3UlEIkanILtuaW5EmGbdhb+BjDsXlfcolcN5iJq/IvE3kkNVOrdiVKkm4aPk+3fbIgT5+Z6/dp9oZ9QWeYdX5qvMYVoxx5sPKP4bjny9982+HklSKZekrNyFK3Zybo6VHHvkef//IUjViwLZ+zAi3ZFp2kaThIegEAAKDcSkrN0NBcJSAkz6i8C05upJpVKWyA0nXneW0kSc8N7CxJuueCdnn6nP/ylFKNCQAqIudcpqR7JY2XtFrSt865lWb2rJldFd3oAJQFb05ar/enb4rY9fzLG/7lq9/U59Vphb7GE/9dEfT47Z8s0Bu/rtMns0KvHVVYWdlOIwuRxChP/JORM9fvC93Ra+ravbrho3l6aVze746hJKdlKiufAW05ZSmLM2Pum3xKUOYuix7u+mPhOJruSez+sixwDbIFwdaXC/EZLN9xbFZaWmZWxGKLBN4CAAAAoFw6lJKu7s9OzHO8oDIUQEka2L25BnZvLkm6+ew4SVLbRjU1d+N+/bjYU3XLOSlu8Gj1PaWJLunUWH/s0VIxMeWvbAgARJtzboykMbmOPRmi74WlEROAgq3bk6STm9QOq+/PS3aoWb3qOiNKJcoLk2dIzSjei/83fl0vSfrdac1Vr0aVYl0rR9VKFXPOS1Jqpm975MKSSex1fmp8vu2vjF8b9PiB5HSdULNwv79geaWbh88L2X/Oxv06u10DSVJmVrZGL9+lq7o1k5npk1mb9cwvqzTn0T7afyRdXZrXLVQsRXHRK1M1+9GLS/w+4SLpBQAAgHIlLTNLD4xYorEr8pag+OrPZ0YhIiB/f+rZUue1b6hVuxIDyp/8unqPfl29R4/8sFxLn7pUdatXjmKUAAAApePS16cHrLmbn/tHLJHkGdiWmpGlapVjA9pDre1ULH6XPJCctxxebue9NFk1q1TS2j1FK3OXW7CBfUVVKTb8rN1rE9dF7L4l7YwXfi1U/49mRm4GXUGG/Lhc793co9jXyW99r+s/nKtpD12o+ZsP6Jv5W/Wbd7bZwO7NNdz7s/Z7c4YOpWQEHRQ6aunOPMeKY+fh1MD9Q0ejusRAxUz1AgAAoMLJynZauztJHR4fF5Dw6tS0juIa1NDiJy7Rue0bRjFCILRm9apr3AO9dV2IxdAXbz0oSfp2wbbgZUUAAAAqkJzScOFasu2QOj4xTpPX5L++U0HOe2myrvtgTtj9f15ScHJg+8GjEUt4RVpaRnbYfRP9Zk+h6I4WYcZfUWarJaVm6qHvl/kSXjsPeRJPid6yiDn/xqatS1Dc4NHasPfY39GnRuVdE1uSxq/YrbjBo5WY6vfvM9d0xy37g6/nllMyUZLu+DTvumqliaQXAAAAyoUbP5qry96YHnDs96c315j7z9fUhy5S/UKWkACi4eHLO+qjW3rq8s4nBhy/7ZMFOuOFX/XwD8v0x/fCfxEDAABwPPhti2eA0PR1ges3mQpXInr7waOauynvAKOMrODJoczs/GeSZRXQjuPPtHUJ2nYgpdTvm7NeWe7k5a0fz5ck9X1tep5zckv2Jq42J/gltnLVXsz9nTzHKU+O823nXo+stFHeEAAAAGXeuj1Jeb6cvn5tN/3utBZRiggomhNqVlHfTk10YYdGev3XdXp3ykZfW0JSmm/7aHqWqleJDXYJAACAMi8pNUNb9qeU+HpCD4xcoro1KuuiDo1D9tmTmKoDyek6pWmdkH3e+HWdHrqso2fHL4+2eldivvffn5yWb3u0kZKLjvNfnqJWJ9RQWmbR1nn7ZFbecoy515e74u2ZRbp2YU1cvTdgPzUjW1/N21Iq9y4qZnoBAACgTLv2/Tm69PXA0WTzh1xMwgvlWqXYGD10WUdd1a1Z0PaXxq0p8pdkAACASLjl4/m6s4hlyu74dIGueHtmyJlQuV/gFyS//m9PWp/vuWe+OEn93pyR77PVrA37CxeQV68XJhXpvNLiyHpFzdYDKdqTWLSk6DO/rMpzrLCzGoP58bfthT4nNT3vv5vHflpR7FhKEkkvAAAAlEkrdhzWu1M2aN7mwBle//v7eWpcp1qUogIi663rT1P8sAHaPLR/wPFPZ8frni8WKTvbBdTHBwAAKC3T1yVo0pq9BXcMYpG3HGFB4gaP1l+/WlRgv/ySN+v2HAmrnNwPi3b4XS/wggFJtQqUKDqSFt0ycyhbxizflefvfjD+/x6qVi58CmnX4dRCnxNJJL0AAABQ5sQNHq0r3p6pV8av9R0b+vtTteCxviVeIgWIBjPTM1d1Djg2ZW2C2g4Zo1OeHKf0zPAXIQcAAChPxizfXazzj6Rl6vyXpxTYL8vvZf/epMLPwNl5OFVxg0crNaP8DEj6dmHhZ/YgulbtDF5S8+MgJQ8L69fVe7V2T1KhzokJMc1y6/7koMfLAtb0AgAAQJmRne306I/L8xyf8n8Xqk3DmlGICCg9N57ZSu0a1VL1KrH6w39mB7T9tvWgGtaqqvaNa0UpOgAAgOjZkxjZmSP5TXbZWsCsscSjzJ5Cyfn31A0lev3XJ64rVP9QpUU/nFH8JFxJYaYXAAAAyoSMrGxd/No0jVy4zXfsog6NFD9sAAkvHBcqxcbovJMaqkfr+hp933nq2uLYrMbrPpirvq9N04Mjl2jS6j1RjBIAAKD0vT99kyRPCegjaZlB+4RTts3XN1cNQ//3+mNXFG/mGVAcxamuedsn8wvsk+JXOv1QSsEJ3KlrE4oRUXSQ9AIAAEBUOed000fzdNJjY7V5X2CJhE9u7xWlqIDo6tysrr6566w8Iyt/XLxDd362UI/9tFy7Dh+NTnAAAAAFKOyL+4ys8Es5/yfETJhbPs7/hX+ICSuetlDTWYBSNj/XmtaFEU6CKiu74H+dT/x3RZFj8Pfjb9s1fGbpzwgj6QUAAICoeuPX9Zq5YV/Ascn/vECbXuwfpYiAsqFm1UraPHSALji5UZ62r+Zt1dlDJ2v2xn1BzgQAACjbcueYTnpsbNjnhnpnP2N90Z+LSHmhrEgownpzhZEZRtJr6fbDEbnXg98u1XP/WxWRaxUGSS8AAABERWpGli5/Y7renLQ+4Hj8sAFq26iWYmL46glI0md39NKSJy8J2vbYT5EZhQkAABBJJfkkX9Rrr96VGLItJw0Q6bXDgLImnJle5R1JLwAAAJQ655w6PjFOa3YnBRxf+cxlUYoIKNvq1aii8Q/0znN8875kHU7JUFIqC6oDAIDjQ0wYpQi/nLslz7Gv5m0N2T/nime+OKmoYQHlQnKINfEqEpJeAAAAKDXOOS3bfkhtHh0TcPzLO89U/LABqlm1UpQiA8q+k5vU0gN9T9INZ7YKON7t2Qk69ekJyj4ORm0CAIDouuvzhfpg+saoxmBWcCnDxwtYk8jlemxiSS8cL3IPPC0pKenRS67xVgEAAACl4nBKhro9OyHP8fhhA6IQDVD+mJke6HuyJOmRyzuq2zOB/57aDhmjv17YTg9f3jEa4QEAgOPAxFV7NHHVHg3q3S5knwPJ6SHX3YqE1buSdCA5vdjXyV3K8KVxa8I6Ly0zu9j3Biq6B0cujdq9mekFAACAErfvSFqehNerf+ymzUP7RykioHyrW72yTm5SS5J8/ytJ/566UX/5clG0wgIAANA1780u0nkrdhwOq9+vq/eEbEtJz9RbudYMDiYjK1uXvj7dt28y/WdqeDPYzn95Slj9gOPZmt2h19AraSS9AAAAUKL+9tVv6vn8r7791g1qKH7YAP2hRwsZdUSAInv6qs66tFMTjb7v/ICSPGNX7I5qOREAAHB825SQXKTzrnh7ZrHv3enJ8Xpt4roC+13wylQdPsqaqEBJid+fErV7k/QCAABAiTln6CSNXr4r4Ni0hy6KUjRAxXJOu4b64JaeqhwboxF3nRXQtikhWYdTeJEDAACiy+VePMvLlP/gtyNpmdp/JK0kQgJQwbGmFwAAACIuMTVDXZ8OLGe45MlLVK9GlShFBFRsZ7ZtoPlDLtaf3p+j+P0puuLtmapbvbKmP3yR6lavHO3wAABAFM1cv0+nNq+rujXKzzPBha9M1b7STHpRgAKoMJjpBQAAgEI7kJyu2Rv2BW2bsHJ3QMLrh7+crc1D+5PwAkpY4zrV9PkdZ/r2Dx/NULdnJmjFjsNKTGXWFwAAx6PE1AzdNHye7vp8YancLyMrOyLXKdWEl6T5mw+U6v0AlBxmegEAAKDQBn2+UAu3HNSa5y5XtcqxkqT1e5J0id9i0JK04YV+qhTLOCugtDSpWzXPsZz1MX574hKdUJPkMwAAx5OMTE8SakPCkVK5X1Z2YDnD9XuPKDPL6dQWdUvl/gBA0gsAAACFtn6v50tzonfx5y/nbtHzo1f72n/4y9nq0fqEqMQGHM+qVorV81d30SlN62jj3iN6+IdlvrZHflimD2/pGcXoAADA8abfmzMkSfHDBgQcN8oJAighJL0AAABQaNUqx+jwUanXi5MCjp9Yp5qmPnShb/YXgNJ301mtJUk9WtdXv1NP1BbvGl8TV+1RSnqmalThayAAAAiUnpmtRVsO6ux2DUrlfmt2J+nkJrWDtm3dn1IqMQComKg1AwAAgEKrWilvUmvFM5dp7pCLSXgBZUjtapXVpXldDenfUZL0j5FLlJ4ZmbU2AABAdBxMTteGveGVK3QFd5EkvTB6la7/cK5W7jxc9MAknfH8r2H1u++bxUGPz9qwT71fmVKsGAAc30h6AQAAoNCqVT72GDmwezPFDxugWlWZPQKUVbef20ZN61bT+JV7dPLjY3Xxq1N1IDk92mEBAIAiuOyN6er72rSIXnPdHk8S7VBKRrGuk5SWWazz1+5OKtb5AEDSCwAAAIVmOlaE/83rTotiJADCUTk2Rt//5Rzf/saEZI1bsTuKEQEAgKLam5QmyVOSMG7waH08c3PIvmV56ayfl+yIdggAKiCSXgAAACi0FvWrS5I+u6NXlCMBEK7m9apr89D+WvBYXzWtW01Dflqus4dO0o5DR6MdGgAAKIJk76yqtyavj3IkBft0Vt7E3P0jluQ5ZmU5SwegXCDpBQAAgEKrUilGJzeppQtObhTtUAAUgpmpUe2qOrutZ5H6XYdT9eDIJXIu3BU/AABAWZWakaVNCYFrfeX3X3jnXLGeAa59f47+9N6csPqu3hVe2UJyXgCKi6QXAAAACi0jK1uVY3mUBMqrO89vo9+f3lxD+nfUvM0H1P6xsZq8Zk+0wwIAAIWQO1318PfL1OfVaUpKDW9drjaPjtHdXywq8v3nbT6g+fEHinw+AJQE3lQAAACUM2Z2uZmtNbMNZjY4GjGkZWaraiUeJYHyqnOzunrtT91153lt9fiAU9SgZhW9OGaNsrKZ8QUAQHmTMztq9sZ9kqQbPpyncSt2BbSFMmFV4KCXYP1zZo/NWJ/gO3YwOd23feNHc5WZlZ3vfWZv2ldAJAAQGbypAAAAKEfMLFbSu5L6Seok6Xoz61TacSSnZapm1UqlfVsAERYbY/rz+W315JWdtGHvEf3z2yXadiClwBdXAACgLPKkrJbvOKx7vvxN9xRhFlew4S99Xp2m5LRM3Tx8vu/Yac9N9G3P2rBfCUfS8r3utgPhrSHKzDEAxUXSCwAAoHzpJWmDc26Tcy5d0ghJA0s7iJT0LNWoElvatwVQQvp1aao/9mih/y7ZqfNfnqKr/z1LaZlZ0Q4LAAAUguWapjVu5W5fEuuA38ysokjPLJ0BMWOW7y6V+wCouEh6AQAAlC/NJW3z29/uPVaq6tWorKZ1q5f2bQGUkNgY04u/P9W3v2JHojo8Po7EFwAAx5lQ5RALKoDsqJAMoIygJg0AAEAFZGaDJA2SpFatWkX8+iMGnR3xawKIrsqxecdEvj1pg/7vsg5RiAYAABTE5co0FbR+V37mbNpfqHvlaS/GvQEgkiIy06ugxdTNrKqZjfS2zzOzOO/xS8xskZkt9/5vn0jEAwAAUIHtkNTSb7+F91gA59wHzrmezrmejRo1KrXgAFQM55/UUJL0zpQN+nxOvCSxzhcAAGWUeesa5i5vWJoKSooh+mr5rcm85rnLoxgJULKKnfQKczH1OyUddM61l/S6pJe8x/dJutI5d6qkWyV9Udx4AAAAKrgFkk4yszZmVkXSdZJGRTkmABXE7MF9NO2hC/XFnWf6jj3580rN2rBP3Z6ZoGXbD0UxOgAAkB8r1lyv/BWU0jrvpSkldm9Exr192vu2gyVIv7sn/2oe7910um/73RtOD2j720XtihdcGDqeWFt/9/sZgomNiWLmF2VGJGZ6hbOY+kBJn3m3v5d0sZmZc26xc26n9/hKSdXNrGoEYgIAAKiQnHOZku6VNF7SaknfOudWRjcqABVFs3rV1bpBTUnSE1ccG8t440fzlJyepdkb9+v9aRs1r4ASSAAAoHSlpGcGTWR8Oiu+1GNB+VSjSmy+7Zd2OtG3PaBr04C2c9s31FltTyiRuHLcdk6c/nx+23z7nNq8bonGgPIhEkmvcBZT9/Xxvqg5LKlBrj5/kPSbcy4t2E3MbJCZLTSzhQkJCREIGwAAoHxyzo1xzp3snGvnnHsh2vEAqJjuPK+NvrnrrIBjSakZGjp2ja79YG6UogIAALkdSE5XpyfHa9fh1Dxt70zZUOD5lCY8Pvj/moPNCizsTMGv/3ysMkCwczs0qZ3nmP8MsaVPXqqRg87K0ye301vV823XrV45ZPvgfh0LvBaODxFZ06u4zKyzPCUP7w7Vh3UpAAAAAKB0dW0ROFr23SkbfdtzNjLbCwCA8iQ1I6vAPhsTjujD6ZuKdP1tB1KKdB6K572beoTVz/kVqQw2K7CgNeFyt5/R5tjMrmBVBR++vEOeYz1a1/dt161RWae2KHhmVp0giS5/OT/VGXH18+2H40ckkl7hLKbu62NmlSTVlbTfu99C0k+SbnHObRQAAAAAoEyoWbWSlj51adC26z+cq8MpGaUcEQAAFVtSaoY2JRwJq29h52cN+Wl50OMrdyb6tp/4eaVeGLNaD323NKBPWmZ2gdf/fE58ISNCJBSUrAqmUpAsVbD1sLq3rJfnWDDVq8T6ZpLlXLtybMGph3Bml8V4f8Ci/Jw4PkUi6RXOYuqjJN3q3b5G0mTnnDOzepJGSxrsnJsVgVgAAAAAABFUp1qlkG3dnp2guMGjGdkNAECEXP/hXPV5dVqJXHvdniTf9qTVe3zbf3p/Tp6+3y3aHrB/7rDJBV7/wxmbixEdiircXFBAecMgGaRgSS//brnP8d/r2qKeLwnr3+2iDvlXbAuWyHrpD6cG7NeqGvpZNLdm9aqF3RcVV7GTXqEWUzezZ83sKm+34ZIamNkGSQ9KGuw9fq+k9pKeNLMl3j+NixsTAAAAACAyzEwT/9FbK565TJd2ahK0z/T1rLsMAEAkrNiRWHAnr8IuxbViR6K27E/Wtwu36c7PFvqOp6QXXPYQZVewZFU46teorD4dj72Kjw2SgSrOcm9m0vBbz9APfznn2PVyzU8saPbWc1d3UdVK4aYwTMP+0LWQUaIiCj9Nmg/n3BhJY3Ide9JvO1XSH4Oc97yk5yMRAwAAAACgZJzkXYj86as6q13jWjq/fUPd8NE8X/tjP63QT7/t0PknNdL9fU+KVpgAAKAAq3Ym6uHvl0U7DERQi/o1wu57X5/2+mLuFknS4ic9JazjBo+WFDx55vLJegWbLZZbTIwFLaXou0aQeWr+tzynXQMt23Yo33tc1a2ZFm89pJb1q6tOtfzX/8LxIRLlDQEAAAAAx4Fm9arrkcs7qluQ9R0Wbjmo139dp/h9yUoPY90PAABQPEVZ4+jbhdsiHwiiqk3DmmH3ffDSDr5kV44OTWqrV5sTFBMkOXVlt2bhBxIiP5adb+Ls2PZZbU/QFV2b+i5zbc+WateoVoG3ve2cOK19/nI1ruMpbfhyrtleP/31nGCnoQIj6QUAAAAAKJSaVSvpnRtOC9p24b+m6rPZ8aUbEAAAx6GilJ6bspaSxBWNf+Louau7BLQ1rXtsjatQs7bG/6O3vr37bFWJDUwVbHqxv67v1SrsOHJKF+aevZXfX1P/niMGna13bjj9WFuIpO5j/U8JvIaZqlaK9e3/6YyWAe2ntaqfTwSoiEh6AQAAAAAK7YquzbTimcv04S0987Qt33E4ChEBON6Y2eVmttbMNpjZ4CDt95jZcu8a8jPNrFM04gSAkuSfG7r5rNYBbX/vc5Ku6NpUUsFJ0ka1qwbsx8SYalatpLYNa+rBS072Hb/zvDZ57uuveytPRYDGtat573vsxrljCFYisaA47+rdNv8Okj65/YwC+0TSymcuK9X7lTeHj2aU6v0isqYXAAAAAOD4U6tqJV3SqYnWPHe5ElMzNGrJTj0/erVGLd2pUUt36rcnLtE387eqX5cT1TaM8jQAEC4zi5X0rqRLJG2XtMDMRjnnVvl1+9o59563/1WSXpN0eakHCwAlKCafOpcxJsU1CL/8YTCT/+9C33b8sAHHrh1j+t/fz1NGlqes9W3ntNGC+IN694bTtT85TR1O9KwJ2/HEOnmueds5cZKCJ858M8ZyNQZb/yvUcmEXdWgc4qc55p0bTtO9Xy8usB8ioAizUouDpBcAAAAAoFiqVY5Vtcqx+vP5bfXDbzu0eleiJOn05yZK8qwfMu2hi6IZIoCKp5ekDc65TZJkZiMkDZTkS3o55xL9+tdUqb92A4CSl9/abi7EdqR0aV7Xtz2ga1MN6OpJivnPGqtZtZJOrFNNuxNTJQUmzoqyLp2/2FBZrwK0qF9dLerXCLv/+zf3UJuGNXXp69ODtueXeIRkpVxvkPKGAAAAAICI+fCWHnmObdmfog17k6IQDYAKrLmkbX77273HApjZ38xso6SXJd1XSrEBQLG1aRjeDK2cEoH9Tz0xT5tzhUsslVTupmk9T6nDSrGBNwhW3rAkDezeTHENaui5gV3U1S9hVxDnpJOb1FbVSsHTKWbS1d2bSZJOb1VPL/7uVEnS+Sc1LH7QFUBsKf+eSXoBAAAAACKmRf0aWv9CvzyLjPd9bbp+XrJD6ZnZUYoMwPHIOfeuc66dpEckPR6sj5kNMrOFZrYwISGhdAMECundKRt82yMXbI1iJChLFjzWV29ce1qe466Q87s2vdg/UiEF+OiWnnrzuu6+db5ya1b32PFja3oVnCgpaP2vghfGDwAAIABJREFU3J6+srOmPnSRLurYWDExpvsuPqlwFwjBTOrY1FPG8Yy4E3Ri3are48wAk0p/JhxJLwAAAABARFWOjdFdvdvqym7NAo7fP2KJ/jN1ox7+fqm+X7Q9StEBqCB2SGrpt9/CeyyUEZKuDtbgnPvAOdfTOdezUaNGEQwRiJysbKeU9Ey9NWm979i/JqyLYkQoSxrVrqoquWYhVYmNUZ+OBa9t5a+kkjQNalXVwO55JuNKkj64uYd+/Ou5vv2cPFZJhFK/ZpXAA0GyZu0b19LwW3vmOpp/di3YemOFNbhfxzzHOjfLux5aeVTauT/W9AIAAAAAlIhXrumqhy/roMXbDum+bzwLhb/+q+cF3bcLt+uaHi2iGR6A8m2BpJPMrI08ya7rJN3g38HMTnLO5WQIBkhaL6Cceuyn5RqxYFuexAaQ26h7z1WlmBh1ypUwKeysqNJyaee8ZRmlY/O8SjLsYNcefmtPtW4QXmnJHEVcWizAPRe007Cxa3z7TetW0+j7ztfZQydp1+HU4t8gipjpBQAAAACoEKpVjlXLE2roqlwzvnL0f3OGUjOySjkqABWBcy5T0r2SxktaLelb59xKM3vWzK7ydrvXzFaa2RJJD0q6NUrhAsU2YsG2gjvhuBA/bEC+7V1b1AtIeB1LHoWXPvrrhe007PenFjW8YrvwZM+M2z/1bBnYUAJ5k9yJwHduOK3QCS8p7wy5Did6Pv8/nB58dpu/KrHBUzS92pwgSXrqys6FjqesiY1EVrAQSHoBAAAAAKJi1a5ELYw/GO0wAJRTzrkxzrmTnXPtnHMveI896Zwb5d2+3znX2TnX3Tl3kXNuZXQjBgonNSNLL45ZreS0zGMHy+hsHQQqS0s5Xd6lqSTpshAzqnJ7+PKOuq5Xq5IMKV8tT6ih+GED1K1lvQL7Vq8SG7Lt7gva6tu7z873/NyJwFOaBi8nGGyW3LntG/i2YyywT/N61RU/bICu7Bp84Je/j287QyufuSzP8Zf+0NUXZbS9ck3Xgjvlo5RzXiS9AAAAAAAl7/M7emnAqU3zHL9p+Dz9Y+QSPfML76IBAMixbk+S3p+26f/Zu+84qarzj+PfZ2Ybyy4Luyy9LL33BUQQRBFBosSOMbHFYGI0MYkFuxILMaaYaFSSmBh/GqOJHRv2ioIggoIKSBWUJtLL7vn9scOyZWbrzL0zO5/367Uv7j3n3HufvQy7w33mOUcz31ihu15dVtpe02od+OvgM/7zR3XyNQ5J6t2miVbOmBQxoZPIerWK/D1dObFXabVUJBNDCcHvDS9J8rXITq9y/MF/fa9deqT+fvbQ0vZylV61TPBkZ6SocXrlVagyUksSevEwLWV913qL1VpxkZD0AgAAAADE3Oju+brrzME6a0THSn2PL1inf7y9Ui4e/lcPAIDHvvp2j8b89lWt3ryrtG38H94oXQfzQPGh34/7i/hd6bfuLbOqHXPwIX/A6xKXBug7/UsSU4M7NCtt+/ePDovKufu2zdHKGZN0y4n9tHLGJGVnpIYdV/FfXYsm6aVJqYM65mVKkjo3PzQ9YnW5nvG9W5araHvypyOrvTaqR9ILAAAAAOCZq47rpV9P7qMu+ZXXS9hRdvomAACSxBML1mnV5l36v/dW+R0KaqBlk4wq+0d1bV46nVt9Ul5VfRjozcvH1uPMieXIHi20csYkdW1xKNnodS6x4l+Fhfmbndi3lf73kxGV1yIro0NuZrn9mWcVltsPN6Vj2WsPLWhWqb86kwcemmLxF+O61/p4qeav495xUk1I0gsAAAAA4JmM1KB+MKJAL//qSI0JLVR+UL8bXtT81VtVVMxnWgEAQHw5eXC7asfcdnJ//fWswrDTuS28brzSUyI/jh/ROS9iX0XtczP1m5P76cYT+tT4mIbIr+k+w1VwmZmGdMyNOJVfqyYZGliDdcom9GlV7j3yUT1baHzvlnrz8rHq3jK79rHW+gjpV8fULTl22YQeldom9q3ZWnLRRNILAAAAAOCL6ZP7VPrE60l/eUe3v/ipTxEBAACEV5NliSb0a6VGaUGdOqQkQZYSPHRQTmZqxHNMHthGfz+nUCcNalup77DOuaVjnvv5EaXtpw/toLMPL6j5N9CAeL1GVJcWlWcoqImycZrVbKrCe34wRPefN6x0v1FaUDPPKlT73ExPUnxH9Wyhi4/uVq6tpre7T5hKrxPDvKZjjaQXAAAAAMAXHfMa643Lx+qhHw0v1373a8t1y7NLfIoKAACgaueP6hS2/eBUdNMn99XiG49VSqBmj99PK2yvzLQU3XRiX10yrpte/tWY0oRJVnrJOlO9WjdRrziZPs5vvds0Ucsm6bp0fOXKomgZ2TVPzTJT9cE149SzVXTu+xlDI099WFvjerWs0bgrJvYs3a5JZVy4/FZmWkqNrhWsMO/k4huP1fg+VHoBAAAAAJLM4V2a67rv9C7XNvONFT5FAwBA3a3ZsktPL/wyqudctXlnVM+Huin7OP+aMu9bFt94rLIzQkmBUE4hGDBlpacoUMuKpMy0FF0yrru65GdVWtPL42Ws4lpWeoreu2qchtdiSsjaevD8w7TguvHKy0qv13my0w8ljA7v2lwrZ0yq87nKviQuOqpr2DEXV2hvndNIPytTuTWhQhKqZ6vyUyZWfMl2zm+scb1a1Ci+iknerPSaJcuijaQXAAAAAMB354X5xPRfXlum5xatVzFrfAEAEsR3/vyWLv73glodU91vuRc+/qruASFqIuWvstJTdN85QzWpX+tDya+Qif1qVuXiqngReDyTH6pS4e+peVZatYe88IvRUbv82B4la31dOr572PXBTi9sr1/Vsvqt4mu2Ynr12Z8dUePpJMtO5/napUfWKo5o8ifVBgAAAABABQ9PPUx/fuVzvb1ssyTptudL1va6YkJPpacENLBDUw3u0MzPEAEAqNK23fv9DgEeGd4pV8f0LplibmhBroYW5FYa071ltl64ZLSyKiUWyqvJtHOIHwdzQM9cfIRWbNrh2XXH92mlT2+aoPSUYKW+edeMU9NGqVUeX1Vy9aCK+a2M1KCKynwArUt+Yy3fWLn69I4pA9W4TGVXQfO6rYMWDSS9AAAAAABx4bDOeTqsc54Wrd2m4+98q7T99hc/Lf3Pdn2mhAEAIB5RyBO/LhrbVXe+uqxc28HEwX8uGFGjc/QoM32cVfjbbp2Toaz0FA0K86Ge1CCTtMWbdrmNtGLjztK/x1Y5GWqVk1HlMbmNS6rBorX+WLiElyQ1r2IaxrKvuqqKtrLTU/STI7tUefyTF41S3+tfqDRm8sC2kU/sMZJeAAAAAIC40q9djj687hgNnD5bksp9unTlpp1q1jhN2ekpCgR4TAgA8NbWnft0/J1v6W9nF6pnqyZ+h4MYK1uBVTFhVReDOjTVO8s3KyVgOlDs1Ldtjv56VmHYsX8/e6j+M2+19h0orvd1ER0P/+gwzV+9VWkpNU9IZqQGK31o69g+LdWySdXJMq8cfF2npwS06MZjqx1fXJNyMZ+RLgYAAAAAxJ2mmWlaMn2Cfn/agHLtR97+mgbc+KKu+N9HPkUGAEhmr3+2UWu37tbdry33OxSEMbZHvi6fEJ2KGql8outgoiMrve51JG2bNpIkjemeX+3YDnmZuuzYnodi4bM+vmvRJEMT+rau93nu/UGhpk/uG4WIwpsytL16tsrWYZ3LT7lZNl116pB25fqqen2V7UuEtXap9AIAAAAAxKVGaUGdNLid/jd/bek6Xwc9+sFaXXlcr9IpYwAASFSL1m3zO4QGIxgI6MIju5auCxpNQzvlql2zRjp9aPuon7sqCVBYgzgz4+T+5farSmgdrGYM1DCrmpIA027Gf4QAAAAAgKQW6T/hg389Wz9/eIG27tzncUQAgGQw7veva3KZNSZrw9UwUzF/9VY989H6Ol0DsVd+ekPpgjFd1DSTD9wg8XVvWbLW3KR+JZVrFd9vnzCgjVJCU4lbmb76VDp6haQXAAAAACCuXTC6ZEHtnEaplfqe/PBL/eOdlR5HBABIBsu+3qGFa+tWhbVq8y7d8uySapNf67/ZU7r90dpv6nQtRMdRPVtUais7vWFGarDe1ziYOzj4qqhJbQ3TGiJqnFPv1iVrEQ7u2EwrbjlOJw4qmeaw4uvsT2cM0rJbjgt7mj+ePjCmYdYXSS8AAAAAQFwb1a25Vs6YpFd+NSZsP8+CAADx5oIHPtDMN1ao05XPas/+IhVMm6WCabP0yZffRjxmzootHkYISbrn+0M062ejJJVPeh2sfumYl1naNq5X5aRYfdWkHpDpDVGd2b8YXeOxF47tqid/OlJDOjZTIGCyUIbo4HpzNfHdQW21csak2obpGZJeAAAACcLMTjWzj82s2MwK/Y4HALyWl5Wu1y87slL7so07EmJRbQBAw+Sc07pvdpdr219UXLr91beHqrmO+9ObWrL+Wz343io9t4hpDaOttlVRZlKfNjlafstxpYkuSbrze4O07OaJap1TkggY2TWv3BRvdTV5YFtJ0tCC3Fofa3zMBxW8dcVYvfiL0eoWmqqwJoIB04D2TUv3m2Sk6s9nDNK/fjgsanGdc3hB1M5VF/E/ASMAAAAOWizpJEn3+h0IAPilY15jvXfV0Trhzrf01bd7JUmzPlqvWR+tV9umjXTNpF6aWOahFQAAsfa/+et06aML9eiPR5S2VfVRjIl3vFm6fdf3BscwsuRT16qoYMDUrPGhtbrMTCnB6CeZRnYtqV5/fvGGGh/Dx3oQSbtmmdUP0qGEaaTX0vED2kQpIsVFBRiVXgAAAAnCObfEOfep33EAgN9aNsnQnCuPrtS+7pvd+smD87Vnf5EPUQEAktUHq7ZKkj77antpW3ENsy8/fWh+TGJCzVSX1nJxlHJibS/U1cEEbiBJXkQkvQAAABogM5tqZvPMbN7GjRv9DgcAos7M9MIl4dcv+NWjCz2OBgCQ3ConRtZtPTTdIWsyeae2z/RTU2r2eDxWUwvW5qy8jlBX544s0NkjOmrq6M4xOf/DUw+LyXnriqQXAABAHDGzl8xscZivybU5j3NupnOu0DlXmJ+fH6twAcBXPVqFX79g1keskQIA8N5D760u3T7AWpMJITVQ9ePxJhmpkqR2zRpF9brdW2ZJko7p3TKq5wXCyUxL0Y2T+6pxemxWuzqsc15MzltXrOkFAAAQR5xz4/yOAQASyZ3fG6Stu/br2icWl2svmDZL9/5giMb3bhmVhecBAIjkYAXOx19+628gqLXUatbtGtC+qe75/hAd2SO6HyTsnJ+lJdMnqFFasMbH8HYGqBkqvQAAAAAACes7/dvoB4d11Iu/qDzV4QUPfKBz/jHXh6gAAIlgxcYd2rZrv99hIApOGdKuTse1aVp9BdeEvq2UkVrz5FRN1SbhBcSj7PQU5TRK9TuMSkh6AQAAJAgzO9HM1koaIWmWmb3gd0wAEC+6t8wOu8bX659t1HF3vKk5Kzb7EBUAIJ4d9bvXNenPb9b7PNVV4BwoLq73NVC1cb1KpgmsTTHU29OOUvvczNgEFEWs5YV4teC6Y/TBNfE3WQ1JLwAAgAThnHvcOdfOOZfunGvpnDvW75gAIJ70aJWtlTMmaeWMSeraIqu0/ZP132rKzDk+RgYAiFdrt+6u9zmqS0r8Yfbn9b4GIjtzeIeIfWXfD0jSiYPalm63rUGVF4DIUoIBpQTjL8UUfxEBAAAAAFBPfzx9oCYPbFOu7UARn7QHAFS2e1+Rtu+J3TSHsxatj9m5k1mzzJJp1QJmGt29uQ7vkqdpE3tKklICJTVfj194uFo2SZckZaYF9btTB1R5zvzsdPVrmxPDqGuvTdMMSSWxoW7umDJQd0wZ6HcY8AhJLwAAAABAg9O3bY7OGFb+k99dr35Ob3y20aeIAADx6ojbXlG/G16s9XG3v/Cpnl+8nunnYuClX1aesrii9JRDa2JlpqXooR8dps755Su7MlKDmnFyf0nSgHZNFQhUPQHi3KvH6emLR9Uh4tg5d2Qn3fP9ITphQJvqByOsyQPbavLAttUPRINA0gsAAAAA0CAVdmymYQW55drOuu99nfk3pjoEAByyace+Oh1356vL9OP/mx/laCCVVG9FS1po+rUontJTwYBpQt9WskT9BgCPkfQCAAAAADRIKcGA/nPBYbr6uF7l2t9etlmzP/nKp6gAAEAkE/q00mMXHl6rY5wotQNwSFSSXmY2wcw+NbNlZjYtTH+6mf0n1P+emRWU6bsy1P6pmbEYOwAAAAAgasxMPxzVSUf1bFGu/Uf/mqe5K7f4FBUAINHt3FvkdwgJ69yRBRH7fnZ0Nw3u0KxG5zmqV8nvdqaXBFBWvZNeZhaUdJekiZJ6SzrDzHpXGPZDSVudc10l/UHSb0LH9pY0RVIfSRMk/SV0PgAAAAAAoiIQMN13zlDdf96wcu2n3vOujrr9NX+CAgAktJeWHKoYptLIez87uptOGlSyRtPQClMZS1LLJhlehwQgTkSj0muYpGXOuRXOuX2SHpY0ucKYyZLuD23/V9LRVjIJ6WRJDzvn9jrnvpC0LHQ+AAAAAACiakz3fF37nfKf0VyxaadP0QAAkJxqUpkVbv2qyyf00KAOTUv3Cwty9d5VR+u7oeRXWY/+eITumDJQqcGA2jZtJEka1a25JOn2Uwfo6YtG1TF6IHn1bJXtdwg1khKFc7SVtKbM/lpJwyONcc4dMLNtkvJC7XMqHFv5pxQAAAAAAFFwxrD2umnWJ5UeuO09UKT0FCYeAQDU3iPz1vodQlJonJaiMd3ztWD1N6VtkSq62jRtpMkDSx4zFzRvrPeuOlotstMlSacMaRf7YIEGZsn0CQoGKiej41FU1vTygplNNbN5ZjZv48aNfocDAAAAAEhAmWkpWnbzceXaCqbNUo9rnteaLbu0Zz9rtAAAKtt3oFh/evlz7T3A74n6MpOaZaaG7cvOiFyjEab4q8ZaNskIWz0GoGYapQWVlpIY6aRoRLlOUvsy++1CbWHHmFmKpBxJm2t4rCTJOTfTOVfonCvMz8+PQtgAAAAAgGQUDJgev/BwFXZsVq79iNteVc9rn9dht7ysfQeKfYoOABCP/m/OKv1+9me6/cXP/A4l4TknvXvl0RpcZqpCSQqY1D43s8rjTCSuAFQtGkmvuZK6mVknM0uTNEXSUxXGPCXp7ND2KZJecc65UPsUM0s3s06Sukl6PwoxAQAAAAAQ0aAOzfSH0weqQ5iHaxu+3aPNO/f6EBUAIF6t+2a3JGnjdn4/RENGalDfG96xXNvB6QglRUxt/fCITpoytL2mju4cw+gAJLJ6J72ccwckXSTpBUlLJD3inPvYzKab2QmhYX+XlGdmyyT9UtK00LEfS3pE0ieSnpf0U+ccNcIAAAAAgJhrn5upNy4fq3u+P7hS3+rNu3yICAAQr5j+NvpGdW1ebr+6Gq5WORnKSk/RjJP7Kys98jSIAJJbVCZhdM4965zr7pzr4py7OdR2nXPuqdD2Hufcqc65rs65Yc65FWWOvTl0XA/n3HPRiAcAAAAAgJqa0Le1/nD6gHJtp8+coy+/2a2SSUoAAEDFJbHqu75Pq5wMrZwxqcz5w6e9zh7RUf86b5jG925Zr+sBSA6JsfIYAAAAAAAx9N2BbTX/2mPKtR0+4xXd9eoyFUybpSc/DLv8NAAAScur1bU6NW+s0d3zIybFAKAskl4AAAAAgKRnZsptnKYXLhldrv32Fz+TJP384Q/9CAsAgLgViHISquzpWjRJlyRNHthGZ40oiOp1ADRsTH4KAAAAAEBIVVM1LVi9VYM6NPMwGgCAHx6Ys0pbduzzO4y4V9ecV6Spg8ueLjMtpdzUhwBQU1R6AQAAAAAQ0rJJesSHeCf+5R0tXPONtwEBADx37ROL9YeXPvM7jLgzpMIHP4Z1yi3dTgvW/zFzRmqw3ucAAJJeAAAAAACEZKal6ItbJ+neHwzR388urNQ/+a639cCcVT5EBgDwU/japOQypnt+uf0OuZml21NHd67zec8dWSBJap6VXudzAMBBJL0AAAAAAKjg2D6tdHSvlmH7rn1isZ78cJ2Ki3kECgBIHj85sku5/bKF0ScObltpfKTK6Z+P615uv3FaSpXjAaA2SHoBAAAAAFBLP3/4Q/3q0YV+hwEAgGdSggF9cM24Go9/47KxYdtzG6eV23ehOjpyXgCigaQXAAAAAAARXDOpl8b3blk69VJZjy9Yp2c++lK3v/CpNu3Y631wQJIzswlm9qmZLTOzaWH6f2lmn5jZR2b2spl19CNOJL7123bLUdwrScqrZgrC7IyUKvt/dUz3Sm2jupZMmziiS17dAwOAEJJeAAAAAABEcP4RnTXzrEL94pjuuuH43pX6L3poge58dZlOv/ddH6IDkpeZBSXdJWmipN6SzjCziv9IF0gqdM71l/RfSbd5GyUSTcG0Wdq590Cl9uuf/FhPL/zSh4gST35WulrnZJTun3N4Qbn+zvlZlY4Z0SVPy26eqMKC3FiHByAJkPQCAAAAAKAaTTJSdc7IThH7l2/c6WE0ACQNk7TMObfCObdP0sOSJpcd4Jx71Tm3K7Q7R1I7j2NEAvp6e+XK3Rc/+Uo7wiTDcEgwtCBXo7SggoFDExXecEKfcuOaZqaGPT4lyGNqANHBTxMAAAAAAGpoUIemfocAoERbSWvK7K8NtUXyQ0nPxTQiIMmYHUpudczL1GXH9tDMswqVVkUCa2TX5l6EBiCJkfQCAAAAAKCGHr9wZMS+gmmz9M+3v9CfX/7cw4gAVMfMvi+pUNJvI/RPNbN5ZjZv48aN3gYHJKBLxnXT5IFtStfv+u7ANjIz/XRsV7Vt2kj3nTNUF43tqnbNGvkcKYBkRNILAAAAAIBaGN4p8pojNzz9iX43+zMPowGS1jpJ7cvstwu1lWNm4yRdLekE51zleeskOedmOucKnXOF+fn5MQkWaEguGdddd0wZVLpfcZ2uguaNdemxPcpVggGAV0h6AQAAAABQC/efN0wLrj1Gt53SP+IY55yHEQFJaa6kbmbWyczSJE2R9FTZAWY2SNK9Kkl4fe1DjEhAtzy7xO8QfFHV7zQASCQkvQAAAAAAqIWM1KCaNU7TCQPaRBzz9fa9Wr5xh4dRAcnFOXdA0kWSXpC0RNIjzrmPzWy6mZ0QGvZbSVmSHjWzD83sqQinQwO1ecdeFRXX7kMIsz/5KkbRxLfTCtvrmYtHxeTcd585OCbnBYBwSHoBAAAAAFAHGalBFXZsFrZv+C0v6+jfva7XPqW4BIgV59yzzrnuzrkuzrmbQ23XOeeeCm2Pc861dM4NDH2dUPUZ0ZBs271fQ256KWkrt+qib9ucmJx3Yr/WOqZ3y5icGwAqIukFAAAAAEAdHXxAOPsXo8P2f/bVdi/DAQCEfLt7vyTphY83+BxJw3UwkXVUzxbVjv3rWYVaOWNSrEMCAJJeAAAAAADU1bSJPfX4hYerW8tsfXjdMZX6b3l2qSb88Q2dcOdb+vKb3T5ECACAYpJw6t+uqVbOmBSzCjEAqAuSXgAAAAnCzH5rZkvN7CMze9zMmvodEwAku4zUoAZ1KJnisGlmmk4a1LbSmKUbtuujtdv04HurvA4PAJKeC7Okl3NOjy9Y630wCebPZwwq3Z71s9is9wUA0UbSCwAAIHHMltTXOddf0meSrvQ5HgBABZeM6x6xb/G6b+XCPX0FAHjqyQ+/1C/+s9DvMOLG+aM6hW3Py0or3e7ThmouAIkhxe8AAAAAUDPOuRfL7M6RdIpfsQAAwuuQl6kvbj1OH6zaqk+/2q6gmaY9tkiS9PpnG9Xpymf17pVHqXVOI58jBYDktWXnPr9DiLlgwFRUXLMPWvRrFyGhFTq8R8vsKEUFALFHpRcAAEBiOk/Sc34HAQCozMxUWJCrM4d31JRhHSr1j7j1Fe3eV+RDZACQfNaFWU/RzIdAPBatyuKPbzxWT108MirnAgAvkPQCAACII2b2kpktDvM1ucyYqyUdkPRgFeeZambzzGzexo0bvQgdAFALC9d+o30Hiv0OAwCSUhLkvBSVlJdJjdNTlJ4SjMbZAMATJL0AAADiiHNunHOub5ivJyXJzM6R9B1JZ7oqPr7pnJvpnCt0zhXm5+d7FD0AIJzhnXIlST8e06W0bcrMOep+zXO67fmlrPMFAIg6frUASFas6QUAAJAgzGyCpMsljXHO7fI7HgBAzfz5e4P09bd71bdtju55fXm5vr+8tlyFBc3UrUW22udm+hQhADQszjntpZq2VHXTOU7s29qbQADAA1R6AQAAJI47JWVLmm1mH5rZPX4HBACoXovsDPVtmyNJeubiURrTvXwF7nn/nKcjbntVc1du8SM8AGhwHnp/tcb9/vXS/fXbduvTDdtL9y0JFvV67MLDS7fnXj2uyrFpKTwiBtBwUOkFAACQIJxzXf2OAQBQP33b5uj+84apYNqsSn2ff7VDQwtyfYgKABqW5xZtKLc/4tZXJEkrZ0zyIxxfDO7QTItuGK/0lGCVSa3bTu7vYVQAEHuk8QEAAAAA8NjPju5WqW3fgSIfIgGAxLd+227t2V/9z9DPviqp9kqCQi9JUnZGapUJryEdm+m0oe3Ltf3jnKE6Y1hJW5/WOTGNDwBigaQXAAAAAAAeO/+ITpXabnj6Ez298Evt2ndAT364Ts45HyIDgMQz4tZXdP7980r3IyW1Jt7xpkcRJYbvDmpbqW1szxa69aT+WjljknIyU32ICgDqh+kNAQAAAADwWHZ6ivq3y1GfNk307vLNWrl5lyTp4n8vKB2zfc8Bff+wjn6FCAAJ5a1lm6odU1TsdNTtr6moAX6ooENuplZv2VXlmPa5jbRmy25J0vJbjlMgSSreACQXkl4AAAAAAHjMzPTURaNK98Ot8XXNE4u1YdsefXdQW3VtkeVleADQYK3YtNPvEOqNM2SCAAAgAElEQVStZ6tsLd2wvVzbG5ePDfu7pKw3Lz9Km3bs1b4DxQqS8QLQQDG9IQAAAAAAPrv1pH5h2+98dZmm/mte2D4AQPLIa5xWuj2iS17YMelVrN91UPOsdLVp2ihqcQFAvCHpBQAAAACAz84Y1iFi34pNO3Xhgx/om137PIwIABAvRnVtrl6tm1Q7rmwFMQAkK5JeAAAAAADEgSd+OlJjuueH7Xt20QYNnD7b44gAIP7MWbFZr3+20e8w4lKPVtl+hwAAviPpBQAAAABAHBjYvqn+ee5Qv8MAgLg2ZeYcnX3f+36HAQCIUyS9AAAAAACIE2amnx/dLWL/7n1F2rO/yMOIAADxxmR+hwAAcSvF7wAAAAAAAMAhvzimuw7vkqfTZ86p1NfruuclSdkZKTpzeEdNm9jT6/AAIG5t37Nf2RmpMmtYSSEzybnybeN7t9SLn3zlT0AAEMeo9AIAAAAAIM4M75ynH47qFLF/+54Duuf15R5GBADx748vfS5J2rpzn8+RxN45Iwv8DgEA4hJJLwAAAAAA4tCl43vo6YtG6Ytbj9Mvj+kedsxvnl+qNVt2eRwZAMSnA0XF2negWIvWbfM7lKhLDZavXktP4bEuAITDT0cAAAAAAOJQo7Sg+rXLkZnpZ0d3029O7ldpzN2vLdcRt72q1Zt3qajYhTkLADRsX327p9z+39/6wqdIYus3J/f3OwQASAgkvQAAAAAASACnD+2gP5w+IGzf6N++qqsfX+RxRADgv1eXfl26ff+7q7S4AVZ5jeraXC2aZFRoPVT5den48NXAAJCM6pX0MrNcM5ttZp+H/mwWYdzZoTGfm9nZobZMM5tlZkvN7GMzm1GfWAAAAAAAaOiGFuRG7Ht47hrtLyr2MBoA8F8gUH7av1mL1vsUSXScMaxDuf2rjuupqaM7VxrXOqckCXbp+O666Khupe3P/fwIvXn52NgGCQBxrL6VXtMkveyc6ybp5dB+OWaWK+l6ScMlDZN0fZnk2O3OuZ6SBkkaaWYT6xkPAAAAAAANVn52uvq3y4nYf8V/P/IwGgDw18I13+iLTTv9DiNqJvVvrVtPKj+VbcsmGTKzSmPbNG2k+dceowuP7FquvVfrJmqfmxnTOAEgntU36TVZ0v2h7fslfTfMmGMlzXbObXHObZU0W9IE59wu59yrkuSc2ydpvqR29YwHAAAAAIAGKz0lqKcuGqUfHdEpbP9jC9bpxY83eBwVAPhj8l1v6+7XlvsdRtRUTm1JLsxyjU4ljbmN0ypVugFAsqtv0qulc+5gzfAGSS3DjGkraU2Z/bWhtlJm1lTS8SqpFgvLzKaa2Twzm7dx48b6RQ0AAAAAQAKbOrqLGqcFw/c98IF++uB8LVn/rcdRAUD0zXhuqe55veEktqpysKLr5V+N8TkSAEhc1Sa9zOwlM1sc5mty2XHOOScpzGcPqj1/iqR/S/qTc25FpHHOuZnOuULnXGF+fn5tLwMAAAAAQIORn52uj6dP0O9OHRC2f9ai9Zp4x5vava/I48gAILrueX25Zjy31O8wPHGwZqtLfpYmD2wj6VBVV1mtmmR4GBUAJJaU6gY458ZF6jOzr8ystXNuvZm1lvR1mGHrJB1ZZr+dpNfK7M+U9Llz7o81ihgAAAAAAEiShnXKrbL/iQ/X6YxhHTyKBgBQH2WX7jq4WXZ6w8y0oHbtK9IZw/m5DgCR1Hd6w6cknR3aPlvSk2HGvCBpvJk1M7NmksaH2mRmN0nKkXRJPeMAAAAAACDp5GSmSpKmju6s28NUfV352CK9/8UWOee0v6hYD8xZpZ17D3gdJgCgBsquznVwqsOySS8LMw4AUF61lV7VmCHpETP7oaRVkk6TJDMrlPRj59z5zrktZvZrSXNDx0wPtbWTdLWkpZLmh36Q3+mc+1s9YwIAAAAAICk0yUjV3KvHqXlWmsxMJw1qq7P/8b7e/HxT6ZjT7n233DFL1n+rW07s53WoAFAr7yzbpMEdm/kdhid+e0p/Xfbfj0oTXVKZSq8y48r2AwDCq1fSyzm3WdLRYdrnSTq/zP59ku6rMGat+GACAAAAAAD1kp+dXrodCJj+dd4wdbry2Yjjt+3a70VYAFBnn27Yru/97T2dMay936F4YmK/1nrxk6902bE9DjVW8dS08ipfAICD6ju9IQAAAAAAiCNmpj+cPkC3ndw/bH96akB9r39Bp1eoAAOAePHNrn2SpGVf7/A5Em9kpgb117MK1aZpoyrHUT0AANUj6QUAAAAAQANz4qB2Om1oez338yPUPrf8Q9TH5q/Tjr0H9N4XW3TSX972KUIAqJ6R5pFz1HUBQG2Q9AIAAAAAoIHq1bqJHr9wZMT++au/0W3PL9VbZdYAAwC/JVuaJ9xSXQcTfuHuBXkwAIiMpBcAAAAAAA1YbmaaJvVrrcO75IXt/8try/X9v7/ncVQAUD979hf5HUKNzPzBkDodFy4RRuEbAFSPpBcAAAAAAA1YIGC668zBeuhHh/kdCgDUiqui5qvntc97GEllM07qV6Nx4/u0qt+FwpZ61e+UANCQkfQCAAAAACBJ/O7UARrSsVnYvisfW+RxNAAQ3sHp++au3OpvIB6xMGVdB1vKJv4o9AKA6pH0AgAAAAAgSZw8pJ0m9i2pOji6Z4tyff9+f7UKps3SorXbtG33fhUXU0oAIH7E28+ksNMPxuD8Zdfv6toiS5IUDJL+AoBISHoBAAAAAJBEDlYUNM9KD9t//J1vacCNL+rOV5d5GRYAaM/+Im3fsz/stIa3PrfEh4iqFgyUTz7ddkr/sOOumNCz1ue2UF1X2Ttx3zlDdf95w5SVnlLr8wFAsiDpBQAAAABAEslILXkU0Lpphk4Z0i7iuN/P/kx7DxR5FRYAaPRtr6rfDS+G7Xt47hr98aXPPI6oamnB8o9Wcxqlhh137siCWp87XCVZ08w0jemeX+tzAUAyIekFAACQIMzs12b2kZl9aGYvmlkbv2MCACSeU4e012XH9tCPx3TR9Ml9NKJzXsSxtz671MPIACS7r7fvjdjnnPTHlz73MJrai+akg98b3kGSNJokFwDUCkkvAACAxPFb51x/59xASc9Ius7vgAAAiSctJaCfju2qjNSgMtNS9O+ph+n1y44MO/af76z0NDYAiMS5+FrTS1K5aRhTAtFdZ6t/u6ZaOWOS2jZtFNXzAkBDxwSwAAAACcI5922Z3cZSmMUOAACog455jSP2Pfz+au09UKwvNu3UyYPbqV+7HA8jA5CM/v3+mkpt8fzG94xhHXT+EZ20YuNOSVKv1k20ZP23lcalpwT06U0TtXjdNrVskqGhN7/kdagA0OBR6QUAAJBAzOxmM1sj6UxR6QUAiKKHzh8etn3aY4t0/VMf65/vrNTxd77lcVRAZGY2wcw+NbNlZjYtTP9oM5tvZgfM7BQ/YkTdPL3wy0ptu/bF1xqDJtPB4rPrvtNbXfKz1Cg1KEnq1Sq7ymP7ts1RfnZ6rEMEgKRE0gsAACCOmNlLZrY4zNdkSXLOXe2cay/pQUkXVXGeqWY2z8zmbdy40avwAQAJ7PCuzTX36nEa3im3ynHrvtmt/UXFHkUFhGdmQUl3SZooqbekM8ysd4VhqyWdI+khb6NDshrZNU/XH99bN07uU649DmdmBIAGi6QXAABAHHHOjXPO9Q3z9WSFoQ9KOrmK88x0zhU65wrz81n8GgBQM/nZ6br91AG6+rheGtsj/O+PkTNe0fSnP/E4MqCSYZKWOedWOOf2SXpY0uSyA5xzK51zH0kiSwtPmJnOHdlJ2RmpfocCAEmLpBcAAECCMLNuZXYnS1rqVywAgIarfW6mfjS6s/5x7jDNv/YYnTuyoNKYB+as0p79Rdq594CKiilhgC/aSiq78NPaUFutUSGP+jKr3NanTZMq+yWF/fkKAKgfkl4AAACJY0ZoqsOPJI2X9HO/AwIANGy5jdM0sW/rsH2XPrpQfa5/QRc88IHHUQHRRYU86qqqlP9/LhihNy8fW+Xx1x/fRytnTIpuUACQ5FL8DgAAAAA145yLOJ0hAACxMqxTrk4a3FaPzV9Xrv2Zj9ZLkl5a8pUkac/+IqUFAwoEIpQ0ANG1TlL7MvvtQm2AN6r5UZeVnqKsdB69AoDXqPQCAAAAAABVatkko8r+e15frp7XPq+bZi3xKCJAcyV1M7NOZpYmaYqkp3yOCfWQaFOl9mnTpOpSLwCAL0h6AQAAAACAKv14dBedN7KTlv56gpbfclyl/hnPlSwzed/bX8g5ngIj9pxzByRdJOkFSUskPeKc+9jMppvZCZJkZkPNbK2kUyXda2Yf+xcxqnPnK8v8DqFW+rTJKd2OtGYXAMB7JL0AAAAAAECVcjJTdd3xvZWRGlSwmukLj/7969q+Z79HkSGZOeeedc51d851cc7dHGq7zjn3VGh7rnOunXOusXMuzznXx9+IUZVF677xO4Qq/eCwjn6HAACoAZJeAAAAAAAgalZs3Kl+N7yoVz/9Wl9+s9vvcAAkiEQsEnU1nN8wEb83AEhUrKYIAAAAAABq5eMbj1UwYLr/nZW6NTS1YUXn/mOuJOmLW4+TMfcXgAbMxM84AIgXVHoBAAAAAIBaaZyeoozUoH50RGctvG68bj2pX8Sxew8UexgZgET18tKv/Q6hSuGqumpawUXeHwC8Q9ILAAAAAADUSSBgyslM1ZSh7SOOmf7MJx5GBCBRbNy+V/96d6X+M3e17n19ud/hRHTZsT1KtzvmZYYdQ1ILAOIHSS8AAAAAAFAvZqZzDi+QJHXILf9Q+KH3VuuyRxfKsagNgDJ++uB8Xffkx7rif4siTpPqhdtO7l9lf3pKyeNT56TXLxtbru+MYR0kSQGyXgAQN0h6AQAAAACAervhhD764tbj9PplRyqvcVq5vkc/WKvJd72tx+av1f/NWaXiYhJgQLLbumuf3yFIkk6rolJVUpVrEt54Qh8t/fUEBQMkvQAgXpD0AgAAAAAAUWFmMjO9+IvRlfo+WrtNv3xkoa55YrEWrNnqQ3QA4snnX+/wO4Sw3p52lBZeP16T+rWWJKVUkdAKBEwZqUGvQgMA1ECK3wEAAAAAAICGJS8rXZ/eNEH/fHulnKQZFaYuO/nud7X01xP06AdrNaprc3Vq3tifQAGggrZNG0mSfn/6AF3znV5qlpmmJeu/1aXjS9b2uvvMweoQYW0vAID/SHoBAAAAAICoS08J6oIxXbRnf5EefG+V1mzZXa6/57XPS5Ky0lO0+MZjdcuzS7Ri4w797eyhfoQLAOWkpwTVOqckATajzLpfE0MVYACA+MT0hgAAAAAAIGYyUoN68/KjIvbv2HtAi9dt08w3VuilJV97GBkAeCM9JaDTC9vroR8N9zsUAGjwSHoBAAAAAICYe//qoyP2fefPb3kYCQCEd985hTE5r5npN6f015COuTE5PwDgEJJeAAAAAAAg5lpkZ2jljEm6Y8pAv0MBgLCO6tnS7xAAAPVE0gsAAAAAAHjmyO4tNKhDU3VtkRW2v98NL2j15l3atGOvx5EBAAAg0aX4HQAAAAAAAEgeOZmpevzCkXLOqdOVz1bq377ngEb/9lVJ0oPnD9fA9k3VOJ3HFwCi54bjeyszLUXtchv5HQoAIMp41wgAAAAAADxnZurfLkcfrd0WccyZf3tPR/dsob+fM9TDyAA0dOeM7OR3CACAGGF6QwAAAAAA4ItHLhih3506oMoxLy/9WjOeW6odew94FBUAAAASFUkvAAAAAADgi4zUoE4c1Lbacfe8vlx9r39Ba7bs8iAqALG2fc/+mF+jbVOmLgSAZETSCwAAAAAA+CYQsNLtppmp+mT6sTqiW/OwY4+47VUVTJul+976wqvwAMTA3a8tj/k1gmV+tkhSQV5mzK8JAPAfSS8AAAAAAOCr5y85QscPaKOnLxqlzLQUTZ/cV82z0iKOn/7MJ3pn+Sat3LRTX3+7x8NIAUTDgWIX82ukVEh6dcxrHPNrAgD8l1Kfg80sV9J/JBVIWinpNOfc1jDjzpZ0TWj3Jufc/RX6n5LU2TnXtz7xAAAAAACAxNOzVRP9+YxBpfudmjfWvGuOUcG0WRGP+d5f3yvdXjljUkzjAxBdzsU+6RWokPRykv557lC1bJIR82sDAPxT30qvaZJeds51k/RyaL+cUGLseknDJQ2TdL2ZNSvTf5KkHfWMAwAAAAAANDBXTOipzLRgteN27D3gQTQAouXlJV/H/BqpwYAuPLJLubYje7RQr9ZNKo2963uDNaFPq5jHBACIvfomvSZLOli1db+k74YZc6yk2c65LaEqsNmSJkiSmWVJ+qWkm+oZBwAAAAAAaGB+cmQXfTJ9QrXjpv5rngfRAIiWFZt2xvwaaUHT5RN6lu5bFWMn9W+te34wJOYxAQBir75Jr5bOufWh7Q2SWoYZ01bSmjL7a0NtkvRrSb+TtKuecQAAAAAAgAZq6ujO6tOmiVo2SQ/b/87yzaXblzy8QNc8scir0IBq7TtQrAl/fEMPv7/a71AapPa5jcK2p6WUPPb82dHdvAwHAOCzapNeZvaSmS0O8zW57DhXMhlvjSfkNbOBkro45x6v4fipZjbPzOZt3LixppcBAAAAAAAJ7qrjemnWz45QnzY5EccUTJuluSu36IkPv9T/zVmtv725Qm98xvMD+M9MWrphuzZu3+t3KHFhz/6iqJ7PItRwHUx6je2RL0nqmJcZ1esCAOJTSnUDnHPjIvWZ2Vdm1to5t97MWksKNyHvOklHltlvJ+k1SSMkFZrZylAcLczsNefckQrDOTdT0kxJKiwsjP1qlwAAAAAAIK78/rQBOvefc7Vg9Tdh+0+9593S7ZtmLZEkrZwxyZPYgEiCVpKUKXI8ztp7oEi3PrvEk2vlNEqVJA3q0Ez3nVOokV2be3JdAIC/6ju94VOSzg5tny3pyTBjXpA03syamVkzSeMlveCcu9s518Y5VyBplKTPIiW8AAAAAAAAmmam6fELR+r+84b5HQpQY4FASdKruDh5k14rNu7Q8o079MC7q3T/u6uidt7rj+8dse/Uwval20f1bKn0lGDUrgsAiF/VVnpVY4akR8zsh5JWSTpNksysUNKPnXPnO+e2mNmvJc0NHTPdObelntcFAAAAAABJakz3/NLtTs0bq2Nepl77NPxUhn99Y4XG9MhX95bZXoUHVBIwKdlyXs45/fv9NZq3aosem78uJtfo2apJ2PZlN09USrC+n/UHACSieiW9nHObJR0dpn2epPPL7N8n6b4qzrNSUt/6xAIAAAAAAJLH0xeNUjBg6tkqW4GA6ZonFun/5qyuNO7mZ5fo5meXMM0hfBUMWNJNb/jG55t01eOLYn4dC7OkFwkvAEhe9a30AgAAAAAA8Fy/djnl9m88oW/YpNdBTy/8UqlB04S+rSWVVKFYuKflQAwEzJJqesP/fbBWew8Ue3Ktg/+KO+Zl6renDFCrJhmeXBcAEJ9IegEAACQYM/uVpNsl5TvnNvkdDwAA8SAYMF11XE/d8uzSsP0X/3uBJGli31ZyTvp6+x49duFIL0NEEgsGTEVJkPQ6UFSsa55YrIfnronpddo1a6RmmWka2L5pads/zx2mTs0bx/S6AID4R60vAABAAjGz9pLGS4r8UXYAAJLU5IFtlZ+drjOHd4g45rnFG/T8xxs0f/U3HkaGZBe0hj+94Vff7lHXq5+LScKrbdNGumJCz9L9fm1z9PTFo9QoLVja5hr4/QUA1AxJLwAAgMTyB0mXS+J/9QAAVNCySYbmXj1ON5/YT0f2yK92fMG0WZrwxze0ZssuHpgjpgKBhjO94bVPLNZ9b31Rrm3uyi0afsvLMbtmbuM0dczLLN0f3im3dJtpSgEAZZH0AgAASBBmNlnSOufcQr9jAQAg3t18Yj9NGdpegzo0rXLc0g3bdcRtr2rKzDnasG2PR9Eh2QQDiVvp9f4XW/TsovWl+w/MWaXpz3yiJeu/1a3PLdH6bbt16j3vxjQGMylQJrd19uEFlcYk5t0FAEQba3oBAADEETN7SVKrMF1XS7pKJVMb1uQ8UyVNlaQOHSJP8QQAQEPVtmkjzTi5v6SSiq7qvPfFFh1268taOWNSrENDEgqYqajY7yhq7/g/v6VF67ZJklbOmKRv9+wv7Zt4x5uSpHtfXxHzOEwl97B0v8x2k4ySx5tBKr4AACLpBQAAEFecc+PCtZtZP0mdJC0M/Se/naT5ZjbMObchzHlmSpopSYWFhXzwFQCQ1E4rbKdH5q1VSsB0oIFMMYfEEgwk5ppTBxNeB/30wfkxvd6RPfJVVOz05uebyrUP7thMzbPTwx5z7w8K9fTCL8tNfwgASF5MbwgAAJAAnHOLnHMtnHMFzrkCSWslDQ6X8AIAAOX95uT++uymifr0polq16xRlWMffG+Vrnp8kb76lqkOET0llV6Jl/Qq61/vrqyUjIq2Nk0bKTtUuXXn9waVtl91XC8N7tAs7DGtcjL0o9GdWdsLACCJSi8AAAAAANDAmZnSUkoeiL91xVG6+N8L9PTCL8OOvfrxxZKkrTv36e7vD/EsRjRsAUu8Nb02bt9bbv+6Jz+O+TU75TXWgl37KrWnBks+t//J9GNjHgMAILFR6QUAAJCAQhVfsf2oLQAADdTPjuqq5llpVY75YtNOj6JBMggGTMUJUunlnNPvX/xUQ29+yfNrm0kmC8VRuT8zLUWZaXyGHwAQGUkvAAAAAACQVLq1zNa8a47ROYcX6KTBbcOOWbphu67470cae/tr2rxjb9gxQE0FA6aiBMh5PfTeap1+7xz96ZVlnl738gk9JEn52eka0SVPktQ5v7GnMQAAGgY+GgEAAAAAAJLSDSf0kSQ9Nn9d2P7/zFsjSTr57nf0j3OHqVNzHsKjbgKmhKj0uurxRb5c94LRXdQpr7Em9G0lSRrXq6Va5WTofz8Zof2JkC0EAMQNkl4AAAAAAABVWLl5l8be/poWXHuMmjWuelpEIJxgwFQU50mvD1Zt8eQ6L1wyWnlZaWqela6CabMkldyfif1al45plZMhSRrSMdeTmAAADQfTGwIAAAAAgKT2j3OG6pjeLXX/ecOqHHfsH9/wKCI0NAEzFYVbpMonRcVOa7bsKt3fs79IJ9/9bsyu1zEvs3S7R6tsNc9Kj9m1AADJjaQXAAAAAABIamN7ttBfzyrUmO75evlXY3TNpF5hx329nbW9UDdLN2zX7E++8juMUr95fqmOuO1VPTZ/rSRp3O9fj+r5M9OCpduTB7bR388uDDvujGEddOXEnlG9NgAguTG9IQAAAAAAQEiX/Cx1yc9SYUGuvnvX25X6v9m1T00zmeIQiWfphm8VMNP4PxyqWPzlIwv1y0cW1vmcy26eqJRgQA++t0pXP764tP3Ny8dqyE0vKSVgumPKIEnSNZN6qX+7puWOv/WkfnW+NgAA4ZD0AgAAAAAAqGBg+6Z65IIROu3e8lO+Ld2wXYd1zvMpKqB6W3fu0wUPfKCxPVvoN88vjem1UoIlk0idObyjzhzesXSNrrysdC28brwCZeaYOv+IzjGNBQAAiaQXAAAAAABAWMM65ZZu92yVraUbtisjNVjFEfCSmU2QdIekoKS/OedmVOhPl/QvSUMkbZZ0unNupddxlvXBqq0a0rFZjcbu2V+kJz9cp0n92+itzzepV+tsjfnta5Kkbi2y9PnXO6o8/v2VW+obblgFeZl68qejlJOZWqnv/vOGqTi0dlm4fgAAYo2kFwAAAAAAQARvXDZW+4uLS6eAKyp2PkcESTKzoKS7JB0jaa2kuWb2lHPukzLDfihpq3Ouq5lNkfQbSad7H+0hJ9/9Tq2PueJ/iyq1VZfwipYfHNZRvzymuwb9enZp2+MXjoyY0BrTPd+TuAAAiISkFwAAAAAAQAQd8jIlScf1baWFa75R65wMnyNCyDBJy5xzKyTJzB6WNFlS2aTXZEk3hLb/K+lOMzPnnOeZywuP7KK/vLbc68uWk5WeorNGdNRfXluu9rmN9NhPRurJD9epY15jbd25TycNbqu9B4q1aN029Wubo8bphx4brpwxSS9+vEG9WjdRs8asaQcAiF8kvQAAAAAAAKoxdXRnnVrYXrk88I8XbSWtKbO/VtLwSGOccwfMbJukPEmbyg4ys6mSpkpShw4dYhLs5RN66vIJPUv3t+/Zr8079ikvK00HipxWbNqhVjmNlJWWooy0gPYeKFZaaL2s9JSAioqdggE7GG+9Yzmo4jpbKcFAxDXrxvdpVa/rAgDgBZJeAAAAAAAA1TAzEl4NlHNupqSZklRYWOhJFVh2RqqyMw5NETikcW65/vSU8mvHpQTrl+gCACBZBPwOAAAAAAAAAKildZLal9lvF2oLO8bMUiTlSNrsSXQAAMAXJL0AAAAAAACQaOZK6mZmncwsTdIUSU9VGPOUpLND26dIesWP9bwAAIB3mN4QAAAAAAAACSW0RtdFkl6QFJR0n3PuYzObLmmec+4pSX+X9ICZLZO0RSWJMQAA0ICR9AIAAAAAAEDCcc49K+nZCm3XldneI+lUr+MCAAD+YXpDAAAAAAAAAAAAJDySXgAAAAAAAAAAAEh4JL0AAAAAAAAAAACQ8Eh6AQAAAAAAAAAAIOGR9AIAAAAAAAAAAEDCI+kFAAAAAAAAAACAhGfOOb9jqDUz2yhpld9xxInmkjb5HUQDxz32Bvc59rjH3uA+H9LROZfvdxCI6XsnXu+xxz32Bvc59rjH3uA+eyNW95n3TnEgxs+c+Dcae9xjb3CfY4977A3uc+zF8h6Hfe+UkEkvHGJm85xzhX7H0ZBxj73BfY497rE3uM9IJrzeY4977A3uc+xxj73BffYG9xl1xWsn9rjH3uA+xx732Bvc59jz4x4zvSEAAAAAAAAAAAASHkkvAAAAAAAAAAAAJDySXolvpt8BJAHusTe4z7HHPfYG9xnJhCKpxWcAAAinSURBVNd77HGPvcF9jj3usTe4z97gPqOueO3EHvfYG9zn2OMee4P7HHue32PW9AIAAAAAAAAAAEDCo9ILAAAAAAAAAAAACY+kVwIws1wzm21mn4f+bBZh3NmhMZ+b2dlh+p8ys8Wxjzjx1Ocem1mmmc0ys6Vm9rGZzfA2+vhnZhPM7FMzW2Zm08L0p5vZf0L975lZQZm+K0Ptn5rZsV7GnUjqeo/N7Bgz+8DMFoX+PMrr2BNJfV7Lof4OZrbDzC71KmYgFqr7t4CqmVl7M3vVzD4JvXf4eag97PsRK/Gn0P3+yMwGlzlXle//kp2ZBc1sgZk9E9rvFPr5vCz08zot1M57kToys6Zm9t/Qe+ElZjaC13J0mdkvQj8rFpvZv80sg9dy/ZnZfWb2tZX5P3I0X7tmNiT0HntZ6Fjz9jtEPOG9U93xvslbvHeKLd43eYP3TrGRUO+dnHN8xfmXpNskTQttT5P0mzBjciWtCP3ZLLTdrEz/SZIekrTY7+8nHr/qc48lZUoaGxqTJulNSRP9/p7i5UtSUNJySZ1D92ehpN4Vxlwo6Z7Q9hRJ/wlt9w6NT5fUKXSeoN/fU7x91fMeD5LUJrTdV9I6v7+feP2qz30u0/9fSY9KutTv74cvvur6VZN/C3xVew9bSxoc2s6W9Fnod17Y9yOSjpP0nCSTdJik90LtVb7/48tJ0i9D74GfCe0/ImlKaPseST8JbfNepO73+H5J54e20yQ15bUc1fvbVtIXkhqF9h+RdA6v5ajc29GSBqvM/5Gj+dqV9H5orIWO5f+ISfol3jvV9/7xvsnb+817p9jeX943xf4e894pdvc2Yd47UemVGCar5IeiQn9+N8yYYyXNds5tcc5tlTRb0gTp/9u7t1C5qjOA4/8PjqatolVBUVNIFNuH+qAQNKKCqAhqawr1QShEqoIvfciDVLw8aBVBqRpFESQiWMSAVmvQB8EbpQVta5QatWq8oInxQovxBl7w68NakX3iOcdkZu89sz3/Hywys2fPZu3vfLPyzayZtSEi9qb8p3V1D30dqpFjnJmfZeYTAJn5BbARWNpDn4fiGGBzZr5e47OeEu+mZvzvA06ps/mrgPWZ+XlmvgFsrsfTbCPHODOfzcx36vYXgB9GxJJeej084+QyEfErSuH1Qk/9lbqyK68FLSAzt2Xmxnr7Y+Alypuz+eqRVcBdWTwF/DgiDmaB+k8QEUuBM4F19X4AJ1PGZ/h2jK1FdlNE7Et583sHlFo4Mz/EXG7bDKVGm6F84W4b5vLYMvOvwP922txK7tbH9snMp7J8inMXc7/H1OJg7TQG66b+WDt1y7qpV9ZOHRhS7eSk1zAclJnb6u13gYPm2OdQ4O3G/S11G8BVwPXAZ531cPjGjTFQfqYM/BJ4rItODtR3xq25T2Z+BWwHDtjF52q8GDf9GtiYmZ931M+hGznO9csHFwNX9tBPqWuOzS2qy2ccDTzN/PXIfDH3b7GwtcDvga/r/QOAD+v4DLPjZS0ymuXAB8CddSmkdRGxF+ZyazJzK/BH4C3KBzbbgWcwl7vSVu4eWm/vvF2Lk6+/llg3dc7aqVvWTT2wdurdVNZOTnpNiYh4tK4zunOb9e2fOtOZu3Hco4DDM/OBtvs8NF3FuHH8GeAe4ObMfL2lbku9iIifA9cCF066L99TVwA3ZuYnk+6IpOlRJ8T/DKzJzI+aj41aj6iIiF8A72fmM5Puy/fcDGWJk9sy82jgU8qyJt8wl8dTr4uwivJB2SHAXvht7l6Yu9J0sW7qlrVTL6ybemDtNDnTlL9Oek2JzDw1M4+coz0IvFd/4kf99/05DrEV+Enj/tK67ThgRUS8CfwN+GlEPNnluUyrDmO8w+3Aq5m5tqtzGKjvitusferk4b7Af3fxuRovxjuWMHgAWJ2Zr3Xe2+EaJ87HAtfVsXgNcGlE/K7rDksdcWxuQUTsQfng5u7MvL9unq8emS/m/i3mdzxwVh1311OWM7mJsqzGTN2nGS9rkdFsAbZk5tP1/n2UD3PM5facCryRmR9k5pfA/ZT8Npe70VbubmX2kvfGe3Hz9Tcm66ZeWDt1z7qpH9ZO/ZrK2slJr2HYAJxbb58LPDjHPo8Ap0XEfnVG+zTgkcy8LTMPycxlwAnAK5l5Ug99HpqRYwwQEVdTBsU1PfR1aP4JHBERyyNiT8pFITfstE8z/mcDj9dvB2wAzomIJRGxHDiCclFDzTZyjOuSnA9TLjr59956PEwjxzkzT8zMZXUsXgtck5m39NVxqWW78lrQAuoa8XcAL2XmDY2H5qtHNgCro1gJbK9LSMxbmyx2mXlJZi6t4+45lPH4N8ATlPEZvh1ja5HdlJnvAm9HxM/qplOAFzGX2/QWsDIiflTHjh0xNpe70Uru1sc+ioiV9e+2mrnfY2pxsHYag3VTP6ydumfd1Btrp35NZ+2UmbYpb5R1RB8DXgUeBfav21cA6xr7nUe5qN5m4LdzHGcZsGnS5zONbZwYU2aek3Ix1edqu2DS5zRNDTgDeAV4DbisbvsDcFa9/QPg3hrXfwCHNZ57WX3ey8Dpkz6XaW2jxhi4nPKT+uca7cBJn8+0tnFyuXGMK4CLJn0uNts4ba7Xgm234ndCrR3+3Rh7z1igHgng1hrv54EVjWMtWP/ZEuAk4KF6+7A6Pm+u4/WSut1aZPT4HgX8q+bzX4D9zOXWY3wl8B9gE/AnYIm53Epc76Fc6+NLyrfvz28zdynvJTfV59wCxKTP2TbRfLN2Gj121k39x9zaqbvYWjf1E2drp27iOpjaKeoBJUmSJEmSJEmSpMFyeUNJkiRJkiRJkiQNnpNekiRJkiRJkiRJGjwnvSRJkiRJkiRJkjR4TnpJkiRJkiRJkiRp8Jz0kiRJkiRJkiRJ0uA56SVJkiRJkiRJkqTBc9JLkiRJkiRJkiRJg+eklyRJkiRJkiRJkgbv/9hpi71XVAgqAAAAAElFTkSuQmCC\n"
          },
          "metadata": {
            "needs_background": "light"
          }
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\u001b[1;30;43mStreaming output truncated to the last 5000 lines.\u001b[0m\n",
            "reward of step 21520 is: 1.0698609933896517\n",
            "reward of step 21521 is: 1.0392960870889965\n",
            "reward of step 21522 is: 1.1090497431898998\n",
            "reward of step 21523 is: 1.1052526472955377\n",
            "reward of step 21524 is: 1.06653651819679\n",
            "reward of step 21525 is: 1.075616862633809\n",
            "reward of step 21526 is: 1.087968776940835\n",
            "reward of step 21527 is: 1.1002173670578828\n",
            "reward of step 21528 is: 1.0599142814021492\n",
            "reward of step 21529 is: 1.0151300321561725\n",
            "reward of step 21530 is: 1.0923512993100681\n",
            "reward of step 21531 is: 1.124106678371477\n",
            "reward of step 21532 is: 1.0477932954351852\n",
            "reward of step 21533 is: 1.0519409206047305\n",
            "reward of step 21534 is: 1.1849101610533164\n",
            "reward of step 21535 is: 1.0938612861857153\n",
            "reward of step 21536 is: 1.127094296938588\n",
            "reward of step 21537 is: 1.0865666131555838\n",
            "reward of step 21538 is: 1.0893811864727012\n",
            "reward of step 21539 is: 1.112596447810527\n",
            "reward of step 21540 is: 1.059037185234566\n",
            "reward of step 21541 is: 1.0999312963725\n",
            "reward of step 21542 is: 1.0588658177213355\n",
            "reward of step 21543 is: 1.0550364794626765\n",
            "reward of step 21544 is: 1.015780758864353\n",
            "reward of step 21545 is: 1.1187380041249755\n",
            "reward of step 21546 is: 1.1075274503314327\n",
            "reward of step 21547 is: 1.116390913057541\n",
            "reward of step 21548 is: 1.0538334623503731\n",
            "reward of step 21549 is: 1.0985899022550536\n",
            "reward of step 21550 is: 1.1119743379829479\n",
            "reward of step 21551 is: 1.0897921411085387\n",
            "reward of step 21552 is: 1.0936596922566093\n",
            "reward of step 21553 is: 1.016059229300764\n",
            "reward of step 21554 is: 1.0260738038800898\n",
            "reward of step 21555 is: 1.117467864261153\n",
            "reward of step 21556 is: 1.025275991340072\n",
            "reward of step 21557 is: 1.0567802417945336\n",
            "reward of step 21558 is: 1.0400024169846729\n",
            "reward of step 21559 is: 1.0632310353294518\n",
            "reward of step 21560 is: 1.0868766840593431\n",
            "reward of step 21561 is: 1.0840841294893124\n",
            "reward of step 21562 is: 1.0262388777264737\n",
            "reward of step 21563 is: 1.0877860145118952\n",
            "reward of step 21564 is: 1.134019214352536\n",
            "reward of step 21565 is: 1.0973367017974291\n",
            "reward of step 21566 is: 1.146337179515712\n",
            "reward of step 21567 is: 1.0993958442204224\n",
            "reward of step 21568 is: 1.1219848290926062\n",
            "reward of step 21569 is: 1.0696051560483915\n",
            "reward of step 21570 is: 1.0672752207289173\n",
            "reward of step 21571 is: 1.0467052058171356\n",
            "reward of step 21572 is: 1.1196134935342943\n",
            "reward of step 21573 is: 1.0005096057844254\n",
            "reward of step 21574 is: 1.0845651991072525\n",
            "reward of step 21575 is: 1.0508435333811164\n",
            "reward of step 21576 is: 1.0677771374919138\n",
            "reward of step 21577 is: 1.0229085229814276\n",
            "reward of step 21578 is: 1.093871626296787\n",
            "reward of step 21579 is: 1.090677137149942\n",
            "reward of step 21580 is: 1.1421809570351216\n",
            "reward of step 21581 is: 1.061319974727584\n",
            "reward of step 21582 is: 1.102816804874116\n",
            "reward of step 21583 is: 1.0829980444539034\n",
            "reward of step 21584 is: 1.0851930613882104\n",
            "reward of step 21585 is: 1.030631908131639\n",
            "reward of step 21586 is: 1.0089724600183076\n",
            "reward of step 21587 is: 1.1120922764914076\n",
            "reward of step 21588 is: 1.0631110030143307\n",
            "reward of step 21589 is: 1.0867021566477149\n",
            "reward of step 21590 is: 1.0510772332531046\n",
            "reward of step 21591 is: 1.0806837981724478\n",
            "reward of step 21592 is: 1.0928365278257735\n",
            "reward of step 21593 is: 1.0873559726781312\n",
            "reward of step 21594 is: 1.1398407937760808\n",
            "reward of step 21595 is: 1.1061226641751252\n",
            "reward of step 21596 is: 1.039280740442985\n",
            "reward of step 21597 is: 1.0341702877799195\n",
            "reward of step 21598 is: 1.1111771512704292\n",
            "reward of step 21599 is: 1.1342953681617751\n",
            "reward of step 21600 is: 1.065778701256422\n",
            "reward of step 21601 is: 1.0499711489916417\n",
            "reward of step 21602 is: 1.0160082157385002\n",
            "reward of step 21603 is: 1.1500338110994108\n",
            "reward of step 21604 is: 1.0932706836767232\n",
            "reward of step 21605 is: 1.0697191996016784\n",
            "reward of step 21606 is: 1.0722883592575783\n",
            "reward of step 21607 is: 1.0958766532766706\n",
            "reward of step 21608 is: 1.107784784458194\n",
            "reward of step 21609 is: 1.1228627040898944\n",
            "reward of step 21610 is: 1.1633433508491686\n",
            "reward of step 21611 is: 1.0837980262585372\n",
            "reward of step 21612 is: 1.054371230133308\n",
            "reward of step 21613 is: 1.0719378864485123\n",
            "reward of step 21614 is: 1.0858408215788258\n",
            "reward of step 21615 is: 1.0723060781771603\n",
            "reward of step 21616 is: 1.0935707072486847\n",
            "reward of step 21617 is: 1.0642633650741842\n",
            "reward of step 21618 is: 1.1083763875563228\n",
            "reward of step 21619 is: 0.989298382960903\n",
            "reward of step 21620 is: 1.0609885137568311\n",
            "reward of step 21621 is: 1.0248895308553825\n",
            "reward of step 21622 is: 1.122436933884753\n",
            "reward of step 21623 is: 1.0974424529667473\n",
            "reward of step 21624 is: 1.0461312327629835\n",
            "reward of step 21625 is: 1.0089682110113969\n",
            "reward of step 21626 is: 1.059886258303202\n",
            "reward of step 21627 is: 1.022257678558633\n",
            "reward of step 21628 is: 1.027732920679555\n",
            "reward of step 21629 is: 1.1252351114139727\n",
            "reward of step 21630 is: 1.0828064401011854\n",
            "reward of step 21631 is: 1.1084317239096837\n",
            "reward of step 21632 is: 1.0142691192159172\n",
            "reward of step 21633 is: 1.0479279058951083\n",
            "reward of step 21634 is: 1.0944309996583907\n",
            "reward of step 21635 is: 1.0402770661174043\n",
            "reward of step 21636 is: 1.0950935129387327\n",
            "reward of step 21637 is: 1.0555064929989153\n",
            "reward of step 21638 is: 1.0202412102667062\n",
            "reward of step 21639 is: 1.0776447573773922\n",
            "reward of step 21640 is: 1.0586755116500615\n",
            "reward of step 21641 is: 1.052646545372574\n",
            "reward of step 21642 is: 1.0937865460874265\n",
            "reward of step 21643 is: 1.0949852861012719\n",
            "reward of step 21644 is: 1.0850610473218656\n",
            "reward of step 21645 is: 1.0489001800882227\n",
            "reward of step 21646 is: 1.0473183866953246\n",
            "reward of step 21647 is: 0.9690155286988842\n",
            "reward of step 21648 is: 1.076553430370401\n",
            "reward of step 21649 is: 1.077568704967673\n",
            "reward of step 21650 is: 1.0692570332156848\n",
            "reward of step 21651 is: 1.0511500154111704\n",
            "reward of step 21652 is: 1.0849213231041337\n",
            "reward of step 21653 is: 1.0147871711832113\n",
            "reward of step 21654 is: 1.0790390868530986\n",
            "reward of step 21655 is: 1.0821016995428065\n",
            "reward of step 21656 is: 1.1335758451186166\n",
            "reward of step 21657 is: 1.0237026980680684\n",
            "reward of step 21658 is: 1.0267627598016524\n",
            "reward of step 21659 is: 1.1042900975774195\n",
            "reward of step 21660 is: 1.0434342840450863\n",
            "reward of step 21661 is: 1.104453713908438\n",
            "reward of step 21662 is: 1.0352192803884093\n",
            "reward of step 21663 is: 1.1392704514936733\n",
            "reward of step 21664 is: 1.1295005600189527\n",
            "reward of step 21665 is: 1.0548511054218528\n",
            "reward of step 21666 is: 1.0823136374824929\n",
            "reward of step 21667 is: 1.096316384133241\n",
            "reward of step 21668 is: 1.1050524314611943\n",
            "reward of step 21669 is: 1.1160083205495497\n",
            "reward of step 21670 is: 1.0692672480811578\n",
            "reward of step 21671 is: 1.005808245324088\n",
            "reward of step 21672 is: 1.0359453587686303\n",
            "reward of step 21673 is: 1.1186185671074078\n",
            "reward of step 21674 is: 1.1426764140589087\n",
            "reward of step 21675 is: 1.0214280741167248\n",
            "reward of step 21676 is: 1.1114093695045222\n",
            "reward of step 21677 is: 1.1127804834636148\n",
            "reward of step 21678 is: 1.1195407260590315\n",
            "reward of step 21679 is: 1.0474910559833301\n",
            "reward of step 21680 is: 1.0997180444532018\n",
            "reward of step 21681 is: 1.060198739232147\n",
            "reward of step 21682 is: 1.0536087969907335\n",
            "reward of step 21683 is: 1.1366712987404752\n",
            "reward of step 21684 is: 1.1193098064255085\n",
            "reward of step 21685 is: 1.0307382093645248\n",
            "reward of step 21686 is: 1.0414523104043976\n",
            "reward of step 21687 is: 1.0479742460142167\n",
            "reward of step 21688 is: 1.0739956481215058\n",
            "reward of step 21689 is: 1.1362332913095643\n",
            "reward of step 21690 is: 0.9976065715350748\n",
            "reward of step 21691 is: 1.0322020653569828\n",
            "reward of step 21692 is: 1.1464180522802798\n",
            "reward of step 21693 is: 1.0597864527640874\n",
            "reward of step 21694 is: 1.0782410183240394\n",
            "reward of step 21695 is: 1.049397938086949\n",
            "reward of step 21696 is: 1.1124850483486486\n",
            "reward of step 21697 is: 1.035336576206153\n",
            "reward of step 21698 is: 1.1187203605480533\n",
            "reward of step 21699 is: 1.0424600701212985\n",
            "reward of step 21700 is: 1.0266207900562612\n",
            "reward of step 21701 is: 0.9861251373191263\n",
            "reward of step 21702 is: 1.0983086937672786\n",
            "reward of step 21703 is: 1.1105825882317726\n",
            "reward of step 21704 is: 1.0413754594110065\n",
            "reward of step 21705 is: 1.042148851137208\n",
            "reward of step 21706 is: 1.0910166091068918\n",
            "reward of step 21707 is: 1.1481820125168738\n",
            "reward of step 21708 is: 1.1066621125725644\n",
            "reward of step 21709 is: 1.0981245085490872\n",
            "reward of step 21710 is: 1.030010858532959\n",
            "reward of step 21711 is: 1.0925479170287031\n",
            "reward of step 21712 is: 1.0724564980632072\n",
            "reward of step 21713 is: 1.143274620415598\n",
            "reward of step 21714 is: 1.063944515479148\n",
            "reward of step 21715 is: 1.0413943743954035\n",
            "reward of step 21716 is: 1.1182073010858509\n",
            "reward of step 21717 is: 1.0617568736623433\n",
            "reward of step 21718 is: 1.0258287958674381\n",
            "reward of step 21719 is: 0.972645938460283\n",
            "reward of step 21720 is: 1.17028630433712\n",
            "reward of step 21721 is: 1.1229501039980254\n",
            "reward of step 21722 is: 1.109715307078574\n",
            "reward of step 21723 is: 1.1005317351187283\n",
            "reward of step 21724 is: 1.0959126044474499\n",
            "reward of step 21725 is: 1.1000989041213294\n",
            "reward of step 21726 is: 1.097013228854805\n",
            "reward of step 21727 is: 1.0390326597843365\n",
            "reward of step 21728 is: 1.0199603030359394\n",
            "reward of step 21729 is: 1.0612904960583176\n",
            "reward of step 21730 is: 1.0539140684552233\n",
            "reward of step 21731 is: 1.1071979899612672\n",
            "reward of step 21732 is: 1.0607080457932763\n",
            "reward of step 21733 is: 0.9777826001690997\n",
            "reward of step 21734 is: 1.059891459105566\n",
            "reward of step 21735 is: 1.0737907408623728\n",
            "reward of step 21736 is: 1.0132919425781717\n",
            "reward of step 21737 is: 1.0506750044031237\n",
            "reward of step 21738 is: 1.08737203417714\n",
            "reward of step 21739 is: 1.0421028313952858\n",
            "reward of step 21740 is: 1.0492559186354335\n",
            "reward of step 21741 is: 1.059545186656118\n",
            "reward of step 21742 is: 1.1607628818368214\n",
            "reward of step 21743 is: 1.0980386485463287\n",
            "reward of step 21744 is: 1.1616930954841291\n",
            "reward of step 21745 is: 1.0641521737056134\n",
            "reward of step 21746 is: 1.0987309574651534\n",
            "reward of step 21747 is: 1.1012251958910406\n",
            "reward of step 21748 is: 1.0732259730024973\n",
            "reward of step 21749 is: 1.1123187487269286\n",
            "reward of step 21750 is: 1.0537231525523056\n",
            "reward of step 21751 is: 1.0832528368514502\n",
            "reward of step 21752 is: 1.0854035134999984\n",
            "reward of step 21753 is: 0.9782217555057136\n",
            "reward of step 21754 is: 1.083214478088192\n",
            "reward of step 21755 is: 1.1019786093804642\n",
            "reward of step 21756 is: 1.1271490731730442\n",
            "reward of step 21757 is: 1.0540005276742084\n",
            "reward of step 21758 is: 1.1111649374042303\n",
            "reward of step 21759 is: 1.1221852674424637\n",
            "reward of step 21760 is: 1.0612539163630483\n",
            "reward of step 21761 is: 1.0250205035536888\n",
            "reward of step 21762 is: 1.1115011455346206\n",
            "reward of step 21763 is: 1.0477409726177314\n",
            "reward of step 21764 is: 1.1130676088743514\n",
            "reward of step 21765 is: 1.0804933899462688\n",
            "reward of step 21766 is: 1.0991369051559507\n",
            "reward of step 21767 is: 1.0791825546563247\n",
            "reward of step 21768 is: 1.0532941818724537\n",
            "reward of step 21769 is: 1.111377843275885\n",
            "reward of step 21770 is: 1.1314247476795578\n",
            "reward of step 21771 is: 1.0577616742118\n",
            "reward of step 21772 is: 1.1429150448013092\n",
            "reward of step 21773 is: 1.129035496258893\n",
            "reward of step 21774 is: 1.0359551852482483\n",
            "reward of step 21775 is: 1.0039723291587341\n",
            "reward of step 21776 is: 1.0422210612382954\n",
            "reward of step 21777 is: 1.0250495116484912\n",
            "reward of step 21778 is: 1.129993210618299\n",
            "reward of step 21779 is: 1.048882471467206\n",
            "reward of step 21780 is: 1.0856639157809123\n",
            "reward of step 21781 is: 1.083387852831923\n",
            "reward of step 21782 is: 1.1428513753509857\n",
            "reward of step 21783 is: 1.0902056974799685\n",
            "reward of step 21784 is: 1.1005234592018422\n",
            "reward of step 21785 is: 1.1327690774374777\n",
            "reward of step 21786 is: 1.06593112670715\n",
            "reward of step 21787 is: 1.1161472548884164\n",
            "reward of step 21788 is: 1.0366836113330369\n",
            "reward of step 21789 is: 1.1681144745564478\n",
            "reward of step 21790 is: 1.065826380860608\n",
            "reward of step 21791 is: 1.099944968075253\n",
            "reward of step 21792 is: 1.0953977995662796\n",
            "reward of step 21793 is: 1.077898281911886\n",
            "reward of step 21794 is: 1.0995102390023646\n",
            "reward of step 21795 is: 1.0678043058712108\n",
            "reward of step 21796 is: 1.0717976884864\n",
            "reward of step 21797 is: 1.0859339363715343\n",
            "reward of step 21798 is: 1.1337644305201828\n",
            "reward of step 21799 is: 1.0608308015225343\n",
            "reward of step 21800 is: 1.084414362563224\n",
            "reward of step 21801 is: 1.0753893276380833\n",
            "reward of step 21802 is: 1.0735720040619938\n",
            "reward of step 21803 is: 1.0697374446132093\n",
            "reward of step 21804 is: 1.1162721594305343\n",
            "reward of step 21805 is: 1.1229018359729692\n",
            "reward of step 21806 is: 1.054944911777533\n",
            "reward of step 21807 is: 1.0641630031607288\n",
            "reward of step 21808 is: 1.1177646070697602\n",
            "reward of step 21809 is: 1.0864142703964208\n",
            "reward of step 21810 is: 1.125965906298298\n",
            "reward of step 21811 is: 1.1636474842151134\n",
            "reward of step 21812 is: 1.0641577171072278\n",
            "reward of step 21813 is: 1.0566497437158704\n",
            "reward of step 21814 is: 1.097710516274399\n",
            "reward of step 21815 is: 1.0970057181312014\n",
            "reward of step 21816 is: 1.0450782574831377\n",
            "reward of step 21817 is: 1.0436001449045702\n",
            "reward of step 21818 is: 1.1187343771388747\n",
            "reward of step 21819 is: 1.158034091596372\n",
            "reward of step 21820 is: 1.124720367698954\n",
            "reward of step 21821 is: 1.087690515715252\n",
            "reward of step 21822 is: 1.0597035846548777\n",
            "reward of step 21823 is: 0.9855707986495531\n",
            "reward of step 21824 is: 1.0565490062913026\n",
            "reward of step 21825 is: 1.1038249293832765\n",
            "reward of step 21826 is: 1.0343614569899544\n",
            "reward of step 21827 is: 1.1162931520528967\n",
            "reward of step 21828 is: 1.0749349043095069\n",
            "reward of step 21829 is: 1.0692296308272526\n",
            "reward of step 21830 is: 1.1159202204769134\n",
            "reward of step 21831 is: 1.0009534147678605\n",
            "reward of step 21832 is: 1.1085882635940734\n",
            "reward of step 21833 is: 1.0337566400975602\n",
            "reward of step 21834 is: 1.122780984679565\n",
            "reward of step 21835 is: 1.0916314908453404\n",
            "reward of step 21836 is: 1.0750022044101022\n",
            "reward of step 21837 is: 1.077396698291684\n",
            "reward of step 21838 is: 1.0949469805454544\n",
            "reward of step 21839 is: 1.1105747239885109\n",
            "reward of step 21840 is: 1.0243598104750955\n",
            "reward of step 21841 is: 1.1318975910376943\n",
            "reward of step 21842 is: 1.0601808962146277\n",
            "reward of step 21843 is: 1.0175469830666142\n",
            "reward of step 21844 is: 1.1109975225815512\n",
            "reward of step 21845 is: 1.0890475525165102\n",
            "reward of step 21846 is: 1.067364894915769\n",
            "reward of step 21847 is: 1.0773173914173975\n",
            "reward of step 21848 is: 1.040138311485049\n",
            "reward of step 21849 is: 1.0510855993626511\n",
            "reward of step 21850 is: 1.025589928053961\n",
            "reward of step 21851 is: 1.0997457495182998\n",
            "reward of step 21852 is: 1.072105626874809\n",
            "reward of step 21853 is: 1.0785793700398694\n",
            "reward of step 21854 is: 1.1061160156931908\n",
            "reward of step 21855 is: 1.0103965749349468\n",
            "reward of step 21856 is: 1.085889643453441\n",
            "reward of step 21857 is: 1.1116211116889922\n",
            "reward of step 21858 is: 1.0501713211405597\n",
            "reward of step 21859 is: 1.0585015241159144\n",
            "reward of step 21860 is: 1.078706336631994\n",
            "reward of step 21861 is: 1.1077133972948112\n",
            "reward of step 21862 is: 1.0688710122606602\n",
            "reward of step 21863 is: 1.1267985662644249\n",
            "reward of step 21864 is: 1.0427389413533126\n",
            "reward of step 21865 is: 1.0807118635389585\n",
            "reward of step 21866 is: 1.1261594739390863\n",
            "reward of step 21867 is: 1.0563461205513531\n",
            "reward of step 21868 is: 1.1125933888786652\n",
            "reward of step 21869 is: 1.0402588994329713\n",
            "reward of step 21870 is: 1.066838410187371\n",
            "reward of step 21871 is: 1.1461590769283623\n",
            "reward of step 21872 is: 1.1458527729530894\n",
            "reward of step 21873 is: 0.997291410812264\n",
            "reward of step 21874 is: 1.1248910602222084\n",
            "reward of step 21875 is: 1.1595276865334712\n",
            "reward of step 21876 is: 1.0476690004270297\n",
            "reward of step 21877 is: 1.0789211804638672\n",
            "reward of step 21878 is: 1.091965961402391\n",
            "reward of step 21879 is: 1.0620101801449398\n",
            "reward of step 21880 is: 1.1127067327967683\n",
            "reward of step 21881 is: 1.0972320450859692\n",
            "reward of step 21882 is: 1.1280383786789634\n",
            "reward of step 21883 is: 1.0788398379424526\n",
            "reward of step 21884 is: 1.0742647681606967\n",
            "reward of step 21885 is: 1.083016330261036\n",
            "reward of step 21886 is: 1.0479747430240391\n",
            "reward of step 21887 is: 1.1124712635410914\n",
            "reward of step 21888 is: 1.0822914757094946\n",
            "reward of step 21889 is: 1.065758814778013\n",
            "reward of step 21890 is: 1.0616902363557337\n",
            "reward of step 21891 is: 1.078528325475498\n",
            "reward of step 21892 is: 1.0481802669569003\n",
            "reward of step 21893 is: 1.1154074118256183\n",
            "reward of step 21894 is: 1.0733235106826557\n",
            "reward of step 21895 is: 1.0252202394632581\n",
            "reward of step 21896 is: 1.0740878853828588\n",
            "reward of step 21897 is: 1.1044914782577737\n",
            "reward of step 21898 is: 1.0641625998633866\n",
            "reward of step 21899 is: 1.0397059632412957\n",
            "reward of step 21900 is: 1.0552624131605084\n",
            "reward of step 21901 is: 1.1164542898432213\n",
            "reward of step 21902 is: 1.0763033852359367\n",
            "reward of step 21903 is: 1.1453424629241247\n",
            "reward of step 21904 is: 1.0891542143184028\n",
            "reward of step 21905 is: 1.033850915037545\n",
            "reward of step 21906 is: 1.032306248189593\n",
            "reward of step 21907 is: 1.0650363155992064\n",
            "reward of step 21908 is: 1.0883213180497309\n",
            "reward of step 21909 is: 1.1183230920654683\n",
            "reward of step 21910 is: 1.0927362272831442\n",
            "reward of step 21911 is: 1.0345690449565799\n",
            "reward of step 21912 is: 1.0822324441542706\n",
            "reward of step 21913 is: 1.0325922655502504\n",
            "reward of step 21914 is: 1.112074706554834\n",
            "reward of step 21915 is: 1.0723878009307284\n",
            "reward of step 21916 is: 1.0527651508492406\n",
            "reward of step 21917 is: 1.0694831307388\n",
            "reward of step 21918 is: 1.1563251843169144\n",
            "reward of step 21919 is: 1.070525469740051\n",
            "reward of step 21920 is: 1.1085768285346635\n",
            "reward of step 21921 is: 0.9759372929936114\n",
            "reward of step 21922 is: 1.1303349653117605\n",
            "reward of step 21923 is: 1.034281281445577\n",
            "reward of step 21924 is: 1.1200956191138083\n",
            "reward of step 21925 is: 1.0450533441747976\n",
            "reward of step 21926 is: 1.0556954165953645\n",
            "reward of step 21927 is: 1.0524561876831708\n",
            "reward of step 21928 is: 1.0837315384198747\n",
            "reward of step 21929 is: 1.138125154295723\n",
            "reward of step 21930 is: 1.081707604666905\n",
            "reward of step 21931 is: 1.1550624715271707\n",
            "reward of step 21932 is: 1.053744987983447\n",
            "reward of step 21933 is: 1.1298549351370104\n",
            "reward of step 21934 is: 1.080935284470979\n",
            "reward of step 21935 is: 1.1112809122222944\n",
            "reward of step 21936 is: 1.0940345970079743\n",
            "reward of step 21937 is: 1.119927851121973\n",
            "reward of step 21938 is: 1.0909602924004798\n",
            "reward of step 21939 is: 1.1077470149883273\n",
            "reward of step 21940 is: 1.1147057814089352\n",
            "reward of step 21941 is: 1.0103813315983552\n",
            "reward of step 21942 is: 1.019212974183624\n",
            "reward of step 21943 is: 1.1320778506187645\n",
            "reward of step 21944 is: 1.0249894303728655\n",
            "reward of step 21945 is: 1.0338953586881123\n",
            "reward of step 21946 is: 1.072484993727548\n",
            "reward of step 21947 is: 1.0886434324762442\n",
            "reward of step 21948 is: 1.099079180539689\n",
            "reward of step 21949 is: 1.0399057570602706\n",
            "reward of step 21950 is: 1.1292188336041014\n",
            "reward of step 21951 is: 1.0892642854956125\n",
            "reward of step 21952 is: 1.0545075124933736\n",
            "reward of step 21953 is: 1.0518348156500106\n",
            "reward of step 21954 is: 1.1491711445836275\n",
            "reward of step 21955 is: 1.1545691699381453\n",
            "reward of step 21956 is: 1.0913729440082305\n",
            "reward of step 21957 is: 1.0679354482481878\n",
            "reward of step 21958 is: 1.0662557438437086\n",
            "reward of step 21959 is: 1.059019490041638\n",
            "reward of step 21960 is: 1.091919895434403\n",
            "reward of step 21961 is: 1.119612921354804\n",
            "reward of step 21962 is: 1.0279976758477825\n",
            "reward of step 21963 is: 1.1198006382379286\n",
            "reward of step 21964 is: 1.0563645723250166\n",
            "reward of step 21965 is: 1.1042343003163353\n",
            "reward of step 21966 is: 1.0612570541316955\n",
            "reward of step 21967 is: 1.0945190602141177\n",
            "reward of step 21968 is: 1.0684035797934976\n",
            "reward of step 21969 is: 1.0516957495181063\n",
            "reward of step 21970 is: 1.1048764699281648\n",
            "reward of step 21971 is: 1.0679287660706478\n",
            "reward of step 21972 is: 1.1146634654613574\n",
            "reward of step 21973 is: 1.1163702567131149\n",
            "reward of step 21974 is: 1.082751500831646\n",
            "reward of step 21975 is: 1.0583451130328743\n",
            "reward of step 21976 is: 1.0687969594040387\n",
            "reward of step 21977 is: 1.0601682527346836\n",
            "reward of step 21978 is: 1.1297978242575055\n",
            "reward of step 21979 is: 1.0535635725343746\n",
            "reward of step 21980 is: 1.082351090542978\n",
            "reward of step 21981 is: 1.0607169409994994\n",
            "reward of step 21982 is: 1.0187633661414375\n",
            "reward of step 21983 is: 1.109473688797027\n",
            "reward of step 21984 is: 1.1303459543478507\n",
            "reward of step 21985 is: 1.095719320274136\n",
            "reward of step 21986 is: 1.087090646084747\n",
            "reward of step 21987 is: 1.016042131522034\n",
            "reward of step 21988 is: 1.1216517652681306\n",
            "reward of step 21989 is: 1.0956642610142195\n",
            "reward of step 21990 is: 1.0296202991635943\n",
            "reward of step 21991 is: 1.1340515796589337\n",
            "reward of step 21992 is: 1.107932586316663\n",
            "reward of step 21993 is: 1.0869490727609763\n",
            "reward of step 21994 is: 1.103527400715013\n",
            "reward of step 21995 is: 1.097781942428121\n",
            "reward of step 21996 is: 1.115860492173935\n",
            "reward of step 21997 is: 1.0637997342327359\n",
            "reward of step 21998 is: 1.0640136529734732\n",
            "reward of step 21999 is: 1.047791227546179\n",
            "reward of step 22000 is: 1.0215321872446947\n",
            "reward of step 22001 is: 1.140975900832445\n",
            "reward of step 22002 is: 1.105637033808731\n",
            "reward of step 22003 is: 1.0355929445329048\n",
            "reward of step 22004 is: 0.9867797788638074\n",
            "reward of step 22005 is: 1.0247611289804337\n",
            "reward of step 22006 is: 1.116204168871806\n",
            "reward of step 22007 is: 1.0183731839271963\n",
            "reward of step 22008 is: 1.1177519822010504\n",
            "reward of step 22009 is: 1.1092987304115667\n",
            "reward of step 22010 is: 1.1216717100794558\n",
            "reward of step 22011 is: 1.1142904489538088\n",
            "reward of step 22012 is: 1.127573213826209\n",
            "reward of step 22013 is: 1.088326921659736\n",
            "reward of step 22014 is: 1.024645940617912\n",
            "reward of step 22015 is: 1.0557248788444613\n",
            "reward of step 22016 is: 1.0666319998869924\n",
            "reward of step 22017 is: 1.0623732383261775\n",
            "reward of step 22018 is: 1.098282221190849\n",
            "reward of step 22019 is: 1.1394180385311168\n",
            "reward of step 22020 is: 1.1516601624433822\n",
            "reward of step 22021 is: 1.005948833282627\n",
            "reward of step 22022 is: 1.0239945921267006\n",
            "reward of step 22023 is: 1.0624184615634138\n",
            "reward of step 22024 is: 1.1002293679184982\n",
            "reward of step 22025 is: 1.0702035435941495\n",
            "reward of step 22026 is: 1.0900783412881059\n",
            "reward of step 22027 is: 1.0601321178872425\n",
            "reward of step 22028 is: 1.0337634521432344\n",
            "reward of step 22029 is: 1.1052906543841923\n",
            "reward of step 22030 is: 1.1099719817515636\n",
            "reward of step 22031 is: 1.0920045359928285\n",
            "reward of step 22032 is: 1.0928282871867183\n",
            "reward of step 22033 is: 1.0555282241955912\n",
            "reward of step 22034 is: 1.011392716454875\n",
            "reward of step 22035 is: 1.1261295922367056\n",
            "reward of step 22036 is: 1.0475298869572844\n",
            "reward of step 22037 is: 1.050330160453633\n",
            "reward of step 22038 is: 1.1056939824057452\n",
            "reward of step 22039 is: 1.0102598741989097\n",
            "reward of step 22040 is: 1.0001844775740203\n",
            "reward of step 22041 is: 1.0975327748754713\n",
            "reward of step 22042 is: 1.0608906110457779\n",
            "reward of step 22043 is: 1.0999566457079848\n",
            "reward of step 22044 is: 0.990452198336123\n",
            "reward of step 22045 is: 1.1040740501049342\n",
            "reward of step 22046 is: 1.0604895025725734\n",
            "reward of step 22047 is: 1.1408751701059168\n",
            "reward of step 22048 is: 1.0187774059985697\n",
            "reward of step 22049 is: 1.0171497118390809\n",
            "reward of step 22050 is: 1.0260990897203368\n",
            "reward of step 22051 is: 1.0807255887558866\n",
            "reward of step 22052 is: 1.1523030655165099\n",
            "reward of step 22053 is: 1.0978642260122404\n",
            "reward of step 22054 is: 1.1131416065983752\n",
            "reward of step 22055 is: 1.035898159724851\n",
            "reward of step 22056 is: 1.0868220985069028\n",
            "reward of step 22057 is: 1.0534277205447427\n",
            "reward of step 22058 is: 1.0867151502861858\n",
            "reward of step 22059 is: 1.060923782500214\n",
            "reward of step 22060 is: 1.086533815055207\n",
            "reward of step 22061 is: 1.076848515905941\n",
            "reward of step 22062 is: 1.1035157463580658\n",
            "reward of step 22063 is: 1.074261858638931\n",
            "reward of step 22064 is: 1.1042286736674403\n",
            "reward of step 22065 is: 1.1028611615611692\n",
            "reward of step 22066 is: 1.0828144178935306\n",
            "reward of step 22067 is: 1.0285591445170732\n",
            "reward of step 22068 is: 1.1026442415892168\n",
            "reward of step 22069 is: 1.0996467628233901\n",
            "reward of step 22070 is: 1.0944609832702037\n",
            "reward of step 22071 is: 1.054976232834675\n",
            "reward of step 22072 is: 1.1077995783088361\n",
            "reward of step 22073 is: 1.1097347630213013\n",
            "reward of step 22074 is: 1.0131920581699938\n",
            "reward of step 22075 is: 1.0948505748317885\n",
            "reward of step 22076 is: 1.0712090918409845\n",
            "reward of step 22077 is: 1.0989534668510572\n",
            "reward of step 22078 is: 1.1242384536822456\n",
            "reward of step 22079 is: 1.1404594593089281\n",
            "reward of step 22080 is: 1.0327677149109362\n",
            "reward of step 22081 is: 1.085366200015781\n",
            "reward of step 22082 is: 1.0802566356699823\n",
            "reward of step 22083 is: 1.013115363709269\n",
            "reward of step 22084 is: 1.1603460430806272\n",
            "reward of step 22085 is: 1.0619849378717494\n",
            "reward of step 22086 is: 1.0156557917070559\n",
            "reward of step 22087 is: 1.104972830146289\n",
            "reward of step 22088 is: 1.0678002833018831\n",
            "reward of step 22089 is: 1.0851577959784962\n",
            "reward of step 22090 is: 1.1485632146248768\n",
            "reward of step 22091 is: 1.045372908928977\n",
            "reward of step 22092 is: 1.040972736033697\n",
            "reward of step 22093 is: 1.0180271642014516\n",
            "reward of step 22094 is: 1.124578003777056\n",
            "reward of step 22095 is: 1.0418881694770374\n",
            "reward of step 22096 is: 1.0732038966142003\n",
            "reward of step 22097 is: 1.041412969862825\n",
            "reward of step 22098 is: 1.1172637276761974\n",
            "reward of step 22099 is: 1.1251797247249509\n",
            "reward of step 22100 is: 1.0934245559725186\n",
            "reward of step 22101 is: 1.1348052594675613\n",
            "reward of step 22102 is: 1.0065996318070738\n",
            "reward of step 22103 is: 1.1151517409298992\n",
            "reward of step 22104 is: 0.9860590854889268\n",
            "reward of step 22105 is: 1.089803594298402\n",
            "reward of step 22106 is: 1.1077447473136353\n",
            "reward of step 22107 is: 1.0568546103853302\n",
            "reward of step 22108 is: 1.093256552358726\n",
            "reward of step 22109 is: 1.0269165079187799\n",
            "reward of step 22110 is: 1.1051923651165416\n",
            "reward of step 22111 is: 1.0538816688559955\n",
            "reward of step 22112 is: 1.045779970068929\n",
            "reward of step 22113 is: 1.049325289497117\n",
            "reward of step 22114 is: 1.0407844241844497\n",
            "reward of step 22115 is: 1.073573239008574\n",
            "reward of step 22116 is: 1.1156031760655092\n",
            "reward of step 22117 is: 1.0610911046799125\n",
            "reward of step 22118 is: 1.0805568631344804\n",
            "reward of step 22119 is: 1.0826476529693418\n",
            "reward of step 22120 is: 1.0804187031652595\n",
            "reward of step 22121 is: 1.051782672883478\n",
            "reward of step 22122 is: 1.1040360913543208\n",
            "reward of step 22123 is: 1.1146846672527786\n",
            "reward of step 22124 is: 1.0859611642939266\n",
            "reward of step 22125 is: 1.101354451313741\n",
            "reward of step 22126 is: 1.085432441682388\n",
            "reward of step 22127 is: 1.0570230293336582\n",
            "reward of step 22128 is: 1.0070997935773445\n",
            "reward of step 22129 is: 1.1628159255745916\n",
            "reward of step 22130 is: 1.1226428246829254\n",
            "reward of step 22131 is: 1.06892037743093\n",
            "reward of step 22132 is: 1.0233721794882125\n",
            "reward of step 22133 is: 1.074139193052788\n",
            "reward of step 22134 is: 1.0246613205346433\n",
            "reward of step 22135 is: 1.088542603491127\n",
            "reward of step 22136 is: 1.086914683504942\n",
            "reward of step 22137 is: 1.1093435852894444\n",
            "reward of step 22138 is: 1.092303638052849\n",
            "reward of step 22139 is: 1.0954705091172232\n",
            "reward of step 22140 is: 1.092910612394779\n",
            "reward of step 22141 is: 1.092802493241242\n",
            "reward of step 22142 is: 1.1027237937609136\n",
            "reward of step 22143 is: 1.0488952617424196\n",
            "reward of step 22144 is: 1.034493304255497\n",
            "reward of step 22145 is: 1.0528150754585635\n",
            "reward of step 22146 is: 1.1076117699719161\n",
            "reward of step 22147 is: 1.0744119939928065\n",
            "reward of step 22148 is: 1.1039064599393464\n",
            "reward of step 22149 is: 1.0424533732629542\n",
            "reward of step 22150 is: 1.0357257807734261\n",
            "reward of step 22151 is: 1.0497090900449262\n",
            "reward of step 22152 is: 1.0165761289060242\n",
            "reward of step 22153 is: 1.0487011501414423\n",
            "reward of step 22154 is: 1.07988682095599\n",
            "reward of step 22155 is: 0.9915328405696897\n",
            "reward of step 22156 is: 1.0628839389123543\n",
            "reward of step 22157 is: 0.9935297628104913\n",
            "reward of step 22158 is: 1.1368011131917322\n",
            "reward of step 22159 is: 1.0811898870940833\n",
            "reward of step 22160 is: 1.0676717147731511\n",
            "reward of step 22161 is: 1.0977889083570136\n",
            "reward of step 22162 is: 1.0323942576129976\n",
            "reward of step 22163 is: 1.0858317494588825\n",
            "reward of step 22164 is: 1.1400290133650341\n",
            "reward of step 22165 is: 1.0597627824416116\n",
            "reward of step 22166 is: 1.02807409699858\n",
            "reward of step 22167 is: 1.0821570552010775\n",
            "reward of step 22168 is: 1.0833248630030456\n",
            "reward of step 22169 is: 1.0894964710716175\n",
            "reward of step 22170 is: 1.0686621127995712\n",
            "reward of step 22171 is: 1.0296706122494532\n",
            "reward of step 22172 is: 1.1258251668836112\n",
            "reward of step 22173 is: 1.1197936146356122\n",
            "reward of step 22174 is: 1.1186669702877066\n",
            "reward of step 22175 is: 1.0727813583813477\n",
            "reward of step 22176 is: 1.0914663778461882\n",
            "reward of step 22177 is: 1.1277779175017681\n",
            "reward of step 22178 is: 1.0980139751558005\n",
            "reward of step 22179 is: 1.053731914368114\n",
            "reward of step 22180 is: 1.033907454668705\n",
            "reward of step 22181 is: 1.0620854923438734\n",
            "reward of step 22182 is: 1.0654086004409211\n",
            "reward of step 22183 is: 1.1650456039887953\n",
            "reward of step 22184 is: 1.0683771110507307\n",
            "reward of step 22185 is: 1.1157748733473145\n",
            "reward of step 22186 is: 1.0441361435789043\n",
            "reward of step 22187 is: 1.111254193809736\n",
            "reward of step 22188 is: 1.1403016625531641\n",
            "reward of step 22189 is: 1.037183931366139\n",
            "reward of step 22190 is: 1.087774465442516\n",
            "reward of step 22191 is: 1.0540426175144202\n",
            "reward of step 22192 is: 1.1046348842287994\n",
            "reward of step 22193 is: 1.0740495450611776\n",
            "reward of step 22194 is: 1.0664502668384226\n",
            "reward of step 22195 is: 1.0787925068729116\n",
            "reward of step 22196 is: 1.0696959721324526\n",
            "reward of step 22197 is: 1.030153838369201\n",
            "reward of step 22198 is: 1.0928301235879547\n",
            "reward of step 22199 is: 1.0715960567329001\n",
            "reward of step 22200 is: 1.0198762766531217\n",
            "reward of step 22201 is: 1.0543902973830053\n",
            "reward of step 22202 is: 1.0839435077758757\n",
            "reward of step 22203 is: 1.0985438051637533\n",
            "reward of step 22204 is: 1.1313918283057705\n",
            "reward of step 22205 is: 1.0529478732911877\n",
            "reward of step 22206 is: 1.0684271448816491\n",
            "reward of step 22207 is: 1.0847462451989645\n",
            "reward of step 22208 is: 1.0892458370112479\n",
            "reward of step 22209 is: 1.0801073069535732\n",
            "reward of step 22210 is: 1.0552079473575728\n",
            "reward of step 22211 is: 1.1352100229660909\n",
            "reward of step 22212 is: 1.1193894491937109\n",
            "reward of step 22213 is: 1.067525907245079\n",
            "reward of step 22214 is: 1.1169779918815232\n",
            "reward of step 22215 is: 1.1033190569685642\n",
            "reward of step 22216 is: 1.0636653889593135\n",
            "reward of step 22217 is: 1.0509028108209086\n",
            "reward of step 22218 is: 1.0820900259906723\n",
            "reward of step 22219 is: 1.0760538103019037\n",
            "reward of step 22220 is: 1.1277879175805157\n",
            "reward of step 22221 is: 1.0450502640314028\n",
            "reward of step 22222 is: 1.0603427031374921\n",
            "reward of step 22223 is: 1.0514046403205466\n",
            "reward of step 22224 is: 1.1137686181792579\n",
            "reward of step 22225 is: 1.0407814530021444\n",
            "reward of step 22226 is: 1.0109985586928234\n",
            "reward of step 22227 is: 1.1005646018978337\n",
            "reward of step 22228 is: 1.1158734949599611\n",
            "reward of step 22229 is: 1.108380545925809\n",
            "reward of step 22230 is: 1.0328948692659154\n",
            "reward of step 22231 is: 1.0838557711898344\n",
            "reward of step 22232 is: 1.1189527866784603\n",
            "reward of step 22233 is: 1.0616040930407544\n",
            "reward of step 22234 is: 1.0538785400435389\n",
            "reward of step 22235 is: 1.060984581166846\n",
            "reward of step 22236 is: 1.0303418724047395\n",
            "reward of step 22237 is: 1.0705481591739106\n",
            "reward of step 22238 is: 1.0050183867444558\n",
            "reward of step 22239 is: 1.0200157132868173\n",
            "reward of step 22240 is: 1.0737332293130644\n",
            "reward of step 22241 is: 1.012686614745569\n",
            "reward of step 22242 is: 1.0624275531081881\n",
            "reward of step 22243 is: 1.122369505608459\n",
            "reward of step 22244 is: 1.0838767684204278\n",
            "reward of step 22245 is: 1.1247415624769448\n",
            "reward of step 22246 is: 1.1138501821107951\n",
            "reward of step 22247 is: 1.0187592453937806\n",
            "reward of step 22248 is: 1.0024816003877906\n",
            "reward of step 22249 is: 1.0657063769891981\n",
            "reward of step 22250 is: 1.0021475957251755\n",
            "reward of step 22251 is: 1.08226675239932\n",
            "reward of step 22252 is: 1.0890948813163153\n",
            "reward of step 22253 is: 1.1116830689585988\n",
            "reward of step 22254 is: 1.0622486510279114\n",
            "reward of step 22255 is: 1.0976310253565909\n",
            "reward of step 22256 is: 1.104652815723785\n",
            "reward of step 22257 is: 1.0843086133394273\n",
            "reward of step 22258 is: 1.070828886441542\n",
            "reward of step 22259 is: 1.07457620028853\n",
            "reward of step 22260 is: 1.0050288664213007\n",
            "reward of step 22261 is: 1.1186624764925894\n",
            "reward of step 22262 is: 1.0729871810857232\n",
            "reward of step 22263 is: 1.072042051931132\n",
            "reward of step 22264 is: 1.0866019828951106\n",
            "reward of step 22265 is: 1.100294871216215\n",
            "reward of step 22266 is: 1.1268141710210782\n",
            "reward of step 22267 is: 1.091069803514525\n",
            "reward of step 22268 is: 1.0661330222352363\n",
            "reward of step 22269 is: 1.0122305466144592\n",
            "reward of step 22270 is: 1.1350422925074173\n",
            "reward of step 22271 is: 1.0723568142183992\n",
            "reward of step 22272 is: 1.13440027561442\n",
            "reward of step 22273 is: 1.097261577160517\n",
            "reward of step 22274 is: 1.096382109733081\n",
            "reward of step 22275 is: 1.1093342911641528\n",
            "reward of step 22276 is: 1.00096919287923\n",
            "reward of step 22277 is: 1.1288691082513425\n",
            "reward of step 22278 is: 1.0893676284445768\n",
            "reward of step 22279 is: 1.0686745897851908\n",
            "reward of step 22280 is: 1.0915901732844442\n",
            "reward of step 22281 is: 1.145757826566395\n",
            "reward of step 22282 is: 1.121827786768569\n",
            "reward of step 22283 is: 1.0442862314019974\n",
            "reward of step 22284 is: 1.165974984993234\n",
            "reward of step 22285 is: 1.1418148783449193\n",
            "reward of step 22286 is: 1.070222712715913\n",
            "reward of step 22287 is: 1.0114210828599917\n",
            "reward of step 22288 is: 1.026703912218748\n",
            "reward of step 22289 is: 1.0329797654382726\n",
            "reward of step 22290 is: 1.0688228203471692\n",
            "reward of step 22291 is: 1.145801020708939\n",
            "reward of step 22292 is: 1.1160489504387425\n",
            "reward of step 22293 is: 1.1160446409655247\n",
            "reward of step 22294 is: 1.0475255504665126\n",
            "reward of step 22295 is: 1.1043653921197527\n",
            "reward of step 22296 is: 1.171721568915912\n",
            "reward of step 22297 is: 1.1193835592070154\n",
            "reward of step 22298 is: 1.1024165401885846\n",
            "reward of step 22299 is: 1.0630016161797629\n",
            "reward of step 22300 is: 1.1082735438150904\n",
            "reward of step 22301 is: 1.052808055223529\n",
            "reward of step 22302 is: 1.0710316822881427\n",
            "reward of step 22303 is: 1.1308592856878381\n",
            "reward of step 22304 is: 1.021191723893427\n",
            "reward of step 22305 is: 1.0322809882773067\n",
            "reward of step 22306 is: 1.039332264504349\n",
            "reward of step 22307 is: 1.1233011414458915\n",
            "reward of step 22308 is: 1.0627124222745705\n",
            "reward of step 22309 is: 1.1364295200607744\n",
            "reward of step 22310 is: 1.0373869318166196\n",
            "reward of step 22311 is: 1.0308067948026118\n",
            "reward of step 22312 is: 1.064518051490976\n",
            "reward of step 22313 is: 1.064247882753056\n",
            "reward of step 22314 is: 1.0963391332673953\n",
            "reward of step 22315 is: 1.0453105841616206\n",
            "reward of step 22316 is: 1.055841171511313\n",
            "reward of step 22317 is: 1.1287311815193344\n",
            "reward of step 22318 is: 1.0891167771460653\n",
            "reward of step 22319 is: 1.0851704955894315\n",
            "reward of step 22320 is: 1.0547920832576847\n",
            "reward of step 22321 is: 1.0819440707624803\n",
            "reward of step 22322 is: 1.1048406069393906\n",
            "reward of step 22323 is: 1.0697650834694152\n",
            "reward of step 22324 is: 1.0462069476201419\n",
            "reward of step 22325 is: 1.0947298617458128\n",
            "reward of step 22326 is: 1.0201049906484378\n",
            "reward of step 22327 is: 1.0569795754245086\n",
            "reward of step 22328 is: 1.1424531958043302\n",
            "reward of step 22329 is: 1.078818908182651\n",
            "reward of step 22330 is: 1.0646080849486537\n",
            "reward of step 22331 is: 1.11093959550074\n",
            "reward of step 22332 is: 1.031715510745469\n",
            "reward of step 22333 is: 1.0600916923919175\n",
            "reward of step 22334 is: 1.1017530011678416\n",
            "reward of step 22335 is: 1.146715659893193\n",
            "reward of step 22336 is: 1.0773627664671537\n",
            "reward of step 22337 is: 1.1307383103598547\n",
            "reward of step 22338 is: 1.0533888488937884\n",
            "reward of step 22339 is: 1.049029807621116\n",
            "reward of step 22340 is: 1.084055808305837\n",
            "reward of step 22341 is: 1.0727625646175998\n",
            "reward of step 22342 is: 1.1581773807073827\n",
            "reward of step 22343 is: 1.0587497507246155\n",
            "reward of step 22344 is: 1.0896025919205181\n",
            "reward of step 22345 is: 1.1544028939256754\n",
            "reward of step 22346 is: 1.0918072800357814\n",
            "reward of step 22347 is: 1.0934445358257987\n",
            "reward of step 22348 is: 1.0683253780774886\n",
            "reward of step 22349 is: 1.0769420847935454\n",
            "reward of step 22350 is: 1.0928586937698137\n",
            "reward of step 22351 is: 1.1314828010187874\n",
            "reward of step 22352 is: 0.9869020235252114\n",
            "reward of step 22353 is: 1.1218597933443\n",
            "reward of step 22354 is: 1.03878895866354\n",
            "reward of step 22355 is: 1.0475019775870111\n",
            "reward of step 22356 is: 1.1202052841720982\n",
            "reward of step 22357 is: 1.0484560652715627\n",
            "reward of step 22358 is: 1.1431291021425114\n",
            "reward of step 22359 is: 1.0752687194230448\n",
            "reward of step 22360 is: 1.0808740782254125\n",
            "reward of step 22361 is: 1.1640577512699086\n",
            "reward of step 22362 is: 1.1341492490583511\n",
            "reward of step 22363 is: 1.064225280758711\n",
            "reward of step 22364 is: 1.1076740833233307\n",
            "reward of step 22365 is: 1.1030751808883945\n",
            "reward of step 22366 is: 1.054529295821556\n",
            "reward of step 22367 is: 0.9934947781960807\n",
            "reward of step 22368 is: 1.1376685186683413\n",
            "reward of step 22369 is: 1.1033478514776123\n",
            "reward of step 22370 is: 1.1061772810672332\n",
            "reward of step 22371 is: 1.0919690852637218\n",
            "reward of step 22372 is: 1.03569892603843\n",
            "reward of step 22373 is: 1.1120006572746202\n",
            "reward of step 22374 is: 1.0904045877425812\n",
            "reward of step 22375 is: 1.0427026090330342\n",
            "reward of step 22376 is: 1.0976538028807852\n",
            "reward of step 22377 is: 1.078640533580795\n",
            "reward of step 22378 is: 1.1044136046538529\n",
            "reward of step 22379 is: 1.033752186363933\n",
            "reward of step 22380 is: 1.132154077632923\n",
            "reward of step 22381 is: 1.0392027921483118\n",
            "reward of step 22382 is: 1.089293122489607\n",
            "reward of step 22383 is: 1.15242203885952\n",
            "reward of step 22384 is: 1.1629345463337126\n",
            "reward of step 22385 is: 1.0734664685502477\n",
            "reward of step 22386 is: 1.1406383691540454\n",
            "reward of step 22387 is: 1.0623717001662143\n",
            "reward of step 22388 is: 1.0873327068803944\n",
            "reward of step 22389 is: 1.0832490127814072\n",
            "reward of step 22390 is: 1.0879956590110065\n",
            "reward of step 22391 is: 1.0970308693936808\n",
            "reward of step 22392 is: 1.1474617626082773\n",
            "reward of step 22393 is: 1.1254576759529067\n",
            "reward of step 22394 is: 1.0977253933300364\n",
            "reward of step 22395 is: 1.0449075319483963\n",
            "reward of step 22396 is: 1.0930401278415594\n",
            "reward of step 22397 is: 1.028656581922872\n",
            "reward of step 22398 is: 1.0666623587973159\n",
            "reward of step 22399 is: 1.0754954152026777\n",
            "reward of step 22400 is: 1.1096553125871529\n",
            "reward of step 22401 is: 1.0062371505624106\n",
            "reward of step 22402 is: 1.0942165823618275\n",
            "reward of step 22403 is: 1.1496607471337006\n",
            "reward of step 22404 is: 1.033412741251731\n",
            "reward of step 22405 is: 1.129786365829311\n",
            "reward of step 22406 is: 1.1335899751790266\n",
            "reward of step 22407 is: 1.1127928925737003\n",
            "reward of step 22408 is: 1.1071529098322839\n",
            "reward of step 22409 is: 1.1514417210318275\n",
            "reward of step 22410 is: 1.0872737936459786\n",
            "reward of step 22411 is: 1.0679057547687338\n",
            "reward of step 22412 is: 1.070399544928909\n",
            "reward of step 22413 is: 1.0386697460597978\n",
            "reward of step 22414 is: 1.0748320456023448\n",
            "reward of step 22415 is: 1.053728572333043\n",
            "reward of step 22416 is: 1.0666671709641697\n",
            "reward of step 22417 is: 1.0092498930809908\n",
            "reward of step 22418 is: 1.0027646104959604\n",
            "reward of step 22419 is: 1.0877754357294283\n",
            "reward of step 22420 is: 1.1486147593590461\n",
            "reward of step 22421 is: 1.035269763949466\n",
            "reward of step 22422 is: 1.0367624158627868\n",
            "reward of step 22423 is: 1.0625576325087158\n",
            "reward of step 22424 is: 1.0923216459751477\n",
            "reward of step 22425 is: 1.0124900204803706\n",
            "reward of step 22426 is: 1.088456832110479\n",
            "reward of step 22427 is: 1.1008834553352806\n",
            "reward of step 22428 is: 1.0713952754424776\n",
            "reward of step 22429 is: 1.0924896659414292\n",
            "reward of step 22430 is: 1.0528188768281397\n",
            "reward of step 22431 is: 1.090837582980573\n",
            "reward of step 22432 is: 1.0561251110185919\n",
            "reward of step 22433 is: 1.0716468654351665\n",
            "reward of step 22434 is: 0.9900440812188211\n",
            "reward of step 22435 is: 1.0497062207397077\n",
            "reward of step 22436 is: 1.0825314564257025\n",
            "reward of step 22437 is: 1.0653777849607622\n",
            "reward of step 22438 is: 1.1017990947456728\n",
            "reward of step 22439 is: 1.0649054329299412\n",
            "reward of step 22440 is: 1.0830578796514296\n",
            "reward of step 22441 is: 1.128891775475989\n",
            "reward of step 22442 is: 1.0139577991687525\n",
            "reward of step 22443 is: 1.057585000728332\n",
            "reward of step 22444 is: 1.0142524780674769\n",
            "reward of step 22445 is: 1.0527157331619126\n",
            "reward of step 22446 is: 1.0543719977717585\n",
            "reward of step 22447 is: 1.0993047250007888\n",
            "reward of step 22448 is: 1.0946134333332926\n",
            "reward of step 22449 is: 1.0910311372991361\n",
            "reward of step 22450 is: 1.0400634921699858\n",
            "reward of step 22451 is: 1.0620424126288022\n",
            "reward of step 22452 is: 1.0368345075591443\n",
            "reward of step 22453 is: 1.0448064117771816\n",
            "reward of step 22454 is: 1.0650725514656814\n",
            "reward of step 22455 is: 1.1154269630281748\n",
            "reward of step 22456 is: 1.1233799217193452\n",
            "reward of step 22457 is: 1.0962863354981658\n",
            "reward of step 22458 is: 1.069240700460515\n",
            "reward of step 22459 is: 1.0640791509263827\n",
            "reward of step 22460 is: 1.0474386151676414\n",
            "reward of step 22461 is: 1.0505284613961987\n",
            "reward of step 22462 is: 1.0542621511937416\n",
            "reward of step 22463 is: 1.062309277119054\n",
            "reward of step 22464 is: 1.0662768973870946\n",
            "reward of step 22465 is: 1.0400335829189056\n",
            "reward of step 22466 is: 1.037521554378673\n",
            "reward of step 22467 is: 1.1225326796856638\n",
            "reward of step 22468 is: 1.094119229020824\n",
            "reward of step 22469 is: 1.0115243002816303\n",
            "reward of step 22470 is: 1.0696190616532022\n",
            "reward of step 22471 is: 1.114713270286827\n",
            "reward of step 22472 is: 1.0610861184345333\n",
            "reward of step 22473 is: 1.0318723811476813\n",
            "reward of step 22474 is: 1.0700478114060747\n",
            "reward of step 22475 is: 1.0427444800808219\n",
            "reward of step 22476 is: 1.0581438346969603\n",
            "reward of step 22477 is: 1.0982110656805375\n",
            "reward of step 22478 is: 1.1153570435412805\n",
            "reward of step 22479 is: 1.0858728551931778\n",
            "reward of step 22480 is: 1.137905930181848\n",
            "reward of step 22481 is: 1.0630950263571406\n",
            "reward of step 22482 is: 1.0747475864354348\n",
            "reward of step 22483 is: 0.9848493457206585\n",
            "reward of step 22484 is: 1.1192190947053513\n",
            "reward of step 22485 is: 1.1275252304723145\n",
            "reward of step 22486 is: 1.000042941052174\n",
            "reward of step 22487 is: 1.1387091665915403\n",
            "reward of step 22488 is: 1.1036786874997198\n",
            "reward of step 22489 is: 1.0213750426281292\n",
            "reward of step 22490 is: 1.0545014280085319\n",
            "reward of step 22491 is: 1.0198225750414076\n",
            "reward of step 22492 is: 1.09625371850074\n",
            "reward of step 22493 is: 1.0451414829983916\n",
            "reward of step 22494 is: 1.0802909442236488\n",
            "reward of step 22495 is: 1.085925629956592\n",
            "reward of step 22496 is: 1.0524514960471487\n",
            "reward of step 22497 is: 1.1275614665211875\n",
            "reward of step 22498 is: 1.085817921197155\n",
            "reward of step 22499 is: 1.1369457620507402\n",
            "reward of step 22500 is: 1.1005741199889356\n",
            "reward of step 22501 is: 1.0427645104970846\n",
            "reward of step 22502 is: 1.1431917434384826\n",
            "reward of step 22503 is: 1.1503712773182724\n",
            "reward of step 22504 is: 1.1181839932070736\n",
            "reward of step 22505 is: 1.0793056693852865\n",
            "reward of step 22506 is: 1.0552768742617695\n",
            "reward of step 22507 is: 1.104353982234827\n",
            "reward of step 22508 is: 1.0607309085771006\n",
            "reward of step 22509 is: 1.0507284748621435\n",
            "reward of step 22510 is: 1.1385357685063044\n",
            "reward of step 22511 is: 1.0511821624113065\n",
            "reward of step 22512 is: 1.0659605929934792\n",
            "reward of step 22513 is: 1.0171438003422613\n",
            "reward of step 22514 is: 0.9946905527448041\n",
            "reward of step 22515 is: 1.0452260216615117\n",
            "reward of step 22516 is: 1.1244044395108796\n",
            "reward of step 22517 is: 1.1143397593356283\n",
            "reward of step 22518 is: 1.0873155009787983\n",
            "reward of step 22519 is: 1.0345761000277287\n",
            "reward of step 22520 is: 1.0437366217342459\n",
            "reward of step 22521 is: 1.090614983875908\n",
            "reward of step 22522 is: 1.0914830948403906\n",
            "reward of step 22523 is: 1.1311315223249137\n",
            "reward of step 22524 is: 1.0892107398198918\n",
            "reward of step 22525 is: 1.1066288048989157\n",
            "reward of step 22526 is: 1.0905763669285\n",
            "reward of step 22527 is: 1.0416574917053012\n",
            "reward of step 22528 is: 1.0439714762985912\n",
            "reward of step 22529 is: 1.0988098039496956\n",
            "reward of step 22530 is: 1.107517834972728\n",
            "reward of step 22531 is: 1.029526778095835\n",
            "reward of step 22532 is: 1.0914070848127744\n",
            "reward of step 22533 is: 1.0803092598081423\n",
            "reward of step 22534 is: 1.039102385618131\n",
            "reward of step 22535 is: 1.085229108398471\n",
            "reward of step 22536 is: 1.0640598242948403\n",
            "reward of step 22537 is: 1.0895739411643874\n",
            "reward of step 22538 is: 1.0791249353937609\n",
            "reward of step 22539 is: 1.0848300382756553\n",
            "reward of step 22540 is: 1.061324936077367\n",
            "reward of step 22541 is: 1.1192668345756314\n",
            "reward of step 22542 is: 1.0760174218168683\n",
            "reward of step 22543 is: 0.9747550251108874\n",
            "reward of step 22544 is: 1.1182109711988604\n",
            "reward of step 22545 is: 1.0160804265951944\n",
            "reward of step 22546 is: 1.0594488645243525\n",
            "reward of step 22547 is: 1.1348516818385121\n",
            "reward of step 22548 is: 1.1281994299724207\n",
            "reward of step 22549 is: 1.1051739172947408\n",
            "reward of step 22550 is: 1.1093279122249238\n",
            "reward of step 22551 is: 1.073927447586918\n",
            "reward of step 22552 is: 1.027160304111092\n",
            "reward of step 22553 is: 1.1187843767056016\n",
            "reward of step 22554 is: 1.1052470364574145\n",
            "reward of step 22555 is: 1.097440194052183\n",
            "reward of step 22556 is: 1.1272115018711955\n",
            "reward of step 22557 is: 1.0552768526510068\n",
            "reward of step 22558 is: 1.1072485215018002\n",
            "reward of step 22559 is: 1.0690496432078729\n",
            "reward of step 22560 is: 1.07811709538668\n",
            "reward of step 22561 is: 1.101694685686071\n",
            "reward of step 22562 is: 1.1429027741968016\n",
            "reward of step 22563 is: 1.072831758450839\n",
            "reward of step 22564 is: 1.0895490260251761\n",
            "reward of step 22565 is: 1.0493681348754738\n",
            "reward of step 22566 is: 1.0815126565930402\n",
            "reward of step 22567 is: 1.0532455950836175\n",
            "reward of step 22568 is: 1.0716998238306443\n",
            "reward of step 22569 is: 1.0945431031972976\n",
            "reward of step 22570 is: 1.1495503581812958\n",
            "reward of step 22571 is: 1.0752893810616624\n",
            "reward of step 22572 is: 1.0351685558330965\n",
            "reward of step 22573 is: 1.0137066183918715\n",
            "reward of step 22574 is: 1.049383475244243\n",
            "reward of step 22575 is: 1.069335355691211\n",
            "reward of step 22576 is: 1.1403408376724753\n",
            "reward of step 22577 is: 1.0519837018016587\n",
            "reward of step 22578 is: 1.0797263805466302\n",
            "reward of step 22579 is: 1.1165715193470869\n",
            "reward of step 22580 is: 1.0364309497650988\n",
            "reward of step 22581 is: 1.0314985397476968\n",
            "reward of step 22582 is: 1.0805137869166357\n",
            "reward of step 22583 is: 1.0526125325359925\n",
            "reward of step 22584 is: 1.1262028158973396\n",
            "reward of step 22585 is: 1.137113136453486\n",
            "reward of step 22586 is: 1.0482420897996687\n",
            "reward of step 22587 is: 1.07087526123124\n",
            "reward of step 22588 is: 1.0139271501227385\n",
            "reward of step 22589 is: 1.0582772804873992\n",
            "reward of step 22590 is: 1.112712577190268\n",
            "reward of step 22591 is: 1.0859532432752785\n",
            "reward of step 22592 is: 1.1163452327690817\n",
            "reward of step 22593 is: 1.0436388933046925\n",
            "reward of step 22594 is: 1.0650868689466857\n",
            "reward of step 22595 is: 1.0058523275184648\n",
            "reward of step 22596 is: 1.0116341732330176\n",
            "reward of step 22597 is: 1.0571898810689353\n",
            "reward of step 22598 is: 1.0800585606140483\n",
            "reward of step 22599 is: 1.0659113806801694\n",
            "reward of step 22600 is: 1.160076254246594\n",
            "reward of step 22601 is: 1.1222594663828862\n",
            "reward of step 22602 is: 1.0087109565969297\n",
            "reward of step 22603 is: 1.1036871694815105\n",
            "reward of step 22604 is: 1.0568323452898376\n",
            "reward of step 22605 is: 1.06336627484808\n",
            "reward of step 22606 is: 1.0671740823342948\n",
            "reward of step 22607 is: 1.0692489641118277\n",
            "reward of step 22608 is: 1.1229871179092497\n",
            "reward of step 22609 is: 1.1277630101321618\n",
            "reward of step 22610 is: 1.121875406569497\n",
            "reward of step 22611 is: 1.1276597384720417\n",
            "reward of step 22612 is: 1.0154515811341533\n",
            "reward of step 22613 is: 1.0312424676471432\n",
            "reward of step 22614 is: 1.1361736858547982\n",
            "reward of step 22615 is: 1.0999636342826127\n",
            "reward of step 22616 is: 1.102627576054085\n",
            "reward of step 22617 is: 1.0577778035713306\n",
            "reward of step 22618 is: 1.0947017773624517\n",
            "reward of step 22619 is: 1.03434441187248\n",
            "reward of step 22620 is: 1.0467701053120633\n",
            "reward of step 22621 is: 1.0470229701538227\n",
            "reward of step 22622 is: 1.1106086265281845\n",
            "reward of step 22623 is: 1.156533721397319\n",
            "reward of step 22624 is: 1.011131664675359\n",
            "reward of step 22625 is: 1.1494713702890222\n",
            "reward of step 22626 is: 1.1137419349812372\n",
            "reward of step 22627 is: 1.1131246924117535\n",
            "reward of step 22628 is: 1.0772309325700304\n",
            "reward of step 22629 is: 1.022104078312195\n",
            "reward of step 22630 is: 1.1000698570382166\n",
            "reward of step 22631 is: 1.0729379052186558\n",
            "reward of step 22632 is: 1.0882055742528665\n",
            "reward of step 22633 is: 1.025426407904741\n",
            "reward of step 22634 is: 1.1061917992044243\n",
            "reward of step 22635 is: 1.0862419879867349\n",
            "reward of step 22636 is: 1.0036649986088748\n",
            "reward of step 22637 is: 1.1026297317287388\n",
            "reward of step 22638 is: 1.084111871928454\n",
            "reward of step 22639 is: 1.0527991045181502\n",
            "reward of step 22640 is: 1.0683185196929714\n",
            "reward of step 22641 is: 1.0724218045175475\n",
            "reward of step 22642 is: 1.0854781546693244\n",
            "reward of step 22643 is: 1.1307591385866473\n",
            "reward of step 22644 is: 1.0100695623650977\n",
            "reward of step 22645 is: 1.14010732655569\n",
            "reward of step 22646 is: 1.0532481915191245\n",
            "reward of step 22647 is: 1.1011988092556377\n",
            "reward of step 22648 is: 1.1637166876750138\n",
            "reward of step 22649 is: 1.1241229019092014\n",
            "reward of step 22650 is: 1.1116495541618803\n",
            "reward of step 22651 is: 1.091766663580115\n",
            "reward of step 22652 is: 1.0943937519009048\n",
            "reward of step 22653 is: 1.060393537354543\n",
            "reward of step 22654 is: 1.124934876446243\n",
            "reward of step 22655 is: 1.0880484913046733\n",
            "reward of step 22656 is: 1.0381915833116078\n",
            "reward of step 22657 is: 1.069487635010777\n",
            "reward of step 22658 is: 1.1140771698444598\n",
            "reward of step 22659 is: 1.083613003689594\n",
            "reward of step 22660 is: 1.074011928042042\n",
            "reward of step 22661 is: 1.0922143333085974\n",
            "reward of step 22662 is: 1.1276368768004816\n",
            "reward of step 22663 is: 1.0244006526109501\n",
            "reward of step 22664 is: 1.0261914801228997\n",
            "reward of step 22665 is: 1.087330233862906\n",
            "reward of step 22666 is: 1.1188954954332875\n",
            "reward of step 22667 is: 1.135103616372302\n",
            "reward of step 22668 is: 1.062310211950287\n",
            "reward of step 22669 is: 1.0662217707045203\n",
            "reward of step 22670 is: 1.1121010498052253\n",
            "reward of step 22671 is: 1.142658894580715\n",
            "reward of step 22672 is: 1.135433896374908\n",
            "reward of step 22673 is: 1.1377691820751206\n",
            "reward of step 22674 is: 1.0134701682774134\n",
            "reward of step 22675 is: 1.0987336907068914\n",
            "reward of step 22676 is: 1.044835656134913\n",
            "reward of step 22677 is: 1.0801167732639447\n",
            "reward of step 22678 is: 1.0665092074788316\n",
            "reward of step 22679 is: 1.0810656388066437\n",
            "reward of step 22680 is: 1.0699634241318348\n",
            "reward of step 22681 is: 1.0724570944490084\n",
            "reward of step 22682 is: 1.093701338966182\n",
            "reward of step 22683 is: 1.0418396273711328\n",
            "reward of step 22684 is: 1.0479998488783886\n",
            "reward of step 22685 is: 1.075047935972436\n",
            "reward of step 22686 is: 1.1444820106649742\n",
            "reward of step 22687 is: 1.1326913789760633\n",
            "reward of step 22688 is: 1.0211466544352392\n",
            "reward of step 22689 is: 1.0314307061456516\n",
            "reward of step 22690 is: 1.109665148798773\n",
            "reward of step 22691 is: 1.096506766684333\n",
            "reward of step 22692 is: 1.10710454320213\n",
            "reward of step 22693 is: 1.125275057845828\n",
            "reward of step 22694 is: 1.068133754525285\n",
            "reward of step 22695 is: 1.0618062856108619\n",
            "reward of step 22696 is: 1.1046081963842944\n",
            "reward of step 22697 is: 1.0446163470186338\n",
            "reward of step 22698 is: 1.0448918816865422\n",
            "reward of step 22699 is: 1.131893480220118\n",
            "reward of step 22700 is: 1.0865291047407248\n",
            "reward of step 22701 is: 1.0910507356373702\n",
            "reward of step 22702 is: 1.0378724271333712\n",
            "reward of step 22703 is: 1.0901115990963548\n",
            "reward of step 22704 is: 1.09387734231725\n",
            "reward of step 22705 is: 1.1544508077220783\n",
            "reward of step 22706 is: 1.0769851152041556\n",
            "reward of step 22707 is: 1.1953927910921331\n",
            "reward of step 22708 is: 1.1054709477574263\n",
            "reward of step 22709 is: 1.067879365818284\n",
            "reward of step 22710 is: 1.086002350428133\n",
            "reward of step 22711 is: 1.1526298981893257\n",
            "reward of step 22712 is: 1.029347882232451\n",
            "reward of step 22713 is: 1.1250549913055923\n",
            "reward of step 22714 is: 1.0932333922970798\n",
            "reward of step 22715 is: 1.0756963272885205\n",
            "reward of step 22716 is: 1.0918700555935232\n",
            "reward of step 22717 is: 1.0186859290527848\n",
            "reward of step 22718 is: 1.069532073391436\n",
            "reward of step 22719 is: 1.1771786990744557\n",
            "reward of step 22720 is: 1.0168863639795693\n",
            "reward of step 22721 is: 1.0754004570426192\n",
            "reward of step 22722 is: 1.0583586685747348\n",
            "reward of step 22723 is: 0.9951869601552001\n",
            "reward of step 22724 is: 1.0980949365588728\n",
            "reward of step 22725 is: 1.0767843214334571\n",
            "reward of step 22726 is: 1.1142389420146808\n",
            "reward of step 22727 is: 1.0703069835663421\n",
            "reward of step 22728 is: 1.0813444047686502\n",
            "reward of step 22729 is: 1.0617663905626769\n",
            "reward of step 22730 is: 1.0945742937468173\n",
            "reward of step 22731 is: 1.049981858540098\n",
            "reward of step 22732 is: 1.0554410405956516\n",
            "reward of step 22733 is: 1.0990068372278965\n",
            "reward of step 22734 is: 1.0674942203344797\n",
            "reward of step 22735 is: 1.0368299302923742\n",
            "reward of step 22736 is: 1.1468453156713212\n",
            "reward of step 22737 is: 1.0739369482094148\n",
            "reward of step 22738 is: 1.108246731280254\n",
            "reward of step 22739 is: 1.0812089384094055\n",
            "reward of step 22740 is: 1.098970579264035\n",
            "reward of step 22741 is: 1.017908522153824\n",
            "reward of step 22742 is: 1.0803926235452164\n",
            "reward of step 22743 is: 1.116087377044638\n",
            "reward of step 22744 is: 1.0937910594149542\n",
            "reward of step 22745 is: 1.0839565473773352\n",
            "reward of step 22746 is: 1.0584307939511177\n",
            "reward of step 22747 is: 1.0964924160177147\n",
            "reward of step 22748 is: 1.0394401030009242\n",
            "reward of step 22749 is: 1.0524889082207962\n",
            "reward of step 22750 is: 1.1140233722615664\n",
            "reward of step 22751 is: 1.1337315613786125\n",
            "reward of step 22752 is: 1.117890188862427\n",
            "reward of step 22753 is: 1.04091828024778\n",
            "reward of step 22754 is: 1.094747749122957\n",
            "reward of step 22755 is: 1.0619601845799496\n",
            "reward of step 22756 is: 1.140675165288328\n",
            "reward of step 22757 is: 1.086500124123762\n",
            "reward of step 22758 is: 1.058675633309833\n",
            "reward of step 22759 is: 1.0902525849906386\n",
            "reward of step 22760 is: 1.0661281623302206\n",
            "reward of step 22761 is: 1.147011786685373\n",
            "reward of step 22762 is: 1.0430253372868834\n",
            "reward of step 22763 is: 1.017322216013004\n",
            "reward of step 22764 is: 1.0372649754730165\n",
            "reward of step 22765 is: 1.0161444593548787\n",
            "reward of step 22766 is: 1.0853133097546344\n",
            "reward of step 22767 is: 1.0695064735321191\n",
            "reward of step 22768 is: 1.0789193723021633\n",
            "reward of step 22769 is: 1.0706770183601189\n",
            "reward of step 22770 is: 1.1144942583209296\n",
            "reward of step 22771 is: 0.9654330938339384\n",
            "reward of step 22772 is: 1.0397126278796718\n",
            "reward of step 22773 is: 1.1128271730371195\n",
            "reward of step 22774 is: 1.0907047678008528\n",
            "reward of step 22775 is: 0.9995319185058705\n",
            "reward of step 22776 is: 1.0319462878677714\n",
            "reward of step 22777 is: 1.1146901196926202\n",
            "reward of step 22778 is: 1.1046813747890063\n",
            "reward of step 22779 is: 1.0468092651224878\n",
            "reward of step 22780 is: 1.1094062446467468\n",
            "reward of step 22781 is: 1.0461478170834493\n",
            "reward of step 22782 is: 1.0964618711773104\n",
            "reward of step 22783 is: 1.05977781585233\n",
            "reward of step 22784 is: 1.1990853936098183\n",
            "reward of step 22785 is: 1.0824372790037988\n",
            "reward of step 22786 is: 1.0142872380416361\n",
            "reward of step 22787 is: 1.06586254595473\n",
            "reward of step 22788 is: 1.1261372587564107\n",
            "reward of step 22789 is: 1.1208248305069928\n",
            "reward of step 22790 is: 1.0796235980644981\n",
            "reward of step 22791 is: 1.0955289178950625\n",
            "reward of step 22792 is: 1.0678946950412103\n",
            "reward of step 22793 is: 0.9988482731850409\n",
            "reward of step 22794 is: 1.0954156841434175\n",
            "reward of step 22795 is: 1.0643230697284567\n",
            "reward of step 22796 is: 1.1071228632409236\n",
            "reward of step 22797 is: 1.077555329862348\n",
            "reward of step 22798 is: 1.1041241604062724\n",
            "reward of step 22799 is: 1.0990893795695602\n",
            "reward of step 22800 is: 1.0695818938141881\n",
            "reward of step 22801 is: 1.1082400410402122\n",
            "reward of step 22802 is: 1.1256335951254235\n",
            "reward of step 22803 is: 1.0778149726719073\n",
            "reward of step 22804 is: 0.9905359079104564\n",
            "reward of step 22805 is: 1.0263139700555772\n",
            "reward of step 22806 is: 0.9891131241136002\n",
            "reward of step 22807 is: 1.0438917963241794\n",
            "reward of step 22808 is: 1.134026651972417\n",
            "reward of step 22809 is: 1.0231341269524141\n",
            "reward of step 22810 is: 0.9869229033505467\n",
            "reward of step 22811 is: 1.0633421518070145\n",
            "reward of step 22812 is: 1.1035697644714566\n",
            "reward of step 22813 is: 1.0572709427752969\n",
            "reward of step 22814 is: 1.1527638585998163\n",
            "reward of step 22815 is: 1.076348419925327\n",
            "reward of step 22816 is: 1.1415666891992022\n",
            "reward of step 22817 is: 1.061603619045529\n",
            "reward of step 22818 is: 1.0521833266191822\n",
            "reward of step 22819 is: 1.0717420032076892\n",
            "reward of step 22820 is: 1.1096689125498067\n",
            "reward of step 22821 is: 1.1428931488680607\n",
            "reward of step 22822 is: 1.080996739931013\n",
            "reward of step 22823 is: 1.1025323391284625\n",
            "reward of step 22824 is: 1.0813468586993862\n",
            "reward of step 22825 is: 1.0759746498139129\n",
            "reward of step 22826 is: 1.114013272271123\n",
            "reward of step 22827 is: 1.0576319090318127\n",
            "reward of step 22828 is: 1.0680200311036954\n",
            "reward of step 22829 is: 1.0971931714916623\n",
            "reward of step 22830 is: 1.062847160369129\n",
            "reward of step 22831 is: 1.0485815249677737\n",
            "reward of step 22832 is: 1.142604243279785\n",
            "reward of step 22833 is: 1.0943678687948086\n",
            "reward of step 22834 is: 1.0352235897279494\n",
            "reward of step 22835 is: 1.078377019050936\n",
            "reward of step 22836 is: 1.0882885346491404\n",
            "reward of step 22837 is: 1.053069284183367\n",
            "reward of step 22838 is: 1.027859033799201\n",
            "reward of step 22839 is: 1.064003677536821\n",
            "reward of step 22840 is: 1.125632052548496\n",
            "reward of step 22841 is: 1.0690108773038776\n",
            "reward of step 22842 is: 1.1246733203854737\n",
            "reward of step 22843 is: 1.0997466973894179\n",
            "reward of step 22844 is: 1.117649328973763\n",
            "reward of step 22845 is: 1.0924176026685777\n",
            "reward of step 22846 is: 1.0389246816989375\n",
            "reward of step 22847 is: 1.1327182934605577\n",
            "reward of step 22848 is: 1.0816655781858298\n",
            "reward of step 22849 is: 1.0744751310700504\n",
            "reward of step 22850 is: 1.0926989694005798\n",
            "reward of step 22851 is: 1.038241834341894\n",
            "reward of step 22852 is: 1.1161753327062733\n",
            "reward of step 22853 is: 1.0761284960975792\n",
            "reward of step 22854 is: 1.1139298277470684\n",
            "reward of step 22855 is: 1.0759194737506654\n",
            "reward of step 22856 is: 1.0259098675932972\n",
            "reward of step 22857 is: 1.027135634757896\n",
            "reward of step 22858 is: 1.0968895197018784\n",
            "reward of step 22859 is: 1.076056256226804\n",
            "reward of step 22860 is: 1.0543027701653414\n",
            "reward of step 22861 is: 1.0699376614847875\n",
            "reward of step 22862 is: 1.0795295256628197\n",
            "reward of step 22863 is: 1.0439140218575091\n",
            "reward of step 22864 is: 1.1082332610527381\n",
            "reward of step 22865 is: 1.0857930759343244\n",
            "reward of step 22866 is: 1.0917831631986343\n",
            "reward of step 22867 is: 1.0445227819377934\n",
            "reward of step 22868 is: 1.077140077656305\n",
            "reward of step 22869 is: 1.1030814291208562\n",
            "reward of step 22870 is: 1.1737872520156265\n",
            "reward of step 22871 is: 1.1304178227534476\n",
            "reward of step 22872 is: 1.0615517159421728\n",
            "reward of step 22873 is: 1.06929843113366\n",
            "reward of step 22874 is: 1.0959254868526627\n",
            "reward of step 22875 is: 1.093108131024354\n",
            "reward of step 22876 is: 1.041160679380669\n",
            "reward of step 22877 is: 1.1126812694826362\n",
            "reward of step 22878 is: 1.1436305359403387\n",
            "reward of step 22879 is: 1.0262183025813447\n",
            "reward of step 22880 is: 1.0938037689007338\n",
            "reward of step 22881 is: 1.0156348582319557\n",
            "reward of step 22882 is: 1.0606233411952406\n",
            "reward of step 22883 is: 1.0510582061058593\n",
            "reward of step 22884 is: 1.110310901298798\n",
            "reward of step 22885 is: 1.1000924112550114\n",
            "reward of step 22886 is: 1.1032158228150295\n",
            "reward of step 22887 is: 1.0766690190900237\n",
            "reward of step 22888 is: 1.102005806066074\n",
            "reward of step 22889 is: 1.0816834013111551\n",
            "reward of step 22890 is: 1.0872984972434652\n",
            "reward of step 22891 is: 1.099458190895675\n",
            "reward of step 22892 is: 1.0653353186517642\n",
            "reward of step 22893 is: 1.087391262551547\n",
            "reward of step 22894 is: 1.111506799914676\n",
            "reward of step 22895 is: 1.1420800926375252\n",
            "reward of step 22896 is: 1.0407120968595236\n",
            "reward of step 22897 is: 1.0711469207020834\n",
            "reward of step 22898 is: 1.037045295556148\n",
            "reward of step 22899 is: 1.1159482272103642\n",
            "reward of step 22900 is: 1.0528028726894079\n",
            "reward of step 22901 is: 1.0537792551792116\n",
            "reward of step 22902 is: 1.0680496940146278\n",
            "reward of step 22903 is: 1.0865959856265222\n",
            "reward of step 22904 is: 1.0795979070039345\n",
            "reward of step 22905 is: 1.034986942039982\n",
            "reward of step 22906 is: 1.1023961162704536\n",
            "reward of step 22907 is: 1.053121780953709\n",
            "reward of step 22908 is: 1.0450291198594739\n",
            "reward of step 22909 is: 1.097154776726183\n",
            "reward of step 22910 is: 1.0407386282262525\n",
            "reward of step 22911 is: 1.0770148933323611\n",
            "reward of step 22912 is: 1.0016337891380693\n",
            "reward of step 22913 is: 1.139512909935807\n",
            "reward of step 22914 is: 1.1071849436808225\n",
            "reward of step 22915 is: 1.1130165734312905\n",
            "reward of step 22916 is: 1.071609767310871\n",
            "reward of step 22917 is: 1.1416434558829722\n",
            "reward of step 22918 is: 1.064242043480329\n",
            "reward of step 22919 is: 1.1114095224202334\n",
            "reward of step 22920 is: 1.099274631588669\n",
            "reward of step 22921 is: 1.0593357006624475\n",
            "reward of step 22922 is: 1.0854171647869468\n",
            "reward of step 22923 is: 1.1152672981214646\n",
            "reward of step 22924 is: 1.0775905611394798\n",
            "reward of step 22925 is: 1.127565365248874\n",
            "reward of step 22926 is: 1.0124801388098739\n",
            "reward of step 22927 is: 1.1267458970302267\n",
            "reward of step 22928 is: 1.0837777905745871\n",
            "reward of step 22929 is: 1.1244615928025998\n",
            "reward of step 22930 is: 1.0864056604213128\n",
            "reward of step 22931 is: 1.0912949724475003\n",
            "reward of step 22932 is: 1.0253146272190716\n",
            "reward of step 22933 is: 1.1098520071855011\n",
            "reward of step 22934 is: 1.0017996418250559\n",
            "reward of step 22935 is: 1.0449642678651045\n",
            "reward of step 22936 is: 1.0371847487656054\n",
            "reward of step 22937 is: 0.9941044228742764\n",
            "reward of step 22938 is: 1.1076757363733467\n",
            "reward of step 22939 is: 1.145718896134127\n",
            "reward of step 22940 is: 1.074529720178554\n",
            "reward of step 22941 is: 1.0974520125447955\n",
            "reward of step 22942 is: 1.098591813094505\n",
            "reward of step 22943 is: 1.1584763091617778\n",
            "reward of step 22944 is: 1.1286282736852242\n",
            "reward of step 22945 is: 1.0951284315626328\n",
            "reward of step 22946 is: 1.0961960974872422\n",
            "reward of step 22947 is: 1.0530662583897135\n",
            "reward of step 22948 is: 1.0489182592490274\n",
            "reward of step 22949 is: 1.0722205299319219\n",
            "reward of step 22950 is: 1.1748382691582837\n",
            "reward of step 22951 is: 1.0292368322736012\n",
            "reward of step 22952 is: 1.095968270273481\n",
            "reward of step 22953 is: 1.1445319896195558\n",
            "reward of step 22954 is: 1.1206969708458194\n",
            "reward of step 22955 is: 1.0231327502904093\n",
            "reward of step 22956 is: 1.0318345028866687\n",
            "reward of step 22957 is: 1.0809899134499603\n",
            "reward of step 22958 is: 1.1360140932315486\n",
            "reward of step 22959 is: 1.0411052007630994\n",
            "reward of step 22960 is: 1.0734056106713048\n",
            "reward of step 22961 is: 1.1001615987043032\n",
            "reward of step 22962 is: 1.0929384603862617\n",
            "reward of step 22963 is: 1.0390100472006145\n",
            "reward of step 22964 is: 1.0733947007299043\n",
            "reward of step 22965 is: 1.0531901658944833\n",
            "reward of step 22966 is: 1.1050529208349165\n",
            "reward of step 22967 is: 1.0851038682481855\n",
            "reward of step 22968 is: 1.0767741572045404\n",
            "reward of step 22969 is: 1.0579437545299128\n",
            "reward of step 22970 is: 1.0804403801922908\n",
            "reward of step 22971 is: 1.0439830483834776\n",
            "reward of step 22972 is: 1.0498211582990122\n",
            "reward of step 22973 is: 1.0577126833887012\n",
            "reward of step 22974 is: 1.0693198161468092\n",
            "reward of step 22975 is: 1.0604738229932498\n",
            "reward of step 22976 is: 1.07757109189845\n",
            "reward of step 22977 is: 1.0652738673882771\n",
            "reward of step 22978 is: 1.0683863066984816\n",
            "reward of step 22979 is: 1.0965410726103186\n",
            "reward of step 22980 is: 1.1760410591179364\n",
            "reward of step 22981 is: 1.1392170317791526\n",
            "reward of step 22982 is: 1.0866146370420051\n",
            "reward of step 22983 is: 1.0934518763198564\n",
            "reward of step 22984 is: 1.0265935410358948\n",
            "reward of step 22985 is: 1.104422913384742\n",
            "reward of step 22986 is: 1.0867964908436734\n",
            "reward of step 22987 is: 1.1387523187595803\n",
            "reward of step 22988 is: 1.1083314241264062\n",
            "reward of step 22989 is: 1.0822783546681225\n",
            "reward of step 22990 is: 1.1110809819513077\n",
            "reward of step 22991 is: 1.100202554944249\n",
            "reward of step 22992 is: 1.1475206398617361\n",
            "reward of step 22993 is: 1.1173358121283774\n",
            "reward of step 22994 is: 1.1069761082299516\n",
            "reward of step 22995 is: 1.076836934610927\n",
            "reward of step 22996 is: 1.0797600035694916\n",
            "reward of step 22997 is: 1.067241711181216\n",
            "reward of step 22998 is: 1.0668688250203\n",
            "reward of step 22999 is: 1.1072799696779088\n",
            "reward of step 23000 is: 1.1366510991806362\n",
            "reward of step 23001 is: 1.0649683336805387\n",
            "reward of step 23002 is: 1.0581733478637432\n",
            "reward of step 23003 is: 1.1435006478614516\n",
            "reward of step 23004 is: 1.0990056851979593\n",
            "reward of step 23005 is: 1.0424236379062055\n",
            "reward of step 23006 is: 1.1382810927872424\n",
            "reward of step 23007 is: 1.1144678634697773\n",
            "reward of step 23008 is: 1.0686216468141356\n",
            "reward of step 23009 is: 1.105965996816161\n",
            "reward of step 23010 is: 1.1306246897185963\n",
            "reward of step 23011 is: 1.0352323056456534\n",
            "reward of step 23012 is: 1.050882714295465\n",
            "reward of step 23013 is: 1.0767215818775546\n",
            "reward of step 23014 is: 1.0560280924643348\n",
            "reward of step 23015 is: 1.118107045553161\n",
            "reward of step 23016 is: 1.0381916238664055\n",
            "reward of step 23017 is: 0.992499646273904\n",
            "reward of step 23018 is: 1.0545437011946284\n",
            "reward of step 23019 is: 1.131072571691415\n",
            "reward of step 23020 is: 1.125459874758359\n",
            "reward of step 23021 is: 1.1000269721090985\n",
            "reward of step 23022 is: 1.1231305285160418\n",
            "reward of step 23023 is: 1.1548913723864045\n",
            "reward of step 23024 is: 1.071809243681434\n",
            "reward of step 23025 is: 1.0888409215047319\n",
            "reward of step 23026 is: 1.0693694787576922\n",
            "reward of step 23027 is: 1.1098608371492764\n",
            "reward of step 23028 is: 1.1439349564894088\n",
            "reward of step 23029 is: 1.07980456751716\n",
            "reward of step 23030 is: 1.1225361615318878\n",
            "reward of step 23031 is: 1.107196578479401\n",
            "reward of step 23032 is: 1.0238136135401303\n",
            "reward of step 23033 is: 1.1361821988236342\n",
            "reward of step 23034 is: 1.0543833890379788\n",
            "reward of step 23035 is: 1.1306867619039123\n",
            "reward of step 23036 is: 1.110927462194657\n",
            "reward of step 23037 is: 1.1080232931696101\n",
            "reward of step 23038 is: 1.1328135744269072\n",
            "reward of step 23039 is: 1.0682444331679473\n",
            "reward of step 23040 is: 1.096837323101072\n",
            "reward of step 23041 is: 1.1579418448450496\n",
            "reward of step 23042 is: 1.069350457934569\n",
            "reward of step 23043 is: 1.1223077508961115\n",
            "reward of step 23044 is: 1.1123760686601947\n",
            "reward of step 23045 is: 1.156454289759242\n",
            "reward of step 23046 is: 1.055952222603782\n",
            "reward of step 23047 is: 1.117206718242603\n",
            "reward of step 23048 is: 1.0497040905519457\n",
            "reward of step 23049 is: 1.123997186007231\n",
            "reward of step 23050 is: 1.093121098173746\n",
            "reward of step 23051 is: 1.048178542466411\n",
            "reward of step 23052 is: 1.0890374695149037\n",
            "reward of step 23053 is: 1.0402279426462089\n",
            "reward of step 23054 is: 1.123386666960764\n",
            "reward of step 23055 is: 1.0485861658374622\n",
            "reward of step 23056 is: 1.11580206277225\n",
            "reward of step 23057 is: 1.0239601549036508\n",
            "reward of step 23058 is: 1.0179127880337098\n",
            "reward of step 23059 is: 1.12842688936187\n",
            "reward of step 23060 is: 1.1319822297135071\n",
            "reward of step 23061 is: 1.0331970444299101\n",
            "reward of step 23062 is: 1.0338979790114224\n",
            "reward of step 23063 is: 1.0381989461590035\n",
            "reward of step 23064 is: 1.0470035583407815\n",
            "reward of step 23065 is: 1.0528585902967071\n",
            "reward of step 23066 is: 1.0691963832113909\n",
            "reward of step 23067 is: 1.100947143277037\n",
            "reward of step 23068 is: 1.1203966742605993\n",
            "reward of step 23069 is: 1.0939586188704458\n",
            "reward of step 23070 is: 1.1330705750256191\n",
            "reward of step 23071 is: 1.1463843805037477\n",
            "reward of step 23072 is: 1.0792489047679226\n",
            "reward of step 23073 is: 1.1043819971398814\n",
            "reward of step 23074 is: 1.0082010152434098\n",
            "reward of step 23075 is: 1.0458809099383293\n",
            "reward of step 23076 is: 1.084070919639859\n",
            "reward of step 23077 is: 1.1566037586346662\n",
            "reward of step 23078 is: 1.0625182364979886\n",
            "reward of step 23079 is: 1.0854456638885082\n",
            "reward of step 23080 is: 1.0412132530146152\n",
            "reward of step 23081 is: 1.1283459103923703\n",
            "reward of step 23082 is: 1.0881446144752838\n",
            "reward of step 23083 is: 1.064965862721802\n",
            "reward of step 23084 is: 1.0564103018062574\n",
            "reward of step 23085 is: 1.0786816823676575\n",
            "reward of step 23086 is: 1.1235973322323636\n",
            "reward of step 23087 is: 1.1258471102460519\n",
            "reward of step 23088 is: 1.035558500053114\n",
            "reward of step 23089 is: 1.1050608481683506\n",
            "reward of step 23090 is: 1.08903700457932\n",
            "reward of step 23091 is: 1.0355959184407832\n",
            "reward of step 23092 is: 1.0660143877736563\n",
            "reward of step 23093 is: 1.1082796123375305\n",
            "reward of step 23094 is: 0.9936372832573144\n",
            "reward of step 23095 is: 1.0943592364335437\n",
            "reward of step 23096 is: 1.1535988205464072\n",
            "reward of step 23097 is: 1.127292818834361\n",
            "reward of step 23098 is: 1.0723549758429862\n",
            "reward of step 23099 is: 1.0802109853076\n",
            "reward of step 23100 is: 1.0804271603285578\n",
            "reward of step 23101 is: 1.0934404402857905\n",
            "reward of step 23102 is: 1.109029827908254\n",
            "reward of step 23103 is: 1.1147555763316301\n",
            "reward of step 23104 is: 1.079747818206425\n",
            "reward of step 23105 is: 1.11744388621218\n",
            "reward of step 23106 is: 1.1764502642683001\n",
            "reward of step 23107 is: 1.0438570065637727\n",
            "reward of step 23108 is: 1.041871424319824\n",
            "reward of step 23109 is: 1.0958094993286518\n",
            "reward of step 23110 is: 1.0907514893818187\n",
            "reward of step 23111 is: 1.0883292718536774\n",
            "reward of step 23112 is: 1.0476759587668942\n",
            "reward of step 23113 is: 1.1268089742659937\n",
            "reward of step 23114 is: 1.0258959332358553\n",
            "reward of step 23115 is: 1.048748038201707\n",
            "reward of step 23116 is: 1.0673590005509248\n",
            "reward of step 23117 is: 1.0988183953774437\n",
            "reward of step 23118 is: 1.05600746901691\n",
            "reward of step 23119 is: 1.0973655188466993\n",
            "reward of step 23120 is: 1.0854161748611921\n",
            "reward of step 23121 is: 1.1248196377219735\n",
            "reward of step 23122 is: 1.0287871344239967\n",
            "reward of step 23123 is: 1.1284671749072772\n",
            "reward of step 23124 is: 1.097422982364134\n",
            "reward of step 23125 is: 1.053665915326845\n",
            "reward of step 23126 is: 1.035162780358378\n",
            "reward of step 23127 is: 1.0345863390370673\n",
            "reward of step 23128 is: 1.1132533366127817\n",
            "reward of step 23129 is: 1.1471555982426684\n",
            "reward of step 23130 is: 1.105856340258967\n",
            "reward of step 23131 is: 1.0865699933189177\n",
            "reward of step 23132 is: 1.0538228032292465\n",
            "reward of step 23133 is: 1.1163914395277694\n",
            "reward of step 23134 is: 1.0764024619378083\n",
            "reward of step 23135 is: 1.0607879687733734\n",
            "reward of step 23136 is: 1.0352448445831972\n",
            "reward of step 23137 is: 1.1045724026346087\n",
            "reward of step 23138 is: 1.0980437332425192\n",
            "reward of step 23139 is: 1.115039516961905\n",
            "reward of step 23140 is: 1.0503339918850982\n",
            "reward of step 23141 is: 1.0810008189895073\n",
            "reward of step 23142 is: 1.0637010688083801\n",
            "reward of step 23143 is: 1.0941334800369562\n",
            "reward of step 23144 is: 1.0515405268978588\n",
            "reward of step 23145 is: 1.093094187985042\n",
            "reward of step 23146 is: 1.085075657271316\n",
            "reward of step 23147 is: 1.1652105180530277\n",
            "reward of step 23148 is: 1.105480625404447\n",
            "reward of step 23149 is: 1.0380722750771763\n",
            "reward of step 23150 is: 1.0392371609215658\n",
            "reward of step 23151 is: 1.08297694530086\n",
            "reward of step 23152 is: 1.0864415427674754\n",
            "reward of step 23153 is: 1.0409376347942216\n",
            "reward of step 23154 is: 1.0368580095038715\n",
            "reward of step 23155 is: 1.0347154188995276\n",
            "reward of step 23156 is: 1.0667086235566132\n",
            "reward of step 23157 is: 1.026011873079883\n",
            "reward of step 23158 is: 1.1501999739790143\n",
            "reward of step 23159 is: 1.1132120770014726\n",
            "reward of step 23160 is: 1.0612991772634963\n",
            "reward of step 23161 is: 1.0602078256392562\n",
            "reward of step 23162 is: 1.042088264958737\n",
            "reward of step 23163 is: 1.0449681059174751\n",
            "reward of step 23164 is: 1.1704563465500026\n",
            "reward of step 23165 is: 1.1053034652218967\n",
            "reward of step 23166 is: 1.1128339701661156\n",
            "reward of step 23167 is: 1.0835894454968429\n",
            "reward of step 23168 is: 1.0889994292517997\n",
            "reward of step 23169 is: 1.1233174327127644\n",
            "reward of step 23170 is: 1.1395579690687982\n",
            "reward of step 23171 is: 1.0572864084154914\n",
            "reward of step 23172 is: 1.134491843410812\n",
            "reward of step 23173 is: 1.0962977625131232\n",
            "reward of step 23174 is: 1.0974434295431803\n",
            "reward of step 23175 is: 0.9843835390313009\n",
            "reward of step 23176 is: 1.1092504711749676\n",
            "reward of step 23177 is: 1.1293455130870775\n",
            "reward of step 23178 is: 1.1002429888090837\n",
            "reward of step 23179 is: 1.064348545980507\n",
            "reward of step 23180 is: 1.0658719297657309\n",
            "reward of step 23181 is: 1.0921744608584305\n",
            "reward of step 23182 is: 1.002862354914049\n",
            "reward of step 23183 is: 1.124285759478202\n",
            "reward of step 23184 is: 1.0715416246787544\n",
            "reward of step 23185 is: 1.0192324592667088\n",
            "reward of step 23186 is: 1.0641518385709177\n",
            "reward of step 23187 is: 1.0731439162411833\n",
            "reward of step 23188 is: 1.1165373143946686\n",
            "reward of step 23189 is: 1.1147616788300718\n",
            "reward of step 23190 is: 1.138359627649403\n",
            "reward of step 23191 is: 1.0721598204641714\n",
            "reward of step 23192 is: 1.0450985645952198\n",
            "reward of step 23193 is: 1.0751242542037804\n",
            "reward of step 23194 is: 1.120536803562353\n",
            "reward of step 23195 is: 1.121705946083913\n",
            "reward of step 23196 is: 1.1153284020446463\n",
            "reward of step 23197 is: 1.0311825921416777\n",
            "reward of step 23198 is: 1.0902224821344082\n",
            "reward of step 23199 is: 1.1223847279561388\n",
            "reward of step 23200 is: 1.0494333627125705\n",
            "reward of step 23201 is: 1.0418892344049928\n",
            "reward of step 23202 is: 1.0409317873397113\n",
            "reward of step 23203 is: 1.0484707206814545\n",
            "reward of step 23204 is: 1.067742561213666\n",
            "reward of step 23205 is: 1.0394201607180635\n",
            "reward of step 23206 is: 1.0940181562278597\n",
            "reward of step 23207 is: 1.0429815125242283\n",
            "reward of step 23208 is: 1.1197831923625008\n",
            "reward of step 23209 is: 1.013408127906893\n",
            "reward of step 23210 is: 1.028876937132171\n",
            "reward of step 23211 is: 1.0610065393725832\n",
            "reward of step 23212 is: 1.1295369937588506\n",
            "reward of step 23213 is: 1.1051816876434837\n",
            "reward of step 23214 is: 1.095248761885681\n",
            "reward of step 23215 is: 1.1032882939464779\n",
            "reward of step 23216 is: 1.1044537889813535\n",
            "reward of step 23217 is: 1.05774889219787\n",
            "reward of step 23218 is: 1.0449934261828702\n",
            "reward of step 23219 is: 1.1454585698192596\n",
            "reward of step 23220 is: 1.1422300362489426\n",
            "reward of step 23221 is: 1.0727548934423177\n",
            "reward of step 23222 is: 1.0796025223732184\n",
            "reward of step 23223 is: 1.0492349014101876\n",
            "reward of step 23224 is: 1.0737984979201856\n",
            "reward of step 23225 is: 1.095718317525\n",
            "reward of step 23226 is: 1.0936632484265107\n",
            "reward of step 23227 is: 1.0583094401478865\n",
            "reward of step 23228 is: 1.0906262005146767\n",
            "reward of step 23229 is: 1.1221051255980443\n",
            "reward of step 23230 is: 1.1296963959810795\n",
            "reward of step 23231 is: 1.0985316590078757\n",
            "reward of step 23232 is: 1.1236750925405863\n",
            "reward of step 23233 is: 1.1201626213354308\n",
            "reward of step 23234 is: 1.0741542233938937\n",
            "reward of step 23235 is: 1.0745360009960558\n",
            "reward of step 23236 is: 1.1295265501191225\n",
            "reward of step 23237 is: 1.0996363083007137\n",
            "reward of step 23238 is: 1.0460288344914894\n",
            "reward of step 23239 is: 1.0805598999599666\n",
            "reward of step 23240 is: 1.0638587950813365\n",
            "reward of step 23241 is: 1.070916112848888\n",
            "reward of step 23242 is: 1.1284171555730294\n",
            "reward of step 23243 is: 1.091206762696717\n",
            "reward of step 23244 is: 1.097595535792844\n",
            "reward of step 23245 is: 1.1176437636365986\n",
            "reward of step 23246 is: 1.1233447603782991\n",
            "reward of step 23247 is: 1.0619340889077198\n",
            "reward of step 23248 is: 1.0520091211055798\n",
            "reward of step 23249 is: 1.0761591621355184\n",
            "reward of step 23250 is: 1.0926563274973324\n",
            "reward of step 23251 is: 1.0777007920813424\n",
            "reward of step 23252 is: 1.0911604000125341\n",
            "reward of step 23253 is: 1.0709667229123128\n",
            "reward of step 23254 is: 1.1095594533232416\n",
            "reward of step 23255 is: 1.0965545592915213\n",
            "reward of step 23256 is: 1.0926100540868098\n",
            "reward of step 23257 is: 1.0179021723393493\n",
            "reward of step 23258 is: 0.9990683523276258\n",
            "reward of step 23259 is: 1.091853210244984\n",
            "reward of step 23260 is: 1.044947738994981\n",
            "reward of step 23261 is: 1.0807428162846993\n",
            "reward of step 23262 is: 1.1240611047047628\n",
            "reward of step 23263 is: 1.0420685319345786\n",
            "reward of step 23264 is: 1.033046032736399\n",
            "reward of step 23265 is: 1.120594306731056\n",
            "reward of step 23266 is: 1.115740111648518\n",
            "reward of step 23267 is: 1.0613831347525924\n",
            "reward of step 23268 is: 1.0367162726059402\n",
            "reward of step 23269 is: 1.090878324448823\n",
            "reward of step 23270 is: 1.1789036553472227\n",
            "reward of step 23271 is: 1.118597696444672\n",
            "reward of step 23272 is: 1.0791568990897433\n",
            "reward of step 23273 is: 1.1009362434327858\n",
            "reward of step 23274 is: 1.0318117785048528\n",
            "reward of step 23275 is: 1.1120275793011873\n",
            "reward of step 23276 is: 1.0258986007331656\n",
            "reward of step 23277 is: 1.0435806437638535\n",
            "reward of step 23278 is: 1.084966631418402\n",
            "reward of step 23279 is: 1.087485002172178\n",
            "reward of step 23280 is: 1.1209959710298116\n",
            "reward of step 23281 is: 1.0516600812775603\n",
            "reward of step 23282 is: 1.1242867625675093\n",
            "reward of step 23283 is: 1.0293703321931877\n",
            "reward of step 23284 is: 1.0283446789798947\n",
            "reward of step 23285 is: 1.0935819012741574\n",
            "reward of step 23286 is: 1.1579160792632095\n",
            "reward of step 23287 is: 1.175244410285547\n",
            "reward of step 23288 is: 1.0606203466509267\n",
            "reward of step 23289 is: 1.147159293253699\n",
            "reward of step 23290 is: 1.1433423964683156\n",
            "reward of step 23291 is: 1.1346949626999971\n",
            "reward of step 23292 is: 1.0518593152413462\n",
            "reward of step 23293 is: 1.1571166705522633\n",
            "reward of step 23294 is: 1.133620902431032\n",
            "reward of step 23295 is: 1.0923217855952934\n",
            "reward of step 23296 is: 1.075869022998057\n",
            "reward of step 23297 is: 1.0829491136287488\n",
            "reward of step 23298 is: 1.0407607076933378\n",
            "reward of step 23299 is: 1.048753125592286\n",
            "reward of step 23300 is: 1.0981684800463969\n",
            "reward of step 23301 is: 1.0803132853378097\n",
            "reward of step 23302 is: 1.0126909478714834\n",
            "reward of step 23303 is: 1.0342122943581777\n",
            "reward of step 23304 is: 1.056665162393355\n",
            "reward of step 23305 is: 1.0518764705135444\n",
            "reward of step 23306 is: 1.1031440462942457\n",
            "reward of step 23307 is: 1.074532857554546\n",
            "reward of step 23308 is: 1.0804253180886465\n",
            "reward of step 23309 is: 1.0894120471159643\n",
            "reward of step 23310 is: 1.0487205849361816\n",
            "reward of step 23311 is: 1.0184414496897647\n",
            "reward of step 23312 is: 1.1225903797825967\n",
            "reward of step 23313 is: 1.082580928781066\n",
            "reward of step 23314 is: 1.0849626081896169\n",
            "reward of step 23315 is: 1.1048774714893699\n",
            "reward of step 23316 is: 1.044036526957973\n",
            "reward of step 23317 is: 1.0415997399776953\n",
            "reward of step 23318 is: 1.070868288471977\n",
            "reward of step 23319 is: 1.0389195119090764\n",
            "reward of step 23320 is: 1.0414280395330353\n",
            "reward of step 23321 is: 1.1083356074537303\n",
            "reward of step 23322 is: 1.072505361701701\n",
            "reward of step 23323 is: 1.1080323532036473\n",
            "reward of step 23324 is: 1.0592139034905155\n",
            "reward of step 23325 is: 1.0166672629792142\n",
            "reward of step 23326 is: 1.0952699473874805\n",
            "reward of step 23327 is: 1.0241838120071327\n",
            "reward of step 23328 is: 1.1021937880910437\n",
            "reward of step 23329 is: 1.034837722256834\n",
            "reward of step 23330 is: 1.0560989348626708\n",
            "reward of step 23331 is: 1.0982349104588223\n",
            "reward of step 23332 is: 1.1138269801850589\n",
            "reward of step 23333 is: 1.0908749572095708\n",
            "reward of step 23334 is: 1.0689840760449312\n",
            "reward of step 23335 is: 1.0350276002865009\n",
            "reward of step 23336 is: 1.1177197813131698\n",
            "reward of step 23337 is: 1.0787076513221634\n",
            "reward of step 23338 is: 1.0552038616336366\n",
            "reward of step 23339 is: 1.0592995965582785\n",
            "reward of step 23340 is: 1.0548112838199661\n",
            "reward of step 23341 is: 1.0488615735394138\n",
            "reward of step 23342 is: 1.07513186075192\n",
            "reward of step 23343 is: 1.0792858089418884\n",
            "reward of step 23344 is: 1.0888891709977322\n",
            "reward of step 23345 is: 1.0621748354798706\n",
            "reward of step 23346 is: 1.0070967342136545\n",
            "reward of step 23347 is: 1.1260706988566087\n",
            "reward of step 23348 is: 1.0427764631406198\n",
            "reward of step 23349 is: 1.0913828663098566\n",
            "reward of step 23350 is: 1.1237947411896034\n",
            "reward of step 23351 is: 1.111209259359501\n",
            "reward of step 23352 is: 1.0619071102192579\n",
            "reward of step 23353 is: 1.0521045453249178\n",
            "reward of step 23354 is: 1.099382538747339\n",
            "reward of step 23355 is: 1.1219067633549997\n",
            "reward of step 23356 is: 1.1127730784905485\n",
            "reward of step 23357 is: 1.1125398424228314\n",
            "reward of step 23358 is: 1.0429728141013594\n",
            "reward of step 23359 is: 1.104525348055451\n",
            "reward of step 23360 is: 1.1111843065752165\n",
            "reward of step 23361 is: 1.1311728874685332\n",
            "reward of step 23362 is: 1.0615256917973448\n",
            "reward of step 23363 is: 1.0409499186346656\n",
            "reward of step 23364 is: 1.0729464423154298\n",
            "reward of step 23365 is: 1.0272512597863575\n",
            "reward of step 23366 is: 0.9303982466372055\n",
            "reward of step 23367 is: 0.884265461929203\n",
            "reward of step 23368 is: 0.7024022840712366\n",
            "reward of step 23369 is: 0.6852247633099772\n",
            "reward of step 23370 is: 0.5380741264452069\n",
            "reward of step 23371 is: 0.439635991791688\n",
            "reward of step 23372 is: 0.30882346221168877\n",
            "reward of step 23373 is: 0.2492763323685926\n",
            "reward of step 23374 is: 0.19668590535050734\n",
            "reward of step 23375 is: 0.190038849759946\n",
            "reward of step 23376 is: 0.10959898478657659\n",
            "reward of step 23377 is: 0.03549493634540868\n",
            "reward of step 23378 is: 0.09885617935328739\n",
            "reward of step 23379 is: 0.06622291125322544\n",
            "reward of step 23380 is: -0.00316372877557769\n",
            "reward of step 23381 is: -0.012136220014562493\n",
            "reward of step 23382 is: 0.009183703937729826\n",
            "reward of step 23383 is: 0.030087423922613876\n",
            "reward of step 23384 is: 0.031696780607783026\n",
            "reward of step 23385 is: 0.02889515083547045\n",
            "reward of step 23386 is: -0.020595911438056613\n",
            "reward of step 23387 is: -0.14110139278585132\n",
            "reward of step 23388 is: 0.011230692427089406\n",
            "reward of step 23389 is: -0.09028453603435449\n",
            "reward of step 23390 is: -0.04589463425554785\n",
            "reward of step 23391 is: -0.014879790934265724\n",
            "reward of step 23392 is: -0.06081616351328101\n",
            "reward of step 23393 is: -0.045382077411259436\n",
            "reward of step 23394 is: 0.00679959753127124\n",
            "reward of step 23395 is: -0.03421740725803479\n",
            "reward of step 23396 is: -0.07174399414612653\n",
            "reward of step 23397 is: -0.11277152697062653\n",
            "reward of step 23398 is: -0.05815422785382174\n",
            "reward of step 23399 is: -0.06198780746603627\n",
            "reward of step 23400 is: -0.07936290949451141\n",
            "reward of step 23401 is: -0.11022638361586712\n",
            "reward of step 23402 is: -0.09052597663018802\n",
            "reward of step 23403 is: 0.03100603457533313\n",
            "reward of step 23404 is: -0.0006383557472807722\n",
            "reward of step 23405 is: -0.033917747478087845\n",
            "reward of step 23406 is: -0.05702804791743854\n",
            "reward of step 23407 is: -0.07148186146075974\n",
            "reward of step 23408 is: -0.09308489900124806\n",
            "reward of step 23409 is: -0.039338813417006846\n",
            "reward of step 23410 is: -0.041020447674283034\n",
            "reward of step 23411 is: -0.02835517205195659\n",
            "reward of step 23412 is: -0.04837064450808437\n",
            "reward of step 23413 is: -0.08896725129081695\n",
            "reward of step 23414 is: -0.10330763122619946\n",
            "reward of step 23415 is: -0.08394611566632515\n",
            "reward of step 23416 is: -0.02968818746947799\n",
            "reward of step 23417 is: -0.06026512246528304\n",
            "reward of step 23418 is: -0.11071321552354396\n",
            "reward of step 23419 is: -0.06820776177581933\n",
            "reward of step 23420 is: -0.018529637491130013\n",
            "reward of step 23421 is: -0.11458011053395978\n",
            "reward of step 23422 is: -0.0822361410018877\n",
            "reward of step 23423 is: -0.08841304449146281\n",
            "reward of step 23424 is: -0.03530571304267671\n",
            "reward of step 23425 is: -0.06120343539150985\n",
            "reward of step 23426 is: -0.10337803524256539\n",
            "reward of step 23427 is: -0.0652836874621785\n",
            "reward of step 23428 is: -0.05509765840919134\n",
            "reward of step 23429 is: -0.09218354287706831\n",
            "reward of step 23430 is: -0.051446100038752385\n",
            "reward of step 23431 is: -0.08452705728274112\n",
            "reward of step 23432 is: -0.021145545289792844\n",
            "reward of step 23433 is: -0.0813187061164451\n",
            "reward of step 23434 is: 0.015681028658652063\n",
            "reward of step 23435 is: -0.11273838543641623\n",
            "reward of step 23436 is: -0.03928597281389745\n",
            "reward of step 23437 is: -0.09141149817280614\n",
            "reward of step 23438 is: -0.10335270371928407\n",
            "reward of step 23439 is: -0.10224302739053392\n",
            "reward of step 23440 is: -0.17189885283347062\n",
            "reward of step 23441 is: -0.07333111837183182\n",
            "reward of step 23442 is: -0.0966361055042606\n",
            "reward of step 23443 is: -0.08430566028077857\n",
            "reward of step 23444 is: -0.0940614217388771\n",
            "reward of step 23445 is: -0.08887194934568499\n",
            "reward of step 23446 is: -0.040398278442674185\n",
            "reward of step 23447 is: -0.01844389765418808\n",
            "reward of step 23448 is: -0.09160298804336986\n",
            "reward of step 23449 is: -0.12414543245846055\n",
            "reward of step 23450 is: -0.05345368974183906\n",
            "reward of step 23451 is: -0.11062546724765987\n",
            "reward of step 23452 is: -0.14901824518873386\n",
            "reward of step 23453 is: -0.12751062118072343\n",
            "reward of step 23454 is: -0.05156574678928061\n",
            "reward of step 23455 is: -0.0883288140826819\n",
            "reward of step 23456 is: -0.13368979711501994\n",
            "reward of step 23457 is: -0.09858765221413257\n",
            "reward of step 23458 is: -0.09074790076126826\n",
            "reward of step 23459 is: -0.07226025110721601\n",
            "reward of step 23460 is: -0.007983179547657704\n",
            "reward of step 23461 is: -0.18473168357656478\n",
            "reward of step 23462 is: -0.0856312520348832\n",
            "reward of step 23463 is: -0.07810850445844675\n",
            "reward of step 23464 is: -0.0511613760323717\n",
            "reward of step 23465 is: 0.0014509720082175326\n",
            "reward of step 23466 is: -0.04648107730633555\n",
            "reward of step 23467 is: -0.1142836448802782\n",
            "reward of step 23468 is: -0.030507957264421126\n",
            "reward of step 23469 is: -0.0626182061680054\n",
            "reward of step 23470 is: -0.021439091838250546\n",
            "reward of step 23471 is: -0.07631900192096175\n",
            "reward of step 23472 is: -0.05429681473814063\n",
            "reward of step 23473 is: -0.09519560070723965\n",
            "reward of step 23474 is: -0.13815273483381585\n",
            "reward of step 23475 is: -0.1271913253395608\n",
            "reward of step 23476 is: -0.06389267065221427\n",
            "reward of step 23477 is: -0.05613260407098153\n",
            "reward of step 23478 is: -0.06346940968551562\n",
            "reward of step 23479 is: -0.12785209256225427\n",
            "reward of step 23480 is: -0.15346558743339744\n",
            "reward of step 23481 is: -0.1325651379870041\n",
            "reward of step 23482 is: -0.06950100883062793\n",
            "reward of step 23483 is: 0.02823499767048765\n",
            "reward of step 23484 is: -0.14089326623791387\n",
            "reward of step 23485 is: -0.08288697218757324\n",
            "reward of step 23486 is: -0.026419152680240754\n",
            "reward of step 23487 is: -0.10068246746277387\n",
            "reward of step 23488 is: -0.08995441355785083\n",
            "reward of step 23489 is: -0.05821506317888803\n",
            "reward of step 23490 is: -0.10161494925724512\n",
            "reward of step 23491 is: -0.09127423503348064\n",
            "reward of step 23492 is: -0.05785317844787652\n",
            "reward of step 23493 is: -0.06218887257158745\n",
            "reward of step 23494 is: -0.04326981516452599\n",
            "reward of step 23495 is: -0.07198649034216031\n",
            "reward of step 23496 is: -0.07045841345222514\n",
            "reward of step 23497 is: -0.056831022772396156\n",
            "reward of step 23498 is: -0.05386565603680815\n",
            "reward of step 23499 is: -0.08333418459396635\n",
            "reward of step 23500 is: -0.11872489365833971\n",
            "reward of step 23501 is: -0.06380153131007738\n",
            "reward of step 23502 is: -0.027743425628157237\n",
            "reward of step 23503 is: 0.009982332022271945\n",
            "reward of step 23504 is: -0.09320996096498624\n",
            "reward of step 23505 is: -0.12037932121837991\n",
            "reward of step 23506 is: -0.09451947563630969\n",
            "reward of step 23507 is: -0.032606216496710694\n",
            "reward of step 23508 is: -0.0602692104055349\n",
            "reward of step 23509 is: -0.08639200710931161\n",
            "reward of step 23510 is: -0.021901760332307174\n",
            "reward of step 23511 is: -0.1111292595624429\n",
            "reward of step 23512 is: -0.06511461920472528\n",
            "reward of step 23513 is: -0.11051207985495903\n",
            "reward of step 23514 is: -0.06638011053070136\n",
            "reward of step 23515 is: -0.07332166161456699\n",
            "reward of step 23516 is: -0.04903170168562154\n",
            "reward of step 23517 is: -0.09582276296839343\n",
            "reward of step 23518 is: -0.1108507620570488\n",
            "reward of step 23519 is: -0.09122735493884726\n",
            "reward of step 23520 is: 0.01468852501895801\n",
            "reward of step 23521 is: -0.06959470786499056\n",
            "reward of step 23522 is: -0.0154927415327899\n",
            "reward of step 23523 is: -0.08477267222315132\n",
            "reward of step 23524 is: -0.09350186639723357\n",
            "reward of step 23525 is: -0.03712393658143043\n",
            "reward of step 23526 is: -0.046071224664515964\n",
            "reward of step 23527 is: -0.11249487812499459\n",
            "reward of step 23528 is: -0.06907151215555285\n",
            "reward of step 23529 is: -0.08289045025948549\n",
            "reward of step 23530 is: -0.08943697282222285\n",
            "reward of step 23531 is: -0.04737596010395406\n",
            "reward of step 23532 is: -0.08284663706205864\n",
            "reward of step 23533 is: -0.11856367135686197\n",
            "reward of step 23534 is: -0.03301064322594094\n",
            "reward of step 23535 is: -0.0130889772950874\n",
            "reward of step 23536 is: -0.07943765310813189\n",
            "reward of step 23537 is: -0.0012264015292186015\n",
            "reward of step 23538 is: -0.14208899846887824\n",
            "reward of step 23539 is: -0.07371133017463094\n",
            "reward of step 23540 is: -0.11101741807855992\n",
            "reward of step 23541 is: -0.13733973512121944\n",
            "reward of step 23542 is: -0.07701323713127761\n",
            "reward of step 23543 is: -0.1350965657499006\n",
            "reward of step 23544 is: -0.06025263449835416\n",
            "reward of step 23545 is: -0.03724491626247328\n",
            "reward of step 23546 is: -0.06512370545755342\n",
            "reward of step 23547 is: -0.09387214872890093\n",
            "reward of step 23548 is: -0.05046354752210114\n",
            "reward of step 23549 is: -0.11763411926312384\n",
            "reward of step 23550 is: -0.05203790840937483\n",
            "reward of step 23551 is: -0.0910102593919051\n",
            "reward of step 23552 is: -0.05653722376906123\n",
            "reward of step 23553 is: -0.07237432942546318\n",
            "reward of step 23554 is: -0.10211829738831957\n",
            "reward of step 23555 is: -0.1432242029200388\n",
            "reward of step 23556 is: -0.028948912180250197\n",
            "reward of step 23557 is: -0.09781742066313304\n",
            "reward of step 23558 is: -0.08986515818907981\n",
            "reward of step 23559 is: -0.04952772270319994\n",
            "reward of step 23560 is: -0.060634606725048834\n",
            "reward of step 23561 is: -0.08187017166721755\n",
            "reward of step 23562 is: -0.10712385550049941\n",
            "reward of step 23563 is: -0.03894307026550048\n",
            "reward of step 23564 is: -0.1058749059237194\n",
            "reward of step 23565 is: -0.08488485278495661\n",
            "reward of step 23566 is: -0.1268222141744375\n",
            "reward of step 23567 is: -0.10838716036115359\n",
            "reward of step 23568 is: -0.07059950997017095\n",
            "reward of step 23569 is: -0.039306700958643304\n",
            "reward of step 23570 is: -0.0445567770872749\n",
            "reward of step 23571 is: -0.11241097219135132\n",
            "reward of step 23572 is: -0.03400546403357829\n",
            "reward of step 23573 is: -0.026939612092791387\n",
            "reward of step 23574 is: -0.03837036375570224\n",
            "reward of step 23575 is: -0.05458763967911362\n",
            "reward of step 23576 is: 0.012679136424471937\n",
            "reward of step 23577 is: -0.0986906217655179\n",
            "reward of step 23578 is: -0.06566717077450035\n",
            "reward of step 23579 is: -0.12897008080443406\n",
            "reward of step 23580 is: -0.11122844642111962\n",
            "reward of step 23581 is: -0.08829539618932636\n",
            "reward of step 23582 is: -0.05880573042426118\n",
            "reward of step 23583 is: -0.10020846536941286\n",
            "reward of step 23584 is: -0.05783523796240142\n",
            "reward of step 23585 is: -0.08819524384716226\n",
            "reward of step 23586 is: -0.05304842202980864\n",
            "reward of step 23587 is: -0.016618321507148925\n",
            "reward of step 23588 is: 0.011589613548479316\n",
            "reward of step 23589 is: -0.04220921711930592\n",
            "reward of step 23590 is: -0.15298753470303683\n",
            "reward of step 23591 is: -0.010799008317530001\n",
            "reward of step 23592 is: -0.03576804254152621\n",
            "reward of step 23593 is: -0.05684145783270811\n",
            "reward of step 23594 is: -0.014328918724502926\n",
            "reward of step 23595 is: 0.002241234720634089\n",
            "reward of step 23596 is: -0.11218990240464466\n",
            "reward of step 23597 is: -0.020761932597564647\n",
            "reward of step 23598 is: -0.13495378004460246\n",
            "reward of step 23599 is: -0.14999506084896008\n",
            "reward of step 23600 is: -0.09809789396865298\n",
            "reward of step 23601 is: -0.08445597069885091\n",
            "reward of step 23602 is: -0.08736958658185667\n",
            "reward of step 23603 is: -0.1202620772178361\n",
            "reward of step 23604 is: -0.055536102159752576\n",
            "reward of step 23605 is: -0.11187174786075549\n",
            "reward of step 23606 is: -0.10315169725814011\n",
            "reward of step 23607 is: -0.06581916738444604\n",
            "reward of step 23608 is: -0.024918003113367893\n",
            "reward of step 23609 is: -0.057454691471751085\n",
            "reward of step 23610 is: -0.05588663429907814\n",
            "reward of step 23611 is: -0.09375597728149576\n",
            "reward of step 23612 is: -0.05622030747878981\n",
            "reward of step 23613 is: -0.022700660743813383\n",
            "reward of step 23614 is: -0.07430821345737149\n",
            "reward of step 23615 is: -0.14478218687545186\n",
            "reward of step 23616 is: -0.04141038036452327\n",
            "reward of step 23617 is: -0.10750297218131455\n",
            "reward of step 23618 is: -0.09339030779117252\n",
            "reward of step 23619 is: -0.06345014119067016\n",
            "reward of step 23620 is: -0.09270697400440076\n",
            "reward of step 23621 is: -0.01962801060139563\n",
            "reward of step 23622 is: -0.11219548630796927\n",
            "reward of step 23623 is: -0.14011242472511554\n",
            "reward of step 23624 is: -0.10510141999469425\n",
            "reward of step 23625 is: -0.04252938753364577\n",
            "reward of step 23626 is: -0.0814444280183737\n",
            "reward of step 23627 is: -0.0680874447462615\n",
            "reward of step 23628 is: -0.04522076935110875\n",
            "reward of step 23629 is: -0.11303454425182802\n",
            "reward of step 23630 is: -0.09011499689739111\n",
            "reward of step 23631 is: -0.03272654401294284\n",
            "reward of step 23632 is: -0.13324786464484173\n",
            "reward of step 23633 is: -0.05807030536745417\n",
            "reward of step 23634 is: -0.03875587964031246\n",
            "reward of step 23635 is: -0.10588704120623083\n",
            "reward of step 23636 is: -0.10000334615196083\n",
            "reward of step 23637 is: -0.0530357115791984\n",
            "reward of step 23638 is: -0.04545448026750121\n",
            "reward of step 23639 is: -0.10618444162145102\n",
            "reward of step 23640 is: -0.055729892472629716\n",
            "reward of step 23641 is: -0.10180547126318684\n",
            "reward of step 23642 is: -0.06921355034903287\n",
            "reward of step 23643 is: -0.10274828195516594\n",
            "reward of step 23644 is: -0.08163888885069714\n",
            "reward of step 23645 is: -0.04268437102977418\n",
            "reward of step 23646 is: -0.03518470130393525\n",
            "reward of step 23647 is: -0.07446077799448214\n",
            "reward of step 23648 is: -0.05226059930426297\n",
            "reward of step 23649 is: 0.01636590361667145\n",
            "reward of step 23650 is: -0.1418522309144281\n",
            "reward of step 23651 is: -0.05753196000933947\n",
            "reward of step 23652 is: -0.1473495944765031\n",
            "reward of step 23653 is: -0.05393971345141857\n",
            "reward of step 23654 is: -0.0670086127670737\n",
            "reward of step 23655 is: -0.11895699502120405\n",
            "reward of step 23656 is: -0.07277149921630643\n",
            "reward of step 23657 is: -0.05460419116206272\n",
            "reward of step 23658 is: -0.03284774301195503\n",
            "reward of step 23659 is: 0.004977250157812252\n",
            "reward of step 23660 is: -0.08950138346209435\n",
            "reward of step 23661 is: -0.09106601860631536\n",
            "reward of step 23662 is: -0.044822490888742905\n",
            "reward of step 23663 is: 0.012188467056654329\n",
            "reward of step 23664 is: -0.08355203084538942\n",
            "reward of step 23665 is: -0.12170748537098108\n",
            "reward of step 23666 is: -0.08866857867824218\n",
            "reward of step 23667 is: -0.14698582914765657\n",
            "reward of step 23668 is: -0.08379846144148784\n",
            "reward of step 23669 is: -0.04544664291566658\n",
            "reward of step 23670 is: -0.08200929358426623\n",
            "reward of step 23671 is: -0.07132621749803969\n",
            "reward of step 23672 is: -0.05693209256395404\n",
            "reward of step 23673 is: -0.09759903285969662\n",
            "reward of step 23674 is: -0.025881574586497313\n",
            "reward of step 23675 is: -0.09945476511301743\n",
            "reward of step 23676 is: -0.06846629033096241\n",
            "reward of step 23677 is: -0.07832396951612819\n",
            "reward of step 23678 is: -0.02651412912757889\n",
            "reward of step 23679 is: -0.07122789970712717\n",
            "reward of step 23680 is: -0.07413188196234544\n",
            "reward of step 23681 is: -0.1298968575767081\n",
            "reward of step 23682 is: -0.02432858535825666\n",
            "reward of step 23683 is: -0.05816861992566269\n",
            "reward of step 23684 is: -0.09857230838601083\n",
            "reward of step 23685 is: -0.009518065435773493\n",
            "reward of step 23686 is: -0.053914813372170634\n",
            "reward of step 23687 is: -0.060317119733299895\n",
            "reward of step 23688 is: -0.05876833488518385\n",
            "reward of step 23689 is: -0.04443463593897801\n",
            "reward of step 23690 is: -0.0848695248481276\n",
            "reward of step 23691 is: -0.10469273323940043\n",
            "reward of step 23692 is: -0.07348209131028216\n",
            "reward of step 23693 is: -0.04771693406414379\n",
            "reward of step 23694 is: -0.08223335012878918\n",
            "reward of step 23695 is: -0.06645216794770692\n",
            "reward of step 23696 is: -0.1046326019935947\n",
            "reward of step 23697 is: -0.11778183968956946\n",
            "reward of step 23698 is: -0.060223012151238375\n",
            "reward of step 23699 is: -0.0841379346976423\n",
            "reward of step 23700 is: -0.08927627240768643\n",
            "reward of step 23701 is: -0.12285042556129755\n",
            "reward of step 23702 is: -0.06700645693124585\n",
            "reward of step 23703 is: 0.007151731733528566\n",
            "reward of step 23704 is: -0.055268155766503124\n",
            "reward of step 23705 is: -0.1366861977599768\n",
            "reward of step 23706 is: -0.04354349577158301\n",
            "reward of step 23707 is: -0.010360180596111324\n",
            "reward of step 23708 is: -0.09576540634084285\n",
            "reward of step 23709 is: -0.07022799495543453\n",
            "reward of step 23710 is: 0.004606055052463032\n",
            "reward of step 23711 is: -0.09565130488553164\n",
            "reward of step 23712 is: -0.1236479137154165\n",
            "reward of step 23713 is: -0.0474970464030634\n",
            "reward of step 23714 is: -0.034856662203150646\n",
            "reward of step 23715 is: -0.0930217109544943\n",
            "reward of step 23716 is: -0.04305155065939281\n",
            "reward of step 23717 is: -0.027445379445622198\n",
            "reward of step 23718 is: 0.026366321842958618\n",
            "reward of step 23719 is: -0.1124099492975762\n",
            "reward of step 23720 is: -0.02887756640888095\n",
            "reward of step 23721 is: -0.1041853786887641\n",
            "reward of step 23722 is: -0.04812468613196941\n",
            "reward of step 23723 is: -0.050206570848438936\n",
            "reward of step 23724 is: -0.006432329512058632\n",
            "reward of step 23725 is: -0.0851195894077511\n",
            "reward of step 23726 is: -0.13888968478064856\n",
            "reward of step 23727 is: -0.00018669673744076132\n",
            "reward of step 23728 is: -0.019309574612977776\n",
            "reward of step 23729 is: -0.09891976006824565\n",
            "reward of step 23730 is: -0.09735469546154485\n",
            "reward of step 23731 is: -0.0888810399805221\n",
            "reward of step 23732 is: -0.019773021823088266\n",
            "reward of step 23733 is: -0.04044998018099233\n",
            "reward of step 23734 is: 0.0160218137545588\n",
            "reward of step 23735 is: 0.01739655448470856\n",
            "reward of step 23736 is: -0.041275958278859104\n",
            "reward of step 23737 is: -0.11107519595832205\n",
            "reward of step 23738 is: -0.07636010442160046\n",
            "reward of step 23739 is: -0.01469820402399069\n",
            "reward of step 23740 is: -0.10415780600872038\n",
            "reward of step 23741 is: -0.09818606407940256\n",
            "reward of step 23742 is: -0.09143330051104404\n",
            "reward of step 23743 is: -0.09492783775745217\n",
            "reward of step 23744 is: -0.12580295696709354\n",
            "reward of step 23745 is: -0.04673804098966117\n",
            "reward of step 23746 is: -0.03949673092234873\n",
            "reward of step 23747 is: -0.07900605324621246\n",
            "reward of step 23748 is: -0.06962390285611242\n",
            "reward of step 23749 is: -0.07553768308679276\n",
            "reward of step 23750 is: -0.1458406539053505\n",
            "reward of step 23751 is: -0.12460356174343268\n",
            "reward of step 23752 is: -0.02779902499239928\n",
            "reward of step 23753 is: -0.0832031892691627\n",
            "reward of step 23754 is: -0.02399543751961486\n",
            "reward of step 23755 is: -0.09122030654380764\n",
            "reward of step 23756 is: -0.09452860216833459\n",
            "reward of step 23757 is: -0.13312308326647704\n",
            "reward of step 23758 is: -0.012397124906466717\n",
            "reward of step 23759 is: -0.08481147305137593\n",
            "reward of step 23760 is: -0.13350407983056045\n",
            "reward of step 23761 is: -0.0801555090624928\n",
            "reward of step 23762 is: -0.06233967162983589\n",
            "reward of step 23763 is: -0.07557685105084067\n",
            "reward of step 23764 is: -0.051372454509650334\n",
            "reward of step 23765 is: -0.07157857808927126\n",
            "reward of step 23766 is: -0.0019443743450225925\n",
            "reward of step 23767 is: -0.06359734504155468\n",
            "reward of step 23768 is: -0.055115162163522524\n",
            "reward of step 23769 is: -0.05061422480412514\n",
            "reward of step 23770 is: -0.1572947378216264\n",
            "reward of step 23771 is: -0.03932374833479624\n",
            "reward of step 23772 is: -0.09660757000004006\n",
            "reward of step 23773 is: -0.07536565239789317\n",
            "reward of step 23774 is: -0.09386577919175654\n",
            "reward of step 23775 is: -0.10674674343569945\n",
            "reward of step 23776 is: -0.04315276575011384\n",
            "reward of step 23777 is: -0.07574119754524555\n",
            "reward of step 23778 is: -0.10744665084655669\n",
            "reward of step 23779 is: -0.006681044871741704\n",
            "reward of step 23780 is: -0.063375612859631\n",
            "reward of step 23781 is: -0.06355001654381365\n",
            "reward of step 23782 is: -0.0707779637012046\n",
            "reward of step 23783 is: -0.14550032420684578\n",
            "reward of step 23784 is: -0.05977903877663915\n",
            "reward of step 23785 is: -0.034644711459716926\n",
            "reward of step 23786 is: -0.10157194726942109\n",
            "reward of step 23787 is: -0.06539483354829179\n",
            "reward of step 23788 is: -0.04481071828626004\n",
            "reward of step 23789 is: -0.13088446622313243\n",
            "reward of step 23790 is: -0.10651653903248759\n",
            "reward of step 23791 is: -0.11311950512560875\n",
            "reward of step 23792 is: -0.023586834639338194\n",
            "reward of step 23793 is: -0.13235101491119017\n",
            "reward of step 23794 is: -0.015446419775624443\n",
            "reward of step 23795 is: -0.03015052016546249\n",
            "reward of step 23796 is: -0.029861866836211837\n",
            "reward of step 23797 is: -0.07600982198979511\n",
            "reward of step 23798 is: -0.08099111955507399\n",
            "reward of step 23799 is: -0.07534411090579674\n",
            "reward of step 23800 is: -0.05025805111015169\n",
            "reward of step 23801 is: -0.13372420720433764\n",
            "reward of step 23802 is: -0.010865012401241603\n",
            "reward of step 23803 is: -0.08915231541037816\n",
            "reward of step 23804 is: -0.056033057355975746\n",
            "reward of step 23805 is: -0.11111481815959401\n",
            "reward of step 23806 is: -0.12491984037943771\n",
            "reward of step 23807 is: -0.10656110725112011\n",
            "reward of step 23808 is: -0.018021719244726397\n",
            "reward of step 23809 is: -0.12606783065549554\n",
            "reward of step 23810 is: -0.09400984846341953\n",
            "reward of step 23811 is: -0.13645905032988026\n",
            "reward of step 23812 is: -0.0577525152909828\n",
            "reward of step 23813 is: -0.11990468026379564\n",
            "reward of step 23814 is: -0.015144892221075734\n",
            "reward of step 23815 is: 0.005727227461696072\n",
            "reward of step 23816 is: -0.08140278972981885\n",
            "reward of step 23817 is: -0.0889121325309492\n",
            "reward of step 23818 is: -0.00700154699289568\n",
            "reward of step 23819 is: -0.04658678466882504\n",
            "reward of step 23820 is: -0.07184140650148008\n",
            "reward of step 23821 is: -0.07215851430253495\n",
            "reward of step 23822 is: -0.13602248714627396\n",
            "reward of step 23823 is: -0.10636345582777562\n",
            "reward of step 23824 is: -0.011583595219449294\n",
            "reward of step 23825 is: -0.1359055225396314\n",
            "reward of step 23826 is: -0.08597667818827837\n",
            "reward of step 23827 is: -0.09529053016178146\n",
            "reward of step 23828 is: -0.07911462937594327\n",
            "reward of step 23829 is: -0.06212959382643879\n",
            "reward of step 23830 is: -0.07514496095204448\n",
            "reward of step 23831 is: -0.09358457077189952\n",
            "reward of step 23832 is: -0.0527394699944862\n",
            "reward of step 23833 is: -0.07648198914856374\n",
            "reward of step 23834 is: -0.02118039249280279\n",
            "reward of step 23835 is: -0.075911302724601\n",
            "reward of step 23836 is: -0.11064205598417853\n",
            "reward of step 23837 is: -0.12392389971739792\n",
            "reward of step 23838 is: -0.1257946799936297\n",
            "reward of step 23839 is: -0.11712335817750086\n",
            "reward of step 23840 is: -0.12815669237521266\n",
            "reward of step 23841 is: -0.0845464786619009\n",
            "reward of step 23842 is: -0.06525351148138725\n",
            "reward of step 23843 is: -0.1267672806788438\n",
            "reward of step 23844 is: -0.10336919410604295\n",
            "reward of step 23845 is: -0.07599042678435253\n",
            "reward of step 23846 is: -0.13762780942938446\n",
            "reward of step 23847 is: -0.07471421711047865\n",
            "reward of step 23848 is: -0.16826065564382642\n",
            "reward of step 23849 is: -0.11103228319494718\n",
            "reward of step 23850 is: -0.06287644694052263\n",
            "reward of step 23851 is: -0.08466580100092946\n",
            "reward of step 23852 is: -0.06974293707789636\n",
            "reward of step 23853 is: -0.06577505700087272\n",
            "reward of step 23854 is: -0.03379682325242728\n",
            "reward of step 23855 is: -0.10092991404127549\n",
            "reward of step 23856 is: -0.12675120563247322\n",
            "reward of step 23857 is: -0.12083523573266008\n",
            "reward of step 23858 is: -0.1647295936490777\n",
            "reward of step 23859 is: -0.12421162503047112\n",
            "reward of step 23860 is: -0.09829657241568401\n",
            "reward of step 23861 is: -0.03183332859393695\n",
            "reward of step 23862 is: -0.08984583296899096\n",
            "reward of step 23863 is: -0.11726828726230887\n",
            "reward of step 23864 is: -0.0983859494837056\n",
            "reward of step 23865 is: -0.1085634463047056\n",
            "reward of step 23866 is: -0.08139421635383826\n",
            "reward of step 23867 is: -0.06303632267816173\n",
            "reward of step 23868 is: -0.059941070931228024\n",
            "reward of step 23869 is: -0.1237239894625356\n",
            "reward of step 23870 is: -0.11890059792376195\n",
            "reward of step 23871 is: -0.03771049391743575\n",
            "reward of step 23872 is: -0.05492169157581761\n",
            "reward of step 23873 is: -0.0639110290880065\n",
            "reward of step 23874 is: -0.1214014478314982\n",
            "reward of step 23875 is: 0.010345859628725584\n",
            "reward of step 23876 is: -0.11060390003721776\n",
            "reward of step 23877 is: -0.12829917506709665\n",
            "reward of step 23878 is: -0.02857140821424664\n",
            "reward of step 23879 is: -0.12847234429283227\n",
            "reward of step 23880 is: -0.1604417405218681\n",
            "reward of step 23881 is: -0.05826547921590031\n",
            "reward of step 23882 is: -0.07346993622015863\n",
            "reward of step 23883 is: -0.027631097701864116\n",
            "reward of step 23884 is: -0.16960053039425105\n",
            "reward of step 23885 is: -0.04505771348591636\n",
            "reward of step 23886 is: -0.06439271298892402\n",
            "reward of step 23887 is: -0.010603618618285071\n",
            "reward of step 23888 is: -0.07131685821484046\n",
            "reward of step 23889 is: -0.026635594603977686\n",
            "reward of step 23890 is: -0.11632303990229986\n",
            "reward of step 23891 is: -0.09368041438284802\n",
            "reward of step 23892 is: -0.15620742745388372\n",
            "reward of step 23893 is: -0.10385582002678184\n",
            "reward of step 23894 is: -0.15335567845739062\n",
            "reward of step 23895 is: -0.030064956364557527\n",
            "reward of step 23896 is: -0.10272403523326068\n",
            "reward of step 23897 is: -0.07206699961799512\n",
            "reward of step 23898 is: -0.14033112852814966\n",
            "reward of step 23899 is: -0.018015303256461057\n",
            "reward of step 23900 is: -0.09962227154368997\n",
            "reward of step 23901 is: -0.1054478616280381\n",
            "reward of step 23902 is: -0.11631647137207735\n",
            "reward of step 23903 is: -0.06329294909503158\n",
            "reward of step 23904 is: -0.10616338994453478\n",
            "reward of step 23905 is: -0.04811011968815959\n",
            "reward of step 23906 is: -0.08385317003145543\n",
            "reward of step 23907 is: -0.008072870809571753\n",
            "reward of step 23908 is: -0.12355112344001351\n",
            "reward of step 23909 is: -0.10008639627604254\n",
            "reward of step 23910 is: -0.06742976878402085\n",
            "reward of step 23911 is: -0.050966240788567374\n",
            "reward of step 23912 is: -0.07542678814578407\n",
            "reward of step 23913 is: -0.09849446612997359\n",
            "reward of step 23914 is: -0.031535684575401346\n",
            "reward of step 23915 is: -0.03896102312145877\n",
            "reward of step 23916 is: -0.06480484882037851\n",
            "reward of step 23917 is: -0.0616345544107868\n",
            "reward of step 23918 is: -0.1139351870612062\n",
            "reward of step 23919 is: -0.06282435658926733\n",
            "reward of step 23920 is: -0.09628402543515213\n",
            "reward of step 23921 is: -0.03836189060000761\n",
            "reward of step 23922 is: -0.038650095575067156\n",
            "reward of step 23923 is: -0.10607966343415176\n",
            "reward of step 23924 is: -0.0608567702657441\n",
            "reward of step 23925 is: -0.06731805441245664\n",
            "reward of step 23926 is: -0.07614145149211127\n",
            "reward of step 23927 is: -0.028979833509273356\n",
            "reward of step 23928 is: -0.027813618296826936\n",
            "reward of step 23929 is: -0.12214246783727245\n",
            "reward of step 23930 is: -0.1062674966285414\n",
            "reward of step 23931 is: -0.11281813525435358\n",
            "reward of step 23932 is: -0.08664887051788006\n",
            "reward of step 23933 is: -0.07174005053929389\n",
            "reward of step 23934 is: -0.08466731584256404\n",
            "reward of step 23935 is: -0.0895552606529515\n",
            "reward of step 23936 is: -0.08503885188084359\n",
            "reward of step 23937 is: -0.057144841582981165\n",
            "reward of step 23938 is: -0.09691105650772758\n",
            "reward of step 23939 is: -0.03824344025195747\n",
            "reward of step 23940 is: -0.05329467904652185\n",
            "reward of step 23941 is: -0.06599720961472011\n",
            "reward of step 23942 is: -0.14216647687866957\n",
            "reward of step 23943 is: -0.04228928032152446\n",
            "reward of step 23944 is: -0.07578272798350272\n",
            "reward of step 23945 is: -0.04778188354997559\n",
            "reward of step 23946 is: -0.09067523807792244\n",
            "reward of step 23947 is: -0.12054149087122557\n",
            "reward of step 23948 is: -0.036488553407869495\n",
            "reward of step 23949 is: -0.11162487477748506\n",
            "reward of step 23950 is: -0.04349163465238781\n",
            "reward of step 23951 is: -0.1353019106506177\n",
            "reward of step 23952 is: -0.0669458772268996\n",
            "reward of step 23953 is: -0.028064519363077634\n",
            "reward of step 23954 is: -0.14986717205187572\n",
            "reward of step 23955 is: -0.07948321824494742\n",
            "reward of step 23956 is: -0.09847452811716717\n",
            "reward of step 23957 is: -0.08762630649881509\n",
            "reward of step 23958 is: -0.016515541164295433\n",
            "reward of step 23959 is: -0.025603958981162078\n",
            "reward of step 23960 is: -0.10286008248818834\n",
            "reward of step 23961 is: -0.0690602141184371\n",
            "reward of step 23962 is: -0.031481562133966356\n",
            "reward of step 23963 is: -0.048118758524327676\n",
            "reward of step 23964 is: -0.08213038128118388\n",
            "reward of step 23965 is: -0.08670465995193044\n",
            "reward of step 23966 is: -0.04698740316949257\n",
            "reward of step 23967 is: -0.09437539099052761\n",
            "reward of step 23968 is: -0.06294181716141578\n",
            "reward of step 23969 is: -0.07943677664445858\n",
            "reward of step 23970 is: -0.02059948605732098\n",
            "reward of step 23971 is: -0.01804183600617415\n",
            "reward of step 23972 is: -0.039578158252389084\n",
            "reward of step 23973 is: -0.05351496892871899\n",
            "reward of step 23974 is: -0.08504466054385229\n",
            "reward of step 23975 is: -0.07144697452158988\n",
            "reward of step 23976 is: -0.07160543403399966\n",
            "reward of step 23977 is: -0.10866319089806886\n",
            "reward of step 23978 is: -0.10417042721092384\n",
            "reward of step 23979 is: -0.11159664792498492\n",
            "reward of step 23980 is: -0.012907413820577474\n",
            "reward of step 23981 is: -0.09153633240186365\n",
            "reward of step 23982 is: -0.03958343365706518\n",
            "reward of step 23983 is: -0.10804420660508351\n",
            "reward of step 23984 is: -0.0814704417892449\n",
            "reward of step 23985 is: -0.0627418334158345\n",
            "reward of step 23986 is: 3.3332969019528313e-05\n",
            "reward of step 23987 is: -0.05571924031721498\n",
            "reward of step 23988 is: -0.06024648438575808\n",
            "reward of step 23989 is: -0.10685068592577762\n",
            "reward of step 23990 is: -0.01695862237782042\n",
            "reward of step 23991 is: -0.06319725760599859\n",
            "reward of step 23992 is: -0.027234130557004632\n",
            "reward of step 23993 is: -0.0942689382178985\n",
            "reward of step 23994 is: -0.07114388763166313\n",
            "reward of step 23995 is: -0.11900070077956992\n",
            "reward of step 23996 is: -0.04619607393449121\n",
            "reward of step 23997 is: -0.06734743672298171\n",
            "reward of step 23998 is: -0.09320427384827423\n",
            "reward of step 23999 is: -0.0725939442001815\n",
            "reward of step 24000 is: -0.05101529239373659\n",
            "reward of step 24001 is: -0.09360122893024936\n",
            "reward of step 24002 is: -0.05290108127562676\n",
            "reward of step 24003 is: -0.08896509070032688\n",
            "reward of step 24004 is: -0.14306844113954575\n",
            "reward of step 24005 is: -0.13979833534355912\n",
            "reward of step 24006 is: -0.08536656897965367\n",
            "reward of step 24007 is: -0.06077756494878028\n",
            "reward of step 24008 is: -0.13843822469951383\n",
            "reward of step 24009 is: -0.07630885787989661\n",
            "reward of step 24010 is: -0.04902604225460261\n",
            "reward of step 24011 is: -0.12523458514209662\n",
            "reward of step 24012 is: -0.12258335957978661\n",
            "reward of step 24013 is: 0.038968965385802856\n",
            "reward of step 24014 is: -0.03764120167470031\n",
            "reward of step 24015 is: -0.12020978760122314\n",
            "reward of step 24016 is: -0.10273412245728053\n",
            "reward of step 24017 is: -0.024086691326309806\n",
            "reward of step 24018 is: -0.06638323449307504\n",
            "reward of step 24019 is: -0.01812405237828041\n",
            "reward of step 24020 is: -0.055994049904066534\n",
            "reward of step 24021 is: -0.010851156984247434\n",
            "reward of step 24022 is: -0.07728840914826263\n",
            "reward of step 24023 is: -0.03221036782075548\n",
            "reward of step 24024 is: -0.056727084298781594\n",
            "reward of step 24025 is: -0.11606243356723878\n",
            "reward of step 24026 is: 0.0027685493610698497\n",
            "reward of step 24027 is: -0.11255474658408215\n",
            "reward of step 24028 is: -0.046191713206516716\n",
            "reward of step 24029 is: -0.08799490107590335\n",
            "reward of step 24030 is: -0.04464706479464153\n",
            "reward of step 24031 is: -0.04670877980861965\n",
            "reward of step 24032 is: 8.408594775399791e-05\n",
            "reward of step 24033 is: -0.07922065763789099\n",
            "reward of step 24034 is: -0.17291575715445318\n",
            "reward of step 24035 is: -0.12606676620158153\n",
            "reward of step 24036 is: -0.11251634415158795\n",
            "reward of step 24037 is: -0.08375468639168981\n",
            "reward of step 24038 is: -0.0853076292841426\n",
            "reward of step 24039 is: -0.09605084041017564\n",
            "reward of step 24040 is: -0.06658523819535822\n",
            "reward of step 24041 is: -0.0873992213928163\n",
            "reward of step 24042 is: -0.07765485271289907\n",
            "reward of step 24043 is: -0.08352544449947141\n",
            "reward of step 24044 is: -0.061820975622394414\n",
            "reward of step 24045 is: -0.0772497443105078\n",
            "reward of step 24046 is: -0.05814607904766811\n",
            "reward of step 24047 is: -0.009928659199162637\n",
            "reward of step 24048 is: -0.08956207888334022\n",
            "reward of step 24049 is: -0.04393893284678174\n",
            "reward of step 24050 is: -0.13756837059661797\n",
            "reward of step 24051 is: -0.09732555781872976\n",
            "reward of step 24052 is: -0.09962407574793852\n",
            "reward of step 24053 is: -0.08219898167094164\n",
            "reward of step 24054 is: -0.09725982720549853\n",
            "reward of step 24055 is: -0.034294985067432115\n",
            "reward of step 24056 is: -0.039416656962041774\n",
            "reward of step 24057 is: -0.02895207196149774\n",
            "reward of step 24058 is: -0.0805436495181463\n",
            "reward of step 24059 is: -0.06799129531860348\n",
            "reward of step 24060 is: -0.13002522333758426\n",
            "reward of step 24061 is: -0.049896187518000734\n",
            "reward of step 24062 is: -0.0820495155147486\n",
            "reward of step 24063 is: -0.07330840366702207\n",
            "reward of step 24064 is: -0.03587380894540615\n",
            "reward of step 24065 is: -0.07800866263114858\n",
            "reward of step 24066 is: -0.10952131577021751\n",
            "reward of step 24067 is: -0.03674631014176766\n",
            "reward of step 24068 is: -0.08337683894341452\n",
            "reward of step 24069 is: -0.02141070143364665\n",
            "reward of step 24070 is: -0.10994667352641863\n",
            "reward of step 24071 is: -0.0706285800785198\n",
            "reward of step 24072 is: -0.09339462075373028\n",
            "reward of step 24073 is: -0.051003971826701466\n",
            "reward of step 24074 is: -0.015398137345146279\n",
            "reward of step 24075 is: -0.08787163756254668\n",
            "reward of step 24076 is: -0.05928114398523454\n",
            "reward of step 24077 is: -0.09911187752724249\n",
            "reward of step 24078 is: -0.09011124615920008\n",
            "reward of step 24079 is: 0.018441576837639806\n",
            "reward of step 24080 is: -0.13614274854460695\n",
            "reward of step 24081 is: -0.07227443136356893\n",
            "reward of step 24082 is: -0.10470651069277936\n",
            "reward of step 24083 is: -0.06729499899962499\n",
            "reward of step 24084 is: -0.05622445789310104\n",
            "reward of step 24085 is: -0.08515111489941951\n",
            "reward of step 24086 is: -0.09504017224474581\n",
            "reward of step 24087 is: -0.03517777108185649\n",
            "reward of step 24088 is: -0.09915753621950729\n",
            "reward of step 24089 is: -0.1207140477896983\n",
            "reward of step 24090 is: -0.0939204574528052\n",
            "reward of step 24091 is: -0.04742061152695509\n",
            "reward of step 24092 is: -0.14830063151453943\n",
            "reward of step 24093 is: -0.11050443049793501\n",
            "reward of step 24094 is: -0.06130124087393818\n",
            "reward of step 24095 is: -0.03392899843435393\n",
            "reward of step 24096 is: -0.01196677284777481\n",
            "reward of step 24097 is: -0.06030368924092122\n",
            "reward of step 24098 is: -0.0930888686338538\n",
            "reward of step 24099 is: -0.13751904652987645\n",
            "reward of step 24100 is: -0.09361666586903894\n",
            "reward of step 24101 is: -0.05752891519772252\n",
            "reward of step 24102 is: -0.013390872151670119\n",
            "reward of step 24103 is: -0.07702534642124603\n",
            "reward of step 24104 is: -0.09274633549833855\n",
            "reward of step 24105 is: -0.02912070104293918\n",
            "reward of step 24106 is: -0.08041160556287974\n",
            "reward of step 24107 is: -0.09695623535039788\n",
            "reward of step 24108 is: -0.10678025561702853\n",
            "reward of step 24109 is: -0.00012648460042519183\n",
            "reward of step 24110 is: -0.05836239199353499\n",
            "reward of step 24111 is: -0.07931535173627624\n",
            "reward of step 24112 is: -0.12039449686464665\n",
            "reward of step 24113 is: -0.06079921343209327\n",
            "reward of step 24114 is: -0.06917177164069888\n",
            "reward of step 24115 is: -0.009188823423201264\n",
            "reward of step 24116 is: -0.08257811975393903\n",
            "reward of step 24117 is: -0.07685900184095673\n",
            "reward of step 24118 is: -0.04772752376368161\n",
            "reward of step 24119 is: -0.1090204342612715\n",
            "reward of step 24120 is: -0.05678663335804668\n",
            "reward of step 24121 is: -0.05365623322401869\n",
            "reward of step 24122 is: -0.05549843782001318\n",
            "reward of step 24123 is: -0.07489912361749673\n",
            "reward of step 24124 is: -0.04738270950348955\n",
            "reward of step 24125 is: -0.14178419807765819\n",
            "reward of step 24126 is: -0.10972286999610692\n",
            "reward of step 24127 is: -0.0538710776051351\n",
            "reward of step 24128 is: -0.06167533135491354\n",
            "reward of step 24129 is: -0.152676878882834\n",
            "reward of step 24130 is: -0.07209692559891878\n",
            "reward of step 24131 is: -0.0397256785873108\n",
            "reward of step 24132 is: -0.100660805327633\n",
            "reward of step 24133 is: -0.018885491814413458\n",
            "reward of step 24134 is: -0.09328513412839956\n",
            "reward of step 24135 is: -0.1076976029193959\n",
            "reward of step 24136 is: -0.14234108508761112\n",
            "reward of step 24137 is: -0.00995749206424601\n",
            "reward of step 24138 is: -0.0660660595985223\n",
            "reward of step 24139 is: -0.08586568784235471\n",
            "reward of step 24140 is: -0.08115357591954353\n",
            "reward of step 24141 is: -0.08884284524784969\n",
            "reward of step 24142 is: -0.11224989682787145\n",
            "reward of step 24143 is: -0.07835840902760305\n",
            "reward of step 24144 is: -0.014754618324383073\n",
            "reward of step 24145 is: -0.12912639298868878\n",
            "reward of step 24146 is: -0.061013365646262474\n",
            "reward of step 24147 is: -0.09441734304275973\n",
            "reward of step 24148 is: -0.08780353831904919\n",
            "reward of step 24149 is: -0.13750178560162551\n",
            "reward of step 24150 is: -0.08554468459306164\n",
            "reward of step 24151 is: -0.0324996344527857\n",
            "reward of step 24152 is: -0.06864239586704113\n",
            "reward of step 24153 is: -0.06288914492486819\n",
            "reward of step 24154 is: -0.017434550367689372\n",
            "reward of step 24155 is: -0.10670612621471509\n",
            "reward of step 24156 is: -0.11390263654267563\n",
            "reward of step 24157 is: -0.12475143967306179\n",
            "reward of step 24158 is: -0.08907729789413221\n",
            "reward of step 24159 is: -0.07160385894163879\n",
            "reward of step 24160 is: -0.06799476968882445\n",
            "reward of step 24161 is: -0.0960540620061231\n",
            "reward of step 24162 is: -0.11839695868114497\n",
            "reward of step 24163 is: -0.09060594251166587\n",
            "reward of step 24164 is: -0.12158780908999411\n",
            "reward of step 24165 is: -0.03206440212013306\n",
            "reward of step 24166 is: -0.05110991252362829\n",
            "reward of step 24167 is: -0.05579885716645172\n",
            "reward of step 24168 is: -0.03069721006348969\n",
            "reward of step 24169 is: -0.07325137482348432\n",
            "reward of step 24170 is: -0.05826595264680723\n",
            "reward of step 24171 is: -0.05155314487778029\n",
            "reward of step 24172 is: 0.014752166462127736\n",
            "reward of step 24173 is: -0.040066829310317464\n",
            "reward of step 24174 is: -0.06110465802007181\n",
            "reward of step 24175 is: -0.1032057049851075\n",
            "reward of step 24176 is: -0.053713312653505874\n",
            "reward of step 24177 is: -0.0724244846464015\n",
            "reward of step 24178 is: -0.07959228501634719\n",
            "reward of step 24179 is: -0.02978004559160019\n",
            "reward of step 24180 is: -0.02240057128390005\n",
            "reward of step 24181 is: -0.16211910653458905\n",
            "reward of step 24182 is: -0.07743473893602049\n",
            "reward of step 24183 is: -0.07883908769488324\n",
            "reward of step 24184 is: -0.13765203232090317\n",
            "reward of step 24185 is: 0.009231319208729993\n",
            "reward of step 24186 is: -0.06342627917613586\n",
            "reward of step 24187 is: -0.11261670183517869\n",
            "reward of step 24188 is: -0.06473459642962198\n",
            "reward of step 24189 is: -0.09510534097664025\n",
            "reward of step 24190 is: -0.022813428723230222\n",
            "reward of step 24191 is: -0.03820878111465753\n",
            "reward of step 24192 is: -0.09705964340739537\n",
            "reward of step 24193 is: -0.11150387748374158\n",
            "reward of step 24194 is: -0.1154384892237531\n",
            "reward of step 24195 is: -0.13412999769758172\n",
            "reward of step 24196 is: -0.09257885539757726\n",
            "reward of step 24197 is: -0.06514815279085184\n",
            "reward of step 24198 is: -0.0541830453386456\n",
            "reward of step 24199 is: -0.048873078970474326\n",
            "reward of step 24200 is: -0.0505390015632492\n",
            "reward of step 24201 is: -0.10624500459319408\n",
            "reward of step 24202 is: -0.04107875462545518\n",
            "reward of step 24203 is: -0.05129368309579829\n",
            "reward of step 24204 is: -0.05099277089106591\n",
            "reward of step 24205 is: -0.09608882630769833\n",
            "reward of step 24206 is: -0.020457225487981345\n",
            "reward of step 24207 is: -0.06793544431752896\n",
            "reward of step 24208 is: -0.04328725770227615\n",
            "reward of step 24209 is: -0.11186920276855561\n",
            "reward of step 24210 is: -0.05975797347330636\n",
            "reward of step 24211 is: -0.0059942035386809644\n",
            "reward of step 24212 is: -0.11997352278339901\n",
            "reward of step 24213 is: -0.041618293078041435\n",
            "reward of step 24214 is: -0.05767883044462785\n",
            "reward of step 24215 is: -0.12739899921281894\n",
            "reward of step 24216 is: -0.11314577202655607\n",
            "reward of step 24217 is: -0.13153590171125895\n",
            "reward of step 24218 is: -0.09953828099020978\n",
            "reward of step 24219 is: -0.05410018487198942\n",
            "reward of step 24220 is: -0.044272767028360804\n",
            "reward of step 24221 is: -0.08690150072694902\n",
            "reward of step 24222 is: -0.08833861683651667\n",
            "reward of step 24223 is: -0.14088883235261374\n",
            "reward of step 24224 is: -0.10440894125523825\n",
            "reward of step 24225 is: -0.08918654469083354\n",
            "reward of step 24226 is: -0.09902571661147719\n",
            "reward of step 24227 is: -0.0812096032170786\n",
            "reward of step 24228 is: -0.07832689504252832\n",
            "reward of step 24229 is: -0.059943197169942675\n",
            "reward of step 24230 is: -0.032581612364066426\n",
            "reward of step 24231 is: -0.07537122532259777\n",
            "reward of step 24232 is: -0.06678747513302863\n",
            "reward of step 24233 is: -0.03201349154134636\n",
            "reward of step 24234 is: -0.07811450986404689\n",
            "reward of step 24235 is: -0.036603680296308116\n",
            "reward of step 24236 is: -0.035995064650429565\n",
            "reward of step 24237 is: -0.009159968777784022\n",
            "reward of step 24238 is: -0.06173185828747396\n",
            "reward of step 24239 is: -0.08461810915470969\n",
            "reward of step 24240 is: 0.021852262144451795\n",
            "reward of step 24241 is: -0.1314892342531503\n",
            "reward of step 24242 is: -0.10839035345216741\n",
            "reward of step 24243 is: -0.13015694112343712\n",
            "reward of step 24244 is: -0.04479062570837844\n",
            "reward of step 24245 is: -0.0656834547364803\n",
            "reward of step 24246 is: -0.13443365171503074\n",
            "reward of step 24247 is: -0.19007303279557308\n",
            "reward of step 24248 is: -0.05862453429405701\n",
            "reward of step 24249 is: -0.08088083585047401\n",
            "reward of step 24250 is: -0.04198953820087625\n",
            "reward of step 24251 is: -0.06022134234237231\n",
            "reward of step 24252 is: -0.05688804387570323\n",
            "reward of step 24253 is: -0.09676440423289112\n",
            "reward of step 24254 is: -0.1357828617938156\n",
            "reward of step 24255 is: -0.1662150037871084\n",
            "reward of step 24256 is: -0.07677317223616653\n",
            "reward of step 24257 is: -0.006914476051639862\n",
            "reward of step 24258 is: -0.035840192833727635\n",
            "reward of step 24259 is: -0.09432122347930616\n",
            "reward of step 24260 is: -0.0818407471903222\n",
            "reward of step 24261 is: -0.06378605066059528\n",
            "reward of step 24262 is: -0.06303407490392554\n",
            "reward of step 24263 is: -0.04968857620756706\n",
            "reward of step 24264 is: -0.1079247784182007\n",
            "reward of step 24265 is: -0.08980764407636144\n",
            "reward of step 24266 is: -0.08683389099703698\n",
            "reward of step 24267 is: -0.059184429351266776\n",
            "reward of step 24268 is: -0.09954938714725181\n",
            "reward of step 24269 is: -0.04076472110787144\n",
            "reward of step 24270 is: -0.021959020037933485\n",
            "reward of step 24271 is: -0.03735815109449836\n",
            "reward of step 24272 is: -0.12562893203769665\n",
            "reward of step 24273 is: -0.008120489669413367\n",
            "reward of step 24274 is: -0.044027652569821996\n",
            "reward of step 24275 is: -0.08085463085101718\n",
            "reward of step 24276 is: -0.14242799885320168\n",
            "reward of step 24277 is: -0.10103282693583815\n",
            "reward of step 24278 is: 0.01091983276636943\n",
            "reward of step 24279 is: 0.0015684511552076508\n",
            "reward of step 24280 is: -0.08287766837285793\n",
            "reward of step 24281 is: -0.012702064133988777\n",
            "reward of step 24282 is: -0.05808079992781978\n",
            "reward of step 24283 is: -0.1177722431497219\n",
            "reward of step 24284 is: -0.026752451127980414\n",
            "reward of step 24285 is: -0.09618528690047434\n",
            "reward of step 24286 is: -0.02644445124360839\n",
            "reward of step 24287 is: -0.14888532268483934\n",
            "reward of step 24288 is: -0.10565996696714541\n",
            "reward of step 24289 is: -0.1053584089455104\n",
            "reward of step 24290 is: -0.12980201780468537\n",
            "reward of step 24291 is: -0.07254948256651716\n",
            "reward of step 24292 is: -0.049821144428995656\n",
            "reward of step 24293 is: -0.072298719555287\n",
            "reward of step 24294 is: -0.07591522864162381\n",
            "reward of step 24295 is: -0.03215115837683169\n",
            "reward of step 24296 is: -0.08536496465390353\n",
            "reward of step 24297 is: -0.1486086010336325\n",
            "reward of step 24298 is: -0.03000780159276284\n",
            "reward of step 24299 is: -0.05917513521844997\n",
            "reward of step 24300 is: -0.06646778614077498\n",
            "reward of step 24301 is: -0.052248735266016366\n",
            "reward of step 24302 is: -0.059840763158601984\n",
            "reward of step 24303 is: -0.04711184320301254\n",
            "reward of step 24304 is: -0.0035785307386528853\n",
            "reward of step 24305 is: -0.02985574384096401\n",
            "reward of step 24306 is: -0.12864075263557062\n",
            "reward of step 24307 is: -0.1245978714082655\n",
            "reward of step 24308 is: -0.060167865923219876\n",
            "reward of step 24309 is: -0.06438420351420737\n",
            "reward of step 24310 is: -0.006025842950418192\n",
            "reward of step 24311 is: -0.026260504620350122\n",
            "reward of step 24312 is: -0.06528100829150207\n",
            "reward of step 24313 is: -0.02787661050410073\n",
            "reward of step 24314 is: -0.05593334154339258\n",
            "reward of step 24315 is: -0.08529157661630138\n",
            "reward of step 24316 is: -0.08542241830989838\n",
            "reward of step 24317 is: -0.09701637108980221\n",
            "reward of step 24318 is: -0.09564531865224823\n",
            "reward of step 24319 is: -0.044809907000315286\n",
            "reward of step 24320 is: -0.04183333676478529\n",
            "reward of step 24321 is: -0.08922259922166109\n",
            "reward of step 24322 is: -0.06470429111649145\n",
            "reward of step 24323 is: -0.09466207167249652\n",
            "reward of step 24324 is: -0.053471411528371315\n",
            "reward of step 24325 is: -0.04433987698529451\n",
            "reward of step 24326 is: -0.08789631367714967\n",
            "reward of step 24327 is: -0.13952507108749412\n",
            "reward of step 24328 is: 0.018774156401666464\n",
            "reward of step 24329 is: -0.07944477893748025\n",
            "reward of step 24330 is: -0.09768782116195895\n",
            "reward of step 24331 is: -0.039992805800387776\n",
            "reward of step 24332 is: -0.10424295340380685\n",
            "reward of step 24333 is: -0.05629580213468355\n",
            "reward of step 24334 is: -0.06066341604863923\n",
            "reward of step 24335 is: -0.07340611348435033\n",
            "reward of step 24336 is: -0.14293359851318732\n",
            "reward of step 24337 is: -0.12144863145996787\n",
            "reward of step 24338 is: -0.06086891258256255\n",
            "reward of step 24339 is: -0.11508657109453113\n",
            "reward of step 24340 is: -0.09460613485677771\n",
            "reward of step 24341 is: -0.05272600112413861\n",
            "reward of step 24342 is: -0.14741556482944274\n",
            "reward of step 24343 is: 0.016960733103786763\n",
            "reward of step 24344 is: -0.10854237783302767\n",
            "reward of step 24345 is: -0.048203135029928124\n",
            "reward of step 24346 is: -0.03140635180521467\n",
            "reward of step 24347 is: -0.09968704164927655\n",
            "reward of step 24348 is: -0.14149356988048178\n",
            "reward of step 24349 is: -0.09777941627552555\n",
            "reward of step 24350 is: -0.07191276690546333\n",
            "reward of step 24351 is: -0.11720261019753453\n",
            "reward of step 24352 is: -0.07166239982263523\n",
            "reward of step 24353 is: -0.00457522070902916\n",
            "reward of step 24354 is: -0.0944977886998779\n",
            "reward of step 24355 is: -0.11684089402603393\n",
            "reward of step 24356 is: -0.07713415233307319\n",
            "reward of step 24357 is: -0.0703784015377803\n",
            "reward of step 24358 is: -0.047049071355974026\n",
            "reward of step 24359 is: -0.033337159012529316\n",
            "reward of step 24360 is: -0.014430285686970357\n",
            "reward of step 24361 is: -0.03822210790784719\n",
            "reward of step 24362 is: -0.10320919232733294\n",
            "reward of step 24363 is: -0.06961831139398456\n",
            "reward of step 24364 is: -0.09431479153890732\n",
            "reward of step 24365 is: -0.05555282660780925\n",
            "reward of step 24366 is: -0.10224147237374959\n",
            "reward of step 24367 is: -0.12418544696528155\n",
            "reward of step 24368 is: -0.11039938650909542\n",
            "reward of step 24369 is: -0.11638329471302833\n",
            "reward of step 24370 is: -0.045211956831051725\n",
            "reward of step 24371 is: -0.04756539250047498\n",
            "reward of step 24372 is: -0.003464605574451163\n",
            "reward of step 24373 is: -0.10495084351497908\n",
            "reward of step 24374 is: -0.06072989823360819\n",
            "reward of step 24375 is: -0.005180778667906782\n",
            "reward of step 24376 is: -0.12761370262395422\n",
            "reward of step 24377 is: -0.028231917110195348\n",
            "reward of step 24378 is: 0.017020776028652485\n",
            "reward of step 24379 is: -0.020039796647396968\n",
            "reward of step 24380 is: -0.028359254720814242\n",
            "reward of step 24381 is: -0.1095138997007371\n",
            "reward of step 24382 is: 0.0007382997258410517\n",
            "reward of step 24383 is: -0.029864380886499675\n",
            "reward of step 24384 is: -0.11304078079581559\n",
            "reward of step 24385 is: -0.10957927544195867\n",
            "reward of step 24386 is: -0.011392193746203816\n",
            "reward of step 24387 is: -0.11648815763404707\n",
            "reward of step 24388 is: -0.055835442360331444\n",
            "reward of step 24389 is: -0.15930593453425024\n",
            "reward of step 24390 is: -0.05216963398640473\n",
            "reward of step 24391 is: -0.1092966933092222\n",
            "reward of step 24392 is: -0.054531992796468365\n",
            "reward of step 24393 is: -0.06587283043324921\n",
            "reward of step 24394 is: -0.028134171510089234\n",
            "reward of step 24395 is: -0.07432249804068547\n",
            "reward of step 24396 is: -0.08099700747373229\n",
            "reward of step 24397 is: -0.0640448541392955\n",
            "reward of step 24398 is: -0.07881984898256889\n",
            "reward of step 24399 is: -0.05441451678556608\n",
            "reward of step 24400 is: -0.083026707610936\n",
            "reward of step 24401 is: -0.0351674859042318\n",
            "reward of step 24402 is: -0.0595196071953904\n",
            "reward of step 24403 is: -0.12784921144963202\n",
            "reward of step 24404 is: 0.010197081766626925\n",
            "reward of step 24405 is: -0.03922067172076327\n",
            "reward of step 24406 is: -0.08273523197904753\n",
            "reward of step 24407 is: -0.1322536947437828\n",
            "reward of step 24408 is: -0.11855847154468568\n",
            "reward of step 24409 is: -0.12343774350492553\n",
            "reward of step 24410 is: -0.09464268073246551\n",
            "reward of step 24411 is: -0.09280865013897888\n",
            "reward of step 24412 is: -0.09690834638011292\n",
            "reward of step 24413 is: -0.04673094153380597\n",
            "reward of step 24414 is: -0.10697508316762427\n",
            "reward of step 24415 is: -0.113870961333567\n",
            "reward of step 24416 is: -0.04765812249988666\n",
            "reward of step 24417 is: -0.04393210020060179\n",
            "reward of step 24418 is: -0.12593835867248726\n",
            "reward of step 24419 is: -0.07739740910905968\n",
            "reward of step 24420 is: -0.08731162850361796\n",
            "reward of step 24421 is: -0.05536043541563629\n",
            "reward of step 24422 is: -0.09209396481232446\n",
            "reward of step 24423 is: -0.12910155016176017\n",
            "reward of step 24424 is: -0.007988575976225598\n",
            "reward of step 24425 is: -0.04665957544847843\n",
            "reward of step 24426 is: -0.0838796560410019\n",
            "reward of step 24427 is: -0.04025928002070123\n",
            "reward of step 24428 is: -0.009953029653176237\n",
            "reward of step 24429 is: -0.06276095448856744\n",
            "reward of step 24430 is: -0.0959212140002137\n",
            "reward of step 24431 is: -0.034374433237920554\n",
            "reward of step 24432 is: -0.13457532168201247\n",
            "reward of step 24433 is: -0.11813820249783014\n",
            "reward of step 24434 is: -0.056278432902340136\n",
            "reward of step 24435 is: -0.1242845121455608\n",
            "reward of step 24436 is: -0.09388002687715491\n",
            "reward of step 24437 is: -0.08819138305082186\n",
            "reward of step 24438 is: -0.02870483129014545\n",
            "reward of step 24439 is: -0.0812958498056443\n",
            "reward of step 24440 is: -0.09191763161933642\n",
            "reward of step 24441 is: -0.037996610016393983\n",
            "reward of step 24442 is: -0.011942554097164715\n",
            "reward of step 24443 is: -0.1070142423118362\n",
            "reward of step 24444 is: -0.04438507874220121\n",
            "reward of step 24445 is: -0.1165373966436496\n",
            "reward of step 24446 is: -0.05630032785431616\n",
            "reward of step 24447 is: -0.12242772724068918\n",
            "reward of step 24448 is: -0.08639314085556571\n",
            "reward of step 24449 is: -0.06610995265718\n",
            "reward of step 24450 is: -0.11656246125012071\n",
            "reward of step 24451 is: -0.11437803114587408\n",
            "reward of step 24452 is: -0.0264102821881792\n",
            "reward of step 24453 is: -0.09906409842919361\n",
            "reward of step 24454 is: -0.07343105477721612\n",
            "reward of step 24455 is: -0.01581344440849386\n",
            "reward of step 24456 is: -0.05294544987202776\n",
            "reward of step 24457 is: -0.07502259868961358\n",
            "reward of step 24458 is: -0.1284265014781205\n",
            "reward of step 24459 is: -0.08415224200218485\n",
            "reward of step 24460 is: -0.11027925726839405\n",
            "reward of step 24461 is: -0.10157764408068903\n",
            "reward of step 24462 is: -0.0944838991112017\n",
            "reward of step 24463 is: -0.13801189172994133\n",
            "reward of step 24464 is: -0.07654635657303976\n",
            "reward of step 24465 is: -0.0892611978604827\n",
            "reward of step 24466 is: -0.13891352527764345\n",
            "reward of step 24467 is: -0.031117926989945865\n",
            "reward of step 24468 is: -0.10138835311080008\n",
            "reward of step 24469 is: -0.08792192433616808\n",
            "reward of step 24470 is: -0.06319652063324444\n",
            "reward of step 24471 is: 0.002196183501794957\n",
            "reward of step 24472 is: -0.07365207606168789\n",
            "reward of step 24473 is: -0.04205320160163395\n",
            "reward of step 24474 is: -0.07037388200654315\n",
            "reward of step 24475 is: -0.14853210942009765\n",
            "reward of step 24476 is: -0.17435825454522003\n",
            "reward of step 24477 is: -0.07654712992525159\n",
            "reward of step 24478 is: -0.03293308401253736\n",
            "reward of step 24479 is: 0.0012924254485771058\n",
            "reward of step 24480 is: -0.02457413190913904\n",
            "reward of step 24481 is: -0.07077978778553318\n",
            "reward of step 24482 is: -0.01486893095908337\n",
            "reward of step 24483 is: -0.11194928908930557\n",
            "reward of step 24484 is: -0.10108206838382561\n",
            "reward of step 24485 is: -0.09818318557690742\n",
            "reward of step 24486 is: -0.04602938857024552\n",
            "reward of step 24487 is: -0.07537955808206953\n",
            "reward of step 24488 is: -0.09044103492974476\n",
            "reward of step 24489 is: -0.06659408805378286\n",
            "reward of step 24490 is: -0.09493371752342106\n",
            "reward of step 24491 is: 0.013843235811533017\n",
            "reward of step 24492 is: -0.06875189555234329\n",
            "reward of step 24493 is: -0.07795015663350513\n",
            "reward of step 24494 is: -0.10885583496988338\n",
            "reward of step 24495 is: -0.05575274514687911\n",
            "reward of step 24496 is: -0.07624743420539559\n",
            "reward of step 24497 is: -0.05272135677426382\n",
            "reward of step 24498 is: -0.012124628512019564\n",
            "reward of step 24499 is: -0.0901407240295472\n",
            "reward of step 24500 is: -0.16248402147315422\n",
            "reward of step 24501 is: -0.04865071277009492\n",
            "reward of step 24502 is: -0.03368145120875288\n",
            "reward of step 24503 is: -0.038174700448770804\n",
            "reward of step 24504 is: -0.03854518829784437\n",
            "reward of step 24505 is: -0.09773062919220687\n",
            "reward of step 24506 is: -0.12417860301420958\n",
            "reward of step 24507 is: -0.046635248547274966\n",
            "reward of step 24508 is: -0.0957308004389702\n",
            "reward of step 24509 is: -0.09325344198735885\n",
            "reward of step 24510 is: -0.060457041368563336\n",
            "reward of step 24511 is: -0.11627627576570376\n",
            "reward of step 24512 is: -0.025618694362314898\n",
            "reward of step 24513 is: 8.489485589269208e-05\n",
            "reward of step 24514 is: -0.09031845858198417\n",
            "reward of step 24515 is: -0.07237324236053111\n",
            "reward of step 24516 is: -0.08565136373578575\n",
            "reward of step 24517 is: -0.08299858189325415\n",
            "reward of step 24518 is: -0.10489754761466052\n",
            "reward of step 24519 is: -0.07196291511610697\n",
            "reward of step 24520 is: -0.15708556093902437\n",
            "reward of step 24521 is: -0.00799325777701343\n",
            "reward of step 24522 is: 0.00711869441143298\n",
            "reward of step 24523 is: -0.02034778165203166\n",
            "reward of step 24524 is: -0.08949167056804852\n",
            "reward of step 24525 is: -0.12466148724454018\n",
            "reward of step 24526 is: -0.013685496794706653\n",
            "reward of step 24527 is: -0.06781447768784843\n",
            "reward of step 24528 is: -0.10491281502615868\n",
            "reward of step 24529 is: -0.0984768075629584\n",
            "reward of step 24530 is: -0.06405036663094765\n",
            "reward of step 24531 is: -0.03894672716235725\n",
            "reward of step 24532 is: -0.0907089108278667\n",
            "reward of step 24533 is: -0.07401231908845984\n",
            "reward of step 24534 is: -0.08132403892421136\n",
            "reward of step 24535 is: -0.07144051156331455\n",
            "reward of step 24536 is: -0.06304102004054812\n",
            "reward of step 24537 is: -0.015795371489008336\n",
            "reward of step 24538 is: -0.11175063018118503\n",
            "reward of step 24539 is: -0.044613057291782665\n",
            "reward of step 24540 is: -0.02886391869708549\n",
            "reward of step 24541 is: -0.0016186888137493805\n",
            "reward of step 24542 is: -0.05687452419594796\n",
            "reward of step 24543 is: -0.10181837063096666\n",
            "reward of step 24544 is: -0.021184768816365662\n",
            "reward of step 24545 is: -0.0545821745349937\n",
            "reward of step 24546 is: -0.06422048481223686\n",
            "reward of step 24547 is: -0.06058162038559123\n",
            "reward of step 24548 is: -0.09729462493931451\n",
            "reward of step 24549 is: -0.0609285130389593\n",
            "reward of step 24550 is: -0.1356052646054896\n",
            "reward of step 24551 is: -0.0964563749722579\n",
            "reward of step 24552 is: -0.08376588753607317\n",
            "reward of step 24553 is: -0.08809215512286284\n",
            "reward of step 24554 is: -0.018337149700321076\n",
            "reward of step 24555 is: -0.05315155954041073\n",
            "reward of step 24556 is: -0.10764032996385509\n",
            "reward of step 24557 is: -0.10760305344762011\n",
            "reward of step 24558 is: -0.03285933679828945\n",
            "reward of step 24559 is: -0.02794135091370742\n",
            "reward of step 24560 is: -0.050349617926454204\n",
            "reward of step 24561 is: -0.07269929864044233\n",
            "reward of step 24562 is: 0.006901105020042153\n",
            "reward of step 24563 is: -0.0808999742632619\n",
            "reward of step 24564 is: -0.06612836670693323\n",
            "reward of step 24565 is: -0.051436515406148686\n",
            "reward of step 24566 is: -0.014121725696598308\n",
            "reward of step 24567 is: -0.09149634469276524\n",
            "reward of step 24568 is: -0.059246562190474306\n",
            "reward of step 24569 is: -0.10784800405454431\n",
            "reward of step 24570 is: -0.08542582131297438\n",
            "reward of step 24571 is: 0.0029982000743653536\n",
            "reward of step 24572 is: -0.10335143006452707\n",
            "reward of step 24573 is: -0.12878811562217196\n",
            "reward of step 24574 is: -0.008217755153900552\n",
            "reward of step 24575 is: -0.07330617508867976\n",
            "reward of step 24576 is: -0.08737408670258384\n",
            "reward of step 24577 is: -0.10492384354154727\n",
            "reward of step 24578 is: 0.02790780347835964\n",
            "reward of step 24579 is: 0.012980484456312946\n",
            "reward of step 24580 is: -0.028826425051078908\n",
            "reward of step 24581 is: -0.0975314398877728\n",
            "reward of step 24582 is: -0.050577457496050915\n",
            "reward of step 24583 is: -0.03544870041163262\n",
            "reward of step 24584 is: -0.10250898712823997\n",
            "reward of step 24585 is: -0.1043668808147763\n",
            "reward of step 24586 is: -0.06902716706159284\n",
            "reward of step 24587 is: -0.024488179615033356\n",
            "reward of step 24588 is: -0.08125270839103382\n",
            "reward of step 24589 is: -0.08177820100607047\n",
            "reward of step 24590 is: -0.07201910619001073\n",
            "reward of step 24591 is: 0.012705765747683007\n",
            "reward of step 24592 is: -0.04501143082632453\n",
            "reward of step 24593 is: -0.13226438092608483\n",
            "reward of step 24594 is: -0.10770042238213318\n",
            "reward of step 24595 is: -0.09032069984122337\n",
            "reward of step 24596 is: -0.05743376369383901\n",
            "reward of step 24597 is: -0.015580292246002325\n",
            "reward of step 24598 is: -0.002563959935356075\n",
            "reward of step 24599 is: -0.10671285045182566\n",
            "reward of step 24600 is: -0.11037684563749794\n",
            "reward of step 24601 is: -0.05305263340978761\n",
            "reward of step 24602 is: -0.03633527502634537\n",
            "reward of step 24603 is: -0.03982622447666073\n",
            "reward of step 24604 is: 0.01247626893597198\n",
            "reward of step 24605 is: -0.009154049831312072\n",
            "reward of step 24606 is: -0.0733699518524441\n",
            "reward of step 24607 is: -0.07783127263294864\n",
            "reward of step 24608 is: -0.06065002158218502\n",
            "reward of step 24609 is: -0.07317627825522899\n",
            "reward of step 24610 is: -0.13321436377638207\n",
            "reward of step 24611 is: -0.1534512558426725\n",
            "reward of step 24612 is: -0.11105072973183605\n",
            "reward of step 24613 is: -0.06265312818424973\n",
            "reward of step 24614 is: -0.09485557179313175\n",
            "reward of step 24615 is: -0.06674671157110523\n",
            "reward of step 24616 is: -0.1168837697774997\n",
            "reward of step 24617 is: -0.11694444876926946\n",
            "reward of step 24618 is: -0.11815370650453816\n",
            "reward of step 24619 is: -0.13675146022928786\n",
            "reward of step 24620 is: -0.031050761154487128\n",
            "reward of step 24621 is: -0.08128748832895982\n",
            "reward of step 24622 is: -0.09137576380538581\n",
            "reward of step 24623 is: -0.09786372023615808\n",
            "reward of step 24624 is: -0.11842109860607464\n",
            "reward of step 24625 is: -0.05745853029284875\n",
            "reward of step 24626 is: -0.09727138705539606\n",
            "reward of step 24627 is: -0.12161282118770178\n",
            "reward of step 24628 is: -0.011374314479970948\n",
            "reward of step 24629 is: -0.15012787364884583\n",
            "reward of step 24630 is: -0.005039460018864239\n",
            "reward of step 24631 is: -0.03650602774205802\n",
            "reward of step 24632 is: -0.031050682364716087\n",
            "reward of step 24633 is: -0.050256355888034454\n",
            "reward of step 24634 is: -0.07749253827015867\n",
            "reward of step 24635 is: -0.01894084860263978\n",
            "reward of step 24636 is: -0.13334511062448873\n",
            "reward of step 24637 is: -0.05631728560555982\n",
            "reward of step 24638 is: -0.06372067989826025\n",
            "reward of step 24639 is: -0.13156230996076113\n",
            "reward of step 24640 is: -0.04987303394463449\n",
            "reward of step 24641 is: -0.10127225638035642\n",
            "reward of step 24642 is: -0.037841869965561825\n",
            "reward of step 24643 is: -0.011640203831843388\n",
            "reward of step 24644 is: -0.13471993434740837\n",
            "reward of step 24645 is: -0.07293524633913429\n",
            "reward of step 24646 is: -0.08558779600206634\n",
            "reward of step 24647 is: -0.05981521831070202\n",
            "reward of step 24648 is: -0.04815989243969443\n",
            "reward of step 24649 is: -0.006051809332173752\n",
            "reward of step 24650 is: -0.021612663951479427\n",
            "reward of step 24651 is: -0.09023595767572912\n",
            "reward of step 24652 is: -0.07817756814215071\n",
            "reward of step 24653 is: -0.18003288190809558\n",
            "reward of step 24654 is: -0.06322730250107922\n",
            "reward of step 24655 is: -0.07796450348901596\n",
            "reward of step 24656 is: -0.03915402058779949\n",
            "reward of step 24657 is: -0.09468329601638747\n",
            "reward of step 24658 is: -0.08048455422719136\n",
            "reward of step 24659 is: -0.023577352249433803\n",
            "reward of step 24660 is: -0.06353523214437318\n",
            "reward of step 24661 is: -0.11143016586062404\n",
            "reward of step 24662 is: -0.061896006975937334\n",
            "reward of step 24663 is: -0.02196564391346756\n",
            "reward of step 24664 is: -0.13456800217985598\n",
            "reward of step 24665 is: -0.03980722833031791\n",
            "reward of step 24666 is: -0.026860895334059887\n",
            "reward of step 24667 is: -0.03202023925815456\n",
            "reward of step 24668 is: -0.08616485770877436\n",
            "reward of step 24669 is: -0.10748399590143598\n",
            "reward of step 24670 is: -0.0770113761123753\n",
            "reward of step 24671 is: -0.13989466092918001\n",
            "reward of step 24672 is: -0.1529579619548903\n",
            "reward of step 24673 is: -0.02976102403521763\n",
            "reward of step 24674 is: -0.07622450461122598\n",
            "reward of step 24675 is: -0.1221870082121439\n",
            "reward of step 24676 is: -0.060565866406392055\n",
            "reward of step 24677 is: -0.07511354589901853\n",
            "reward of step 24678 is: -0.056949458983582724\n",
            "reward of step 24679 is: -0.07387497026039225\n",
            "reward of step 24680 is: -0.07976399893618091\n",
            "reward of step 24681 is: -0.06887540203547504\n",
            "reward of step 24682 is: -0.04119410950811242\n",
            "reward of step 24683 is: -0.01147195943863566\n",
            "reward of step 24684 is: -0.06820557484419554\n",
            "reward of step 24685 is: -0.07025092922682463\n",
            "reward of step 24686 is: -0.16075051316999822\n",
            "reward of step 24687 is: -0.09162338688033411\n",
            "reward of step 24688 is: -0.0970047137502269\n",
            "reward of step 24689 is: -0.12436019465073722\n",
            "reward of step 24690 is: -0.07810943488524313\n",
            "reward of step 24691 is: -0.05782024851265766\n",
            "reward of step 24692 is: -0.08150956207934401\n",
            "reward of step 24693 is: -0.1027750986766457\n",
            "reward of step 24694 is: -0.029834957192524048\n",
            "reward of step 24695 is: -0.09010577692759225\n",
            "reward of step 24696 is: -0.0776747360702964\n",
            "reward of step 24697 is: -0.005943406707748067\n",
            "reward of step 24698 is: -0.04466720247646294\n",
            "reward of step 24699 is: -0.05548361489502973\n",
            "reward of step 24700 is: -0.11796608390989749\n",
            "reward of step 24701 is: -0.13171434267569881\n",
            "reward of step 24702 is: -0.12577629589060113\n",
            "reward of step 24703 is: -0.0777971379658946\n",
            "reward of step 24704 is: -0.1467741608230786\n",
            "reward of step 24705 is: -0.008526880358584288\n",
            "reward of step 24706 is: -0.0801833320160491\n",
            "reward of step 24707 is: -0.12276622621931865\n",
            "reward of step 24708 is: -0.12411058784469808\n",
            "reward of step 24709 is: -0.03584454915657942\n",
            "reward of step 24710 is: -0.07678716373580874\n",
            "reward of step 24711 is: -0.06734930144368279\n",
            "reward of step 24712 is: -0.11632488366789517\n",
            "reward of step 24713 is: -0.08422549174936933\n",
            "reward of step 24714 is: -0.08933802219743514\n",
            "reward of step 24715 is: -0.08151703112981501\n",
            "reward of step 24716 is: -0.018446744836808304\n",
            "reward of step 24717 is: -0.07191273516750107\n",
            "reward of step 24718 is: -0.06782978854196775\n",
            "reward of step 24719 is: -0.08763430475906764\n",
            "reward of step 24720 is: -0.08456259397703536\n",
            "reward of step 24721 is: -0.024974289186826426\n",
            "reward of step 24722 is: -0.03597512420702609\n",
            "reward of step 24723 is: -0.11541475534697643\n",
            "reward of step 24724 is: -0.07009453969429358\n",
            "reward of step 24725 is: -0.09293216207614907\n",
            "reward of step 24726 is: -0.059080947090605074\n",
            "reward of step 24727 is: -0.0683156291540461\n",
            "reward of step 24728 is: -0.11199379842233181\n",
            "reward of step 24729 is: -0.11606440025698095\n",
            "reward of step 24730 is: -0.09451923496603587\n",
            "reward of step 24731 is: -0.009298738702289144\n",
            "reward of step 24732 is: -0.11630860802809562\n",
            "reward of step 24733 is: -0.05764794568736942\n",
            "reward of step 24734 is: -0.09589680522233224\n",
            "reward of step 24735 is: -0.09710939505603955\n",
            "reward of step 24736 is: -0.05891308259851602\n",
            "reward of step 24737 is: -0.10299094250540375\n",
            "reward of step 24738 is: -0.16924515710068677\n",
            "reward of step 24739 is: -0.048542111557018264\n",
            "reward of step 24740 is: 0.01539010153390341\n",
            "reward of step 24741 is: -0.06574328506581517\n",
            "reward of step 24742 is: -0.07723983617629315\n",
            "reward of step 24743 is: -0.059263273916328574\n",
            "reward of step 24744 is: -0.025242764080056213\n",
            "reward of step 24745 is: -0.1224092472994982\n",
            "reward of step 24746 is: 0.0011142310523137855\n",
            "reward of step 24747 is: -0.07801121834697111\n",
            "reward of step 24748 is: -0.13536803134099873\n",
            "reward of step 24749 is: -0.05720498576684163\n",
            "reward of step 24750 is: -0.1617032704090231\n",
            "reward of step 24751 is: -0.10210983405919938\n",
            "reward of step 24752 is: -0.12168141481454764\n",
            "reward of step 24753 is: -0.14171816135921722\n",
            "reward of step 24754 is: -0.11920668447255356\n",
            "reward of step 24755 is: -0.030779097435499447\n",
            "reward of step 24756 is: -0.1038628460033818\n",
            "reward of step 24757 is: -0.07685766592525778\n",
            "reward of step 24758 is: -0.020227829297910116\n",
            "reward of step 24759 is: -0.175046873535729\n",
            "reward of step 24760 is: -0.11034942486037347\n",
            "reward of step 24761 is: -0.03323707268348619\n",
            "reward of step 24762 is: -0.028383186504542413\n",
            "reward of step 24763 is: -0.07264235109209294\n",
            "reward of step 24764 is: -0.10492108994058247\n",
            "reward of step 24765 is: -0.07056486987407173\n",
            "reward of step 24766 is: -0.11389922134340336\n",
            "reward of step 24767 is: -0.1276477650577572\n",
            "reward of step 24768 is: -0.1081823867028795\n",
            "reward of step 24769 is: -0.023993146602479754\n",
            "reward of step 24770 is: -0.11535308086159002\n",
            "reward of step 24771 is: -0.02962402775032258\n",
            "reward of step 24772 is: -0.05581410329740355\n",
            "reward of step 24773 is: -0.12040002674318884\n",
            "reward of step 24774 is: -0.10916464093242273\n",
            "reward of step 24775 is: -0.11395098860474351\n",
            "reward of step 24776 is: -0.06153786502416336\n",
            "reward of step 24777 is: -0.05358370980645377\n",
            "reward of step 24778 is: -0.048109967379523666\n",
            "reward of step 24779 is: -0.07807488874797586\n",
            "reward of step 24780 is: -0.0859107780722348\n",
            "reward of step 24781 is: -0.049133007032677756\n",
            "reward of step 24782 is: -0.08460975986973573\n",
            "reward of step 24783 is: -0.0700579606718531\n",
            "reward of step 24784 is: -0.05457294915192035\n",
            "reward of step 24785 is: -0.05689523786762052\n",
            "reward of step 24786 is: -0.05655101652334993\n",
            "reward of step 24787 is: -0.043619304428429095\n",
            "reward of step 24788 is: -0.10498180622384701\n",
            "reward of step 24789 is: -0.019768405227402086\n",
            "reward of step 24790 is: -0.07941477952902587\n",
            "reward of step 24791 is: -0.0593966088975737\n",
            "reward of step 24792 is: -0.022495610433754032\n",
            "reward of step 24793 is: -0.0493692456062107\n",
            "reward of step 24794 is: -0.03932284979566425\n",
            "reward of step 24795 is: -0.06347954251068133\n",
            "reward of step 24796 is: -0.0015570441586436612\n",
            "reward of step 24797 is: -0.12631320916116118\n",
            "reward of step 24798 is: -0.04797697103804055\n",
            "reward of step 24799 is: -0.011716648452793965\n",
            "reward of step 24800 is: 0.0054676627447514115\n",
            "reward of step 24801 is: -0.1106737982658631\n",
            "reward of step 24802 is: -0.07601690461297517\n",
            "reward of step 24803 is: -0.04777278824097697\n",
            "reward of step 24804 is: -0.07132577851483712\n",
            "reward of step 24805 is: -0.013379732678813094\n",
            "reward of step 24806 is: -0.06731084260190423\n",
            "reward of step 24807 is: -0.11484955787105522\n",
            "reward of step 24808 is: -0.11479599781518857\n",
            "reward of step 24809 is: -0.04560829100759911\n",
            "reward of step 24810 is: -0.10842564339610772\n",
            "reward of step 24811 is: -0.0625994824274565\n",
            "reward of step 24812 is: -0.15154294298771054\n",
            "reward of step 24813 is: -0.12496623644722504\n",
            "reward of step 24814 is: -0.03746603666610193\n",
            "reward of step 24815 is: -0.03381533292567718\n",
            "reward of step 24816 is: -0.07110272940129103\n",
            "reward of step 24817 is: -0.08393638207288956\n",
            "reward of step 24818 is: -0.07315775185826967\n",
            "reward of step 24819 is: -0.06742927145216238\n",
            "reward of step 24820 is: -0.13429395319422355\n",
            "reward of step 24821 is: -0.11156556877034374\n",
            "reward of step 24822 is: -0.07973225264461148\n",
            "reward of step 24823 is: -0.163174546361589\n",
            "reward of step 24824 is: -0.09599397032040002\n",
            "reward of step 24825 is: -0.12726187471937167\n",
            "reward of step 24826 is: -0.08992224919829994\n",
            "reward of step 24827 is: -0.0660524746634763\n",
            "reward of step 24828 is: -0.06352428174385516\n",
            "reward of step 24829 is: -0.06988247074193854\n",
            "reward of step 24830 is: -0.04785337134267309\n",
            "reward of step 24831 is: -0.07880146544793765\n",
            "reward of step 24832 is: -0.12992339095601302\n",
            "reward of step 24833 is: -0.049327330415649406\n",
            "reward of step 24834 is: -0.11531520119712435\n",
            "reward of step 24835 is: -0.08565004935358334\n",
            "reward of step 24836 is: -0.09699781497160498\n",
            "reward of step 24837 is: -0.010862039489224262\n",
            "reward of step 24838 is: -0.0821907705367293\n",
            "reward of step 24839 is: -0.013888281173916828\n",
            "reward of step 24840 is: -0.02637254431363012\n",
            "reward of step 24841 is: -0.0839177706004699\n",
            "reward of step 24842 is: -0.06976329520554236\n",
            "reward of step 24843 is: -0.05398770245852802\n",
            "reward of step 24844 is: -0.0796108801557065\n",
            "reward of step 24845 is: -0.0185207168762328\n",
            "reward of step 24846 is: -0.10795930689928157\n",
            "reward of step 24847 is: -0.05605711146359715\n",
            "reward of step 24848 is: -0.11550272422746\n",
            "reward of step 24849 is: -0.0818568296621911\n",
            "reward of step 24850 is: -0.03249947225715444\n",
            "reward of step 24851 is: -0.0776721932683474\n",
            "reward of step 24852 is: -0.09711210033476236\n",
            "reward of step 24853 is: -0.0573800517955706\n",
            "reward of step 24854 is: -0.1166912697344088\n",
            "reward of step 24855 is: -0.027264736793560118\n",
            "reward of step 24856 is: -0.0007280187517852976\n",
            "reward of step 24857 is: -0.020836206217986875\n",
            "reward of step 24858 is: -0.05009703700538637\n",
            "reward of step 24859 is: 0.009208257921975438\n",
            "reward of step 24860 is: -0.13294935400306496\n",
            "reward of step 24861 is: -0.06830525860673553\n",
            "reward of step 24862 is: -0.13329932602495598\n",
            "reward of step 24863 is: 0.002480616526942425\n",
            "reward of step 24864 is: -0.11757149044259141\n",
            "reward of step 24865 is: -0.08711085778315142\n",
            "reward of step 24866 is: -0.03743659524996401\n",
            "reward of step 24867 is: 0.005552117561425818\n",
            "reward of step 24868 is: 0.010380082849645422\n",
            "reward of step 24869 is: -0.11245211274417488\n",
            "reward of step 24870 is: -0.09857924812275798\n",
            "reward of step 24871 is: -0.08848382281877065\n",
            "reward of step 24872 is: -0.02895857875444785\n",
            "reward of step 24873 is: -0.059587656973558945\n",
            "reward of step 24874 is: -0.02162880332960082\n",
            "reward of step 24875 is: -0.07967856897291647\n",
            "reward of step 24876 is: -0.1334702874032485\n",
            "reward of step 24877 is: -0.060299632008840476\n",
            "reward of step 24878 is: 0.004923900857685903\n",
            "reward of step 24879 is: -0.08033581791715949\n",
            "reward of step 24880 is: -0.08042289443194672\n",
            "reward of step 24881 is: 0.012422072989997313\n",
            "reward of step 24882 is: -0.09613293537068057\n",
            "reward of step 24883 is: -0.04243628839587621\n",
            "reward of step 24884 is: -0.0423752864581286\n",
            "reward of step 24885 is: -0.04990720168414764\n",
            "reward of step 24886 is: -0.12063091193036479\n",
            "reward of step 24887 is: -0.02463761150592292\n",
            "reward of step 24888 is: -0.04011488133993946\n",
            "reward of step 24889 is: -0.04869368058509538\n",
            "reward of step 24890 is: -0.14430474309149666\n",
            "reward of step 24891 is: -0.04775720678449613\n",
            "reward of step 24892 is: -0.0815271491466043\n",
            "reward of step 24893 is: -0.035737759040623596\n",
            "reward of step 24894 is: -0.017503106052664696\n",
            "reward of step 24895 is: -0.06389631590512179\n",
            "reward of step 24896 is: -0.10703266668128752\n",
            "reward of step 24897 is: -0.07146171395818401\n",
            "reward of step 24898 is: -0.022240840641009063\n",
            "reward of step 24899 is: -0.13526265211056943\n",
            "reward of step 24900 is: -0.08363447838251448\n",
            "reward of step 24901 is: -0.13107398585587304\n",
            "reward of step 24902 is: -0.08374068031489268\n",
            "reward of step 24903 is: -0.05211413023674916\n",
            "reward of step 24904 is: -0.10613320495260825\n",
            "reward of step 24905 is: -0.11171391518385465\n",
            "reward of step 24906 is: -0.029031691934856996\n",
            "reward of step 24907 is: -0.032923548802672564\n",
            "reward of step 24908 is: -0.09933751102526922\n",
            "reward of step 24909 is: -0.09600115974540857\n",
            "reward of step 24910 is: -0.061015458029252456\n",
            "reward of step 24911 is: -0.06257448410955813\n",
            "reward of step 24912 is: -0.07819530952425524\n",
            "reward of step 24913 is: -0.0902390854212628\n",
            "reward of step 24914 is: -0.06023372650219938\n",
            "reward of step 24915 is: -0.125812451992238\n",
            "reward of step 24916 is: -0.03309196740517151\n",
            "reward of step 24917 is: -0.08052621236100554\n",
            "reward of step 24918 is: -0.06453181971519562\n",
            "reward of step 24919 is: -0.11730922743373895\n",
            "reward of step 24920 is: -0.043877340343594695\n",
            "reward of step 24921 is: -0.08553222163025032\n",
            "reward of step 24922 is: -0.1150155561460855\n",
            "reward of step 24923 is: -0.0725612993672341\n",
            "reward of step 24924 is: -0.08700907402782188\n",
            "reward of step 24925 is: -0.07586369929398928\n",
            "reward of step 24926 is: -0.08881000532403116\n",
            "reward of step 24927 is: -0.0460061096777149\n",
            "reward of step 24928 is: -0.05146348837932946\n",
            "reward of step 24929 is: -0.09291271469439\n",
            "reward of step 24930 is: -0.06792073158421674\n",
            "reward of step 24931 is: -0.042639963349459076\n",
            "reward of step 24932 is: -0.0329572634045624\n",
            "reward of step 24933 is: -0.10447995813102717\n",
            "reward of step 24934 is: -0.12810485886441203\n",
            "reward of step 24935 is: -0.023225634808181628\n",
            "reward of step 24936 is: -0.059492417203359516\n",
            "reward of step 24937 is: -0.09989245277272829\n",
            "reward of step 24938 is: -0.06115110184288375\n",
            "reward of step 24939 is: -0.00587228425801356\n",
            "reward of step 24940 is: -0.11845579600422251\n",
            "reward of step 24941 is: -0.028017688952184705\n",
            "reward of step 24942 is: -0.10263115138379642\n",
            "reward of step 24943 is: -0.058322549458439066\n",
            "reward of step 24944 is: -0.0656482205989386\n",
            "reward of step 24945 is: -0.08849914324530461\n",
            "reward of step 24946 is: -0.10957183959046102\n",
            "reward of step 24947 is: -0.16233529836254967\n",
            "reward of step 24948 is: -0.059132249208089016\n",
            "reward of step 24949 is: -0.07027199856792321\n",
            "reward of step 24950 is: -0.06710241867523947\n",
            "reward of step 24951 is: -0.06296607209686567\n",
            "reward of step 24952 is: -0.04315412035249799\n",
            "reward of step 24953 is: -0.0577741081813159\n",
            "reward of step 24954 is: -0.16311185184558052\n",
            "reward of step 24955 is: -0.06725101261526112\n",
            "reward of step 24956 is: -0.07023037842437707\n",
            "reward of step 24957 is: -0.06216983367517226\n",
            "reward of step 24958 is: -0.0661060659845214\n",
            "reward of step 24959 is: -0.07046446744236357\n",
            "reward of step 24960 is: -0.1097469633490038\n",
            "reward of step 24961 is: -0.06300194179641139\n",
            "reward of step 24962 is: -0.0540876656801883\n",
            "reward of step 24963 is: -0.09029859865828438\n",
            "reward of step 24964 is: -0.0743697220334959\n",
            "reward of step 24965 is: -0.10784198500467879\n",
            "reward of step 24966 is: 0.014272150747010381\n",
            "reward of step 24967 is: -0.08768230821571243\n",
            "reward of step 24968 is: 0.018600986088598503\n",
            "reward of step 24969 is: -0.04648682685554717\n",
            "reward of step 24970 is: -0.01842071820222324\n",
            "reward of step 24971 is: -0.017011946984265336\n",
            "reward of step 24972 is: -0.14971705699986948\n",
            "reward of step 24973 is: -0.0575555714002024\n",
            "reward of step 24974 is: -0.04910034280990405\n",
            "reward of step 24975 is: 0.010473465004165905\n",
            "reward of step 24976 is: -0.09898487545524248\n",
            "reward of step 24977 is: -0.07058824889566073\n",
            "reward of step 24978 is: -0.06837077914128653\n",
            "reward of step 24979 is: -0.0825933467371529\n",
            "reward of step 24980 is: -0.006235058429565954\n",
            "reward of step 24981 is: -0.05781185809556344\n",
            "reward of step 24982 is: -0.09513096290361012\n",
            "reward of step 24983 is: -0.022907452775040693\n",
            "reward of step 24984 is: -0.04291746635047755\n",
            "reward of step 24985 is: -0.0352751050283826\n",
            "reward of step 24986 is: -0.09182303314601303\n",
            "reward of step 24987 is: -0.05750284758486868\n",
            "reward of step 24988 is: -0.05033141143313258\n",
            "reward of step 24989 is: -0.06355610116067978\n",
            "reward of step 24990 is: 0.009710150312332133\n",
            "reward of step 24991 is: -0.07842263956414142\n",
            "reward of step 24992 is: 0.008364973198506043\n",
            "reward of step 24993 is: -0.10167733634171239\n",
            "reward of step 24994 is: -0.1579422852182002\n",
            "reward of step 24995 is: -0.10060143168335822\n",
            "reward of step 24996 is: -0.05261250777613413\n",
            "reward of step 24997 is: -0.11570085938639463\n",
            "reward of step 24998 is: -0.08538642302907662\n",
            "reward of step 24999 is: -0.12556024881963623\n",
            "reward of step 25000 is: -0.02223147020333993\n",
            "reward of step 25001 is: -0.050149974737660696\n",
            "reward of step 25002 is: -0.07797727321037684\n",
            "reward of step 25003 is: -0.14664008694660913\n",
            "reward of step 25004 is: -0.04419120542186006\n",
            "reward of step 25005 is: -0.09489998637384123\n",
            "reward of step 25006 is: -0.031214978662066906\n",
            "reward of step 25007 is: -0.007990088759670777\n",
            "reward of step 25008 is: -0.06036189105919221\n",
            "reward of step 25009 is: -0.1512301144343665\n",
            "reward of step 25010 is: -0.040771250375054136\n",
            "reward of step 25011 is: -0.05215757814748945\n",
            "reward of step 25012 is: -0.09212472316757447\n",
            "reward of step 25013 is: -0.11551268811387838\n",
            "reward of step 25014 is: -0.06881122932912465\n",
            "reward of step 25015 is: -0.16436171966008906\n",
            "reward of step 25016 is: -0.07216965308271872\n",
            "reward of step 25017 is: -0.10162910472529707\n",
            "reward of step 25018 is: -0.07779806309200354\n",
            "reward of step 25019 is: -0.08503060150312303\n",
            "reward of step 25020 is: -0.07027177494890768\n",
            "reward of step 25021 is: -0.10881633624062637\n",
            "reward of step 25022 is: -0.06738313446093014\n",
            "reward of step 25023 is: -0.05530236553643786\n",
            "reward of step 25024 is: -0.11980118882827095\n",
            "reward of step 25025 is: -0.03266256538597456\n",
            "reward of step 25026 is: -0.1242354441861564\n",
            "reward of step 25027 is: -0.015074020975151536\n",
            "reward of step 25028 is: -0.07932920779558372\n",
            "reward of step 25029 is: -0.019232410956820045\n",
            "reward of step 25030 is: -0.08698803073842787\n",
            "reward of step 25031 is: -0.05040726146702157\n",
            "reward of step 25032 is: 0.008203540945519006\n",
            "reward of step 25033 is: -0.10183137419137078\n",
            "reward of step 25034 is: -0.04204856864037532\n",
            "reward of step 25035 is: -0.03274057550871179\n",
            "reward of step 25036 is: -0.12476261190688331\n",
            "reward of step 25037 is: -0.07117094794863776\n",
            "reward of step 25038 is: -0.07619217925678012\n",
            "reward of step 25039 is: 0.019299538681912365\n",
            "reward of step 25040 is: -0.10655380770332046\n",
            "reward of step 25041 is: -0.006645947733653679\n",
            "reward of step 25042 is: -0.03600710492238446\n",
            "reward of step 25043 is: -0.06701131478045108\n",
            "reward of step 25044 is: -0.16739325635514657\n",
            "reward of step 25045 is: -0.10000976174884857\n",
            "reward of step 25046 is: -0.00622845143624795\n",
            "reward of step 25047 is: -0.060682742486418695\n",
            "reward of step 25048 is: -0.12731448168397896\n",
            "reward of step 25049 is: -0.1752028457998192\n",
            "reward of step 25050 is: -0.053861640414145984\n",
            "reward of step 25051 is: -0.07659487585694758\n",
            "reward of step 25052 is: -0.08629061041262487\n",
            "reward of step 25053 is: -0.06477699556715666\n",
            "reward of step 25054 is: -0.09402079045734235\n",
            "reward of step 25055 is: -0.04432018712417862\n",
            "reward of step 25056 is: -0.12260862915250237\n",
            "reward of step 25057 is: -0.08023276480402897\n",
            "reward of step 25058 is: -0.040636720654346536\n",
            "reward of step 25059 is: -0.0752292715790821\n",
            "reward of step 25060 is: -0.031812278792998305\n",
            "reward of step 25061 is: -0.03088363021274443\n",
            "reward of step 25062 is: -0.052435207798403494\n",
            "reward of step 25063 is: -0.04991968795773405\n",
            "reward of step 25064 is: 0.02659958323833511\n",
            "reward of step 25065 is: -0.12099953429678922\n",
            "reward of step 25066 is: -0.08457251726803594\n",
            "reward of step 25067 is: -0.05708853474946529\n",
            "reward of step 25068 is: -0.05991529190542222\n",
            "reward of step 25069 is: -0.06966276420864137\n",
            "reward of step 25070 is: -0.04080334116447315\n",
            "reward of step 25071 is: -0.023421888889333387\n",
            "reward of step 25072 is: -0.08319282461725808\n",
            "reward of step 25073 is: -0.07531115150338352\n",
            "reward of step 25074 is: -0.03693347325066121\n",
            "reward of step 25075 is: -0.0762684963978334\n",
            "reward of step 25076 is: -0.05581283384658209\n",
            "reward of step 25077 is: -0.06369246033475418\n",
            "reward of step 25078 is: -0.07188294325886269\n",
            "reward of step 25079 is: -0.006212956679791826\n",
            "reward of step 25080 is: -0.07237323655324335\n",
            "reward of step 25081 is: -0.06473436054420145\n",
            "reward of step 25082 is: -0.10397337793452543\n",
            "reward of step 25083 is: -0.0419402398940244\n",
            "reward of step 25084 is: -0.024462248713241652\n",
            "reward of step 25085 is: -0.09643397003076815\n",
            "reward of step 25086 is: -0.09094155507155843\n",
            "reward of step 25087 is: -0.04515072017287802\n",
            "reward of step 25088 is: -0.05836526912159112\n",
            "reward of step 25089 is: -0.045805290388110986\n",
            "reward of step 25090 is: -0.08078013720925692\n",
            "reward of step 25091 is: -0.05641502331838344\n",
            "reward of step 25092 is: -0.10169248548755805\n",
            "reward of step 25093 is: -0.07236428864064404\n",
            "reward of step 25094 is: -0.13850251922286216\n",
            "reward of step 25095 is: -0.09930740805786409\n",
            "reward of step 25096 is: -0.035423167817082546\n",
            "reward of step 25097 is: -0.03303346825880227\n",
            "reward of step 25098 is: -0.020022415431503737\n",
            "reward of step 25099 is: -0.049180054138298956\n",
            "reward of step 25100 is: -0.08896486561980299\n",
            "reward of step 25101 is: -0.08700180534387503\n",
            "reward of step 25102 is: -3.654127663610396e-05\n",
            "reward of step 25103 is: -0.02385911516018835\n",
            "reward of step 25104 is: -0.005980041897565624\n",
            "reward of step 25105 is: -0.01278153099457291\n",
            "reward of step 25106 is: -0.15301369823028188\n",
            "reward of step 25107 is: -0.0059138821092629135\n",
            "reward of step 25108 is: -0.10738639622109869\n",
            "reward of step 25109 is: -0.12766992042473768\n",
            "reward of step 25110 is: -0.046750117973323735\n",
            "reward of step 25111 is: -0.09948307967077608\n",
            "reward of step 25112 is: -0.04030972185440518\n",
            "reward of step 25113 is: -0.032360827573560846\n",
            "reward of step 25114 is: -0.11670555039915831\n",
            "reward of step 25115 is: -0.09409647983798641\n",
            "reward of step 25116 is: -0.11955118690074518\n",
            "reward of step 25117 is: -0.05170933542582534\n",
            "reward of step 25118 is: -0.1290148017045457\n",
            "reward of step 25119 is: -0.011860411792194436\n",
            "reward of step 25120 is: -0.017727574429169723\n",
            "reward of step 25121 is: -0.04277244918533307\n",
            "reward of step 25122 is: -0.08880893018577307\n",
            "reward of step 25123 is: -0.10684247738990216\n",
            "reward of step 25124 is: -0.09813880263312713\n",
            "reward of step 25125 is: -0.07402634069479841\n",
            "reward of step 25126 is: -0.05378798483426794\n",
            "reward of step 25127 is: -0.1136785522377205\n",
            "reward of step 25128 is: -0.03825119857986181\n",
            "reward of step 25129 is: -0.09648289594863102\n",
            "reward of step 25130 is: -0.1442198226540996\n",
            "reward of step 25131 is: -0.10329223585448222\n",
            "reward of step 25132 is: -0.07351205368381886\n",
            "reward of step 25133 is: -0.08894900362009361\n",
            "reward of step 25134 is: -0.05729596941215975\n",
            "reward of step 25135 is: -0.14000099256054188\n",
            "reward of step 25136 is: -0.07071378851700338\n",
            "reward of step 25137 is: -0.019056749635468218\n",
            "reward of step 25138 is: -0.04732189322191571\n",
            "reward of step 25139 is: -0.03620536126601814\n",
            "reward of step 25140 is: -0.055436173730625704\n",
            "reward of step 25141 is: -0.008631712302920036\n",
            "reward of step 25142 is: -0.10274006113141443\n",
            "reward of step 25143 is: -0.06886384425523995\n",
            "reward of step 25144 is: -0.09743030638577554\n",
            "reward of step 25145 is: -0.0031108865213449466\n",
            "reward of step 25146 is: -0.061209306682158404\n",
            "reward of step 25147 is: -0.040839008275915134\n",
            "reward of step 25148 is: -0.06845600750308722\n",
            "reward of step 25149 is: -0.08252669264980939\n",
            "reward of step 25150 is: -0.11399824055116636\n",
            "reward of step 25151 is: -0.0713093838278368\n",
            "reward of step 25152 is: -0.08113251126256005\n",
            "reward of step 25153 is: -0.07525864199715604\n",
            "reward of step 25154 is: -0.08919457187206037\n",
            "reward of step 25155 is: -0.03208255146162731\n",
            "reward of step 25156 is: -0.042833559812840205\n",
            "reward of step 25157 is: -0.13508344073204592\n",
            "reward of step 25158 is: 0.0008974873436499164\n",
            "reward of step 25159 is: -0.04115392793522654\n",
            "reward of step 25160 is: -0.0518905849292991\n",
            "reward of step 25161 is: -0.06040781236462389\n",
            "reward of step 25162 is: -0.028894058574575165\n",
            "reward of step 25163 is: -0.10178601125710418\n",
            "reward of step 25164 is: -0.029471565671910782\n",
            "reward of step 25165 is: -0.08588441138959368\n",
            "reward of step 25166 is: -0.06503511445314958\n",
            "reward of step 25167 is: -0.10598320292300589\n",
            "reward of step 25168 is: -0.1259742818587456\n",
            "reward of step 25169 is: -0.07383863623881426\n",
            "reward of step 25170 is: -0.08071664662402211\n",
            "reward of step 25171 is: -0.03255833844129463\n",
            "reward of step 25172 is: -0.08134431463262304\n",
            "reward of step 25173 is: -0.04749483760964368\n",
            "reward of step 25174 is: 0.0015212789586912523\n",
            "reward of step 25175 is: -0.05567552732799819\n",
            "reward of step 25176 is: -0.1324631153689949\n",
            "reward of step 25177 is: -0.11515213729026075\n",
            "reward of step 25178 is: -0.09221117700621828\n",
            "reward of step 25179 is: -0.08781899729632858\n",
            "reward of step 25180 is: -0.08959306472934991\n",
            "reward of step 25181 is: -0.06876267710411543\n",
            "reward of step 25182 is: -0.03221236204530864\n",
            "reward of step 25183 is: -0.1511486903352649\n",
            "reward of step 25184 is: -0.033052674938507054\n",
            "reward of step 25185 is: -0.08357899054977802\n",
            "reward of step 25186 is: -0.08864892547254\n",
            "reward of step 25187 is: -0.09503675175155368\n",
            "reward of step 25188 is: -0.08631800111430143\n",
            "reward of step 25189 is: -0.10467621395950066\n",
            "reward of step 25190 is: -0.023242368528245017\n",
            "reward of step 25191 is: -0.06442423053039681\n",
            "reward of step 25192 is: -0.07319411324029623\n",
            "reward of step 25193 is: -0.009387139981064108\n",
            "reward of step 25194 is: -0.029094066492087456\n",
            "reward of step 25195 is: -0.07337137014794237\n",
            "reward of step 25196 is: -0.048839590167718216\n",
            "reward of step 25197 is: -0.07597514076763234\n",
            "reward of step 25198 is: -0.1108607094814229\n",
            "reward of step 25199 is: -0.10645678329661534\n",
            "reward of step 25200 is: 0.009503013567260465\n",
            "reward of step 25201 is: -0.05300370422976752\n",
            "reward of step 25202 is: -0.08465638933210318\n",
            "reward of step 25203 is: -0.16096053044295966\n",
            "reward of step 25204 is: -0.10800244358960376\n",
            "reward of step 25205 is: -0.024967561675751315\n",
            "reward of step 25206 is: -0.08669304964930868\n",
            "reward of step 25207 is: -0.13466050460603318\n",
            "reward of step 25208 is: -0.01711535728870961\n",
            "reward of step 25209 is: -0.07411114289773191\n",
            "reward of step 25210 is: -0.1314448394042328\n",
            "reward of step 25211 is: -0.12719306578873524\n",
            "reward of step 25212 is: -0.08768167453750753\n",
            "reward of step 25213 is: -0.11337003582571792\n",
            "reward of step 25214 is: -0.055657535180498874\n",
            "reward of step 25215 is: -0.13814417184678296\n",
            "reward of step 25216 is: -0.07160313532909868\n",
            "reward of step 25217 is: -0.10808196290018712\n",
            "reward of step 25218 is: -0.09511658394957045\n",
            "reward of step 25219 is: -0.04471469360010427\n",
            "reward of step 25220 is: -0.0699698297367104\n",
            "reward of step 25221 is: -0.12553949630713435\n",
            "reward of step 25222 is: -0.0499765610886308\n",
            "reward of step 25223 is: -0.08089671112311658\n",
            "reward of step 25224 is: -0.06011187875461954\n",
            "reward of step 25225 is: -0.004440448156279642\n",
            "reward of step 25226 is: -0.09843720609676665\n",
            "reward of step 25227 is: -0.07527884209562108\n",
            "reward of step 25228 is: -0.0968908040821933\n",
            "reward of step 25229 is: -0.09150040086393019\n",
            "reward of step 25230 is: -0.035275274654497646\n",
            "reward of step 25231 is: -0.1136418285274784\n",
            "reward of step 25232 is: -0.028397406522615243\n",
            "reward of step 25233 is: -0.06940194957114454\n",
            "reward of step 25234 is: -0.07681023237559803\n",
            "reward of step 25235 is: -0.13232452154963703\n",
            "reward of step 25236 is: -0.05582899350706205\n",
            "reward of step 25237 is: -0.06531496700448847\n",
            "reward of step 25238 is: -0.11074385029416\n",
            "reward of step 25239 is: -0.11322675818339634\n",
            "reward of step 25240 is: -0.05176282005814692\n",
            "reward of step 25241 is: -0.04519940219060681\n",
            "reward of step 25242 is: -0.08880162433079009\n",
            "reward of step 25243 is: -0.08343708030529551\n",
            "reward of step 25244 is: -0.05741427280362765\n",
            "reward of step 25245 is: -0.04488534676269129\n",
            "reward of step 25246 is: -0.07353104507787755\n",
            "reward of step 25247 is: -0.08229799826053863\n",
            "reward of step 25248 is: -0.07643409811657831\n",
            "reward of step 25249 is: -0.13833298806953143\n",
            "reward of step 25250 is: -0.043377658002453745\n",
            "reward of step 25251 is: -0.14168554895842345\n",
            "reward of step 25252 is: -0.08003152172201211\n",
            "reward of step 25253 is: 0.006607865228977494\n",
            "reward of step 25254 is: -0.16332545514511765\n",
            "reward of step 25255 is: -0.12488138657659031\n",
            "reward of step 25256 is: -0.12360553176463018\n",
            "reward of step 25257 is: -0.08480084053612424\n",
            "reward of step 25258 is: -0.1052854842635329\n",
            "reward of step 25259 is: -0.02454415356855888\n",
            "reward of step 25260 is: -0.14003589753448287\n",
            "reward of step 25261 is: -0.04734411262130489\n",
            "reward of step 25262 is: -0.09521794171323139\n",
            "reward of step 25263 is: -0.03553327668801387\n",
            "reward of step 25264 is: -0.02664591701193675\n",
            "reward of step 25265 is: -0.01157038740997618\n",
            "reward of step 25266 is: 0.0002460033299150899\n",
            "reward of step 25267 is: -0.07448926158923341\n",
            "reward of step 25268 is: -0.048719741172824516\n",
            "reward of step 25269 is: -0.09048425691047779\n",
            "reward of step 25270 is: -0.048877703727073785\n",
            "reward of step 25271 is: -0.034069244680189126\n",
            "reward of step 25272 is: -0.039097006587123895\n",
            "reward of step 25273 is: -0.08048092968046305\n",
            "reward of step 25274 is: -0.08147835330142594\n",
            "reward of step 25275 is: -0.06946393887400248\n",
            "reward of step 25276 is: -0.06080159235791038\n",
            "reward of step 25277 is: -0.12158935068464227\n",
            "reward of step 25278 is: -0.055402988365066186\n",
            "reward of step 25279 is: -0.14621819822099968\n",
            "reward of step 25280 is: -0.06565700987693956\n",
            "reward of step 25281 is: -0.1082326798026606\n",
            "reward of step 25282 is: -0.0018766463359793928\n",
            "reward of step 25283 is: -0.1508057179938357\n",
            "reward of step 25284 is: -0.02823965363384351\n",
            "reward of step 25285 is: -0.07050464538453016\n",
            "reward of step 25286 is: -0.12738794478943105\n",
            "reward of step 25287 is: 0.002331960278506018\n",
            "reward of step 25288 is: -0.061534481697341326\n",
            "reward of step 25289 is: -0.14806825950120905\n",
            "reward of step 25290 is: -0.11384351420203376\n",
            "reward of step 25291 is: -0.04813327563267167\n",
            "reward of step 25292 is: -0.13012469439335483\n",
            "reward of step 25293 is: -0.05987982628642874\n",
            "reward of step 25294 is: -0.10995125600613209\n",
            "reward of step 25295 is: -0.1450668724852\n",
            "reward of step 25296 is: -0.061462675269317146\n",
            "reward of step 25297 is: -0.12359013186945289\n",
            "reward of step 25298 is: -0.04257576698026189\n",
            "reward of step 25299 is: -0.026717544408233818\n",
            "reward of step 25300 is: -0.07326678884930005\n",
            "reward of step 25301 is: -0.03903416300194196\n",
            "reward of step 25302 is: -0.07932271729948115\n",
            "reward of step 25303 is: -0.07097542937238921\n",
            "reward of step 25304 is: -0.085728798474897\n",
            "reward of step 25305 is: -0.014169154895936176\n",
            "reward of step 25306 is: -0.10776116051569073\n",
            "reward of step 25307 is: -0.10077771111839351\n",
            "reward of step 25308 is: 0.00993352129258629\n",
            "reward of step 25309 is: -0.10749448641017934\n",
            "reward of step 25310 is: -0.010043482858418162\n",
            "reward of step 25311 is: -0.02387937892530989\n",
            "reward of step 25312 is: -0.028035842585057758\n",
            "reward of step 25313 is: -0.14827889186780996\n",
            "reward of step 25314 is: -0.13933688965591162\n",
            "reward of step 25315 is: -0.027794410854769636\n",
            "reward of step 25316 is: -0.0442638100220023\n",
            "reward of step 25317 is: -0.026484384762586632\n",
            "reward of step 25318 is: -0.07611445358022317\n",
            "reward of step 25319 is: -0.06316574962144372\n",
            "reward of step 25320 is: -0.08191533158582076\n",
            "reward of step 25321 is: -0.06228988890483289\n",
            "reward of step 25322 is: -0.14748704476819807\n",
            "reward of step 25323 is: -0.034075152679185816\n",
            "reward of step 25324 is: -0.10843251864052683\n",
            "reward of step 25325 is: -0.11653998365130624\n",
            "reward of step 25326 is: -0.06916663188939054\n",
            "reward of step 25327 is: -0.11779402898505742\n",
            "reward of step 25328 is: -0.10262707272672911\n",
            "reward of step 25329 is: -0.0773829137339842\n",
            "reward of step 25330 is: -0.1255002130658388\n",
            "reward of step 25331 is: -0.11636518104101445\n",
            "reward of step 25332 is: -0.03361121537597778\n",
            "reward of step 25333 is: -0.09445923878364726\n",
            "reward of step 25334 is: -0.02025786574809796\n",
            "reward of step 25335 is: -0.10724155178923767\n",
            "reward of step 25336 is: -0.07253349817460797\n",
            "reward of step 25337 is: -0.03841484250404492\n",
            "reward of step 25338 is: 0.005367619638641852\n",
            "reward of step 25339 is: -0.1055283496437549\n",
            "reward of step 25340 is: -0.044633873464472784\n",
            "reward of step 25341 is: -0.04032819809209176\n",
            "reward of step 25342 is: -0.06304745841310289\n",
            "reward of step 25343 is: -0.0696449301641181\n",
            "reward of step 25344 is: -0.06850279968775641\n",
            "reward of step 25345 is: -0.09921193675799078\n",
            "reward of step 25346 is: -0.059479962527280605\n",
            "reward of step 25347 is: -0.0460579345382941\n",
            "reward of step 25348 is: -0.10094296816579751\n",
            "reward of step 25349 is: -0.0077677679303886915\n",
            "reward of step 25350 is: -0.018896652834105243\n",
            "reward of step 25351 is: -0.005138648057622541\n",
            "reward of step 25352 is: -0.04982926024123624\n",
            "reward of step 25353 is: -0.07379642319164892\n",
            "reward of step 25354 is: -0.10712096078112499\n",
            "reward of step 25355 is: -0.042382940283418824\n",
            "reward of step 25356 is: -0.016257369461103455\n",
            "reward of step 25357 is: -0.053323304480431766\n",
            "reward of step 25358 is: -0.07797883632579106\n",
            "reward of step 25359 is: -0.044619223402785346\n",
            "reward of step 25360 is: -0.11396889232920604\n",
            "reward of step 25361 is: -0.09603429823105047\n",
            "reward of step 25362 is: -0.04804153761149099\n",
            "reward of step 25363 is: -0.09535356962969499\n",
            "reward of step 25364 is: -0.10635000754607693\n",
            "reward of step 25365 is: -0.14578640442804125\n",
            "reward of step 25366 is: -0.13054348501586888\n",
            "reward of step 25367 is: -0.03145614854598866\n",
            "reward of step 25368 is: -0.13959704119145033\n",
            "reward of step 25369 is: -0.11412290880122555\n",
            "reward of step 25370 is: -0.06276309617740372\n",
            "reward of step 25371 is: -0.10848302870365067\n",
            "reward of step 25372 is: -0.04812204534775211\n",
            "reward of step 25373 is: -0.13467617146192346\n",
            "reward of step 25374 is: -0.036053290765931556\n",
            "reward of step 25375 is: -0.08010227243732038\n",
            "reward of step 25376 is: -0.04207150985156993\n",
            "reward of step 25377 is: -0.05266961600864151\n",
            "reward of step 25378 is: -0.1026844168190928\n",
            "reward of step 25379 is: -0.07672402830115777\n",
            "reward of step 25380 is: -0.06761583315986075\n",
            "reward of step 25381 is: -0.10718224844664548\n",
            "reward of step 25382 is: -0.12677795472982079\n",
            "reward of step 25383 is: -0.07707752541815438\n",
            "reward of step 25384 is: -0.07756225933553262\n",
            "reward of step 25385 is: -0.01396592117183737\n",
            "reward of step 25386 is: -0.08536091533834334\n",
            "reward of step 25387 is: -0.05168148831500019\n",
            "reward of step 25388 is: -0.09229637692496784\n",
            "reward of step 25389 is: -0.05839380494386048\n",
            "reward of step 25390 is: -0.12233124366309944\n",
            "reward of step 25391 is: -0.03681840958537308\n",
            "reward of step 25392 is: -0.07898935678652885\n",
            "reward of step 25393 is: -0.18543027810596224\n",
            "reward of step 25394 is: -0.0626724534627584\n",
            "reward of step 25395 is: -0.004532129332728285\n",
            "reward of step 25396 is: -0.05407003924639697\n",
            "reward of step 25397 is: -0.07606354080778754\n",
            "reward of step 25398 is: -0.06812838493565021\n",
            "reward of step 25399 is: -0.1070994175174621\n",
            "reward of step 25400 is: -0.07384136114622075\n",
            "reward of step 25401 is: -0.008990034138150582\n",
            "reward of step 25402 is: -0.09101491724838684\n",
            "reward of step 25403 is: -0.09454874421030424\n",
            "reward of step 25404 is: -0.0452736069481533\n",
            "reward of step 25405 is: -0.05246345319706103\n",
            "reward of step 25406 is: -0.11661738211061179\n",
            "reward of step 25407 is: -0.07841631036208363\n",
            "reward of step 25408 is: -0.04970864734459357\n",
            "reward of step 25409 is: -0.044854271646552646\n",
            "reward of step 25410 is: -0.06950022084742324\n",
            "reward of step 25411 is: -0.04718400681311763\n",
            "reward of step 25412 is: -0.0859079570104514\n",
            "reward of step 25413 is: -0.08706349261728952\n",
            "reward of step 25414 is: -0.09248953899261458\n",
            "reward of step 25415 is: -0.03181978541448749\n",
            "reward of step 25416 is: -0.11979438021025679\n",
            "reward of step 25417 is: -0.08330969112283593\n",
            "reward of step 25418 is: 0.0026498174516651485\n",
            "reward of step 25419 is: -0.06069929457680179\n",
            "reward of step 25420 is: -0.11819201716299987\n",
            "reward of step 25421 is: -0.054989509788103685\n",
            "reward of step 25422 is: -0.13026783944403653\n",
            "reward of step 25423 is: -0.1171600757497201\n",
            "reward of step 25424 is: -0.02375492128696266\n",
            "reward of step 25425 is: -0.028502965695570426\n",
            "reward of step 25426 is: -0.05276647093368625\n",
            "reward of step 25427 is: -0.11985994300655656\n",
            "reward of step 25428 is: -0.08171138777758102\n",
            "reward of step 25429 is: -0.08103854440392189\n",
            "reward of step 25430 is: -0.06910337101327568\n",
            "reward of step 25431 is: -0.015344426470902173\n",
            "reward of step 25432 is: -0.10687810894514327\n",
            "reward of step 25433 is: -0.044284764513062624\n",
            "reward of step 25434 is: -0.06906525816367437\n",
            "reward of step 25435 is: -0.0895399297394599\n",
            "reward of step 25436 is: -0.128582793130533\n",
            "reward of step 25437 is: -0.10698636800592243\n",
            "reward of step 25438 is: -0.04769506007443791\n",
            "reward of step 25439 is: -0.07417292646591533\n",
            "reward of step 25440 is: -0.11691014838896185\n",
            "reward of step 25441 is: -0.03849141884420981\n",
            "reward of step 25442 is: -0.10143309405877365\n",
            "reward of step 25443 is: -0.03625215942032556\n",
            "reward of step 25444 is: -0.07938072019193032\n",
            "reward of step 25445 is: 0.024860082259299254\n",
            "reward of step 25446 is: -0.011158265614630625\n",
            "reward of step 25447 is: -0.09862466756979049\n",
            "reward of step 25448 is: -0.06938562639158119\n",
            "reward of step 25449 is: -0.086037808538725\n",
            "reward of step 25450 is: -0.09634861404421857\n",
            "reward of step 25451 is: -0.0694392066363716\n",
            "reward of step 25452 is: -0.0739123742632134\n",
            "reward of step 25453 is: 0.02225568572704273\n",
            "reward of step 25454 is: -0.049139432001609196\n",
            "reward of step 25455 is: -0.16047978501624305\n",
            "reward of step 25456 is: -0.09704383762856161\n",
            "reward of step 25457 is: -0.07809449353351683\n",
            "reward of step 25458 is: -0.034176665335805545\n",
            "reward of step 25459 is: -0.06366965024994542\n",
            "reward of step 25460 is: -0.15839572759732823\n",
            "reward of step 25461 is: -0.1330204269555334\n",
            "reward of step 25462 is: -0.09707330997762842\n",
            "reward of step 25463 is: -0.03155244950292868\n",
            "reward of step 25464 is: -0.13435588389235686\n",
            "reward of step 25465 is: -0.050802598172895896\n",
            "reward of step 25466 is: -0.058274406008374524\n",
            "reward of step 25467 is: -0.08402911462658247\n",
            "reward of step 25468 is: -0.06335349029822557\n",
            "reward of step 25469 is: -0.1367094134711997\n",
            "reward of step 25470 is: -0.07534800255914176\n",
            "reward of step 25471 is: -0.09383634578319688\n",
            "reward of step 25472 is: -0.028282201811043683\n",
            "reward of step 25473 is: -0.018474983446476467\n",
            "reward of step 25474 is: -0.05284520174342955\n",
            "reward of step 25475 is: -0.0712551620327283\n",
            "reward of step 25476 is: 0.017438019837190732\n",
            "reward of step 25477 is: -0.07983286920298094\n",
            "reward of step 25478 is: -0.10486201100320147\n",
            "reward of step 25479 is: -0.021278015288583974\n",
            "reward of step 25480 is: -0.0030819909826896863\n",
            "reward of step 25481 is: -0.1065475746993656\n",
            "reward of step 25482 is: -0.06882489900671285\n",
            "reward of step 25483 is: -0.05364298366896292\n",
            "reward of step 25484 is: -0.07486712356760583\n",
            "reward of step 25485 is: -0.07358133271774492\n",
            "reward of step 25486 is: -0.04888252650556779\n",
            "reward of step 25487 is: -0.1089630591290972\n",
            "reward of step 25488 is: -0.032044317214708795\n",
            "reward of step 25489 is: -0.06258923839575736\n",
            "reward of step 25490 is: -0.04479469411438486\n",
            "reward of step 25491 is: -0.028350870735182943\n",
            "reward of step 25492 is: -0.08863523570646581\n",
            "reward of step 25493 is: -0.11145009426156582\n",
            "reward of step 25494 is: -0.10368738213493167\n",
            "reward of step 25495 is: -0.05100508317656183\n",
            "reward of step 25496 is: -0.06584168378242727\n",
            "reward of step 25497 is: -0.021584864493381373\n",
            "reward of step 25498 is: -0.05024123297131944\n",
            "reward of step 25499 is: -0.04294749155106736\n",
            "reward of step 25500 is: -0.0747061286243671\n",
            "reward of step 25501 is: -0.10237173485080442\n",
            "reward of step 25502 is: -0.026852629339995437\n",
            "reward of step 25503 is: -0.053056856183609\n",
            "reward of step 25504 is: -0.038628842162192756\n",
            "reward of step 25505 is: -0.08583812005549363\n",
            "reward of step 25506 is: -0.06335624752491176\n",
            "reward of step 25507 is: -0.06291229251074737\n",
            "reward of step 25508 is: -0.11572685244398073\n",
            "reward of step 25509 is: -0.01591597704524894\n",
            "reward of step 25510 is: -0.041392790032414584\n",
            "reward of step 25511 is: -0.09208098214764926\n",
            "reward of step 25512 is: -0.09236940682249417\n",
            "reward of step 25513 is: -0.09221917988275707\n",
            "reward of step 25514 is: -0.04634111276233088\n",
            "reward of step 25515 is: -0.06086143110363118\n",
            "reward of step 25516 is: -0.04109096979497262\n",
            "reward of step 25517 is: -0.08993074171612814\n",
            "reward of step 25518 is: -0.049029974795250286\n",
            "reward of step 25519 is: -0.09909481083155958\n",
            "reward of step 25520 is: -0.03862006432432585\n",
            "reward of step 25521 is: -0.14618652397050802\n",
            "reward of step 25522 is: -0.05282600147810257\n",
            "reward of step 25523 is: -0.04677035175040822\n",
            "reward of step 25524 is: -0.038908235030393534\n",
            "reward of step 25525 is: -0.0449651160910538\n",
            "reward of step 25526 is: -0.03270671873666575\n",
            "reward of step 25527 is: -0.10685655216033763\n",
            "reward of step 25528 is: -0.06573341001425659\n",
            "reward of step 25529 is: -0.11675100318580034\n",
            "reward of step 25530 is: -0.05459776307580044\n",
            "reward of step 25531 is: -0.04482856360615284\n",
            "reward of step 25532 is: -0.07174901375959575\n",
            "reward of step 25533 is: -0.1383737719626068\n",
            "reward of step 25534 is: -0.034343271744681325\n",
            "reward of step 25535 is: -0.03482501217680556\n",
            "reward of step 25536 is: -0.1230805137050166\n",
            "reward of step 25537 is: -0.11687083029921685\n",
            "reward of step 25538 is: -0.036962305958967545\n",
            "reward of step 25539 is: -0.12956984041360586\n",
            "reward of step 25540 is: -0.11889970363133051\n",
            "reward of step 25541 is: -0.14815438954777116\n",
            "reward of step 25542 is: -0.06044650743403934\n",
            "reward of step 25543 is: -0.0030310022809607595\n",
            "reward of step 25544 is: -0.08526998374013561\n",
            "reward of step 25545 is: -0.05177049988674953\n",
            "reward of step 25546 is: -0.09695805995165963\n",
            "reward of step 25547 is: -0.11039577028807768\n",
            "reward of step 25548 is: -0.12786974488967873\n",
            "reward of step 25549 is: -0.10386270064477032\n",
            "reward of step 25550 is: -0.04934031196637101\n",
            "reward of step 25551 is: -0.13406236432511942\n",
            "reward of step 25552 is: 0.00461397476300196\n",
            "reward of step 25553 is: -0.05981173286094965\n",
            "reward of step 25554 is: -0.06100674324973343\n",
            "reward of step 25555 is: -0.08899708998331446\n",
            "reward of step 25556 is: -0.11329044697190538\n",
            "reward of step 25557 is: -0.1149622982924231\n",
            "reward of step 25558 is: -0.02768346576724212\n",
            "reward of step 25559 is: -0.08513865368324824\n",
            "reward of step 25560 is: -0.05585415473758104\n",
            "reward of step 25561 is: -0.10645942037093659\n",
            "reward of step 25562 is: -0.06705805622760108\n",
            "reward of step 25563 is: -0.047826115747046316\n",
            "reward of step 25564 is: -0.02149578743078151\n",
            "reward of step 25565 is: -0.04584691446555256\n",
            "reward of step 25566 is: -0.14576651336385693\n",
            "reward of step 25567 is: -0.03782649483643341\n",
            "reward of step 25568 is: -0.08155825740868206\n",
            "reward of step 25569 is: -0.10069294632433756\n",
            "reward of step 25570 is: -0.10006762905080013\n",
            "reward of step 25571 is: -0.07809613106984004\n",
            "reward of step 25572 is: -0.09515562945792844\n",
            "reward of step 25573 is: -0.06966644162205082\n",
            "reward of step 25574 is: -0.07466659689904442\n",
            "reward of step 25575 is: -0.14362539427235632\n",
            "reward of step 25576 is: -0.028355001037578487\n",
            "reward of step 25577 is: 0.01563298677068159\n",
            "reward of step 25578 is: -0.10159753299666596\n",
            "reward of step 25579 is: -0.13219121116149346\n",
            "reward of step 25580 is: -0.11813939114340577\n",
            "reward of step 25581 is: -0.12637927456504083\n",
            "reward of step 25582 is: -0.05140179930975852\n",
            "reward of step 25583 is: 0.0011084030957566604\n",
            "reward of step 25584 is: -0.06529681391914133\n",
            "reward of step 25585 is: -0.04933524850710558\n",
            "reward of step 25586 is: -0.02974572032768419\n",
            "reward of step 25587 is: -0.04618489972342288\n",
            "reward of step 25588 is: -0.09169783771799056\n",
            "reward of step 25589 is: 0.018247756693477224\n",
            "reward of step 25590 is: -0.05039478050615975\n",
            "reward of step 25591 is: -0.09033531439351905\n",
            "reward of step 25592 is: -0.103270950289863\n",
            "reward of step 25593 is: -0.09577707164524674\n",
            "reward of step 25594 is: -0.036960396447716426\n",
            "reward of step 25595 is: -0.06465785956001746\n",
            "reward of step 25596 is: -0.020231073949600664\n",
            "reward of step 25597 is: -0.011766962722627339\n",
            "reward of step 25598 is: -0.06095044514502446\n",
            "reward of step 25599 is: -0.02813508956794719\n",
            "reward of step 25600 is: -0.097331296836767\n",
            "reward of step 25601 is: -0.016917908307056417\n",
            "reward of step 25602 is: -0.10356290928736256\n",
            "reward of step 25603 is: -0.05749611100601726\n",
            "reward of step 25604 is: -0.1317032252065623\n",
            "reward of step 25605 is: -0.09212326819151151\n",
            "reward of step 25606 is: -0.0671394640888151\n",
            "reward of step 25607 is: -0.03383127984757828\n",
            "reward of step 25608 is: -0.11899706254914011\n",
            "reward of step 25609 is: -0.08975022193104443\n",
            "reward of step 25610 is: -0.05270938617852683\n",
            "reward of step 25611 is: -0.003857841691831343\n",
            "reward of step 25612 is: 0.021982630549288285\n",
            "reward of step 25613 is: -0.06111133169368144\n",
            "reward of step 25614 is: -0.05135819373843875\n",
            "reward of step 25615 is: -0.06097519118377248\n",
            "reward of step 25616 is: -0.0312672452093844\n",
            "reward of step 25617 is: -0.11038741001868502\n",
            "reward of step 25618 is: -0.058579333640052966\n",
            "reward of step 25619 is: -0.08427384398272164\n",
            "reward of step 25620 is: -0.04969884447439232\n",
            "reward of step 25621 is: -0.07982084678475199\n",
            "reward of step 25622 is: -0.09473648147081337\n",
            "reward of step 25623 is: -0.068765777241936\n",
            "reward of step 25624 is: -0.037541144437835294\n",
            "reward of step 25625 is: -0.05155259451166305\n",
            "reward of step 25626 is: -0.05561138698576429\n",
            "reward of step 25627 is: -0.05407867283886825\n",
            "reward of step 25628 is: -0.16700621979415275\n",
            "reward of step 25629 is: -0.058337603080700684\n",
            "reward of step 25630 is: 0.0014330710881947928\n",
            "reward of step 25631 is: -0.10876306042525963\n",
            "reward of step 25632 is: -0.10330649831070882\n",
            "reward of step 25633 is: -0.09963581717983339\n",
            "reward of step 25634 is: -0.05774254660360045\n",
            "reward of step 25635 is: -0.03756872961614666\n",
            "reward of step 25636 is: -0.04274836252041947\n",
            "reward of step 25637 is: -0.15857433481223038\n",
            "reward of step 25638 is: -0.07545226476351141\n",
            "reward of step 25639 is: -0.1400663425336296\n",
            "reward of step 25640 is: -0.03945998712953758\n",
            "reward of step 25641 is: -0.07751952906896598\n",
            "reward of step 25642 is: -0.04880449210353044\n",
            "reward of step 25643 is: -0.14596147911298873\n",
            "reward of step 25644 is: -0.09772627907290021\n",
            "reward of step 25645 is: -0.015723845266613967\n",
            "reward of step 25646 is: -0.04687008885760746\n",
            "reward of step 25647 is: -0.03588638699458102\n",
            "reward of step 25648 is: -0.09632484924978868\n",
            "reward of step 25649 is: -0.0750339975704073\n",
            "reward of step 25650 is: -0.013125559728550784\n",
            "reward of step 25651 is: -0.036873418750312026\n",
            "reward of step 25652 is: -0.12841332285195173\n",
            "reward of step 25653 is: -0.06262078296363371\n",
            "reward of step 25654 is: -0.02275071477127044\n",
            "reward of step 25655 is: -0.07036080702378589\n",
            "reward of step 25656 is: -0.0517661109418871\n",
            "reward of step 25657 is: -0.1254130806619087\n",
            "reward of step 25658 is: -0.026582297915971997\n",
            "reward of step 25659 is: 0.007813066255605072\n",
            "reward of step 25660 is: -0.09296440958767882\n",
            "reward of step 25661 is: -0.11940729186370158\n",
            "reward of step 25662 is: -0.031099343204580077\n",
            "reward of step 25663 is: -0.09034470875046285\n",
            "reward of step 25664 is: -0.03808297034684882\n",
            "reward of step 25665 is: -0.030445504134241852\n",
            "reward of step 25666 is: -0.10664502115665131\n",
            "reward of step 25667 is: -0.06517055645772629\n",
            "reward of step 25668 is: -0.09858393063194282\n",
            "reward of step 25669 is: 0.0014776634120013954\n",
            "reward of step 25670 is: -0.07664496249901698\n",
            "reward of step 25671 is: 0.01856785408396555\n",
            "reward of step 25672 is: -0.09101727571655427\n",
            "reward of step 25673 is: -0.037815308152253424\n",
            "reward of step 25674 is: -0.05806678506387042\n",
            "reward of step 25675 is: -0.07086486550739213\n",
            "reward of step 25676 is: -0.05123763344519061\n",
            "reward of step 25677 is: -0.07585445354112641\n",
            "reward of step 25678 is: -0.0004989516579596565\n",
            "reward of step 25679 is: -0.07484192258860578\n",
            "reward of step 25680 is: -0.09362729876836828\n",
            "reward of step 25681 is: -0.11204794348899327\n",
            "reward of step 25682 is: -0.04222048816932722\n",
            "reward of step 25683 is: -0.1347954341323493\n",
            "reward of step 25684 is: -0.07254271355802677\n",
            "reward of step 25685 is: -0.04632447219784375\n",
            "reward of step 25686 is: -0.04924735591300322\n",
            "reward of step 25687 is: -0.07620281183705346\n",
            "reward of step 25688 is: -0.045716032357409286\n",
            "reward of step 25689 is: -0.047560798954346506\n",
            "reward of step 25690 is: -0.10049860548378942\n",
            "reward of step 25691 is: -0.031564149899697025\n",
            "reward of step 25692 is: -0.0232092185483862\n",
            "reward of step 25693 is: -0.06903370806413822\n",
            "reward of step 25694 is: -0.07728940842718535\n",
            "reward of step 25695 is: -0.11216949636735984\n",
            "reward of step 25696 is: -0.09902469027058425\n",
            "reward of step 25697 is: -0.06463796837250713\n",
            "reward of step 25698 is: -0.030829794214664252\n",
            "reward of step 25699 is: -0.012468003940495853\n",
            "reward of step 25700 is: 0.00024047883480493581\n",
            "reward of step 25701 is: -0.10043582667755624\n",
            "reward of step 25702 is: -0.12549093607442663\n",
            "reward of step 25703 is: -0.11359920865085593\n",
            "reward of step 25704 is: -0.059314422967606695\n",
            "reward of step 25705 is: -0.06341539624350001\n",
            "reward of step 25706 is: -0.008204049546849501\n",
            "reward of step 25707 is: -0.059947473985940136\n",
            "reward of step 25708 is: -0.04507197571730981\n",
            "reward of step 25709 is: -0.08682564773773693\n",
            "reward of step 25710 is: -0.07326627753156822\n",
            "reward of step 25711 is: -0.0553004272200901\n",
            "reward of step 25712 is: -0.060498830837538486\n",
            "reward of step 25713 is: -0.09225586787050188\n",
            "reward of step 25714 is: -0.05356044646766611\n",
            "reward of step 25715 is: -0.06893355258017309\n",
            "reward of step 25716 is: -0.08498659208991266\n",
            "reward of step 25717 is: -0.1513358777968944\n",
            "reward of step 25718 is: -0.06983367498451554\n",
            "reward of step 25719 is: -0.0005571381939396547\n",
            "reward of step 25720 is: -0.08579501237894505\n",
            "reward of step 25721 is: -0.0537620898897464\n",
            "reward of step 25722 is: -0.07315619106326854\n",
            "reward of step 25723 is: -0.04347361770498659\n",
            "reward of step 25724 is: -0.08661461167919504\n",
            "reward of step 25725 is: -0.10223843897043794\n",
            "reward of step 25726 is: -0.0748730289728412\n",
            "reward of step 25727 is: -0.05786393952460933\n",
            "reward of step 25728 is: -0.11719908580188942\n",
            "reward of step 25729 is: -0.08719956475850121\n",
            "reward of step 25730 is: -0.12495486855549376\n",
            "reward of step 25731 is: -0.053508188274416946\n",
            "reward of step 25732 is: -0.08989275262000718\n",
            "reward of step 25733 is: -0.12489986456693547\n",
            "reward of step 25734 is: -0.07292530597507552\n",
            "reward of step 25735 is: -0.11566089643262756\n",
            "reward of step 25736 is: -0.14827041695591048\n",
            "reward of step 25737 is: -0.12288805866472086\n",
            "reward of step 25738 is: -0.02338651194275776\n",
            "reward of step 25739 is: -0.06967523851281043\n",
            "reward of step 25740 is: -0.03693791534015711\n",
            "reward of step 25741 is: -0.10625956487318422\n",
            "reward of step 25742 is: -0.09256337107931845\n",
            "reward of step 25743 is: -0.015650023004138203\n",
            "reward of step 25744 is: -0.07040891409801175\n",
            "reward of step 25745 is: -0.14551656126507595\n",
            "reward of step 25746 is: -0.11617806558356991\n",
            "reward of step 25747 is: -0.09677469641693315\n",
            "reward of step 25748 is: 0.027691688601253817\n",
            "reward of step 25749 is: 0.00011700593691443828\n",
            "reward of step 25750 is: -0.029651486111791137\n",
            "reward of step 25751 is: -0.1195318553413719\n",
            "reward of step 25752 is: -0.03982156120812097\n",
            "reward of step 25753 is: -0.1393832414745526\n",
            "reward of step 25754 is: -0.08398861422123127\n",
            "reward of step 25755 is: -0.12448281875172218\n",
            "reward of step 25756 is: -0.06743050797881223\n",
            "reward of step 25757 is: -0.12846535114824076\n",
            "reward of step 25758 is: -0.052038054827872826\n",
            "reward of step 25759 is: -0.07470197583668936\n",
            "reward of step 25760 is: -0.03886923666594677\n",
            "reward of step 25761 is: -0.03965281433312917\n",
            "reward of step 25762 is: -0.13094424913478875\n",
            "reward of step 25763 is: -0.038237223618946214\n",
            "reward of step 25764 is: -0.07649609064599283\n",
            "reward of step 25765 is: -0.011002854899392012\n",
            "reward of step 25766 is: -0.03643369955025655\n",
            "reward of step 25767 is: -0.08064618154779624\n",
            "reward of step 25768 is: -0.10475867872679434\n",
            "reward of step 25769 is: -0.053150714773676166\n",
            "reward of step 25770 is: -0.05328279397879143\n",
            "reward of step 25771 is: -0.03737760853975547\n",
            "reward of step 25772 is: -0.06736162119738898\n",
            "reward of step 25773 is: -0.08369083567676405\n",
            "reward of step 25774 is: -0.03204429434932565\n",
            "reward of step 25775 is: -0.09800868927630124\n",
            "reward of step 25776 is: -0.06628525230321325\n",
            "reward of step 25777 is: -0.06230623413475389\n",
            "reward of step 25778 is: -0.03878147924102404\n",
            "reward of step 25779 is: -0.05484475998543903\n",
            "reward of step 25780 is: -0.09316794395523642\n",
            "reward of step 25781 is: -0.08231729499751506\n",
            "reward of step 25782 is: -0.04590181668836735\n",
            "reward of step 25783 is: -0.02780129327648362\n",
            "reward of step 25784 is: 0.0003131353304236528\n",
            "reward of step 25785 is: -0.04683902232165371\n",
            "reward of step 25786 is: 0.018464583880492547\n",
            "reward of step 25787 is: -0.0598608281831291\n",
            "reward of step 25788 is: -0.1004185134324933\n",
            "reward of step 25789 is: 0.0007419237696003611\n",
            "reward of step 25790 is: -0.1361655708861902\n",
            "reward of step 25791 is: -0.09465239674269255\n",
            "reward of step 25792 is: -0.07133697118869042\n",
            "reward of step 25793 is: -0.047151546668772526\n",
            "reward of step 25794 is: -0.13920847777902534\n",
            "reward of step 25795 is: 0.014202270740742762\n",
            "reward of step 25796 is: -0.0023497665828410286\n",
            "reward of step 25797 is: -0.09290101099689896\n",
            "reward of step 25798 is: -0.13107277633320935\n",
            "reward of step 25799 is: -0.09531609595941903\n",
            "reward of step 25800 is: -0.03299863640611145\n",
            "reward of step 25801 is: -0.07777715658701467\n",
            "reward of step 25802 is: -0.02229561751608211\n",
            "reward of step 25803 is: -0.08832656397448035\n",
            "reward of step 25804 is: -0.011713651753053833\n",
            "reward of step 25805 is: -0.0682940884285459\n",
            "reward of step 25806 is: -0.10296487013953748\n",
            "reward of step 25807 is: -0.10071194468067124\n",
            "reward of step 25808 is: -0.03339894531960619\n",
            "reward of step 25809 is: -0.029238451088289374\n",
            "reward of step 25810 is: -0.07141755361695656\n",
            "reward of step 25811 is: -0.06507172352136759\n",
            "reward of step 25812 is: -0.05165952012809538\n",
            "reward of step 25813 is: -0.05086045538919681\n",
            "reward of step 25814 is: -0.08899430429040223\n",
            "reward of step 25815 is: -0.07706773038493764\n",
            "reward of step 25816 is: -0.027881785945721882\n",
            "reward of step 25817 is: -0.03261121562159908\n",
            "reward of step 25818 is: -0.0808531033069595\n",
            "reward of step 25819 is: -0.060789537139962935\n",
            "reward of step 25820 is: -0.043573740654769266\n",
            "reward of step 25821 is: -0.12003105113833601\n",
            "reward of step 25822 is: -0.016232306233909033\n",
            "reward of step 25823 is: -0.09500075184000434\n",
            "reward of step 25824 is: -0.030763885250126854\n",
            "reward of step 25825 is: -0.06396617368513324\n",
            "reward of step 25826 is: -0.06604895202239303\n",
            "reward of step 25827 is: -0.05245327358930518\n",
            "reward of step 25828 is: -0.08376683540398966\n",
            "reward of step 25829 is: -0.14942749219335638\n",
            "reward of step 25830 is: -0.10487925376597351\n",
            "reward of step 25831 is: -0.0926464390810442\n",
            "reward of step 25832 is: -0.1263921402441237\n",
            "reward of step 25833 is: -0.09163676626475803\n",
            "reward of step 25834 is: -0.10801588055729872\n",
            "reward of step 25835 is: -0.0911128729070243\n",
            "reward of step 25836 is: -0.09569510959138017\n",
            "reward of step 25837 is: -0.12078728603709421\n",
            "reward of step 25838 is: -0.05634929764068353\n",
            "reward of step 25839 is: -0.12743950188017572\n",
            "reward of step 25840 is: -0.01644963740193095\n",
            "reward of step 25841 is: -0.09578734825658841\n",
            "reward of step 25842 is: -0.12176184604293117\n",
            "reward of step 25843 is: -0.1157517606664572\n",
            "reward of step 25844 is: -0.139976058608416\n",
            "reward of step 25845 is: -0.028410858958205498\n",
            "reward of step 25846 is: -0.0974223419815804\n",
            "reward of step 25847 is: -0.04391118644969161\n",
            "reward of step 25848 is: -0.002718930699780575\n",
            "reward of step 25849 is: -0.06312344123629554\n",
            "reward of step 25850 is: -0.04437355196156256\n",
            "reward of step 25851 is: -0.07436837920971595\n",
            "reward of step 25852 is: -0.08658864861249183\n",
            "reward of step 25853 is: -0.13818377789275083\n",
            "reward of step 25854 is: -0.04354141130413003\n",
            "reward of step 25855 is: -0.04683201684776217\n",
            "reward of step 25856 is: -0.07246388338121657\n",
            "reward of step 25857 is: -0.09627840394658693\n",
            "reward of step 25858 is: -0.06289135368785115\n",
            "reward of step 25859 is: -0.09057707526258818\n",
            "reward of step 25860 is: -0.1545145034226818\n",
            "reward of step 25861 is: -0.07121374631731936\n",
            "reward of step 25862 is: -0.04800212158457129\n",
            "reward of step 25863 is: -0.02657111970594761\n",
            "reward of step 25864 is: -0.0641003522518856\n",
            "reward of step 25865 is: -0.11014095713902261\n",
            "reward of step 25866 is: -0.03953776675347487\n",
            "reward of step 25867 is: -0.08514664833296659\n",
            "reward of step 25868 is: -0.10165058903801549\n",
            "reward of step 25869 is: -0.11231543333112715\n",
            "reward of step 25870 is: -0.08657031389428882\n",
            "reward of step 25871 is: -0.043028923386575046\n",
            "reward of step 25872 is: -0.1014816017108442\n",
            "reward of step 25873 is: -0.026513974378876703\n",
            "reward of step 25874 is: 0.0015256585665583966\n",
            "reward of step 25875 is: -0.11264985453470044\n",
            "reward of step 25876 is: -0.04902292204631031\n",
            "reward of step 25877 is: -0.06332602562923861\n",
            "reward of step 25878 is: -0.13512128035723114\n",
            "reward of step 25879 is: -0.10164874486276387\n",
            "reward of step 25880 is: -0.09619314158440517\n",
            "reward of step 25881 is: -0.10456660055774081\n",
            "reward of step 25882 is: -0.03841893962066645\n",
            "reward of step 25883 is: -0.06556594847123376\n",
            "reward of step 25884 is: -0.036482883252978326\n",
            "reward of step 25885 is: -0.042725554398307275\n",
            "reward of step 25886 is: -0.03915468964313662\n",
            "reward of step 25887 is: 0.002805658288568269\n",
            "reward of step 25888 is: -0.07390786804406724\n",
            "reward of step 25889 is: -0.10127849740669148\n",
            "reward of step 25890 is: -0.04330742218702932\n",
            "reward of step 25891 is: -0.04912671766706633\n",
            "reward of step 25892 is: -0.08176883444074856\n",
            "reward of step 25893 is: -0.00857275195534557\n",
            "reward of step 25894 is: -0.028852019900638126\n",
            "reward of step 25895 is: -0.09528160414221387\n",
            "reward of step 25896 is: -0.050515774778905786\n",
            "reward of step 25897 is: -0.04838651931484195\n",
            "reward of step 25898 is: -0.11295913266971502\n",
            "reward of step 25899 is: -0.06311552951277366\n",
            "reward of step 25900 is: -0.002493594280977507\n",
            "reward of step 25901 is: -0.04336579739080881\n",
            "reward of step 25902 is: -0.07079579757989296\n",
            "reward of step 25903 is: -0.05735595123928672\n",
            "reward of step 25904 is: -0.06812809664953845\n",
            "reward of step 25905 is: -0.07165844334561655\n",
            "reward of step 25906 is: -0.10273910768777783\n",
            "reward of step 25907 is: -0.0825115127993502\n",
            "reward of step 25908 is: -0.0663381356906565\n",
            "reward of step 25909 is: -0.07331744381816974\n",
            "reward of step 25910 is: -0.059308086225664525\n",
            "reward of step 25911 is: -0.022597796816307247\n",
            "reward of step 25912 is: 0.002612584246265648\n",
            "reward of step 25913 is: -0.0450429073471611\n",
            "reward of step 25914 is: -0.07825093331827604\n",
            "reward of step 25915 is: -0.11361323995345207\n",
            "reward of step 25916 is: -0.08662748767005068\n",
            "reward of step 25917 is: -0.17375149662259604\n",
            "reward of step 25918 is: -0.09353643851219262\n",
            "reward of step 25919 is: -0.051116715080989694\n",
            "reward of step 25920 is: -0.07882587045033884\n",
            "reward of step 25921 is: -0.025939046814482447\n",
            "reward of step 25922 is: -0.10619242167544996\n",
            "reward of step 25923 is: -0.060699476733029556\n",
            "reward of step 25924 is: -0.06397248578323556\n",
            "reward of step 25925 is: -0.11546002685272161\n",
            "reward of step 25926 is: -0.05319048298950957\n",
            "reward of step 25927 is: -0.028903598881287906\n",
            "reward of step 25928 is: -0.053749520130949135\n",
            "reward of step 25929 is: -0.07969587576559345\n",
            "reward of step 25930 is: -0.0386708272494134\n",
            "reward of step 25931 is: -0.0641925396835179\n",
            "reward of step 25932 is: -0.0758638285405242\n",
            "reward of step 25933 is: -0.023094131403051166\n",
            "reward of step 25934 is: -0.08444821612509812\n",
            "reward of step 25935 is: -0.11931969783119933\n",
            "reward of step 25936 is: -0.0959413631581062\n",
            "reward of step 25937 is: -0.06193020683090389\n",
            "reward of step 25938 is: -0.0839835325966376\n",
            "reward of step 25939 is: -0.040216837593206156\n",
            "reward of step 25940 is: -0.08116332210155741\n",
            "reward of step 25941 is: -0.11600894677420237\n",
            "reward of step 25942 is: -0.06744130405692605\n",
            "reward of step 25943 is: -0.034941184345475995\n",
            "reward of step 25944 is: -0.021182351768856633\n",
            "reward of step 25945 is: -0.08654273155782088\n",
            "reward of step 25946 is: 0.002267958457409347\n",
            "reward of step 25947 is: -0.04975255353874475\n",
            "reward of step 25948 is: -0.13144660414018805\n",
            "reward of step 25949 is: -0.1646420616709552\n",
            "reward of step 25950 is: -0.1314575455057555\n",
            "reward of step 25951 is: -0.125080367719271\n",
            "reward of step 25952 is: -0.06291588014560989\n",
            "reward of step 25953 is: -0.10313308147983058\n",
            "reward of step 25954 is: -0.1210059710664726\n",
            "reward of step 25955 is: -0.10745012000475795\n",
            "reward of step 25956 is: -0.08678691478153266\n",
            "reward of step 25957 is: -0.03651334196651068\n",
            "reward of step 25958 is: -0.054943561645486416\n",
            "reward of step 25959 is: -0.01514161336562736\n",
            "reward of step 25960 is: -0.09747596933076519\n",
            "reward of step 25961 is: -0.11490357222610137\n",
            "reward of step 25962 is: -0.01878945618300032\n",
            "reward of step 25963 is: -0.09507304434046016\n",
            "reward of step 25964 is: -0.08688364315893227\n",
            "reward of step 25965 is: -0.04499723862372462\n",
            "reward of step 25966 is: -0.038996415722467814\n",
            "reward of step 25967 is: -0.045817913789172326\n",
            "reward of step 25968 is: -0.09908334220553627\n",
            "reward of step 25969 is: -0.019270828483634594\n",
            "reward of step 25970 is: -0.09400589778172974\n",
            "reward of step 25971 is: -0.10887396082984291\n",
            "reward of step 25972 is: -0.08121289743222082\n",
            "reward of step 25973 is: -0.02116694412940512\n",
            "reward of step 25974 is: -0.1050339335618854\n",
            "reward of step 25975 is: -0.11505843603716914\n",
            "reward of step 25976 is: -0.05671541211008191\n",
            "reward of step 25977 is: -0.06685588705779022\n",
            "reward of step 25978 is: -0.12270983402991853\n",
            "reward of step 25979 is: -0.0864234781482135\n",
            "reward of step 25980 is: -0.07792163130246232\n",
            "reward of step 25981 is: -0.09323182609396208\n",
            "reward of step 25982 is: -0.11075647111951203\n",
            "reward of step 25983 is: -0.06082182419147286\n",
            "reward of step 25984 is: -0.05696803110451487\n",
            "reward of step 25985 is: -0.05757097394209032\n",
            "reward of step 25986 is: -0.0707516342846255\n",
            "reward of step 25987 is: -0.09448814576967202\n",
            "reward of step 25988 is: -0.029117790018869272\n",
            "reward of step 25989 is: -0.07581943379222933\n",
            "reward of step 25990 is: -0.140682589043038\n",
            "reward of step 25991 is: -0.07797368929455883\n",
            "reward of step 25992 is: -0.06357959433100346\n",
            "reward of step 25993 is: -0.028563353104320677\n",
            "reward of step 25994 is: -0.10088910592600453\n",
            "reward of step 25995 is: -0.08366868473242217\n",
            "reward of step 25996 is: -0.0647108737636709\n",
            "reward of step 25997 is: -0.12451071932003333\n",
            "reward of step 25998 is: -0.13361167858349088\n",
            "reward of step 25999 is: -0.08513262758388929\n",
            "reward of step 26000 is: -0.03229614645160761\n",
            "reward of step 26001 is: -0.04562955160361337\n",
            "reward of step 26002 is: -0.05459401430028499\n",
            "reward of step 26003 is: -0.07519557415214762\n",
            "reward of step 26004 is: -0.04642337876542013\n",
            "reward of step 26005 is: -0.1589361095554238\n",
            "reward of step 26006 is: -0.06867186324204233\n",
            "reward of step 26007 is: -0.07421280848873502\n",
            "reward of step 26008 is: -0.01799067006555588\n",
            "reward of step 26009 is: -0.09889997445521115\n",
            "reward of step 26010 is: -0.09340073583740194\n",
            "reward of step 26011 is: -0.05183028741896534\n",
            "reward of step 26012 is: -0.05619956996100972\n",
            "reward of step 26013 is: -0.05423476107392944\n",
            "reward of step 26014 is: -0.0755328920458459\n",
            "reward of step 26015 is: -0.01092822615578004\n",
            "reward of step 26016 is: -0.07677578390422335\n",
            "reward of step 26017 is: -0.08972642823254162\n",
            "reward of step 26018 is: -0.017268509560950163\n",
            "reward of step 26019 is: -0.032875320936987085\n",
            "reward of step 26020 is: -0.10501294541328576\n",
            "reward of step 26021 is: -0.03904109095712549\n",
            "reward of step 26022 is: 0.00042808476052047517\n",
            "reward of step 26023 is: -0.012271538099991597\n",
            "reward of step 26024 is: -0.03411548554470867\n",
            "reward of step 26025 is: -0.08752227646742694\n",
            "reward of step 26026 is: -0.07320795571753902\n",
            "reward of step 26027 is: -0.05582486533441344\n",
            "reward of step 26028 is: -0.12977262198914308\n",
            "reward of step 26029 is: -0.07215987572903437\n",
            "reward of step 26030 is: -0.08445795098717179\n",
            "reward of step 26031 is: -0.037915134883995893\n",
            "reward of step 26032 is: -0.05865315366293522\n",
            "reward of step 26033 is: 0.0034403318118921833\n",
            "reward of step 26034 is: -0.0665241010241634\n",
            "reward of step 26035 is: -0.13268485511141503\n",
            "reward of step 26036 is: -0.04379877764274487\n",
            "reward of step 26037 is: -0.038455772462634874\n",
            "reward of step 26038 is: -0.04730930713603909\n",
            "reward of step 26039 is: -0.06914628450527738\n",
            "reward of step 26040 is: -0.0809170164320907\n",
            "reward of step 26041 is: -0.062003411160314714\n",
            "reward of step 26042 is: -0.06719690310440629\n",
            "reward of step 26043 is: -0.14099680039411455\n",
            "reward of step 26044 is: -0.06848419852882925\n",
            "reward of step 26045 is: -0.01705146691238113\n",
            "reward of step 26046 is: -0.1312720271966189\n",
            "reward of step 26047 is: -0.05265259803591482\n",
            "reward of step 26048 is: -0.06719670660363031\n",
            "reward of step 26049 is: -0.08154448520655366\n",
            "reward of step 26050 is: -0.0415045123989598\n",
            "reward of step 26051 is: -0.07439212754457247\n",
            "reward of step 26052 is: -0.0948355508224139\n",
            "reward of step 26053 is: -0.11685612233359344\n",
            "reward of step 26054 is: -0.14947160464155818\n",
            "reward of step 26055 is: -0.11453646560277086\n",
            "reward of step 26056 is: -0.02596119800181884\n",
            "reward of step 26057 is: -0.00011261599518652687\n",
            "reward of step 26058 is: -0.027740937473473837\n",
            "reward of step 26059 is: -0.05224976340094367\n",
            "reward of step 26060 is: -0.05227418456741806\n",
            "reward of step 26061 is: -0.07181044256294145\n",
            "reward of step 26062 is: -0.03848829363978323\n",
            "reward of step 26063 is: -0.0828139038021104\n",
            "reward of step 26064 is: -0.06969797361715646\n",
            "reward of step 26065 is: -0.03787436573877212\n",
            "reward of step 26066 is: -0.08642714219133296\n",
            "reward of step 26067 is: -0.07119361820912251\n",
            "reward of step 26068 is: -0.0907719425884751\n",
            "reward of step 26069 is: 0.04220840886615895\n",
            "reward of step 26070 is: -0.03492341581829794\n",
            "reward of step 26071 is: -0.1639861433134585\n",
            "reward of step 26072 is: -0.08248980927105964\n",
            "reward of step 26073 is: -0.013256935268369463\n",
            "reward of step 26074 is: -0.01274530330435475\n",
            "reward of step 26075 is: -0.04178568149116346\n",
            "reward of step 26076 is: -0.0774217638781084\n",
            "reward of step 26077 is: -0.07961298736358469\n",
            "reward of step 26078 is: -0.0772586225952816\n",
            "reward of step 26079 is: -0.10677117366808042\n",
            "reward of step 26080 is: -0.05124783692354051\n",
            "reward of step 26081 is: -0.02378228527335624\n",
            "reward of step 26082 is: -0.17425593266800277\n",
            "reward of step 26083 is: -0.04570956351242217\n",
            "reward of step 26084 is: -0.0618920867701368\n",
            "reward of step 26085 is: -0.10933751417500448\n",
            "reward of step 26086 is: -0.05677211005182248\n",
            "reward of step 26087 is: -0.015988676337720054\n",
            "reward of step 26088 is: -0.09874659940806096\n",
            "reward of step 26089 is: -0.1298776960665381\n",
            "reward of step 26090 is: -0.051710554408072484\n",
            "reward of step 26091 is: -0.07989264854527067\n",
            "reward of step 26092 is: -0.037021707187496866\n",
            "reward of step 26093 is: -0.09947668200690263\n",
            "reward of step 26094 is: -0.10577566611245726\n",
            "reward of step 26095 is: -0.10368050105850424\n",
            "reward of step 26096 is: -0.14413575485273344\n",
            "reward of step 26097 is: -0.04765072458503283\n",
            "reward of step 26098 is: -0.08842964908606532\n",
            "reward of step 26099 is: -0.11999338247513902\n",
            "reward of step 26100 is: -0.08285406578370713\n",
            "reward of step 26101 is: -0.04304675534522251\n",
            "reward of step 26102 is: -0.0818520421421094\n",
            "reward of step 26103 is: -0.08045245318106631\n",
            "reward of step 26104 is: -0.04685538423992408\n",
            "reward of step 26105 is: -0.10221332007884987\n",
            "reward of step 26106 is: -0.06215132121073319\n",
            "reward of step 26107 is: -0.11408538971811111\n",
            "reward of step 26108 is: -0.10098371059913358\n",
            "reward of step 26109 is: -0.13897741323609847\n",
            "reward of step 26110 is: -0.07526574254598284\n",
            "reward of step 26111 is: -0.017490168613123358\n",
            "reward of step 26112 is: -0.048979746487172404\n",
            "reward of step 26113 is: -0.09802566336404994\n",
            "reward of step 26114 is: -0.10754465225868626\n",
            "reward of step 26115 is: -0.052475297071363225\n",
            "reward of step 26116 is: -0.09758010489812174\n",
            "reward of step 26117 is: -0.07818945268003952\n",
            "reward of step 26118 is: -0.031401728985437916\n",
            "reward of step 26119 is: -0.02325056882368126\n",
            "reward of step 26120 is: -0.03214174377009649\n",
            "reward of step 26121 is: -0.12363510046998383\n",
            "reward of step 26122 is: -0.07470758036715763\n",
            "reward of step 26123 is: -0.1466897605958437\n",
            "reward of step 26124 is: -0.05895996126960912\n",
            "reward of step 26125 is: -0.07388020200969114\n",
            "reward of step 26126 is: -0.057046556770496304\n",
            "reward of step 26127 is: -0.05427978634317632\n",
            "reward of step 26128 is: -0.03887323585037061\n",
            "reward of step 26129 is: -0.021681292641387073\n",
            "reward of step 26130 is: -0.10637086146570607\n",
            "reward of step 26131 is: -0.1180072624407077\n",
            "reward of step 26132 is: -0.1204088978689486\n",
            "reward of step 26133 is: -0.062116890515092726\n",
            "reward of step 26134 is: -0.12265885451218739\n",
            "reward of step 26135 is: -0.04374552086092953\n",
            "reward of step 26136 is: -0.05899015299273269\n",
            "reward of step 26137 is: -0.06525131671389506\n",
            "reward of step 26138 is: -0.012693204006506531\n",
            "reward of step 26139 is: -0.10561918898046063\n",
            "reward of step 26140 is: -0.14128231878702602\n",
            "reward of step 26141 is: -0.044342543121713485\n",
            "reward of step 26142 is: -0.07303885488392658\n",
            "reward of step 26143 is: -0.1279971579932908\n",
            "reward of step 26144 is: -0.10394974605880447\n",
            "reward of step 26145 is: -0.07605171053115678\n",
            "reward of step 26146 is: -0.06977269455371449\n",
            "reward of step 26147 is: -0.0718309261496316\n",
            "reward of step 26148 is: -0.026140302740202848\n",
            "reward of step 26149 is: -0.10374856050987746\n",
            "reward of step 26150 is: -0.07252130724158223\n",
            "reward of step 26151 is: -0.05418299279957761\n",
            "reward of step 26152 is: 0.007651303525098241\n",
            "reward of step 26153 is: -0.061975125831481015\n",
            "reward of step 26154 is: -0.01870369325445942\n",
            "reward of step 26155 is: -0.07018301185055276\n",
            "reward of step 26156 is: -0.17096952310431035\n",
            "reward of step 26157 is: -0.09991746112727085\n",
            "reward of step 26158 is: -0.05972004261024455\n",
            "reward of step 26159 is: -0.05695029415462016\n",
            "reward of step 26160 is: -0.02478601251102175\n",
            "reward of step 26161 is: -0.1026473992574366\n",
            "reward of step 26162 is: -0.09586247592282382\n",
            "reward of step 26163 is: -0.05873095530279382\n",
            "reward of step 26164 is: -0.06012776013474774\n",
            "reward of step 26165 is: -0.0575968077575324\n",
            "reward of step 26166 is: -0.06257571974813603\n",
            "reward of step 26167 is: -0.04814608655298225\n",
            "reward of step 26168 is: -0.07457107866386958\n",
            "reward of step 26169 is: -0.04872915767093744\n",
            "reward of step 26170 is: -0.09954959271479502\n",
            "reward of step 26171 is: -0.10360652543170701\n",
            "reward of step 26172 is: -0.07076557617408086\n",
            "reward of step 26173 is: -0.06600463567123982\n",
            "reward of step 26174 is: -0.06762524062845054\n",
            "reward of step 26175 is: -0.06338375738951929\n",
            "reward of step 26176 is: -0.024547365379495756\n",
            "reward of step 26177 is: -0.10651499996625569\n",
            "reward of step 26178 is: -0.09424078089375298\n",
            "reward of step 26179 is: -0.17363322722336283\n",
            "reward of step 26180 is: -0.0517915714164785\n",
            "reward of step 26181 is: -0.10075668415241412\n",
            "reward of step 26182 is: -0.06338915975915704\n",
            "reward of step 26183 is: -0.048797424474152695\n",
            "reward of step 26184 is: -0.12889253919432708\n",
            "reward of step 26185 is: -0.14455392460873173\n",
            "reward of step 26186 is: -0.10316820945703342\n",
            "reward of step 26187 is: -0.07666204517224351\n",
            "reward of step 26188 is: -0.033580502603961615\n",
            "reward of step 26189 is: -0.077716556426491\n",
            "reward of step 26190 is: -0.024170244873683466\n",
            "reward of step 26191 is: -0.14594038238205498\n",
            "reward of step 26192 is: -0.1121388524582907\n",
            "reward of step 26193 is: -0.06141931445439175\n",
            "reward of step 26194 is: -0.104157714753079\n",
            "reward of step 26195 is: -0.04178603071420772\n",
            "reward of step 26196 is: 0.01193075002560784\n",
            "reward of step 26197 is: -0.10374646536802357\n",
            "reward of step 26198 is: -0.051044537041850524\n",
            "reward of step 26199 is: -0.028123539064972336\n",
            "reward of step 26200 is: -0.07413358986084595\n",
            "reward of step 26201 is: -0.09350031249892221\n",
            "reward of step 26202 is: -0.038842196640052506\n",
            "reward of step 26203 is: -0.014701738246753138\n",
            "reward of step 26204 is: -0.05122505106383257\n",
            "reward of step 26205 is: -0.09772489332628176\n",
            "reward of step 26206 is: -0.053116789556021726\n",
            "reward of step 26207 is: -0.05167898684871486\n",
            "reward of step 26208 is: -0.058936812898051194\n",
            "reward of step 26209 is: -0.059488651052928776\n",
            "reward of step 26210 is: -0.06674974618697682\n",
            "reward of step 26211 is: -0.032149076233640606\n",
            "reward of step 26212 is: -0.06359430224290585\n",
            "reward of step 26213 is: -0.12015819599972377\n",
            "reward of step 26214 is: -0.09901755610749874\n",
            "reward of step 26215 is: -0.038867752651116305\n",
            "reward of step 26216 is: -0.05347206692145301\n",
            "reward of step 26217 is: -0.05950150909687857\n",
            "reward of step 26218 is: -0.0734316341023552\n",
            "reward of step 26219 is: 0.03961808669936151\n",
            "reward of step 26220 is: -0.050485924320325615\n",
            "reward of step 26221 is: -0.01627041309602395\n",
            "reward of step 26222 is: -0.04642743973047969\n",
            "reward of step 26223 is: -0.08592749879532868\n",
            "reward of step 26224 is: -0.11690758299220383\n",
            "reward of step 26225 is: -0.03891162480180532\n",
            "reward of step 26226 is: -0.05766652577749032\n",
            "reward of step 26227 is: -0.05839929091607965\n",
            "reward of step 26228 is: -0.11292008000471321\n",
            "reward of step 26229 is: -0.043110764670789226\n",
            "reward of step 26230 is: -0.10481739726555728\n",
            "reward of step 26231 is: -0.1516585215479035\n",
            "reward of step 26232 is: -0.12010293449594933\n",
            "reward of step 26233 is: -0.09249422783959216\n",
            "reward of step 26234 is: -0.051920899779967944\n",
            "reward of step 26235 is: -0.050173606984151964\n",
            "reward of step 26236 is: -0.030839560250613407\n",
            "reward of step 26237 is: -0.1354107958390267\n",
            "reward of step 26238 is: -0.030180357902353183\n",
            "reward of step 26239 is: -0.034690318867233616\n",
            "reward of step 26240 is: -0.056410185937947555\n",
            "reward of step 26241 is: -0.09922700736119472\n",
            "reward of step 26242 is: -0.03338758028995381\n",
            "reward of step 26243 is: -0.04819172229081925\n",
            "reward of step 26244 is: -0.05597828429336915\n",
            "reward of step 26245 is: -0.1147116548165279\n",
            "reward of step 26246 is: -0.1570013015463817\n",
            "reward of step 26247 is: -0.028089212413809905\n",
            "reward of step 26248 is: -0.040591316630522845\n",
            "reward of step 26249 is: -0.060531710083517876\n",
            "reward of step 26250 is: -0.041580099913372415\n",
            "reward of step 26251 is: -0.044865969422107854\n",
            "reward of step 26252 is: -0.08621208901433652\n",
            "reward of step 26253 is: -0.05419249653156144\n",
            "reward of step 26254 is: -0.14007311582721982\n",
            "reward of step 26255 is: -0.11931108204830088\n",
            "reward of step 26256 is: -0.10835904902073845\n",
            "reward of step 26257 is: -0.0633968915194465\n",
            "reward of step 26258 is: -0.10701690475733383\n",
            "reward of step 26259 is: -0.05892841932026327\n",
            "reward of step 26260 is: -0.03554348718721079\n",
            "reward of step 26261 is: -0.0717857189572324\n",
            "reward of step 26262 is: -0.08957621651594072\n",
            "reward of step 26263 is: -0.10291418024722543\n",
            "reward of step 26264 is: -0.08840425693860676\n",
            "reward of step 26265 is: -0.08645457321724381\n",
            "reward of step 26266 is: -0.0651843870807971\n",
            "reward of step 26267 is: -0.09522519440412469\n",
            "reward of step 26268 is: -0.10826889502722625\n",
            "reward of step 26269 is: -0.05951374980702573\n",
            "reward of step 26270 is: -0.053880742789231606\n",
            "reward of step 26271 is: -0.02047742172213174\n",
            "reward of step 26272 is: -0.02579650539531464\n",
            "reward of step 26273 is: -0.02356938225064731\n",
            "reward of step 26274 is: -0.08279100440585319\n",
            "reward of step 26275 is: -0.03076896107729432\n",
            "reward of step 26276 is: -0.1285956828674818\n",
            "reward of step 26277 is: -0.021926647979518266\n",
            "reward of step 26278 is: -0.09189538748921666\n",
            "reward of step 26279 is: -0.08797239450072547\n",
            "reward of step 26280 is: -0.13973551069237766\n",
            "reward of step 26281 is: 0.00487966595975875\n",
            "reward of step 26282 is: -0.1379061425363607\n",
            "reward of step 26283 is: -0.0470399430773758\n",
            "reward of step 26284 is: -0.07014566379037213\n",
            "reward of step 26285 is: -0.08029236146975471\n",
            "reward of step 26286 is: -0.07303564225198267\n",
            "reward of step 26287 is: -0.10015583883488521\n",
            "reward of step 26288 is: -0.015779613620020116\n",
            "reward of step 26289 is: -0.12896498907178056\n",
            "reward of step 26290 is: -0.052462351533619556\n",
            "reward of step 26291 is: -0.09637668389504328\n",
            "reward of step 26292 is: -0.07822818876393645\n",
            "reward of step 26293 is: -0.044963447452410654\n",
            "reward of step 26294 is: -0.1046369420445522\n",
            "reward of step 26295 is: -0.11476169898042865\n",
            "reward of step 26296 is: -0.08771999824627308\n",
            "reward of step 26297 is: -0.039432757495892146\n",
            "reward of step 26298 is: -0.05008608701161865\n",
            "reward of step 26299 is: -0.05570805167772952\n",
            "reward of step 26300 is: -0.10960426451691585\n",
            "reward of step 26301 is: -0.11216837583910377\n",
            "reward of step 26302 is: -0.02193784778120389\n",
            "reward of step 26303 is: -0.05683674655053872\n",
            "reward of step 26304 is: -0.043372485525692506\n",
            "reward of step 26305 is: 0.03230112455521805\n",
            "reward of step 26306 is: 0.0026637741997339592\n",
            "reward of step 26307 is: -0.023424199298820003\n",
            "reward of step 26308 is: -0.05808106826809989\n",
            "reward of step 26309 is: -0.08413693646887022\n",
            "reward of step 26310 is: -0.05600536914648557\n",
            "reward of step 26311 is: -0.1352131741508762\n",
            "reward of step 26312 is: -0.10414314222900034\n",
            "reward of step 26313 is: -0.10802942655366987\n",
            "reward of step 26314 is: -0.12920243963715705\n",
            "reward of step 26315 is: -0.04047203654695575\n",
            "reward of step 26316 is: -0.08087941424902412\n",
            "reward of step 26317 is: -0.07665283826247205\n",
            "reward of step 26318 is: -0.15338726841125117\n",
            "reward of step 26319 is: -0.13101349422305708\n",
            "reward of step 26320 is: -0.12315597591545269\n",
            "reward of step 26321 is: -0.0788762306099392\n",
            "reward of step 26322 is: -0.05154862826463802\n",
            "reward of step 26323 is: -0.15783782285866121\n",
            "reward of step 26324 is: -0.027427747645653255\n",
            "reward of step 26325 is: -0.10847945113081625\n",
            "reward of step 26326 is: -0.12136318276656699\n",
            "reward of step 26327 is: -0.0787220853619146\n",
            "reward of step 26328 is: -0.031766308695181156\n",
            "reward of step 26329 is: -0.1627471338783094\n",
            "reward of step 26330 is: -0.08955974723946192\n",
            "reward of step 26331 is: -0.010933584484035364\n",
            "reward of step 26332 is: -0.039039282893358784\n",
            "reward of step 26333 is: -0.13468399795804775\n",
            "reward of step 26334 is: 0.012462559286354091\n",
            "reward of step 26335 is: -0.14496279853274252\n",
            "reward of step 26336 is: -0.09325233045879056\n",
            "reward of step 26337 is: -0.05593887760715277\n",
            "reward of step 26338 is: -0.04579184402238634\n",
            "reward of step 26339 is: -0.04677200362596845\n",
            "reward of step 26340 is: -0.05393279181689259\n",
            "reward of step 26341 is: -0.07349663300644027\n",
            "reward of step 26342 is: -0.03384739460202235\n",
            "reward of step 26343 is: -0.09144107638413601\n",
            "reward of step 26344 is: -0.0963304286544413\n",
            "reward of step 26345 is: -0.03015091246239021\n",
            "reward of step 26346 is: -0.07460842220096142\n",
            "reward of step 26347 is: -0.02573873572838381\n",
            "reward of step 26348 is: -0.03656451479146994\n",
            "reward of step 26349 is: -0.061798569836580275\n",
            "reward of step 26350 is: -0.06134017749542575\n",
            "reward of step 26351 is: -0.07434502198718906\n",
            "reward of step 26352 is: -0.06170674483634353\n",
            "reward of step 26353 is: -0.07469673032266255\n",
            "reward of step 26354 is: -0.12940408634878586\n",
            "reward of step 26355 is: -0.11830918880193186\n",
            "reward of step 26356 is: -0.022169666808026012\n",
            "reward of step 26357 is: -0.11216473266496463\n",
            "reward of step 26358 is: -0.11585117887864615\n",
            "reward of step 26359 is: -0.0419167228184113\n",
            "reward of step 26360 is: -0.16682992082921055\n",
            "reward of step 26361 is: -0.11556359288325535\n",
            "reward of step 26362 is: -0.08667148824358384\n",
            "reward of step 26363 is: -0.08059693338501539\n",
            "reward of step 26364 is: -0.05203340985640659\n",
            "reward of step 26365 is: -0.09122043395465551\n",
            "reward of step 26366 is: -0.13908289392484452\n",
            "reward of step 26367 is: -0.07423846084590635\n",
            "reward of step 26368 is: -0.12157648818692468\n",
            "reward of step 26369 is: -0.0029239225545039105\n",
            "reward of step 26370 is: -0.0792608496141064\n",
            "reward of step 26371 is: -0.04842955991161235\n",
            "reward of step 26372 is: -0.1371441286309274\n",
            "reward of step 26373 is: -0.12289935650699801\n",
            "reward of step 26374 is: -0.00895870592657566\n",
            "reward of step 26375 is: -0.071168186097887\n",
            "reward of step 26376 is: -0.07357684558486166\n",
            "reward of step 26377 is: -0.09223167480591277\n",
            "reward of step 26378 is: -0.08757682814651568\n",
            "reward of step 26379 is: -0.17242783909727288\n",
            "reward of step 26380 is: -0.06359896022265343\n",
            "reward of step 26381 is: -0.10905735433220187\n",
            "reward of step 26382 is: -0.027035290242770293\n",
            "reward of step 26383 is: -0.042592917500443805\n",
            "reward of step 26384 is: -0.031651727419790054\n",
            "reward of step 26385 is: 0.015634116530063302\n",
            "reward of step 26386 is: -0.10106025069182367\n",
            "reward of step 26387 is: -0.03467616097803605\n",
            "reward of step 26388 is: -0.0732401787788497\n",
            "reward of step 26389 is: -0.0734204108176163\n",
            "reward of step 26390 is: -0.12192896138738285\n",
            "reward of step 26391 is: -0.003976167088458937\n",
            "reward of step 26392 is: -0.002575525068652884\n",
            "reward of step 26393 is: -0.12713889127255995\n",
            "reward of step 26394 is: -0.07954760228204749\n",
            "reward of step 26395 is: -0.07814425263126956\n",
            "reward of step 26396 is: -0.05583745084407021\n",
            "reward of step 26397 is: -0.1008131632101078\n",
            "reward of step 26398 is: -0.11434745327183493\n",
            "reward of step 26399 is: -0.08711091147641392\n",
            "reward of step 26400 is: -0.10275021622914438\n",
            "reward of step 26401 is: -0.057062612636377685\n",
            "reward of step 26402 is: -0.03470222761382635\n",
            "reward of step 26403 is: -0.009044233822546821\n",
            "reward of step 26404 is: -0.09776457102830738\n",
            "reward of step 26405 is: -0.012519734316631115\n",
            "reward of step 26406 is: -0.06433807854630713\n",
            "reward of step 26407 is: -0.0796624327038783\n",
            "reward of step 26408 is: 0.008975747068321493\n",
            "reward of step 26409 is: -0.03121034113441168\n",
            "reward of step 26410 is: -0.11226195689789153\n",
            "reward of step 26411 is: -0.048135673701924286\n",
            "reward of step 26412 is: -0.08899511180537956\n",
            "reward of step 26413 is: -0.10318594395820091\n",
            "reward of step 26414 is: -0.14069461277346096\n",
            "reward of step 26415 is: -0.08203747098532488\n",
            "reward of step 26416 is: -0.11133478466637359\n",
            "reward of step 26417 is: -0.06973705454279289\n",
            "reward of step 26418 is: -0.04455600881825128\n",
            "reward of step 26419 is: -0.13065562837518407\n",
            "reward of step 26420 is: 0.013235678935283324\n",
            "reward of step 26421 is: -0.04217352787495543\n",
            "reward of step 26422 is: -0.06192055899288018\n",
            "reward of step 26423 is: -0.01285580358651206\n",
            "reward of step 26424 is: -0.007228368592151813\n",
            "reward of step 26425 is: -0.09715755665764092\n",
            "reward of step 26426 is: -0.018332613892323657\n",
            "reward of step 26427 is: -0.10148800869679986\n",
            "reward of step 26428 is: -0.08804988638380673\n",
            "reward of step 26429 is: -0.05003902971650276\n",
            "reward of step 26430 is: -0.13732739757048384\n",
            "reward of step 26431 is: -0.08121595177759922\n",
            "reward of step 26432 is: -0.04533830785575632\n",
            "reward of step 26433 is: -0.02284503311105912\n",
            "reward of step 26434 is: -0.1323928473756969\n",
            "reward of step 26435 is: -0.02965268619399064\n",
            "reward of step 26436 is: -0.10637710899972308\n",
            "reward of step 26437 is: -0.0893879924650417\n",
            "reward of step 26438 is: -0.013616253929422784\n",
            "reward of step 26439 is: -0.016453549581355698\n",
            "reward of step 26440 is: -0.1312472512679459\n",
            "reward of step 26441 is: -0.05007822953470764\n",
            "reward of step 26442 is: -0.013581943106958483\n",
            "reward of step 26443 is: -0.01460405806850118\n",
            "reward of step 26444 is: -0.04621343790787669\n",
            "reward of step 26445 is: -0.05316779018562401\n",
            "reward of step 26446 is: -0.09832961315511057\n",
            "reward of step 26447 is: -0.06831893445313841\n",
            "reward of step 26448 is: -0.15586988777675348\n",
            "reward of step 26449 is: -0.06278210258684569\n",
            "reward of step 26450 is: -0.027666232337531182\n",
            "reward of step 26451 is: -0.024359708418470905\n",
            "reward of step 26452 is: -0.053676439547624466\n",
            "reward of step 26453 is: -0.038593158973592834\n",
            "reward of step 26454 is: -0.029601867692992445\n",
            "reward of step 26455 is: -0.03605664201216052\n",
            "reward of step 26456 is: -0.0828104081414126\n",
            "reward of step 26457 is: -0.0037529744820974376\n",
            "reward of step 26458 is: -0.07802128979873191\n",
            "reward of step 26459 is: -0.07993847879989113\n",
            "reward of step 26460 is: -0.10909336352276489\n",
            "reward of step 26461 is: -0.0672680732216282\n",
            "reward of step 26462 is: -0.1335671900762484\n",
            "reward of step 26463 is: -0.03792578184951256\n",
            "reward of step 26464 is: -0.04959314432372963\n",
            "reward of step 26465 is: -0.1181268368781031\n",
            "reward of step 26466 is: -0.035316883575280955\n",
            "reward of step 26467 is: -0.10170442963536108\n",
            "reward of step 26468 is: -0.11929372932731908\n",
            "reward of step 26469 is: -0.13479737775994782\n",
            "reward of step 26470 is: -0.09057072911663455\n",
            "reward of step 26471 is: -0.15625659001905867\n",
            "reward of step 26472 is: -0.15811262907242418\n",
            "reward of step 26473 is: -0.002605766994532077\n",
            "reward of step 26474 is: -0.11656981367675967\n",
            "reward of step 26475 is: -0.027819079389303836\n",
            "reward of step 26476 is: -0.13216793919143177\n",
            "reward of step 26477 is: -0.04887151735526529\n",
            "reward of step 26478 is: -0.05628644438524888\n",
            "reward of step 26479 is: -0.05264631878332693\n",
            "reward of step 26480 is: -0.09772799723253756\n",
            "reward of step 26481 is: -0.07843377449575262\n",
            "reward of step 26482 is: -0.07398186964424047\n",
            "reward of step 26483 is: -0.06203128042298989\n",
            "reward of step 26484 is: -0.09427514927422886\n",
            "reward of step 26485 is: -0.11970509084016034\n",
            "reward of step 26486 is: -0.08797285775377506\n",
            "reward of step 26487 is: -0.04785772091919327\n",
            "reward of step 26488 is: -0.06632764677831848\n",
            "reward of step 26489 is: -0.045425052755345585\n",
            "reward of step 26490 is: -0.06846105836750527\n",
            "reward of step 26491 is: -0.12021099697671445\n",
            "reward of step 26492 is: -0.0747599480857154\n",
            "reward of step 26493 is: -0.1383102512704527\n",
            "reward of step 26494 is: -0.1751872184905967\n",
            "reward of step 26495 is: -0.06603953125385076\n",
            "reward of step 26496 is: -0.007423526936145564\n",
            "reward of step 26497 is: -0.10654938498063393\n",
            "reward of step 26498 is: -0.12935302373223634\n",
            "reward of step 26499 is: -0.03181440982447736\n",
            "reward of step 26500 is: -0.06013900132823202\n",
            "reward of step 26501 is: -0.03849234982323291\n",
            "reward of step 26502 is: -0.07314909459063756\n",
            "reward of step 26503 is: -0.06431262088828948\n",
            "reward of step 26504 is: -0.041553059592270514\n",
            "reward of step 26505 is: -0.02385932858535811\n",
            "reward of step 26506 is: -0.12128295926400334\n",
            "reward of step 26507 is: -0.11313302712720141\n",
            "reward of step 26508 is: -0.026635527442228413\n",
            "reward of step 26509 is: -0.006222369154733731\n",
            "reward of step 26510 is: -0.018795053215917457\n",
            "reward of step 26511 is: -0.07465815008520793\n",
            "reward of step 26512 is: -0.08017232439492472\n",
            "reward of step 26513 is: -0.0760632308599345\n",
            "reward of step 26514 is: -0.09272497362410448\n",
            "reward of step 26515 is: -0.056921928302982594\n",
            "reward of step 26516 is: -0.034344549412831715\n",
            "reward of step 26517 is: -0.07344693830527294\n",
            "reward of step 26518 is: -0.08342591968956692\n",
            "reward of step 26519 is: -0.0006274436152003959\n"
          ]
        },
        {
          "output_type": "error",
          "ename": "KeyboardInterrupt",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-29-4326fee8443c>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0magent\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtrain\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnum_frames\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
            "\u001b[0;32m<ipython-input-27-86275e2ecc46>\u001b[0m in \u001b[0;36mtrain\u001b[0;34m(self, num_frames, plotting_interval)\u001b[0m\n\u001b[1;32m    183\u001b[0m                 \u001b[0;32mand\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtotal_step\u001b[0m \u001b[0;34m>\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0minitial_random_steps\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    184\u001b[0m             ):\n\u001b[0;32m--> 185\u001b[0;31m                 \u001b[0mactor_loss\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcritic_loss\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mupdate_model\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    186\u001b[0m                 \u001b[0mactor_losses\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mactor_loss\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    187\u001b[0m                 \u001b[0mcritic_losses\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcritic_loss\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m<ipython-input-27-86275e2ecc46>\u001b[0m in \u001b[0;36mupdate_model\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    137\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    138\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcritic_optimizer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mzero_grad\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 139\u001b[0;31m         \u001b[0mcritic_loss\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbackward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    140\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcritic_optimizer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstep\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    141\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/torch/_tensor.py\u001b[0m in \u001b[0;36mbackward\u001b[0;34m(self, gradient, retain_graph, create_graph, inputs)\u001b[0m\n\u001b[1;32m    361\u001b[0m                 \u001b[0mcreate_graph\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mcreate_graph\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    362\u001b[0m                 inputs=inputs)\n\u001b[0;32m--> 363\u001b[0;31m         \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mautograd\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbackward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mgradient\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mretain_graph\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcreate_graph\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minputs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0minputs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    364\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    365\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mregister_hook\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhook\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/torch/autograd/__init__.py\u001b[0m in \u001b[0;36mbackward\u001b[0;34m(tensors, grad_tensors, retain_graph, create_graph, grad_variables, inputs)\u001b[0m\n\u001b[1;32m    173\u001b[0m     Variable._execution_engine.run_backward(  # Calls into the C++ engine to run the backward pass\n\u001b[1;32m    174\u001b[0m         \u001b[0mtensors\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mgrad_tensors_\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mretain_graph\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcreate_graph\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minputs\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 175\u001b[0;31m         allow_unreachable=True, accumulate_grad=True)  # Calls into the C++ engine to run the backward pass\n\u001b[0m\u001b[1;32m    176\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    177\u001b[0m def grad(\n",
            "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Test"
      ],
      "metadata": {
        "id": "7p0roQPkH46C"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "A = np.array([ -57.53854223, -44.63363074,0.        ])\n",
        "B = np.array([-407.76417867,-564.62837335,0.        ])\n",
        "\n",
        "print(\"Expand dims\")\n",
        "A = np.expand_dims(A, axis=0)\n",
        "B = np.expand_dims(B, axis=0)\n",
        "print(A-B)\n",
        "np.sqrt(np.sum((A - B)**2,axis=1))"
      ],
      "metadata": {
        "id": "azo5xqDdUaw7"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "power_radar   = 1000\n",
        "power_bs_comm = 100\n",
        "power_bs_priv = np.ones((5,1))\n",
        "sigma2 = -10**(-18)  \n",
        "B = 10**6\n",
        "H_radar_uav   = np.array([7.32715132e-12])\n",
        "H_user_radar  = np.array([[8.58882632e-14, 1.90202363e-13, 7.27181848e-14, 1.50638726e-13, 9.32527962e-14],\n",
        "                [8.58882632e-14, 1.90202363e-13, 7.27181848e-14, 1.50638726e-13, 9.32527962e-14],\n",
        "                [8.58882632e-14, 1.90202363e-13, 7.27181848e-14, 1.50638726e-13, 9.32527962e-14],\n",
        "                [8.58882632e-14, 1.90202363e-13, 7.27181848e-14, 1.50638726e-13, 9.32527962e-14],\n",
        "                [8.58882632e-14, 1.90202363e-13, 7.27181848e-14, 1.50638726e-13, 9.32527962e-14]])\n",
        "# print(H_radar_uav)\n",
        "# print(np.sum(power_bs_priv))\n",
        "numerator = (H_radar_uav**2)*power_radar\n",
        "print(numerator)\n",
        "denominator = (H_user_radar**2) * (power_bs_comm + np.sum(power_bs_priv)) + (B*(sigma2))\n",
        "print(denominator)\n",
        "SINR = numerator/denominator\n",
        "print(SINR)"
      ],
      "metadata": {
        "id": "3btEVIJsPowE"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "BS_x = 0\n",
        "BS_y = 0\n",
        "theta  = 0\n",
        "UAV_trajectory = []\n",
        "for i in range(1000):\n",
        "    theta  = theta + np.pi/360\n",
        "    print(theta)\n",
        "    r      = 500*np.sin(theta/2)\n",
        "    UAV_vx = r*np.cos(theta)\n",
        "    UAV_vy = r*np.sin(theta)\n",
        "    UAV_vh = 0\n",
        "    UAV_trajectory.append([UAV_vx, UAV_vy, UAV_vh])\n",
        "    fig, ax = plt.subplots()\n",
        "    circle1 = plt.Circle((BS_x, BS_y), 1, color='r', fill=True)\n",
        "    circle2 = plt.Circle((BS_x, BS_y), 100, color='g', fill=False)\n",
        "    circle3 = plt.Circle((BS_x, BS_y), 1000, color='b', fill=False)\n",
        "    ax.add_patch(circle1)\n",
        "    ax.add_patch(circle2)\n",
        "    ax.add_patch(circle3)\n",
        "    gu = np.array(UAV_trajectory).transpose()[0:2].transpose()\n",
        "    plt.scatter(gu[:,0], gu[:,1])\n",
        "    plt.show()    "
      ],
      "metadata": {
        "id": "n8IeJExwwyLj"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "import numpy as np\n",
        "\n",
        "V = np.array([1,2,3,4,5])\n",
        "\n",
        "# Private user exclude sum of user at this point\n",
        "# Clone P_BS_U vector into P_BS_U matrix \n",
        "clonedM = np.ones((len(V),1))*V\n",
        "print(f\"clonedM: {clonedM}\")\n",
        "# Remove diagonal\n",
        "diagonalExcludedM = clonedM -np.diag(np.diag(clonedM))\n",
        "print(f\"diagonalExcludedM: {diagonalExcludedM}\")\n",
        "# Sum over self-excluded matrix\n",
        "rowWise_SumM = np.matrix(diagonalExcludedM).sum(axis=1)\n",
        "print(f\"rowWise_SumM: {rowWise_SumM}\")\n",
        "\n",
        "print(np.multiply(h.transpose(),rowWise_SumM))"
      ],
      "metadata": {
        "id": "H-z4cB9oPGn-"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        ""
      ],
      "metadata": {
        "id": "RvJVuOhCUvuw"
      },
      "execution_count": null,
      "outputs": []
    }
  ]
}