PPO.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "name": "PPO.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "toc_visible": true
    },
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.6.8"
    }
  },
  "cells": [
    {
      "cell_type": "code",
      "metadata": {
        "id": "6s7lP8w_qT-g"
      },
      "source": [
        "import gym\n",
        "import numpy as np\n",
        "from matplotlib import pyplot as plt\n",
        "%matplotlib inline"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "qMmeQkfjMtNo",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "cee7b3ab-e7e5-4953-cd17-e98034d858db"
      },
      "source": [
        "!nvidia-smi"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Tue Jun  8 06:00:56 2021       \n",
            "+-----------------------------------------------------------------------------+\n",
            "| NVIDIA-SMI 465.27       Driver Version: 460.32.03    CUDA Version: 11.2     |\n",
            "|-------------------------------+----------------------+----------------------+\n",
            "| GPU  Name        Persistence-M| Bus-Id        Disp.A | Volatile Uncorr. ECC |\n",
            "| Fan  Temp  Perf  Pwr:Usage/Cap|         Memory-Usage | GPU-Util  Compute M. |\n",
            "|                               |                      |               MIG M. |\n",
            "|===============================+======================+======================|\n",
            "|   0  Tesla V100-SXM2...  Off  | 00000000:00:04.0 Off |                    0 |\n",
            "| N/A   38C    P0    37W / 300W |   1033MiB / 16160MiB |      0%      Default |\n",
            "|                               |                      |                  N/A |\n",
            "+-------------------------------+----------------------+----------------------+\n",
            "                                                                               \n",
            "+-----------------------------------------------------------------------------+\n",
            "| Processes:                                                                  |\n",
            "|  GPU   GI   CI        PID   Type   Process name                  GPU Memory |\n",
            "|        ID   ID                                                   Usage      |\n",
            "|=============================================================================|\n",
            "+-----------------------------------------------------------------------------+\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "isafSyFJqqbC"
      },
      "source": [
        "# Auswahl des Spiels\n",
        "\n",
        "[Hier](https://gym.openai.com/envs/#atari) ist eine vollständige Liste der verfügbaren Spiele zu finden. Um ein Environment zu erstellen muss der vollständige Name des Spiels als String übergeben werden.\n",
        "\n",
        "Beispiel:\n",
        "\n",
        "```python\n",
        "game = \"MsPacman-v0\"\n",
        "```"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "9oFwznyfqtY8"
      },
      "source": [
        "# Hier kann das Spiel übergeben werden\n",
        "game = \"Pong-v0\""
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Irvr4YgYEh7n"
      },
      "source": [
        "try: \n",
        "    env = gym.make(game)\n",
        "except:\n",
        "    from google.colab import drive\n",
        "    drive.mount('/content/drive')\n",
        "    !python -m atari_py.import_roms \"drive/MyDrive/Atari_Roms\"\n",
        "    env = gym.make(game)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "JZFpCCiUqxec"
      },
      "source": [
        "# **Preprocessing**\n",
        "[Stable Baselines](https://github.com/openai/baselines/blob/master/baselines/common/atari_wrappers.py)\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "K56-9gA8q2cM"
      },
      "source": [
        "## Fire Wrapper"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "H7xfJ9C1q47O"
      },
      "source": [
        "class FireResetEnv(gym.Wrapper):\n",
        "    def __init__(self, env):\n",
        "        gym.Wrapper.__init__(self, env) \n",
        "        self.env.reset()\n",
        "\n",
        "    def reset(self, **kwargs):\n",
        "        self.env.reset(**kwargs)\n",
        "        observation, _, _, _ = self.env.step(env.unwrapped.get_action_meanings().index('FIRE'))\n",
        "\n",
        "        return observation"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "iUX3_jS2MtNx"
      },
      "source": [
        "## Max And Skip Env Wrapper"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "KGNoWqC6MtNy"
      },
      "source": [
        "class MaxAndSkipEnv(gym.Wrapper):\n",
        "    def __init__(self, env, skip=4):\n",
        "        \"\"\"Return only every `skip`-th frame\"\"\"\n",
        "        gym.Wrapper.__init__(self, env)\n",
        "        # most recent raw observations (for max pooling across time steps)\n",
        "        self._obs_buffer = np.zeros((2,)+env.observation_space.shape, dtype=np.uint8)\n",
        "        self._skip       = skip\n",
        "\n",
        "    def step(self, action):\n",
        "        \"\"\"Repeat action, sum reward, and max over last observations.\"\"\"\n",
        "        total_reward = 0.0\n",
        "        done = None\n",
        "        for i in range(self._skip):\n",
        "            obs, reward, done, info = self.env.step(action)\n",
        "            if i == self._skip - 2: self._obs_buffer[0] = obs\n",
        "            if i == self._skip - 1: self._obs_buffer[1] = obs\n",
        "            total_reward += reward\n",
        "            if done:\n",
        "                break\n",
        "        # Note that the observation on the done=True frame\n",
        "        # doesn't matter\n",
        "        max_frame = self._obs_buffer.max(axis=0)\n",
        "\n",
        "        return max_frame, total_reward, done, info\n",
        "    \n",
        "    def reset(self, **kwargs):\n",
        "        return self.env.reset(**kwargs)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Nzhr2UTyMtNz"
      },
      "source": [
        "## Noop Reset Env Wrapper"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "YHDVQS_nMtN0"
      },
      "source": [
        "class NoopResetEnv(gym.Wrapper):\n",
        "    def __init__(self, env, noop_max=30):\n",
        "        \"\"\"Sample initial states by taking random number of no-ops on reset.\n",
        "        No-op is assumed to be action 0.\n",
        "        \"\"\"\n",
        "        gym.Wrapper.__init__(self, env)\n",
        "        self.noop_max = noop_max\n",
        "        self.override_num_noops = None\n",
        "        self.noop_action = 0\n",
        "        assert env.unwrapped.get_action_meanings()[0] == 'NOOP'\n",
        "\n",
        "    def reset(self, **kwargs):\n",
        "        \"\"\" Do no-op action for a number of steps in [1, noop_max].\"\"\"\n",
        "        self.env.reset(**kwargs)\n",
        "        if self.override_num_noops is not None:\n",
        "            noops = self.override_num_noops\n",
        "        else:\n",
        "            noops = self.unwrapped.np_random.randint(1, self.noop_max + 1) #pylint: disable=E1101\n",
        "        assert noops > 0\n",
        "        obs = None\n",
        "        for _ in range(noops):\n",
        "            obs, _, done, _ = self.env.step(self.noop_action)\n",
        "            if done:\n",
        "                obs = self.env.reset(**kwargs)\n",
        "        return obs\n",
        "\n",
        "    def step(self, ac):\n",
        "        return self.env.step(ac)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "5Csj73MlMtN1"
      },
      "source": [
        "## Scaled Float Frame Wrapper"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "rBYeRdvrMtN1"
      },
      "source": [
        "class ScaledFloatFrame(gym.ObservationWrapper):\n",
        "    def __init__(self, env):\n",
        "        gym.ObservationWrapper.__init__(self, env)\n",
        "        self.observation_space = gym.spaces.Box(low=0, high=1, shape=env.observation_space.shape, dtype=np.float32)\n",
        "\n",
        "    def observation(self, observation):\n",
        "        # careful! This undoes the memory optimization, use\n",
        "        # with smaller replay buffers only.\n",
        "        return np.array(observation).astype(np.float32) / 255.0"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "_vd03Hv1MtN2"
      },
      "source": [
        "## Episodic Life Wrapper"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "K3_tCT4fMtN2"
      },
      "source": [
        "class EpisodicLifeEnv(gym.Wrapper):\n",
        "    def __init__(self, env):\n",
        "        \"\"\"Make end-of-life == end-of-episode, but only reset on true game over.\n",
        "        Done by DeepMind for the DQN and co. since it helps value estimation.\n",
        "        \"\"\"\n",
        "        gym.Wrapper.__init__(self, env)\n",
        "        self.lives = 0\n",
        "        self.was_real_done  = True\n",
        "\n",
        "    def step(self, action):\n",
        "        obs, reward, done, info = self.env.step(action)\n",
        "        self.was_real_done = done\n",
        "        # check current lives, make loss of life terminal,\n",
        "        # then update lives to handle bonus lives\n",
        "        lives = self.env.unwrapped.ale.lives()\n",
        "        if lives < self.lives and lives > 0:\n",
        "            # for Qbert sometimes we stay in lives == 0 condition for a few frames\n",
        "            # so it's important to keep lives > 0, so that we only reset once\n",
        "            # the environment advertises done.\n",
        "            done = True\n",
        "        self.lives = lives\n",
        "        return obs, reward, done, info\n",
        "\n",
        "    def reset(self, **kwargs):\n",
        "        \"\"\"Reset only when lives are exhausted.\n",
        "        This way all states are still reachable even though lives are episodic,\n",
        "        and the learner need not know about any of this behind-the-scenes.\n",
        "        \"\"\"\n",
        "        if self.was_real_done:\n",
        "            obs = self.env.reset(**kwargs)\n",
        "        else:\n",
        "            # no-op step to advance from terminal/lost life state\n",
        "            obs, _, _, _ = self.env.step(0)\n",
        "        self.lives = self.env.unwrapped.ale.lives()\n",
        "        return obs"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Z_zaDmRJMtN4"
      },
      "source": [
        "## Clip Reward Wrapper"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "yTsOOYyEMtN5"
      },
      "source": [
        "class ClipRewardEnv(gym.RewardWrapper):\n",
        "    def __init__(self, env):\n",
        "        gym.RewardWrapper.__init__(self, env)\n",
        "\n",
        "    def reward(self, reward):\n",
        "        \"\"\"Bin reward to {+1, 0, -1} by its sign.\"\"\"\n",
        "        return np.sign(reward)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ukAIUaihrUBd"
      },
      "source": [
        "## Resize & Grayscale Wrapper"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "IFE3KyHWrN-1"
      },
      "source": [
        "from gym import spaces\n",
        "import cv2\n",
        "\n",
        "class WarpFrame(gym.ObservationWrapper):\n",
        "    def __init__(self, env):\n",
        "        \"\"\"\n",
        "        Warp frames to 84x84 as done in the Nature paper and later work.\n",
        "        :param env: (Gym Environment) the environment\n",
        "        \"\"\"\n",
        "        gym.ObservationWrapper.__init__(self, env)\n",
        "        self.width = 84\n",
        "        self.height = 84\n",
        "        self.observation_space = spaces.Box(low=0, high=255, shape=(self.height, self.width, 1),\n",
        "                                            dtype=env.observation_space.dtype)\n",
        "        \n",
        "    def observation(self, frame):\n",
        "        \"\"\"\n",
        "        returns the current observation from a frame\n",
        "        :param frame: ([int] or [float]) environment frame\n",
        "        :return: ([int] or [float]) the observation\n",
        "        \"\"\"\n",
        "        frame = cv2.cvtColor(frame, cv2.COLOR_RGB2GRAY)\n",
        "        frame = cv2.resize(frame, (self.width, self.height), interpolation=cv2.INTER_AREA)\n",
        "        return frame[:, :, None]"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 285
        },
        "id": "kb1LXTXKs6qH",
        "outputId": "21805253-adbf-4ddd-dea2-f4e34f4cee04"
      },
      "source": [
        "\"\"\" \n",
        "[OPTIONAL]\n",
        "\n",
        "Diese Zelle ist optional ausführbar und dient zur Visualisierung des Wrappers.\n",
        "Die Zelle hat keinen Einfluss auf den Agenten\n",
        "\"\"\"\n",
        "\n",
        "def WarpFrameEnv(env_name):\n",
        "    env = gym.make(env_name)\n",
        "    env = WarpFrame(env)\n",
        "    return env\n",
        "\n",
        "normal_env = gym.make(game)\n",
        "wrapped_env = WarpFrameEnv(game)\n",
        "\n",
        "normal_env.reset()\n",
        "wrapped_env.reset()\n",
        "action = normal_env.action_space.sample()\n",
        "\n",
        "normal_state, _, _, _ = normal_env.step(action)\n",
        "wrapped_state, _, _, _ = wrapped_env.step(action)\n",
        "\n",
        "wrapped_state = wrapped_state[: , :, 0]\n",
        "\n",
        "fig, axs = plt.subplots(1, 2)\n",
        "fig.suptitle('Warp Frame', fontsize=20)\n",
        "axs[0].imshow(normal_state)\n",
        "axs[0].set_title(\"Normal\", fontsize=16)\n",
        "axs[1].imshow(wrapped_state, cmap=\"gray\")\n",
        "axs[1].set_title(\"Warp Frame\", fontsize=16)\n",
        "plt.show()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAEMCAYAAAA/Jfb8AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3de5hcVZnv8e+vu9MdciEJJGSQAAkQkYsaMINcJMMQRcQL6nGQHAfxMgd9BAfUOQM6HuE4w4jjBXVUNEoERwQZEeUoIzDhEphRIAEGgYAQCCSZkAQIkA5JOp1+zx97VdhUqrqru6q6qiu/z/PUU11r397ahLd2rVp7vYoIzMystbQ1OgAzM6s9J3czsxbk5G5m1oKc3M3MWpCTu5lZC3JyNzNrQU7uZmYtyMndypL0Bkkh6c4yy+el5SFpRonlu0jaLOklSV31j7g6ki7LvZ9Sj1sbHaNZpToaHYA1tXuB9cAbJO0aES8WLZ8LBCDgeODSouXHAF3ATRGxpd7B1tCvgPtKtC8f5jjMhszJ3cqKiL50tfoe4M+A/1e0yvHArcDrKJ3cj0/PC+sXZV38MiIua3QQZtVwt4wNpJCYj883SpoOzEjLbwP+vMS2r0jukl4t6SJJiyWtk7RF0pOS5kuaVryxpONSd8gFko6Q9BtJz6W26Wmd5ekxQdK3Ja1KXUEPSfprSarBOSiO69YUQ6ekL0h6JL2Xy9LyCZL+t6SbJa2U1JPe73WSjiqzz0j7nSppgaQ1kjZK+k9Jx6Z1xkr6SjpnWyQ9KOkv+olznqRbJD2fzslSSZ8fCV1kVj1fudtAbk7Pc4va5+aWvwC8V9LBEfEQgKRdgdlk3Tr3pHXfC3wcuAX4T6AHOAT4K+CdkmZHxKoSMRwFfBa4A1gATE7bFnQC/w5MBK5Kr/8H8E3gQODMQb/rylwD/Cnwb8AvgbWp/SDgQmAR8Buyc7AP8C7gbZLeGRG/LbG/icB/ABuAK4HdgFOBG9KHwvdT26+BUcA84GeSVkTE7/M7krQA+DCwMsX5PHAk8PfAXElviYjeWpwEa1IR4Ycf/T6A/wb6gCm5tivIklAHWYIO4Kzc8nemtl/k2vYCukrs/wRgG3BJUftxaR8BfKxMbMvT8jvy+yZLgsvSsjkVvs/L0vq/BC4o8ZiY1rs1rXc/MLnEfiaUaZ+WzuXSEssK7/N7QFuu/bTU/hxZt9jo3LJj07Jri/b1ocK5B3YpWnZBWnZ2o/9d+VHfR8MD8KP5H8C/pIRwSq7tv4Hrc6/XFCXyi9M2Z1Z4jPuBx4vaCsn93n62KyT3Y0ssKyS5H1UYQyG5l3tMT+sVkvvJQziX30rb7lPUHsBGYHxRezuwNS3fr8T+ngCeKGq7N20zscT67cAzwF2N/nflR30f7paxStwM/CVZH/rVkg4C9iRL4AW3Am+R1BYRfZT4MTX1f3+ALOm+HphElmwK8l0teXcNEF8vWTdPsVvT82EDbF/sw1HZD6pl45J0DHA2WZfSHmRdRXl7AU8Vtf0xIjbkGyJim6Q1wNiIeLzEoVYBb8wddwzZuX0GOKfMTw5byLqOrIU5uVslCgl6btHzzbl1bgVOAQ6T9BTwWmBVRDycW+frwDnAauAGssS0KS37ELBvmeM/PUB8z0TEtn62mzDA9kNVMi5J7wF+DmwGbiLrHtpI1rV1HNnIo1I/ar5Q5ji9AyzL/388iWxo6hTg/H6jt5bm5G4DioinJC0DDpC0N9lV+fNkX/8LbknPxwNPkiWY/FX7HsBfAw8ARxdfoUqa118IA4Q4WVJ7iQT/J+m5XGKsSkSUi+vvyb6FzI6IpfkFkr5PltzrpfBe742Iw+t4HGtyHgpplSok6jeTXX3elrpfAEhX6E+TJfdS49v3I/v3dmOJxD4tLR+qDuDoEu3Hped7SyyrpwOAh0ok9jbgTfU8cER0Aw8Ch0jarZ7Hsubm5G6VKnTBfIrsq/8tJda5hWwExwnpdT65L0/Pb5K0vZ9d0jjgB1T/LfJL+fHbKbF9Pr38UZX7HqzlwExJr8rFI7KRKgcPw/G/TtbHv0DSxOKFkiZJ8lV9i3O3jFXqZrLukdfmXhe7hWzs9QzgkciNWY+IpyVdRTZu+z5JN5L1hb+FrG/6PmDWEGNbTdaH/YCk68jGgL+P7Eff70bEoiHud6guJhvSeK+ka8hGrhxDltj/H9kw0bqJiAWS3gB8Algm6QayH293I/tvM4fsA+/j9YzDGstX7laRiFgH/CG9fIas77xY/mq+1JQDHwX+EdiF7Mait5LdkHM01fWL95B1F91I9uHxsbS/s4GzqtjvkETE98luIFoNnE42QmgF2aiWe/rZtJYxnEn2IfI7snPzabKbqCYAXwG+MRxxWOOo/G9CZs1P0nKAiJje2EjMmouv3M3MWpCTu5lZC3JyNzNrQe5zNzNrQb5yrxFJH0pzcj8vaVLRso7CvOQNCm9IUtm55Y2Ow8wGz8m99iYA5zY6CDPbuTm5196NwCclTa3Hzl1Fx8wq4eRee/+Qnj/f30qpbNy/S+pO5dQWSjqiaJ3LUpm2o1K5tU3AP0manrp5Pi7pS5KelrRB0k8kjZF0gKQb0r4fk3R60X4PkPQvkp6QtEnS45IuKe5OMrORy8m99lYD3wbOkFRyCltJryOrOzqJbKrbDwK7ArdJen3R6hPISsddCbwN+Glu2WeBV5HdBfkF4P1kt71fS1be7T1kRTB+JOmQ3HavIrtj8hyyu0S/SDaN7/VDecNm1nw8t0x9fJnsFvjzgY+UWP4FsoIJcyPieQBJN5FNOHU+Wa3RgnHAX0bErwoNSsWhgWURUbgqvyEVUj4NOC0ifpLWXUx22/n7yGYLJM21sii3v/8EHgNul3RYRAz3LIpmVmO+cq+DiHgO+BrwQUkHllhlDvDrQmJP27wIXMeOc31vJZt/pZR/K3pdKIxxQ26/68kKN+9daJPUKelzkh5OXT1bgdvT4lLxmtkI4+RePxeTFTX+Yollu5F13xR7mqyrJm9dmSpDAOuLXvf00z469/pLZNPP/gR4O3AEL39bGI2ZjXjulqmTiOiW9CWyK/ivFC1+jperBOX9CTsm5nrcZXYq8OOIKPz4W5hX3cxahK/c6+u7ZHVC/6Go/TbgJEnjCw3p73fyclHnehpD1hWT9+FhOK6ZDRNfuddRRGyR9EVgftGivwfeASyU9GWyq/NzyZJuqW6cWvstcLqkP5D9kPpeSpepM7MRylfu9fcj4NF8Q0TcT1bf80XgcuBfgG7gzyLiv4Yhpk+S/Xh7IfAzYDxZBSUzaxGeOMzMrAX5yt3MrAXVLblLOlHSI+n29/PqdRwzM9tRXbplJLUDfySrbL8SuBuYFxEP1fxgZma2g3pduR8BPBYRj0dED9ncKCfX6VhmZlakXkMh9yKbmKpgJfDGcitLKvv1YXSHmDLGPw00o6F851PNo6jMihe3PRMRUxp0eLNh17Bx7pLOAM4AmDS6jfOPm9CoULY7eP/92WvqHhWtu23bNm6+8646R9Tc+trEPeecVPH6M35zL7s/8t91jKi8c367/smGHNisQep1SbyK3ERVwLTUtl1EzI+I2RExe1xno67nzMxaU72S+93ATEkzJHWSzWVyXZ2OZWZmRerSLRMRvZLOIpt6th1YEBEP1uNY9fTS5s1E38s9y2N2GY3kbxnlaFsfXS+8tP11b1cHvWM9yaRZI9Stzz0irmeEV/a5d+lSXtq0efvruUe+0cm9H13Pb+TQy7fXAGHdoXvz5Amva2BEZjsvD0MxM2tBTu5mLULSvFQ4fU5R+9TUvqbENmemZYcOX6SlSbogxVLqcUCj4xtpPOWvWeso9InNyf1deP0SsIek10TEw0XLniXV120SbwKKq4+tKLWilefkbtYiImKVpGVkCTtvDnAzcFD6O5/cjwXuiCrnIZHUFRFbqtlHzp0R0duA47YUd8uYtZZFwFGS8hduc8gKoN9BLvFLmgnsSVYZDEl/KunnklZK2pQm/vtHSbvkDyDpVkl3SHqnpHslbQE+kZaFpAsl/V1uP4skzar2jeW6bQ6VdIOkbuDqtOwESddLWi3pJUkPSPpMmucqv4/lkn4i6bT0/jZJul3STEljJX1f0rOS1kj6WtF5RNIUSd+TtErSllRk/oxq31s9+MrdrLUsIiuZeDhwl6SJwKFkyf1Z4Au5defktgHYB7gPuAzYAByS1t+P7F6VvFcD3yKrKvY4WV3ggg8CTwFnAV1k1cUWSpoZEc8xsPaiUWl9EdGXe/0r4FLgy0ChfT9gIfDPwGZgNlkR+ClA8ay0c4D9yaqfdQLfAK5J7+Ox9F7nAJ8HlpGVy0TSrmQfkLukfT8BvBW4JH2D+OcK3tuwcXI3ay23pec5wF1k3S5bgCVkyX0fSdMjYnla50WyhE5EXFPYibLs+h9p+Y8lnRkRz+aOMxk4ISLuKxHDLmnZxrSvO8mqkX0K+D8VvIfNRa+vAP4y9/pbEfHN/AoR8b2i2G8nS9x/I+lzRR8O44ATI+KFtP6fAN8E7oqIv0nr3CTp7cBfkJI7cDawL/DaiChUV/v39AF6vqRLKu1OGg5O7mYtJCKekLSSLHF/NT3fmWZn/aOktalteXr+j4jYBtuvTP8OeB/Z9CGjcrueSfbhULC8TGIHuL6Q2FNMyyX9HjiqwrdxJK/8QfXZouXXFm8gaU+yq+kTgVfxyty2B/B07vXvCok9KfwGcUPRbh8mm+G24ETgTuCJou6aG4C/Ag4G7t/x7TSGk7tZ61kEvC1dwc7hlUnrDmCOpJuB6cD3c8t+BLyZrCvmPmAjWXL7DlB8q/Hqfo6/w5DL1HZIhfEvGeAK+BXHltRGNr3Jq8gS/MPAJuDdZB9WxbGvL3rd0097fts9gAOArWXi2r2fmIedk7tZ67kN+J9kV8CHk/UdF9xO9uPnn6XXiwAkjSaruXBBvstD0mvLHKO/0TVTy7StKtE+FMXH3p+sj/20iPhJoVHSO2t0vIJngbVk3TOlPFLj41XFyb0ff3roofTlRoh56oH+bZ40lv86Y+72130d7f2sbXVU+IH0PLIp9H+XW3YHcDFwCtnY97tTexfZPFDFV6UfGsLxT5I0NtfnPp3sg+aiIeyrEmPS8/bYJY0CPlDj4/wW+CTwVESsrfG+a87JvR+do0YNvJK9rK2NreM8UVijRcTDqW/9nWRdHN25xfcC3WnZLRGxNW3zQuoX/4yk1cAzwEfICu8M1ibgRklfIfvQ+L9kP8xePNT3NIClwJPAhZK2kSX5T9XhOBcD7wdul3Qx2ZX6WOA1wLER0VTV5jzO3aw1LSK7ar8935h+PP1dWraoaJt5ZKNqvkM2HPJpyndB9OfHwG+AbwOXA+uAuRUOgxy09GPxu8ni/TFZ/Iuo8TeF9CPs0WQTIp5L9lvGArLurFtqeaxaqEuB7MHaZ0JHfOboXRsdBgfvvz+v2qOySmx9fX2uxNQm7jn7bRWvP+P6+xpZiWlJRMxuyMF3Iqlk5oUR8fkBV7a6crdMzkPLlvHQsmWNDmPEaOsL3nDxiJ7V2axlDblbRtLekm6R9JCkByWdndovSLfm3pcelRfZtBFHg3yY2fCo5sq9F/hMRNwjaTywRNJNadnFEfHVSnektnY6xza+QLa1suIhzEMj6USyuxnbgR9GRL1GgIxIEeHP8CYx5OQeEatJNxNExAZJSxnaL+tM2vcgTvn+wqGGYjagT1wzuep9pEmovgO8BVgJ3C3puoh4qOqdm9VYTfrc0zjWw8huzT0GOEvSB4HFZFf3O1w2pZnUzgCYNm1aLcIwq7cjgMci4nEASVeRjZQom9wlRVtb6d7Pjo4O2tra6nr/RGHfo0aNorOzs6Jtent72bp1K319fTTDgIvhJglJjB49mnL/7fIigi1bttDX10dfX9+A69dST08Pvb29Jf8BVZ3cJY0jm1HtnIh4UdIlZDPFRXr+Gtl42VeIiPnAfIBZs2btfP+CbCTai1cWjVgJvLF4pfyFy6hRozjooINK7qyQROqpra2Nzs5Opk+fziGHVHb3/5NPPsmyZcvYuHEjPT09A2/QYjo7O+ns7OSoo45i/PjxA67f09PDkiVL2LBhA93d3QOuX0uPPFL+ptiqknu6C+wa4IqI+AVARKzJLf8B8OtqjmE20uQvXMaMGVP2yn24FeLo6+tj3bp1vPTSS0D2ATR16lTa29vr/k1ipCich8K5eOmll1i3bt325ZMmTWLcuHFNfb6GnNzTpESXAksj4uu59j1TfzzAe4AHqgvRrGmsIpstsWAatZsvZVg9+uijrF6d/W86YcIEJkyYQFdXV8VdNzubF198kXvuuWd7t8trX/ta9t1336Y+X9VcuR8DnAb8QVJh6s/PAfNS1ZUgm1b0Y1VFaNY87gZmSppBltRPJZuga8TZGfvSdzbVjJa5g9JDl31Xi7WkiOiVdBbZbeftwIKIaKbC0mbb+Q5Vs0GIiOvxBYyNAM3xS4+ZmdVUU1y5d699itu+cWajwzAzaxlNkdx7Nr7IisU3NjoMs52GJNrbs2IqhSGSzTqkrxlIesWwx5FwrpoiuZvZ8BozZsz2G3TGjh07IpJVI7W3tzN+/Pjto4w6Ozub/pw5uZvthCZOnLj9in306NHbp0Kw0rq6upgyZcr25D5mzJimP19O7mY7mba2NqZPn75Dm5U3fvx4Dj300O2vm/2qHZzczVpeX18fGzduZM2aNQOvTHY3ZmHisJ1RX18fvb29PPvss2zatGnA9bdu3UpPT0/TnS8nd7MW1tfXx7Zt29iwYQMrVqwYeAOgu7ubrVu37rR3sfb19dHT08OaNWsYNWpURetv3ryZ3t7eYYiuck7uZnU0ZswYDjvssIbG0NbWxqhRoxg9enRF62/ZsoXe3t6GTGHbDAqjYnbZZZftI4r609fXx9SpU7d/kA6n/j6wndzN6mj06NEceOCBjQ7DWlR/H9hO7mZ1tNtuu/H+97+/0WFYi7riiivKLnNyN6ujQqEMs3rob8phj38yM2tBTu5mZi2oFjVUlwMbgG1Ab0TMlrQb8DNgOlnBjlNKFck2M7P6qFWf+59HxDO51+cBCyPiIknnpdfn1uhYZiNGT09PxePLzQarvwLm9fpB9WTguPT35cCtOLnbTmjjxo3cddddjQ7DWtTGjRvLLqtFcg/gRkkBfD9Vfp+aK5L9NDC1eCNJZwBnAEwa7a5/a03d3d3cdtttjQ7DWlR3d3fZZbVI7m+KiFWS9gBukvRwfmFEREr8FLXPB+YD7DOhY+e8z9la3oYNG7j99tsbHYa1qA0bNpRdVvUlc0SsSs9rgWuBI4A1kvYESM9rqz2OmZlVrqrkLmmspPGFv4ETgAeA64DT02qnA7+q5jhmZjY41XbLTAWuTXMbdwA/jYjfSrobuFrSR4EngVOqPI6ZmQ1CVck9Ih4HXl+i/VlgbjX7NjOzofMwFTOzFuTkbmbWgpzczcxakJO7mVkLcnI3KyJpb0m3SHpI0oOSzk7tu0m6SdKj6XlSo2M1K8fJ3WxHvcBnIuJg4EjgTEkH8/KEeDOBhem1WVNycjcrEhGrI+Ke9PcGYCmwF9mEeJen1S4H3t2YCM0G5jJ7Zv2QNB04DLiTCibES9tsnxRv1KhR9Q/SrARfuZuVIWkccA1wTkS8mF8WEUE2I+oOImJ+RMyOiNkdHb5+ssZwcjcrQdIossR+RUT8IjV7QjwbMZzczYoomyzpUmBpRHw9t8gT4tmI4e+MZjs6BjgN+IOk+1Lb54CL8IR4NkI4uZsViYg7AJVZ7AnxbERwt4yZWQtycjcza0FD7paRdCDws1zTfsAXgInA/wLWpfbPRcT1Q47QzMwGbcjJPSIeAWYBSGoHVpHVUP0wcHFEfLUmEZqZ2aDVqltmLrAsIp6s0f7MzKwKtUrupwJX5l6fJel+SQvKzZwn6QxJiyUt7u4peaOfmZkNUdXJXVIn8C7gX1PTJcD+ZF02q4Gvldouf4v2uM5yo87MzGwoanHl/jbgnohYAxARayJiW0T0AT8AjqjBMczMbBBqkdznkeuSKcy9kbwHeKAGxzAzs0Go6g5VSWOBtwAfyzX/k6RZZDPmLS9aZmZmw6Cq5B4RG4Hdi9pOqyoiMzOrmu9QNTNrQU7uZmYtyMndzKwFObmbmbUgJ3czsxbk5G5m1oKc3M3MWpCTu5lZC3JyNzNrQU7uZmYtyMndzKwFObmbmbUgJ3czsxbk5G5m1oIqSu6pFupaSQ/k2naTdJOkR9PzpNQuSd+S9Fiqo3p4vYI3M7PSKr1yvww4sajtPGBhRMwEFqbXkJXdm5keZ5DVVDUzM6C9vZ2Ojg66urro7Oyko6ODtrbad6JUVKwjIhZJml7UfDJwXPr7cuBW4NzU/uOICOD3kiZK2jMiVtciYLPhIqkdWAysioh3SJoBXEVWoGYJcFpE9DQyRhtZOjo6mDJlCuPGjWPy5Mls2rSJDRs28Mwzz/DCCy/U9FjVfFxMzSXsp4Gp6e+9gBW59VamNrOR5mxgae71l4GLI+IAYD3w0YZEZSNaV1cXo0ePZvz48YwdO5bRo0fT0VFVUbySavJdIF2lx2C2kXSGpMWSFnf3DGpTs7qTNA14O/DD9FrA8cDP0yqXA+9uTHQ2UrW1tTFq1ChGjx7NuHHjGDNmDJ2dnbS3t9f+WFVsu0bSngDpeW1qXwXsnVtvWmp7hYiYHxGzI2L2uE5VEYZZXXwD+FugL73eHXg+InrT67LfSPMXLr29vaVWsZ1Ydp2QPRf+rodqkvt1wOnp79OBX+XaP5hGzRwJvOD+dhtJJL0DWBsRS4ayff7CpR5ft80qUdG/PElXkv14OlnSSuB84CLgakkfBZ4ETkmrXw+cBDwGvAR8uMYx205gW2cH937ihO2vx/33el5z9e+G6/DHAO+SdBIwGtgV+CYwUVJHunov+Y3UrFlUOlpmXplFc0usG8CZ1QRlBkDby19ZYxh77iLis8BnASQdB/xNRHxA0r8C7yMbMZP/tmrWdHyHqlnlzgU+Lekxsj74Sxscj1lZ7hA060dE3Ep2DwcR8ThwRCPjMauUr9zNzFqQk7uZ2TDq6+ujt7eXrVu3snXrVvr6+sh+qqwtd8uYmQ2Tvr4+uru72bZtGx0dHWzevJnu7m62bNlS82M5uZuZDZOIYMuWLUQE69evZ+vWrWzevJl63Ozm5G5mNky2bdvG+vXrAVi3bh19fX0DbDF07nM3M2uAeiZ2cHI3M2tJTu5mZi3Iyd3MrAU5uZuZtSAndzOzFuShkNaU2rb2ctBP7njFazOrnJO7NSUFjF1b24LBZjuTAbtlJC2QtFbSA7m2r0h6WNL9kq6VNDG1T5e0SdJ96fG9egZvZmalVdLnfhlwYlHbTcChEfE64I+kwgbJsoiYlR4fr02YZmY2GAMm94hYBDxX1HZjrlDw78lKjpmZWZOoxWiZjwD/lns9Q9K9km6TdGy5jfIV4rt7aj/dpZnZzqyqH1Ql/R3QC1yRmlYD+0TEs5LeAPxS0iER8WLxthExH5gPsM+EDmd3M7MaGvKVu6QPAe8APpCKYhMRWyLi2fT3EmAZ8OoaxGlmZoMwpOQu6UTgb4F3RcRLufYpktrT3/sBM4HHaxGomZlVbsBuGUlXAscBkyWtBM4nGx3TBdwkCeD3aWTMHOCLkrYCfcDHI+K5kjs2M7O6GTC5R8S8Es2Xlln3GuCaaoMyM7PqeG4ZM7MW5ORuZtaCnNzNzFqQk7uZWQtycjcza0FO7mYlSJoo6edp9tOlko6StJukmyQ9mp4nNTpOs3Kc3M1K+ybw24h4DfB6YClwHrAwImYCC9Nrs6bk5G5WRNIEshvyLgWIiJ6IeB44Gbg8rXY58O7GRGg2MCd3sx3NANYBP0oznP5Q0lhgakSsTus8DUwttXF+xtPeXpcHtMZwcjfbUQdwOHBJRBwGbKSoCyZNlldyNtOImB8RsyNidkeHK1laYzi5m+1oJbAyIu5Mr39OluzXSNoTID2vbVB8ZgNycjcrEhFPAyskHZia5gIPAdcBp6e204FfNSA8s4r4O6NZaZ8ErpDUSTZt9YfJLoaulvRR4EnglAbGZ9YvJ3ezEiLiPmB2iUVzhzsWs6Fwt4yZWQsaMLlLWiBpraQHcm0XSFol6b70OCm37LOSHpP0iKS31itwMzMrr5Ir98uAE0u0XxwRs9LjegBJBwOnAoekbb5bKLtnZmbDZ8DkHhGLgEpL5Z0MXJUKZT8BPAYcUUV8ZmY2BNX0uZ8l6f7UbVOYQGkvYEVunZWpbQf5u/i6e0reC2JmZkM01OR+CbA/MAtYDXxtsDvI38U3rlNDDMPMzEoZUnKPiDURsS0i+oAf8HLXyypg79yq01KbmZkNoyEl98It2Ml7gMJImuuAUyV1SZoBzATuqi5EMzMbrAFvYpJ0JXAcMFnSSuB84DhJs8gmTloOfAwgIh6UdDXZrdq9wJkRsa0+oZuZWTkDJveImFei+dJ+1r8QuLCaoMzMrDq+Q9XMrAU5uZuZtSAndzOzFuTkbmbWgpzczcxakJO7mVkLcnI3M2tBTu5mZi3Iyd3MrAU5uZuZtSAndzOzFuTkbmbWgpzczcxa0ICzQprZ0Emirc3XUFYfUvkqdk7uZnU0duxYjj766EaHYS1qxYoVZZdVUqxjAfAOYG1EHJrafgYcmFaZCDwfEbMkTQeWAo+kZb+PiI8POXKzEa6rq4v999+/0WFYi+rq6iq7rJIr98uAbwM/LjRExPsLf0v6GvBCbv1lETFr0FGataDdd9+d0047rdFhWIu67LLLyi6rpBLTonRFvgNlHT6nAMcPLTSz5iTpU8BfkZWS/APwYWBP4Cpgd2AJcFpE9PS3n46ODnbfffc6R2s7q46O8im82l96jgXWRMSjubYZku6VdJukY8ttKOkMSYslLe7uiSrDMKsdSXsBfw3MTl2R7cCpwJeBiyPiAGA98NHGRWnWv2qT+zzgytzr1cA+EXEY8Gngp5J2LbVhRMyPiNkRMXtcZ/lffM0apAPYRVIHMIbs3/bxwM/T8suBdzcoNrMBDTm5p3/07wV+VmiLiC0R8Wz6ewmwDHh1tUGaDaeIWAV8FXiKLKm/QJ8o7CQAAAd9SURBVNYN83xE9KbVVgJ7NSZCs4FVc+X+ZuDhiFhZaJA0RVJ7+ns/YCbweHUhmg0vSZOAk4EZwKuAscCJg9h+e5fjunXr6hSlWf8GTO6SrgR+BxwoaaWkQj/jqbyySwZgDnC/pPvIvr5+PCKeq2XAZsPgzcATEbEuIrYCvwCOASamb6wA04BVpTbOdzlOmTJleCI2K1LJaJl5Zdo/VKLtGuCa6sMya6ingCMljQE2AXOBxcAtwPvIRsycDvyqYRGaDcD3RZsViYg7yb553kM2DLINmA+cC3xa0mNkwyEvbViQZgPw9ANmJUTE+cD5Rc2PA0c0IByzQfOVu5lZC/KVu1kdLVmy5BlJG4FnGh1LzmQcz0CaLaZy8exbbgMnd7M6iogpkhZHxOxGx1LgeAbWbDENJR53y5iZtSAndzOzFuTkblZ/8xsdQBHHM7Bmi2nQ8Ti5m9VZRDRVonA8A2u2mIYSj5O7mVkLcnI3M2tBTTEUsm1UF7vuuV+jw7CWtmTYjyjpROCbZMU+fhgRFzUghr3JSmROJasqNT8ivilpN7LpuqcDy4FTImL9MMbVTjZfz6qIeIekGQyyylUNY5kI/BA4lOwcfYSsDnRDzk+tqoApovFVkGbNmhULFy5sdBjWwiZPnrxkOMctp+T1R+AtZHO/3w3Mi4iHhiuGFMeewJ4RcY+k8WSJ4d3Ah4DnIuIiSecBkyLi3GGM69PAbGDXlNyvBn4REVdJ+h7wXxFxyTDFcjlwe0T8UFInWXGWz9GA85OqgN0BHBwRm9J5uR44iUGeH3fLmNXHEcBjEfF4usK6imyO+GEVEasj4p709wZgKVmRkZPJqknBMFeVkjQNeDvZ1XKhFnNDqlxJmkA2VfmlABHRExHP08DzQ42qgDm5m9XHXsCK3OuGV25Khe4PA+4EpkbE6rToabJum+HyDeBvgb70encaV+VqBrAO+FGq/fxDSWNp0PmpZRWwSop17C3pFkkPSXpQ0tmpfTdJN0l6ND1PSu2S9C1Jj0m6X9LhQ3ubZlYrksaR1Vo4JyJezC+LrG92WPpnJb0DWJvKcDaDDuBw4JJU+3kjcF5+hWE+P1VVAcur5Mq9F/hMRBwMHAmcKelgshOwMCJmAgt5+YS8jay83kzgDGBY+s3MmswqYO/c67KVm+pN0iiyxH5FRPwiNa9J/fGFfvm1wxTOMcC7JC0n66o6nuxH54qqXNXBSmBlmsMfsq6Pw2nc+amqCljegMl9CH12JwM/jszvU1B7VvzWzFrD3cBMSTPSj3SnAtcNdxCpP/tSYGlEfD236DqyalIwjFWlIuKzETEtIqaTnZObI+IDvFzlarjjeRpYIenA1DQXeIgGnR9yVcDSf7tCPIM+P4MaCllhn125vsbVuTYknUF2Zc+0adMGE4ZZ04uIXklnATeQDYVcEBEPNiCUY4DTgD+k2saQjQS5CLg61UR+EjilAbHlnQtcJekfgHsZ3ipXnwSuSB/Cj5MNPWyjAecnIu6UVKgC1kt2LuYDv2GQ56fioZCpz+424MKI+IWk5yNiYm75+oiYJOnXwEURcUdqXwicGxGLy+3bQyGt3oZ7KKRZo1U0WmaQfXZN09doZrazqmS0zGD77K4DPphGzRwJvJDrvjEzs2FQSZ/7YPvsCndTPQa8RNZ/ZWZmw2jA5J76zlVm8dwS6wdwZpVxmZlZFXyHqplZC3JyNzNrQU7uZmYtqCmm/JW0jmxOh2caHcsQTWbkxg4jO/5KY983IqbUOxizZtEUyR1A0uKRepPJSI4dRnb8Izl2s3pyt4yZWQtycjcza0HNlNznNzqAKozk2GFkxz+SYzerm6bpczczs9pppit3MzOrESd3M7MW1PDkLulESY+kmqvnDbxF40laLukPku6TtDi1lawp2wwkLZC0VtIDubYRUQO3TOwXSFqVzv99kk7KLftsiv0RSW9tTNRmjdfQ5C6pHfgOWd3Vg4F5qT7rSPDnETErN8a6XE3ZZnAZOxbZHSk1cC+jdIHgi9P5nxUR1wOkfzunAoekbb6b/o2Z7XQafeV+BPBYRDweET1kBXNPbnBMQ1WupmzDRcQi4Lmi5hFRA7dM7OWcDFwVEVsi4gmyaaePqFtwZk2s0cm9XL3VZhfAjZKWpFqwUL6mbLMabA3cZnNW6jZakOsCGymxm9Vdo5P7SPWmiDicrAvjTElz8gvTnPYjZozpSIuXrKtof2AWWeH1rzU2HLPm0+jkPiLrrUbEqvS8FriW7Kt/uZqyzWrE1sCNiDURsS0i+oAf8HLXS9PHbjZcGp3c7wZmSpohqZPsx7DrGhxTvySNlTS+8DdwAvAA5WvKNqsRWwO36DeA95Cdf8hiP1VSl6QZZD8K3zXc8Zk1g0pqqNZNRPRKOgu4AWgHFkTEg42MqQJTgWuzuuF0AD+NiN9KupvSNWUbTtKVwHHAZEkrgfMZITVwy8R+nKRZZF1Jy4GPAUTEg5KuBh4CeoEzI2JbI+I2azRPP2Bm1oIa3S1jZmZ14ORuZtaCnNzNzFqQk7uZWQtycjcza0FO7mZmLcjJ3cysBf1/eoVR84y+mGUAAAAASUVORK5CYII=\n",
            "text/plain": [
              "<Figure size 432x288 with 2 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "FjZQ6BOPq90B"
      },
      "source": [
        "## Frame Stack Wrapper"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "qYTs63ANrFyb"
      },
      "source": [
        "from collections import deque\n",
        "\n",
        "class FrameStack(gym.Wrapper):\n",
        "    def __init__(self, env):\n",
        "        super().__init__(env)\n",
        "        self.frames = deque(maxlen=4)\n",
        "        low = np.repeat(self.observation_space.low[np.newaxis, ...], repeats=4, axis=0)\n",
        "        high = np.repeat(self.observation_space.high[np.newaxis, ...], repeats=4, axis=0)\n",
        "        self.observation_space = gym.spaces.Box(low=low, high=high, dtype=self.observation_space.dtype)\n",
        "\n",
        "    def step(self, action):\n",
        "        obs, reward, done, info = self.env.step(action)\n",
        "        self.frames.append(obs)\n",
        "        frame_stack = np.asarray(self.frames, dtype=np.float32)\n",
        "        frame_stack = np.moveaxis(frame_stack, 0, -1).reshape(1, 84, 84, -1)\n",
        "        return frame_stack, reward, done, info\n",
        "\n",
        "    def reset(self, **kwargs):\n",
        "        obs = self.env.reset(**kwargs)\n",
        "        for _ in range(4):\n",
        "            self.frames.append(obs)\n",
        "        frame_stack = np.asarray(self.frames, dtype=np.float32)\n",
        "        frame_stack = np.moveaxis(frame_stack, 0, -1).reshape(1, 84, 84, -1)\n",
        "        return frame_stack"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 311
        },
        "id": "BFsMn2h-s04j",
        "outputId": "6ce5f29a-25b8-4954-8980-a7daafed5f3d"
      },
      "source": [
        "\"\"\"\n",
        "[OPTIONAL]\n",
        "\n",
        "Diese Zelle ist optional ausführbar und dient zur Visualisierung des Wrappers.\n",
        "Die Zelle hat keinen Einfluss auf den Agenten\n",
        "\"\"\"\n",
        "\n",
        "def FrameStackEnv(env_name):\n",
        "    env = gym.make(env_name)\n",
        "    env = WarpFrame(env)\n",
        "    if 'FIRE' in env.unwrapped.get_action_meanings():\n",
        "        env = FireResetEnv(env)\n",
        "    env = FrameStack(env)\n",
        "    return env\n",
        "\n",
        "env = FrameStackEnv(game)\n",
        "env.reset()\n",
        "\n",
        "for _ in range(1, 5):\n",
        "  # Führe eine zufällige Aktion aus\n",
        "  state, _, _, _ = env.step(env.action_space.sample())\n",
        "\n",
        "# Stack umformen, damit das Plotten der vier Bilder gelingt\n",
        "state = state.reshape(84, 84,4)\n",
        "\n",
        "# Frame Stack plotten\n",
        "fig, axs = plt.subplots(1,4, figsize=(15, 5))\n",
        "fig.suptitle('Frame Stack', fontsize=20)\n",
        "for i in range(state.shape[2]):\n",
        "    axs[i].imshow(state[:, :, i], cmap=\"gray\")\n",
        "    axs[i].set_title(\"Frame \"+str(i+1), fontsize=16)\n",
        "plt.show()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAA2cAAAEmCAYAAADm0OiFAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3de5hlBXnn++9bVV1dfYOmoenToRFwwAsxEWMP0dFjHIEoxggmxEFjpkk4IZOTi04yE3GSMzHPJBM9k4xxZnKccERhEsZLiAbGOBBBMJo5ohhbVBBBBO2moUG6m75VVVfVe/5Yq9qiuqpr1669Vq299/fzPPvZte7vXlW/3v3uddmRmUiSJEmSltfAchcgSZIkSbI5kyRJkqRGsDmTJEmSpAawOZMkSZKkBrA5kyRJkqQGsDmTJEmSpAawOZMkqQEiIiPizuWuQ5K0fGzOJKkHlP+xP97jiuWucTlExM9ExC0RsTsijkTE9yLi3oj4i4jYNmveV5b76p3LVK4kqc8NLXcBkqSO+r15xm+vtYoGiIhrgF8EDgN/A3wbCOB5wE8CrwSuX676JEmazeZMknpIZr5zuWtogoh4OUVjtgN4aWbumDV9BUVzJklSY3haoyT1iYg4szxt77qIeE5EfKQ83W8qIl5ZzvPiiHhvRHwlIp6KiNGIeCAi/jgiTppjnVdMnzYZERdFxGcj4kBEPBERH4yI9eV8L4qIT0TEnnL6zRFx5jx1boiIP4yI+yLicETsi4jbI+LHF/Fy/0n5/FezGzOAzDySmZ+asc3rgDvKwd+ddUro9L45MSL+dUR8OiJ2RMR4+TpvjoiXzldIRDwvIj4QEQ9HxFi5zz8bEb/cygsptzkVEX8fERtafP2SpC7kkTNJ6j//CLgL+CZwA7AKeLqc9ovAG4DPALdRfIj3YuA3gIsj4kczc/8c63w98DrgE8B/pWiOrgDOjIh3ALcDnwWuBX6I4rTCZ0fED2fm1PRKIuIM4E7gzHL+W4A15bpviYhfysz/t4XX+L3y+ZwW5gX46/J5G8Vrv3PGtIfL5+cDfwD8HcVpknuAZ1G89osj4icz85aZK42InwD+ElhZvpYPAeuBFwK/BbxvvoIiYgD4E+DXgI8BP5uZoy2+HklSF4rMXO4aJElLFBHT/5jPdc3Zw5l5XXmk6tvluD/MzH8zx3rOAHZk5uSs8VcC7weuzsx3zxh/BfBBYBK4IDM/U44fAG4FLqRoYn4tM2+Ysdy1wC8Al2bmTTPG3wm8AnhzZn54xvj1FA3Tc4EzM/PxBfbHacDXgROB/wF8GPgi8GDO88ZXHiG7A/i9uU4PjYgTgRWZ+eSs8VuALwD7MvP5M8afAnyLovm9aHrfzFxu5lG98nf4mcx8ZUSMUDTOPwX8F+CtM5tYSVJv8rRGSeotvzvH44pZ8zzOPDcOycxHZjdmpQ9QHF179Tzb/dDM5qNsJP68HPzazMas9N/K5/OmR0TEC4EfozgV8cMzZ87MveVrGQF+ep4aZs6/k+II4LcojtLdQHGkcF9598a3RMTgQuuZtc59sxuzcvwO4EbgeRHxrBmTtgEnAO+b3ZjNWO4Y5amLt5X1vz0zf83GTJL6g6c1SlIPycxoYbavZObYXBPKG2X8EnA5cC7FkaeZH+SdNs86755j3KPl85fmmLazfN4yY9z0dVsnznM7+43l8/PnmHaMzLwjIp4DvIyi6XtR+fOry8e2iHjdfPtiLhHxMuCtZa2nAsOzZjkN+E7580vK5//Z6vqBTcDfA88G3pKZ/30Ry0qSupzNmST1n8eOM+0jFEdsHgJuKuedbl7eRnHt1Fz2zTFuooVpK2aMO7l8vqh8zGftcaY9Q3nE6bPlg4iIct3XU5xy+csU13UtKCLeQHGEbBT4FMVRuYPAFMWdH3+MZ+6f9eXzTlr3v1EcbdsBfG4Ry0mSeoDNmST1n/muudpK0ZjdBlycmRMzpg1Q3MCiStNN3Fsz8z9VsYHyerO/jYjfobiG7lW02JwB/w4YB7Zm5n0zJ0TEn1E0ZzPtLZ9PA77a4ja+UtZ1HfB3EfGqzHyoxWUlSV3Oa84kSdPOLp9vntmYlc6nuLFFlT5fPv/vFW8HYPqOkzNPA52+1m6+a9HOBu6dozEbAF4+x/zTr+fixRSWmX9BcVrpD1A0aM9ZzPKSpO5lcyZJmvZw+fzKmSMj4lTgT6veeGbeTXH64U9FxC/MNU9E/FBZz3FFxGsi4qfKa+hmT1tLcYomFLfFnzZ9+/1nMbeHgXMi4gdmrCuAd1Jcnzfb9RQ3UfnliHjFHHVsOXaRQmbeCFwGnAJ8JiJ+cL55JUm9w9MaJUnTvkhxM4qfioj/RXHN0yaKIz/38/0bfFTpzcCngWsj4tcpvo9tL8WNQ34YeAHFzTh2L7Ce5wHvAfZExGeBByiuc9sC/ATF9WB3Udymftr9FNeHXR4RR4BHKE4B/fPMfKRc338FvhwRfwUcobjByLkUt+v/yZkFZOaTEfFmiuvU7oiI/wncQ3FN2Q8DpwNnzfcCMvPmiLgE+DhwZ0RcmJlfWeB1S5K6mM2ZJAmAzJyMiNcDvw+8Fvh1imbl/eW4e2uoYUdEvJjii5d/GvhZitMMHyu3/59p7fqtv6A4anURxRc+v4LiRiJ7ge0UXwz9/swcn7HtyfKmH+8CfgZYR3Ha4+eARzLzzyJijOKo2zbgMMWRvp8va31Gc1au82/Ka/neDlwA/DjF9759A/jDFvbHrRHxWorm746IeHVmfrGF1y9J6kJ+CbUkSZIkNYDXnEmSJElSA9icSZIkSVID2JxJkiRJUgPYnEmSJElSA9icSZIkSVID2JxJkiRJUgPYnEmSJElSA9icSZIkSVID2JxJkiRJUgPYnEmSJElSA9icSZIkSVID2JxJkiRJUgPYnEmSJElSA9icSZIkSVID2JxJkiRJUgPYnEmSJElSA9icSZIkSVID2JxJkiRJUgPYnEmSJElSA9icSZIkSVID2JxJkiRJUgPYnEmSJElSA9icSZIkSVID2JxJkiRJUgPYnEmSJElSA9icSZIkSVID2JxJkiRJUgPYnEmSJElSA9icSZIkSVID2JxJkiRJUgPYnEmSJElSA9icSZIkSVID2JxJkiRJUgPYnEmSJElSA9icSZIkSVID2JxJkiRJUgPYnEmSJElSA9icSZIkSVID2JxJkiRJUgPYnEmSJElSA9icSZIkSVID2JxJkiRJUgPYnEmSJElSA9icSZIkSVID2JxJkiRJUgPYnEmSJElSA9icSZIkSVID2JxVJCKuiIic53Hhcte3VBHxGxHxPyJiV/ma3rncNan/9HLOIuI5EfHeiLgnIg6UWbs5Il643LWp//R41tZFxEcj4sGIOBgReyPiCxHxluWuTf2ll3M2W0RcXr6uHctdS9MMLXcBfeBngNl/ePcuRyEd9ovA08BfA/9imWuRejFnPw78U+B64B+A9cBvAZ+PiJdn5peWszj1rV7M2jAwAfwh8DCwEvhnwJ9HxMbMfM8y1qb+1Is5Oyoi1gN/Ajy23LU0kc1Z9bZn5oOtzBgRKzNzrOqCOuQHM3MqIoawOdPy68WcfRj408zM6RER8WmK/zy+Ffjny1SX+lvPZS0zvwe8edboT0bEc4BfAGzOVLeey9ks/zfwFWAX0FNHBDvB0xqXyYxD16+IiL+MiL3AXeW0fxwRN0bEjog4HBH3R8S/j4hVs9ZxZ0R8LiJeExHby3m/HBE/GhFD5TK7IuKpiLguItbMWn51RLw7Ir4dEePl829HxIJ/F5k51dEdIlWgm3OWmU/ObMzKcfuAbwKndWQHSR3SzVk7ju9RHFGTGqEXchYRLwPeAvxKh3ZLz/HIWfUGozi6NC0zc3LG8A3Ah4DL+P7v41nAduA6YD/wg8C/BZ4NXD5r/WcD/wH4A+AAxacRN5ePIeAK4PnlPLspTouirOlW4Fzg3wFfBV4C/F/ABuA3l/KipZr1Rc4iYgPwAuCDi1lO6qCezVpEBDAInAj8NPBq4MqFlpMq0JM5i4gVwDXAf8jMB4vI6RiZ6aOCB8Ufds7x+Nys6e9ZYD1BEZS3AFPAyTOm3QkcAZ49Y9zry/XeNms9HwO+PWP458r5XjFrvt8GxoFTW3ydQ+V63rnc+9xH/z36JWczlrsBOAScvdz73kd/Pfoha8Cvznhd48D/udz73Ud/PXo9Z8DvAA8CI+XwdcCO5d7vTXt45Kx6b+CZF3XunzX947MXiIgTKP7QLwNOB1bMmHwOxakW076ZmQ/NGP5G+XzrrNV+A/jJiIgsEvEa4BHgf836dOZvgd+n+CTk5uO8LqlJej5nEfEOiutirswWr0WQKtDLWfsI8HngFIr/rP7niJjMzD9bYDmp03ouZxFxdlnfGzJzdK55VLA5q97XFviP1K45xn2Q4gLJf0txiPogcD7wp8DIrHn3zBoeP874IYpTNiaAU4EzKD49mcvJx6lZapqezllE/Avg3wO/k5kfaGUZqSI9m7XMfAJ4ohy8JSJWA38UER/IzPnWK1WhF3P2n4BPU9xxeH05bpjijOL1wFhmHj7O8n3D5mz5PeOC/4gYAS6hOE3wvTPG/1CHt/s94NvAG+eZ/nCHtyctp67NWUT8HPD/AH+cmX/QudKkSnRt1uZwN7AN2MSxtzWXllM35uxcisZudgNIOe69wNuWUlyvsDlrnpUUn1DM/lTiig5v5xaKC54PZOY3FppZ6jFdkbOIeAPFp6Hvz8x/1eHapDp0Rdbm8WMUN0vY3aH1SVXphpxdzrFH8K4GXszc3+vWt2zOGiYz90XE54HfjIhdwJMU37PS6Vtn3wD8PHB7RPwxxfdNDAP/iOJc+0sz89B8C0fEVuBMvv91DOdGxGXlz5883rLScuuGnEXEKyjuxvUV4LqIeMmMyWOZ+eUO1yp1XJdk7ZcorpW5jeI/iCdTHBm4DLg6M8fnWk5qim7IWWZ+fva4iLiC4v3szg7X2dVszprpTcD7KM4TPgx8lOJLZz/RqQ1k5pGIeDXFpxZXAWdRnJ/8LeBv+P75x/P5VYrTPab9TPmgXNfDnapVqkjTc/Yqik9DfwT4+1nTHqH4cETqBk3P2lcpTgn7I4rbgT8J3Ae8LjP/plM1ShVres7UoihvZSlJkiRJWkYtfZu3JEmSJKlaNmeSJEmS1ABLas4i4jURcX9EPBgRV3eqKEnPZNak6pkzqXrmTDq+tq85i4hB4JvARRR3N/oi8KbMvLdz5Ukya1L1zJlUPXMmLWwpR87OBx7MzIfK28x+mOJuR5I6y6xJ1TNnUvXMmbSApdxK/zTguzOGdwA/erwFIuK4h+kGBgZYsWIFEUFELKG0xZve5vDwMIODgy0vl5mMj48zNTVFZuLdLxc2c18PDCzu84Em7uvDhw8/mZkbK9zEorJmzgTmrA2+p2HW2tFLWRsfH2diYqLKP1ZzhjlrRy/lDI7/nlb595xFxFUU34VQbHBo/k2ecMIJnHLKKQwNDR0NWl2GhoYYHh7mjDPO4IQTTmh5uYmJCb773e9y4MCB6X/UKqyyN8zc12vXrm15uYmJCXbu3Nm4fb19+/ZHlrsGc6bZzFk1zJpm66Ws3X///ctdAmDOdKxeyhkc/z1tKc3ZTuD0GcNbynHPkJnXANcAjIyM5JYtW+Zd4XSwBgYGav/0Y3BwkKGhIdavX8/JJ5/c8nJHjhzhySefZGxsrDG/8Kabua83bNjQ8nLj4+M89dRT/bivF8yaOdNs5mzRfE/DrLXDrC2KOcOctaOfcraU5uyLwDkRcRZFsC4H3ny8BQYGBli3bt0SNlmN6cOjAwMDjIyMsGrVqqPTpg+DTouIZxxOnf5jWY7D6d1oet9FBCtXrmRkZOTotLkON7uvgUVmzZzJnLXF9zT6+vffFrO2aOaMvv3dt63fctZ2c5aZExHxq8CtwCDwgcz8escqa4hHH32UXbt2HR1et24dZ5xxxtFPadQ5u3bt4tFHHz06vG7dOk4//XSGh4ePe0pDr+uHrJmz+pizufVDzsCs1cmsHcucmbNO68WcLanqzPwk8MkO1dJIe/fu5eGHHz46vHHjRjZv3szAwIAB67C9e/fyyCPfPwX3lFNOYfPmzQwNDTXmAs7l0utZM2f1MWfz6/WcgVmrk1mbmzkzZ53UiznzL0SSJEmSGsDmTJIkSZIawOZMkiRJkhrA5kySJEmSGsDmTJIkSZIawOZMkiRJkhqgO78AoEaDg4MMDg4eHfYWqNUZGBh4xr6e+bN6mzmrjznrb2atPmatf5mz+vRizmzOFrBixQrWrFlzdHhkZISBgYGu+ZbxbuK+7l/+7uvjvu5v/v7r477uX/7u69OL+9rmbAErVqxg1apVR4dXrlxJRHT1L72p5trX0582ub97mzmrjznrb2atPmatf5mz+vRizmzOFnDqqaeybt26o8NDQ0OsWrWKiPAwdYedeuqprF279ujwihUrnhEy9S5zVh9z1t/MWn3MWv8yZ/XpxZzZnM2QmUxNTTE1NXV03MqVK1m5cuWc80/POz1/ZtZSZ6+Ympp6xj473r6e/btxX3cvc1Yvc9a/zFq9zFp/Mmf16oec2ZxR/KInJycZGxvj8ccf59ChQy0vOzExwcGDBxkfH39GMDW3mft69+7d7us+Ys7qY876m1mrj1nrX+asPv2WM5uz0nR3vW/fPsbGxlpebmpqitHRUSYnJ7umI19u7uv+5e++Pu7r/ubvvz7u6/7l774+/bSvF2zOIuIDwOuA3Zn5gnLcBuAjwJnAw8AbM3PPQutatWoVz3/+85dSbyWm7+oSEaxevZqhodZ71qmpKTZs2MDk5OTRzl7z68V9vX379o6sp1NZM2fqxX3dtJyBWVPv7evvfOc7HVmPOTu+Jv7um6wX9/Xx3tNaeXXXAf8F+G8zxl0N3J6Z74qIq8vhty+0opGREc4999wWNin1pevoQNbMmXRc1+F7mjSnz3zmM51a1XWYM6ktCzZnmfl3EXHmrNGXAK8sf74euJMWArZ69WrOO++8RRUo9YtOZc2cSfPzPU2a3+rVqzuyHnMmta/da842Zeau8ufHgE2tLHTiiSdy8cUXt7lJqS8tOmvmTFo039Mk4J3vfGeVqzdnUguWfEOQzMyImPcKu4i4CrgK4FnPehaDg4NL3aTUl46XNXMmdYbvaVL1zJk0v3a/oe3xiNgMUD7vnm/GzLwmM7dm5taNGze2uTmpb7WUNXMmLYnvaVL1zJnUgnabs5uBbeXP24CbOlOOpFnMmlQ9cyZVz5xJLViwOYuIDwH/H/DciNgREVcC7wIuiogHgAvLYUlLYNak6pkzqXrmTGpfK3drfNM8ky5Y7MYyc1FfHCf1k05lzZxJ8/M9TZpfp76k15xJ7VvyDUEW4+DBg3zhC1+oc5NS3zFnUj3MmnrNwYMHl7uEY5gz9Ztam7MjR47w6KOP1rlJqe+YM6keZk295siRI8tdwjHMmfpNrc3Zvn37+OQnP1nnJqW+Y86kepg19Zp9+/YtdwnHMGfqN7U2Z4cOHeKee+6pc5NS3zFnUj3MmnrNoUOHlruEY5gz9Zt2b6UvSZIkSeogmzNJkiRJagCbM0mSJElqAJszSZIkSWoAmzNJkiRJagCbM0mSJElqAJszSZIkSWoAmzNJkiRJagCbM0mSJElqgAWbs4g4PSLuiIh7I+LrEfHWcvyGiPhURDxQPp9UfblSbzJnUj3MmlQ9cya1r5UjZxPAb2bmucBLgF+JiHOBq4HbM/Mc4PZyWFJ7zJlUD7MmVc+cSW1asDnLzF2Z+Q/lz/uB+4DTgEuA68vZrgcurapIqdeZM6keZk2qnjmT2je0mJkj4kzgRcBdwKbM3FVOegzYNM8yVwFXAaxYsaLdOqW+Yc6kepg1qXrmTFqclm8IEhFrgb8C3paZT8+clpkJ5FzLZeY1mbk1M7cODS2qF5T6jjmT6mHWpOqZM2nxWmrOImIFRbhuyMyPlaMfj4jN5fTNwO5qSpT6gzmT6mHWpOqZM6k9rdytMYBrgfsy8z/OmHQzsK38eRtwU+fLk/qDOZPqYdak6pkzqX2tHCt+GfBzwFcjYns57t8A7wI+GhFXAo8Ab6ymRKkvmDOpHmZNqp45k9q0YHOWmZ8DYp7JF3S2HKk/mTOpHmZNqp45k9rX8g1BJEmSJEnVsTmTJEmSpAawOZMkSZKkBrA5kyRJkqQGsDmTJEmSpAbwa9clSZIkaZaBge8fx5qamqplmzZnkiRJkkTRkK1evZqhoSFWrVrFypUrGRsbY3x8nLGxMQ4cOFDp9m3OJEmSJKm0cuVKhoeHWb9+PSMjI4yOjnLo0CEAmzNJkiRJqkNEMDw8zOrVqznhhBNYu3Yt+/fvB2B8fLzy7ducSZIkSRLfb85GRkZYt24dJ554IgBHjhxheHiYgYGBSq8/826NkiRJktQANmeSJEmS1AALNmcRMRIRX4iIr0TE1yPi98rxZ0XEXRHxYER8JCKGqy9X6k3mTKqHWVO3GxoaYmhoiJGREUZGRo7euGBwcHC5SzvKnEnta+XI2Rjwqsx8IXAe8JqIeAnwbuA9mXk2sAe4sroypZ5nzqR6mDV1rYGBAVauXMnatWs56aST2LRpEyeddBJr1qxhZGRkucubyZxJbVqwOcvC9D0jV5SPBF4F3FiOvx64tJIKpT5gzqR6mDV1uxUrVhy9WcHq1atZtWoVIyMjDA015x5v5kxqX0vXnEXEYERsB3YDnwK+BezNzIlylh3AadWUKPUHcybVw6ypWw0MDBw9arZlyxae/exnc9ppp3HKKaewbt06BgaacysBcya1p6UUZ+ZkZp4HbAHOB57X6gYi4qqIuDsi7p6YmFh4AalPmTOpHmZN3WxwcJChoSFWrlx59JqzoaGhRl1zBuZMateiPmLJzL3AHcBLgfURMX0MfQuwc55lrsnMrZm5tUmH3KWmMmdSPcyaVD1zJi1OK3dr3BgR68ufVwEXAfdRBO2ycrZtwE1VFSn1OnOmXjAwMMDg4ODRR5NOsZpm1qTqmTP1iswEqPRLp2dr5eOIzcD1ETFI0cx9NDM/ERH3Ah+OiN8HvgxcW2GdUq8zZ+pag4ODrF+/npGREdavX8+qVas4ePAghw8f5sCBAzz55JPLXeJMZk2qnjlTV5uYmODIkSOMjo5y6NAhxsfHOXLkCHWcZrtgc5aZ9wAvmmP8QxTnEEtaInOmbhYRrF27lrVr17J582ZOPPFE9u7dy1NPPUVm8tRTT9X6qePxmDWpeuZM3SwzmZqaYnJykiNHjjA+Ps74+DgTExNMTU1V/n7mibySJEmSRNGc7d+/n9HRUUZHRxkeHmZ8fJyxsTFGR0cr377NmSRJkiRRXF924EDxNX179uypffvNu1pbkiRJkvqQzZkkSZIkNYDNmSRJkiQ1gM2ZJEmSJDWANwRRzxseHmZgYIChoSEGBweZnJw8ejvU8fHx5S5PkqSWZObRu8YdOHCAiODAgQOMjo4yPj7emK+skNQ+mzP1tMHBQU4++WTWrFnDxo0bWbduHU8//TT79u1j7969PP74476ZSZK6wuTkJE888QRPPfUUu3btOvqB4/SHjpK6n82ZelpEMDw8zMqVK1mzZg3r1q1jcnKSsbExhoeHl7s8SZIWZWJigomJCcbGxpa7FEkV8JozSZIkSWoAmzNJ0pJlJlNTU0xOTh79eXq8JElqjac1SpKWJDM5ePAgk5OTAOzfv5+DBw9y6NAhDh486HWdkiS1yOZMkrQkmcnY2NjRJmx0dJTR0VGOHDnidTGSJC1Cy6c1RsRgRHw5Ij5RDp8VEXdFxIMR8ZGI8O4K0hKZM3Wj6a+lOHz4MPv372fv3r3s37+fQ4cONfLrKsyZVA+zJi3eYq45eytw34zhdwPvycyzgT3AlZ0sTOpT5kxdaXx8nNHRUZ5++mn27NnDgQMHGtucYc6kupg1aZFaas4iYgvwE8D7y+EAXgXcWM5yPXBpFQVK/cKcSdUzZ1I9zJrUnlaPnP0J8FvA9FXdJwN7M3P6Gw93AKd1uDap35gzqXrmTKqHWZPasGBzFhGvA3Zn5pfa2UBEXBURd0fE3X57vTQ3cyZVb6k5K9dh1qQF+J4mta+VuzW+DHh9RLwWGAFOAN4LrI+IofITkC3AzrkWzsxrgGsAVq9e7RfeSHMzZ1L1lpQzMGtSi3xPk9q04JGzzHxHZm7JzDOBy4FPZ+bPAncAl5WzbQNuqqxKaQkmJyeZmJhgbGyM0dFRxsbGmJiYOPqdTE1gzqTqmTOpHmZNat9i7tY429uB34iIBynOI762MyVJnTM1NcWBAwfYs2cPO3bs4KGHHmLnzp08+eST7N+/vxu+HNecSdUzZ1I9zJq0gEV9CXVm3gncWf78EHB+50uSOuvIkSMADA4OMjk5yfj4+NGjZ01kzqTqmTOpHmZNWpxFNWdSt5mamjp6KuPhw4eJCDKTyclJMj2NXZIkSc1hc6aeN31tWVOPlEmSJEmwtGvOJEmSJEkdYnMmSZIkSQ1gcyZJkiRJDWBzJkmSJEkNYHMmSZIkSQ1gcyZJkiRJDWBzJkmSJEkNYHMmSZIkSQ1gcyZJkiRJDWBzJkmSJEkNYHMmSZIkSQ0w1MpMEfEwsB+YBCYyc2tEbAA+ApwJPAy8MTP3VFOm1PvMmVQPsyZVz5xJ7VnMkbN/mpnnZebWcvhq4PbMPAe4vRyWtDTmTKqHWZOqZ86kRVrKaY2XANeXP18PXLr0ciTNYs6kepg1qXrmTFpAq81ZAn8bEV+KiKvKcZsyc1f582PAprkWjIirIuLuiLh7YmJiieVKPc2cSfUwa1L1zJnUhpauOQNenpk7I+JU4FMR8Y2ZEzMzIyLnWjAzrwGuAVi9evWc80gCzJlUF7MmVc+cSW1o6chZZu4sn3cDHwfOBx6PiM0A5fPuqoqU+oE5k+ph1qTqmTOpPQs2ZxGxJiLWTf8M/DjwNeBmYFs52zbgpqqKlHqdOZPqYdak6pkzqX2tnNa4Cfh4REzP/98z85aI+CLw0Yi4EngEeGN1ZUo9z5xJ9TBrUvXMmdSmBZuzzHwIeOEc478HXFBFUVK/MWdSPcyaVD1zJrVvKbfSlyRJkiR1iM2ZJDkRgrYAAAscSURBVEmSJDWAzZkkSZIkNYDNmSRJkiQ1gM2ZJEmSJDWAzZkkSZIkNYDNmSRJkiQ1gM2ZJEmSJDWAzZkkSZIkNYDNmSRJkiQ1gM2ZJEmSJDWAzZkkSZIkNUBLzVlErI+IGyPiGxFxX0S8NCI2RMSnIuKB8vmkqouVepk5k+ph1qTqmTOpPa0eOXsvcEtmPg94IXAfcDVwe2aeA9xeDktqnzmT6mHWpOqZM6kNCzZnEXEi8ArgWoDMHM/MvcAlwPXlbNcDl1ZVpNTrzJlUD7MmVc+cSe1r5cjZWcATwAcj4ssR8f6IWANsysxd5TyPAZuqKlLqA+ZMqodZk6pnzqQ2tdKcDQE/ArwvM18EHGTWYejMTCDnWjgiroqIuyPi7omJiaXWK/UqcybVw6xJ1TNnUptaac52ADsy865y+EaKwD0eEZsByufdcy2cmddk5tbM3Do0NNSJmqVeZM6kepg1qXrmTGrTgs1ZZj4GfDcinluOugC4F7gZ2FaO2wbcVEmFUh8wZ1I9zJpUPXMmta/VjyN+DbghIoaBh4Cfp2jsPhoRVwKPAG+spkSpb5gzqR5mTaqeOZPa0FJzlpnbga1zTLqgs+VI/cucSfUwa1L1zJnUnla/50ySJEmSVCGbM0mSJElqAJszSZIkSWoAmzNJkiRJagCbM0mSJElqAJszSZIkSWoAmzNJkiRJagCbM0mSJElqAJszSZIkSWoAmzNJkiRJagCbM0mSJElqAJszSZIkSWoAmzNJkiRJaoAFm7OIeG5EbJ/xeDoi3hYRGyLiUxHxQPl8Uh0FS73InEn1MGtS9cyZ1L4Fm7PMvD8zz8vM84AXA4eAjwNXA7dn5jnA7eWwpDaYM6keZk2qnjmT2rfY0xovAL6VmY8AlwDXl+OvBy7tZGFSHzNnUj3MmlQ9cyYtwmKbs8uBD5U/b8rMXeXPjwGbOlaV1N/MmVQPsyZVz5xJi9BycxYRw8Drgb+cPS0zE8h5lrsqIu6OiLsnJibaLlTqB+ZMqodZk6pnzqTFW8yRs4uBf8jMx8vhxyNiM0D5vHuuhTLzmszcmplbh4aGllat1PvMmVQPsyZVz5xJi7SY5uxNfP+wNMDNwLby523ATZ0qSupj5kyqh1mTqmfOpEVqqTmLiDXARcDHZox+F3BRRDwAXFgOS2qTOZPqYdak6pkzqT0tHSvOzIPAybPGfY/iDjySOsCcSfUwa1L1zJnUnsXerVGSJEmSVAGbM0mSJElqAJszSZIkSWoAmzNJkiRJagCbM0mSJElqAJszSZIkSWoAmzNJkiRJagCbM0mSJElqgJa+hLpTVqxYwaZNm+rcpNR3zJlUD7OmXvPQQw8tdwnHMGfqN7U2Z2vWrOH888+vc5NS5W699dblLuEZzJl6UdNyBmZNveeee+5Z7hKOYc7Ui473nlb7kbMtW7bUuUmp75gzqR5mTb1mxYoVy13CMcyZ+k1kZm0bO++88/K2226rbXtSHTZu3PilzNy63HVMM2fqRU3LGZg19Z4LL7yQ7du3x3LXMZM5Uy863ntaS0fOIuJfAv8HkMBXgZ8HNgMfBk4GvgT8XGaOH289Q0NDnHLKKYsoXeof5kyqh1mT5jY01LkTqsyZ1J4F79YYEacBvw5szcwXAIPA5cC7gfdk5tnAHuDKKguVepk5k+ph1qTqmTOpfa3eSn8IWBURQ8BqYBfwKuDGcvr1wKWdL0/qK+ZMqodZk6pnzqQ2LNicZeZO4I+A71AEax/Foei9mTlRzrYDOK2qIqVeZ86kepg1qXrmTGpfK6c1ngRcApwF/ACwBnhNqxuIiKsi4u6IuPuJJ55ou1Cpl5kzqR5mTaqeOZPa18ppjRcC387MJzLzCPAx4GXA+vJQNcAWYOdcC2fmNZm5NTO3bty4sSNFSz3InEn1MGtS9cyZ1KZWmrPvAC+JiNUREcAFwL3AHcBl5TzbgJuqKVHqC+ZMqodZk6pnzqQ2tXLN2V0UF2/+A8WtUAeAa4C3A78REQ9S3BL12grrlHqaOZPqYdak6pkzqX0tfaFFZv4u8LuzRj8EnN/xiqQ+Zc6kepg1qXrmTGpPq7fSlyRJkiRVyOZMkiRJkhrA5kySJEmSGiAys76NRTwBHASerG2jS3MK3VMrdFe9vVTrGZnZmHv9dmHOoLf+Hpqkm2qF49fbqJxBV2atm/4euqlW6K56zVm1uulvAbqr3l6qdd6s1dqcAUTE3Zm5tdaNtqmbaoXuqtdaq9VtNXdTvdZanW6rF7qrZmutTjfV2021TuummrupVuiuevulVk9rlCRJkqQGsDmTJEmSpAZYjubsmmXYZru6qVbornqttVrdVnM31Wut1em2eqG7arbW6nRTvd1U67RuqrmbaoXuqrcvaq39mjNJkiRJ0rE8rVGSJEmSGqC25iwiXhMR90fEgxFxdV3bbVVEnB4Rd0TEvRHx9Yh4azl+Q0R8KiIeKJ9PWu5ap0XEYER8OSI+UQ6fFRF3lfv4IxExvNw1AkTE+oi4MSK+ERH3RcRLG75f/2X5N/C1iPhQRIw0dd/OpclZM2fV6qasmbPqmLNqmbP6NDlnYNaq1E05g85mrZbmLCIGgT8FLgbOBd4UEefWse1FmAB+MzPPBV4C/EpZ49XA7Zl5DnB7OdwUbwXumzH8buA9mXk2sAe4clmqOtZ7gVsy83nACylqbuR+jYjTgF8HtmbmC4BB4HKau2+foQuyZs6q1RVZM2eVM2fVMmc16IKcgVmrUlfkDCrIWmZW/gBeCtw6Y/gdwDvq2PYSar4JuAi4H9hcjtsM3L/ctZW1bKH4w3wV8AkgKL7sbmiufb6MdZ4IfJvy+sYZ45u6X08DvgtsAIbKffvqJu7beervqqyZs47W2jVZM2e112vOOlerOauv/q7KWVmjWetMnV2Ts7KWjmatrtMap4uetqMc10gRcSbwIuAuYFNm7ionPQZsWqayZvsT4LeAqXL4ZGBvZk6Uw03Zx2cBTwAfLA+jvz8i1tDQ/ZqZO4E/Ar4D7AL2AV+imft2Ll2TNXPWcV2TNXNWH3PWceasPl2TMzBrHdY1OYPOZ80bgswSEWuBvwLelplPz5yWReu77Le3jIjXAbsz80vLXUsLhoAfAd6XmS8CDjLrMHRT9itAef7yJRT/MPwAsAZ4zbIW1YPMWSW6JmvmrB7mrBLmTMcwax3XNTmDzmetruZsJ3D6jOEt5bhGiYgVFOG6ITM/Vo5+PCI2l9M3A7uXq74ZXga8PiIeBj5McXj6vcD6iBgq52nKPt4B7MjMu8rhGykC18T9CnAh8O3MfCIzjwAfo9jfTdy3c2l81sxZZbopa+asYuasMuasPo3PGZi1inRTzqDDWaurOfsicE5515Jhiovkbq5p2y2JiACuBe7LzP84Y9LNwLby520U5xMvq8x8R2ZuycwzKfblpzPzZ4E7gMvK2ZpS62PAdyPiueWoC4B7aeB+LX0HeElErC7/Jqbrbdy+nUejs2bOqtNlWTNnFTJn1TFntWp0zsCsVaXLcgadzlqNF8u9Fvgm8C3gt+va7iLqeznF4dF7gO3l47UU5+PeDjwA3AZsWO5aZ9X9SuAT5c/PBr4APAj8JbByuesr6zoPuLvct38NnNTk/Qr8HvAN4GvAnwMrm7pv56m/sVkzZ5XX2TVZM2eV1mbOqq3TnNVXf2NzVtZn1qqrsWtyVtbbsaxFuUJJkiRJ0jLyhiCSJEmS1AA2Z5IkSZLUADZnkiRJktQANmeSJEmS1AA2Z5IkSZLUADZnkiRJktQANmeSJEmS1AA2Z5IkSZLUAP8/kdOnsiacNwMAAAAASUVORK5CYII=\n",
            "text/plain": [
              "<Figure size 1080x360 with 4 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "D1hORw-SrRKu"
      },
      "source": [
        "## Erstellen des Environments"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "uyVNp_0hrOPK",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "98505a32-0ec2-4699-92cc-f08e97758770"
      },
      "source": [
        "def make_env(env_name):\n",
        "    env = gym.make(env_name)\n",
        "    #env = EpisodicLifeEnv(env)\n",
        "    env = MaxAndSkipEnv(env)\n",
        "    env = ScaledFloatFrame(env)\n",
        "    env = NoopResetEnv(env)\n",
        "    #env = ClipRewardEnv(env)\n",
        "    env = WarpFrame(env)\n",
        "    if 'FIRE' in env.unwrapped.get_action_meanings():\n",
        "        env = FireResetEnv(env)\n",
        "    env = FrameStack(env)\n",
        "    return env\n",
        "\n",
        "env = make_env(game)\n",
        "\n",
        "\"\"\" saving the properties for csv \"\"\"\n",
        "\n",
        "MODE = \"NoEpisodicLife_NoClipReward_lr_1e-3_5e-3\"\n",
        "PATH = \"WEIGHTS/\" + game + \"/\" + MODE + \"/\"\n",
        "print(PATH)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "WEIGHTS/Pong-v0/NoEpisodicLife_NoClipReward_lr_1e-3_5e-3/\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "0uAIJ4QYrja0"
      },
      "source": [
        "# Actor Network und Critic Network"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "uXn8gDrenW9T"
      },
      "source": [
        "from tensorflow.keras import backend as K\n",
        "\n",
        "\n",
        "# PPO Loss\n",
        "def ppo(y_true, y_pred):\n",
        "    advantages, prediction_picks, actions = y_true[:, :1], y_true[:, 1:1+env.action_space.n], y_true[:, 1+env.action_space.n:]\n",
        "\n",
        "    LOSS_CLIPPING = 0.2\n",
        "    ENTROPY_LOSS = 5e-3\n",
        "\n",
        "    prob = y_pred * actions\n",
        "    old_prob = actions * prediction_picks\n",
        "    r = prob/(old_prob + 1e-10)\n",
        "    p1 = r * advantages\n",
        "    p2 = K.clip(r, min_value=1 - LOSS_CLIPPING, max_value=1 + LOSS_CLIPPING) * advantages\n",
        "    loss =  -K.mean(K.minimum(p1, p2) + ENTROPY_LOSS * -(prob * K.log(prob + 1e-10)))\n",
        "    return loss"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "aGNNFrCgr0MM",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "f408f20d-f334-48af-c597-8c4ea1523a32"
      },
      "source": [
        "import tensorflow as tf\n",
        "from tensorflow.keras.layers import Activation\n",
        "from tensorflow.keras.layers import Conv2D\n",
        "from tensorflow.keras.layers import Dense\n",
        "from tensorflow.keras.layers import Flatten\n",
        "from tensorflow.keras.layers import Input\n",
        "from tensorflow.keras.models import Model\n",
        "from tensorflow.keras.optimizers import Adam\n",
        "\n",
        "# Network Parameter\n",
        "INPUT_SHAPE = (84, 84, 4) # (Höhe, Breite, Frames)\n",
        "ACTOR_OUTPUT = env.action_space.n # Anzahl der möglichen Aktionen\n",
        "CRITIC_OUTPUT = 1 # Bewertung der gewählten Aktion\n",
        "ACTOR_LEARNING_RATE = 25e-6\n",
        "CRITIC_LEARNING_RATE = 25e-6\n",
        "\n",
        "# neuronales Netz\n",
        "net_input = Input(shape=INPUT_SHAPE)\n",
        "x = Conv2D(filters=32, kernel_size=(8, 8), strides=(4, 4), padding=\"same\")(net_input)\n",
        "x = Activation(\"relu\")(x)\n",
        "x = Conv2D(filters=64, kernel_size=(4, 4), strides=(2, 2), padding=\"same\")(x)\n",
        "x = Activation(\"relu\")(x)\n",
        "x = Conv2D(filters=64, kernel_size=(3, 3), strides=(1, 1), padding=\"same\")(x)\n",
        "x = Activation(\"relu\")(x)\n",
        "x = Flatten()(x)\n",
        "x = Dense(512)(x)\n",
        "x = Activation(\"relu\")(x)\n",
        "\n",
        "\n",
        "'''Aufspalten des Netzes in Actor und Critic'''\n",
        "\n",
        "# Actor - wählt eine Aktion\n",
        "actor_x = Dense(ACTOR_OUTPUT)(x)\n",
        "actor_output = Activation(\"softmax\")(actor_x)\n",
        "\n",
        "ACTOR = Model(inputs=net_input, outputs=actor_output)\n",
        "ACTOR.compile(loss=ppo, optimizer=Adam(lr=ACTOR_LEARNING_RATE))\n",
        "\n",
        "# Critic - bewertet gewählte Aktion\n",
        "critic_x = Dense(CRITIC_OUTPUT)(x)\n",
        "critic_output = Activation(\"linear\")(critic_x)\n",
        "\n",
        "CRITIC = Model(inputs=net_input, outputs=critic_output)\n",
        "CRITIC.compile(loss=\"mse\", optimizer=Adam(lr=CRITIC_LEARNING_RATE))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/tensorflow/python/keras/optimizer_v2/optimizer_v2.py:375: UserWarning: The `lr` argument is deprecated, use `learning_rate` instead.\n",
            "  \"The `lr` argument is deprecated, use `learning_rate` instead.\")\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "s30U2V0CMtN_"
      },
      "source": [
        "# Aktion wählen"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "fEwfPKljMtN_"
      },
      "source": [
        "def get_action(state):\n",
        "    prediction = ACTOR.predict(state)[0]\n",
        "    action = np.random.choice(env.action_space.n, p=prediction) # Aktionen, welche Wahrscheinlichkeit zu Aktion\n",
        "    return action, prediction"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "yLfgqEutNr10"
      },
      "source": [
        "def remember (state, action, reward, prediction):\n",
        "    STATES.append(state)\n",
        "    # Erstellen eines One Hot Labels für die Aktion\n",
        "    action_onehot = np.zeros([env.action_space.n])\n",
        "    action_onehot[action] = 1\n",
        "    ACTIONS.append(action_onehot)\n",
        "    REWARDS.append(reward)\n",
        "    PREDICTIONS.append(prediction)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "CRPapa6lR4-K"
      },
      "source": [
        "def discount_rewards(rewards):\n",
        "    gamma = 0.99\n",
        "    running_add = 0\n",
        "    discounted_r = np.zeros_like(rewards)\n",
        "    for i in reversed(range(0, len(rewards))):\n",
        "        if rewards[i] != 0: # Pong\n",
        "            running_add = 0\n",
        "        running_add = running_add * gamma + rewards[i]\n",
        "        discounted_r[i] = running_add\n",
        "\n",
        "    discounted_r -= np.mean(discounted_r) # normalisieren\n",
        "    discounted_r /= np.std(discounted_r) # teilen durch Standardabweichung\n",
        "\n",
        "    return discounted_r"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "D3SGt8rvRTPT"
      },
      "source": [
        "def replay(STATES, ACTIONS, REWARDS, PREDICTIONS):\n",
        "    states = np.vstack(STATES)\n",
        "    actions = np.vstack(ACTIONS)\n",
        "    predictions = np.vstack(PREDICTIONS)\n",
        "\n",
        "    # discount rewards\n",
        "    discounted_r = np.vstack(discount_rewards(REWARDS))\n",
        "\n",
        "    # Critic predictions\n",
        "    values = CRITIC.predict(states)\n",
        "\n",
        "    # Advantagewerte berechnen\n",
        "    advantages = discounted_r - values\n",
        "\n",
        "    y_true = np.hstack([advantages, predictions, actions])\n",
        "\n",
        "    # Trainieren der Netzwerke ACTOR und CRITIC\n",
        "    ACTOR.fit(states, y_true, epochs=1, verbose=0, batch_size=len(REWARDS))\n",
        "    CRITIC.fit(states, discounted_r, epochs=1, verbose=0, batch_size=len(REWARDS))\n",
        "\n",
        "    # leeren des Episoden Buffers\n",
        "    STATES, ACTIONS, REWARDS, PREDICTIONS = [], [], [], []"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "JwLX4bvUMtOA"
      },
      "source": [
        "# Training /Spielen"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "fV_l4s5aMtOA",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "bfd85086-23c1-4802-89f1-f9a58b6ea8c1"
      },
      "source": [
        "# Initial Mean Reward -> Mean Reward kann nich auf 0 gesetzt werden (Pong -21)\n",
        "INITIAL_MEAN_REWARD = 0.0\n",
        "env.reset()\n",
        "while True:\n",
        "    _, reward, done, _ = env.step(env.action_space.sample())\n",
        "    INITIAL_MEAN_REWARD += reward\n",
        "    if done:\n",
        "        break\n",
        "INITIAL_MEAN_REWARD"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "-19.0"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 64
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "zXHmDN4bMtOA",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "a1962190-a461-4450-b0fb-2653f613c18d"
      },
      "source": [
        "EPISODES = 15_000\n",
        "REWARD_LIST = []\n",
        "MEAN_LIST = []\n",
        "BEST_MEAN_REWARD = INITIAL_MEAN_REWARD\n",
        "\n",
        "for episode in range(EPISODES):\n",
        "    EPISODE_REWARD = 0.0\n",
        "    state = env.reset()\n",
        "    done = False\n",
        "\n",
        "    # Episoden Buffer\n",
        "    STATES, ACTIONS, REWARDS, PREDICTIONS = [], [], [], []\n",
        "    \n",
        "    while not done:        \n",
        "        action, prediction = get_action(state)\n",
        "        \n",
        "        next_state, reward, done, _ = env.step(action)\n",
        "        \n",
        "        # Transition im Episoden Buffer speichern\n",
        "        remember(state, action, reward, prediction)\n",
        "\n",
        "        # aktualisieren des States\n",
        "        state = next_state\n",
        "        \n",
        "        # Reward einer Aktion zum gesamten Reward der Episode addieren\n",
        "        EPISODE_REWARD += reward\n",
        "\n",
        "\n",
        "        if done:\n",
        "            REWARD_LIST.append(EPISODE_REWARD)\n",
        "            current_mean_reward = np.mean(REWARD_LIST[-min(len(REWARD_LIST), 10):])\n",
        "            MEAN_LIST.append(np.mean(REWARD_LIST))\n",
        "            \n",
        "            print(\"Episode:\", episode+1, \"\\tReward:\", EPISODE_REWARD, \"\\tMean:\", round(current_mean_reward, 2),\"\\tBestMean:\", BEST_MEAN_REWARD)\n",
        "\n",
        "            # Übernahme des höchsten Mean Rewards\n",
        "            if current_mean_reward > BEST_MEAN_REWARD:\n",
        "                BEST_MEAN_REWARD = current_mean_reward\n",
        "        \n",
        "                # trainierte Gewichte speichern\n",
        "                import os\n",
        "                try:\n",
        "                    os.makedirs(PATH)\n",
        "                except FileExistsError:\n",
        "                    # Pfad existiert bereits\n",
        "                    pass\n",
        "                ACTOR.save_weights(PATH + \"Best_ACTOR.h5\")\n",
        "                CRITIC.save_weights(PATH + \"Best_CRITIC.h5\")\n",
        "\n",
        "            replay(STATES, ACTIONS, REWARDS, PREDICTIONS)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Episode: 1 \tReward: -20.0 \tMean: -20.0 \tBestMean: -19.0\n",
            "Episode: 2 \tReward: -20.0 \tMean: -20.0 \tBestMean: -19.0\n",
            "Episode: 3 \tReward: -20.0 \tMean: -20.0 \tBestMean: -19.0\n",
            "WARNING:tensorflow:5 out of the last 5 calls to <function Model.make_train_function.<locals>.train_function at 0x7f2f95524200> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings could be due to (1) creating @tf.function repeatedly in a loop, (2) passing tensors with different shapes, (3) passing Python objects instead of tensors. For (1), please define your @tf.function outside of the loop. For (2), @tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. For (3), please refer to https://www.tensorflow.org/guide/function#controlling_retracing and https://www.tensorflow.org/api_docs/python/tf/function for  more details.\n",
            "WARNING:tensorflow:6 out of the last 6 calls to <function Model.make_train_function.<locals>.train_function at 0x7f2f9558dcb0> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings could be due to (1) creating @tf.function repeatedly in a loop, (2) passing tensors with different shapes, (3) passing Python objects instead of tensors. For (1), please define your @tf.function outside of the loop. For (2), @tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. For (3), please refer to https://www.tensorflow.org/guide/function#controlling_retracing and https://www.tensorflow.org/api_docs/python/tf/function for  more details.\n",
            "Episode: 4 \tReward: -20.0 \tMean: -20.0 \tBestMean: -19.0\n",
            "Episode: 5 \tReward: -21.0 \tMean: -20.2 \tBestMean: -19.0\n",
            "Episode: 6 \tReward: -21.0 \tMean: -20.33 \tBestMean: -19.0\n",
            "Episode: 7 \tReward: -20.0 \tMean: -20.29 \tBestMean: -19.0\n",
            "Episode: 8 \tReward: -20.0 \tMean: -20.25 \tBestMean: -19.0\n",
            "Episode: 9 \tReward: -21.0 \tMean: -20.33 \tBestMean: -19.0\n",
            "Episode: 10 \tReward: -21.0 \tMean: -20.4 \tBestMean: -19.0\n",
            "Episode: 11 \tReward: -21.0 \tMean: -20.5 \tBestMean: -19.0\n",
            "Episode: 12 \tReward: -20.0 \tMean: -20.5 \tBestMean: -19.0\n",
            "Episode: 13 \tReward: -21.0 \tMean: -20.6 \tBestMean: -19.0\n",
            "Episode: 14 \tReward: -20.0 \tMean: -20.6 \tBestMean: -19.0\n",
            "Episode: 15 \tReward: -21.0 \tMean: -20.6 \tBestMean: -19.0\n",
            "Episode: 16 \tReward: -20.0 \tMean: -20.5 \tBestMean: -19.0\n",
            "Episode: 17 \tReward: -21.0 \tMean: -20.6 \tBestMean: -19.0\n",
            "Episode: 18 \tReward: -20.0 \tMean: -20.6 \tBestMean: -19.0\n",
            "Episode: 19 \tReward: -20.0 \tMean: -20.5 \tBestMean: -19.0\n",
            "Episode: 20 \tReward: -20.0 \tMean: -20.4 \tBestMean: -19.0\n",
            "Episode: 21 \tReward: -21.0 \tMean: -20.4 \tBestMean: -19.0\n",
            "Episode: 22 \tReward: -19.0 \tMean: -20.3 \tBestMean: -19.0\n",
            "Episode: 23 \tReward: -20.0 \tMean: -20.2 \tBestMean: -19.0\n",
            "Episode: 24 \tReward: -21.0 \tMean: -20.3 \tBestMean: -19.0\n",
            "Episode: 25 \tReward: -20.0 \tMean: -20.2 \tBestMean: -19.0\n",
            "Episode: 26 \tReward: -20.0 \tMean: -20.2 \tBestMean: -19.0\n",
            "Episode: 27 \tReward: -21.0 \tMean: -20.2 \tBestMean: -19.0\n",
            "Episode: 28 \tReward: -21.0 \tMean: -20.3 \tBestMean: -19.0\n",
            "Episode: 29 \tReward: -21.0 \tMean: -20.4 \tBestMean: -19.0\n",
            "Episode: 30 \tReward: -21.0 \tMean: -20.5 \tBestMean: -19.0\n",
            "Episode: 31 \tReward: -20.0 \tMean: -20.4 \tBestMean: -19.0\n",
            "Episode: 32 \tReward: -20.0 \tMean: -20.5 \tBestMean: -19.0\n",
            "Episode: 33 \tReward: -20.0 \tMean: -20.5 \tBestMean: -19.0\n",
            "Episode: 34 \tReward: -20.0 \tMean: -20.4 \tBestMean: -19.0\n",
            "Episode: 35 \tReward: -19.0 \tMean: -20.3 \tBestMean: -19.0\n",
            "Episode: 36 \tReward: -21.0 \tMean: -20.4 \tBestMean: -19.0\n",
            "Episode: 37 \tReward: -19.0 \tMean: -20.2 \tBestMean: -19.0\n",
            "Episode: 38 \tReward: -18.0 \tMean: -19.9 \tBestMean: -19.0\n",
            "Episode: 39 \tReward: -20.0 \tMean: -19.8 \tBestMean: -19.0\n",
            "Episode: 40 \tReward: -20.0 \tMean: -19.7 \tBestMean: -19.0\n",
            "Episode: 41 \tReward: -19.0 \tMean: -19.6 \tBestMean: -19.0\n",
            "Episode: 42 \tReward: -20.0 \tMean: -19.6 \tBestMean: -19.0\n",
            "Episode: 43 \tReward: -21.0 \tMean: -19.7 \tBestMean: -19.0\n",
            "Episode: 44 \tReward: -20.0 \tMean: -19.7 \tBestMean: -19.0\n",
            "Episode: 45 \tReward: -21.0 \tMean: -19.9 \tBestMean: -19.0\n",
            "Episode: 46 \tReward: -19.0 \tMean: -19.7 \tBestMean: -19.0\n",
            "Episode: 47 \tReward: -20.0 \tMean: -19.8 \tBestMean: -19.0\n",
            "Episode: 48 \tReward: -21.0 \tMean: -20.1 \tBestMean: -19.0\n",
            "Episode: 49 \tReward: -21.0 \tMean: -20.2 \tBestMean: -19.0\n",
            "Episode: 50 \tReward: -20.0 \tMean: -20.2 \tBestMean: -19.0\n",
            "Episode: 51 \tReward: -19.0 \tMean: -20.2 \tBestMean: -19.0\n",
            "Episode: 52 \tReward: -20.0 \tMean: -20.2 \tBestMean: -19.0\n",
            "Episode: 53 \tReward: -21.0 \tMean: -20.2 \tBestMean: -19.0\n",
            "Episode: 54 \tReward: -20.0 \tMean: -20.2 \tBestMean: -19.0\n",
            "Episode: 55 \tReward: -20.0 \tMean: -20.1 \tBestMean: -19.0\n",
            "Episode: 56 \tReward: -20.0 \tMean: -20.2 \tBestMean: -19.0\n",
            "Episode: 57 \tReward: -21.0 \tMean: -20.3 \tBestMean: -19.0\n",
            "Episode: 58 \tReward: -21.0 \tMean: -20.3 \tBestMean: -19.0\n",
            "Episode: 59 \tReward: -20.0 \tMean: -20.2 \tBestMean: -19.0\n",
            "Episode: 60 \tReward: -21.0 \tMean: -20.3 \tBestMean: -19.0\n",
            "Episode: 61 \tReward: -20.0 \tMean: -20.4 \tBestMean: -19.0\n",
            "Episode: 62 \tReward: -20.0 \tMean: -20.4 \tBestMean: -19.0\n",
            "Episode: 63 \tReward: -19.0 \tMean: -20.2 \tBestMean: -19.0\n",
            "Episode: 64 \tReward: -21.0 \tMean: -20.3 \tBestMean: -19.0\n",
            "Episode: 65 \tReward: -19.0 \tMean: -20.2 \tBestMean: -19.0\n",
            "Episode: 66 \tReward: -21.0 \tMean: -20.3 \tBestMean: -19.0\n",
            "Episode: 67 \tReward: -18.0 \tMean: -20.0 \tBestMean: -19.0\n",
            "Episode: 68 \tReward: -20.0 \tMean: -19.9 \tBestMean: -19.0\n",
            "Episode: 69 \tReward: -17.0 \tMean: -19.6 \tBestMean: -19.0\n",
            "Episode: 70 \tReward: -21.0 \tMean: -19.6 \tBestMean: -19.0\n",
            "Episode: 71 \tReward: -21.0 \tMean: -19.7 \tBestMean: -19.0\n",
            "Episode: 72 \tReward: -20.0 \tMean: -19.7 \tBestMean: -19.0\n",
            "Episode: 73 \tReward: -20.0 \tMean: -19.8 \tBestMean: -19.0\n",
            "Episode: 74 \tReward: -20.0 \tMean: -19.7 \tBestMean: -19.0\n",
            "Episode: 75 \tReward: -20.0 \tMean: -19.8 \tBestMean: -19.0\n",
            "Episode: 76 \tReward: -21.0 \tMean: -19.8 \tBestMean: -19.0\n",
            "Episode: 77 \tReward: -19.0 \tMean: -19.9 \tBestMean: -19.0\n",
            "Episode: 78 \tReward: -20.0 \tMean: -19.9 \tBestMean: -19.0\n",
            "Episode: 79 \tReward: -19.0 \tMean: -20.1 \tBestMean: -19.0\n",
            "Episode: 80 \tReward: -19.0 \tMean: -19.9 \tBestMean: -19.0\n",
            "Episode: 81 \tReward: -21.0 \tMean: -19.9 \tBestMean: -19.0\n",
            "Episode: 82 \tReward: -21.0 \tMean: -20.0 \tBestMean: -19.0\n",
            "Episode: 83 \tReward: -20.0 \tMean: -20.0 \tBestMean: -19.0\n",
            "Episode: 84 \tReward: -20.0 \tMean: -20.0 \tBestMean: -19.0\n",
            "Episode: 85 \tReward: -19.0 \tMean: -19.9 \tBestMean: -19.0\n",
            "Episode: 86 \tReward: -20.0 \tMean: -19.8 \tBestMean: -19.0\n",
            "Episode: 87 \tReward: -20.0 \tMean: -19.9 \tBestMean: -19.0\n",
            "Episode: 88 \tReward: -21.0 \tMean: -20.0 \tBestMean: -19.0\n",
            "Episode: 89 \tReward: -20.0 \tMean: -20.1 \tBestMean: -19.0\n",
            "Episode: 90 \tReward: -20.0 \tMean: -20.2 \tBestMean: -19.0\n",
            "Episode: 91 \tReward: -21.0 \tMean: -20.2 \tBestMean: -19.0\n",
            "Episode: 92 \tReward: -20.0 \tMean: -20.1 \tBestMean: -19.0\n",
            "Episode: 93 \tReward: -21.0 \tMean: -20.2 \tBestMean: -19.0\n",
            "Episode: 94 \tReward: -20.0 \tMean: -20.2 \tBestMean: -19.0\n",
            "Episode: 95 \tReward: -20.0 \tMean: -20.3 \tBestMean: -19.0\n",
            "Episode: 96 \tReward: -20.0 \tMean: -20.3 \tBestMean: -19.0\n",
            "Episode: 97 \tReward: -20.0 \tMean: -20.3 \tBestMean: -19.0\n",
            "Episode: 98 \tReward: -20.0 \tMean: -20.2 \tBestMean: -19.0\n",
            "Episode: 99 \tReward: -20.0 \tMean: -20.2 \tBestMean: -19.0\n",
            "Episode: 100 \tReward: -20.0 \tMean: -20.2 \tBestMean: -19.0\n",
            "Episode: 101 \tReward: -21.0 \tMean: -20.2 \tBestMean: -19.0\n",
            "Episode: 102 \tReward: -19.0 \tMean: -20.1 \tBestMean: -19.0\n",
            "Episode: 103 \tReward: -19.0 \tMean: -19.9 \tBestMean: -19.0\n",
            "Episode: 104 \tReward: -19.0 \tMean: -19.8 \tBestMean: -19.0\n",
            "Episode: 105 \tReward: -20.0 \tMean: -19.8 \tBestMean: -19.0\n",
            "Episode: 106 \tReward: -19.0 \tMean: -19.7 \tBestMean: -19.0\n",
            "Episode: 107 \tReward: -21.0 \tMean: -19.8 \tBestMean: -19.0\n",
            "Episode: 108 \tReward: -20.0 \tMean: -19.8 \tBestMean: -19.0\n",
            "Episode: 109 \tReward: -18.0 \tMean: -19.6 \tBestMean: -19.0\n",
            "Episode: 110 \tReward: -19.0 \tMean: -19.5 \tBestMean: -19.0\n",
            "Episode: 111 \tReward: -21.0 \tMean: -19.5 \tBestMean: -19.0\n",
            "Episode: 112 \tReward: -21.0 \tMean: -19.7 \tBestMean: -19.0\n",
            "Episode: 113 \tReward: -20.0 \tMean: -19.8 \tBestMean: -19.0\n",
            "Episode: 114 \tReward: -20.0 \tMean: -19.9 \tBestMean: -19.0\n",
            "Episode: 115 \tReward: -19.0 \tMean: -19.8 \tBestMean: -19.0\n",
            "Episode: 116 \tReward: -19.0 \tMean: -19.8 \tBestMean: -19.0\n",
            "Episode: 117 \tReward: -21.0 \tMean: -19.8 \tBestMean: -19.0\n",
            "Episode: 118 \tReward: -19.0 \tMean: -19.7 \tBestMean: -19.0\n",
            "Episode: 119 \tReward: -21.0 \tMean: -20.0 \tBestMean: -19.0\n",
            "Episode: 120 \tReward: -19.0 \tMean: -20.0 \tBestMean: -19.0\n",
            "Episode: 121 \tReward: -21.0 \tMean: -20.0 \tBestMean: -19.0\n",
            "Episode: 122 \tReward: -21.0 \tMean: -20.0 \tBestMean: -19.0\n",
            "Episode: 123 \tReward: -21.0 \tMean: -20.1 \tBestMean: -19.0\n",
            "Episode: 124 \tReward: -19.0 \tMean: -20.0 \tBestMean: -19.0\n",
            "Episode: 125 \tReward: -19.0 \tMean: -20.0 \tBestMean: -19.0\n",
            "Episode: 126 \tReward: -20.0 \tMean: -20.1 \tBestMean: -19.0\n",
            "Episode: 127 \tReward: -20.0 \tMean: -20.0 \tBestMean: -19.0\n",
            "Episode: 128 \tReward: -20.0 \tMean: -20.1 \tBestMean: -19.0\n",
            "Episode: 129 \tReward: -20.0 \tMean: -20.0 \tBestMean: -19.0\n",
            "Episode: 130 \tReward: -20.0 \tMean: -20.1 \tBestMean: -19.0\n",
            "Episode: 131 \tReward: -21.0 \tMean: -20.1 \tBestMean: -19.0\n",
            "Episode: 132 \tReward: -21.0 \tMean: -20.1 \tBestMean: -19.0\n",
            "Episode: 133 \tReward: -20.0 \tMean: -20.0 \tBestMean: -19.0\n",
            "Episode: 134 \tReward: -21.0 \tMean: -20.2 \tBestMean: -19.0\n",
            "Episode: 135 \tReward: -21.0 \tMean: -20.4 \tBestMean: -19.0\n",
            "Episode: 136 \tReward: -20.0 \tMean: -20.4 \tBestMean: -19.0\n",
            "Episode: 137 \tReward: -21.0 \tMean: -20.5 \tBestMean: -19.0\n",
            "Episode: 138 \tReward: -21.0 \tMean: -20.6 \tBestMean: -19.0\n",
            "Episode: 139 \tReward: -21.0 \tMean: -20.7 \tBestMean: -19.0\n",
            "Episode: 140 \tReward: -20.0 \tMean: -20.7 \tBestMean: -19.0\n",
            "Episode: 141 \tReward: -20.0 \tMean: -20.6 \tBestMean: -19.0\n",
            "Episode: 142 \tReward: -20.0 \tMean: -20.5 \tBestMean: -19.0\n",
            "Episode: 143 \tReward: -21.0 \tMean: -20.6 \tBestMean: -19.0\n",
            "Episode: 144 \tReward: -20.0 \tMean: -20.5 \tBestMean: -19.0\n",
            "Episode: 145 \tReward: -18.0 \tMean: -20.2 \tBestMean: -19.0\n",
            "Episode: 146 \tReward: -20.0 \tMean: -20.2 \tBestMean: -19.0\n",
            "Episode: 147 \tReward: -21.0 \tMean: -20.2 \tBestMean: -19.0\n",
            "Episode: 148 \tReward: -21.0 \tMean: -20.2 \tBestMean: -19.0\n",
            "Episode: 149 \tReward: -21.0 \tMean: -20.2 \tBestMean: -19.0\n",
            "Episode: 150 \tReward: -20.0 \tMean: -20.2 \tBestMean: -19.0\n",
            "Episode: 151 \tReward: -19.0 \tMean: -20.1 \tBestMean: -19.0\n",
            "Episode: 152 \tReward: -19.0 \tMean: -20.0 \tBestMean: -19.0\n",
            "Episode: 153 \tReward: -21.0 \tMean: -20.0 \tBestMean: -19.0\n",
            "Episode: 154 \tReward: -20.0 \tMean: -20.0 \tBestMean: -19.0\n",
            "Episode: 155 \tReward: -18.0 \tMean: -20.0 \tBestMean: -19.0\n",
            "Episode: 156 \tReward: -20.0 \tMean: -20.0 \tBestMean: -19.0\n",
            "Episode: 157 \tReward: -19.0 \tMean: -19.8 \tBestMean: -19.0\n",
            "Episode: 158 \tReward: -21.0 \tMean: -19.8 \tBestMean: -19.0\n",
            "Episode: 159 \tReward: -20.0 \tMean: -19.7 \tBestMean: -19.0\n",
            "Episode: 160 \tReward: -20.0 \tMean: -19.7 \tBestMean: -19.0\n",
            "Episode: 161 \tReward: -21.0 \tMean: -19.9 \tBestMean: -19.0\n",
            "Episode: 162 \tReward: -18.0 \tMean: -19.8 \tBestMean: -19.0\n",
            "Episode: 163 \tReward: -20.0 \tMean: -19.7 \tBestMean: -19.0\n",
            "Episode: 164 \tReward: -21.0 \tMean: -19.8 \tBestMean: -19.0\n",
            "Episode: 165 \tReward: -19.0 \tMean: -19.9 \tBestMean: -19.0\n",
            "Episode: 166 \tReward: -18.0 \tMean: -19.7 \tBestMean: -19.0\n",
            "Episode: 167 \tReward: -20.0 \tMean: -19.8 \tBestMean: -19.0\n",
            "Episode: 168 \tReward: -21.0 \tMean: -19.8 \tBestMean: -19.0\n",
            "Episode: 169 \tReward: -19.0 \tMean: -19.7 \tBestMean: -19.0\n",
            "Episode: 170 \tReward: -21.0 \tMean: -19.8 \tBestMean: -19.0\n",
            "Episode: 171 \tReward: -21.0 \tMean: -19.8 \tBestMean: -19.0\n",
            "Episode: 172 \tReward: -21.0 \tMean: -20.1 \tBestMean: -19.0\n",
            "Episode: 173 \tReward: -21.0 \tMean: -20.2 \tBestMean: -19.0\n",
            "Episode: 174 \tReward: -18.0 \tMean: -19.9 \tBestMean: -19.0\n",
            "Episode: 175 \tReward: -20.0 \tMean: -20.0 \tBestMean: -19.0\n",
            "Episode: 176 \tReward: -20.0 \tMean: -20.2 \tBestMean: -19.0\n",
            "Episode: 177 \tReward: -19.0 \tMean: -20.1 \tBestMean: -19.0\n",
            "Episode: 178 \tReward: -20.0 \tMean: -20.0 \tBestMean: -19.0\n",
            "Episode: 179 \tReward: -20.0 \tMean: -20.1 \tBestMean: -19.0\n",
            "Episode: 180 \tReward: -20.0 \tMean: -20.0 \tBestMean: -19.0\n",
            "Episode: 181 \tReward: -21.0 \tMean: -20.0 \tBestMean: -19.0\n",
            "Episode: 182 \tReward: -20.0 \tMean: -19.9 \tBestMean: -19.0\n",
            "Episode: 183 \tReward: -20.0 \tMean: -19.8 \tBestMean: -19.0\n",
            "Episode: 184 \tReward: -21.0 \tMean: -20.1 \tBestMean: -19.0\n",
            "Episode: 185 \tReward: -21.0 \tMean: -20.2 \tBestMean: -19.0\n",
            "Episode: 186 \tReward: -20.0 \tMean: -20.2 \tBestMean: -19.0\n",
            "Episode: 187 \tReward: -21.0 \tMean: -20.4 \tBestMean: -19.0\n",
            "Episode: 188 \tReward: -21.0 \tMean: -20.5 \tBestMean: -19.0\n",
            "Episode: 189 \tReward: -21.0 \tMean: -20.6 \tBestMean: -19.0\n",
            "Episode: 190 \tReward: -18.0 \tMean: -20.4 \tBestMean: -19.0\n",
            "Episode: 191 \tReward: -20.0 \tMean: -20.3 \tBestMean: -19.0\n",
            "Episode: 192 \tReward: -21.0 \tMean: -20.4 \tBestMean: -19.0\n",
            "Episode: 193 \tReward: -21.0 \tMean: -20.5 \tBestMean: -19.0\n",
            "Episode: 194 \tReward: -21.0 \tMean: -20.5 \tBestMean: -19.0\n",
            "Episode: 195 \tReward: -21.0 \tMean: -20.5 \tBestMean: -19.0\n",
            "Episode: 196 \tReward: -20.0 \tMean: -20.5 \tBestMean: -19.0\n",
            "Episode: 197 \tReward: -20.0 \tMean: -20.4 \tBestMean: -19.0\n",
            "Episode: 198 \tReward: -21.0 \tMean: -20.4 \tBestMean: -19.0\n",
            "Episode: 199 \tReward: -19.0 \tMean: -20.2 \tBestMean: -19.0\n",
            "Episode: 200 \tReward: -20.0 \tMean: -20.4 \tBestMean: -19.0\n",
            "Episode: 201 \tReward: -20.0 \tMean: -20.4 \tBestMean: -19.0\n",
            "Episode: 202 \tReward: -20.0 \tMean: -20.3 \tBestMean: -19.0\n",
            "Episode: 203 \tReward: -21.0 \tMean: -20.3 \tBestMean: -19.0\n",
            "Episode: 204 \tReward: -21.0 \tMean: -20.3 \tBestMean: -19.0\n",
            "Episode: 205 \tReward: -20.0 \tMean: -20.2 \tBestMean: -19.0\n",
            "Episode: 206 \tReward: -21.0 \tMean: -20.3 \tBestMean: -19.0\n",
            "Episode: 207 \tReward: -21.0 \tMean: -20.4 \tBestMean: -19.0\n",
            "Episode: 208 \tReward: -21.0 \tMean: -20.4 \tBestMean: -19.0\n",
            "Episode: 209 \tReward: -19.0 \tMean: -20.4 \tBestMean: -19.0\n",
            "Episode: 210 \tReward: -20.0 \tMean: -20.4 \tBestMean: -19.0\n",
            "Episode: 211 \tReward: -21.0 \tMean: -20.5 \tBestMean: -19.0\n",
            "Episode: 212 \tReward: -19.0 \tMean: -20.4 \tBestMean: -19.0\n",
            "Episode: 213 \tReward: -20.0 \tMean: -20.3 \tBestMean: -19.0\n",
            "Episode: 214 \tReward: -20.0 \tMean: -20.2 \tBestMean: -19.0\n",
            "Episode: 215 \tReward: -21.0 \tMean: -20.3 \tBestMean: -19.0\n",
            "Episode: 216 \tReward: -21.0 \tMean: -20.3 \tBestMean: -19.0\n",
            "Episode: 217 \tReward: -21.0 \tMean: -20.3 \tBestMean: -19.0\n",
            "Episode: 218 \tReward: -19.0 \tMean: -20.1 \tBestMean: -19.0\n",
            "Episode: 219 \tReward: -20.0 \tMean: -20.2 \tBestMean: -19.0\n",
            "Episode: 220 \tReward: -20.0 \tMean: -20.2 \tBestMean: -19.0\n",
            "Episode: 221 \tReward: -20.0 \tMean: -20.1 \tBestMean: -19.0\n",
            "Episode: 222 \tReward: -19.0 \tMean: -20.1 \tBestMean: -19.0\n",
            "Episode: 223 \tReward: -21.0 \tMean: -20.2 \tBestMean: -19.0\n",
            "Episode: 224 \tReward: -19.0 \tMean: -20.1 \tBestMean: -19.0\n",
            "Episode: 225 \tReward: -19.0 \tMean: -19.9 \tBestMean: -19.0\n",
            "Episode: 226 \tReward: -21.0 \tMean: -19.9 \tBestMean: -19.0\n",
            "Episode: 227 \tReward: -21.0 \tMean: -19.9 \tBestMean: -19.0\n",
            "Episode: 228 \tReward: -20.0 \tMean: -20.0 \tBestMean: -19.0\n",
            "Episode: 229 \tReward: -21.0 \tMean: -20.1 \tBestMean: -19.0\n",
            "Episode: 230 \tReward: -20.0 \tMean: -20.1 \tBestMean: -19.0\n",
            "Episode: 231 \tReward: -21.0 \tMean: -20.2 \tBestMean: -19.0\n",
            "Episode: 232 \tReward: -21.0 \tMean: -20.4 \tBestMean: -19.0\n",
            "Episode: 233 \tReward: -20.0 \tMean: -20.3 \tBestMean: -19.0\n",
            "Episode: 234 \tReward: -21.0 \tMean: -20.5 \tBestMean: -19.0\n",
            "Episode: 235 \tReward: -21.0 \tMean: -20.7 \tBestMean: -19.0\n",
            "Episode: 236 \tReward: -20.0 \tMean: -20.6 \tBestMean: -19.0\n",
            "Episode: 237 \tReward: -19.0 \tMean: -20.4 \tBestMean: -19.0\n",
            "Episode: 238 \tReward: -16.0 \tMean: -20.0 \tBestMean: -19.0\n",
            "Episode: 239 \tReward: -21.0 \tMean: -20.0 \tBestMean: -19.0\n",
            "Episode: 240 \tReward: -20.0 \tMean: -20.0 \tBestMean: -19.0\n",
            "Episode: 241 \tReward: -21.0 \tMean: -20.0 \tBestMean: -19.0\n",
            "Episode: 242 \tReward: -20.0 \tMean: -19.9 \tBestMean: -19.0\n",
            "Episode: 243 \tReward: -21.0 \tMean: -20.0 \tBestMean: -19.0\n",
            "Episode: 244 \tReward: -20.0 \tMean: -19.9 \tBestMean: -19.0\n",
            "Episode: 245 \tReward: -20.0 \tMean: -19.8 \tBestMean: -19.0\n",
            "Episode: 246 \tReward: -19.0 \tMean: -19.7 \tBestMean: -19.0\n",
            "Episode: 247 \tReward: -21.0 \tMean: -19.9 \tBestMean: -19.0\n",
            "Episode: 248 \tReward: -21.0 \tMean: -20.4 \tBestMean: -19.0\n",
            "Episode: 249 \tReward: -21.0 \tMean: -20.4 \tBestMean: -19.0\n",
            "Episode: 250 \tReward: -21.0 \tMean: -20.5 \tBestMean: -19.0\n",
            "Episode: 251 \tReward: -21.0 \tMean: -20.5 \tBestMean: -19.0\n",
            "Episode: 252 \tReward: -19.0 \tMean: -20.4 \tBestMean: -19.0\n",
            "Episode: 253 \tReward: -21.0 \tMean: -20.4 \tBestMean: -19.0\n",
            "Episode: 254 \tReward: -19.0 \tMean: -20.3 \tBestMean: -19.0\n",
            "Episode: 255 \tReward: -20.0 \tMean: -20.3 \tBestMean: -19.0\n",
            "Episode: 256 \tReward: -21.0 \tMean: -20.5 \tBestMean: -19.0\n",
            "Episode: 257 \tReward: -21.0 \tMean: -20.5 \tBestMean: -19.0\n",
            "Episode: 258 \tReward: -21.0 \tMean: -20.5 \tBestMean: -19.0\n",
            "Episode: 259 \tReward: -21.0 \tMean: -20.5 \tBestMean: -19.0\n",
            "Episode: 260 \tReward: -20.0 \tMean: -20.4 \tBestMean: -19.0\n",
            "Episode: 261 \tReward: -21.0 \tMean: -20.4 \tBestMean: -19.0\n",
            "Episode: 262 \tReward: -21.0 \tMean: -20.6 \tBestMean: -19.0\n",
            "Episode: 263 \tReward: -21.0 \tMean: -20.6 \tBestMean: -19.0\n",
            "Episode: 264 \tReward: -20.0 \tMean: -20.7 \tBestMean: -19.0\n",
            "Episode: 265 \tReward: -21.0 \tMean: -20.8 \tBestMean: -19.0\n",
            "Episode: 266 \tReward: -21.0 \tMean: -20.8 \tBestMean: -19.0\n",
            "Episode: 267 \tReward: -19.0 \tMean: -20.6 \tBestMean: -19.0\n",
            "Episode: 268 \tReward: -20.0 \tMean: -20.5 \tBestMean: -19.0\n",
            "Episode: 269 \tReward: -21.0 \tMean: -20.5 \tBestMean: -19.0\n",
            "Episode: 270 \tReward: -20.0 \tMean: -20.5 \tBestMean: -19.0\n",
            "Episode: 271 \tReward: -20.0 \tMean: -20.4 \tBestMean: -19.0\n",
            "Episode: 272 \tReward: -21.0 \tMean: -20.4 \tBestMean: -19.0\n",
            "Episode: 273 \tReward: -21.0 \tMean: -20.4 \tBestMean: -19.0\n",
            "Episode: 274 \tReward: -21.0 \tMean: -20.5 \tBestMean: -19.0\n",
            "Episode: 275 \tReward: -21.0 \tMean: -20.5 \tBestMean: -19.0\n",
            "Episode: 276 \tReward: -20.0 \tMean: -20.4 \tBestMean: -19.0\n",
            "Episode: 277 \tReward: -21.0 \tMean: -20.6 \tBestMean: -19.0\n",
            "Episode: 278 \tReward: -19.0 \tMean: -20.5 \tBestMean: -19.0\n",
            "Episode: 279 \tReward: -21.0 \tMean: -20.5 \tBestMean: -19.0\n",
            "Episode: 280 \tReward: -19.0 \tMean: -20.4 \tBestMean: -19.0\n",
            "Episode: 281 \tReward: -20.0 \tMean: -20.4 \tBestMean: -19.0\n",
            "Episode: 282 \tReward: -20.0 \tMean: -20.3 \tBestMean: -19.0\n",
            "Episode: 283 \tReward: -21.0 \tMean: -20.3 \tBestMean: -19.0\n",
            "Episode: 284 \tReward: -20.0 \tMean: -20.2 \tBestMean: -19.0\n",
            "Episode: 285 \tReward: -20.0 \tMean: -20.1 \tBestMean: -19.0\n",
            "Episode: 286 \tReward: -21.0 \tMean: -20.2 \tBestMean: -19.0\n",
            "Episode: 287 \tReward: -21.0 \tMean: -20.2 \tBestMean: -19.0\n",
            "Episode: 288 \tReward: -20.0 \tMean: -20.3 \tBestMean: -19.0\n",
            "Episode: 289 \tReward: -20.0 \tMean: -20.2 \tBestMean: -19.0\n",
            "Episode: 290 \tReward: -21.0 \tMean: -20.4 \tBestMean: -19.0\n",
            "Episode: 291 \tReward: -19.0 \tMean: -20.3 \tBestMean: -19.0\n",
            "Episode: 292 \tReward: -20.0 \tMean: -20.3 \tBestMean: -19.0\n",
            "Episode: 293 \tReward: -21.0 \tMean: -20.3 \tBestMean: -19.0\n",
            "Episode: 294 \tReward: -20.0 \tMean: -20.3 \tBestMean: -19.0\n",
            "Episode: 295 \tReward: -21.0 \tMean: -20.4 \tBestMean: -19.0\n",
            "Episode: 296 \tReward: -20.0 \tMean: -20.3 \tBestMean: -19.0\n",
            "Episode: 297 \tReward: -21.0 \tMean: -20.3 \tBestMean: -19.0\n",
            "Episode: 298 \tReward: -20.0 \tMean: -20.3 \tBestMean: -19.0\n",
            "Episode: 299 \tReward: -19.0 \tMean: -20.2 \tBestMean: -19.0\n",
            "Episode: 300 \tReward: -19.0 \tMean: -20.0 \tBestMean: -19.0\n",
            "Episode: 301 \tReward: -20.0 \tMean: -20.1 \tBestMean: -19.0\n",
            "Episode: 302 \tReward: -20.0 \tMean: -20.1 \tBestMean: -19.0\n",
            "Episode: 303 \tReward: -20.0 \tMean: -20.0 \tBestMean: -19.0\n",
            "Episode: 304 \tReward: -21.0 \tMean: -20.1 \tBestMean: -19.0\n",
            "Episode: 305 \tReward: -21.0 \tMean: -20.1 \tBestMean: -19.0\n",
            "Episode: 306 \tReward: -21.0 \tMean: -20.2 \tBestMean: -19.0\n",
            "Episode: 307 \tReward: -20.0 \tMean: -20.1 \tBestMean: -19.0\n",
            "Episode: 308 \tReward: -20.0 \tMean: -20.1 \tBestMean: -19.0\n",
            "Episode: 309 \tReward: -19.0 \tMean: -20.1 \tBestMean: -19.0\n",
            "Episode: 310 \tReward: -21.0 \tMean: -20.3 \tBestMean: -19.0\n",
            "Episode: 311 \tReward: -21.0 \tMean: -20.4 \tBestMean: -19.0\n",
            "Episode: 312 \tReward: -19.0 \tMean: -20.3 \tBestMean: -19.0\n",
            "Episode: 313 \tReward: -20.0 \tMean: -20.3 \tBestMean: -19.0\n",
            "Episode: 314 \tReward: -21.0 \tMean: -20.3 \tBestMean: -19.0\n",
            "Episode: 315 \tReward: -19.0 \tMean: -20.1 \tBestMean: -19.0\n",
            "Episode: 316 \tReward: -21.0 \tMean: -20.1 \tBestMean: -19.0\n",
            "Episode: 317 \tReward: -20.0 \tMean: -20.1 \tBestMean: -19.0\n",
            "Episode: 318 \tReward: -18.0 \tMean: -19.9 \tBestMean: -19.0\n",
            "Episode: 319 \tReward: -20.0 \tMean: -20.0 \tBestMean: -19.0\n",
            "Episode: 320 \tReward: -20.0 \tMean: -19.9 \tBestMean: -19.0\n",
            "Episode: 321 \tReward: -20.0 \tMean: -19.8 \tBestMean: -19.0\n",
            "Episode: 322 \tReward: -20.0 \tMean: -19.9 \tBestMean: -19.0\n",
            "Episode: 323 \tReward: -19.0 \tMean: -19.8 \tBestMean: -19.0\n",
            "Episode: 324 \tReward: -21.0 \tMean: -19.8 \tBestMean: -19.0\n",
            "Episode: 325 \tReward: -21.0 \tMean: -20.0 \tBestMean: -19.0\n",
            "Episode: 326 \tReward: -20.0 \tMean: -19.9 \tBestMean: -19.0\n",
            "Episode: 327 \tReward: -20.0 \tMean: -19.9 \tBestMean: -19.0\n",
            "Episode: 328 \tReward: -20.0 \tMean: -20.1 \tBestMean: -19.0\n",
            "Episode: 329 \tReward: -18.0 \tMean: -19.9 \tBestMean: -19.0\n",
            "Episode: 330 \tReward: -20.0 \tMean: -19.9 \tBestMean: -19.0\n",
            "Episode: 331 \tReward: -20.0 \tMean: -19.9 \tBestMean: -19.0\n",
            "Episode: 332 \tReward: -20.0 \tMean: -19.9 \tBestMean: -19.0\n",
            "Episode: 333 \tReward: -20.0 \tMean: -20.0 \tBestMean: -19.0\n",
            "Episode: 334 \tReward: -21.0 \tMean: -20.0 \tBestMean: -19.0\n",
            "Episode: 335 \tReward: -21.0 \tMean: -20.0 \tBestMean: -19.0\n",
            "Episode: 336 \tReward: -21.0 \tMean: -20.1 \tBestMean: -19.0\n",
            "Episode: 337 \tReward: -20.0 \tMean: -20.1 \tBestMean: -19.0\n",
            "Episode: 338 \tReward: -19.0 \tMean: -20.0 \tBestMean: -19.0\n",
            "Episode: 339 \tReward: -20.0 \tMean: -20.2 \tBestMean: -19.0\n",
            "Episode: 340 \tReward: -21.0 \tMean: -20.3 \tBestMean: -19.0\n",
            "Episode: 341 \tReward: -21.0 \tMean: -20.4 \tBestMean: -19.0\n",
            "Episode: 342 \tReward: -19.0 \tMean: -20.3 \tBestMean: -19.0\n",
            "Episode: 343 \tReward: -21.0 \tMean: -20.4 \tBestMean: -19.0\n",
            "Episode: 344 \tReward: -19.0 \tMean: -20.2 \tBestMean: -19.0\n",
            "Episode: 345 \tReward: -19.0 \tMean: -20.0 \tBestMean: -19.0\n",
            "Episode: 346 \tReward: -21.0 \tMean: -20.0 \tBestMean: -19.0\n",
            "Episode: 347 \tReward: -21.0 \tMean: -20.1 \tBestMean: -19.0\n",
            "Episode: 348 \tReward: -20.0 \tMean: -20.2 \tBestMean: -19.0\n",
            "Episode: 349 \tReward: -19.0 \tMean: -20.1 \tBestMean: -19.0\n",
            "Episode: 350 \tReward: -19.0 \tMean: -19.9 \tBestMean: -19.0\n",
            "Episode: 351 \tReward: -21.0 \tMean: -19.9 \tBestMean: -19.0\n",
            "Episode: 352 \tReward: -20.0 \tMean: -20.0 \tBestMean: -19.0\n",
            "Episode: 353 \tReward: -20.0 \tMean: -19.9 \tBestMean: -19.0\n",
            "Episode: 354 \tReward: -21.0 \tMean: -20.1 \tBestMean: -19.0\n",
            "Episode: 355 \tReward: -20.0 \tMean: -20.2 \tBestMean: -19.0\n",
            "Episode: 356 \tReward: -21.0 \tMean: -20.2 \tBestMean: -19.0\n",
            "Episode: 357 \tReward: -21.0 \tMean: -20.2 \tBestMean: -19.0\n",
            "Episode: 358 \tReward: -21.0 \tMean: -20.3 \tBestMean: -19.0\n",
            "Episode: 359 \tReward: -21.0 \tMean: -20.5 \tBestMean: -19.0\n",
            "Episode: 360 \tReward: -20.0 \tMean: -20.6 \tBestMean: -19.0\n",
            "Episode: 361 \tReward: -20.0 \tMean: -20.5 \tBestMean: -19.0\n",
            "Episode: 362 \tReward: -21.0 \tMean: -20.6 \tBestMean: -19.0\n",
            "Episode: 363 \tReward: -20.0 \tMean: -20.6 \tBestMean: -19.0\n",
            "Episode: 364 \tReward: -20.0 \tMean: -20.5 \tBestMean: -19.0\n",
            "Episode: 365 \tReward: -21.0 \tMean: -20.6 \tBestMean: -19.0\n",
            "Episode: 366 \tReward: -20.0 \tMean: -20.5 \tBestMean: -19.0\n",
            "Episode: 367 \tReward: -20.0 \tMean: -20.4 \tBestMean: -19.0\n",
            "Episode: 368 \tReward: -20.0 \tMean: -20.3 \tBestMean: -19.0\n",
            "Episode: 369 \tReward: -21.0 \tMean: -20.3 \tBestMean: -19.0\n",
            "Episode: 370 \tReward: -21.0 \tMean: -20.4 \tBestMean: -19.0\n",
            "Episode: 371 \tReward: -20.0 \tMean: -20.4 \tBestMean: -19.0\n",
            "Episode: 372 \tReward: -19.0 \tMean: -20.2 \tBestMean: -19.0\n",
            "Episode: 373 \tReward: -20.0 \tMean: -20.2 \tBestMean: -19.0\n",
            "Episode: 374 \tReward: -21.0 \tMean: -20.3 \tBestMean: -19.0\n",
            "Episode: 375 \tReward: -20.0 \tMean: -20.2 \tBestMean: -19.0\n",
            "Episode: 376 \tReward: -21.0 \tMean: -20.3 \tBestMean: -19.0\n",
            "Episode: 377 \tReward: -21.0 \tMean: -20.4 \tBestMean: -19.0\n",
            "Episode: 378 \tReward: -20.0 \tMean: -20.4 \tBestMean: -19.0\n",
            "Episode: 379 \tReward: -21.0 \tMean: -20.4 \tBestMean: -19.0\n",
            "Episode: 380 \tReward: -21.0 \tMean: -20.4 \tBestMean: -19.0\n",
            "Episode: 381 \tReward: -21.0 \tMean: -20.5 \tBestMean: -19.0\n",
            "Episode: 382 \tReward: -20.0 \tMean: -20.6 \tBestMean: -19.0\n",
            "Episode: 383 \tReward: -21.0 \tMean: -20.7 \tBestMean: -19.0\n",
            "Episode: 384 \tReward: -20.0 \tMean: -20.6 \tBestMean: -19.0\n",
            "Episode: 385 \tReward: -20.0 \tMean: -20.6 \tBestMean: -19.0\n",
            "Episode: 386 \tReward: -20.0 \tMean: -20.5 \tBestMean: -19.0\n",
            "Episode: 387 \tReward: -20.0 \tMean: -20.4 \tBestMean: -19.0\n",
            "Episode: 388 \tReward: -21.0 \tMean: -20.5 \tBestMean: -19.0\n",
            "Episode: 389 \tReward: -20.0 \tMean: -20.4 \tBestMean: -19.0\n",
            "Episode: 390 \tReward: -19.0 \tMean: -20.2 \tBestMean: -19.0\n",
            "Episode: 391 \tReward: -20.0 \tMean: -20.1 \tBestMean: -19.0\n",
            "Episode: 392 \tReward: -19.0 \tMean: -20.0 \tBestMean: -19.0\n",
            "Episode: 393 \tReward: -19.0 \tMean: -19.8 \tBestMean: -19.0\n",
            "Episode: 394 \tReward: -20.0 \tMean: -19.8 \tBestMean: -19.0\n",
            "Episode: 395 \tReward: -21.0 \tMean: -19.9 \tBestMean: -19.0\n",
            "Episode: 396 \tReward: -20.0 \tMean: -19.9 \tBestMean: -19.0\n",
            "Episode: 397 \tReward: -19.0 \tMean: -19.8 \tBestMean: -19.0\n",
            "Episode: 398 \tReward: -21.0 \tMean: -19.8 \tBestMean: -19.0\n",
            "Episode: 399 \tReward: -19.0 \tMean: -19.7 \tBestMean: -19.0\n",
            "Episode: 400 \tReward: -20.0 \tMean: -19.8 \tBestMean: -19.0\n",
            "Episode: 401 \tReward: -20.0 \tMean: -19.8 \tBestMean: -19.0\n",
            "Episode: 402 \tReward: -19.0 \tMean: -19.8 \tBestMean: -19.0\n",
            "Episode: 403 \tReward: -20.0 \tMean: -19.9 \tBestMean: -19.0\n",
            "Episode: 404 \tReward: -21.0 \tMean: -20.0 \tBestMean: -19.0\n",
            "Episode: 405 \tReward: -21.0 \tMean: -20.0 \tBestMean: -19.0\n",
            "Episode: 406 \tReward: -20.0 \tMean: -20.0 \tBestMean: -19.0\n",
            "Episode: 407 \tReward: -20.0 \tMean: -20.1 \tBestMean: -19.0\n",
            "Episode: 408 \tReward: -21.0 \tMean: -20.1 \tBestMean: -19.0\n",
            "Episode: 409 \tReward: -19.0 \tMean: -20.1 \tBestMean: -19.0\n",
            "Episode: 410 \tReward: -21.0 \tMean: -20.2 \tBestMean: -19.0\n",
            "Episode: 411 \tReward: -19.0 \tMean: -20.1 \tBestMean: -19.0\n",
            "Episode: 412 \tReward: -21.0 \tMean: -20.3 \tBestMean: -19.0\n",
            "Episode: 413 \tReward: -20.0 \tMean: -20.3 \tBestMean: -19.0\n",
            "Episode: 414 \tReward: -21.0 \tMean: -20.3 \tBestMean: -19.0\n",
            "Episode: 415 \tReward: -21.0 \tMean: -20.3 \tBestMean: -19.0\n",
            "Episode: 416 \tReward: -21.0 \tMean: -20.4 \tBestMean: -19.0\n",
            "Episode: 417 \tReward: -19.0 \tMean: -20.3 \tBestMean: -19.0\n",
            "Episode: 418 \tReward: -20.0 \tMean: -20.2 \tBestMean: -19.0\n",
            "Episode: 419 \tReward: -21.0 \tMean: -20.4 \tBestMean: -19.0\n",
            "Episode: 420 \tReward: -20.0 \tMean: -20.3 \tBestMean: -19.0\n",
            "Episode: 421 \tReward: -21.0 \tMean: -20.5 \tBestMean: -19.0\n",
            "Episode: 422 \tReward: -20.0 \tMean: -20.4 \tBestMean: -19.0\n",
            "Episode: 423 \tReward: -21.0 \tMean: -20.5 \tBestMean: -19.0\n",
            "Episode: 424 \tReward: -20.0 \tMean: -20.4 \tBestMean: -19.0\n",
            "Episode: 425 \tReward: -21.0 \tMean: -20.4 \tBestMean: -19.0\n",
            "Episode: 426 \tReward: -20.0 \tMean: -20.3 \tBestMean: -19.0\n",
            "Episode: 427 \tReward: -19.0 \tMean: -20.3 \tBestMean: -19.0\n",
            "Episode: 428 \tReward: -21.0 \tMean: -20.4 \tBestMean: -19.0\n",
            "Episode: 429 \tReward: -20.0 \tMean: -20.3 \tBestMean: -19.0\n",
            "Episode: 430 \tReward: -19.0 \tMean: -20.2 \tBestMean: -19.0\n",
            "Episode: 431 \tReward: -20.0 \tMean: -20.1 \tBestMean: -19.0\n",
            "Episode: 432 \tReward: -19.0 \tMean: -20.0 \tBestMean: -19.0\n",
            "Episode: 433 \tReward: -21.0 \tMean: -20.0 \tBestMean: -19.0\n",
            "Episode: 434 \tReward: -21.0 \tMean: -20.1 \tBestMean: -19.0\n",
            "Episode: 435 \tReward: -21.0 \tMean: -20.1 \tBestMean: -19.0\n",
            "Episode: 436 \tReward: -21.0 \tMean: -20.2 \tBestMean: -19.0\n",
            "Episode: 437 \tReward: -21.0 \tMean: -20.4 \tBestMean: -19.0\n",
            "Episode: 438 \tReward: -21.0 \tMean: -20.4 \tBestMean: -19.0\n",
            "Episode: 439 \tReward: -19.0 \tMean: -20.3 \tBestMean: -19.0\n",
            "Episode: 440 \tReward: -20.0 \tMean: -20.4 \tBestMean: -19.0\n",
            "Episode: 441 \tReward: -20.0 \tMean: -20.4 \tBestMean: -19.0\n",
            "Episode: 442 \tReward: -18.0 \tMean: -20.3 \tBestMean: -19.0\n",
            "Episode: 443 \tReward: -20.0 \tMean: -20.2 \tBestMean: -19.0\n",
            "Episode: 444 \tReward: -20.0 \tMean: -20.1 \tBestMean: -19.0\n",
            "Episode: 445 \tReward: -20.0 \tMean: -20.0 \tBestMean: -19.0\n",
            "Episode: 446 \tReward: -17.0 \tMean: -19.6 \tBestMean: -19.0\n",
            "Episode: 447 \tReward: -20.0 \tMean: -19.5 \tBestMean: -19.0\n",
            "Episode: 448 \tReward: -20.0 \tMean: -19.4 \tBestMean: -19.0\n",
            "Episode: 449 \tReward: -20.0 \tMean: -19.5 \tBestMean: -19.0\n",
            "Episode: 450 \tReward: -21.0 \tMean: -19.6 \tBestMean: -19.0\n",
            "Episode: 451 \tReward: -21.0 \tMean: -19.7 \tBestMean: -19.0\n",
            "Episode: 452 \tReward: -19.0 \tMean: -19.8 \tBestMean: -19.0\n",
            "Episode: 453 \tReward: -21.0 \tMean: -19.9 \tBestMean: -19.0\n",
            "Episode: 454 \tReward: -20.0 \tMean: -19.9 \tBestMean: -19.0\n",
            "Episode: 455 \tReward: -19.0 \tMean: -19.8 \tBestMean: -19.0\n",
            "Episode: 456 \tReward: -20.0 \tMean: -20.1 \tBestMean: -19.0\n",
            "Episode: 457 \tReward: -21.0 \tMean: -20.2 \tBestMean: -19.0\n",
            "Episode: 458 \tReward: -21.0 \tMean: -20.3 \tBestMean: -19.0\n",
            "Episode: 459 \tReward: -20.0 \tMean: -20.3 \tBestMean: -19.0\n",
            "Episode: 460 \tReward: -20.0 \tMean: -20.2 \tBestMean: -19.0\n",
            "Episode: 461 \tReward: -21.0 \tMean: -20.2 \tBestMean: -19.0\n",
            "Episode: 462 \tReward: -20.0 \tMean: -20.3 \tBestMean: -19.0\n",
            "Episode: 463 \tReward: -20.0 \tMean: -20.2 \tBestMean: -19.0\n",
            "Episode: 464 \tReward: -21.0 \tMean: -20.3 \tBestMean: -19.0\n",
            "Episode: 465 \tReward: -19.0 \tMean: -20.3 \tBestMean: -19.0\n",
            "Episode: 466 \tReward: -19.0 \tMean: -20.2 \tBestMean: -19.0\n",
            "Episode: 467 \tReward: -20.0 \tMean: -20.1 \tBestMean: -19.0\n",
            "Episode: 468 \tReward: -21.0 \tMean: -20.1 \tBestMean: -19.0\n",
            "Episode: 469 \tReward: -19.0 \tMean: -20.0 \tBestMean: -19.0\n",
            "Episode: 470 \tReward: -21.0 \tMean: -20.1 \tBestMean: -19.0\n",
            "Episode: 471 \tReward: -19.0 \tMean: -19.9 \tBestMean: -19.0\n",
            "Episode: 472 \tReward: -20.0 \tMean: -19.9 \tBestMean: -19.0\n",
            "Episode: 473 \tReward: -20.0 \tMean: -19.9 \tBestMean: -19.0\n",
            "Episode: 474 \tReward: -19.0 \tMean: -19.7 \tBestMean: -19.0\n",
            "Episode: 475 \tReward: -20.0 \tMean: -19.8 \tBestMean: -19.0\n",
            "Episode: 476 \tReward: -21.0 \tMean: -20.0 \tBestMean: -19.0\n",
            "Episode: 477 \tReward: -18.0 \tMean: -19.8 \tBestMean: -19.0\n",
            "Episode: 478 \tReward: -19.0 \tMean: -19.6 \tBestMean: -19.0\n",
            "Episode: 479 \tReward: -19.0 \tMean: -19.6 \tBestMean: -19.0\n",
            "Episode: 480 \tReward: -20.0 \tMean: -19.5 \tBestMean: -19.0\n",
            "Episode: 481 \tReward: -21.0 \tMean: -19.7 \tBestMean: -19.0\n",
            "Episode: 482 \tReward: -20.0 \tMean: -19.7 \tBestMean: -19.0\n",
            "Episode: 483 \tReward: -21.0 \tMean: -19.8 \tBestMean: -19.0\n",
            "Episode: 484 \tReward: -21.0 \tMean: -20.0 \tBestMean: -19.0\n",
            "Episode: 485 \tReward: -19.0 \tMean: -19.9 \tBestMean: -19.0\n",
            "Episode: 486 \tReward: -20.0 \tMean: -19.8 \tBestMean: -19.0\n",
            "Episode: 487 \tReward: -17.0 \tMean: -19.7 \tBestMean: -19.0\n",
            "Episode: 488 \tReward: -20.0 \tMean: -19.8 \tBestMean: -19.0\n",
            "Episode: 489 \tReward: -21.0 \tMean: -20.0 \tBestMean: -19.0\n",
            "Episode: 490 \tReward: -20.0 \tMean: -20.0 \tBestMean: -19.0\n",
            "Episode: 491 \tReward: -20.0 \tMean: -19.9 \tBestMean: -19.0\n",
            "Episode: 492 \tReward: -20.0 \tMean: -19.9 \tBestMean: -19.0\n",
            "Episode: 493 \tReward: -21.0 \tMean: -19.9 \tBestMean: -19.0\n",
            "Episode: 494 \tReward: -20.0 \tMean: -19.8 \tBestMean: -19.0\n",
            "Episode: 495 \tReward: -21.0 \tMean: -20.0 \tBestMean: -19.0\n",
            "Episode: 496 \tReward: -20.0 \tMean: -20.0 \tBestMean: -19.0\n",
            "Episode: 497 \tReward: -20.0 \tMean: -20.3 \tBestMean: -19.0\n",
            "Episode: 498 \tReward: -20.0 \tMean: -20.3 \tBestMean: -19.0\n",
            "Episode: 499 \tReward: -20.0 \tMean: -20.2 \tBestMean: -19.0\n",
            "Episode: 500 \tReward: -21.0 \tMean: -20.3 \tBestMean: -19.0\n",
            "Episode: 501 \tReward: -21.0 \tMean: -20.4 \tBestMean: -19.0\n",
            "Episode: 502 \tReward: -20.0 \tMean: -20.4 \tBestMean: -19.0\n",
            "Episode: 503 \tReward: -21.0 \tMean: -20.4 \tBestMean: -19.0\n",
            "Episode: 504 \tReward: -19.0 \tMean: -20.3 \tBestMean: -19.0\n",
            "Episode: 505 \tReward: -21.0 \tMean: -20.3 \tBestMean: -19.0\n",
            "Episode: 506 \tReward: -19.0 \tMean: -20.2 \tBestMean: -19.0\n",
            "Episode: 507 \tReward: -21.0 \tMean: -20.3 \tBestMean: -19.0\n",
            "Episode: 508 \tReward: -19.0 \tMean: -20.2 \tBestMean: -19.0\n",
            "Episode: 509 \tReward: -18.0 \tMean: -20.0 \tBestMean: -19.0\n",
            "Episode: 510 \tReward: -18.0 \tMean: -19.7 \tBestMean: -19.0\n",
            "Episode: 511 \tReward: -21.0 \tMean: -19.7 \tBestMean: -19.0\n",
            "Episode: 512 \tReward: -18.0 \tMean: -19.5 \tBestMean: -19.0\n",
            "Episode: 513 \tReward: -20.0 \tMean: -19.4 \tBestMean: -19.0\n",
            "Episode: 514 \tReward: -20.0 \tMean: -19.5 \tBestMean: -19.0\n",
            "Episode: 515 \tReward: -21.0 \tMean: -19.5 \tBestMean: -19.0\n",
            "Episode: 516 \tReward: -20.0 \tMean: -19.6 \tBestMean: -19.0\n",
            "Episode: 517 \tReward: -20.0 \tMean: -19.5 \tBestMean: -19.0\n",
            "Episode: 518 \tReward: -20.0 \tMean: -19.6 \tBestMean: -19.0\n",
            "Episode: 519 \tReward: -19.0 \tMean: -19.7 \tBestMean: -19.0\n",
            "Episode: 520 \tReward: -20.0 \tMean: -19.9 \tBestMean: -19.0\n",
            "Episode: 521 \tReward: -21.0 \tMean: -19.9 \tBestMean: -19.0\n",
            "Episode: 522 \tReward: -21.0 \tMean: -20.2 \tBestMean: -19.0\n",
            "Episode: 523 \tReward: -21.0 \tMean: -20.3 \tBestMean: -19.0\n",
            "Episode: 524 \tReward: -20.0 \tMean: -20.3 \tBestMean: -19.0\n",
            "Episode: 525 \tReward: -21.0 \tMean: -20.3 \tBestMean: -19.0\n",
            "Episode: 526 \tReward: -21.0 \tMean: -20.4 \tBestMean: -19.0\n",
            "Episode: 527 \tReward: -19.0 \tMean: -20.3 \tBestMean: -19.0\n",
            "Episode: 528 \tReward: -21.0 \tMean: -20.4 \tBestMean: -19.0\n",
            "Episode: 529 \tReward: -20.0 \tMean: -20.5 \tBestMean: -19.0\n",
            "Episode: 530 \tReward: -19.0 \tMean: -20.4 \tBestMean: -19.0\n",
            "Episode: 531 \tReward: -20.0 \tMean: -20.3 \tBestMean: -19.0\n",
            "Episode: 532 \tReward: -21.0 \tMean: -20.3 \tBestMean: -19.0\n",
            "Episode: 533 \tReward: -21.0 \tMean: -20.3 \tBestMean: -19.0\n",
            "Episode: 534 \tReward: -21.0 \tMean: -20.4 \tBestMean: -19.0\n",
            "Episode: 535 \tReward: -20.0 \tMean: -20.3 \tBestMean: -19.0\n",
            "Episode: 536 \tReward: -20.0 \tMean: -20.2 \tBestMean: -19.0\n",
            "Episode: 537 \tReward: -20.0 \tMean: -20.3 \tBestMean: -19.0\n",
            "Episode: 538 \tReward: -20.0 \tMean: -20.2 \tBestMean: -19.0\n",
            "Episode: 539 \tReward: -20.0 \tMean: -20.2 \tBestMean: -19.0\n",
            "Episode: 540 \tReward: -19.0 \tMean: -20.2 \tBestMean: -19.0\n",
            "Episode: 541 \tReward: -20.0 \tMean: -20.2 \tBestMean: -19.0\n",
            "Episode: 542 \tReward: -21.0 \tMean: -20.2 \tBestMean: -19.0\n",
            "Episode: 543 \tReward: -21.0 \tMean: -20.2 \tBestMean: -19.0\n",
            "Episode: 544 \tReward: -21.0 \tMean: -20.2 \tBestMean: -19.0\n",
            "Episode: 545 \tReward: -21.0 \tMean: -20.3 \tBestMean: -19.0\n",
            "Episode: 546 \tReward: -21.0 \tMean: -20.4 \tBestMean: -19.0\n",
            "Episode: 547 \tReward: -21.0 \tMean: -20.5 \tBestMean: -19.0\n",
            "Episode: 548 \tReward: -21.0 \tMean: -20.6 \tBestMean: -19.0\n",
            "Episode: 549 \tReward: -19.0 \tMean: -20.5 \tBestMean: -19.0\n",
            "Episode: 550 \tReward: -21.0 \tMean: -20.7 \tBestMean: -19.0\n",
            "Episode: 551 \tReward: -20.0 \tMean: -20.7 \tBestMean: -19.0\n",
            "Episode: 552 \tReward: -21.0 \tMean: -20.7 \tBestMean: -19.0\n",
            "Episode: 553 \tReward: -21.0 \tMean: -20.7 \tBestMean: -19.0\n",
            "Episode: 554 \tReward: -19.0 \tMean: -20.5 \tBestMean: -19.0\n",
            "Episode: 555 \tReward: -20.0 \tMean: -20.4 \tBestMean: -19.0\n",
            "Episode: 556 \tReward: -20.0 \tMean: -20.3 \tBestMean: -19.0\n",
            "Episode: 557 \tReward: -19.0 \tMean: -20.1 \tBestMean: -19.0\n",
            "Episode: 558 \tReward: -21.0 \tMean: -20.1 \tBestMean: -19.0\n",
            "Episode: 559 \tReward: -21.0 \tMean: -20.3 \tBestMean: -19.0\n",
            "Episode: 560 \tReward: -21.0 \tMean: -20.3 \tBestMean: -19.0\n",
            "Episode: 561 \tReward: -20.0 \tMean: -20.3 \tBestMean: -19.0\n",
            "Episode: 562 \tReward: -19.0 \tMean: -20.1 \tBestMean: -19.0\n",
            "Episode: 563 \tReward: -19.0 \tMean: -19.9 \tBestMean: -19.0\n",
            "Episode: 564 \tReward: -21.0 \tMean: -20.1 \tBestMean: -19.0\n",
            "Episode: 565 \tReward: -21.0 \tMean: -20.2 \tBestMean: -19.0\n",
            "Episode: 566 \tReward: -20.0 \tMean: -20.2 \tBestMean: -19.0\n",
            "Episode: 567 \tReward: -20.0 \tMean: -20.3 \tBestMean: -19.0\n",
            "Episode: 568 \tReward: -21.0 \tMean: -20.3 \tBestMean: -19.0\n",
            "Episode: 569 \tReward: -20.0 \tMean: -20.2 \tBestMean: -19.0\n",
            "Episode: 570 \tReward: -20.0 \tMean: -20.1 \tBestMean: -19.0\n",
            "Episode: 571 \tReward: -20.0 \tMean: -20.1 \tBestMean: -19.0\n",
            "Episode: 572 \tReward: -20.0 \tMean: -20.2 \tBestMean: -19.0\n",
            "Episode: 573 \tReward: -21.0 \tMean: -20.4 \tBestMean: -19.0\n",
            "Episode: 574 \tReward: -21.0 \tMean: -20.4 \tBestMean: -19.0\n",
            "Episode: 575 \tReward: -19.0 \tMean: -20.2 \tBestMean: -19.0\n",
            "Episode: 576 \tReward: -20.0 \tMean: -20.2 \tBestMean: -19.0\n",
            "Episode: 577 \tReward: -21.0 \tMean: -20.3 \tBestMean: -19.0\n",
            "Episode: 578 \tReward: -21.0 \tMean: -20.3 \tBestMean: -19.0\n",
            "Episode: 579 \tReward: -20.0 \tMean: -20.3 \tBestMean: -19.0\n",
            "Episode: 580 \tReward: -19.0 \tMean: -20.2 \tBestMean: -19.0\n",
            "Episode: 581 \tReward: -21.0 \tMean: -20.3 \tBestMean: -19.0\n",
            "Episode: 582 \tReward: -19.0 \tMean: -20.2 \tBestMean: -19.0\n",
            "Episode: 583 \tReward: -20.0 \tMean: -20.1 \tBestMean: -19.0\n",
            "Episode: 584 \tReward: -19.0 \tMean: -19.9 \tBestMean: -19.0\n",
            "Episode: 585 \tReward: -20.0 \tMean: -20.0 \tBestMean: -19.0\n",
            "Episode: 586 \tReward: -21.0 \tMean: -20.1 \tBestMean: -19.0\n",
            "Episode: 587 \tReward: -20.0 \tMean: -20.0 \tBestMean: -19.0\n",
            "Episode: 588 \tReward: -20.0 \tMean: -19.9 \tBestMean: -19.0\n",
            "Episode: 589 \tReward: -20.0 \tMean: -19.9 \tBestMean: -19.0\n",
            "Episode: 590 \tReward: -18.0 \tMean: -19.8 \tBestMean: -19.0\n",
            "Episode: 591 \tReward: -20.0 \tMean: -19.7 \tBestMean: -19.0\n",
            "Episode: 592 \tReward: -19.0 \tMean: -19.7 \tBestMean: -19.0\n",
            "Episode: 593 \tReward: -20.0 \tMean: -19.7 \tBestMean: -19.0\n",
            "Episode: 594 \tReward: -19.0 \tMean: -19.7 \tBestMean: -19.0\n",
            "Episode: 595 \tReward: -18.0 \tMean: -19.5 \tBestMean: -19.0\n",
            "Episode: 596 \tReward: -21.0 \tMean: -19.5 \tBestMean: -19.0\n",
            "Episode: 597 \tReward: -19.0 \tMean: -19.4 \tBestMean: -19.0\n",
            "Episode: 598 \tReward: -21.0 \tMean: -19.5 \tBestMean: -19.0\n",
            "Episode: 599 \tReward: -20.0 \tMean: -19.5 \tBestMean: -19.0\n",
            "Episode: 600 \tReward: -20.0 \tMean: -19.7 \tBestMean: -19.0\n",
            "Episode: 601 \tReward: -21.0 \tMean: -19.8 \tBestMean: -19.0\n",
            "Episode: 602 \tReward: -20.0 \tMean: -19.9 \tBestMean: -19.0\n",
            "Episode: 603 \tReward: -21.0 \tMean: -20.0 \tBestMean: -19.0\n",
            "Episode: 604 \tReward: -19.0 \tMean: -20.0 \tBestMean: -19.0\n",
            "Episode: 605 \tReward: -21.0 \tMean: -20.3 \tBestMean: -19.0\n",
            "Episode: 606 \tReward: -21.0 \tMean: -20.3 \tBestMean: -19.0\n",
            "Episode: 607 \tReward: -21.0 \tMean: -20.5 \tBestMean: -19.0\n",
            "Episode: 608 \tReward: -21.0 \tMean: -20.5 \tBestMean: -19.0\n",
            "Episode: 609 \tReward: -19.0 \tMean: -20.4 \tBestMean: -19.0\n",
            "Episode: 610 \tReward: -21.0 \tMean: -20.5 \tBestMean: -19.0\n",
            "Episode: 611 \tReward: -19.0 \tMean: -20.3 \tBestMean: -19.0\n",
            "Episode: 612 \tReward: -19.0 \tMean: -20.2 \tBestMean: -19.0\n",
            "Episode: 613 \tReward: -20.0 \tMean: -20.1 \tBestMean: -19.0\n",
            "Episode: 614 \tReward: -20.0 \tMean: -20.2 \tBestMean: -19.0\n",
            "Episode: 615 \tReward: -18.0 \tMean: -19.9 \tBestMean: -19.0\n",
            "Episode: 616 \tReward: -18.0 \tMean: -19.6 \tBestMean: -19.0\n",
            "Episode: 617 \tReward: -20.0 \tMean: -19.5 \tBestMean: -19.0\n",
            "Episode: 618 \tReward: -19.0 \tMean: -19.3 \tBestMean: -19.0\n",
            "Episode: 619 \tReward: -20.0 \tMean: -19.4 \tBestMean: -19.0\n",
            "Episode: 620 \tReward: -20.0 \tMean: -19.3 \tBestMean: -19.0\n",
            "Episode: 621 \tReward: -20.0 \tMean: -19.4 \tBestMean: -19.0\n",
            "Episode: 622 \tReward: -21.0 \tMean: -19.6 \tBestMean: -19.0\n",
            "Episode: 623 \tReward: -20.0 \tMean: -19.6 \tBestMean: -19.0\n",
            "Episode: 624 \tReward: -21.0 \tMean: -19.7 \tBestMean: -19.0\n",
            "Episode: 625 \tReward: -21.0 \tMean: -20.0 \tBestMean: -19.0\n",
            "Episode: 626 \tReward: -20.0 \tMean: -20.2 \tBestMean: -19.0\n",
            "Episode: 627 \tReward: -20.0 \tMean: -20.2 \tBestMean: -19.0\n",
            "Episode: 628 \tReward: -19.0 \tMean: -20.2 \tBestMean: -19.0\n",
            "Episode: 629 \tReward: -21.0 \tMean: -20.3 \tBestMean: -19.0\n",
            "Episode: 630 \tReward: -20.0 \tMean: -20.3 \tBestMean: -19.0\n",
            "Episode: 631 \tReward: -21.0 \tMean: -20.4 \tBestMean: -19.0\n",
            "Episode: 632 \tReward: -21.0 \tMean: -20.4 \tBestMean: -19.0\n",
            "Episode: 633 \tReward: -20.0 \tMean: -20.4 \tBestMean: -19.0\n",
            "Episode: 634 \tReward: -20.0 \tMean: -20.3 \tBestMean: -19.0\n",
            "Episode: 635 \tReward: -20.0 \tMean: -20.2 \tBestMean: -19.0\n",
            "Episode: 636 \tReward: -21.0 \tMean: -20.3 \tBestMean: -19.0\n",
            "Episode: 637 \tReward: -21.0 \tMean: -20.4 \tBestMean: -19.0\n",
            "Episode: 638 \tReward: -20.0 \tMean: -20.5 \tBestMean: -19.0\n",
            "Episode: 639 \tReward: -21.0 \tMean: -20.5 \tBestMean: -19.0\n",
            "Episode: 640 \tReward: -20.0 \tMean: -20.5 \tBestMean: -19.0\n",
            "Episode: 641 \tReward: -20.0 \tMean: -20.4 \tBestMean: -19.0\n",
            "Episode: 642 \tReward: -21.0 \tMean: -20.4 \tBestMean: -19.0\n",
            "Episode: 643 \tReward: -20.0 \tMean: -20.4 \tBestMean: -19.0\n",
            "Episode: 644 \tReward: -21.0 \tMean: -20.5 \tBestMean: -19.0\n",
            "Episode: 645 \tReward: -21.0 \tMean: -20.6 \tBestMean: -19.0\n",
            "Episode: 646 \tReward: -20.0 \tMean: -20.5 \tBestMean: -19.0\n",
            "Episode: 647 \tReward: -21.0 \tMean: -20.5 \tBestMean: -19.0\n",
            "Episode: 648 \tReward: -20.0 \tMean: -20.5 \tBestMean: -19.0\n",
            "Episode: 649 \tReward: -21.0 \tMean: -20.5 \tBestMean: -19.0\n",
            "Episode: 650 \tReward: -18.0 \tMean: -20.3 \tBestMean: -19.0\n",
            "Episode: 651 \tReward: -20.0 \tMean: -20.3 \tBestMean: -19.0\n",
            "Episode: 652 \tReward: -21.0 \tMean: -20.3 \tBestMean: -19.0\n",
            "Episode: 653 \tReward: -19.0 \tMean: -20.2 \tBestMean: -19.0\n",
            "Episode: 654 \tReward: -21.0 \tMean: -20.2 \tBestMean: -19.0\n",
            "Episode: 655 \tReward: -20.0 \tMean: -20.1 \tBestMean: -19.0\n",
            "Episode: 656 \tReward: -20.0 \tMean: -20.1 \tBestMean: -19.0\n",
            "Episode: 657 \tReward: -20.0 \tMean: -20.0 \tBestMean: -19.0\n",
            "Episode: 658 \tReward: -20.0 \tMean: -20.0 \tBestMean: -19.0\n",
            "Episode: 659 \tReward: -19.0 \tMean: -19.8 \tBestMean: -19.0\n",
            "Episode: 660 \tReward: -21.0 \tMean: -20.1 \tBestMean: -19.0\n",
            "Episode: 661 \tReward: -21.0 \tMean: -20.2 \tBestMean: -19.0\n",
            "Episode: 662 \tReward: -20.0 \tMean: -20.1 \tBestMean: -19.0\n",
            "Episode: 663 \tReward: -20.0 \tMean: -20.2 \tBestMean: -19.0\n",
            "Episode: 664 \tReward: -19.0 \tMean: -20.0 \tBestMean: -19.0\n",
            "Episode: 665 \tReward: -20.0 \tMean: -20.0 \tBestMean: -19.0\n",
            "Episode: 666 \tReward: -19.0 \tMean: -19.9 \tBestMean: -19.0\n",
            "Episode: 667 \tReward: -21.0 \tMean: -20.0 \tBestMean: -19.0\n",
            "Episode: 668 \tReward: -20.0 \tMean: -20.0 \tBestMean: -19.0\n",
            "Episode: 669 \tReward: -20.0 \tMean: -20.1 \tBestMean: -19.0\n",
            "Episode: 670 \tReward: -21.0 \tMean: -20.1 \tBestMean: -19.0\n",
            "Episode: 671 \tReward: -21.0 \tMean: -20.1 \tBestMean: -19.0\n",
            "Episode: 672 \tReward: -20.0 \tMean: -20.1 \tBestMean: -19.0\n",
            "Episode: 673 \tReward: -21.0 \tMean: -20.2 \tBestMean: -19.0\n",
            "Episode: 674 \tReward: -21.0 \tMean: -20.4 \tBestMean: -19.0\n",
            "Episode: 675 \tReward: -20.0 \tMean: -20.4 \tBestMean: -19.0\n",
            "Episode: 676 \tReward: -20.0 \tMean: -20.5 \tBestMean: -19.0\n",
            "Episode: 677 \tReward: -21.0 \tMean: -20.5 \tBestMean: -19.0\n",
            "Episode: 678 \tReward: -21.0 \tMean: -20.6 \tBestMean: -19.0\n",
            "Episode: 679 \tReward: -20.0 \tMean: -20.6 \tBestMean: -19.0\n",
            "Episode: 680 \tReward: -21.0 \tMean: -20.6 \tBestMean: -19.0\n",
            "Episode: 681 \tReward: -20.0 \tMean: -20.5 \tBestMean: -19.0\n",
            "Episode: 682 \tReward: -21.0 \tMean: -20.6 \tBestMean: -19.0\n",
            "Episode: 683 \tReward: -20.0 \tMean: -20.5 \tBestMean: -19.0\n",
            "Episode: 684 \tReward: -19.0 \tMean: -20.3 \tBestMean: -19.0\n",
            "Episode: 685 \tReward: -20.0 \tMean: -20.3 \tBestMean: -19.0\n",
            "Episode: 686 \tReward: -19.0 \tMean: -20.2 \tBestMean: -19.0\n",
            "Episode: 687 \tReward: -18.0 \tMean: -19.9 \tBestMean: -19.0\n",
            "Episode: 688 \tReward: -21.0 \tMean: -19.9 \tBestMean: -19.0\n",
            "Episode: 689 \tReward: -20.0 \tMean: -19.9 \tBestMean: -19.0\n",
            "Episode: 690 \tReward: -21.0 \tMean: -19.9 \tBestMean: -19.0\n",
            "Episode: 691 \tReward: -20.0 \tMean: -19.9 \tBestMean: -19.0\n",
            "Episode: 692 \tReward: -21.0 \tMean: -19.9 \tBestMean: -19.0\n",
            "Episode: 693 \tReward: -19.0 \tMean: -19.8 \tBestMean: -19.0\n",
            "Episode: 694 \tReward: -19.0 \tMean: -19.8 \tBestMean: -19.0\n",
            "Episode: 695 \tReward: -20.0 \tMean: -19.8 \tBestMean: -19.0\n",
            "Episode: 696 \tReward: -21.0 \tMean: -20.0 \tBestMean: -19.0\n",
            "Episode: 697 \tReward: -19.0 \tMean: -20.1 \tBestMean: -19.0\n",
            "Episode: 698 \tReward: -21.0 \tMean: -20.1 \tBestMean: -19.0\n",
            "Episode: 699 \tReward: -20.0 \tMean: -20.1 \tBestMean: -19.0\n",
            "Episode: 700 \tReward: -21.0 \tMean: -20.1 \tBestMean: -19.0\n",
            "Episode: 701 \tReward: -21.0 \tMean: -20.2 \tBestMean: -19.0\n",
            "Episode: 702 \tReward: -19.0 \tMean: -20.0 \tBestMean: -19.0\n",
            "Episode: 703 \tReward: -20.0 \tMean: -20.1 \tBestMean: -19.0\n",
            "Episode: 704 \tReward: -19.0 \tMean: -20.1 \tBestMean: -19.0\n",
            "Episode: 705 \tReward: -21.0 \tMean: -20.2 \tBestMean: -19.0\n",
            "Episode: 706 \tReward: -17.0 \tMean: -19.8 \tBestMean: -19.0\n",
            "Episode: 707 \tReward: -21.0 \tMean: -20.0 \tBestMean: -19.0\n",
            "Episode: 708 \tReward: -21.0 \tMean: -20.0 \tBestMean: -19.0\n",
            "Episode: 709 \tReward: -18.0 \tMean: -19.8 \tBestMean: -19.0\n",
            "Episode: 710 \tReward: -21.0 \tMean: -19.8 \tBestMean: -19.0\n",
            "Episode: 711 \tReward: -20.0 \tMean: -19.7 \tBestMean: -19.0\n",
            "Episode: 712 \tReward: -20.0 \tMean: -19.8 \tBestMean: -19.0\n",
            "Episode: 713 \tReward: -21.0 \tMean: -19.9 \tBestMean: -19.0\n",
            "Episode: 714 \tReward: -20.0 \tMean: -20.0 \tBestMean: -19.0\n",
            "Episode: 715 \tReward: -18.0 \tMean: -19.7 \tBestMean: -19.0\n",
            "Episode: 716 \tReward: -20.0 \tMean: -20.0 \tBestMean: -19.0\n",
            "Episode: 717 \tReward: -20.0 \tMean: -19.9 \tBestMean: -19.0\n",
            "Episode: 718 \tReward: -21.0 \tMean: -19.9 \tBestMean: -19.0\n",
            "Episode: 719 \tReward: -20.0 \tMean: -20.1 \tBestMean: -19.0\n",
            "Episode: 720 \tReward: -21.0 \tMean: -20.1 \tBestMean: -19.0\n",
            "Episode: 721 \tReward: -21.0 \tMean: -20.2 \tBestMean: -19.0\n",
            "Episode: 722 \tReward: -19.0 \tMean: -20.1 \tBestMean: -19.0\n",
            "Episode: 723 \tReward: -21.0 \tMean: -20.1 \tBestMean: -19.0\n",
            "Episode: 724 \tReward: -20.0 \tMean: -20.1 \tBestMean: -19.0\n",
            "Episode: 725 \tReward: -21.0 \tMean: -20.4 \tBestMean: -19.0\n",
            "Episode: 726 \tReward: -21.0 \tMean: -20.5 \tBestMean: -19.0\n",
            "Episode: 727 \tReward: -20.0 \tMean: -20.5 \tBestMean: -19.0\n",
            "Episode: 728 \tReward: -20.0 \tMean: -20.4 \tBestMean: -19.0\n",
            "Episode: 729 \tReward: -21.0 \tMean: -20.5 \tBestMean: -19.0\n",
            "Episode: 730 \tReward: -19.0 \tMean: -20.3 \tBestMean: -19.0\n",
            "Episode: 731 \tReward: -20.0 \tMean: -20.2 \tBestMean: -19.0\n",
            "Episode: 732 \tReward: -20.0 \tMean: -20.3 \tBestMean: -19.0\n",
            "Episode: 733 \tReward: -20.0 \tMean: -20.2 \tBestMean: -19.0\n",
            "Episode: 734 \tReward: -20.0 \tMean: -20.2 \tBestMean: -19.0\n",
            "Episode: 735 \tReward: -18.0 \tMean: -19.9 \tBestMean: -19.0\n",
            "Episode: 736 \tReward: -20.0 \tMean: -19.8 \tBestMean: -19.0\n",
            "Episode: 737 \tReward: -21.0 \tMean: -19.9 \tBestMean: -19.0\n",
            "Episode: 738 \tReward: -21.0 \tMean: -20.0 \tBestMean: -19.0\n",
            "Episode: 739 \tReward: -20.0 \tMean: -19.9 \tBestMean: -19.0\n",
            "Episode: 740 \tReward: -20.0 \tMean: -20.0 \tBestMean: -19.0\n",
            "Episode: 741 \tReward: -20.0 \tMean: -20.0 \tBestMean: -19.0\n",
            "Episode: 742 \tReward: -19.0 \tMean: -19.9 \tBestMean: -19.0\n",
            "Episode: 743 \tReward: -21.0 \tMean: -20.0 \tBestMean: -19.0\n",
            "Episode: 744 \tReward: -21.0 \tMean: -20.1 \tBestMean: -19.0\n",
            "Episode: 745 \tReward: -20.0 \tMean: -20.3 \tBestMean: -19.0\n",
            "Episode: 746 \tReward: -20.0 \tMean: -20.3 \tBestMean: -19.0\n",
            "Episode: 747 \tReward: -21.0 \tMean: -20.3 \tBestMean: -19.0\n",
            "Episode: 748 \tReward: -19.0 \tMean: -20.1 \tBestMean: -19.0\n",
            "Episode: 749 \tReward: -21.0 \tMean: -20.2 \tBestMean: -19.0\n",
            "Episode: 750 \tReward: -19.0 \tMean: -20.1 \tBestMean: -19.0\n",
            "Episode: 751 \tReward: -19.0 \tMean: -20.0 \tBestMean: -19.0\n",
            "Episode: 752 \tReward: -20.0 \tMean: -20.1 \tBestMean: -19.0\n",
            "Episode: 753 \tReward: -20.0 \tMean: -20.0 \tBestMean: -19.0\n",
            "Episode: 754 \tReward: -19.0 \tMean: -19.8 \tBestMean: -19.0\n",
            "Episode: 755 \tReward: -21.0 \tMean: -19.9 \tBestMean: -19.0\n",
            "Episode: 756 \tReward: -21.0 \tMean: -20.0 \tBestMean: -19.0\n",
            "Episode: 757 \tReward: -20.0 \tMean: -19.9 \tBestMean: -19.0\n",
            "Episode: 758 \tReward: -21.0 \tMean: -20.1 \tBestMean: -19.0\n",
            "Episode: 759 \tReward: -20.0 \tMean: -20.0 \tBestMean: -19.0\n",
            "Episode: 760 \tReward: -19.0 \tMean: -20.0 \tBestMean: -19.0\n",
            "Episode: 761 \tReward: -20.0 \tMean: -20.1 \tBestMean: -19.0\n",
            "Episode: 762 \tReward: -20.0 \tMean: -20.1 \tBestMean: -19.0\n",
            "Episode: 763 \tReward: -19.0 \tMean: -20.0 \tBestMean: -19.0\n",
            "Episode: 764 \tReward: -18.0 \tMean: -19.9 \tBestMean: -19.0\n",
            "Episode: 765 \tReward: -18.0 \tMean: -19.6 \tBestMean: -19.0\n",
            "Episode: 766 \tReward: -21.0 \tMean: -19.6 \tBestMean: -19.0\n",
            "Episode: 767 \tReward: -20.0 \tMean: -19.6 \tBestMean: -19.0\n",
            "Episode: 768 \tReward: -21.0 \tMean: -19.6 \tBestMean: -19.0\n",
            "Episode: 769 \tReward: -20.0 \tMean: -19.6 \tBestMean: -19.0\n",
            "Episode: 770 \tReward: -19.0 \tMean: -19.6 \tBestMean: -19.0\n",
            "Episode: 771 \tReward: -21.0 \tMean: -19.7 \tBestMean: -19.0\n",
            "Episode: 772 \tReward: -21.0 \tMean: -19.8 \tBestMean: -19.0\n",
            "Episode: 773 \tReward: -20.0 \tMean: -19.9 \tBestMean: -19.0\n",
            "Episode: 774 \tReward: -20.0 \tMean: -20.1 \tBestMean: -19.0\n",
            "Episode: 775 \tReward: -20.0 \tMean: -20.3 \tBestMean: -19.0\n",
            "Episode: 776 \tReward: -21.0 \tMean: -20.3 \tBestMean: -19.0\n",
            "Episode: 777 \tReward: -20.0 \tMean: -20.3 \tBestMean: -19.0\n",
            "Episode: 778 \tReward: -19.0 \tMean: -20.1 \tBestMean: -19.0\n",
            "Episode: 779 \tReward: -21.0 \tMean: -20.2 \tBestMean: -19.0\n",
            "Episode: 780 \tReward: -19.0 \tMean: -20.2 \tBestMean: -19.0\n",
            "Episode: 781 \tReward: -17.0 \tMean: -19.8 \tBestMean: -19.0\n",
            "Episode: 782 \tReward: -20.0 \tMean: -19.7 \tBestMean: -19.0\n",
            "Episode: 783 \tReward: -20.0 \tMean: -19.7 \tBestMean: -19.0\n",
            "Episode: 784 \tReward: -20.0 \tMean: -19.7 \tBestMean: -19.0\n",
            "Episode: 785 \tReward: -21.0 \tMean: -19.8 \tBestMean: -19.0\n",
            "Episode: 786 \tReward: -19.0 \tMean: -19.6 \tBestMean: -19.0\n",
            "Episode: 787 \tReward: -19.0 \tMean: -19.5 \tBestMean: -19.0\n",
            "Episode: 788 \tReward: -19.0 \tMean: -19.5 \tBestMean: -19.0\n",
            "Episode: 789 \tReward: -21.0 \tMean: -19.5 \tBestMean: -19.0\n",
            "Episode: 790 \tReward: -20.0 \tMean: -19.6 \tBestMean: -19.0\n",
            "Episode: 791 \tReward: -21.0 \tMean: -20.0 \tBestMean: -19.0\n",
            "Episode: 792 \tReward: -21.0 \tMean: -20.1 \tBestMean: -19.0\n",
            "Episode: 793 \tReward: -21.0 \tMean: -20.2 \tBestMean: -19.0\n",
            "Episode: 794 \tReward: -21.0 \tMean: -20.3 \tBestMean: -19.0\n",
            "Episode: 795 \tReward: -21.0 \tMean: -20.3 \tBestMean: -19.0\n",
            "Episode: 796 \tReward: -20.0 \tMean: -20.4 \tBestMean: -19.0\n",
            "Episode: 797 \tReward: -20.0 \tMean: -20.5 \tBestMean: -19.0\n",
            "Episode: 798 \tReward: -20.0 \tMean: -20.6 \tBestMean: -19.0\n",
            "Episode: 799 \tReward: -21.0 \tMean: -20.6 \tBestMean: -19.0\n",
            "Episode: 800 \tReward: -21.0 \tMean: -20.7 \tBestMean: -19.0\n",
            "Episode: 801 \tReward: -21.0 \tMean: -20.7 \tBestMean: -19.0\n",
            "Episode: 802 \tReward: -20.0 \tMean: -20.6 \tBestMean: -19.0\n",
            "Episode: 803 \tReward: -20.0 \tMean: -20.5 \tBestMean: -19.0\n",
            "Episode: 804 \tReward: -21.0 \tMean: -20.5 \tBestMean: -19.0\n",
            "Episode: 805 \tReward: -21.0 \tMean: -20.5 \tBestMean: -19.0\n",
            "Episode: 806 \tReward: -21.0 \tMean: -20.6 \tBestMean: -19.0\n",
            "Episode: 807 \tReward: -20.0 \tMean: -20.6 \tBestMean: -19.0\n",
            "Episode: 808 \tReward: -20.0 \tMean: -20.6 \tBestMean: -19.0\n",
            "Episode: 809 \tReward: -21.0 \tMean: -20.6 \tBestMean: -19.0\n",
            "Episode: 810 \tReward: -21.0 \tMean: -20.6 \tBestMean: -19.0\n",
            "Episode: 811 \tReward: -21.0 \tMean: -20.6 \tBestMean: -19.0\n",
            "Episode: 812 \tReward: -19.0 \tMean: -20.5 \tBestMean: -19.0\n",
            "Episode: 813 \tReward: -20.0 \tMean: -20.5 \tBestMean: -19.0\n",
            "Episode: 814 \tReward: -21.0 \tMean: -20.5 \tBestMean: -19.0\n",
            "Episode: 815 \tReward: -20.0 \tMean: -20.4 \tBestMean: -19.0\n",
            "Episode: 816 \tReward: -19.0 \tMean: -20.2 \tBestMean: -19.0\n",
            "Episode: 817 \tReward: -19.0 \tMean: -20.1 \tBestMean: -19.0\n",
            "Episode: 818 \tReward: -21.0 \tMean: -20.2 \tBestMean: -19.0\n",
            "Episode: 819 \tReward: -21.0 \tMean: -20.2 \tBestMean: -19.0\n",
            "Episode: 820 \tReward: -19.0 \tMean: -20.0 \tBestMean: -19.0\n",
            "Episode: 821 \tReward: -18.0 \tMean: -19.7 \tBestMean: -19.0\n",
            "Episode: 822 \tReward: -21.0 \tMean: -19.9 \tBestMean: -19.0\n",
            "Episode: 823 \tReward: -19.0 \tMean: -19.8 \tBestMean: -19.0\n",
            "Episode: 824 \tReward: -20.0 \tMean: -19.7 \tBestMean: -19.0\n",
            "Episode: 825 \tReward: -19.0 \tMean: -19.6 \tBestMean: -19.0\n",
            "Episode: 826 \tReward: -20.0 \tMean: -19.7 \tBestMean: -19.0\n",
            "Episode: 827 \tReward: -20.0 \tMean: -19.8 \tBestMean: -19.0\n",
            "Episode: 828 \tReward: -20.0 \tMean: -19.7 \tBestMean: -19.0\n",
            "Episode: 829 \tReward: -21.0 \tMean: -19.7 \tBestMean: -19.0\n",
            "Episode: 830 \tReward: -21.0 \tMean: -19.9 \tBestMean: -19.0\n",
            "Episode: 831 \tReward: -20.0 \tMean: -20.1 \tBestMean: -19.0\n",
            "Episode: 832 \tReward: -21.0 \tMean: -20.1 \tBestMean: -19.0\n",
            "Episode: 833 \tReward: -21.0 \tMean: -20.3 \tBestMean: -19.0\n",
            "Episode: 834 \tReward: -20.0 \tMean: -20.3 \tBestMean: -19.0\n",
            "Episode: 835 \tReward: -19.0 \tMean: -20.3 \tBestMean: -19.0\n",
            "Episode: 836 \tReward: -20.0 \tMean: -20.3 \tBestMean: -19.0\n",
            "Episode: 837 \tReward: -20.0 \tMean: -20.3 \tBestMean: -19.0\n",
            "Episode: 838 \tReward: -21.0 \tMean: -20.4 \tBestMean: -19.0\n",
            "Episode: 839 \tReward: -21.0 \tMean: -20.4 \tBestMean: -19.0\n",
            "Episode: 840 \tReward: -20.0 \tMean: -20.3 \tBestMean: -19.0\n",
            "Episode: 841 \tReward: -20.0 \tMean: -20.3 \tBestMean: -19.0\n",
            "Episode: 842 \tReward: -20.0 \tMean: -20.2 \tBestMean: -19.0\n",
            "Episode: 843 \tReward: -20.0 \tMean: -20.1 \tBestMean: -19.0\n",
            "Episode: 844 \tReward: -20.0 \tMean: -20.1 \tBestMean: -19.0\n",
            "Episode: 845 \tReward: -20.0 \tMean: -20.2 \tBestMean: -19.0\n",
            "Episode: 846 \tReward: -20.0 \tMean: -20.2 \tBestMean: -19.0\n",
            "Episode: 847 \tReward: -19.0 \tMean: -20.1 \tBestMean: -19.0\n",
            "Episode: 848 \tReward: -21.0 \tMean: -20.1 \tBestMean: -19.0\n",
            "Episode: 849 \tReward: -20.0 \tMean: -20.0 \tBestMean: -19.0\n",
            "Episode: 850 \tReward: -20.0 \tMean: -20.0 \tBestMean: -19.0\n",
            "Episode: 851 \tReward: -21.0 \tMean: -20.1 \tBestMean: -19.0\n",
            "Episode: 852 \tReward: -20.0 \tMean: -20.1 \tBestMean: -19.0\n",
            "Episode: 853 \tReward: -20.0 \tMean: -20.1 \tBestMean: -19.0\n",
            "Episode: 854 \tReward: -20.0 \tMean: -20.1 \tBestMean: -19.0\n",
            "Episode: 855 \tReward: -20.0 \tMean: -20.1 \tBestMean: -19.0\n",
            "Episode: 856 \tReward: -19.0 \tMean: -20.0 \tBestMean: -19.0\n",
            "Episode: 857 \tReward: -20.0 \tMean: -20.1 \tBestMean: -19.0\n",
            "Episode: 858 \tReward: -20.0 \tMean: -20.0 \tBestMean: -19.0\n",
            "Episode: 859 \tReward: -21.0 \tMean: -20.1 \tBestMean: -19.0\n",
            "Episode: 860 \tReward: -20.0 \tMean: -20.1 \tBestMean: -19.0\n",
            "Episode: 861 \tReward: -20.0 \tMean: -20.0 \tBestMean: -19.0\n",
            "Episode: 862 \tReward: -20.0 \tMean: -20.0 \tBestMean: -19.0\n",
            "Episode: 863 \tReward: -20.0 \tMean: -20.0 \tBestMean: -19.0\n",
            "Episode: 864 \tReward: -21.0 \tMean: -20.1 \tBestMean: -19.0\n",
            "Episode: 865 \tReward: -21.0 \tMean: -20.2 \tBestMean: -19.0\n",
            "Episode: 866 \tReward: -19.0 \tMean: -20.2 \tBestMean: -19.0\n",
            "Episode: 867 \tReward: -20.0 \tMean: -20.2 \tBestMean: -19.0\n",
            "Episode: 868 \tReward: -21.0 \tMean: -20.3 \tBestMean: -19.0\n",
            "Episode: 869 \tReward: -20.0 \tMean: -20.2 \tBestMean: -19.0\n",
            "Episode: 870 \tReward: -20.0 \tMean: -20.2 \tBestMean: -19.0\n",
            "Episode: 871 \tReward: -21.0 \tMean: -20.3 \tBestMean: -19.0\n",
            "Episode: 872 \tReward: -19.0 \tMean: -20.2 \tBestMean: -19.0\n",
            "Episode: 873 \tReward: -21.0 \tMean: -20.3 \tBestMean: -19.0\n",
            "Episode: 874 \tReward: -21.0 \tMean: -20.3 \tBestMean: -19.0\n",
            "Episode: 875 \tReward: -21.0 \tMean: -20.3 \tBestMean: -19.0\n",
            "Episode: 876 \tReward: -20.0 \tMean: -20.4 \tBestMean: -19.0\n",
            "Episode: 877 \tReward: -20.0 \tMean: -20.4 \tBestMean: -19.0\n",
            "Episode: 878 \tReward: -21.0 \tMean: -20.4 \tBestMean: -19.0\n",
            "Episode: 879 \tReward: -20.0 \tMean: -20.4 \tBestMean: -19.0\n",
            "Episode: 880 \tReward: -21.0 \tMean: -20.5 \tBestMean: -19.0\n",
            "Episode: 881 \tReward: -20.0 \tMean: -20.4 \tBestMean: -19.0\n",
            "Episode: 882 \tReward: -21.0 \tMean: -20.6 \tBestMean: -19.0\n",
            "Episode: 883 \tReward: -21.0 \tMean: -20.6 \tBestMean: -19.0\n",
            "Episode: 884 \tReward: -19.0 \tMean: -20.4 \tBestMean: -19.0\n",
            "Episode: 885 \tReward: -20.0 \tMean: -20.3 \tBestMean: -19.0\n",
            "Episode: 886 \tReward: -20.0 \tMean: -20.3 \tBestMean: -19.0\n",
            "Episode: 887 \tReward: -19.0 \tMean: -20.2 \tBestMean: -19.0\n",
            "Episode: 888 \tReward: -20.0 \tMean: -20.1 \tBestMean: -19.0\n",
            "Episode: 889 \tReward: -20.0 \tMean: -20.1 \tBestMean: -19.0\n",
            "Episode: 890 \tReward: -20.0 \tMean: -20.0 \tBestMean: -19.0\n",
            "Episode: 891 \tReward: -20.0 \tMean: -20.0 \tBestMean: -19.0\n",
            "Episode: 892 \tReward: -20.0 \tMean: -19.9 \tBestMean: -19.0\n",
            "Episode: 893 \tReward: -21.0 \tMean: -19.9 \tBestMean: -19.0\n",
            "Episode: 894 \tReward: -20.0 \tMean: -20.0 \tBestMean: -19.0\n",
            "Episode: 895 \tReward: -21.0 \tMean: -20.1 \tBestMean: -19.0\n",
            "Episode: 896 \tReward: -20.0 \tMean: -20.1 \tBestMean: -19.0\n",
            "Episode: 897 \tReward: -20.0 \tMean: -20.2 \tBestMean: -19.0\n",
            "Episode: 898 \tReward: -19.0 \tMean: -20.1 \tBestMean: -19.0\n",
            "Episode: 899 \tReward: -21.0 \tMean: -20.2 \tBestMean: -19.0\n",
            "Episode: 900 \tReward: -20.0 \tMean: -20.2 \tBestMean: -19.0\n",
            "Episode: 901 \tReward: -21.0 \tMean: -20.3 \tBestMean: -19.0\n",
            "Episode: 902 \tReward: -21.0 \tMean: -20.4 \tBestMean: -19.0\n",
            "Episode: 903 \tReward: -20.0 \tMean: -20.3 \tBestMean: -19.0\n",
            "Episode: 904 \tReward: -21.0 \tMean: -20.4 \tBestMean: -19.0\n",
            "Episode: 905 \tReward: -21.0 \tMean: -20.4 \tBestMean: -19.0\n",
            "Episode: 906 \tReward: -21.0 \tMean: -20.5 \tBestMean: -19.0\n",
            "Episode: 907 \tReward: -19.0 \tMean: -20.4 \tBestMean: -19.0\n",
            "Episode: 908 \tReward: -18.0 \tMean: -20.3 \tBestMean: -19.0\n",
            "Episode: 909 \tReward: -21.0 \tMean: -20.3 \tBestMean: -19.0\n",
            "Episode: 910 \tReward: -20.0 \tMean: -20.3 \tBestMean: -19.0\n",
            "Episode: 911 \tReward: -20.0 \tMean: -20.2 \tBestMean: -19.0\n",
            "Episode: 912 \tReward: -18.0 \tMean: -19.9 \tBestMean: -19.0\n",
            "Episode: 913 \tReward: -20.0 \tMean: -19.9 \tBestMean: -19.0\n",
            "Episode: 914 \tReward: -20.0 \tMean: -19.8 \tBestMean: -19.0\n",
            "Episode: 915 \tReward: -21.0 \tMean: -19.8 \tBestMean: -19.0\n",
            "Episode: 916 \tReward: -18.0 \tMean: -19.5 \tBestMean: -19.0\n",
            "Episode: 917 \tReward: -21.0 \tMean: -19.7 \tBestMean: -19.0\n",
            "Episode: 918 \tReward: -20.0 \tMean: -19.9 \tBestMean: -19.0\n",
            "Episode: 919 \tReward: -21.0 \tMean: -19.9 \tBestMean: -19.0\n",
            "Episode: 920 \tReward: -21.0 \tMean: -20.0 \tBestMean: -19.0\n",
            "Episode: 921 \tReward: -20.0 \tMean: -20.0 \tBestMean: -19.0\n",
            "Episode: 922 \tReward: -21.0 \tMean: -20.3 \tBestMean: -19.0\n",
            "Episode: 923 \tReward: -20.0 \tMean: -20.3 \tBestMean: -19.0\n",
            "Episode: 924 \tReward: -21.0 \tMean: -20.4 \tBestMean: -19.0\n",
            "Episode: 925 \tReward: -19.0 \tMean: -20.2 \tBestMean: -19.0\n",
            "Episode: 926 \tReward: -21.0 \tMean: -20.5 \tBestMean: -19.0\n",
            "Episode: 927 \tReward: -20.0 \tMean: -20.4 \tBestMean: -19.0\n",
            "Episode: 928 \tReward: -21.0 \tMean: -20.5 \tBestMean: -19.0\n",
            "Episode: 929 \tReward: -20.0 \tMean: -20.4 \tBestMean: -19.0\n",
            "Episode: 930 \tReward: -19.0 \tMean: -20.2 \tBestMean: -19.0\n",
            "Episode: 931 \tReward: -21.0 \tMean: -20.3 \tBestMean: -19.0\n",
            "Episode: 932 \tReward: -20.0 \tMean: -20.2 \tBestMean: -19.0\n",
            "Episode: 933 \tReward: -20.0 \tMean: -20.2 \tBestMean: -19.0\n",
            "Episode: 934 \tReward: -19.0 \tMean: -20.0 \tBestMean: -19.0\n",
            "Episode: 935 \tReward: -19.0 \tMean: -20.0 \tBestMean: -19.0\n",
            "Episode: 936 \tReward: -20.0 \tMean: -19.9 \tBestMean: -19.0\n",
            "Episode: 937 \tReward: -19.0 \tMean: -19.8 \tBestMean: -19.0\n",
            "Episode: 938 \tReward: -20.0 \tMean: -19.7 \tBestMean: -19.0\n",
            "Episode: 939 \tReward: -20.0 \tMean: -19.7 \tBestMean: -19.0\n",
            "Episode: 940 \tReward: -21.0 \tMean: -19.9 \tBestMean: -19.0\n",
            "Episode: 941 \tReward: -20.0 \tMean: -19.8 \tBestMean: -19.0\n",
            "Episode: 942 \tReward: -20.0 \tMean: -19.8 \tBestMean: -19.0\n",
            "Episode: 943 \tReward: -21.0 \tMean: -19.9 \tBestMean: -19.0\n",
            "Episode: 944 \tReward: -21.0 \tMean: -20.1 \tBestMean: -19.0\n",
            "Episode: 945 \tReward: -21.0 \tMean: -20.3 \tBestMean: -19.0\n",
            "Episode: 946 \tReward: -21.0 \tMean: -20.4 \tBestMean: -19.0\n",
            "Episode: 947 \tReward: -21.0 \tMean: -20.6 \tBestMean: -19.0\n",
            "Episode: 948 \tReward: -21.0 \tMean: -20.7 \tBestMean: -19.0\n",
            "Episode: 949 \tReward: -19.0 \tMean: -20.6 \tBestMean: -19.0\n",
            "Episode: 950 \tReward: -21.0 \tMean: -20.6 \tBestMean: -19.0\n",
            "Episode: 951 \tReward: -21.0 \tMean: -20.7 \tBestMean: -19.0\n",
            "Episode: 952 \tReward: -21.0 \tMean: -20.8 \tBestMean: -19.0\n",
            "Episode: 953 \tReward: -21.0 \tMean: -20.8 \tBestMean: -19.0\n",
            "Episode: 954 \tReward: -21.0 \tMean: -20.8 \tBestMean: -19.0\n",
            "Episode: 955 \tReward: -20.0 \tMean: -20.7 \tBestMean: -19.0\n",
            "Episode: 956 \tReward: -20.0 \tMean: -20.6 \tBestMean: -19.0\n",
            "Episode: 957 \tReward: -19.0 \tMean: -20.4 \tBestMean: -19.0\n",
            "Episode: 958 \tReward: -20.0 \tMean: -20.3 \tBestMean: -19.0\n",
            "Episode: 959 \tReward: -20.0 \tMean: -20.4 \tBestMean: -19.0\n",
            "Episode: 960 \tReward: -21.0 \tMean: -20.4 \tBestMean: -19.0\n",
            "Episode: 961 \tReward: -20.0 \tMean: -20.3 \tBestMean: -19.0\n",
            "Episode: 962 \tReward: -21.0 \tMean: -20.3 \tBestMean: -19.0\n",
            "Episode: 963 \tReward: -18.0 \tMean: -20.0 \tBestMean: -19.0\n",
            "Episode: 964 \tReward: -19.0 \tMean: -19.8 \tBestMean: -19.0\n",
            "Episode: 965 \tReward: -20.0 \tMean: -19.8 \tBestMean: -19.0\n",
            "Episode: 966 \tReward: -18.0 \tMean: -19.6 \tBestMean: -19.0\n",
            "Episode: 967 \tReward: -21.0 \tMean: -19.8 \tBestMean: -19.0\n",
            "Episode: 968 \tReward: -21.0 \tMean: -19.9 \tBestMean: -19.0\n",
            "Episode: 969 \tReward: -20.0 \tMean: -19.9 \tBestMean: -19.0\n",
            "Episode: 970 \tReward: -20.0 \tMean: -19.8 \tBestMean: -19.0\n",
            "Episode: 971 \tReward: -19.0 \tMean: -19.7 \tBestMean: -19.0\n",
            "Episode: 972 \tReward: -20.0 \tMean: -19.6 \tBestMean: -19.0\n",
            "Episode: 973 \tReward: -20.0 \tMean: -19.8 \tBestMean: -19.0\n",
            "Episode: 974 \tReward: -21.0 \tMean: -20.0 \tBestMean: -19.0\n",
            "Episode: 975 \tReward: -20.0 \tMean: -20.0 \tBestMean: -19.0\n",
            "Episode: 976 \tReward: -21.0 \tMean: -20.3 \tBestMean: -19.0\n",
            "Episode: 977 \tReward: -21.0 \tMean: -20.3 \tBestMean: -19.0\n",
            "Episode: 978 \tReward: -19.0 \tMean: -20.1 \tBestMean: -19.0\n",
            "Episode: 979 \tReward: -20.0 \tMean: -20.1 \tBestMean: -19.0\n",
            "Episode: 980 \tReward: -21.0 \tMean: -20.2 \tBestMean: -19.0\n",
            "Episode: 981 \tReward: -21.0 \tMean: -20.4 \tBestMean: -19.0\n",
            "Episode: 982 \tReward: -20.0 \tMean: -20.4 \tBestMean: -19.0\n",
            "Episode: 983 \tReward: -20.0 \tMean: -20.4 \tBestMean: -19.0\n",
            "Episode: 984 \tReward: -21.0 \tMean: -20.4 \tBestMean: -19.0\n",
            "Episode: 985 \tReward: -17.0 \tMean: -20.1 \tBestMean: -19.0\n",
            "Episode: 986 \tReward: -20.0 \tMean: -20.0 \tBestMean: -19.0\n",
            "Episode: 987 \tReward: -19.0 \tMean: -19.8 \tBestMean: -19.0\n",
            "Episode: 988 \tReward: -20.0 \tMean: -19.9 \tBestMean: -19.0\n",
            "Episode: 989 \tReward: -20.0 \tMean: -19.9 \tBestMean: -19.0\n",
            "Episode: 990 \tReward: -19.0 \tMean: -19.7 \tBestMean: -19.0\n",
            "Episode: 991 \tReward: -21.0 \tMean: -19.7 \tBestMean: -19.0\n",
            "Episode: 992 \tReward: -20.0 \tMean: -19.7 \tBestMean: -19.0\n",
            "Episode: 993 \tReward: -20.0 \tMean: -19.7 \tBestMean: -19.0\n",
            "Episode: 994 \tReward: -20.0 \tMean: -19.6 \tBestMean: -19.0\n",
            "Episode: 995 \tReward: -20.0 \tMean: -19.9 \tBestMean: -19.0\n",
            "Episode: 996 \tReward: -21.0 \tMean: -20.0 \tBestMean: -19.0\n",
            "Episode: 997 \tReward: -19.0 \tMean: -20.0 \tBestMean: -19.0\n",
            "Episode: 998 \tReward: -19.0 \tMean: -19.9 \tBestMean: -19.0\n",
            "Episode: 999 \tReward: -20.0 \tMean: -19.9 \tBestMean: -19.0\n",
            "Episode: 1000 \tReward: -21.0 \tMean: -20.1 \tBestMean: -19.0\n",
            "Episode: 1001 \tReward: -21.0 \tMean: -20.1 \tBestMean: -19.0\n",
            "Episode: 1002 \tReward: -20.0 \tMean: -20.1 \tBestMean: -19.0\n",
            "Episode: 1003 \tReward: -21.0 \tMean: -20.2 \tBestMean: -19.0\n",
            "Episode: 1004 \tReward: -21.0 \tMean: -20.3 \tBestMean: -19.0\n",
            "Episode: 1005 \tReward: -19.0 \tMean: -20.2 \tBestMean: -19.0\n",
            "Episode: 1006 \tReward: -19.0 \tMean: -20.0 \tBestMean: -19.0\n",
            "Episode: 1007 \tReward: -20.0 \tMean: -20.1 \tBestMean: -19.0\n",
            "Episode: 1008 \tReward: -20.0 \tMean: -20.2 \tBestMean: -19.0\n",
            "Episode: 1009 \tReward: -18.0 \tMean: -20.0 \tBestMean: -19.0\n",
            "Episode: 1010 \tReward: -20.0 \tMean: -19.9 \tBestMean: -19.0\n",
            "Episode: 1011 \tReward: -21.0 \tMean: -19.9 \tBestMean: -19.0\n",
            "Episode: 1012 \tReward: -21.0 \tMean: -20.0 \tBestMean: -19.0\n",
            "Episode: 1013 \tReward: -20.0 \tMean: -19.9 \tBestMean: -19.0\n",
            "Episode: 1014 \tReward: -19.0 \tMean: -19.7 \tBestMean: -19.0\n",
            "Episode: 1015 \tReward: -19.0 \tMean: -19.7 \tBestMean: -19.0\n",
            "Episode: 1016 \tReward: -20.0 \tMean: -19.8 \tBestMean: -19.0\n",
            "Episode: 1017 \tReward: -20.0 \tMean: -19.8 \tBestMean: -19.0\n",
            "Episode: 1018 \tReward: -20.0 \tMean: -19.8 \tBestMean: -19.0\n",
            "Episode: 1019 \tReward: -19.0 \tMean: -19.9 \tBestMean: -19.0\n",
            "Episode: 1020 \tReward: -20.0 \tMean: -19.9 \tBestMean: -19.0\n",
            "Episode: 1021 \tReward: -21.0 \tMean: -19.9 \tBestMean: -19.0\n",
            "Episode: 1022 \tReward: -21.0 \tMean: -19.9 \tBestMean: -19.0\n",
            "Episode: 1023 \tReward: -20.0 \tMean: -19.9 \tBestMean: -19.0\n",
            "Episode: 1024 \tReward: -21.0 \tMean: -20.1 \tBestMean: -19.0\n",
            "Episode: 1025 \tReward: -21.0 \tMean: -20.3 \tBestMean: -19.0\n",
            "Episode: 1026 \tReward: -20.0 \tMean: -20.3 \tBestMean: -19.0\n",
            "Episode: 1027 \tReward: -20.0 \tMean: -20.3 \tBestMean: -19.0\n",
            "Episode: 1028 \tReward: -21.0 \tMean: -20.4 \tBestMean: -19.0\n",
            "Episode: 1029 \tReward: -20.0 \tMean: -20.5 \tBestMean: -19.0\n",
            "Episode: 1030 \tReward: -18.0 \tMean: -20.3 \tBestMean: -19.0\n",
            "Episode: 1031 \tReward: -18.0 \tMean: -20.0 \tBestMean: -19.0\n",
            "Episode: 1032 \tReward: -21.0 \tMean: -20.0 \tBestMean: -19.0\n",
            "Episode: 1033 \tReward: -21.0 \tMean: -20.1 \tBestMean: -19.0\n",
            "Episode: 1034 \tReward: -20.0 \tMean: -20.0 \tBestMean: -19.0\n",
            "Episode: 1035 \tReward: -21.0 \tMean: -20.0 \tBestMean: -19.0\n",
            "Episode: 1036 \tReward: -18.0 \tMean: -19.8 \tBestMean: -19.0\n",
            "Episode: 1037 \tReward: -20.0 \tMean: -19.8 \tBestMean: -19.0\n",
            "Episode: 1038 \tReward: -21.0 \tMean: -19.8 \tBestMean: -19.0\n",
            "Episode: 1039 \tReward: -21.0 \tMean: -19.9 \tBestMean: -19.0\n",
            "Episode: 1040 \tReward: -21.0 \tMean: -20.2 \tBestMean: -19.0\n",
            "Episode: 1041 \tReward: -21.0 \tMean: -20.5 \tBestMean: -19.0\n",
            "Episode: 1042 \tReward: -20.0 \tMean: -20.4 \tBestMean: -19.0\n",
            "Episode: 1043 \tReward: -20.0 \tMean: -20.3 \tBestMean: -19.0\n",
            "Episode: 1044 \tReward: -21.0 \tMean: -20.4 \tBestMean: -19.0\n",
            "Episode: 1045 \tReward: -21.0 \tMean: -20.4 \tBestMean: -19.0\n",
            "Episode: 1046 \tReward: -20.0 \tMean: -20.6 \tBestMean: -19.0\n",
            "Episode: 1047 \tReward: -20.0 \tMean: -20.6 \tBestMean: -19.0\n",
            "Episode: 1048 \tReward: -19.0 \tMean: -20.4 \tBestMean: -19.0\n",
            "Episode: 1049 \tReward: -20.0 \tMean: -20.3 \tBestMean: -19.0\n",
            "Episode: 1050 \tReward: -20.0 \tMean: -20.2 \tBestMean: -19.0\n",
            "Episode: 1051 \tReward: -20.0 \tMean: -20.1 \tBestMean: -19.0\n",
            "Episode: 1052 \tReward: -21.0 \tMean: -20.2 \tBestMean: -19.0\n",
            "Episode: 1053 \tReward: -21.0 \tMean: -20.3 \tBestMean: -19.0\n",
            "Episode: 1054 \tReward: -21.0 \tMean: -20.3 \tBestMean: -19.0\n",
            "Episode: 1055 \tReward: -21.0 \tMean: -20.3 \tBestMean: -19.0\n",
            "Episode: 1056 \tReward: -19.0 \tMean: -20.2 \tBestMean: -19.0\n",
            "Episode: 1057 \tReward: -20.0 \tMean: -20.2 \tBestMean: -19.0\n",
            "Episode: 1058 \tReward: -19.0 \tMean: -20.2 \tBestMean: -19.0\n",
            "Episode: 1059 \tReward: -21.0 \tMean: -20.3 \tBestMean: -19.0\n",
            "Episode: 1060 \tReward: -19.0 \tMean: -20.2 \tBestMean: -19.0\n",
            "Episode: 1061 \tReward: -20.0 \tMean: -20.2 \tBestMean: -19.0\n",
            "Episode: 1062 \tReward: -20.0 \tMean: -20.1 \tBestMean: -19.0\n",
            "Episode: 1063 \tReward: -20.0 \tMean: -20.0 \tBestMean: -19.0\n",
            "Episode: 1064 \tReward: -21.0 \tMean: -20.0 \tBestMean: -19.0\n",
            "Episode: 1065 \tReward: -21.0 \tMean: -20.0 \tBestMean: -19.0\n",
            "Episode: 1066 \tReward: -20.0 \tMean: -20.1 \tBestMean: -19.0\n",
            "Episode: 1067 \tReward: -18.0 \tMean: -19.9 \tBestMean: -19.0\n",
            "Episode: 1068 \tReward: -20.0 \tMean: -20.0 \tBestMean: -19.0\n",
            "Episode: 1069 \tReward: -21.0 \tMean: -20.0 \tBestMean: -19.0\n",
            "Episode: 1070 \tReward: -18.0 \tMean: -19.9 \tBestMean: -19.0\n",
            "Episode: 1071 \tReward: -19.0 \tMean: -19.8 \tBestMean: -19.0\n",
            "Episode: 1072 \tReward: -18.0 \tMean: -19.6 \tBestMean: -19.0\n",
            "Episode: 1073 \tReward: -20.0 \tMean: -19.6 \tBestMean: -19.0\n",
            "Episode: 1074 \tReward: -20.0 \tMean: -19.5 \tBestMean: -19.0\n",
            "Episode: 1075 \tReward: -21.0 \tMean: -19.5 \tBestMean: -19.0\n",
            "Episode: 1076 \tReward: -20.0 \tMean: -19.5 \tBestMean: -19.0\n",
            "Episode: 1077 \tReward: -19.0 \tMean: -19.6 \tBestMean: -19.0\n",
            "Episode: 1078 \tReward: -20.0 \tMean: -19.6 \tBestMean: -19.0\n",
            "Episode: 1079 \tReward: -20.0 \tMean: -19.5 \tBestMean: -19.0\n",
            "Episode: 1080 \tReward: -19.0 \tMean: -19.6 \tBestMean: -19.0\n",
            "Episode: 1081 \tReward: -21.0 \tMean: -19.8 \tBestMean: -19.0\n",
            "Episode: 1082 \tReward: -21.0 \tMean: -20.1 \tBestMean: -19.0\n",
            "Episode: 1083 \tReward: -19.0 \tMean: -20.0 \tBestMean: -19.0\n",
            "Episode: 1084 \tReward: -20.0 \tMean: -20.0 \tBestMean: -19.0\n",
            "Episode: 1085 \tReward: -19.0 \tMean: -19.8 \tBestMean: -19.0\n",
            "Episode: 1086 \tReward: -20.0 \tMean: -19.8 \tBestMean: -19.0\n",
            "Episode: 1087 \tReward: -21.0 \tMean: -20.0 \tBestMean: -19.0\n",
            "Episode: 1088 \tReward: -21.0 \tMean: -20.1 \tBestMean: -19.0\n",
            "Episode: 1089 \tReward: -21.0 \tMean: -20.2 \tBestMean: -19.0\n",
            "Episode: 1090 \tReward: -21.0 \tMean: -20.4 \tBestMean: -19.0\n",
            "Episode: 1091 \tReward: -20.0 \tMean: -20.3 \tBestMean: -19.0\n",
            "Episode: 1092 \tReward: -21.0 \tMean: -20.3 \tBestMean: -19.0\n",
            "Episode: 1093 \tReward: -20.0 \tMean: -20.4 \tBestMean: -19.0\n",
            "Episode: 1094 \tReward: -20.0 \tMean: -20.4 \tBestMean: -19.0\n",
            "Episode: 1095 \tReward: -20.0 \tMean: -20.5 \tBestMean: -19.0\n",
            "Episode: 1096 \tReward: -20.0 \tMean: -20.5 \tBestMean: -19.0\n",
            "Episode: 1097 \tReward: -19.0 \tMean: -20.3 \tBestMean: -19.0\n",
            "Episode: 1098 \tReward: -19.0 \tMean: -20.1 \tBestMean: -19.0\n",
            "Episode: 1099 \tReward: -20.0 \tMean: -20.0 \tBestMean: -19.0\n",
            "Episode: 1100 \tReward: -21.0 \tMean: -20.0 \tBestMean: -19.0\n",
            "Episode: 1101 \tReward: -19.0 \tMean: -19.9 \tBestMean: -19.0\n",
            "Episode: 1102 \tReward: -20.0 \tMean: -19.8 \tBestMean: -19.0\n",
            "Episode: 1103 \tReward: -21.0 \tMean: -19.9 \tBestMean: -19.0\n",
            "Episode: 1104 \tReward: -21.0 \tMean: -20.0 \tBestMean: -19.0\n",
            "Episode: 1105 \tReward: -21.0 \tMean: -20.1 \tBestMean: -19.0\n",
            "Episode: 1106 \tReward: -20.0 \tMean: -20.1 \tBestMean: -19.0\n",
            "Episode: 1107 \tReward: -21.0 \tMean: -20.3 \tBestMean: -19.0\n",
            "Episode: 1108 \tReward: -19.0 \tMean: -20.3 \tBestMean: -19.0\n",
            "Episode: 1109 \tReward: -21.0 \tMean: -20.4 \tBestMean: -19.0\n",
            "Episode: 1110 \tReward: -21.0 \tMean: -20.4 \tBestMean: -19.0\n",
            "Episode: 1111 \tReward: -21.0 \tMean: -20.6 \tBestMean: -19.0\n",
            "Episode: 1112 \tReward: -21.0 \tMean: -20.7 \tBestMean: -19.0\n",
            "Episode: 1113 \tReward: -20.0 \tMean: -20.6 \tBestMean: -19.0\n",
            "Episode: 1114 \tReward: -18.0 \tMean: -20.3 \tBestMean: -19.0\n",
            "Episode: 1115 \tReward: -21.0 \tMean: -20.3 \tBestMean: -19.0\n",
            "Episode: 1116 \tReward: -21.0 \tMean: -20.4 \tBestMean: -19.0\n",
            "Episode: 1117 \tReward: -20.0 \tMean: -20.3 \tBestMean: -19.0\n",
            "Episode: 1118 \tReward: -21.0 \tMean: -20.5 \tBestMean: -19.0\n",
            "Episode: 1119 \tReward: -21.0 \tMean: -20.5 \tBestMean: -19.0\n",
            "Episode: 1120 \tReward: -21.0 \tMean: -20.5 \tBestMean: -19.0\n",
            "Episode: 1121 \tReward: -21.0 \tMean: -20.5 \tBestMean: -19.0\n",
            "Episode: 1122 \tReward: -20.0 \tMean: -20.4 \tBestMean: -19.0\n",
            "Episode: 1123 \tReward: -19.0 \tMean: -20.3 \tBestMean: -19.0\n",
            "Episode: 1124 \tReward: -21.0 \tMean: -20.6 \tBestMean: -19.0\n",
            "Episode: 1125 \tReward: -20.0 \tMean: -20.5 \tBestMean: -19.0\n",
            "Episode: 1126 \tReward: -21.0 \tMean: -20.5 \tBestMean: -19.0\n",
            "Episode: 1127 \tReward: -21.0 \tMean: -20.6 \tBestMean: -19.0\n",
            "Episode: 1128 \tReward: -20.0 \tMean: -20.5 \tBestMean: -19.0\n",
            "Episode: 1129 \tReward: -20.0 \tMean: -20.4 \tBestMean: -19.0\n",
            "Episode: 1130 \tReward: -19.0 \tMean: -20.2 \tBestMean: -19.0\n",
            "Episode: 1131 \tReward: -20.0 \tMean: -20.1 \tBestMean: -19.0\n",
            "Episode: 1132 \tReward: -19.0 \tMean: -20.0 \tBestMean: -19.0\n",
            "Episode: 1133 \tReward: -21.0 \tMean: -20.2 \tBestMean: -19.0\n",
            "Episode: 1134 \tReward: -20.0 \tMean: -20.1 \tBestMean: -19.0\n",
            "Episode: 1135 \tReward: -20.0 \tMean: -20.1 \tBestMean: -19.0\n",
            "Episode: 1136 \tReward: -21.0 \tMean: -20.1 \tBestMean: -19.0\n",
            "Episode: 1137 \tReward: -20.0 \tMean: -20.0 \tBestMean: -19.0\n",
            "Episode: 1138 \tReward: -20.0 \tMean: -20.0 \tBestMean: -19.0\n",
            "Episode: 1139 \tReward: -21.0 \tMean: -20.1 \tBestMean: -19.0\n",
            "Episode: 1140 \tReward: -21.0 \tMean: -20.3 \tBestMean: -19.0\n",
            "Episode: 1141 \tReward: -20.0 \tMean: -20.3 \tBestMean: -19.0\n",
            "Episode: 1142 \tReward: -18.0 \tMean: -20.2 \tBestMean: -19.0\n",
            "Episode: 1143 \tReward: -21.0 \tMean: -20.2 \tBestMean: -19.0\n",
            "Episode: 1144 \tReward: -21.0 \tMean: -20.3 \tBestMean: -19.0\n",
            "Episode: 1145 \tReward: -21.0 \tMean: -20.4 \tBestMean: -19.0\n",
            "Episode: 1146 \tReward: -19.0 \tMean: -20.2 \tBestMean: -19.0\n",
            "Episode: 1147 \tReward: -21.0 \tMean: -20.3 \tBestMean: -19.0\n",
            "Episode: 1148 \tReward: -20.0 \tMean: -20.3 \tBestMean: -19.0\n",
            "Episode: 1149 \tReward: -20.0 \tMean: -20.2 \tBestMean: -19.0\n",
            "Episode: 1150 \tReward: -20.0 \tMean: -20.1 \tBestMean: -19.0\n",
            "Episode: 1151 \tReward: -21.0 \tMean: -20.2 \tBestMean: -19.0\n",
            "Episode: 1152 \tReward: -21.0 \tMean: -20.5 \tBestMean: -19.0\n",
            "Episode: 1153 \tReward: -20.0 \tMean: -20.4 \tBestMean: -19.0\n",
            "Episode: 1154 \tReward: -21.0 \tMean: -20.4 \tBestMean: -19.0\n",
            "Episode: 1155 \tReward: -21.0 \tMean: -20.4 \tBestMean: -19.0\n",
            "Episode: 1156 \tReward: -19.0 \tMean: -20.4 \tBestMean: -19.0\n",
            "Episode: 1157 \tReward: -20.0 \tMean: -20.3 \tBestMean: -19.0\n",
            "Episode: 1158 \tReward: -21.0 \tMean: -20.4 \tBestMean: -19.0\n",
            "Episode: 1159 \tReward: -21.0 \tMean: -20.5 \tBestMean: -19.0\n",
            "Episode: 1160 \tReward: -20.0 \tMean: -20.5 \tBestMean: -19.0\n",
            "Episode: 1161 \tReward: -19.0 \tMean: -20.3 \tBestMean: -19.0\n",
            "Episode: 1162 \tReward: -21.0 \tMean: -20.3 \tBestMean: -19.0\n",
            "Episode: 1163 \tReward: -20.0 \tMean: -20.3 \tBestMean: -19.0\n",
            "Episode: 1164 \tReward: -21.0 \tMean: -20.3 \tBestMean: -19.0\n",
            "Episode: 1165 \tReward: -19.0 \tMean: -20.1 \tBestMean: -19.0\n",
            "Episode: 1166 \tReward: -20.0 \tMean: -20.2 \tBestMean: -19.0\n",
            "Episode: 1167 \tReward: -20.0 \tMean: -20.2 \tBestMean: -19.0\n",
            "Episode: 1168 \tReward: -21.0 \tMean: -20.2 \tBestMean: -19.0\n",
            "Episode: 1169 \tReward: -20.0 \tMean: -20.1 \tBestMean: -19.0\n",
            "Episode: 1170 \tReward: -19.0 \tMean: -20.0 \tBestMean: -19.0\n",
            "Episode: 1171 \tReward: -19.0 \tMean: -20.0 \tBestMean: -19.0\n",
            "Episode: 1172 \tReward: -18.0 \tMean: -19.7 \tBestMean: -19.0\n",
            "Episode: 1173 \tReward: -19.0 \tMean: -19.6 \tBestMean: -19.0\n",
            "Episode: 1174 \tReward: -21.0 \tMean: -19.6 \tBestMean: -19.0\n",
            "Episode: 1175 \tReward: -19.0 \tMean: -19.6 \tBestMean: -19.0\n",
            "Episode: 1176 \tReward: -20.0 \tMean: -19.6 \tBestMean: -19.0\n",
            "Episode: 1177 \tReward: -19.0 \tMean: -19.5 \tBestMean: -19.0\n",
            "Episode: 1178 \tReward: -18.0 \tMean: -19.2 \tBestMean: -19.0\n",
            "Episode: 1179 \tReward: -20.0 \tMean: -19.2 \tBestMean: -19.0\n",
            "Episode: 1180 \tReward: -21.0 \tMean: -19.4 \tBestMean: -19.0\n",
            "Episode: 1181 \tReward: -21.0 \tMean: -19.6 \tBestMean: -19.0\n",
            "Episode: 1182 \tReward: -20.0 \tMean: -19.8 \tBestMean: -19.0\n",
            "Episode: 1183 \tReward: -20.0 \tMean: -19.9 \tBestMean: -19.0\n",
            "Episode: 1184 \tReward: -20.0 \tMean: -19.8 \tBestMean: -19.0\n",
            "Episode: 1185 \tReward: -21.0 \tMean: -20.0 \tBestMean: -19.0\n",
            "Episode: 1186 \tReward: -20.0 \tMean: -20.0 \tBestMean: -19.0\n",
            "Episode: 1187 \tReward: -19.0 \tMean: -20.0 \tBestMean: -19.0\n",
            "Episode: 1188 \tReward: -19.0 \tMean: -20.1 \tBestMean: -19.0\n",
            "Episode: 1189 \tReward: -20.0 \tMean: -20.1 \tBestMean: -19.0\n",
            "Episode: 1190 \tReward: -21.0 \tMean: -20.1 \tBestMean: -19.0\n",
            "Episode: 1191 \tReward: -21.0 \tMean: -20.1 \tBestMean: -19.0\n",
            "Episode: 1192 \tReward: -20.0 \tMean: -20.1 \tBestMean: -19.0\n",
            "Episode: 1193 \tReward: -21.0 \tMean: -20.2 \tBestMean: -19.0\n",
            "Episode: 1194 \tReward: -20.0 \tMean: -20.2 \tBestMean: -19.0\n",
            "Episode: 1195 \tReward: -20.0 \tMean: -20.1 \tBestMean: -19.0\n",
            "Episode: 1196 \tReward: -19.0 \tMean: -20.0 \tBestMean: -19.0\n",
            "Episode: 1197 \tReward: -21.0 \tMean: -20.2 \tBestMean: -19.0\n",
            "Episode: 1198 \tReward: -20.0 \tMean: -20.3 \tBestMean: -19.0\n",
            "Episode: 1199 \tReward: -21.0 \tMean: -20.4 \tBestMean: -19.0\n",
            "Episode: 1200 \tReward: -20.0 \tMean: -20.3 \tBestMean: -19.0\n",
            "Episode: 1201 \tReward: -19.0 \tMean: -20.1 \tBestMean: -19.0\n",
            "Episode: 1202 \tReward: -18.0 \tMean: -19.9 \tBestMean: -19.0\n",
            "Episode: 1203 \tReward: -17.0 \tMean: -19.5 \tBestMean: -19.0\n",
            "Episode: 1204 \tReward: -20.0 \tMean: -19.5 \tBestMean: -19.0\n",
            "Episode: 1205 \tReward: -21.0 \tMean: -19.6 \tBestMean: -19.0\n",
            "Episode: 1206 \tReward: -21.0 \tMean: -19.8 \tBestMean: -19.0\n",
            "Episode: 1207 \tReward: -20.0 \tMean: -19.7 \tBestMean: -19.0\n",
            "Episode: 1208 \tReward: -21.0 \tMean: -19.8 \tBestMean: -19.0\n",
            "Episode: 1209 \tReward: -21.0 \tMean: -19.8 \tBestMean: -19.0\n",
            "Episode: 1210 \tReward: -21.0 \tMean: -19.9 \tBestMean: -19.0\n",
            "Episode: 1211 \tReward: -20.0 \tMean: -20.0 \tBestMean: -19.0\n",
            "Episode: 1212 \tReward: -20.0 \tMean: -20.2 \tBestMean: -19.0\n",
            "Episode: 1213 \tReward: -19.0 \tMean: -20.4 \tBestMean: -19.0\n",
            "Episode: 1214 \tReward: -18.0 \tMean: -20.2 \tBestMean: -19.0\n",
            "Episode: 1215 \tReward: -21.0 \tMean: -20.2 \tBestMean: -19.0\n",
            "Episode: 1216 \tReward: -18.0 \tMean: -19.9 \tBestMean: -19.0\n",
            "Episode: 1217 \tReward: -21.0 \tMean: -20.0 \tBestMean: -19.0\n",
            "Episode: 1218 \tReward: -20.0 \tMean: -19.9 \tBestMean: -19.0\n",
            "Episode: 1219 \tReward: -21.0 \tMean: -19.9 \tBestMean: -19.0\n",
            "Episode: 1220 \tReward: -21.0 \tMean: -19.9 \tBestMean: -19.0\n",
            "Episode: 1221 \tReward: -21.0 \tMean: -20.0 \tBestMean: -19.0\n",
            "Episode: 1222 \tReward: -18.0 \tMean: -19.8 \tBestMean: -19.0\n",
            "Episode: 1223 \tReward: -20.0 \tMean: -19.9 \tBestMean: -19.0\n",
            "Episode: 1224 \tReward: -19.0 \tMean: -20.0 \tBestMean: -19.0\n",
            "Episode: 1225 \tReward: -19.0 \tMean: -19.8 \tBestMean: -19.0\n",
            "Episode: 1226 \tReward: -20.0 \tMean: -20.0 \tBestMean: -19.0\n",
            "Episode: 1227 \tReward: -20.0 \tMean: -19.9 \tBestMean: -19.0\n",
            "Episode: 1228 \tReward: -20.0 \tMean: -19.9 \tBestMean: -19.0\n",
            "Episode: 1229 \tReward: -20.0 \tMean: -19.8 \tBestMean: -19.0\n",
            "Episode: 1230 \tReward: -20.0 \tMean: -19.7 \tBestMean: -19.0\n",
            "Episode: 1231 \tReward: -20.0 \tMean: -19.6 \tBestMean: -19.0\n",
            "Episode: 1232 \tReward: -20.0 \tMean: -19.8 \tBestMean: -19.0\n",
            "Episode: 1233 \tReward: -20.0 \tMean: -19.8 \tBestMean: -19.0\n",
            "Episode: 1234 \tReward: -21.0 \tMean: -20.0 \tBestMean: -19.0\n",
            "Episode: 1235 \tReward: -21.0 \tMean: -20.2 \tBestMean: -19.0\n",
            "Episode: 1236 \tReward: -20.0 \tMean: -20.2 \tBestMean: -19.0\n",
            "Episode: 1237 \tReward: -19.0 \tMean: -20.1 \tBestMean: -19.0\n",
            "Episode: 1238 \tReward: -20.0 \tMean: -20.1 \tBestMean: -19.0\n",
            "Episode: 1239 \tReward: -21.0 \tMean: -20.2 \tBestMean: -19.0\n",
            "Episode: 1240 \tReward: -21.0 \tMean: -20.3 \tBestMean: -19.0\n",
            "Episode: 1241 \tReward: -20.0 \tMean: -20.3 \tBestMean: -19.0\n",
            "Episode: 1242 \tReward: -21.0 \tMean: -20.4 \tBestMean: -19.0\n",
            "Episode: 1243 \tReward: -21.0 \tMean: -20.5 \tBestMean: -19.0\n",
            "Episode: 1244 \tReward: -20.0 \tMean: -20.4 \tBestMean: -19.0\n",
            "Episode: 1245 \tReward: -20.0 \tMean: -20.3 \tBestMean: -19.0\n",
            "Episode: 1246 \tReward: -19.0 \tMean: -20.2 \tBestMean: -19.0\n",
            "Episode: 1247 \tReward: -21.0 \tMean: -20.4 \tBestMean: -19.0\n",
            "Episode: 1248 \tReward: -21.0 \tMean: -20.5 \tBestMean: -19.0\n",
            "Episode: 1249 \tReward: -20.0 \tMean: -20.4 \tBestMean: -19.0\n",
            "Episode: 1250 \tReward: -21.0 \tMean: -20.4 \tBestMean: -19.0\n",
            "Episode: 1251 \tReward: -21.0 \tMean: -20.5 \tBestMean: -19.0\n",
            "Episode: 1252 \tReward: -21.0 \tMean: -20.5 \tBestMean: -19.0\n",
            "Episode: 1253 \tReward: -19.0 \tMean: -20.3 \tBestMean: -19.0\n",
            "Episode: 1254 \tReward: -20.0 \tMean: -20.3 \tBestMean: -19.0\n",
            "Episode: 1255 \tReward: -20.0 \tMean: -20.3 \tBestMean: -19.0\n",
            "Episode: 1256 \tReward: -20.0 \tMean: -20.4 \tBestMean: -19.0\n",
            "Episode: 1257 \tReward: -21.0 \tMean: -20.4 \tBestMean: -19.0\n",
            "Episode: 1258 \tReward: -20.0 \tMean: -20.3 \tBestMean: -19.0\n",
            "Episode: 1259 \tReward: -21.0 \tMean: -20.4 \tBestMean: -19.0\n",
            "Episode: 1260 \tReward: -20.0 \tMean: -20.3 \tBestMean: -19.0\n",
            "Episode: 1261 \tReward: -20.0 \tMean: -20.2 \tBestMean: -19.0\n",
            "Episode: 1262 \tReward: -19.0 \tMean: -20.0 \tBestMean: -19.0\n",
            "Episode: 1263 \tReward: -20.0 \tMean: -20.1 \tBestMean: -19.0\n",
            "Episode: 1264 \tReward: -20.0 \tMean: -20.1 \tBestMean: -19.0\n",
            "Episode: 1265 \tReward: -20.0 \tMean: -20.1 \tBestMean: -19.0\n",
            "Episode: 1266 \tReward: -21.0 \tMean: -20.2 \tBestMean: -19.0\n",
            "Episode: 1267 \tReward: -21.0 \tMean: -20.2 \tBestMean: -19.0\n",
            "Episode: 1268 \tReward: -19.0 \tMean: -20.1 \tBestMean: -19.0\n",
            "Episode: 1269 \tReward: -20.0 \tMean: -20.0 \tBestMean: -19.0\n",
            "Episode: 1270 \tReward: -21.0 \tMean: -20.1 \tBestMean: -19.0\n",
            "Episode: 1271 \tReward: -21.0 \tMean: -20.2 \tBestMean: -19.0\n",
            "Episode: 1272 \tReward: -19.0 \tMean: -20.2 \tBestMean: -19.0\n",
            "Episode: 1273 \tReward: -21.0 \tMean: -20.3 \tBestMean: -19.0\n",
            "Episode: 1274 \tReward: -20.0 \tMean: -20.3 \tBestMean: -19.0\n",
            "Episode: 1275 \tReward: -21.0 \tMean: -20.4 \tBestMean: -19.0\n",
            "Episode: 1276 \tReward: -20.0 \tMean: -20.3 \tBestMean: -19.0\n",
            "Episode: 1277 \tReward: -20.0 \tMean: -20.2 \tBestMean: -19.0\n",
            "Episode: 1278 \tReward: -21.0 \tMean: -20.4 \tBestMean: -19.0\n",
            "Episode: 1279 \tReward: -20.0 \tMean: -20.4 \tBestMean: -19.0\n",
            "Episode: 1280 \tReward: -21.0 \tMean: -20.4 \tBestMean: -19.0\n",
            "Episode: 1281 \tReward: -21.0 \tMean: -20.4 \tBestMean: -19.0\n",
            "Episode: 1282 \tReward: -20.0 \tMean: -20.5 \tBestMean: -19.0\n",
            "Episode: 1283 \tReward: -21.0 \tMean: -20.5 \tBestMean: -19.0\n",
            "Episode: 1284 \tReward: -21.0 \tMean: -20.6 \tBestMean: -19.0\n",
            "Episode: 1285 \tReward: -20.0 \tMean: -20.5 \tBestMean: -19.0\n",
            "Episode: 1286 \tReward: -21.0 \tMean: -20.6 \tBestMean: -19.0\n",
            "Episode: 1287 \tReward: -20.0 \tMean: -20.6 \tBestMean: -19.0\n",
            "Episode: 1288 \tReward: -19.0 \tMean: -20.4 \tBestMean: -19.0\n",
            "Episode: 1289 \tReward: -20.0 \tMean: -20.4 \tBestMean: -19.0\n",
            "Episode: 1290 \tReward: -20.0 \tMean: -20.3 \tBestMean: -19.0\n",
            "Episode: 1291 \tReward: -20.0 \tMean: -20.2 \tBestMean: -19.0\n",
            "Episode: 1292 \tReward: -21.0 \tMean: -20.3 \tBestMean: -19.0\n",
            "Episode: 1293 \tReward: -20.0 \tMean: -20.2 \tBestMean: -19.0\n",
            "Episode: 1294 \tReward: -21.0 \tMean: -20.2 \tBestMean: -19.0\n",
            "Episode: 1295 \tReward: -21.0 \tMean: -20.3 \tBestMean: -19.0\n",
            "Episode: 1296 \tReward: -21.0 \tMean: -20.3 \tBestMean: -19.0\n",
            "Episode: 1297 \tReward: -20.0 \tMean: -20.3 \tBestMean: -19.0\n",
            "Episode: 1298 \tReward: -21.0 \tMean: -20.5 \tBestMean: -19.0\n",
            "Episode: 1299 \tReward: -21.0 \tMean: -20.6 \tBestMean: -19.0\n",
            "Episode: 1300 \tReward: -21.0 \tMean: -20.7 \tBestMean: -19.0\n",
            "Episode: 1301 \tReward: -21.0 \tMean: -20.8 \tBestMean: -19.0\n",
            "Episode: 1302 \tReward: -20.0 \tMean: -20.7 \tBestMean: -19.0\n",
            "Episode: 1303 \tReward: -21.0 \tMean: -20.8 \tBestMean: -19.0\n",
            "Episode: 1304 \tReward: -20.0 \tMean: -20.7 \tBestMean: -19.0\n",
            "Episode: 1305 \tReward: -21.0 \tMean: -20.7 \tBestMean: -19.0\n",
            "Episode: 1306 \tReward: -21.0 \tMean: -20.7 \tBestMean: -19.0\n",
            "Episode: 1307 \tReward: -21.0 \tMean: -20.8 \tBestMean: -19.0\n",
            "Episode: 1308 \tReward: -20.0 \tMean: -20.7 \tBestMean: -19.0\n",
            "Episode: 1309 \tReward: -20.0 \tMean: -20.6 \tBestMean: -19.0\n",
            "Episode: 1310 \tReward: -20.0 \tMean: -20.5 \tBestMean: -19.0\n",
            "Episode: 1311 \tReward: -19.0 \tMean: -20.3 \tBestMean: -19.0\n",
            "Episode: 1312 \tReward: -19.0 \tMean: -20.2 \tBestMean: -19.0\n",
            "Episode: 1313 \tReward: -19.0 \tMean: -20.0 \tBestMean: -19.0\n",
            "Episode: 1314 \tReward: -21.0 \tMean: -20.1 \tBestMean: -19.0\n",
            "Episode: 1315 \tReward: -21.0 \tMean: -20.1 \tBestMean: -19.0\n",
            "Episode: 1316 \tReward: -21.0 \tMean: -20.1 \tBestMean: -19.0\n",
            "Episode: 1317 \tReward: -21.0 \tMean: -20.1 \tBestMean: -19.0\n",
            "Episode: 1318 \tReward: -19.0 \tMean: -20.0 \tBestMean: -19.0\n",
            "Episode: 1319 \tReward: -21.0 \tMean: -20.1 \tBestMean: -19.0\n",
            "Episode: 1320 \tReward: -20.0 \tMean: -20.1 \tBestMean: -19.0\n",
            "Episode: 1321 \tReward: -20.0 \tMean: -20.2 \tBestMean: -19.0\n",
            "Episode: 1322 \tReward: -20.0 \tMean: -20.3 \tBestMean: -19.0\n",
            "Episode: 1323 \tReward: -20.0 \tMean: -20.4 \tBestMean: -19.0\n",
            "Episode: 1324 \tReward: -21.0 \tMean: -20.4 \tBestMean: -19.0\n",
            "Episode: 1325 \tReward: -20.0 \tMean: -20.3 \tBestMean: -19.0\n",
            "Episode: 1326 \tReward: -20.0 \tMean: -20.2 \tBestMean: -19.0\n",
            "Episode: 1327 \tReward: -21.0 \tMean: -20.2 \tBestMean: -19.0\n",
            "Episode: 1328 \tReward: -19.0 \tMean: -20.2 \tBestMean: -19.0\n",
            "Episode: 1329 \tReward: -18.0 \tMean: -19.9 \tBestMean: -19.0\n",
            "Episode: 1330 \tReward: -20.0 \tMean: -19.9 \tBestMean: -19.0\n",
            "Episode: 1331 \tReward: -20.0 \tMean: -19.9 \tBestMean: -19.0\n",
            "Episode: 1332 \tReward: -21.0 \tMean: -20.0 \tBestMean: -19.0\n",
            "Episode: 1333 \tReward: -21.0 \tMean: -20.1 \tBestMean: -19.0\n",
            "Episode: 1334 \tReward: -21.0 \tMean: -20.1 \tBestMean: -19.0\n",
            "Episode: 1335 \tReward: -20.0 \tMean: -20.1 \tBestMean: -19.0\n",
            "Episode: 1336 \tReward: -19.0 \tMean: -20.0 \tBestMean: -19.0\n",
            "Episode: 1337 \tReward: -20.0 \tMean: -19.9 \tBestMean: -19.0\n",
            "Episode: 1338 \tReward: -20.0 \tMean: -20.0 \tBestMean: -19.0\n",
            "Episode: 1339 \tReward: -21.0 \tMean: -20.3 \tBestMean: -19.0\n",
            "Episode: 1340 \tReward: -21.0 \tMean: -20.4 \tBestMean: -19.0\n",
            "Episode: 1341 \tReward: -20.0 \tMean: -20.4 \tBestMean: -19.0\n",
            "Episode: 1342 \tReward: -19.0 \tMean: -20.2 \tBestMean: -19.0\n",
            "Episode: 1343 \tReward: -20.0 \tMean: -20.1 \tBestMean: -19.0\n",
            "Episode: 1344 \tReward: -21.0 \tMean: -20.1 \tBestMean: -19.0\n",
            "Episode: 1345 \tReward: -21.0 \tMean: -20.2 \tBestMean: -19.0\n",
            "Episode: 1346 \tReward: -20.0 \tMean: -20.3 \tBestMean: -19.0\n",
            "Episode: 1347 \tReward: -21.0 \tMean: -20.4 \tBestMean: -19.0\n",
            "Episode: 1348 \tReward: -21.0 \tMean: -20.5 \tBestMean: -19.0\n",
            "Episode: 1349 \tReward: -20.0 \tMean: -20.4 \tBestMean: -19.0\n",
            "Episode: 1350 \tReward: -21.0 \tMean: -20.4 \tBestMean: -19.0\n",
            "Episode: 1351 \tReward: -20.0 \tMean: -20.4 \tBestMean: -19.0\n",
            "Episode: 1352 \tReward: -20.0 \tMean: -20.5 \tBestMean: -19.0\n",
            "Episode: 1353 \tReward: -21.0 \tMean: -20.6 \tBestMean: -19.0\n",
            "Episode: 1354 \tReward: -20.0 \tMean: -20.5 \tBestMean: -19.0\n",
            "Episode: 1355 \tReward: -20.0 \tMean: -20.4 \tBestMean: -19.0\n",
            "Episode: 1356 \tReward: -21.0 \tMean: -20.5 \tBestMean: -19.0\n",
            "Episode: 1357 \tReward: -20.0 \tMean: -20.4 \tBestMean: -19.0\n",
            "Episode: 1358 \tReward: -20.0 \tMean: -20.3 \tBestMean: -19.0\n",
            "Episode: 1359 \tReward: -21.0 \tMean: -20.4 \tBestMean: -19.0\n",
            "Episode: 1360 \tReward: -20.0 \tMean: -20.3 \tBestMean: -19.0\n",
            "Episode: 1361 \tReward: -21.0 \tMean: -20.4 \tBestMean: -19.0\n",
            "Episode: 1362 \tReward: -19.0 \tMean: -20.3 \tBestMean: -19.0\n",
            "Episode: 1363 \tReward: -20.0 \tMean: -20.2 \tBestMean: -19.0\n",
            "Episode: 1364 \tReward: -21.0 \tMean: -20.3 \tBestMean: -19.0\n",
            "Episode: 1365 \tReward: -20.0 \tMean: -20.3 \tBestMean: -19.0\n",
            "Episode: 1366 \tReward: -20.0 \tMean: -20.2 \tBestMean: -19.0\n",
            "Episode: 1367 \tReward: -20.0 \tMean: -20.2 \tBestMean: -19.0\n",
            "Episode: 1368 \tReward: -21.0 \tMean: -20.3 \tBestMean: -19.0\n",
            "Episode: 1369 \tReward: -21.0 \tMean: -20.3 \tBestMean: -19.0\n",
            "Episode: 1370 \tReward: -20.0 \tMean: -20.3 \tBestMean: -19.0\n",
            "Episode: 1371 \tReward: -21.0 \tMean: -20.3 \tBestMean: -19.0\n",
            "Episode: 1372 \tReward: -21.0 \tMean: -20.5 \tBestMean: -19.0\n",
            "Episode: 1373 \tReward: -21.0 \tMean: -20.6 \tBestMean: -19.0\n",
            "Episode: 1374 \tReward: -19.0 \tMean: -20.4 \tBestMean: -19.0\n",
            "Episode: 1375 \tReward: -20.0 \tMean: -20.4 \tBestMean: -19.0\n",
            "Episode: 1376 \tReward: -21.0 \tMean: -20.5 \tBestMean: -19.0\n",
            "Episode: 1377 \tReward: -19.0 \tMean: -20.4 \tBestMean: -19.0\n",
            "Episode: 1378 \tReward: -21.0 \tMean: -20.4 \tBestMean: -19.0\n",
            "Episode: 1379 \tReward: -21.0 \tMean: -20.4 \tBestMean: -19.0\n",
            "Episode: 1380 \tReward: -21.0 \tMean: -20.5 \tBestMean: -19.0\n",
            "Episode: 1381 \tReward: -21.0 \tMean: -20.5 \tBestMean: -19.0\n",
            "Episode: 1382 \tReward: -21.0 \tMean: -20.5 \tBestMean: -19.0\n",
            "Episode: 1383 \tReward: -19.0 \tMean: -20.3 \tBestMean: -19.0\n",
            "Episode: 1384 \tReward: -21.0 \tMean: -20.5 \tBestMean: -19.0\n",
            "Episode: 1385 \tReward: -19.0 \tMean: -20.4 \tBestMean: -19.0\n",
            "Episode: 1386 \tReward: -20.0 \tMean: -20.3 \tBestMean: -19.0\n",
            "Episode: 1387 \tReward: -20.0 \tMean: -20.4 \tBestMean: -19.0\n",
            "Episode: 1388 \tReward: -20.0 \tMean: -20.3 \tBestMean: -19.0\n",
            "Episode: 1389 \tReward: -21.0 \tMean: -20.3 \tBestMean: -19.0\n",
            "Episode: 1390 \tReward: -20.0 \tMean: -20.2 \tBestMean: -19.0\n",
            "Episode: 1391 \tReward: -20.0 \tMean: -20.1 \tBestMean: -19.0\n",
            "Episode: 1392 \tReward: -21.0 \tMean: -20.1 \tBestMean: -19.0\n",
            "Episode: 1393 \tReward: -20.0 \tMean: -20.2 \tBestMean: -19.0\n",
            "Episode: 1394 \tReward: -21.0 \tMean: -20.2 \tBestMean: -19.0\n",
            "Episode: 1395 \tReward: -21.0 \tMean: -20.4 \tBestMean: -19.0\n",
            "Episode: 1396 \tReward: -19.0 \tMean: -20.3 \tBestMean: -19.0\n",
            "Episode: 1397 \tReward: -21.0 \tMean: -20.4 \tBestMean: -19.0\n",
            "Episode: 1398 \tReward: -20.0 \tMean: -20.4 \tBestMean: -19.0\n",
            "Episode: 1399 \tReward: -19.0 \tMean: -20.2 \tBestMean: -19.0\n",
            "Episode: 1400 \tReward: -19.0 \tMean: -20.1 \tBestMean: -19.0\n",
            "Episode: 1401 \tReward: -20.0 \tMean: -20.1 \tBestMean: -19.0\n",
            "Episode: 1402 \tReward: -20.0 \tMean: -20.0 \tBestMean: -19.0\n",
            "Episode: 1403 \tReward: -21.0 \tMean: -20.1 \tBestMean: -19.0\n",
            "Episode: 1404 \tReward: -20.0 \tMean: -20.0 \tBestMean: -19.0\n",
            "Episode: 1405 \tReward: -18.0 \tMean: -19.7 \tBestMean: -19.0\n",
            "Episode: 1406 \tReward: -18.0 \tMean: -19.6 \tBestMean: -19.0\n",
            "Episode: 1407 \tReward: -20.0 \tMean: -19.5 \tBestMean: -19.0\n",
            "Episode: 1408 \tReward: -20.0 \tMean: -19.5 \tBestMean: -19.0\n",
            "Episode: 1409 \tReward: -21.0 \tMean: -19.7 \tBestMean: -19.0\n",
            "Episode: 1410 \tReward: -21.0 \tMean: -19.9 \tBestMean: -19.0\n",
            "Episode: 1411 \tReward: -21.0 \tMean: -20.0 \tBestMean: -19.0\n",
            "Episode: 1412 \tReward: -21.0 \tMean: -20.1 \tBestMean: -19.0\n",
            "Episode: 1413 \tReward: -21.0 \tMean: -20.1 \tBestMean: -19.0\n",
            "Episode: 1414 \tReward: -21.0 \tMean: -20.2 \tBestMean: -19.0\n",
            "Episode: 1415 \tReward: -20.0 \tMean: -20.4 \tBestMean: -19.0\n",
            "Episode: 1416 \tReward: -21.0 \tMean: -20.7 \tBestMean: -19.0\n",
            "Episode: 1417 \tReward: -20.0 \tMean: -20.7 \tBestMean: -19.0\n",
            "Episode: 1418 \tReward: -21.0 \tMean: -20.8 \tBestMean: -19.0\n",
            "Episode: 1419 \tReward: -20.0 \tMean: -20.7 \tBestMean: -19.0\n",
            "Episode: 1420 \tReward: -20.0 \tMean: -20.6 \tBestMean: -19.0\n",
            "Episode: 1421 \tReward: -21.0 \tMean: -20.6 \tBestMean: -19.0\n",
            "Episode: 1422 \tReward: -20.0 \tMean: -20.5 \tBestMean: -19.0\n",
            "Episode: 1423 \tReward: -20.0 \tMean: -20.4 \tBestMean: -19.0\n",
            "Episode: 1424 \tReward: -21.0 \tMean: -20.4 \tBestMean: -19.0\n",
            "Episode: 1425 \tReward: -17.0 \tMean: -20.1 \tBestMean: -19.0\n",
            "Episode: 1426 \tReward: -20.0 \tMean: -20.0 \tBestMean: -19.0\n",
            "Episode: 1427 \tReward: -20.0 \tMean: -20.0 \tBestMean: -19.0\n",
            "Episode: 1428 \tReward: -20.0 \tMean: -19.9 \tBestMean: -19.0\n",
            "Episode: 1429 \tReward: -20.0 \tMean: -19.9 \tBestMean: -19.0\n",
            "Episode: 1430 \tReward: -20.0 \tMean: -19.9 \tBestMean: -19.0\n",
            "Episode: 1431 \tReward: -19.0 \tMean: -19.7 \tBestMean: -19.0\n",
            "Episode: 1432 \tReward: -20.0 \tMean: -19.7 \tBestMean: -19.0\n",
            "Episode: 1433 \tReward: -21.0 \tMean: -19.8 \tBestMean: -19.0\n",
            "Episode: 1434 \tReward: -20.0 \tMean: -19.7 \tBestMean: -19.0\n",
            "Episode: 1435 \tReward: -21.0 \tMean: -20.1 \tBestMean: -19.0\n",
            "Episode: 1436 \tReward: -19.0 \tMean: -20.0 \tBestMean: -19.0\n",
            "Episode: 1437 \tReward: -21.0 \tMean: -20.1 \tBestMean: -19.0\n",
            "Episode: 1438 \tReward: -20.0 \tMean: -20.1 \tBestMean: -19.0\n",
            "Episode: 1439 \tReward: -18.0 \tMean: -19.9 \tBestMean: -19.0\n",
            "Episode: 1440 \tReward: -20.0 \tMean: -19.9 \tBestMean: -19.0\n",
            "Episode: 1441 \tReward: -21.0 \tMean: -20.1 \tBestMean: -19.0\n",
            "Episode: 1442 \tReward: -20.0 \tMean: -20.1 \tBestMean: -19.0\n",
            "Episode: 1443 \tReward: -21.0 \tMean: -20.1 \tBestMean: -19.0\n",
            "Episode: 1444 \tReward: -21.0 \tMean: -20.2 \tBestMean: -19.0\n",
            "Episode: 1445 \tReward: -21.0 \tMean: -20.2 \tBestMean: -19.0\n",
            "Episode: 1446 \tReward: -21.0 \tMean: -20.4 \tBestMean: -19.0\n",
            "Episode: 1447 \tReward: -18.0 \tMean: -20.1 \tBestMean: -19.0\n",
            "Episode: 1448 \tReward: -19.0 \tMean: -20.0 \tBestMean: -19.0\n",
            "Episode: 1449 \tReward: -19.0 \tMean: -20.1 \tBestMean: -19.0\n",
            "Episode: 1450 \tReward: -21.0 \tMean: -20.2 \tBestMean: -19.0\n",
            "Episode: 1451 \tReward: -20.0 \tMean: -20.1 \tBestMean: -19.0\n",
            "Episode: 1452 \tReward: -19.0 \tMean: -20.0 \tBestMean: -19.0\n",
            "Episode: 1453 \tReward: -21.0 \tMean: -20.0 \tBestMean: -19.0\n",
            "Episode: 1454 \tReward: -21.0 \tMean: -20.0 \tBestMean: -19.0\n",
            "Episode: 1455 \tReward: -19.0 \tMean: -19.8 \tBestMean: -19.0\n",
            "Episode: 1456 \tReward: -20.0 \tMean: -19.7 \tBestMean: -19.0\n",
            "Episode: 1457 \tReward: -21.0 \tMean: -20.0 \tBestMean: -19.0\n",
            "Episode: 1458 \tReward: -21.0 \tMean: -20.2 \tBestMean: -19.0\n",
            "Episode: 1459 \tReward: -20.0 \tMean: -20.3 \tBestMean: -19.0\n",
            "Episode: 1460 \tReward: -20.0 \tMean: -20.2 \tBestMean: -19.0\n",
            "Episode: 1461 \tReward: -19.0 \tMean: -20.1 \tBestMean: -19.0\n",
            "Episode: 1462 \tReward: -21.0 \tMean: -20.3 \tBestMean: -19.0\n",
            "Episode: 1463 \tReward: -21.0 \tMean: -20.3 \tBestMean: -19.0\n",
            "Episode: 1464 \tReward: -20.0 \tMean: -20.2 \tBestMean: -19.0\n",
            "Episode: 1465 \tReward: -21.0 \tMean: -20.4 \tBestMean: -19.0\n",
            "Episode: 1466 \tReward: -20.0 \tMean: -20.4 \tBestMean: -19.0\n",
            "Episode: 1467 \tReward: -21.0 \tMean: -20.4 \tBestMean: -19.0\n",
            "Episode: 1468 \tReward: -21.0 \tMean: -20.4 \tBestMean: -19.0\n",
            "Episode: 1469 \tReward: -21.0 \tMean: -20.5 \tBestMean: -19.0\n",
            "Episode: 1470 \tReward: -20.0 \tMean: -20.5 \tBestMean: -19.0\n",
            "Episode: 1471 \tReward: -20.0 \tMean: -20.6 \tBestMean: -19.0\n",
            "Episode: 1472 \tReward: -21.0 \tMean: -20.6 \tBestMean: -19.0\n",
            "Episode: 1473 \tReward: -21.0 \tMean: -20.6 \tBestMean: -19.0\n",
            "Episode: 1474 \tReward: -21.0 \tMean: -20.7 \tBestMean: -19.0\n",
            "Episode: 1475 \tReward: -20.0 \tMean: -20.6 \tBestMean: -19.0\n",
            "Episode: 1476 \tReward: -18.0 \tMean: -20.4 \tBestMean: -19.0\n",
            "Episode: 1477 \tReward: -20.0 \tMean: -20.3 \tBestMean: -19.0\n",
            "Episode: 1478 \tReward: -21.0 \tMean: -20.3 \tBestMean: -19.0\n",
            "Episode: 1479 \tReward: -20.0 \tMean: -20.2 \tBestMean: -19.0\n",
            "Episode: 1480 \tReward: -21.0 \tMean: -20.3 \tBestMean: -19.0\n",
            "Episode: 1481 \tReward: -19.0 \tMean: -20.2 \tBestMean: -19.0\n",
            "Episode: 1482 \tReward: -20.0 \tMean: -20.1 \tBestMean: -19.0\n",
            "Episode: 1483 \tReward: -21.0 \tMean: -20.1 \tBestMean: -19.0\n",
            "Episode: 1484 \tReward: -20.0 \tMean: -20.0 \tBestMean: -19.0\n",
            "Episode: 1485 \tReward: -21.0 \tMean: -20.1 \tBestMean: -19.0\n",
            "Episode: 1486 \tReward: -20.0 \tMean: -20.3 \tBestMean: -19.0\n",
            "Episode: 1487 \tReward: -20.0 \tMean: -20.3 \tBestMean: -19.0\n",
            "Episode: 1488 \tReward: -19.0 \tMean: -20.1 \tBestMean: -19.0\n",
            "Episode: 1489 \tReward: -20.0 \tMean: -20.1 \tBestMean: -19.0\n",
            "Episode: 1490 \tReward: -20.0 \tMean: -20.0 \tBestMean: -19.0\n",
            "Episode: 1491 \tReward: -19.0 \tMean: -20.0 \tBestMean: -19.0\n",
            "Episode: 1492 \tReward: -20.0 \tMean: -20.0 \tBestMean: -19.0\n",
            "Episode: 1493 \tReward: -20.0 \tMean: -19.9 \tBestMean: -19.0\n",
            "Episode: 1494 \tReward: -20.0 \tMean: -19.9 \tBestMean: -19.0\n",
            "Episode: 1495 \tReward: -21.0 \tMean: -19.9 \tBestMean: -19.0\n",
            "Episode: 1496 \tReward: -19.0 \tMean: -19.8 \tBestMean: -19.0\n",
            "Episode: 1497 \tReward: -21.0 \tMean: -19.9 \tBestMean: -19.0\n",
            "Episode: 1498 \tReward: -20.0 \tMean: -20.0 \tBestMean: -19.0\n",
            "Episode: 1499 \tReward: -21.0 \tMean: -20.1 \tBestMean: -19.0\n",
            "Episode: 1500 \tReward: -20.0 \tMean: -20.1 \tBestMean: -19.0\n",
            "Episode: 1501 \tReward: -19.0 \tMean: -20.1 \tBestMean: -19.0\n",
            "Episode: 1502 \tReward: -19.0 \tMean: -20.0 \tBestMean: -19.0\n",
            "Episode: 1503 \tReward: -21.0 \tMean: -20.1 \tBestMean: -19.0\n",
            "Episode: 1504 \tReward: -20.0 \tMean: -20.1 \tBestMean: -19.0\n",
            "Episode: 1505 \tReward: -19.0 \tMean: -19.9 \tBestMean: -19.0\n",
            "Episode: 1506 \tReward: -20.0 \tMean: -20.0 \tBestMean: -19.0\n",
            "Episode: 1507 \tReward: -21.0 \tMean: -20.0 \tBestMean: -19.0\n",
            "Episode: 1508 \tReward: -21.0 \tMean: -20.1 \tBestMean: -19.0\n",
            "Episode: 1509 \tReward: -21.0 \tMean: -20.1 \tBestMean: -19.0\n",
            "Episode: 1510 \tReward: -20.0 \tMean: -20.1 \tBestMean: -19.0\n",
            "Episode: 1511 \tReward: -21.0 \tMean: -20.3 \tBestMean: -19.0\n",
            "Episode: 1512 \tReward: -21.0 \tMean: -20.5 \tBestMean: -19.0\n",
            "Episode: 1513 \tReward: -21.0 \tMean: -20.5 \tBestMean: -19.0\n",
            "Episode: 1514 \tReward: -21.0 \tMean: -20.6 \tBestMean: -19.0\n",
            "Episode: 1515 \tReward: -20.0 \tMean: -20.7 \tBestMean: -19.0\n",
            "Episode: 1516 \tReward: -20.0 \tMean: -20.7 \tBestMean: -19.0\n",
            "Episode: 1517 \tReward: -21.0 \tMean: -20.7 \tBestMean: -19.0\n",
            "Episode: 1518 \tReward: -21.0 \tMean: -20.7 \tBestMean: -19.0\n",
            "Episode: 1519 \tReward: -21.0 \tMean: -20.7 \tBestMean: -19.0\n",
            "Episode: 1520 \tReward: -20.0 \tMean: -20.7 \tBestMean: -19.0\n",
            "Episode: 1521 \tReward: -21.0 \tMean: -20.7 \tBestMean: -19.0\n",
            "Episode: 1522 \tReward: -20.0 \tMean: -20.6 \tBestMean: -19.0\n",
            "Episode: 1523 \tReward: -21.0 \tMean: -20.6 \tBestMean: -19.0\n",
            "Episode: 1524 \tReward: -21.0 \tMean: -20.6 \tBestMean: -19.0\n",
            "Episode: 1525 \tReward: -21.0 \tMean: -20.7 \tBestMean: -19.0\n",
            "Episode: 1526 \tReward: -21.0 \tMean: -20.8 \tBestMean: -19.0\n",
            "Episode: 1527 \tReward: -20.0 \tMean: -20.7 \tBestMean: -19.0\n",
            "Episode: 1528 \tReward: -21.0 \tMean: -20.7 \tBestMean: -19.0\n",
            "Episode: 1529 \tReward: -20.0 \tMean: -20.6 \tBestMean: -19.0\n",
            "Episode: 1530 \tReward: -20.0 \tMean: -20.6 \tBestMean: -19.0\n",
            "Episode: 1531 \tReward: -20.0 \tMean: -20.5 \tBestMean: -19.0\n",
            "Episode: 1532 \tReward: -21.0 \tMean: -20.6 \tBestMean: -19.0\n",
            "Episode: 1533 \tReward: -20.0 \tMean: -20.5 \tBestMean: -19.0\n",
            "Episode: 1534 \tReward: -21.0 \tMean: -20.5 \tBestMean: -19.0\n",
            "Episode: 1535 \tReward: -19.0 \tMean: -20.3 \tBestMean: -19.0\n",
            "Episode: 1536 \tReward: -20.0 \tMean: -20.2 \tBestMean: -19.0\n",
            "Episode: 1537 \tReward: -20.0 \tMean: -20.2 \tBestMean: -19.0\n",
            "Episode: 1538 \tReward: -20.0 \tMean: -20.1 \tBestMean: -19.0\n",
            "Episode: 1539 \tReward: -21.0 \tMean: -20.2 \tBestMean: -19.0\n",
            "Episode: 1540 \tReward: -19.0 \tMean: -20.1 \tBestMean: -19.0\n",
            "Episode: 1541 \tReward: -20.0 \tMean: -20.1 \tBestMean: -19.0\n",
            "Episode: 1542 \tReward: -19.0 \tMean: -19.9 \tBestMean: -19.0\n",
            "Episode: 1543 \tReward: -21.0 \tMean: -20.0 \tBestMean: -19.0\n",
            "Episode: 1544 \tReward: -21.0 \tMean: -20.0 \tBestMean: -19.0\n",
            "Episode: 1545 \tReward: -21.0 \tMean: -20.2 \tBestMean: -19.0\n",
            "Episode: 1546 \tReward: -20.0 \tMean: -20.2 \tBestMean: -19.0\n",
            "Episode: 1547 \tReward: -21.0 \tMean: -20.3 \tBestMean: -19.0\n",
            "Episode: 1548 \tReward: -21.0 \tMean: -20.4 \tBestMean: -19.0\n",
            "Episode: 1549 \tReward: -21.0 \tMean: -20.4 \tBestMean: -19.0\n",
            "Episode: 1550 \tReward: -21.0 \tMean: -20.6 \tBestMean: -19.0\n",
            "Episode: 1551 \tReward: -21.0 \tMean: -20.7 \tBestMean: -19.0\n",
            "Episode: 1552 \tReward: -21.0 \tMean: -20.9 \tBestMean: -19.0\n",
            "Episode: 1553 \tReward: -20.0 \tMean: -20.8 \tBestMean: -19.0\n",
            "Episode: 1554 \tReward: -19.0 \tMean: -20.6 \tBestMean: -19.0\n",
            "Episode: 1555 \tReward: -21.0 \tMean: -20.6 \tBestMean: -19.0\n",
            "Episode: 1556 \tReward: -21.0 \tMean: -20.7 \tBestMean: -19.0\n",
            "Episode: 1557 \tReward: -21.0 \tMean: -20.7 \tBestMean: -19.0\n",
            "Episode: 1558 \tReward: -21.0 \tMean: -20.7 \tBestMean: -19.0\n",
            "Episode: 1559 \tReward: -19.0 \tMean: -20.5 \tBestMean: -19.0\n",
            "Episode: 1560 \tReward: -19.0 \tMean: -20.3 \tBestMean: -19.0\n",
            "Episode: 1561 \tReward: -21.0 \tMean: -20.3 \tBestMean: -19.0\n",
            "Episode: 1562 \tReward: -20.0 \tMean: -20.2 \tBestMean: -19.0\n",
            "Episode: 1563 \tReward: -21.0 \tMean: -20.3 \tBestMean: -19.0\n",
            "Episode: 1564 \tReward: -21.0 \tMean: -20.5 \tBestMean: -19.0\n",
            "Episode: 1565 \tReward: -21.0 \tMean: -20.5 \tBestMean: -19.0\n",
            "Episode: 1566 \tReward: -19.0 \tMean: -20.3 \tBestMean: -19.0\n",
            "Episode: 1567 \tReward: -20.0 \tMean: -20.2 \tBestMean: -19.0\n",
            "Episode: 1568 \tReward: -21.0 \tMean: -20.2 \tBestMean: -19.0\n",
            "Episode: 1569 \tReward: -21.0 \tMean: -20.4 \tBestMean: -19.0\n",
            "Episode: 1570 \tReward: -19.0 \tMean: -20.4 \tBestMean: -19.0\n",
            "Episode: 1571 \tReward: -21.0 \tMean: -20.4 \tBestMean: -19.0\n",
            "Episode: 1572 \tReward: -20.0 \tMean: -20.4 \tBestMean: -19.0\n",
            "Episode: 1573 \tReward: -21.0 \tMean: -20.4 \tBestMean: -19.0\n",
            "Episode: 1574 \tReward: -21.0 \tMean: -20.4 \tBestMean: -19.0\n",
            "Episode: 1575 \tReward: -18.0 \tMean: -20.1 \tBestMean: -19.0\n",
            "Episode: 1576 \tReward: -21.0 \tMean: -20.3 \tBestMean: -19.0\n",
            "Episode: 1577 \tReward: -21.0 \tMean: -20.4 \tBestMean: -19.0\n",
            "Episode: 1578 \tReward: -21.0 \tMean: -20.4 \tBestMean: -19.0\n",
            "Episode: 1579 \tReward: -20.0 \tMean: -20.3 \tBestMean: -19.0\n",
            "Episode: 1580 \tReward: -21.0 \tMean: -20.5 \tBestMean: -19.0\n",
            "Episode: 1581 \tReward: -21.0 \tMean: -20.5 \tBestMean: -19.0\n",
            "Episode: 1582 \tReward: -18.0 \tMean: -20.3 \tBestMean: -19.0\n",
            "Episode: 1583 \tReward: -20.0 \tMean: -20.2 \tBestMean: -19.0\n",
            "Episode: 1584 \tReward: -21.0 \tMean: -20.2 \tBestMean: -19.0\n",
            "Episode: 1585 \tReward: -19.0 \tMean: -20.3 \tBestMean: -19.0\n",
            "Episode: 1586 \tReward: -21.0 \tMean: -20.3 \tBestMean: -19.0\n",
            "Episode: 1587 \tReward: -20.0 \tMean: -20.2 \tBestMean: -19.0\n",
            "Episode: 1588 \tReward: -20.0 \tMean: -20.1 \tBestMean: -19.0\n",
            "Episode: 1589 \tReward: -21.0 \tMean: -20.2 \tBestMean: -19.0\n",
            "Episode: 1590 \tReward: -20.0 \tMean: -20.1 \tBestMean: -19.0\n",
            "Episode: 1591 \tReward: -20.0 \tMean: -20.0 \tBestMean: -19.0\n",
            "Episode: 1592 \tReward: -21.0 \tMean: -20.3 \tBestMean: -19.0\n",
            "Episode: 1593 \tReward: -19.0 \tMean: -20.2 \tBestMean: -19.0\n",
            "Episode: 1594 \tReward: -21.0 \tMean: -20.2 \tBestMean: -19.0\n",
            "Episode: 1595 \tReward: -19.0 \tMean: -20.2 \tBestMean: -19.0\n",
            "Episode: 1596 \tReward: -20.0 \tMean: -20.1 \tBestMean: -19.0\n",
            "Episode: 1597 \tReward: -20.0 \tMean: -20.1 \tBestMean: -19.0\n",
            "Episode: 1598 \tReward: -19.0 \tMean: -20.0 \tBestMean: -19.0\n",
            "Episode: 1599 \tReward: -20.0 \tMean: -19.9 \tBestMean: -19.0\n",
            "Episode: 1600 \tReward: -21.0 \tMean: -20.0 \tBestMean: -19.0\n",
            "Episode: 1601 \tReward: -21.0 \tMean: -20.1 \tBestMean: -19.0\n",
            "Episode: 1602 \tReward: -20.0 \tMean: -20.0 \tBestMean: -19.0\n",
            "Episode: 1603 \tReward: -20.0 \tMean: -20.1 \tBestMean: -19.0\n",
            "Episode: 1604 \tReward: -20.0 \tMean: -20.0 \tBestMean: -19.0\n",
            "Episode: 1605 \tReward: -20.0 \tMean: -20.1 \tBestMean: -19.0\n",
            "Episode: 1606 \tReward: -19.0 \tMean: -20.0 \tBestMean: -19.0\n",
            "Episode: 1607 \tReward: -21.0 \tMean: -20.1 \tBestMean: -19.0\n",
            "Episode: 1608 \tReward: -20.0 \tMean: -20.2 \tBestMean: -19.0\n",
            "Episode: 1609 \tReward: -21.0 \tMean: -20.3 \tBestMean: -19.0\n",
            "Episode: 1610 \tReward: -20.0 \tMean: -20.2 \tBestMean: -19.0\n",
            "Episode: 1611 \tReward: -21.0 \tMean: -20.2 \tBestMean: -19.0\n",
            "Episode: 1612 \tReward: -21.0 \tMean: -20.3 \tBestMean: -19.0\n",
            "Episode: 1613 \tReward: -20.0 \tMean: -20.3 \tBestMean: -19.0\n",
            "Episode: 1614 \tReward: -18.0 \tMean: -20.1 \tBestMean: -19.0\n",
            "Episode: 1615 \tReward: -21.0 \tMean: -20.2 \tBestMean: -19.0\n",
            "Episode: 1616 \tReward: -21.0 \tMean: -20.4 \tBestMean: -19.0\n",
            "Episode: 1617 \tReward: -20.0 \tMean: -20.3 \tBestMean: -19.0\n",
            "Episode: 1618 \tReward: -19.0 \tMean: -20.2 \tBestMean: -19.0\n",
            "Episode: 1619 \tReward: -19.0 \tMean: -20.0 \tBestMean: -19.0\n",
            "Episode: 1620 \tReward: -21.0 \tMean: -20.1 \tBestMean: -19.0\n",
            "Episode: 1621 \tReward: -21.0 \tMean: -20.1 \tBestMean: -19.0\n",
            "Episode: 1622 \tReward: -21.0 \tMean: -20.1 \tBestMean: -19.0\n",
            "Episode: 1623 \tReward: -20.0 \tMean: -20.1 \tBestMean: -19.0\n",
            "Episode: 1624 \tReward: -19.0 \tMean: -20.2 \tBestMean: -19.0\n",
            "Episode: 1625 \tReward: -21.0 \tMean: -20.2 \tBestMean: -19.0\n",
            "Episode: 1626 \tReward: -21.0 \tMean: -20.2 \tBestMean: -19.0\n",
            "Episode: 1627 \tReward: -19.0 \tMean: -20.1 \tBestMean: -19.0\n",
            "Episode: 1628 \tReward: -21.0 \tMean: -20.3 \tBestMean: -19.0\n",
            "Episode: 1629 \tReward: -21.0 \tMean: -20.5 \tBestMean: -19.0\n",
            "Episode: 1630 \tReward: -20.0 \tMean: -20.4 \tBestMean: -19.0\n",
            "Episode: 1631 \tReward: -21.0 \tMean: -20.4 \tBestMean: -19.0\n",
            "Episode: 1632 \tReward: -19.0 \tMean: -20.2 \tBestMean: -19.0\n",
            "Episode: 1633 \tReward: -21.0 \tMean: -20.3 \tBestMean: -19.0\n",
            "Episode: 1634 \tReward: -20.0 \tMean: -20.4 \tBestMean: -19.0\n",
            "Episode: 1635 \tReward: -18.0 \tMean: -20.1 \tBestMean: -19.0\n",
            "Episode: 1636 \tReward: -21.0 \tMean: -20.1 \tBestMean: -19.0\n",
            "Episode: 1637 \tReward: -21.0 \tMean: -20.3 \tBestMean: -19.0\n",
            "Episode: 1638 \tReward: -20.0 \tMean: -20.2 \tBestMean: -19.0\n",
            "Episode: 1639 \tReward: -21.0 \tMean: -20.2 \tBestMean: -19.0\n",
            "Episode: 1640 \tReward: -20.0 \tMean: -20.2 \tBestMean: -19.0\n",
            "Episode: 1641 \tReward: -20.0 \tMean: -20.1 \tBestMean: -19.0\n",
            "Episode: 1642 \tReward: -21.0 \tMean: -20.3 \tBestMean: -19.0\n",
            "Episode: 1643 \tReward: -20.0 \tMean: -20.2 \tBestMean: -19.0\n",
            "Episode: 1644 \tReward: -20.0 \tMean: -20.2 \tBestMean: -19.0\n",
            "Episode: 1645 \tReward: -21.0 \tMean: -20.5 \tBestMean: -19.0\n",
            "Episode: 1646 \tReward: -19.0 \tMean: -20.3 \tBestMean: -19.0\n",
            "Episode: 1647 \tReward: -20.0 \tMean: -20.2 \tBestMean: -19.0\n",
            "Episode: 1648 \tReward: -21.0 \tMean: -20.3 \tBestMean: -19.0\n",
            "Episode: 1649 \tReward: -21.0 \tMean: -20.3 \tBestMean: -19.0\n",
            "Episode: 1650 \tReward: -21.0 \tMean: -20.4 \tBestMean: -19.0\n",
            "Episode: 1651 \tReward: -20.0 \tMean: -20.4 \tBestMean: -19.0\n",
            "Episode: 1652 \tReward: -21.0 \tMean: -20.4 \tBestMean: -19.0\n",
            "Episode: 1653 \tReward: -21.0 \tMean: -20.5 \tBestMean: -19.0\n",
            "Episode: 1654 \tReward: -21.0 \tMean: -20.6 \tBestMean: -19.0\n",
            "Episode: 1655 \tReward: -18.0 \tMean: -20.3 \tBestMean: -19.0\n",
            "Episode: 1656 \tReward: -19.0 \tMean: -20.3 \tBestMean: -19.0\n",
            "Episode: 1657 \tReward: -21.0 \tMean: -20.4 \tBestMean: -19.0\n",
            "Episode: 1658 \tReward: -20.0 \tMean: -20.3 \tBestMean: -19.0\n",
            "Episode: 1659 \tReward: -19.0 \tMean: -20.1 \tBestMean: -19.0\n",
            "Episode: 1660 \tReward: -19.0 \tMean: -19.9 \tBestMean: -19.0\n",
            "Episode: 1661 \tReward: -18.0 \tMean: -19.7 \tBestMean: -19.0\n",
            "Episode: 1662 \tReward: -19.0 \tMean: -19.5 \tBestMean: -19.0\n",
            "Episode: 1663 \tReward: -19.0 \tMean: -19.3 \tBestMean: -19.0\n",
            "Episode: 1664 \tReward: -20.0 \tMean: -19.2 \tBestMean: -19.0\n",
            "Episode: 1665 \tReward: -21.0 \tMean: -19.5 \tBestMean: -19.0\n",
            "Episode: 1666 \tReward: -20.0 \tMean: -19.6 \tBestMean: -19.0\n",
            "Episode: 1667 \tReward: -21.0 \tMean: -19.6 \tBestMean: -19.0\n",
            "Episode: 1668 \tReward: -20.0 \tMean: -19.6 \tBestMean: -19.0\n",
            "Episode: 1669 \tReward: -21.0 \tMean: -19.8 \tBestMean: -19.0\n",
            "Episode: 1670 \tReward: -21.0 \tMean: -20.0 \tBestMean: -19.0\n",
            "Episode: 1671 \tReward: -19.0 \tMean: -20.1 \tBestMean: -19.0\n",
            "Episode: 1672 \tReward: -21.0 \tMean: -20.3 \tBestMean: -19.0\n",
            "Episode: 1673 \tReward: -18.0 \tMean: -20.2 \tBestMean: -19.0\n",
            "Episode: 1674 \tReward: -21.0 \tMean: -20.3 \tBestMean: -19.0\n",
            "Episode: 1675 \tReward: -20.0 \tMean: -20.2 \tBestMean: -19.0\n",
            "Episode: 1676 \tReward: -21.0 \tMean: -20.3 \tBestMean: -19.0\n",
            "Episode: 1677 \tReward: -21.0 \tMean: -20.3 \tBestMean: -19.0\n",
            "Episode: 1678 \tReward: -21.0 \tMean: -20.4 \tBestMean: -19.0\n",
            "Episode: 1679 \tReward: -20.0 \tMean: -20.3 \tBestMean: -19.0\n",
            "Episode: 1680 \tReward: -19.0 \tMean: -20.1 \tBestMean: -19.0\n",
            "Episode: 1681 \tReward: -20.0 \tMean: -20.2 \tBestMean: -19.0\n",
            "Episode: 1682 \tReward: -21.0 \tMean: -20.2 \tBestMean: -19.0\n",
            "Episode: 1683 \tReward: -20.0 \tMean: -20.4 \tBestMean: -19.0\n",
            "Episode: 1684 \tReward: -20.0 \tMean: -20.3 \tBestMean: -19.0\n",
            "Episode: 1685 \tReward: -19.0 \tMean: -20.2 \tBestMean: -19.0\n",
            "Episode: 1686 \tReward: -18.0 \tMean: -19.9 \tBestMean: -19.0\n",
            "Episode: 1687 \tReward: -21.0 \tMean: -19.9 \tBestMean: -19.0\n",
            "Episode: 1688 \tReward: -20.0 \tMean: -19.8 \tBestMean: -19.0\n",
            "Episode: 1689 \tReward: -20.0 \tMean: -19.8 \tBestMean: -19.0\n",
            "Episode: 1690 \tReward: -19.0 \tMean: -19.8 \tBestMean: -19.0\n",
            "Episode: 1691 \tReward: -20.0 \tMean: -19.8 \tBestMean: -19.0\n",
            "Episode: 1692 \tReward: -21.0 \tMean: -19.8 \tBestMean: -19.0\n",
            "Episode: 1693 \tReward: -20.0 \tMean: -19.8 \tBestMean: -19.0\n",
            "Episode: 1694 \tReward: -19.0 \tMean: -19.7 \tBestMean: -19.0\n",
            "Episode: 1695 \tReward: -18.0 \tMean: -19.6 \tBestMean: -19.0\n",
            "Episode: 1696 \tReward: -20.0 \tMean: -19.8 \tBestMean: -19.0\n",
            "Episode: 1697 \tReward: -21.0 \tMean: -19.8 \tBestMean: -19.0\n",
            "Episode: 1698 \tReward: -21.0 \tMean: -19.9 \tBestMean: -19.0\n",
            "Episode: 1699 \tReward: -18.0 \tMean: -19.7 \tBestMean: -19.0\n",
            "Episode: 1700 \tReward: -19.0 \tMean: -19.7 \tBestMean: -19.0\n",
            "Episode: 1701 \tReward: -20.0 \tMean: -19.7 \tBestMean: -19.0\n",
            "Episode: 1702 \tReward: -20.0 \tMean: -19.6 \tBestMean: -19.0\n",
            "Episode: 1703 \tReward: -20.0 \tMean: -19.6 \tBestMean: -19.0\n",
            "Episode: 1704 \tReward: -20.0 \tMean: -19.7 \tBestMean: -19.0\n",
            "Episode: 1705 \tReward: -19.0 \tMean: -19.8 \tBestMean: -19.0\n",
            "Episode: 1706 \tReward: -21.0 \tMean: -19.9 \tBestMean: -19.0\n",
            "Episode: 1707 \tReward: -20.0 \tMean: -19.8 \tBestMean: -19.0\n",
            "Episode: 1708 \tReward: -21.0 \tMean: -19.8 \tBestMean: -19.0\n",
            "Episode: 1709 \tReward: -21.0 \tMean: -20.1 \tBestMean: -19.0\n",
            "Episode: 1710 \tReward: -18.0 \tMean: -20.0 \tBestMean: -19.0\n",
            "Episode: 1711 \tReward: -21.0 \tMean: -20.1 \tBestMean: -19.0\n",
            "Episode: 1712 \tReward: -20.0 \tMean: -20.1 \tBestMean: -19.0\n",
            "Episode: 1713 \tReward: -21.0 \tMean: -20.2 \tBestMean: -19.0\n",
            "Episode: 1714 \tReward: -19.0 \tMean: -20.1 \tBestMean: -19.0\n",
            "Episode: 1715 \tReward: -21.0 \tMean: -20.3 \tBestMean: -19.0\n",
            "Episode: 1716 \tReward: -21.0 \tMean: -20.3 \tBestMean: -19.0\n",
            "Episode: 1717 \tReward: -20.0 \tMean: -20.3 \tBestMean: -19.0\n",
            "Episode: 1718 \tReward: -20.0 \tMean: -20.2 \tBestMean: -19.0\n",
            "Episode: 1719 \tReward: -20.0 \tMean: -20.1 \tBestMean: -19.0\n",
            "Episode: 1720 \tReward: -20.0 \tMean: -20.3 \tBestMean: -19.0\n",
            "Episode: 1721 \tReward: -20.0 \tMean: -20.2 \tBestMean: -19.0\n",
            "Episode: 1722 \tReward: -20.0 \tMean: -20.2 \tBestMean: -19.0\n",
            "Episode: 1723 \tReward: -21.0 \tMean: -20.2 \tBestMean: -19.0\n",
            "Episode: 1724 \tReward: -21.0 \tMean: -20.4 \tBestMean: -19.0\n",
            "Episode: 1725 \tReward: -19.0 \tMean: -20.2 \tBestMean: -19.0\n",
            "Episode: 1726 \tReward: -21.0 \tMean: -20.2 \tBestMean: -19.0\n",
            "Episode: 1727 \tReward: -21.0 \tMean: -20.3 \tBestMean: -19.0\n",
            "Episode: 1728 \tReward: -20.0 \tMean: -20.3 \tBestMean: -19.0\n",
            "Episode: 1729 \tReward: -21.0 \tMean: -20.4 \tBestMean: -19.0\n",
            "Episode: 1730 \tReward: -21.0 \tMean: -20.5 \tBestMean: -19.0\n",
            "Episode: 1731 \tReward: -21.0 \tMean: -20.6 \tBestMean: -19.0\n",
            "Episode: 1732 \tReward: -20.0 \tMean: -20.6 \tBestMean: -19.0\n",
            "Episode: 1733 \tReward: -20.0 \tMean: -20.5 \tBestMean: -19.0\n",
            "Episode: 1734 \tReward: -21.0 \tMean: -20.5 \tBestMean: -19.0\n",
            "Episode: 1735 \tReward: -19.0 \tMean: -20.5 \tBestMean: -19.0\n",
            "Episode: 1736 \tReward: -19.0 \tMean: -20.3 \tBestMean: -19.0\n",
            "Episode: 1737 \tReward: -21.0 \tMean: -20.3 \tBestMean: -19.0\n",
            "Episode: 1738 \tReward: -21.0 \tMean: -20.4 \tBestMean: -19.0\n",
            "Episode: 1739 \tReward: -20.0 \tMean: -20.3 \tBestMean: -19.0\n",
            "Episode: 1740 \tReward: -20.0 \tMean: -20.2 \tBestMean: -19.0\n",
            "Episode: 1741 \tReward: -20.0 \tMean: -20.1 \tBestMean: -19.0\n",
            "Episode: 1742 \tReward: -20.0 \tMean: -20.1 \tBestMean: -19.0\n",
            "Episode: 1743 \tReward: -21.0 \tMean: -20.2 \tBestMean: -19.0\n",
            "Episode: 1744 \tReward: -21.0 \tMean: -20.2 \tBestMean: -19.0\n",
            "Episode: 1745 \tReward: -21.0 \tMean: -20.4 \tBestMean: -19.0\n",
            "Episode: 1746 \tReward: -20.0 \tMean: -20.5 \tBestMean: -19.0\n",
            "Episode: 1747 \tReward: -20.0 \tMean: -20.4 \tBestMean: -19.0\n",
            "Episode: 1748 \tReward: -21.0 \tMean: -20.4 \tBestMean: -19.0\n",
            "Episode: 1749 \tReward: -19.0 \tMean: -20.3 \tBestMean: -19.0\n",
            "Episode: 1750 \tReward: -21.0 \tMean: -20.4 \tBestMean: -19.0\n",
            "Episode: 1751 \tReward: -20.0 \tMean: -20.4 \tBestMean: -19.0\n",
            "Episode: 1752 \tReward: -20.0 \tMean: -20.4 \tBestMean: -19.0\n",
            "Episode: 1753 \tReward: -21.0 \tMean: -20.4 \tBestMean: -19.0\n",
            "Episode: 1754 \tReward: -21.0 \tMean: -20.4 \tBestMean: -19.0\n",
            "Episode: 1755 \tReward: -21.0 \tMean: -20.4 \tBestMean: -19.0\n",
            "Episode: 1756 \tReward: -21.0 \tMean: -20.5 \tBestMean: -19.0\n",
            "Episode: 1757 \tReward: -21.0 \tMean: -20.6 \tBestMean: -19.0\n",
            "Episode: 1758 \tReward: -20.0 \tMean: -20.5 \tBestMean: -19.0\n",
            "Episode: 1759 \tReward: -19.0 \tMean: -20.5 \tBestMean: -19.0\n",
            "Episode: 1760 \tReward: -21.0 \tMean: -20.5 \tBestMean: -19.0\n",
            "Episode: 1761 \tReward: -20.0 \tMean: -20.5 \tBestMean: -19.0\n",
            "Episode: 1762 \tReward: -20.0 \tMean: -20.5 \tBestMean: -19.0\n",
            "Episode: 1763 \tReward: -19.0 \tMean: -20.3 \tBestMean: -19.0\n",
            "Episode: 1764 \tReward: -21.0 \tMean: -20.3 \tBestMean: -19.0\n",
            "Episode: 1765 \tReward: -21.0 \tMean: -20.3 \tBestMean: -19.0\n",
            "Episode: 1766 \tReward: -21.0 \tMean: -20.3 \tBestMean: -19.0\n",
            "Episode: 1767 \tReward: -21.0 \tMean: -20.3 \tBestMean: -19.0\n",
            "Episode: 1768 \tReward: -20.0 \tMean: -20.3 \tBestMean: -19.0\n",
            "Episode: 1769 \tReward: -19.0 \tMean: -20.3 \tBestMean: -19.0\n",
            "Episode: 1770 \tReward: -21.0 \tMean: -20.3 \tBestMean: -19.0\n",
            "Episode: 1771 \tReward: -20.0 \tMean: -20.3 \tBestMean: -19.0\n",
            "Episode: 1772 \tReward: -20.0 \tMean: -20.3 \tBestMean: -19.0\n",
            "Episode: 1773 \tReward: -21.0 \tMean: -20.5 \tBestMean: -19.0\n",
            "Episode: 1774 \tReward: -21.0 \tMean: -20.5 \tBestMean: -19.0\n",
            "Episode: 1775 \tReward: -21.0 \tMean: -20.5 \tBestMean: -19.0\n",
            "Episode: 1776 \tReward: -21.0 \tMean: -20.5 \tBestMean: -19.0\n",
            "Episode: 1777 \tReward: -21.0 \tMean: -20.5 \tBestMean: -19.0\n",
            "Episode: 1778 \tReward: -21.0 \tMean: -20.6 \tBestMean: -19.0\n",
            "Episode: 1779 \tReward: -21.0 \tMean: -20.8 \tBestMean: -19.0\n",
            "Episode: 1780 \tReward: -21.0 \tMean: -20.8 \tBestMean: -19.0\n",
            "Episode: 1781 \tReward: -20.0 \tMean: -20.8 \tBestMean: -19.0\n",
            "Episode: 1782 \tReward: -19.0 \tMean: -20.7 \tBestMean: -19.0\n",
            "Episode: 1783 \tReward: -20.0 \tMean: -20.6 \tBestMean: -19.0\n",
            "Episode: 1784 \tReward: -20.0 \tMean: -20.5 \tBestMean: -19.0\n",
            "Episode: 1785 \tReward: -20.0 \tMean: -20.4 \tBestMean: -19.0\n",
            "Episode: 1786 \tReward: -20.0 \tMean: -20.3 \tBestMean: -19.0\n",
            "Episode: 1787 \tReward: -19.0 \tMean: -20.1 \tBestMean: -19.0\n",
            "Episode: 1788 \tReward: -21.0 \tMean: -20.1 \tBestMean: -19.0\n",
            "Episode: 1789 \tReward: -20.0 \tMean: -20.0 \tBestMean: -19.0\n",
            "Episode: 1790 \tReward: -21.0 \tMean: -20.0 \tBestMean: -19.0\n",
            "Episode: 1791 \tReward: -21.0 \tMean: -20.1 \tBestMean: -19.0\n",
            "Episode: 1792 \tReward: -21.0 \tMean: -20.3 \tBestMean: -19.0\n",
            "Episode: 1793 \tReward: -20.0 \tMean: -20.3 \tBestMean: -19.0\n",
            "Episode: 1794 \tReward: -21.0 \tMean: -20.4 \tBestMean: -19.0\n",
            "Episode: 1795 \tReward: -20.0 \tMean: -20.4 \tBestMean: -19.0\n",
            "Episode: 1796 \tReward: -20.0 \tMean: -20.4 \tBestMean: -19.0\n",
            "Episode: 1797 \tReward: -20.0 \tMean: -20.5 \tBestMean: -19.0\n",
            "Episode: 1798 \tReward: -21.0 \tMean: -20.5 \tBestMean: -19.0\n",
            "Episode: 1799 \tReward: -20.0 \tMean: -20.5 \tBestMean: -19.0\n",
            "Episode: 1800 \tReward: -19.0 \tMean: -20.3 \tBestMean: -19.0\n",
            "Episode: 1801 \tReward: -21.0 \tMean: -20.3 \tBestMean: -19.0\n",
            "Episode: 1802 \tReward: -21.0 \tMean: -20.3 \tBestMean: -19.0\n",
            "Episode: 1803 \tReward: -20.0 \tMean: -20.3 \tBestMean: -19.0\n",
            "Episode: 1804 \tReward: -21.0 \tMean: -20.3 \tBestMean: -19.0\n",
            "Episode: 1805 \tReward: -21.0 \tMean: -20.4 \tBestMean: -19.0\n",
            "Episode: 1806 \tReward: -21.0 \tMean: -20.5 \tBestMean: -19.0\n",
            "Episode: 1807 \tReward: -20.0 \tMean: -20.5 \tBestMean: -19.0\n",
            "Episode: 1808 \tReward: -18.0 \tMean: -20.2 \tBestMean: -19.0\n",
            "Episode: 1809 \tReward: -17.0 \tMean: -19.9 \tBestMean: -19.0\n",
            "Episode: 1810 \tReward: -18.0 \tMean: -19.8 \tBestMean: -19.0\n",
            "Episode: 1811 \tReward: -20.0 \tMean: -19.7 \tBestMean: -19.0\n",
            "Episode: 1812 \tReward: -19.0 \tMean: -19.5 \tBestMean: -19.0\n",
            "Episode: 1813 \tReward: -21.0 \tMean: -19.6 \tBestMean: -19.0\n",
            "Episode: 1814 \tReward: -19.0 \tMean: -19.4 \tBestMean: -19.0\n",
            "Episode: 1815 \tReward: -19.0 \tMean: -19.2 \tBestMean: -19.0\n",
            "Episode: 1816 \tReward: -20.0 \tMean: -19.1 \tBestMean: -19.0\n",
            "Episode: 1817 \tReward: -20.0 \tMean: -19.1 \tBestMean: -19.0\n",
            "Episode: 1818 \tReward: -20.0 \tMean: -19.3 \tBestMean: -19.0\n",
            "Episode: 1819 \tReward: -19.0 \tMean: -19.5 \tBestMean: -19.0\n",
            "Episode: 1820 \tReward: -21.0 \tMean: -19.8 \tBestMean: -19.0\n",
            "Episode: 1821 \tReward: -20.0 \tMean: -19.8 \tBestMean: -19.0\n",
            "Episode: 1822 \tReward: -21.0 \tMean: -20.0 \tBestMean: -19.0\n",
            "Episode: 1823 \tReward: -21.0 \tMean: -20.0 \tBestMean: -19.0\n",
            "Episode: 1824 \tReward: -20.0 \tMean: -20.1 \tBestMean: -19.0\n",
            "Episode: 1825 \tReward: -20.0 \tMean: -20.2 \tBestMean: -19.0\n",
            "Episode: 1826 \tReward: -19.0 \tMean: -20.1 \tBestMean: -19.0\n",
            "Episode: 1827 \tReward: -19.0 \tMean: -20.0 \tBestMean: -19.0\n",
            "Episode: 1828 \tReward: -21.0 \tMean: -20.1 \tBestMean: -19.0\n",
            "Episode: 1829 \tReward: -19.0 \tMean: -20.1 \tBestMean: -19.0\n",
            "Episode: 1830 \tReward: -20.0 \tMean: -20.0 \tBestMean: -19.0\n",
            "Episode: 1831 \tReward: -19.0 \tMean: -19.9 \tBestMean: -19.0\n",
            "Episode: 1832 \tReward: -21.0 \tMean: -19.9 \tBestMean: -19.0\n",
            "Episode: 1833 \tReward: -20.0 \tMean: -19.8 \tBestMean: -19.0\n",
            "Episode: 1834 \tReward: -21.0 \tMean: -19.9 \tBestMean: -19.0\n",
            "Episode: 1835 \tReward: -21.0 \tMean: -20.0 \tBestMean: -19.0\n",
            "Episode: 1836 \tReward: -21.0 \tMean: -20.2 \tBestMean: -19.0\n",
            "Episode: 1837 \tReward: -20.0 \tMean: -20.3 \tBestMean: -19.0\n",
            "Episode: 1838 \tReward: -21.0 \tMean: -20.3 \tBestMean: -19.0\n",
            "Episode: 1839 \tReward: -19.0 \tMean: -20.3 \tBestMean: -19.0\n",
            "Episode: 1840 \tReward: -21.0 \tMean: -20.4 \tBestMean: -19.0\n",
            "Episode: 1841 \tReward: -20.0 \tMean: -20.5 \tBestMean: -19.0\n",
            "Episode: 1842 \tReward: -20.0 \tMean: -20.4 \tBestMean: -19.0\n",
            "Episode: 1843 \tReward: -20.0 \tMean: -20.4 \tBestMean: -19.0\n",
            "Episode: 1844 \tReward: -20.0 \tMean: -20.3 \tBestMean: -19.0\n",
            "Episode: 1845 \tReward: -21.0 \tMean: -20.3 \tBestMean: -19.0\n",
            "Episode: 1846 \tReward: -21.0 \tMean: -20.3 \tBestMean: -19.0\n",
            "Episode: 1847 \tReward: -20.0 \tMean: -20.3 \tBestMean: -19.0\n",
            "Episode: 1848 \tReward: -21.0 \tMean: -20.3 \tBestMean: -19.0\n",
            "Episode: 1849 \tReward: -21.0 \tMean: -20.5 \tBestMean: -19.0\n",
            "Episode: 1850 \tReward: -21.0 \tMean: -20.5 \tBestMean: -19.0\n",
            "Episode: 1851 \tReward: -19.0 \tMean: -20.4 \tBestMean: -19.0\n",
            "Episode: 1852 \tReward: -21.0 \tMean: -20.5 \tBestMean: -19.0\n",
            "Episode: 1853 \tReward: -20.0 \tMean: -20.5 \tBestMean: -19.0\n",
            "Episode: 1854 \tReward: -21.0 \tMean: -20.6 \tBestMean: -19.0\n",
            "Episode: 1855 \tReward: -20.0 \tMean: -20.5 \tBestMean: -19.0\n",
            "Episode: 1856 \tReward: -20.0 \tMean: -20.4 \tBestMean: -19.0\n",
            "Episode: 1857 \tReward: -20.0 \tMean: -20.4 \tBestMean: -19.0\n",
            "Episode: 1858 \tReward: -19.0 \tMean: -20.2 \tBestMean: -19.0\n",
            "Episode: 1859 \tReward: -18.0 \tMean: -19.9 \tBestMean: -19.0\n",
            "Episode: 1860 \tReward: -19.0 \tMean: -19.7 \tBestMean: -19.0\n",
            "Episode: 1861 \tReward: -20.0 \tMean: -19.8 \tBestMean: -19.0\n",
            "Episode: 1862 \tReward: -20.0 \tMean: -19.7 \tBestMean: -19.0\n",
            "Episode: 1863 \tReward: -20.0 \tMean: -19.7 \tBestMean: -19.0\n",
            "Episode: 1864 \tReward: -20.0 \tMean: -19.6 \tBestMean: -19.0\n",
            "Episode: 1865 \tReward: -19.0 \tMean: -19.5 \tBestMean: -19.0\n",
            "Episode: 1866 \tReward: -21.0 \tMean: -19.6 \tBestMean: -19.0\n",
            "Episode: 1867 \tReward: -20.0 \tMean: -19.6 \tBestMean: -19.0\n",
            "Episode: 1868 \tReward: -21.0 \tMean: -19.8 \tBestMean: -19.0\n",
            "Episode: 1869 \tReward: -21.0 \tMean: -20.1 \tBestMean: -19.0\n",
            "Episode: 1870 \tReward: -21.0 \tMean: -20.3 \tBestMean: -19.0\n",
            "Episode: 1871 \tReward: -19.0 \tMean: -20.2 \tBestMean: -19.0\n",
            "Episode: 1872 \tReward: -21.0 \tMean: -20.3 \tBestMean: -19.0\n",
            "Episode: 1873 \tReward: -21.0 \tMean: -20.4 \tBestMean: -19.0\n",
            "Episode: 1874 \tReward: -19.0 \tMean: -20.3 \tBestMean: -19.0\n",
            "Episode: 1875 \tReward: -20.0 \tMean: -20.4 \tBestMean: -19.0\n",
            "Episode: 1876 \tReward: -20.0 \tMean: -20.3 \tBestMean: -19.0\n",
            "Episode: 1877 \tReward: -21.0 \tMean: -20.4 \tBestMean: -19.0\n",
            "Episode: 1878 \tReward: -19.0 \tMean: -20.2 \tBestMean: -19.0\n",
            "Episode: 1879 \tReward: -19.0 \tMean: -20.0 \tBestMean: -19.0\n",
            "Episode: 1880 \tReward: -20.0 \tMean: -19.9 \tBestMean: -19.0\n",
            "Episode: 1881 \tReward: -20.0 \tMean: -20.0 \tBestMean: -19.0\n",
            "Episode: 1882 \tReward: -20.0 \tMean: -19.9 \tBestMean: -19.0\n",
            "Episode: 1883 \tReward: -21.0 \tMean: -19.9 \tBestMean: -19.0\n",
            "Episode: 1884 \tReward: -21.0 \tMean: -20.1 \tBestMean: -19.0\n",
            "Episode: 1885 \tReward: -21.0 \tMean: -20.2 \tBestMean: -19.0\n",
            "Episode: 1886 \tReward: -20.0 \tMean: -20.2 \tBestMean: -19.0\n",
            "Episode: 1887 \tReward: -20.0 \tMean: -20.1 \tBestMean: -19.0\n",
            "Episode: 1888 \tReward: -21.0 \tMean: -20.3 \tBestMean: -19.0\n",
            "Episode: 1889 \tReward: -20.0 \tMean: -20.4 \tBestMean: -19.0\n",
            "Episode: 1890 \tReward: -20.0 \tMean: -20.4 \tBestMean: -19.0\n",
            "Episode: 1891 \tReward: -20.0 \tMean: -20.4 \tBestMean: -19.0\n",
            "Episode: 1892 \tReward: -21.0 \tMean: -20.5 \tBestMean: -19.0\n",
            "Episode: 1893 \tReward: -21.0 \tMean: -20.5 \tBestMean: -19.0\n",
            "Episode: 1894 \tReward: -19.0 \tMean: -20.3 \tBestMean: -19.0\n",
            "Episode: 1895 \tReward: -19.0 \tMean: -20.1 \tBestMean: -19.0\n",
            "Episode: 1896 \tReward: -20.0 \tMean: -20.1 \tBestMean: -19.0\n",
            "Episode: 1897 \tReward: -21.0 \tMean: -20.2 \tBestMean: -19.0\n",
            "Episode: 1898 \tReward: -20.0 \tMean: -20.1 \tBestMean: -19.0\n",
            "Episode: 1899 \tReward: -20.0 \tMean: -20.1 \tBestMean: -19.0\n",
            "Episode: 1900 \tReward: -21.0 \tMean: -20.2 \tBestMean: -19.0\n",
            "Episode: 1901 \tReward: -21.0 \tMean: -20.3 \tBestMean: -19.0\n",
            "Episode: 1902 \tReward: -20.0 \tMean: -20.2 \tBestMean: -19.0\n",
            "Episode: 1903 \tReward: -20.0 \tMean: -20.1 \tBestMean: -19.0\n",
            "Episode: 1904 \tReward: -19.0 \tMean: -20.1 \tBestMean: -19.0\n",
            "Episode: 1905 \tReward: -19.0 \tMean: -20.1 \tBestMean: -19.0\n",
            "Episode: 1906 \tReward: -20.0 \tMean: -20.1 \tBestMean: -19.0\n",
            "Episode: 1907 \tReward: -21.0 \tMean: -20.1 \tBestMean: -19.0\n",
            "Episode: 1908 \tReward: -20.0 \tMean: -20.1 \tBestMean: -19.0\n",
            "Episode: 1909 \tReward: -21.0 \tMean: -20.2 \tBestMean: -19.0\n",
            "Episode: 1910 \tReward: -20.0 \tMean: -20.1 \tBestMean: -19.0\n",
            "Episode: 1911 \tReward: -21.0 \tMean: -20.1 \tBestMean: -19.0\n",
            "Episode: 1912 \tReward: -20.0 \tMean: -20.1 \tBestMean: -19.0\n",
            "Episode: 1913 \tReward: -20.0 \tMean: -20.1 \tBestMean: -19.0\n",
            "Episode: 1914 \tReward: -18.0 \tMean: -20.0 \tBestMean: -19.0\n",
            "Episode: 1915 \tReward: -18.0 \tMean: -19.9 \tBestMean: -19.0\n",
            "Episode: 1916 \tReward: -20.0 \tMean: -19.9 \tBestMean: -19.0\n",
            "Episode: 1917 \tReward: -20.0 \tMean: -19.8 \tBestMean: -19.0\n",
            "Episode: 1918 \tReward: -20.0 \tMean: -19.8 \tBestMean: -19.0\n",
            "Episode: 1919 \tReward: -19.0 \tMean: -19.6 \tBestMean: -19.0\n",
            "Episode: 1920 \tReward: -21.0 \tMean: -19.7 \tBestMean: -19.0\n",
            "Episode: 1921 \tReward: -20.0 \tMean: -19.6 \tBestMean: -19.0\n",
            "Episode: 1922 \tReward: -21.0 \tMean: -19.7 \tBestMean: -19.0\n",
            "Episode: 1923 \tReward: -21.0 \tMean: -19.8 \tBestMean: -19.0\n",
            "Episode: 1924 \tReward: -20.0 \tMean: -20.0 \tBestMean: -19.0\n",
            "Episode: 1925 \tReward: -19.0 \tMean: -20.1 \tBestMean: -19.0\n",
            "Episode: 1926 \tReward: -21.0 \tMean: -20.2 \tBestMean: -19.0\n",
            "Episode: 1927 \tReward: -21.0 \tMean: -20.3 \tBestMean: -19.0\n",
            "Episode: 1928 \tReward: -20.0 \tMean: -20.3 \tBestMean: -19.0\n",
            "Episode: 1929 \tReward: -20.0 \tMean: -20.4 \tBestMean: -19.0\n",
            "Episode: 1930 \tReward: -20.0 \tMean: -20.3 \tBestMean: -19.0\n",
            "Episode: 1931 \tReward: -19.0 \tMean: -20.2 \tBestMean: -19.0\n",
            "Episode: 1932 \tReward: -20.0 \tMean: -20.1 \tBestMean: -19.0\n",
            "Episode: 1933 \tReward: -21.0 \tMean: -20.1 \tBestMean: -19.0\n",
            "Episode: 1934 \tReward: -21.0 \tMean: -20.2 \tBestMean: -19.0\n",
            "Episode: 1935 \tReward: -18.0 \tMean: -20.1 \tBestMean: -19.0\n",
            "Episode: 1936 \tReward: -21.0 \tMean: -20.1 \tBestMean: -19.0\n",
            "Episode: 1937 \tReward: -20.0 \tMean: -20.0 \tBestMean: -19.0\n",
            "Episode: 1938 \tReward: -20.0 \tMean: -20.0 \tBestMean: -19.0\n",
            "Episode: 1939 \tReward: -21.0 \tMean: -20.1 \tBestMean: -19.0\n",
            "Episode: 1940 \tReward: -20.0 \tMean: -20.1 \tBestMean: -19.0\n",
            "Episode: 1941 \tReward: -20.0 \tMean: -20.2 \tBestMean: -19.0\n",
            "Episode: 1942 \tReward: -21.0 \tMean: -20.3 \tBestMean: -19.0\n",
            "Episode: 1943 \tReward: -21.0 \tMean: -20.3 \tBestMean: -19.0\n",
            "Episode: 1944 \tReward: -20.0 \tMean: -20.2 \tBestMean: -19.0\n",
            "Episode: 1945 \tReward: -21.0 \tMean: -20.5 \tBestMean: -19.0\n",
            "Episode: 1946 \tReward: -21.0 \tMean: -20.5 \tBestMean: -19.0\n",
            "Episode: 1947 \tReward: -21.0 \tMean: -20.6 \tBestMean: -19.0\n",
            "Episode: 1948 \tReward: -21.0 \tMean: -20.7 \tBestMean: -19.0\n",
            "Episode: 1949 \tReward: -20.0 \tMean: -20.6 \tBestMean: -19.0\n",
            "Episode: 1950 \tReward: -20.0 \tMean: -20.6 \tBestMean: -19.0\n",
            "Episode: 1951 \tReward: -21.0 \tMean: -20.7 \tBestMean: -19.0\n",
            "Episode: 1952 \tReward: -20.0 \tMean: -20.6 \tBestMean: -19.0\n",
            "Episode: 1953 \tReward: -21.0 \tMean: -20.6 \tBestMean: -19.0\n",
            "Episode: 1954 \tReward: -21.0 \tMean: -20.7 \tBestMean: -19.0\n",
            "Episode: 1955 \tReward: -21.0 \tMean: -20.7 \tBestMean: -19.0\n",
            "Episode: 1956 \tReward: -18.0 \tMean: -20.4 \tBestMean: -19.0\n",
            "Episode: 1957 \tReward: -20.0 \tMean: -20.3 \tBestMean: -19.0\n",
            "Episode: 1958 \tReward: -21.0 \tMean: -20.3 \tBestMean: -19.0\n",
            "Episode: 1959 \tReward: -21.0 \tMean: -20.4 \tBestMean: -19.0\n",
            "Episode: 1960 \tReward: -21.0 \tMean: -20.5 \tBestMean: -19.0\n",
            "Episode: 1961 \tReward: -20.0 \tMean: -20.4 \tBestMean: -19.0\n",
            "Episode: 1962 \tReward: -20.0 \tMean: -20.4 \tBestMean: -19.0\n",
            "Episode: 1963 \tReward: -21.0 \tMean: -20.4 \tBestMean: -19.0\n",
            "Episode: 1964 \tReward: -20.0 \tMean: -20.3 \tBestMean: -19.0\n",
            "Episode: 1965 \tReward: -20.0 \tMean: -20.2 \tBestMean: -19.0\n",
            "Episode: 1966 \tReward: -21.0 \tMean: -20.5 \tBestMean: -19.0\n",
            "Episode: 1967 \tReward: -21.0 \tMean: -20.6 \tBestMean: -19.0\n",
            "Episode: 1968 \tReward: -21.0 \tMean: -20.6 \tBestMean: -19.0\n",
            "Episode: 1969 \tReward: -20.0 \tMean: -20.5 \tBestMean: -19.0\n",
            "Episode: 1970 \tReward: -20.0 \tMean: -20.4 \tBestMean: -19.0\n",
            "Episode: 1971 \tReward: -21.0 \tMean: -20.5 \tBestMean: -19.0\n",
            "Episode: 1972 \tReward: -21.0 \tMean: -20.6 \tBestMean: -19.0\n",
            "Episode: 1973 \tReward: -20.0 \tMean: -20.5 \tBestMean: -19.0\n",
            "Episode: 1974 \tReward: -21.0 \tMean: -20.6 \tBestMean: -19.0\n",
            "Episode: 1975 \tReward: -21.0 \tMean: -20.7 \tBestMean: -19.0\n",
            "Episode: 1976 \tReward: -18.0 \tMean: -20.4 \tBestMean: -19.0\n",
            "Episode: 1977 \tReward: -19.0 \tMean: -20.2 \tBestMean: -19.0\n",
            "Episode: 1978 \tReward: -21.0 \tMean: -20.2 \tBestMean: -19.0\n",
            "Episode: 1979 \tReward: -20.0 \tMean: -20.2 \tBestMean: -19.0\n",
            "Episode: 1980 \tReward: -19.0 \tMean: -20.1 \tBestMean: -19.0\n",
            "Episode: 1981 \tReward: -20.0 \tMean: -20.0 \tBestMean: -19.0\n",
            "Episode: 1982 \tReward: -21.0 \tMean: -20.0 \tBestMean: -19.0\n",
            "Episode: 1983 \tReward: -21.0 \tMean: -20.1 \tBestMean: -19.0\n",
            "Episode: 1984 \tReward: -20.0 \tMean: -20.0 \tBestMean: -19.0\n",
            "Episode: 1985 \tReward: -19.0 \tMean: -19.8 \tBestMean: -19.0\n",
            "Episode: 1986 \tReward: -19.0 \tMean: -19.9 \tBestMean: -19.0\n",
            "Episode: 1987 \tReward: -20.0 \tMean: -20.0 \tBestMean: -19.0\n",
            "Episode: 1988 \tReward: -18.0 \tMean: -19.7 \tBestMean: -19.0\n",
            "Episode: 1989 \tReward: -20.0 \tMean: -19.7 \tBestMean: -19.0\n",
            "Episode: 1990 \tReward: -20.0 \tMean: -19.8 \tBestMean: -19.0\n",
            "Episode: 1991 \tReward: -21.0 \tMean: -19.9 \tBestMean: -19.0\n",
            "Episode: 1992 \tReward: -20.0 \tMean: -19.8 \tBestMean: -19.0\n",
            "Episode: 1993 \tReward: -20.0 \tMean: -19.7 \tBestMean: -19.0\n",
            "Episode: 1994 \tReward: -20.0 \tMean: -19.7 \tBestMean: -19.0\n",
            "Episode: 1995 \tReward: -20.0 \tMean: -19.8 \tBestMean: -19.0\n",
            "Episode: 1996 \tReward: -21.0 \tMean: -20.0 \tBestMean: -19.0\n",
            "Episode: 1997 \tReward: -20.0 \tMean: -20.0 \tBestMean: -19.0\n",
            "Episode: 1998 \tReward: -20.0 \tMean: -20.2 \tBestMean: -19.0\n",
            "Episode: 1999 \tReward: -20.0 \tMean: -20.2 \tBestMean: -19.0\n",
            "Episode: 2000 \tReward: -21.0 \tMean: -20.3 \tBestMean: -19.0\n",
            "Episode: 2001 \tReward: -21.0 \tMean: -20.3 \tBestMean: -19.0\n",
            "Episode: 2002 \tReward: -21.0 \tMean: -20.4 \tBestMean: -19.0\n",
            "Episode: 2003 \tReward: -20.0 \tMean: -20.4 \tBestMean: -19.0\n",
            "Episode: 2004 \tReward: -19.0 \tMean: -20.3 \tBestMean: -19.0\n",
            "Episode: 2005 \tReward: -20.0 \tMean: -20.3 \tBestMean: -19.0\n",
            "Episode: 2006 \tReward: -20.0 \tMean: -20.2 \tBestMean: -19.0\n",
            "Episode: 2007 \tReward: -20.0 \tMean: -20.2 \tBestMean: -19.0\n",
            "Episode: 2008 \tReward: -19.0 \tMean: -20.1 \tBestMean: -19.0\n",
            "Episode: 2009 \tReward: -21.0 \tMean: -20.2 \tBestMean: -19.0\n",
            "Episode: 2010 \tReward: -21.0 \tMean: -20.2 \tBestMean: -19.0\n",
            "Episode: 2011 \tReward: -19.0 \tMean: -20.0 \tBestMean: -19.0\n",
            "Episode: 2012 \tReward: -21.0 \tMean: -20.0 \tBestMean: -19.0\n",
            "Episode: 2013 \tReward: -20.0 \tMean: -20.0 \tBestMean: -19.0\n",
            "Episode: 2014 \tReward: -19.0 \tMean: -20.0 \tBestMean: -19.0\n",
            "Episode: 2015 \tReward: -20.0 \tMean: -20.0 \tBestMean: -19.0\n",
            "Episode: 2016 \tReward: -20.0 \tMean: -20.0 \tBestMean: -19.0\n",
            "Episode: 2017 \tReward: -18.0 \tMean: -19.8 \tBestMean: -19.0\n",
            "Episode: 2018 \tReward: -20.0 \tMean: -19.9 \tBestMean: -19.0\n",
            "Episode: 2019 \tReward: -20.0 \tMean: -19.8 \tBestMean: -19.0\n",
            "Episode: 2020 \tReward: -20.0 \tMean: -19.7 \tBestMean: -19.0\n",
            "Episode: 2021 \tReward: -19.0 \tMean: -19.7 \tBestMean: -19.0\n",
            "Episode: 2022 \tReward: -20.0 \tMean: -19.6 \tBestMean: -19.0\n",
            "Episode: 2023 \tReward: -20.0 \tMean: -19.6 \tBestMean: -19.0\n",
            "Episode: 2024 \tReward: -21.0 \tMean: -19.8 \tBestMean: -19.0\n",
            "Episode: 2025 \tReward: -21.0 \tMean: -19.9 \tBestMean: -19.0\n",
            "Episode: 2026 \tReward: -20.0 \tMean: -19.9 \tBestMean: -19.0\n",
            "Episode: 2027 \tReward: -20.0 \tMean: -20.1 \tBestMean: -19.0\n",
            "Episode: 2028 \tReward: -20.0 \tMean: -20.1 \tBestMean: -19.0\n",
            "Episode: 2029 \tReward: -21.0 \tMean: -20.2 \tBestMean: -19.0\n",
            "Episode: 2030 \tReward: -21.0 \tMean: -20.3 \tBestMean: -19.0\n",
            "Episode: 2031 \tReward: -20.0 \tMean: -20.4 \tBestMean: -19.0\n",
            "Episode: 2032 \tReward: -18.0 \tMean: -20.2 \tBestMean: -19.0\n",
            "Episode: 2033 \tReward: -21.0 \tMean: -20.3 \tBestMean: -19.0\n",
            "Episode: 2034 \tReward: -20.0 \tMean: -20.2 \tBestMean: -19.0\n",
            "Episode: 2035 \tReward: -21.0 \tMean: -20.2 \tBestMean: -19.0\n",
            "Episode: 2036 \tReward: -19.0 \tMean: -20.1 \tBestMean: -19.0\n",
            "Episode: 2037 \tReward: -21.0 \tMean: -20.2 \tBestMean: -19.0\n",
            "Episode: 2038 \tReward: -21.0 \tMean: -20.3 \tBestMean: -19.0\n",
            "Episode: 2039 \tReward: -20.0 \tMean: -20.2 \tBestMean: -19.0\n",
            "Episode: 2040 \tReward: -20.0 \tMean: -20.1 \tBestMean: -19.0\n",
            "Episode: 2041 \tReward: -20.0 \tMean: -20.1 \tBestMean: -19.0\n",
            "Episode: 2042 \tReward: -21.0 \tMean: -20.4 \tBestMean: -19.0\n",
            "Episode: 2043 \tReward: -20.0 \tMean: -20.3 \tBestMean: -19.0\n",
            "Episode: 2044 \tReward: -20.0 \tMean: -20.3 \tBestMean: -19.0\n",
            "Episode: 2045 \tReward: -20.0 \tMean: -20.2 \tBestMean: -19.0\n",
            "Episode: 2046 \tReward: -20.0 \tMean: -20.3 \tBestMean: -19.0\n",
            "Episode: 2047 \tReward: -17.0 \tMean: -19.9 \tBestMean: -19.0\n",
            "Episode: 2048 \tReward: -20.0 \tMean: -19.8 \tBestMean: -19.0\n",
            "Episode: 2049 \tReward: -21.0 \tMean: -19.9 \tBestMean: -19.0\n",
            "Episode: 2050 \tReward: -20.0 \tMean: -19.9 \tBestMean: -19.0\n",
            "Episode: 2051 \tReward: -20.0 \tMean: -19.9 \tBestMean: -19.0\n",
            "Episode: 2052 \tReward: -21.0 \tMean: -19.9 \tBestMean: -19.0\n",
            "Episode: 2053 \tReward: -20.0 \tMean: -19.9 \tBestMean: -19.0\n",
            "Episode: 2054 \tReward: -17.0 \tMean: -19.6 \tBestMean: -19.0\n",
            "Episode: 2055 \tReward: -20.0 \tMean: -19.6 \tBestMean: -19.0\n",
            "Episode: 2056 \tReward: -20.0 \tMean: -19.6 \tBestMean: -19.0\n",
            "Episode: 2057 \tReward: -21.0 \tMean: -20.0 \tBestMean: -19.0\n",
            "Episode: 2058 \tReward: -20.0 \tMean: -20.0 \tBestMean: -19.0\n",
            "Episode: 2059 \tReward: -20.0 \tMean: -19.9 \tBestMean: -19.0\n",
            "Episode: 2060 \tReward: -20.0 \tMean: -19.9 \tBestMean: -19.0\n",
            "Episode: 2061 \tReward: -21.0 \tMean: -20.0 \tBestMean: -19.0\n",
            "Episode: 2062 \tReward: -19.0 \tMean: -19.8 \tBestMean: -19.0\n",
            "Episode: 2063 \tReward: -19.0 \tMean: -19.7 \tBestMean: -19.0\n",
            "Episode: 2064 \tReward: -20.0 \tMean: -20.0 \tBestMean: -19.0\n",
            "Episode: 2065 \tReward: -20.0 \tMean: -20.0 \tBestMean: -19.0\n",
            "Episode: 2066 \tReward: -19.0 \tMean: -19.9 \tBestMean: -19.0\n",
            "Episode: 2067 \tReward: -19.0 \tMean: -19.7 \tBestMean: -19.0\n",
            "Episode: 2068 \tReward: -20.0 \tMean: -19.7 \tBestMean: -19.0\n",
            "Episode: 2069 \tReward: -20.0 \tMean: -19.7 \tBestMean: -19.0\n",
            "Episode: 2070 \tReward: -18.0 \tMean: -19.5 \tBestMean: -19.0\n",
            "Episode: 2071 \tReward: -20.0 \tMean: -19.4 \tBestMean: -19.0\n",
            "Episode: 2072 \tReward: -20.0 \tMean: -19.5 \tBestMean: -19.0\n",
            "Episode: 2073 \tReward: -21.0 \tMean: -19.7 \tBestMean: -19.0\n",
            "Episode: 2074 \tReward: -21.0 \tMean: -19.8 \tBestMean: -19.0\n",
            "Episode: 2075 \tReward: -19.0 \tMean: -19.7 \tBestMean: -19.0\n",
            "Episode: 2076 \tReward: -20.0 \tMean: -19.8 \tBestMean: -19.0\n",
            "Episode: 2077 \tReward: -20.0 \tMean: -19.9 \tBestMean: -19.0\n",
            "Episode: 2078 \tReward: -19.0 \tMean: -19.8 \tBestMean: -19.0\n",
            "Episode: 2079 \tReward: -21.0 \tMean: -19.9 \tBestMean: -19.0\n",
            "Episode: 2080 \tReward: -19.0 \tMean: -20.0 \tBestMean: -19.0\n",
            "Episode: 2081 \tReward: -20.0 \tMean: -20.0 \tBestMean: -19.0\n",
            "Episode: 2082 \tReward: -21.0 \tMean: -20.1 \tBestMean: -19.0\n",
            "Episode: 2083 \tReward: -18.0 \tMean: -19.8 \tBestMean: -19.0\n",
            "Episode: 2084 \tReward: -19.0 \tMean: -19.6 \tBestMean: -19.0\n",
            "Episode: 2085 \tReward: -20.0 \tMean: -19.7 \tBestMean: -19.0\n",
            "Episode: 2086 \tReward: -21.0 \tMean: -19.8 \tBestMean: -19.0\n",
            "Episode: 2087 \tReward: -17.0 \tMean: -19.5 \tBestMean: -19.0\n",
            "Episode: 2088 \tReward: -20.0 \tMean: -19.6 \tBestMean: -19.0\n",
            "Episode: 2089 \tReward: -19.0 \tMean: -19.4 \tBestMean: -19.0\n",
            "Episode: 2090 \tReward: -20.0 \tMean: -19.5 \tBestMean: -19.0\n",
            "Episode: 2091 \tReward: -20.0 \tMean: -19.5 \tBestMean: -19.0\n",
            "Episode: 2092 \tReward: -19.0 \tMean: -19.3 \tBestMean: -19.0\n",
            "Episode: 2093 \tReward: -21.0 \tMean: -19.6 \tBestMean: -19.0\n",
            "Episode: 2094 \tReward: -20.0 \tMean: -19.7 \tBestMean: -19.0\n",
            "Episode: 2095 \tReward: -21.0 \tMean: -19.8 \tBestMean: -19.0\n",
            "Episode: 2096 \tReward: -21.0 \tMean: -19.8 \tBestMean: -19.0\n",
            "Episode: 2097 \tReward: -20.0 \tMean: -20.1 \tBestMean: -19.0\n",
            "Episode: 2098 \tReward: -18.0 \tMean: -19.9 \tBestMean: -19.0\n",
            "Episode: 2099 \tReward: -19.0 \tMean: -19.9 \tBestMean: -19.0\n",
            "Episode: 2100 \tReward: -20.0 \tMean: -19.9 \tBestMean: -19.0\n",
            "Episode: 2101 \tReward: -20.0 \tMean: -19.9 \tBestMean: -19.0\n",
            "Episode: 2102 \tReward: -20.0 \tMean: -20.0 \tBestMean: -19.0\n",
            "Episode: 2103 \tReward: -20.0 \tMean: -19.9 \tBestMean: -19.0\n",
            "Episode: 2104 \tReward: -20.0 \tMean: -19.9 \tBestMean: -19.0\n",
            "Episode: 2105 \tReward: -20.0 \tMean: -19.8 \tBestMean: -19.0\n",
            "Episode: 2106 \tReward: -20.0 \tMean: -19.7 \tBestMean: -19.0\n",
            "Episode: 2107 \tReward: -19.0 \tMean: -19.6 \tBestMean: -19.0\n",
            "Episode: 2108 \tReward: -20.0 \tMean: -19.8 \tBestMean: -19.0\n",
            "Episode: 2109 \tReward: -20.0 \tMean: -19.9 \tBestMean: -19.0\n",
            "Episode: 2110 \tReward: -21.0 \tMean: -20.0 \tBestMean: -19.0\n",
            "Episode: 2111 \tReward: -20.0 \tMean: -20.0 \tBestMean: -19.0\n",
            "Episode: 2112 \tReward: -21.0 \tMean: -20.1 \tBestMean: -19.0\n",
            "Episode: 2113 \tReward: -20.0 \tMean: -20.1 \tBestMean: -19.0\n",
            "Episode: 2114 \tReward: -19.0 \tMean: -20.0 \tBestMean: -19.0\n",
            "Episode: 2115 \tReward: -20.0 \tMean: -20.0 \tBestMean: -19.0\n",
            "Episode: 2116 \tReward: -21.0 \tMean: -20.1 \tBestMean: -19.0\n",
            "Episode: 2117 \tReward: -21.0 \tMean: -20.3 \tBestMean: -19.0\n",
            "Episode: 2118 \tReward: -20.0 \tMean: -20.3 \tBestMean: -19.0\n",
            "Episode: 2119 \tReward: -20.0 \tMean: -20.3 \tBestMean: -19.0\n",
            "Episode: 2120 \tReward: -20.0 \tMean: -20.2 \tBestMean: -19.0\n",
            "Episode: 2121 \tReward: -20.0 \tMean: -20.2 \tBestMean: -19.0\n",
            "Episode: 2122 \tReward: -21.0 \tMean: -20.2 \tBestMean: -19.0\n",
            "Episode: 2123 \tReward: -19.0 \tMean: -20.1 \tBestMean: -19.0\n",
            "Episode: 2124 \tReward: -19.0 \tMean: -20.1 \tBestMean: -19.0\n",
            "Episode: 2125 \tReward: -21.0 \tMean: -20.2 \tBestMean: -19.0\n",
            "Episode: 2126 \tReward: -19.0 \tMean: -20.0 \tBestMean: -19.0\n",
            "Episode: 2127 \tReward: -21.0 \tMean: -20.0 \tBestMean: -19.0\n",
            "Episode: 2128 \tReward: -19.0 \tMean: -19.9 \tBestMean: -19.0\n",
            "Episode: 2129 \tReward: -21.0 \tMean: -20.0 \tBestMean: -19.0\n",
            "Episode: 2130 \tReward: -21.0 \tMean: -20.1 \tBestMean: -19.0\n",
            "Episode: 2131 \tReward: -21.0 \tMean: -20.2 \tBestMean: -19.0\n",
            "Episode: 2132 \tReward: -20.0 \tMean: -20.1 \tBestMean: -19.0\n",
            "Episode: 2133 \tReward: -21.0 \tMean: -20.3 \tBestMean: -19.0\n",
            "Episode: 2134 \tReward: -21.0 \tMean: -20.5 \tBestMean: -19.0\n",
            "Episode: 2135 \tReward: -19.0 \tMean: -20.3 \tBestMean: -19.0\n",
            "Episode: 2136 \tReward: -21.0 \tMean: -20.5 \tBestMean: -19.0\n",
            "Episode: 2137 \tReward: -20.0 \tMean: -20.4 \tBestMean: -19.0\n",
            "Episode: 2138 \tReward: -21.0 \tMean: -20.6 \tBestMean: -19.0\n",
            "Episode: 2139 \tReward: -21.0 \tMean: -20.6 \tBestMean: -19.0\n",
            "Episode: 2140 \tReward: -20.0 \tMean: -20.5 \tBestMean: -19.0\n",
            "Episode: 2141 \tReward: -20.0 \tMean: -20.4 \tBestMean: -19.0\n",
            "Episode: 2142 \tReward: -20.0 \tMean: -20.4 \tBestMean: -19.0\n",
            "Episode: 2143 \tReward: -20.0 \tMean: -20.3 \tBestMean: -19.0\n",
            "Episode: 2144 \tReward: -18.0 \tMean: -20.0 \tBestMean: -19.0\n",
            "Episode: 2145 \tReward: -18.0 \tMean: -19.9 \tBestMean: -19.0\n",
            "Episode: 2146 \tReward: -20.0 \tMean: -19.8 \tBestMean: -19.0\n",
            "Episode: 2147 \tReward: -19.0 \tMean: -19.7 \tBestMean: -19.0\n",
            "Episode: 2148 \tReward: -19.0 \tMean: -19.5 \tBestMean: -19.0\n",
            "Episode: 2149 \tReward: -18.0 \tMean: -19.2 \tBestMean: -19.0\n",
            "Episode: 2150 \tReward: -18.0 \tMean: -19.0 \tBestMean: -19.0\n",
            "Episode: 2151 \tReward: -20.0 \tMean: -19.0 \tBestMean: -19.0\n",
            "Episode: 2152 \tReward: -19.0 \tMean: -18.9 \tBestMean: -19.0\n",
            "Episode: 2153 \tReward: -19.0 \tMean: -18.8 \tBestMean: -18.9\n",
            "Episode: 2154 \tReward: -20.0 \tMean: -19.0 \tBestMean: -18.8\n",
            "Episode: 2155 \tReward: -20.0 \tMean: -19.2 \tBestMean: -18.8\n",
            "Episode: 2156 \tReward: -20.0 \tMean: -19.2 \tBestMean: -18.8\n",
            "Episode: 2157 \tReward: -20.0 \tMean: -19.3 \tBestMean: -18.8\n",
            "Episode: 2158 \tReward: -20.0 \tMean: -19.4 \tBestMean: -18.8\n",
            "Episode: 2159 \tReward: -21.0 \tMean: -19.7 \tBestMean: -18.8\n",
            "Episode: 2160 \tReward: -21.0 \tMean: -20.0 \tBestMean: -18.8\n",
            "Episode: 2161 \tReward: -20.0 \tMean: -20.0 \tBestMean: -18.8\n",
            "Episode: 2162 \tReward: -21.0 \tMean: -20.2 \tBestMean: -18.8\n",
            "Episode: 2163 \tReward: -20.0 \tMean: -20.3 \tBestMean: -18.8\n",
            "Episode: 2164 \tReward: -19.0 \tMean: -20.2 \tBestMean: -18.8\n",
            "Episode: 2165 \tReward: -20.0 \tMean: -20.2 \tBestMean: -18.8\n",
            "Episode: 2166 \tReward: -20.0 \tMean: -20.2 \tBestMean: -18.8\n",
            "Episode: 2167 \tReward: -20.0 \tMean: -20.2 \tBestMean: -18.8\n",
            "Episode: 2168 \tReward: -19.0 \tMean: -20.1 \tBestMean: -18.8\n",
            "Episode: 2169 \tReward: -20.0 \tMean: -20.0 \tBestMean: -18.8\n",
            "Episode: 2170 \tReward: -21.0 \tMean: -20.0 \tBestMean: -18.8\n",
            "Episode: 2171 \tReward: -20.0 \tMean: -20.0 \tBestMean: -18.8\n",
            "Episode: 2172 \tReward: -21.0 \tMean: -20.0 \tBestMean: -18.8\n",
            "Episode: 2173 \tReward: -20.0 \tMean: -20.0 \tBestMean: -18.8\n",
            "Episode: 2174 \tReward: -20.0 \tMean: -20.1 \tBestMean: -18.8\n",
            "Episode: 2175 \tReward: -21.0 \tMean: -20.2 \tBestMean: -18.8\n",
            "Episode: 2176 \tReward: -20.0 \tMean: -20.2 \tBestMean: -18.8\n",
            "Episode: 2177 \tReward: -20.0 \tMean: -20.2 \tBestMean: -18.8\n",
            "Episode: 2178 \tReward: -20.0 \tMean: -20.3 \tBestMean: -18.8\n",
            "Episode: 2179 \tReward: -19.0 \tMean: -20.2 \tBestMean: -18.8\n",
            "Episode: 2180 \tReward: -21.0 \tMean: -20.2 \tBestMean: -18.8\n",
            "Episode: 2181 \tReward: -19.0 \tMean: -20.1 \tBestMean: -18.8\n",
            "Episode: 2182 \tReward: -21.0 \tMean: -20.1 \tBestMean: -18.8\n",
            "Episode: 2183 \tReward: -20.0 \tMean: -20.1 \tBestMean: -18.8\n",
            "Episode: 2184 \tReward: -21.0 \tMean: -20.2 \tBestMean: -18.8\n",
            "Episode: 2185 \tReward: -20.0 \tMean: -20.1 \tBestMean: -18.8\n",
            "Episode: 2186 \tReward: -20.0 \tMean: -20.1 \tBestMean: -18.8\n",
            "Episode: 2187 \tReward: -20.0 \tMean: -20.1 \tBestMean: -18.8\n",
            "Episode: 2188 \tReward: -20.0 \tMean: -20.1 \tBestMean: -18.8\n",
            "Episode: 2189 \tReward: -18.0 \tMean: -20.0 \tBestMean: -18.8\n",
            "Episode: 2190 \tReward: -21.0 \tMean: -20.0 \tBestMean: -18.8\n",
            "Episode: 2191 \tReward: -21.0 \tMean: -20.2 \tBestMean: -18.8\n",
            "Episode: 2192 \tReward: -19.0 \tMean: -20.0 \tBestMean: -18.8\n",
            "Episode: 2193 \tReward: -21.0 \tMean: -20.1 \tBestMean: -18.8\n",
            "Episode: 2194 \tReward: -20.0 \tMean: -20.0 \tBestMean: -18.8\n",
            "Episode: 2195 \tReward: -21.0 \tMean: -20.1 \tBestMean: -18.8\n",
            "Episode: 2196 \tReward: -19.0 \tMean: -20.0 \tBestMean: -18.8\n",
            "Episode: 2197 \tReward: -20.0 \tMean: -20.0 \tBestMean: -18.8\n",
            "Episode: 2198 \tReward: -21.0 \tMean: -20.1 \tBestMean: -18.8\n",
            "Episode: 2199 \tReward: -21.0 \tMean: -20.4 \tBestMean: -18.8\n",
            "Episode: 2200 \tReward: -19.0 \tMean: -20.2 \tBestMean: -18.8\n",
            "Episode: 2201 \tReward: -20.0 \tMean: -20.1 \tBestMean: -18.8\n",
            "Episode: 2202 \tReward: -21.0 \tMean: -20.3 \tBestMean: -18.8\n",
            "Episode: 2203 \tReward: -21.0 \tMean: -20.3 \tBestMean: -18.8\n",
            "Episode: 2204 \tReward: -20.0 \tMean: -20.3 \tBestMean: -18.8\n",
            "Episode: 2205 \tReward: -19.0 \tMean: -20.1 \tBestMean: -18.8\n",
            "Episode: 2206 \tReward: -20.0 \tMean: -20.2 \tBestMean: -18.8\n",
            "Episode: 2207 \tReward: -21.0 \tMean: -20.3 \tBestMean: -18.8\n",
            "Episode: 2208 \tReward: -20.0 \tMean: -20.2 \tBestMean: -18.8\n",
            "Episode: 2209 \tReward: -21.0 \tMean: -20.2 \tBestMean: -18.8\n",
            "Episode: 2210 \tReward: -19.0 \tMean: -20.2 \tBestMean: -18.8\n",
            "Episode: 2211 \tReward: -19.0 \tMean: -20.1 \tBestMean: -18.8\n",
            "Episode: 2212 \tReward: -21.0 \tMean: -20.1 \tBestMean: -18.8\n",
            "Episode: 2213 \tReward: -21.0 \tMean: -20.1 \tBestMean: -18.8\n",
            "Episode: 2214 \tReward: -19.0 \tMean: -20.0 \tBestMean: -18.8\n",
            "Episode: 2215 \tReward: -21.0 \tMean: -20.2 \tBestMean: -18.8\n",
            "Episode: 2216 \tReward: -21.0 \tMean: -20.3 \tBestMean: -18.8\n",
            "Episode: 2217 \tReward: -21.0 \tMean: -20.3 \tBestMean: -18.8\n",
            "Episode: 2218 \tReward: -19.0 \tMean: -20.2 \tBestMean: -18.8\n",
            "Episode: 2219 \tReward: -21.0 \tMean: -20.2 \tBestMean: -18.8\n",
            "Episode: 2220 \tReward: -20.0 \tMean: -20.3 \tBestMean: -18.8\n",
            "Episode: 2221 \tReward: -20.0 \tMean: -20.4 \tBestMean: -18.8\n",
            "Episode: 2222 \tReward: -17.0 \tMean: -20.0 \tBestMean: -18.8\n",
            "Episode: 2223 \tReward: -21.0 \tMean: -20.0 \tBestMean: -18.8\n",
            "Episode: 2224 \tReward: -21.0 \tMean: -20.2 \tBestMean: -18.8\n",
            "Episode: 2225 \tReward: -20.0 \tMean: -20.1 \tBestMean: -18.8\n",
            "Episode: 2226 \tReward: -21.0 \tMean: -20.1 \tBestMean: -18.8\n",
            "Episode: 2227 \tReward: -20.0 \tMean: -20.0 \tBestMean: -18.8\n",
            "Episode: 2228 \tReward: -17.0 \tMean: -19.8 \tBestMean: -18.8\n",
            "Episode: 2229 \tReward: -21.0 \tMean: -19.8 \tBestMean: -18.8\n",
            "Episode: 2230 \tReward: -21.0 \tMean: -19.9 \tBestMean: -18.8\n",
            "Episode: 2231 \tReward: -21.0 \tMean: -20.0 \tBestMean: -18.8\n",
            "Episode: 2232 \tReward: -20.0 \tMean: -20.3 \tBestMean: -18.8\n",
            "Episode: 2233 \tReward: -20.0 \tMean: -20.2 \tBestMean: -18.8\n",
            "Episode: 2234 \tReward: -19.0 \tMean: -20.0 \tBestMean: -18.8\n",
            "Episode: 2235 \tReward: -19.0 \tMean: -19.9 \tBestMean: -18.8\n",
            "Episode: 2236 \tReward: -18.0 \tMean: -19.6 \tBestMean: -18.8\n",
            "Episode: 2237 \tReward: -20.0 \tMean: -19.6 \tBestMean: -18.8\n",
            "Episode: 2238 \tReward: -19.0 \tMean: -19.8 \tBestMean: -18.8\n",
            "Episode: 2239 \tReward: -20.0 \tMean: -19.7 \tBestMean: -18.8\n",
            "Episode: 2240 \tReward: -20.0 \tMean: -19.6 \tBestMean: -18.8\n",
            "Episode: 2241 \tReward: -20.0 \tMean: -19.5 \tBestMean: -18.8\n",
            "Episode: 2242 \tReward: -20.0 \tMean: -19.5 \tBestMean: -18.8\n",
            "Episode: 2243 \tReward: -21.0 \tMean: -19.6 \tBestMean: -18.8\n",
            "Episode: 2244 \tReward: -21.0 \tMean: -19.8 \tBestMean: -18.8\n",
            "Episode: 2245 \tReward: -19.0 \tMean: -19.8 \tBestMean: -18.8\n",
            "Episode: 2246 \tReward: -19.0 \tMean: -19.9 \tBestMean: -18.8\n",
            "Episode: 2247 \tReward: -21.0 \tMean: -20.0 \tBestMean: -18.8\n",
            "Episode: 2248 \tReward: -19.0 \tMean: -20.0 \tBestMean: -18.8\n",
            "Episode: 2249 \tReward: -19.0 \tMean: -19.9 \tBestMean: -18.8\n",
            "Episode: 2250 \tReward: -20.0 \tMean: -19.9 \tBestMean: -18.8\n",
            "Episode: 2251 \tReward: -20.0 \tMean: -19.9 \tBestMean: -18.8\n",
            "Episode: 2252 \tReward: -20.0 \tMean: -19.9 \tBestMean: -18.8\n",
            "Episode: 2253 \tReward: -20.0 \tMean: -19.8 \tBestMean: -18.8\n",
            "Episode: 2254 \tReward: -18.0 \tMean: -19.5 \tBestMean: -18.8\n",
            "Episode: 2255 \tReward: -21.0 \tMean: -19.7 \tBestMean: -18.8\n",
            "Episode: 2256 \tReward: -21.0 \tMean: -19.9 \tBestMean: -18.8\n",
            "Episode: 2257 \tReward: -20.0 \tMean: -19.8 \tBestMean: -18.8\n",
            "Episode: 2258 \tReward: -21.0 \tMean: -20.0 \tBestMean: -18.8\n",
            "Episode: 2259 \tReward: -20.0 \tMean: -20.1 \tBestMean: -18.8\n",
            "Episode: 2260 \tReward: -19.0 \tMean: -20.0 \tBestMean: -18.8\n",
            "Episode: 2261 \tReward: -20.0 \tMean: -20.0 \tBestMean: -18.8\n",
            "Episode: 2262 \tReward: -20.0 \tMean: -20.0 \tBestMean: -18.8\n",
            "Episode: 2263 \tReward: -20.0 \tMean: -20.0 \tBestMean: -18.8\n",
            "Episode: 2264 \tReward: -20.0 \tMean: -20.2 \tBestMean: -18.8\n",
            "Episode: 2265 \tReward: -21.0 \tMean: -20.2 \tBestMean: -18.8\n",
            "Episode: 2266 \tReward: -20.0 \tMean: -20.1 \tBestMean: -18.8\n",
            "Episode: 2267 \tReward: -19.0 \tMean: -20.0 \tBestMean: -18.8\n",
            "Episode: 2268 \tReward: -21.0 \tMean: -20.0 \tBestMean: -18.8\n",
            "Episode: 2269 \tReward: -21.0 \tMean: -20.1 \tBestMean: -18.8\n",
            "Episode: 2270 \tReward: -20.0 \tMean: -20.2 \tBestMean: -18.8\n",
            "Episode: 2271 \tReward: -20.0 \tMean: -20.2 \tBestMean: -18.8\n",
            "Episode: 2272 \tReward: -19.0 \tMean: -20.1 \tBestMean: -18.8\n",
            "Episode: 2273 \tReward: -20.0 \tMean: -20.1 \tBestMean: -18.8\n",
            "Episode: 2274 \tReward: -20.0 \tMean: -20.1 \tBestMean: -18.8\n",
            "Episode: 2275 \tReward: -20.0 \tMean: -20.0 \tBestMean: -18.8\n",
            "Episode: 2276 \tReward: -20.0 \tMean: -20.0 \tBestMean: -18.8\n",
            "Episode: 2277 \tReward: -21.0 \tMean: -20.2 \tBestMean: -18.8\n",
            "Episode: 2278 \tReward: -20.0 \tMean: -20.1 \tBestMean: -18.8\n",
            "Episode: 2279 \tReward: -20.0 \tMean: -20.0 \tBestMean: -18.8\n",
            "Episode: 2280 \tReward: -19.0 \tMean: -19.9 \tBestMean: -18.8\n",
            "Episode: 2281 \tReward: -21.0 \tMean: -20.0 \tBestMean: -18.8\n",
            "Episode: 2282 \tReward: -20.0 \tMean: -20.1 \tBestMean: -18.8\n",
            "Episode: 2283 \tReward: -18.0 \tMean: -19.9 \tBestMean: -18.8\n",
            "Episode: 2284 \tReward: -20.0 \tMean: -19.9 \tBestMean: -18.8\n",
            "Episode: 2285 \tReward: -21.0 \tMean: -20.0 \tBestMean: -18.8\n",
            "Episode: 2286 \tReward: -21.0 \tMean: -20.1 \tBestMean: -18.8\n",
            "Episode: 2287 \tReward: -20.0 \tMean: -20.0 \tBestMean: -18.8\n",
            "Episode: 2288 \tReward: -21.0 \tMean: -20.1 \tBestMean: -18.8\n",
            "Episode: 2289 \tReward: -20.0 \tMean: -20.1 \tBestMean: -18.8\n",
            "Episode: 2290 \tReward: -21.0 \tMean: -20.3 \tBestMean: -18.8\n",
            "Episode: 2291 \tReward: -21.0 \tMean: -20.3 \tBestMean: -18.8\n",
            "Episode: 2292 \tReward: -21.0 \tMean: -20.4 \tBestMean: -18.8\n",
            "Episode: 2293 \tReward: -20.0 \tMean: -20.6 \tBestMean: -18.8\n",
            "Episode: 2294 \tReward: -20.0 \tMean: -20.6 \tBestMean: -18.8\n",
            "Episode: 2295 \tReward: -21.0 \tMean: -20.6 \tBestMean: -18.8\n",
            "Episode: 2296 \tReward: -20.0 \tMean: -20.5 \tBestMean: -18.8\n",
            "Episode: 2297 \tReward: -20.0 \tMean: -20.5 \tBestMean: -18.8\n",
            "Episode: 2298 \tReward: -21.0 \tMean: -20.5 \tBestMean: -18.8\n",
            "Episode: 2299 \tReward: -19.0 \tMean: -20.4 \tBestMean: -18.8\n",
            "Episode: 2300 \tReward: -20.0 \tMean: -20.3 \tBestMean: -18.8\n",
            "Episode: 2301 \tReward: -19.0 \tMean: -20.1 \tBestMean: -18.8\n",
            "Episode: 2302 \tReward: -19.0 \tMean: -19.9 \tBestMean: -18.8\n",
            "Episode: 2303 \tReward: -20.0 \tMean: -19.9 \tBestMean: -18.8\n",
            "Episode: 2304 \tReward: -21.0 \tMean: -20.0 \tBestMean: -18.8\n",
            "Episode: 2305 \tReward: -21.0 \tMean: -20.0 \tBestMean: -18.8\n",
            "Episode: 2306 \tReward: -20.0 \tMean: -20.0 \tBestMean: -18.8\n",
            "Episode: 2307 \tReward: -19.0 \tMean: -19.9 \tBestMean: -18.8\n",
            "Episode: 2308 \tReward: -20.0 \tMean: -19.8 \tBestMean: -18.8\n",
            "Episode: 2309 \tReward: -20.0 \tMean: -19.9 \tBestMean: -18.8\n",
            "Episode: 2310 \tReward: -21.0 \tMean: -20.0 \tBestMean: -18.8\n",
            "Episode: 2311 \tReward: -20.0 \tMean: -20.1 \tBestMean: -18.8\n",
            "Episode: 2312 \tReward: -19.0 \tMean: -20.1 \tBestMean: -18.8\n",
            "Episode: 2313 \tReward: -21.0 \tMean: -20.2 \tBestMean: -18.8\n",
            "Episode: 2314 \tReward: -20.0 \tMean: -20.1 \tBestMean: -18.8\n",
            "Episode: 2315 \tReward: -19.0 \tMean: -19.9 \tBestMean: -18.8\n",
            "Episode: 2316 \tReward: -20.0 \tMean: -19.9 \tBestMean: -18.8\n",
            "Episode: 2317 \tReward: -20.0 \tMean: -20.0 \tBestMean: -18.8\n",
            "Episode: 2318 \tReward: -21.0 \tMean: -20.1 \tBestMean: -18.8\n",
            "Episode: 2319 \tReward: -19.0 \tMean: -20.0 \tBestMean: -18.8\n",
            "Episode: 2320 \tReward: -20.0 \tMean: -19.9 \tBestMean: -18.8\n",
            "Episode: 2321 \tReward: -20.0 \tMean: -19.9 \tBestMean: -18.8\n",
            "Episode: 2322 \tReward: -21.0 \tMean: -20.1 \tBestMean: -18.8\n",
            "Episode: 2323 \tReward: -21.0 \tMean: -20.1 \tBestMean: -18.8\n",
            "Episode: 2324 \tReward: -21.0 \tMean: -20.2 \tBestMean: -18.8\n",
            "Episode: 2325 \tReward: -17.0 \tMean: -20.0 \tBestMean: -18.8\n",
            "Episode: 2326 \tReward: -20.0 \tMean: -20.0 \tBestMean: -18.8\n",
            "Episode: 2327 \tReward: -21.0 \tMean: -20.1 \tBestMean: -18.8\n",
            "Episode: 2328 \tReward: -21.0 \tMean: -20.1 \tBestMean: -18.8\n",
            "Episode: 2329 \tReward: -19.0 \tMean: -20.1 \tBestMean: -18.8\n",
            "Episode: 2330 \tReward: -21.0 \tMean: -20.2 \tBestMean: -18.8\n",
            "Episode: 2331 \tReward: -21.0 \tMean: -20.3 \tBestMean: -18.8\n",
            "Episode: 2332 \tReward: -19.0 \tMean: -20.1 \tBestMean: -18.8\n",
            "Episode: 2333 \tReward: -21.0 \tMean: -20.1 \tBestMean: -18.8\n",
            "Episode: 2334 \tReward: -21.0 \tMean: -20.1 \tBestMean: -18.8\n",
            "Episode: 2335 \tReward: -21.0 \tMean: -20.5 \tBestMean: -18.8\n",
            "Episode: 2336 \tReward: -21.0 \tMean: -20.6 \tBestMean: -18.8\n",
            "Episode: 2337 \tReward: -20.0 \tMean: -20.5 \tBestMean: -18.8\n",
            "Episode: 2338 \tReward: -19.0 \tMean: -20.3 \tBestMean: -18.8\n",
            "Episode: 2339 \tReward: -19.0 \tMean: -20.3 \tBestMean: -18.8\n",
            "Episode: 2340 \tReward: -21.0 \tMean: -20.3 \tBestMean: -18.8\n",
            "Episode: 2341 \tReward: -20.0 \tMean: -20.2 \tBestMean: -18.8\n",
            "Episode: 2342 \tReward: -20.0 \tMean: -20.3 \tBestMean: -18.8\n",
            "Episode: 2343 \tReward: -19.0 \tMean: -20.1 \tBestMean: -18.8\n",
            "Episode: 2344 \tReward: -21.0 \tMean: -20.1 \tBestMean: -18.8\n",
            "Episode: 2345 \tReward: -21.0 \tMean: -20.1 \tBestMean: -18.8\n",
            "Episode: 2346 \tReward: -20.0 \tMean: -20.0 \tBestMean: -18.8\n",
            "Episode: 2347 \tReward: -19.0 \tMean: -19.9 \tBestMean: -18.8\n",
            "Episode: 2348 \tReward: -19.0 \tMean: -19.9 \tBestMean: -18.8\n",
            "Episode: 2349 \tReward: -21.0 \tMean: -20.1 \tBestMean: -18.8\n",
            "Episode: 2350 \tReward: -19.0 \tMean: -19.9 \tBestMean: -18.8\n",
            "Episode: 2351 \tReward: -20.0 \tMean: -19.9 \tBestMean: -18.8\n",
            "Episode: 2352 \tReward: -19.0 \tMean: -19.8 \tBestMean: -18.8\n",
            "Episode: 2353 \tReward: -21.0 \tMean: -20.0 \tBestMean: -18.8\n",
            "Episode: 2354 \tReward: -20.0 \tMean: -19.9 \tBestMean: -18.8\n",
            "Episode: 2355 \tReward: -20.0 \tMean: -19.8 \tBestMean: -18.8\n",
            "Episode: 2356 \tReward: -19.0 \tMean: -19.7 \tBestMean: -18.8\n",
            "Episode: 2357 \tReward: -20.0 \tMean: -19.8 \tBestMean: -18.8\n",
            "Episode: 2358 \tReward: -20.0 \tMean: -19.9 \tBestMean: -18.8\n",
            "Episode: 2359 \tReward: -19.0 \tMean: -19.7 \tBestMean: -18.8\n",
            "Episode: 2360 \tReward: -21.0 \tMean: -19.9 \tBestMean: -18.8\n",
            "Episode: 2361 \tReward: -20.0 \tMean: -19.9 \tBestMean: -18.8\n",
            "Episode: 2362 \tReward: -20.0 \tMean: -20.0 \tBestMean: -18.8\n",
            "Episode: 2363 \tReward: -20.0 \tMean: -19.9 \tBestMean: -18.8\n",
            "Episode: 2364 \tReward: -18.0 \tMean: -19.7 \tBestMean: -18.8\n",
            "Episode: 2365 \tReward: -21.0 \tMean: -19.8 \tBestMean: -18.8\n",
            "Episode: 2366 \tReward: -21.0 \tMean: -20.0 \tBestMean: -18.8\n",
            "Episode: 2367 \tReward: -20.0 \tMean: -20.0 \tBestMean: -18.8\n",
            "Episode: 2368 \tReward: -21.0 \tMean: -20.1 \tBestMean: -18.8\n",
            "Episode: 2369 \tReward: -21.0 \tMean: -20.3 \tBestMean: -18.8\n",
            "Episode: 2370 \tReward: -20.0 \tMean: -20.2 \tBestMean: -18.8\n",
            "Episode: 2371 \tReward: -20.0 \tMean: -20.2 \tBestMean: -18.8\n",
            "Episode: 2372 \tReward: -20.0 \tMean: -20.2 \tBestMean: -18.8\n",
            "Episode: 2373 \tReward: -19.0 \tMean: -20.1 \tBestMean: -18.8\n",
            "Episode: 2374 \tReward: -20.0 \tMean: -20.3 \tBestMean: -18.8\n",
            "Episode: 2375 \tReward: -19.0 \tMean: -20.1 \tBestMean: -18.8\n",
            "Episode: 2376 \tReward: -21.0 \tMean: -20.1 \tBestMean: -18.8\n",
            "Episode: 2377 \tReward: -20.0 \tMean: -20.1 \tBestMean: -18.8\n",
            "Episode: 2378 \tReward: -20.0 \tMean: -20.0 \tBestMean: -18.8\n",
            "Episode: 2379 \tReward: -21.0 \tMean: -20.0 \tBestMean: -18.8\n",
            "Episode: 2380 \tReward: -20.0 \tMean: -20.0 \tBestMean: -18.8\n",
            "Episode: 2381 \tReward: -20.0 \tMean: -20.0 \tBestMean: -18.8\n",
            "Episode: 2382 \tReward: -21.0 \tMean: -20.1 \tBestMean: -18.8\n",
            "Episode: 2383 \tReward: -20.0 \tMean: -20.2 \tBestMean: -18.8\n",
            "Episode: 2384 \tReward: -20.0 \tMean: -20.2 \tBestMean: -18.8\n",
            "Episode: 2385 \tReward: -19.0 \tMean: -20.2 \tBestMean: -18.8\n",
            "Episode: 2386 \tReward: -21.0 \tMean: -20.2 \tBestMean: -18.8\n",
            "Episode: 2387 \tReward: -20.0 \tMean: -20.2 \tBestMean: -18.8\n",
            "Episode: 2388 \tReward: -21.0 \tMean: -20.3 \tBestMean: -18.8\n",
            "Episode: 2389 \tReward: -21.0 \tMean: -20.3 \tBestMean: -18.8\n",
            "Episode: 2390 \tReward: -20.0 \tMean: -20.3 \tBestMean: -18.8\n",
            "Episode: 2391 \tReward: -20.0 \tMean: -20.3 \tBestMean: -18.8\n",
            "Episode: 2392 \tReward: -21.0 \tMean: -20.3 \tBestMean: -18.8\n",
            "Episode: 2393 \tReward: -21.0 \tMean: -20.4 \tBestMean: -18.8\n",
            "Episode: 2394 \tReward: -18.0 \tMean: -20.2 \tBestMean: -18.8\n",
            "Episode: 2395 \tReward: -19.0 \tMean: -20.2 \tBestMean: -18.8\n",
            "Episode: 2396 \tReward: -21.0 \tMean: -20.2 \tBestMean: -18.8\n",
            "Episode: 2397 \tReward: -21.0 \tMean: -20.3 \tBestMean: -18.8\n",
            "Episode: 2398 \tReward: -20.0 \tMean: -20.2 \tBestMean: -18.8\n",
            "Episode: 2399 \tReward: -21.0 \tMean: -20.2 \tBestMean: -18.8\n",
            "Episode: 2400 \tReward: -19.0 \tMean: -20.1 \tBestMean: -18.8\n",
            "Episode: 2401 \tReward: -18.0 \tMean: -19.9 \tBestMean: -18.8\n",
            "Episode: 2402 \tReward: -20.0 \tMean: -19.8 \tBestMean: -18.8\n",
            "Episode: 2403 \tReward: -21.0 \tMean: -19.8 \tBestMean: -18.8\n",
            "Episode: 2404 \tReward: -19.0 \tMean: -19.9 \tBestMean: -18.8\n",
            "Episode: 2405 \tReward: -18.0 \tMean: -19.8 \tBestMean: -18.8\n",
            "Episode: 2406 \tReward: -20.0 \tMean: -19.7 \tBestMean: -18.8\n",
            "Episode: 2407 \tReward: -21.0 \tMean: -19.7 \tBestMean: -18.8\n",
            "Episode: 2408 \tReward: -21.0 \tMean: -19.8 \tBestMean: -18.8\n",
            "Episode: 2409 \tReward: -21.0 \tMean: -19.8 \tBestMean: -18.8\n",
            "Episode: 2410 \tReward: -21.0 \tMean: -20.0 \tBestMean: -18.8\n",
            "Episode: 2411 \tReward: -21.0 \tMean: -20.3 \tBestMean: -18.8\n",
            "Episode: 2412 \tReward: -21.0 \tMean: -20.4 \tBestMean: -18.8\n",
            "Episode: 2413 \tReward: -20.0 \tMean: -20.3 \tBestMean: -18.8\n",
            "Episode: 2414 \tReward: -20.0 \tMean: -20.4 \tBestMean: -18.8\n",
            "Episode: 2415 \tReward: -20.0 \tMean: -20.6 \tBestMean: -18.8\n",
            "Episode: 2416 \tReward: -19.0 \tMean: -20.5 \tBestMean: -18.8\n",
            "Episode: 2417 \tReward: -20.0 \tMean: -20.4 \tBestMean: -18.8\n",
            "Episode: 2418 \tReward: -21.0 \tMean: -20.4 \tBestMean: -18.8\n",
            "Episode: 2419 \tReward: -20.0 \tMean: -20.3 \tBestMean: -18.8\n",
            "Episode: 2420 \tReward: -20.0 \tMean: -20.2 \tBestMean: -18.8\n",
            "Episode: 2421 \tReward: -20.0 \tMean: -20.1 \tBestMean: -18.8\n",
            "Episode: 2422 \tReward: -21.0 \tMean: -20.1 \tBestMean: -18.8\n",
            "Episode: 2423 \tReward: -20.0 \tMean: -20.1 \tBestMean: -18.8\n",
            "Episode: 2424 \tReward: -21.0 \tMean: -20.2 \tBestMean: -18.8\n",
            "Episode: 2425 \tReward: -18.0 \tMean: -20.0 \tBestMean: -18.8\n",
            "Episode: 2426 \tReward: -19.0 \tMean: -20.0 \tBestMean: -18.8\n",
            "Episode: 2427 \tReward: -20.0 \tMean: -20.0 \tBestMean: -18.8\n",
            "Episode: 2428 \tReward: -19.0 \tMean: -19.8 \tBestMean: -18.8\n",
            "Episode: 2429 \tReward: -21.0 \tMean: -19.9 \tBestMean: -18.8\n",
            "Episode: 2430 \tReward: -20.0 \tMean: -19.9 \tBestMean: -18.8\n",
            "Episode: 2431 \tReward: -20.0 \tMean: -19.9 \tBestMean: -18.8\n",
            "Episode: 2432 \tReward: -19.0 \tMean: -19.7 \tBestMean: -18.8\n",
            "Episode: 2433 \tReward: -21.0 \tMean: -19.8 \tBestMean: -18.8\n",
            "Episode: 2434 \tReward: -21.0 \tMean: -19.8 \tBestMean: -18.8\n",
            "Episode: 2435 \tReward: -19.0 \tMean: -19.9 \tBestMean: -18.8\n",
            "Episode: 2436 \tReward: -21.0 \tMean: -20.1 \tBestMean: -18.8\n",
            "Episode: 2437 \tReward: -21.0 \tMean: -20.2 \tBestMean: -18.8\n",
            "Episode: 2438 \tReward: -20.0 \tMean: -20.3 \tBestMean: -18.8\n",
            "Episode: 2439 \tReward: -20.0 \tMean: -20.2 \tBestMean: -18.8\n",
            "Episode: 2440 \tReward: -19.0 \tMean: -20.1 \tBestMean: -18.8\n",
            "Episode: 2441 \tReward: -18.0 \tMean: -19.9 \tBestMean: -18.8\n",
            "Episode: 2442 \tReward: -21.0 \tMean: -20.1 \tBestMean: -18.8\n",
            "Episode: 2443 \tReward: -19.0 \tMean: -19.9 \tBestMean: -18.8\n",
            "Episode: 2444 \tReward: -21.0 \tMean: -19.9 \tBestMean: -18.8\n",
            "Episode: 2445 \tReward: -21.0 \tMean: -20.1 \tBestMean: -18.8\n",
            "Episode: 2446 \tReward: -19.0 \tMean: -19.9 \tBestMean: -18.8\n",
            "Episode: 2447 \tReward: -20.0 \tMean: -19.8 \tBestMean: -18.8\n",
            "Episode: 2448 \tReward: -21.0 \tMean: -19.9 \tBestMean: -18.8\n",
            "Episode: 2449 \tReward: -20.0 \tMean: -19.9 \tBestMean: -18.8\n",
            "Episode: 2450 \tReward: -19.0 \tMean: -19.9 \tBestMean: -18.8\n",
            "Episode: 2451 \tReward: -21.0 \tMean: -20.2 \tBestMean: -18.8\n",
            "Episode: 2452 \tReward: -20.0 \tMean: -20.1 \tBestMean: -18.8\n",
            "Episode: 2453 \tReward: -21.0 \tMean: -20.3 \tBestMean: -18.8\n",
            "Episode: 2454 \tReward: -19.0 \tMean: -20.1 \tBestMean: -18.8\n",
            "Episode: 2455 \tReward: -20.0 \tMean: -20.0 \tBestMean: -18.8\n",
            "Episode: 2456 \tReward: -21.0 \tMean: -20.2 \tBestMean: -18.8\n",
            "Episode: 2457 \tReward: -21.0 \tMean: -20.3 \tBestMean: -18.8\n",
            "Episode: 2458 \tReward: -20.0 \tMean: -20.2 \tBestMean: -18.8\n",
            "Episode: 2459 \tReward: -20.0 \tMean: -20.2 \tBestMean: -18.8\n",
            "Episode: 2460 \tReward: -21.0 \tMean: -20.4 \tBestMean: -18.8\n",
            "Episode: 2461 \tReward: -21.0 \tMean: -20.4 \tBestMean: -18.8\n",
            "Episode: 2462 \tReward: -21.0 \tMean: -20.5 \tBestMean: -18.8\n",
            "Episode: 2463 \tReward: -20.0 \tMean: -20.4 \tBestMean: -18.8\n",
            "Episode: 2464 \tReward: -20.0 \tMean: -20.5 \tBestMean: -18.8\n",
            "Episode: 2465 \tReward: -20.0 \tMean: -20.5 \tBestMean: -18.8\n",
            "Episode: 2466 \tReward: -21.0 \tMean: -20.5 \tBestMean: -18.8\n",
            "Episode: 2467 \tReward: -20.0 \tMean: -20.4 \tBestMean: -18.8\n",
            "Episode: 2468 \tReward: -18.0 \tMean: -20.2 \tBestMean: -18.8\n",
            "Episode: 2469 \tReward: -19.0 \tMean: -20.1 \tBestMean: -18.8\n",
            "Episode: 2470 \tReward: -20.0 \tMean: -20.0 \tBestMean: -18.8\n",
            "Episode: 2471 \tReward: -20.0 \tMean: -19.9 \tBestMean: -18.8\n",
            "Episode: 2472 \tReward: -19.0 \tMean: -19.7 \tBestMean: -18.8\n",
            "Episode: 2473 \tReward: -20.0 \tMean: -19.7 \tBestMean: -18.8\n",
            "Episode: 2474 \tReward: -20.0 \tMean: -19.7 \tBestMean: -18.8\n",
            "Episode: 2475 \tReward: -21.0 \tMean: -19.8 \tBestMean: -18.8\n",
            "Episode: 2476 \tReward: -21.0 \tMean: -19.8 \tBestMean: -18.8\n",
            "Episode: 2477 \tReward: -20.0 \tMean: -19.8 \tBestMean: -18.8\n",
            "Episode: 2478 \tReward: -21.0 \tMean: -20.1 \tBestMean: -18.8\n",
            "Episode: 2479 \tReward: -21.0 \tMean: -20.3 \tBestMean: -18.8\n",
            "Episode: 2480 \tReward: -18.0 \tMean: -20.1 \tBestMean: -18.8\n",
            "Episode: 2481 \tReward: -18.0 \tMean: -19.9 \tBestMean: -18.8\n",
            "Episode: 2482 \tReward: -21.0 \tMean: -20.1 \tBestMean: -18.8\n",
            "Episode: 2483 \tReward: -20.0 \tMean: -20.1 \tBestMean: -18.8\n",
            "Episode: 2484 \tReward: -20.0 \tMean: -20.1 \tBestMean: -18.8\n",
            "Episode: 2485 \tReward: -20.0 \tMean: -20.0 \tBestMean: -18.8\n",
            "Episode: 2486 \tReward: -19.0 \tMean: -19.8 \tBestMean: -18.8\n",
            "Episode: 2487 \tReward: -21.0 \tMean: -19.9 \tBestMean: -18.8\n",
            "Episode: 2488 \tReward: -20.0 \tMean: -19.8 \tBestMean: -18.8\n",
            "Episode: 2489 \tReward: -18.0 \tMean: -19.5 \tBestMean: -18.8\n",
            "Episode: 2490 \tReward: -21.0 \tMean: -19.8 \tBestMean: -18.8\n",
            "Episode: 2491 \tReward: -21.0 \tMean: -20.1 \tBestMean: -18.8\n",
            "Episode: 2492 \tReward: -21.0 \tMean: -20.1 \tBestMean: -18.8\n",
            "Episode: 2493 \tReward: -20.0 \tMean: -20.1 \tBestMean: -18.8\n",
            "Episode: 2494 \tReward: -19.0 \tMean: -20.0 \tBestMean: -18.8\n",
            "Episode: 2495 \tReward: -21.0 \tMean: -20.1 \tBestMean: -18.8\n",
            "Episode: 2496 \tReward: -19.0 \tMean: -20.1 \tBestMean: -18.8\n",
            "Episode: 2497 \tReward: -20.0 \tMean: -20.0 \tBestMean: -18.8\n",
            "Episode: 2498 \tReward: -21.0 \tMean: -20.1 \tBestMean: -18.8\n",
            "Episode: 2499 \tReward: -20.0 \tMean: -20.3 \tBestMean: -18.8\n",
            "Episode: 2500 \tReward: -18.0 \tMean: -20.0 \tBestMean: -18.8\n",
            "Episode: 2501 \tReward: -19.0 \tMean: -19.8 \tBestMean: -18.8\n",
            "Episode: 2502 \tReward: -20.0 \tMean: -19.7 \tBestMean: -18.8\n",
            "Episode: 2503 \tReward: -20.0 \tMean: -19.7 \tBestMean: -18.8\n",
            "Episode: 2504 \tReward: -20.0 \tMean: -19.8 \tBestMean: -18.8\n",
            "Episode: 2505 \tReward: -18.0 \tMean: -19.5 \tBestMean: -18.8\n",
            "Episode: 2506 \tReward: -21.0 \tMean: -19.7 \tBestMean: -18.8\n",
            "Episode: 2507 \tReward: -20.0 \tMean: -19.7 \tBestMean: -18.8\n",
            "Episode: 2508 \tReward: -19.0 \tMean: -19.5 \tBestMean: -18.8\n",
            "Episode: 2509 \tReward: -20.0 \tMean: -19.5 \tBestMean: -18.8\n",
            "Episode: 2510 \tReward: -20.0 \tMean: -19.7 \tBestMean: -18.8\n",
            "Episode: 2511 \tReward: -21.0 \tMean: -19.9 \tBestMean: -18.8\n",
            "Episode: 2512 \tReward: -20.0 \tMean: -19.9 \tBestMean: -18.8\n",
            "Episode: 2513 \tReward: -21.0 \tMean: -20.0 \tBestMean: -18.8\n",
            "Episode: 2514 \tReward: -21.0 \tMean: -20.1 \tBestMean: -18.8\n",
            "Episode: 2515 \tReward: -21.0 \tMean: -20.4 \tBestMean: -18.8\n",
            "Episode: 2516 \tReward: -19.0 \tMean: -20.2 \tBestMean: -18.8\n",
            "Episode: 2517 \tReward: -19.0 \tMean: -20.1 \tBestMean: -18.8\n",
            "Episode: 2518 \tReward: -19.0 \tMean: -20.1 \tBestMean: -18.8\n",
            "Episode: 2519 \tReward: -21.0 \tMean: -20.2 \tBestMean: -18.8\n",
            "Episode: 2520 \tReward: -19.0 \tMean: -20.1 \tBestMean: -18.8\n",
            "Episode: 2521 \tReward: -18.0 \tMean: -19.8 \tBestMean: -18.8\n",
            "Episode: 2522 \tReward: -21.0 \tMean: -19.9 \tBestMean: -18.8\n",
            "Episode: 2523 \tReward: -19.0 \tMean: -19.7 \tBestMean: -18.8\n",
            "Episode: 2524 \tReward: -21.0 \tMean: -19.7 \tBestMean: -18.8\n",
            "Episode: 2525 \tReward: -19.0 \tMean: -19.5 \tBestMean: -18.8\n",
            "Episode: 2526 \tReward: -20.0 \tMean: -19.6 \tBestMean: -18.8\n",
            "Episode: 2527 \tReward: -21.0 \tMean: -19.8 \tBestMean: -18.8\n",
            "Episode: 2528 \tReward: -19.0 \tMean: -19.8 \tBestMean: -18.8\n",
            "Episode: 2529 \tReward: -21.0 \tMean: -19.8 \tBestMean: -18.8\n",
            "Episode: 2530 \tReward: -21.0 \tMean: -20.0 \tBestMean: -18.8\n",
            "Episode: 2531 \tReward: -19.0 \tMean: -20.1 \tBestMean: -18.8\n",
            "Episode: 2532 \tReward: -18.0 \tMean: -19.8 \tBestMean: -18.8\n",
            "Episode: 2533 \tReward: -19.0 \tMean: -19.8 \tBestMean: -18.8\n",
            "Episode: 2534 \tReward: -21.0 \tMean: -19.8 \tBestMean: -18.8\n",
            "Episode: 2535 \tReward: -20.0 \tMean: -19.9 \tBestMean: -18.8\n",
            "Episode: 2536 \tReward: -20.0 \tMean: -19.9 \tBestMean: -18.8\n",
            "Episode: 2537 \tReward: -20.0 \tMean: -19.8 \tBestMean: -18.8\n",
            "Episode: 2538 \tReward: -18.0 \tMean: -19.7 \tBestMean: -18.8\n",
            "Episode: 2539 \tReward: -20.0 \tMean: -19.6 \tBestMean: -18.8\n",
            "Episode: 2540 \tReward: -21.0 \tMean: -19.6 \tBestMean: -18.8\n",
            "Episode: 2541 \tReward: -18.0 \tMean: -19.5 \tBestMean: -18.8\n",
            "Episode: 2542 \tReward: -19.0 \tMean: -19.6 \tBestMean: -18.8\n",
            "Episode: 2543 \tReward: -20.0 \tMean: -19.7 \tBestMean: -18.8\n",
            "Episode: 2544 \tReward: -20.0 \tMean: -19.6 \tBestMean: -18.8\n",
            "Episode: 2545 \tReward: -19.0 \tMean: -19.5 \tBestMean: -18.8\n",
            "Episode: 2546 \tReward: -21.0 \tMean: -19.6 \tBestMean: -18.8\n",
            "Episode: 2547 \tReward: -21.0 \tMean: -19.7 \tBestMean: -18.8\n",
            "Episode: 2548 \tReward: -19.0 \tMean: -19.8 \tBestMean: -18.8\n",
            "Episode: 2549 \tReward: -21.0 \tMean: -19.9 \tBestMean: -18.8\n",
            "Episode: 2550 \tReward: -18.0 \tMean: -19.6 \tBestMean: -18.8\n",
            "Episode: 2551 \tReward: -19.0 \tMean: -19.7 \tBestMean: -18.8\n",
            "Episode: 2552 \tReward: -18.0 \tMean: -19.6 \tBestMean: -18.8\n",
            "Episode: 2553 \tReward: -18.0 \tMean: -19.4 \tBestMean: -18.8\n",
            "Episode: 2554 \tReward: -19.0 \tMean: -19.3 \tBestMean: -18.8\n",
            "Episode: 2555 \tReward: -21.0 \tMean: -19.5 \tBestMean: -18.8\n",
            "Episode: 2556 \tReward: -21.0 \tMean: -19.5 \tBestMean: -18.8\n",
            "Episode: 2557 \tReward: -21.0 \tMean: -19.5 \tBestMean: -18.8\n",
            "Episode: 2558 \tReward: -21.0 \tMean: -19.7 \tBestMean: -18.8\n",
            "Episode: 2559 \tReward: -20.0 \tMean: -19.6 \tBestMean: -18.8\n",
            "Episode: 2560 \tReward: -20.0 \tMean: -19.8 \tBestMean: -18.8\n",
            "Episode: 2561 \tReward: -20.0 \tMean: -19.9 \tBestMean: -18.8\n",
            "Episode: 2562 \tReward: -20.0 \tMean: -20.1 \tBestMean: -18.8\n",
            "Episode: 2563 \tReward: -21.0 \tMean: -20.4 \tBestMean: -18.8\n",
            "Episode: 2564 \tReward: -21.0 \tMean: -20.6 \tBestMean: -18.8\n",
            "Episode: 2565 \tReward: -21.0 \tMean: -20.6 \tBestMean: -18.8\n",
            "Episode: 2566 \tReward: -21.0 \tMean: -20.6 \tBestMean: -18.8\n",
            "Episode: 2567 \tReward: -21.0 \tMean: -20.6 \tBestMean: -18.8\n",
            "Episode: 2568 \tReward: -20.0 \tMean: -20.5 \tBestMean: -18.8\n",
            "Episode: 2569 \tReward: -21.0 \tMean: -20.6 \tBestMean: -18.8\n",
            "Episode: 2570 \tReward: -20.0 \tMean: -20.6 \tBestMean: -18.8\n",
            "Episode: 2571 \tReward: -18.0 \tMean: -20.4 \tBestMean: -18.8\n",
            "Episode: 2572 \tReward: -18.0 \tMean: -20.2 \tBestMean: -18.8\n",
            "Episode: 2573 \tReward: -21.0 \tMean: -20.2 \tBestMean: -18.8\n",
            "Episode: 2574 \tReward: -18.0 \tMean: -19.9 \tBestMean: -18.8\n",
            "Episode: 2575 \tReward: -21.0 \tMean: -19.9 \tBestMean: -18.8\n",
            "Episode: 2576 \tReward: -21.0 \tMean: -19.9 \tBestMean: -18.8\n",
            "Episode: 2577 \tReward: -21.0 \tMean: -19.9 \tBestMean: -18.8\n",
            "Episode: 2578 \tReward: -20.0 \tMean: -19.9 \tBestMean: -18.8\n",
            "Episode: 2579 \tReward: -20.0 \tMean: -19.8 \tBestMean: -18.8\n",
            "Episode: 2580 \tReward: -21.0 \tMean: -19.9 \tBestMean: -18.8\n",
            "Episode: 2581 \tReward: -21.0 \tMean: -20.2 \tBestMean: -18.8\n",
            "Episode: 2582 \tReward: -21.0 \tMean: -20.5 \tBestMean: -18.8\n",
            "Episode: 2583 \tReward: -21.0 \tMean: -20.5 \tBestMean: -18.8\n",
            "Episode: 2584 \tReward: -20.0 \tMean: -20.7 \tBestMean: -18.8\n",
            "Episode: 2585 \tReward: -20.0 \tMean: -20.6 \tBestMean: -18.8\n",
            "Episode: 2586 \tReward: -20.0 \tMean: -20.5 \tBestMean: -18.8\n",
            "Episode: 2587 \tReward: -21.0 \tMean: -20.5 \tBestMean: -18.8\n",
            "Episode: 2588 \tReward: -18.0 \tMean: -20.3 \tBestMean: -18.8\n",
            "Episode: 2589 \tReward: -21.0 \tMean: -20.4 \tBestMean: -18.8\n",
            "Episode: 2590 \tReward: -21.0 \tMean: -20.4 \tBestMean: -18.8\n",
            "Episode: 2591 \tReward: -21.0 \tMean: -20.4 \tBestMean: -18.8\n",
            "Episode: 2592 \tReward: -20.0 \tMean: -20.3 \tBestMean: -18.8\n",
            "Episode: 2593 \tReward: -21.0 \tMean: -20.3 \tBestMean: -18.8\n",
            "Episode: 2594 \tReward: -20.0 \tMean: -20.3 \tBestMean: -18.8\n",
            "Episode: 2595 \tReward: -21.0 \tMean: -20.4 \tBestMean: -18.8\n",
            "Episode: 2596 \tReward: -20.0 \tMean: -20.4 \tBestMean: -18.8\n",
            "Episode: 2597 \tReward: -20.0 \tMean: -20.3 \tBestMean: -18.8\n",
            "Episode: 2598 \tReward: -20.0 \tMean: -20.5 \tBestMean: -18.8\n",
            "Episode: 2599 \tReward: -20.0 \tMean: -20.4 \tBestMean: -18.8\n",
            "Episode: 2600 \tReward: -20.0 \tMean: -20.3 \tBestMean: -18.8\n",
            "Episode: 2601 \tReward: -20.0 \tMean: -20.2 \tBestMean: -18.8\n",
            "Episode: 2602 \tReward: -20.0 \tMean: -20.2 \tBestMean: -18.8\n",
            "Episode: 2603 \tReward: -21.0 \tMean: -20.2 \tBestMean: -18.8\n",
            "Episode: 2604 \tReward: -20.0 \tMean: -20.2 \tBestMean: -18.8\n",
            "Episode: 2605 \tReward: -19.0 \tMean: -20.0 \tBestMean: -18.8\n",
            "Episode: 2606 \tReward: -21.0 \tMean: -20.1 \tBestMean: -18.8\n",
            "Episode: 2607 \tReward: -21.0 \tMean: -20.2 \tBestMean: -18.8\n",
            "Episode: 2608 \tReward: -19.0 \tMean: -20.1 \tBestMean: -18.8\n",
            "Episode: 2609 \tReward: -21.0 \tMean: -20.2 \tBestMean: -18.8\n",
            "Episode: 2610 \tReward: -21.0 \tMean: -20.3 \tBestMean: -18.8\n",
            "Episode: 2611 \tReward: -21.0 \tMean: -20.4 \tBestMean: -18.8\n",
            "Episode: 2612 \tReward: -20.0 \tMean: -20.4 \tBestMean: -18.8\n",
            "Episode: 2613 \tReward: -20.0 \tMean: -20.3 \tBestMean: -18.8\n",
            "Episode: 2614 \tReward: -20.0 \tMean: -20.3 \tBestMean: -18.8\n",
            "Episode: 2615 \tReward: -20.0 \tMean: -20.4 \tBestMean: -18.8\n",
            "Episode: 2616 \tReward: -21.0 \tMean: -20.4 \tBestMean: -18.8\n",
            "Episode: 2617 \tReward: -19.0 \tMean: -20.2 \tBestMean: -18.8\n",
            "Episode: 2618 \tReward: -20.0 \tMean: -20.3 \tBestMean: -18.8\n",
            "Episode: 2619 \tReward: -20.0 \tMean: -20.2 \tBestMean: -18.8\n",
            "Episode: 2620 \tReward: -21.0 \tMean: -20.2 \tBestMean: -18.8\n",
            "Episode: 2621 \tReward: -21.0 \tMean: -20.2 \tBestMean: -18.8\n",
            "Episode: 2622 \tReward: -19.0 \tMean: -20.1 \tBestMean: -18.8\n",
            "Episode: 2623 \tReward: -18.0 \tMean: -19.9 \tBestMean: -18.8\n",
            "Episode: 2624 \tReward: -21.0 \tMean: -20.0 \tBestMean: -18.8\n",
            "Episode: 2625 \tReward: -19.0 \tMean: -19.9 \tBestMean: -18.8\n",
            "Episode: 2626 \tReward: -21.0 \tMean: -19.9 \tBestMean: -18.8\n",
            "Episode: 2627 \tReward: -21.0 \tMean: -20.1 \tBestMean: -18.8\n",
            "Episode: 2628 \tReward: -21.0 \tMean: -20.2 \tBestMean: -18.8\n",
            "Episode: 2629 \tReward: -20.0 \tMean: -20.2 \tBestMean: -18.8\n",
            "Episode: 2630 \tReward: -20.0 \tMean: -20.1 \tBestMean: -18.8\n",
            "Episode: 2631 \tReward: -21.0 \tMean: -20.1 \tBestMean: -18.8\n",
            "Episode: 2632 \tReward: -21.0 \tMean: -20.3 \tBestMean: -18.8\n",
            "Episode: 2633 \tReward: -20.0 \tMean: -20.5 \tBestMean: -18.8\n",
            "Episode: 2634 \tReward: -20.0 \tMean: -20.4 \tBestMean: -18.8\n",
            "Episode: 2635 \tReward: -21.0 \tMean: -20.6 \tBestMean: -18.8\n",
            "Episode: 2636 \tReward: -21.0 \tMean: -20.6 \tBestMean: -18.8\n",
            "Episode: 2637 \tReward: -20.0 \tMean: -20.5 \tBestMean: -18.8\n",
            "Episode: 2638 \tReward: -20.0 \tMean: -20.4 \tBestMean: -18.8\n",
            "Episode: 2639 \tReward: -20.0 \tMean: -20.4 \tBestMean: -18.8\n",
            "Episode: 2640 \tReward: -20.0 \tMean: -20.4 \tBestMean: -18.8\n",
            "Episode: 2641 \tReward: -20.0 \tMean: -20.3 \tBestMean: -18.8\n",
            "Episode: 2642 \tReward: -21.0 \tMean: -20.3 \tBestMean: -18.8\n",
            "Episode: 2643 \tReward: -17.0 \tMean: -20.0 \tBestMean: -18.8\n",
            "Episode: 2644 \tReward: -20.0 \tMean: -20.0 \tBestMean: -18.8\n",
            "Episode: 2645 \tReward: -21.0 \tMean: -20.0 \tBestMean: -18.8\n",
            "Episode: 2646 \tReward: -20.0 \tMean: -19.9 \tBestMean: -18.8\n",
            "Episode: 2647 \tReward: -18.0 \tMean: -19.7 \tBestMean: -18.8\n",
            "Episode: 2648 \tReward: -21.0 \tMean: -19.8 \tBestMean: -18.8\n",
            "Episode: 2649 \tReward: -20.0 \tMean: -19.8 \tBestMean: -18.8\n",
            "Episode: 2650 \tReward: -19.0 \tMean: -19.7 \tBestMean: -18.8\n",
            "Episode: 2651 \tReward: -20.0 \tMean: -19.7 \tBestMean: -18.8\n",
            "Episode: 2652 \tReward: -19.0 \tMean: -19.5 \tBestMean: -18.8\n",
            "Episode: 2653 \tReward: -21.0 \tMean: -19.9 \tBestMean: -18.8\n",
            "Episode: 2654 \tReward: -21.0 \tMean: -20.0 \tBestMean: -18.8\n",
            "Episode: 2655 \tReward: -21.0 \tMean: -20.0 \tBestMean: -18.8\n",
            "Episode: 2656 \tReward: -21.0 \tMean: -20.1 \tBestMean: -18.8\n",
            "Episode: 2657 \tReward: -20.0 \tMean: -20.3 \tBestMean: -18.8\n",
            "Episode: 2658 \tReward: -20.0 \tMean: -20.2 \tBestMean: -18.8\n",
            "Episode: 2659 \tReward: -21.0 \tMean: -20.3 \tBestMean: -18.8\n",
            "Episode: 2660 \tReward: -20.0 \tMean: -20.4 \tBestMean: -18.8\n",
            "Episode: 2661 \tReward: -21.0 \tMean: -20.5 \tBestMean: -18.8\n",
            "Episode: 2662 \tReward: -21.0 \tMean: -20.7 \tBestMean: -18.8\n",
            "Episode: 2663 \tReward: -20.0 \tMean: -20.6 \tBestMean: -18.8\n",
            "Episode: 2664 \tReward: -21.0 \tMean: -20.6 \tBestMean: -18.8\n",
            "Episode: 2665 \tReward: -21.0 \tMean: -20.6 \tBestMean: -18.8\n",
            "Episode: 2666 \tReward: -21.0 \tMean: -20.6 \tBestMean: -18.8\n",
            "Episode: 2667 \tReward: -21.0 \tMean: -20.7 \tBestMean: -18.8\n",
            "Episode: 2668 \tReward: -20.0 \tMean: -20.7 \tBestMean: -18.8\n",
            "Episode: 2669 \tReward: -21.0 \tMean: -20.7 \tBestMean: -18.8\n",
            "Episode: 2670 \tReward: -20.0 \tMean: -20.7 \tBestMean: -18.8\n",
            "Episode: 2671 \tReward: -21.0 \tMean: -20.7 \tBestMean: -18.8\n",
            "Episode: 2672 \tReward: -20.0 \tMean: -20.6 \tBestMean: -18.8\n",
            "Episode: 2673 \tReward: -21.0 \tMean: -20.7 \tBestMean: -18.8\n",
            "Episode: 2674 \tReward: -19.0 \tMean: -20.5 \tBestMean: -18.8\n",
            "Episode: 2675 \tReward: -20.0 \tMean: -20.4 \tBestMean: -18.8\n",
            "Episode: 2676 \tReward: -20.0 \tMean: -20.3 \tBestMean: -18.8\n",
            "Episode: 2677 \tReward: -19.0 \tMean: -20.1 \tBestMean: -18.8\n",
            "Episode: 2678 \tReward: -20.0 \tMean: -20.1 \tBestMean: -18.8\n",
            "Episode: 2679 \tReward: -21.0 \tMean: -20.1 \tBestMean: -18.8\n",
            "Episode: 2680 \tReward: -20.0 \tMean: -20.1 \tBestMean: -18.8\n",
            "Episode: 2681 \tReward: -18.0 \tMean: -19.8 \tBestMean: -18.8\n",
            "Episode: 2682 \tReward: -20.0 \tMean: -19.8 \tBestMean: -18.8\n",
            "Episode: 2683 \tReward: -21.0 \tMean: -19.8 \tBestMean: -18.8\n",
            "Episode: 2684 \tReward: -21.0 \tMean: -20.0 \tBestMean: -18.8\n",
            "Episode: 2685 \tReward: -21.0 \tMean: -20.1 \tBestMean: -18.8\n",
            "Episode: 2686 \tReward: -19.0 \tMean: -20.0 \tBestMean: -18.8\n",
            "Episode: 2687 \tReward: -19.0 \tMean: -20.0 \tBestMean: -18.8\n",
            "Episode: 2688 \tReward: -20.0 \tMean: -20.0 \tBestMean: -18.8\n",
            "Episode: 2689 \tReward: -20.0 \tMean: -19.9 \tBestMean: -18.8\n",
            "Episode: 2690 \tReward: -20.0 \tMean: -19.9 \tBestMean: -18.8\n",
            "Episode: 2691 \tReward: -21.0 \tMean: -20.2 \tBestMean: -18.8\n",
            "Episode: 2692 \tReward: -20.0 \tMean: -20.2 \tBestMean: -18.8\n",
            "Episode: 2693 \tReward: -20.0 \tMean: -20.1 \tBestMean: -18.8\n",
            "Episode: 2694 \tReward: -20.0 \tMean: -20.0 \tBestMean: -18.8\n",
            "Episode: 2695 \tReward: -19.0 \tMean: -19.8 \tBestMean: -18.8\n",
            "Episode: 2696 \tReward: -20.0 \tMean: -19.9 \tBestMean: -18.8\n",
            "Episode: 2697 \tReward: -21.0 \tMean: -20.1 \tBestMean: -18.8\n",
            "Episode: 2698 \tReward: -19.0 \tMean: -20.0 \tBestMean: -18.8\n",
            "Episode: 2699 \tReward: -21.0 \tMean: -20.1 \tBestMean: -18.8\n",
            "Episode: 2700 \tReward: -20.0 \tMean: -20.1 \tBestMean: -18.8\n",
            "Episode: 2701 \tReward: -21.0 \tMean: -20.1 \tBestMean: -18.8\n",
            "Episode: 2702 \tReward: -19.0 \tMean: -20.0 \tBestMean: -18.8\n",
            "Episode: 2703 \tReward: -20.0 \tMean: -20.0 \tBestMean: -18.8\n",
            "Episode: 2704 \tReward: -21.0 \tMean: -20.1 \tBestMean: -18.8\n",
            "Episode: 2705 \tReward: -21.0 \tMean: -20.3 \tBestMean: -18.8\n",
            "Episode: 2706 \tReward: -21.0 \tMean: -20.4 \tBestMean: -18.8\n",
            "Episode: 2707 \tReward: -20.0 \tMean: -20.3 \tBestMean: -18.8\n",
            "Episode: 2708 \tReward: -21.0 \tMean: -20.5 \tBestMean: -18.8\n",
            "Episode: 2709 \tReward: -21.0 \tMean: -20.5 \tBestMean: -18.8\n",
            "Episode: 2710 \tReward: -18.0 \tMean: -20.3 \tBestMean: -18.8\n",
            "Episode: 2711 \tReward: -20.0 \tMean: -20.2 \tBestMean: -18.8\n",
            "Episode: 2712 \tReward: -20.0 \tMean: -20.3 \tBestMean: -18.8\n",
            "Episode: 2713 \tReward: -20.0 \tMean: -20.3 \tBestMean: -18.8\n",
            "Episode: 2714 \tReward: -20.0 \tMean: -20.2 \tBestMean: -18.8\n",
            "Episode: 2715 \tReward: -20.0 \tMean: -20.1 \tBestMean: -18.8\n",
            "Episode: 2716 \tReward: -20.0 \tMean: -20.0 \tBestMean: -18.8\n",
            "Episode: 2717 \tReward: -21.0 \tMean: -20.1 \tBestMean: -18.8\n",
            "Episode: 2718 \tReward: -19.0 \tMean: -19.9 \tBestMean: -18.8\n",
            "Episode: 2719 \tReward: -21.0 \tMean: -19.9 \tBestMean: -18.8\n",
            "Episode: 2720 \tReward: -21.0 \tMean: -20.2 \tBestMean: -18.8\n",
            "Episode: 2721 \tReward: -20.0 \tMean: -20.2 \tBestMean: -18.8\n",
            "Episode: 2722 \tReward: -20.0 \tMean: -20.2 \tBestMean: -18.8\n",
            "Episode: 2723 \tReward: -21.0 \tMean: -20.3 \tBestMean: -18.8\n",
            "Episode: 2724 \tReward: -19.0 \tMean: -20.2 \tBestMean: -18.8\n",
            "Episode: 2725 \tReward: -18.0 \tMean: -20.0 \tBestMean: -18.8\n",
            "Episode: 2726 \tReward: -15.0 \tMean: -19.5 \tBestMean: -18.8\n",
            "Episode: 2727 \tReward: -18.0 \tMean: -19.2 \tBestMean: -18.8\n",
            "Episode: 2728 \tReward: -20.0 \tMean: -19.3 \tBestMean: -18.8\n",
            "Episode: 2729 \tReward: -19.0 \tMean: -19.1 \tBestMean: -18.8\n",
            "Episode: 2730 \tReward: -21.0 \tMean: -19.1 \tBestMean: -18.8\n",
            "Episode: 2731 \tReward: -19.0 \tMean: -19.0 \tBestMean: -18.8\n",
            "Episode: 2732 \tReward: -21.0 \tMean: -19.1 \tBestMean: -18.8\n",
            "Episode: 2733 \tReward: -21.0 \tMean: -19.1 \tBestMean: -18.8\n",
            "Episode: 2734 \tReward: -20.0 \tMean: -19.2 \tBestMean: -18.8\n",
            "Episode: 2735 \tReward: -20.0 \tMean: -19.4 \tBestMean: -18.8\n",
            "Episode: 2736 \tReward: -19.0 \tMean: -19.8 \tBestMean: -18.8\n",
            "Episode: 2737 \tReward: -20.0 \tMean: -20.0 \tBestMean: -18.8\n",
            "Episode: 2738 \tReward: -20.0 \tMean: -20.0 \tBestMean: -18.8\n",
            "Episode: 2739 \tReward: -20.0 \tMean: -20.1 \tBestMean: -18.8\n",
            "Episode: 2740 \tReward: -21.0 \tMean: -20.1 \tBestMean: -18.8\n",
            "Episode: 2741 \tReward: -20.0 \tMean: -20.2 \tBestMean: -18.8\n",
            "Episode: 2742 \tReward: -20.0 \tMean: -20.1 \tBestMean: -18.8\n",
            "Episode: 2743 \tReward: -21.0 \tMean: -20.1 \tBestMean: -18.8\n",
            "Episode: 2744 \tReward: -20.0 \tMean: -20.1 \tBestMean: -18.8\n",
            "Episode: 2745 \tReward: -21.0 \tMean: -20.2 \tBestMean: -18.8\n",
            "Episode: 2746 \tReward: -20.0 \tMean: -20.3 \tBestMean: -18.8\n",
            "Episode: 2747 \tReward: -21.0 \tMean: -20.4 \tBestMean: -18.8\n",
            "Episode: 2748 \tReward: -21.0 \tMean: -20.5 \tBestMean: -18.8\n",
            "Episode: 2749 \tReward: -21.0 \tMean: -20.6 \tBestMean: -18.8\n",
            "Episode: 2750 \tReward: -19.0 \tMean: -20.4 \tBestMean: -18.8\n",
            "Episode: 2751 \tReward: -21.0 \tMean: -20.5 \tBestMean: -18.8\n",
            "Episode: 2752 \tReward: -20.0 \tMean: -20.5 \tBestMean: -18.8\n",
            "Episode: 2753 \tReward: -20.0 \tMean: -20.4 \tBestMean: -18.8\n",
            "Episode: 2754 \tReward: -20.0 \tMean: -20.4 \tBestMean: -18.8\n",
            "Episode: 2755 \tReward: -20.0 \tMean: -20.3 \tBestMean: -18.8\n",
            "Episode: 2756 \tReward: -20.0 \tMean: -20.3 \tBestMean: -18.8\n",
            "Episode: 2757 \tReward: -19.0 \tMean: -20.1 \tBestMean: -18.8\n",
            "Episode: 2758 \tReward: -21.0 \tMean: -20.1 \tBestMean: -18.8\n",
            "Episode: 2759 \tReward: -21.0 \tMean: -20.1 \tBestMean: -18.8\n",
            "Episode: 2760 \tReward: -19.0 \tMean: -20.1 \tBestMean: -18.8\n",
            "Episode: 2761 \tReward: -20.0 \tMean: -20.0 \tBestMean: -18.8\n",
            "Episode: 2762 \tReward: -21.0 \tMean: -20.1 \tBestMean: -18.8\n",
            "Episode: 2763 \tReward: -21.0 \tMean: -20.2 \tBestMean: -18.8\n",
            "Episode: 2764 \tReward: -20.0 \tMean: -20.2 \tBestMean: -18.8\n",
            "Episode: 2765 \tReward: -21.0 \tMean: -20.3 \tBestMean: -18.8\n",
            "Episode: 2766 \tReward: -20.0 \tMean: -20.3 \tBestMean: -18.8\n",
            "Episode: 2767 \tReward: -19.0 \tMean: -20.3 \tBestMean: -18.8\n",
            "Episode: 2768 \tReward: -21.0 \tMean: -20.3 \tBestMean: -18.8\n",
            "Episode: 2769 \tReward: -20.0 \tMean: -20.2 \tBestMean: -18.8\n",
            "Episode: 2770 \tReward: -19.0 \tMean: -20.2 \tBestMean: -18.8\n",
            "Episode: 2771 \tReward: -20.0 \tMean: -20.2 \tBestMean: -18.8\n",
            "Episode: 2772 \tReward: -21.0 \tMean: -20.2 \tBestMean: -18.8\n",
            "Episode: 2773 \tReward: -19.0 \tMean: -20.0 \tBestMean: -18.8\n",
            "Episode: 2774 \tReward: -21.0 \tMean: -20.1 \tBestMean: -18.8\n",
            "Episode: 2775 \tReward: -21.0 \tMean: -20.1 \tBestMean: -18.8\n",
            "Episode: 2776 \tReward: -20.0 \tMean: -20.1 \tBestMean: -18.8\n",
            "Episode: 2777 \tReward: -20.0 \tMean: -20.2 \tBestMean: -18.8\n",
            "Episode: 2778 \tReward: -19.0 \tMean: -20.0 \tBestMean: -18.8\n",
            "Episode: 2779 \tReward: -21.0 \tMean: -20.1 \tBestMean: -18.8\n",
            "Episode: 2780 \tReward: -20.0 \tMean: -20.2 \tBestMean: -18.8\n",
            "Episode: 2781 \tReward: -21.0 \tMean: -20.3 \tBestMean: -18.8\n",
            "Episode: 2782 \tReward: -20.0 \tMean: -20.2 \tBestMean: -18.8\n",
            "Episode: 2783 \tReward: -21.0 \tMean: -20.4 \tBestMean: -18.8\n",
            "Episode: 2784 \tReward: -20.0 \tMean: -20.3 \tBestMean: -18.8\n",
            "Episode: 2785 \tReward: -21.0 \tMean: -20.3 \tBestMean: -18.8\n",
            "Episode: 2786 \tReward: -18.0 \tMean: -20.1 \tBestMean: -18.8\n",
            "Episode: 2787 \tReward: -21.0 \tMean: -20.2 \tBestMean: -18.8\n",
            "Episode: 2788 \tReward: -21.0 \tMean: -20.4 \tBestMean: -18.8\n",
            "Episode: 2789 \tReward: -20.0 \tMean: -20.3 \tBestMean: -18.8\n",
            "Episode: 2790 \tReward: -21.0 \tMean: -20.4 \tBestMean: -18.8\n",
            "Episode: 2791 \tReward: -21.0 \tMean: -20.4 \tBestMean: -18.8\n",
            "Episode: 2792 \tReward: -21.0 \tMean: -20.5 \tBestMean: -18.8\n",
            "Episode: 2793 \tReward: -18.0 \tMean: -20.2 \tBestMean: -18.8\n",
            "Episode: 2794 \tReward: -20.0 \tMean: -20.2 \tBestMean: -18.8\n",
            "Episode: 2795 \tReward: -21.0 \tMean: -20.2 \tBestMean: -18.8\n",
            "Episode: 2796 \tReward: -20.0 \tMean: -20.4 \tBestMean: -18.8\n",
            "Episode: 2797 \tReward: -20.0 \tMean: -20.3 \tBestMean: -18.8\n",
            "Episode: 2798 \tReward: -21.0 \tMean: -20.3 \tBestMean: -18.8\n",
            "Episode: 2799 \tReward: -21.0 \tMean: -20.4 \tBestMean: -18.8\n",
            "Episode: 2800 \tReward: -21.0 \tMean: -20.4 \tBestMean: -18.8\n",
            "Episode: 2801 \tReward: -21.0 \tMean: -20.4 \tBestMean: -18.8\n",
            "Episode: 2802 \tReward: -20.0 \tMean: -20.3 \tBestMean: -18.8\n",
            "Episode: 2803 \tReward: -21.0 \tMean: -20.6 \tBestMean: -18.8\n",
            "Episode: 2804 \tReward: -20.0 \tMean: -20.6 \tBestMean: -18.8\n",
            "Episode: 2805 \tReward: -20.0 \tMean: -20.5 \tBestMean: -18.8\n",
            "Episode: 2806 \tReward: -19.0 \tMean: -20.4 \tBestMean: -18.8\n",
            "Episode: 2807 \tReward: -20.0 \tMean: -20.4 \tBestMean: -18.8\n",
            "Episode: 2808 \tReward: -19.0 \tMean: -20.2 \tBestMean: -18.8\n",
            "Episode: 2809 \tReward: -19.0 \tMean: -20.0 \tBestMean: -18.8\n",
            "Episode: 2810 \tReward: -20.0 \tMean: -19.9 \tBestMean: -18.8\n",
            "Episode: 2811 \tReward: -19.0 \tMean: -19.7 \tBestMean: -18.8\n",
            "Episode: 2812 \tReward: -21.0 \tMean: -19.8 \tBestMean: -18.8\n",
            "Episode: 2813 \tReward: -19.0 \tMean: -19.6 \tBestMean: -18.8\n",
            "Episode: 2814 \tReward: -21.0 \tMean: -19.7 \tBestMean: -18.8\n",
            "Episode: 2815 \tReward: -21.0 \tMean: -19.8 \tBestMean: -18.8\n",
            "Episode: 2816 \tReward: -21.0 \tMean: -20.0 \tBestMean: -18.8\n",
            "Episode: 2817 \tReward: -20.0 \tMean: -20.0 \tBestMean: -18.8\n",
            "Episode: 2818 \tReward: -18.0 \tMean: -19.9 \tBestMean: -18.8\n",
            "Episode: 2819 \tReward: -21.0 \tMean: -20.1 \tBestMean: -18.8\n",
            "Episode: 2820 \tReward: -20.0 \tMean: -20.1 \tBestMean: -18.8\n",
            "Episode: 2821 \tReward: -21.0 \tMean: -20.3 \tBestMean: -18.8\n",
            "Episode: 2822 \tReward: -18.0 \tMean: -20.0 \tBestMean: -18.8\n",
            "Episode: 2823 \tReward: -21.0 \tMean: -20.2 \tBestMean: -18.8\n",
            "Episode: 2824 \tReward: -20.0 \tMean: -20.1 \tBestMean: -18.8\n",
            "Episode: 2825 \tReward: -20.0 \tMean: -20.0 \tBestMean: -18.8\n",
            "Episode: 2826 \tReward: -21.0 \tMean: -20.0 \tBestMean: -18.8\n",
            "Episode: 2827 \tReward: -20.0 \tMean: -20.0 \tBestMean: -18.8\n",
            "Episode: 2828 \tReward: -21.0 \tMean: -20.3 \tBestMean: -18.8\n",
            "Episode: 2829 \tReward: -21.0 \tMean: -20.3 \tBestMean: -18.8\n",
            "Episode: 2830 \tReward: -21.0 \tMean: -20.4 \tBestMean: -18.8\n",
            "Episode: 2831 \tReward: -20.0 \tMean: -20.3 \tBestMean: -18.8\n",
            "Episode: 2832 \tReward: -20.0 \tMean: -20.5 \tBestMean: -18.8\n",
            "Episode: 2833 \tReward: -21.0 \tMean: -20.5 \tBestMean: -18.8\n",
            "Episode: 2834 \tReward: -21.0 \tMean: -20.6 \tBestMean: -18.8\n",
            "Episode: 2835 \tReward: -21.0 \tMean: -20.7 \tBestMean: -18.8\n",
            "Episode: 2836 \tReward: -20.0 \tMean: -20.6 \tBestMean: -18.8\n",
            "Episode: 2837 \tReward: -20.0 \tMean: -20.6 \tBestMean: -18.8\n",
            "Episode: 2838 \tReward: -18.0 \tMean: -20.3 \tBestMean: -18.8\n",
            "Episode: 2839 \tReward: -21.0 \tMean: -20.3 \tBestMean: -18.8\n",
            "Episode: 2840 \tReward: -20.0 \tMean: -20.2 \tBestMean: -18.8\n",
            "Episode: 2841 \tReward: -20.0 \tMean: -20.2 \tBestMean: -18.8\n",
            "Episode: 2842 \tReward: -20.0 \tMean: -20.2 \tBestMean: -18.8\n",
            "Episode: 2843 \tReward: -21.0 \tMean: -20.2 \tBestMean: -18.8\n",
            "Episode: 2844 \tReward: -21.0 \tMean: -20.2 \tBestMean: -18.8\n",
            "Episode: 2845 \tReward: -19.0 \tMean: -20.0 \tBestMean: -18.8\n",
            "Episode: 2846 \tReward: -21.0 \tMean: -20.1 \tBestMean: -18.8\n",
            "Episode: 2847 \tReward: -20.0 \tMean: -20.1 \tBestMean: -18.8\n",
            "Episode: 2848 \tReward: -19.0 \tMean: -20.2 \tBestMean: -18.8\n",
            "Episode: 2849 \tReward: -20.0 \tMean: -20.1 \tBestMean: -18.8\n",
            "Episode: 2850 \tReward: -20.0 \tMean: -20.1 \tBestMean: -18.8\n",
            "Episode: 2851 \tReward: -21.0 \tMean: -20.2 \tBestMean: -18.8\n",
            "Episode: 2852 \tReward: -19.0 \tMean: -20.1 \tBestMean: -18.8\n",
            "Episode: 2853 \tReward: -18.0 \tMean: -19.8 \tBestMean: -18.8\n",
            "Episode: 2854 \tReward: -20.0 \tMean: -19.7 \tBestMean: -18.8\n",
            "Episode: 2855 \tReward: -20.0 \tMean: -19.8 \tBestMean: -18.8\n",
            "Episode: 2856 \tReward: -20.0 \tMean: -19.7 \tBestMean: -18.8\n",
            "Episode: 2857 \tReward: -19.0 \tMean: -19.6 \tBestMean: -18.8\n",
            "Episode: 2858 \tReward: -20.0 \tMean: -19.7 \tBestMean: -18.8\n",
            "Episode: 2859 \tReward: -21.0 \tMean: -19.8 \tBestMean: -18.8\n",
            "Episode: 2860 \tReward: -20.0 \tMean: -19.8 \tBestMean: -18.8\n",
            "Episode: 2861 \tReward: -21.0 \tMean: -19.8 \tBestMean: -18.8\n",
            "Episode: 2862 \tReward: -19.0 \tMean: -19.8 \tBestMean: -18.8\n",
            "Episode: 2863 \tReward: -21.0 \tMean: -20.1 \tBestMean: -18.8\n",
            "Episode: 2864 \tReward: -19.0 \tMean: -20.0 \tBestMean: -18.8\n",
            "Episode: 2865 \tReward: -20.0 \tMean: -20.0 \tBestMean: -18.8\n",
            "Episode: 2866 \tReward: -21.0 \tMean: -20.1 \tBestMean: -18.8\n",
            "Episode: 2867 \tReward: -20.0 \tMean: -20.2 \tBestMean: -18.8\n",
            "Episode: 2868 \tReward: -20.0 \tMean: -20.2 \tBestMean: -18.8\n",
            "Episode: 2869 \tReward: -20.0 \tMean: -20.1 \tBestMean: -18.8\n",
            "Episode: 2870 \tReward: -20.0 \tMean: -20.1 \tBestMean: -18.8\n",
            "Episode: 2871 \tReward: -20.0 \tMean: -20.0 \tBestMean: -18.8\n",
            "Episode: 2872 \tReward: -21.0 \tMean: -20.2 \tBestMean: -18.8\n",
            "Episode: 2873 \tReward: -19.0 \tMean: -20.0 \tBestMean: -18.8\n",
            "Episode: 2874 \tReward: -20.0 \tMean: -20.1 \tBestMean: -18.8\n",
            "Episode: 2875 \tReward: -20.0 \tMean: -20.1 \tBestMean: -18.8\n",
            "Episode: 2876 \tReward: -20.0 \tMean: -20.0 \tBestMean: -18.8\n",
            "Episode: 2877 \tReward: -20.0 \tMean: -20.0 \tBestMean: -18.8\n",
            "Episode: 2878 \tReward: -21.0 \tMean: -20.1 \tBestMean: -18.8\n",
            "Episode: 2879 \tReward: -21.0 \tMean: -20.2 \tBestMean: -18.8\n",
            "Episode: 2880 \tReward: -20.0 \tMean: -20.2 \tBestMean: -18.8\n",
            "Episode: 2881 \tReward: -19.0 \tMean: -20.1 \tBestMean: -18.8\n",
            "Episode: 2882 \tReward: -21.0 \tMean: -20.1 \tBestMean: -18.8\n",
            "Episode: 2883 \tReward: -19.0 \tMean: -20.1 \tBestMean: -18.8\n",
            "Episode: 2884 \tReward: -21.0 \tMean: -20.2 \tBestMean: -18.8\n",
            "Episode: 2885 \tReward: -20.0 \tMean: -20.2 \tBestMean: -18.8\n",
            "Episode: 2886 \tReward: -20.0 \tMean: -20.2 \tBestMean: -18.8\n",
            "Episode: 2887 \tReward: -20.0 \tMean: -20.2 \tBestMean: -18.8\n",
            "Episode: 2888 \tReward: -20.0 \tMean: -20.1 \tBestMean: -18.8\n",
            "Episode: 2889 \tReward: -20.0 \tMean: -20.0 \tBestMean: -18.8\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Q-k4snaUMtOB"
      },
      "source": [
        "import pandas as pd\n",
        "from datetime import datetime\n",
        "date = datetime.now().date()\n",
        "\n",
        "df = pd.DataFrame(list(zip(REWARD_LIST, MEAN_LIST)), \n",
        "               columns =['Rewards', 'Mean Reward']) \n",
        "df.to_csv(PATH + game + \"_\" + str(date) + \"_\"+ MODE + \".csv\", mode=\"w\", index=False)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "t1FmesqdMtOB"
      },
      "source": [
        "ACTOR.save_weights(PATH + \"End.h5\")\n",
        "CRITIC.save_weights(PATH + \"End.h5\")"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "oKSlrmpRMtOC"
      },
      "source": [
        "# Auswertung"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "mzDVaCvDMtOC"
      },
      "source": [
        "plt.figure(figsize=(25, 12))\n",
        "plt.plot(REWARD_LIST, label=\"erhaltene Rewards\")\n",
        "plt.plot(MEAN_LIST, label=\"durchschnittler Reward\")\n",
        "plt.title(\"Rewards während des Trainings\", fontsize=25)\n",
        "plt.xlabel(\"Episoden\", fontsize=20)\n",
        "plt.ylabel(\"Rewards\", fontsize=20)\n",
        "plt.legend(fontsize=15)\n",
        "plt.show()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "aqvMdh7kMtOC"
      },
      "source": [
        "# Spielen"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "s23zjIfZMtOD"
      },
      "source": [
        "import gym\n",
        "from IPython import display\n",
        "import matplotlib\n",
        "\n",
        "for i in range(1):\n",
        "    state = env.reset()\n",
        "    done = False\n",
        "    while not done:\n",
        "        img = plt.imshow(env.render(mode='rgb_array'))\n",
        "        img.set_data(env.render(mode='rgb_array'))\n",
        "        display.display(plt.gcf())\n",
        "        display.clear_output(wait=True)\n",
        "        action = np.argmax(ACTOR.predict(state))\n",
        "        state, reward, done, info = env.step(action)"
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}