Disertatie - RL v2.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Disertatie - RLv2.ipynb",
      "provenance": [],
      "collapsed_sections": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "eIXyU6XOCIAi",
        "outputId": "8cd3927b-ecd3-426a-f163-c094ecf5850d"
      },
      "source": [
        "!pip install tf-agents\n",
        "!pip install tensorflow"
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Collecting tf-agents\n",
            "  Downloading tf_agents-0.9.0-py3-none-any.whl (1.3 MB)\n",
            "\u001b[?25l\r\u001b[K     |▎                               | 10 kB 20.1 MB/s eta 0:00:01\r\u001b[K     |▌                               | 20 kB 24.0 MB/s eta 0:00:01\r\u001b[K     |▉                               | 30 kB 28.9 MB/s eta 0:00:01\r\u001b[K     |█                               | 40 kB 33.1 MB/s eta 0:00:01\r\u001b[K     |█▎                              | 51 kB 36.3 MB/s eta 0:00:01\r\u001b[K     |█▋                              | 61 kB 37.4 MB/s eta 0:00:01\r\u001b[K     |█▉                              | 71 kB 39.4 MB/s eta 0:00:01\r\u001b[K     |██                              | 81 kB 41.8 MB/s eta 0:00:01\r\u001b[K     |██▍                             | 92 kB 34.9 MB/s eta 0:00:01\r\u001b[K     |██▋                             | 102 kB 36.9 MB/s eta 0:00:01\r\u001b[K     |██▉                             | 112 kB 36.9 MB/s eta 0:00:01\r\u001b[K     |███▏                            | 122 kB 36.9 MB/s eta 0:00:01\r\u001b[K     |███▍                            | 133 kB 36.9 MB/s eta 0:00:01\r\u001b[K     |███▋                            | 143 kB 36.9 MB/s eta 0:00:01\r\u001b[K     |████                            | 153 kB 36.9 MB/s eta 0:00:01\r\u001b[K     |████▏                           | 163 kB 36.9 MB/s eta 0:00:01\r\u001b[K     |████▍                           | 174 kB 36.9 MB/s eta 0:00:01\r\u001b[K     |████▊                           | 184 kB 36.9 MB/s eta 0:00:01\r\u001b[K     |█████                           | 194 kB 36.9 MB/s eta 0:00:01\r\u001b[K     |█████▏                          | 204 kB 36.9 MB/s eta 0:00:01\r\u001b[K     |█████▌                          | 215 kB 36.9 MB/s eta 0:00:01\r\u001b[K     |█████▊                          | 225 kB 36.9 MB/s eta 0:00:01\r\u001b[K     |██████                          | 235 kB 36.9 MB/s eta 0:00:01\r\u001b[K     |██████▎                         | 245 kB 36.9 MB/s eta 0:00:01\r\u001b[K     |██████▌                         | 256 kB 36.9 MB/s eta 0:00:01\r\u001b[K     |██████▊                         | 266 kB 36.9 MB/s eta 0:00:01\r\u001b[K     |███████                         | 276 kB 36.9 MB/s eta 0:00:01\r\u001b[K     |███████▎                        | 286 kB 36.9 MB/s eta 0:00:01\r\u001b[K     |███████▌                        | 296 kB 36.9 MB/s eta 0:00:01\r\u001b[K     |███████▉                        | 307 kB 36.9 MB/s eta 0:00:01\r\u001b[K     |████████                        | 317 kB 36.9 MB/s eta 0:00:01\r\u001b[K     |████████▎                       | 327 kB 36.9 MB/s eta 0:00:01\r\u001b[K     |████████▋                       | 337 kB 36.9 MB/s eta 0:00:01\r\u001b[K     |████████▉                       | 348 kB 36.9 MB/s eta 0:00:01\r\u001b[K     |█████████                       | 358 kB 36.9 MB/s eta 0:00:01\r\u001b[K     |█████████▍                      | 368 kB 36.9 MB/s eta 0:00:01\r\u001b[K     |█████████▋                      | 378 kB 36.9 MB/s eta 0:00:01\r\u001b[K     |█████████▉                      | 389 kB 36.9 MB/s eta 0:00:01\r\u001b[K     |██████████▏                     | 399 kB 36.9 MB/s eta 0:00:01\r\u001b[K     |██████████▍                     | 409 kB 36.9 MB/s eta 0:00:01\r\u001b[K     |██████████▋                     | 419 kB 36.9 MB/s eta 0:00:01\r\u001b[K     |███████████                     | 430 kB 36.9 MB/s eta 0:00:01\r\u001b[K     |███████████▏                    | 440 kB 36.9 MB/s eta 0:00:01\r\u001b[K     |███████████▍                    | 450 kB 36.9 MB/s eta 0:00:01\r\u001b[K     |███████████▊                    | 460 kB 36.9 MB/s eta 0:00:01\r\u001b[K     |████████████                    | 471 kB 36.9 MB/s eta 0:00:01\r\u001b[K     |████████████▏                   | 481 kB 36.9 MB/s eta 0:00:01\r\u001b[K     |████████████▌                   | 491 kB 36.9 MB/s eta 0:00:01\r\u001b[K     |████████████▊                   | 501 kB 36.9 MB/s eta 0:00:01\r\u001b[K     |█████████████                   | 512 kB 36.9 MB/s eta 0:00:01\r\u001b[K     |█████████████▎                  | 522 kB 36.9 MB/s eta 0:00:01\r\u001b[K     |█████████████▌                  | 532 kB 36.9 MB/s eta 0:00:01\r\u001b[K     |█████████████▊                  | 542 kB 36.9 MB/s eta 0:00:01\r\u001b[K     |██████████████                  | 552 kB 36.9 MB/s eta 0:00:01\r\u001b[K     |██████████████▎                 | 563 kB 36.9 MB/s eta 0:00:01\r\u001b[K     |██████████████▌                 | 573 kB 36.9 MB/s eta 0:00:01\r\u001b[K     |██████████████▉                 | 583 kB 36.9 MB/s eta 0:00:01\r\u001b[K     |███████████████                 | 593 kB 36.9 MB/s eta 0:00:01\r\u001b[K     |███████████████▎                | 604 kB 36.9 MB/s eta 0:00:01\r\u001b[K     |███████████████▋                | 614 kB 36.9 MB/s eta 0:00:01\r\u001b[K     |███████████████▉                | 624 kB 36.9 MB/s eta 0:00:01\r\u001b[K     |████████████████                | 634 kB 36.9 MB/s eta 0:00:01\r\u001b[K     |████████████████▍               | 645 kB 36.9 MB/s eta 0:00:01\r\u001b[K     |████████████████▋               | 655 kB 36.9 MB/s eta 0:00:01\r\u001b[K     |████████████████▉               | 665 kB 36.9 MB/s eta 0:00:01\r\u001b[K     |█████████████████▏              | 675 kB 36.9 MB/s eta 0:00:01\r\u001b[K     |█████████████████▍              | 686 kB 36.9 MB/s eta 0:00:01\r\u001b[K     |█████████████████▋              | 696 kB 36.9 MB/s eta 0:00:01\r\u001b[K     |██████████████████              | 706 kB 36.9 MB/s eta 0:00:01\r\u001b[K     |██████████████████▏             | 716 kB 36.9 MB/s eta 0:00:01\r\u001b[K     |██████████████████▍             | 727 kB 36.9 MB/s eta 0:00:01\r\u001b[K     |██████████████████▊             | 737 kB 36.9 MB/s eta 0:00:01\r\u001b[K     |███████████████████             | 747 kB 36.9 MB/s eta 0:00:01\r\u001b[K     |███████████████████▏            | 757 kB 36.9 MB/s eta 0:00:01\r\u001b[K     |███████████████████▌            | 768 kB 36.9 MB/s eta 0:00:01\r\u001b[K     |███████████████████▊            | 778 kB 36.9 MB/s eta 0:00:01\r\u001b[K     |████████████████████            | 788 kB 36.9 MB/s eta 0:00:01\r\u001b[K     |████████████████████▎           | 798 kB 36.9 MB/s eta 0:00:01\r\u001b[K     |████████████████████▌           | 808 kB 36.9 MB/s eta 0:00:01\r\u001b[K     |████████████████████▊           | 819 kB 36.9 MB/s eta 0:00:01\r\u001b[K     |█████████████████████           | 829 kB 36.9 MB/s eta 0:00:01\r\u001b[K     |█████████████████████▎          | 839 kB 36.9 MB/s eta 0:00:01\r\u001b[K     |█████████████████████▌          | 849 kB 36.9 MB/s eta 0:00:01\r\u001b[K     |█████████████████████▉          | 860 kB 36.9 MB/s eta 0:00:01\r\u001b[K     |██████████████████████          | 870 kB 36.9 MB/s eta 0:00:01\r\u001b[K     |██████████████████████▎         | 880 kB 36.9 MB/s eta 0:00:01\r\u001b[K     |██████████████████████▋         | 890 kB 36.9 MB/s eta 0:00:01\r\u001b[K     |██████████████████████▉         | 901 kB 36.9 MB/s eta 0:00:01\r\u001b[K     |███████████████████████         | 911 kB 36.9 MB/s eta 0:00:01\r\u001b[K     |███████████████████████▍        | 921 kB 36.9 MB/s eta 0:00:01\r\u001b[K     |███████████████████████▋        | 931 kB 36.9 MB/s eta 0:00:01\r\u001b[K     |███████████████████████▉        | 942 kB 36.9 MB/s eta 0:00:01\r\u001b[K     |████████████████████████▏       | 952 kB 36.9 MB/s eta 0:00:01\r\u001b[K     |████████████████████████▍       | 962 kB 36.9 MB/s eta 0:00:01\r\u001b[K     |████████████████████████▋       | 972 kB 36.9 MB/s eta 0:00:01\r\u001b[K     |█████████████████████████       | 983 kB 36.9 MB/s eta 0:00:01\r\u001b[K     |█████████████████████████▏      | 993 kB 36.9 MB/s eta 0:00:01\r\u001b[K     |█████████████████████████▍      | 1.0 MB 36.9 MB/s eta 0:00:01\r\u001b[K     |█████████████████████████▊      | 1.0 MB 36.9 MB/s eta 0:00:01\r\u001b[K     |██████████████████████████      | 1.0 MB 36.9 MB/s eta 0:00:01\r\u001b[K     |██████████████████████████▏     | 1.0 MB 36.9 MB/s eta 0:00:01\r\u001b[K     |██████████████████████████▌     | 1.0 MB 36.9 MB/s eta 0:00:01\r\u001b[K     |██████████████████████████▊     | 1.1 MB 36.9 MB/s eta 0:00:01\r\u001b[K     |███████████████████████████     | 1.1 MB 36.9 MB/s eta 0:00:01\r\u001b[K     |███████████████████████████▎    | 1.1 MB 36.9 MB/s eta 0:00:01\r\u001b[K     |███████████████████████████▌    | 1.1 MB 36.9 MB/s eta 0:00:01\r\u001b[K     |███████████████████████████▊    | 1.1 MB 36.9 MB/s eta 0:00:01\r\u001b[K     |████████████████████████████    | 1.1 MB 36.9 MB/s eta 0:00:01\r\u001b[K     |████████████████████████████▎   | 1.1 MB 36.9 MB/s eta 0:00:01\r\u001b[K     |████████████████████████████▌   | 1.1 MB 36.9 MB/s eta 0:00:01\r\u001b[K     |████████████████████████████▉   | 1.1 MB 36.9 MB/s eta 0:00:01\r\u001b[K     |█████████████████████████████   | 1.1 MB 36.9 MB/s eta 0:00:01\r\u001b[K     |█████████████████████████████▎  | 1.2 MB 36.9 MB/s eta 0:00:01\r\u001b[K     |█████████████████████████████▋  | 1.2 MB 36.9 MB/s eta 0:00:01\r\u001b[K     |█████████████████████████████▉  | 1.2 MB 36.9 MB/s eta 0:00:01\r\u001b[K     |██████████████████████████████  | 1.2 MB 36.9 MB/s eta 0:00:01\r\u001b[K     |██████████████████████████████▍ | 1.2 MB 36.9 MB/s eta 0:00:01\r\u001b[K     |██████████████████████████████▋ | 1.2 MB 36.9 MB/s eta 0:00:01\r\u001b[K     |██████████████████████████████▉ | 1.2 MB 36.9 MB/s eta 0:00:01\r\u001b[K     |███████████████████████████████▏| 1.2 MB 36.9 MB/s eta 0:00:01\r\u001b[K     |███████████████████████████████▍| 1.2 MB 36.9 MB/s eta 0:00:01\r\u001b[K     |███████████████████████████████▋| 1.2 MB 36.9 MB/s eta 0:00:01\r\u001b[K     |████████████████████████████████| 1.3 MB 36.9 MB/s eta 0:00:01\r\u001b[K     |████████████████████████████████| 1.3 MB 36.9 MB/s \n",
            "\u001b[?25hRequirement already satisfied: gym>=0.17.0 in /usr/local/lib/python3.7/dist-packages (from tf-agents) (0.17.3)\n",
            "Requirement already satisfied: typing-extensions>=3.7.4.3 in /usr/local/lib/python3.7/dist-packages (from tf-agents) (3.7.4.3)\n",
            "Requirement already satisfied: absl-py>=0.6.1 in /usr/local/lib/python3.7/dist-packages (from tf-agents) (0.12.0)\n",
            "Requirement already satisfied: gin-config>=0.4.0 in /usr/local/lib/python3.7/dist-packages (from tf-agents) (0.4.0)\n",
            "Requirement already satisfied: tensorflow-probability>=0.13.0 in /usr/local/lib/python3.7/dist-packages (from tf-agents) (0.13.0)\n",
            "Requirement already satisfied: protobuf>=3.11.3 in /usr/local/lib/python3.7/dist-packages (from tf-agents) (3.17.3)\n",
            "Requirement already satisfied: six>=1.10.0 in /usr/local/lib/python3.7/dist-packages (from tf-agents) (1.15.0)\n",
            "Requirement already satisfied: wrapt>=1.11.1 in /usr/local/lib/python3.7/dist-packages (from tf-agents) (1.12.1)\n",
            "Requirement already satisfied: numpy>=1.13.3 in /usr/local/lib/python3.7/dist-packages (from tf-agents) (1.19.5)\n",
            "Requirement already satisfied: cloudpickle>=1.3 in /usr/local/lib/python3.7/dist-packages (from tf-agents) (1.3.0)\n",
            "Requirement already satisfied: scipy in /usr/local/lib/python3.7/dist-packages (from gym>=0.17.0->tf-agents) (1.4.1)\n",
            "Requirement already satisfied: pyglet<=1.5.0,>=1.4.0 in /usr/local/lib/python3.7/dist-packages (from gym>=0.17.0->tf-agents) (1.5.0)\n",
            "Requirement already satisfied: future in /usr/local/lib/python3.7/dist-packages (from pyglet<=1.5.0,>=1.4.0->gym>=0.17.0->tf-agents) (0.16.0)\n",
            "Requirement already satisfied: decorator in /usr/local/lib/python3.7/dist-packages (from tensorflow-probability>=0.13.0->tf-agents) (4.4.2)\n",
            "Requirement already satisfied: gast>=0.3.2 in /usr/local/lib/python3.7/dist-packages (from tensorflow-probability>=0.13.0->tf-agents) (0.4.0)\n",
            "Requirement already satisfied: dm-tree in /usr/local/lib/python3.7/dist-packages (from tensorflow-probability>=0.13.0->tf-agents) (0.1.6)\n",
            "Installing collected packages: tf-agents\n",
            "Successfully installed tf-agents-0.9.0\n",
            "Requirement already satisfied: tensorflow in /usr/local/lib/python3.7/dist-packages (2.6.0)\n",
            "Requirement already satisfied: tensorflow-estimator~=2.6 in /usr/local/lib/python3.7/dist-packages (from tensorflow) (2.6.0)\n",
            "Requirement already satisfied: keras~=2.6 in /usr/local/lib/python3.7/dist-packages (from tensorflow) (2.6.0)\n",
            "Requirement already satisfied: numpy~=1.19.2 in /usr/local/lib/python3.7/dist-packages (from tensorflow) (1.19.5)\n",
            "Requirement already satisfied: six~=1.15.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow) (1.15.0)\n",
            "Requirement already satisfied: wrapt~=1.12.1 in /usr/local/lib/python3.7/dist-packages (from tensorflow) (1.12.1)\n",
            "Requirement already satisfied: h5py~=3.1.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow) (3.1.0)\n",
            "Requirement already satisfied: protobuf>=3.9.2 in /usr/local/lib/python3.7/dist-packages (from tensorflow) (3.17.3)\n",
            "Requirement already satisfied: flatbuffers~=1.12.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow) (1.12)\n",
            "Requirement already satisfied: astunparse~=1.6.3 in /usr/local/lib/python3.7/dist-packages (from tensorflow) (1.6.3)\n",
            "Requirement already satisfied: keras-preprocessing~=1.1.2 in /usr/local/lib/python3.7/dist-packages (from tensorflow) (1.1.2)\n",
            "Requirement already satisfied: google-pasta~=0.2 in /usr/local/lib/python3.7/dist-packages (from tensorflow) (0.2.0)\n",
            "Requirement already satisfied: termcolor~=1.1.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow) (1.1.0)\n",
            "Requirement already satisfied: absl-py~=0.10 in /usr/local/lib/python3.7/dist-packages (from tensorflow) (0.12.0)\n",
            "Requirement already satisfied: gast==0.4.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow) (0.4.0)\n",
            "Requirement already satisfied: tensorboard~=2.6 in /usr/local/lib/python3.7/dist-packages (from tensorflow) (2.6.0)\n",
            "Requirement already satisfied: grpcio<2.0,>=1.37.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow) (1.39.0)\n",
            "Requirement already satisfied: clang~=5.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow) (5.0)\n",
            "Requirement already satisfied: opt-einsum~=3.3.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow) (3.3.0)\n",
            "Requirement already satisfied: typing-extensions~=3.7.4 in /usr/local/lib/python3.7/dist-packages (from tensorflow) (3.7.4.3)\n",
            "Requirement already satisfied: wheel~=0.35 in /usr/local/lib/python3.7/dist-packages (from tensorflow) (0.37.0)\n",
            "Requirement already satisfied: cached-property in /usr/local/lib/python3.7/dist-packages (from h5py~=3.1.0->tensorflow) (1.5.2)\n",
            "Requirement already satisfied: markdown>=2.6.8 in /usr/local/lib/python3.7/dist-packages (from tensorboard~=2.6->tensorflow) (3.3.4)\n",
            "Requirement already satisfied: werkzeug>=0.11.15 in /usr/local/lib/python3.7/dist-packages (from tensorboard~=2.6->tensorflow) (1.0.1)\n",
            "Requirement already satisfied: setuptools>=41.0.0 in /usr/local/lib/python3.7/dist-packages (from tensorboard~=2.6->tensorflow) (57.4.0)\n",
            "Requirement already satisfied: tensorboard-data-server<0.7.0,>=0.6.0 in /usr/local/lib/python3.7/dist-packages (from tensorboard~=2.6->tensorflow) (0.6.1)\n",
            "Requirement already satisfied: requests<3,>=2.21.0 in /usr/local/lib/python3.7/dist-packages (from tensorboard~=2.6->tensorflow) (2.23.0)\n",
            "Requirement already satisfied: tensorboard-plugin-wit>=1.6.0 in /usr/local/lib/python3.7/dist-packages (from tensorboard~=2.6->tensorflow) (1.8.0)\n",
            "Requirement already satisfied: google-auth<2,>=1.6.3 in /usr/local/lib/python3.7/dist-packages (from tensorboard~=2.6->tensorflow) (1.34.0)\n",
            "Requirement already satisfied: google-auth-oauthlib<0.5,>=0.4.1 in /usr/local/lib/python3.7/dist-packages (from tensorboard~=2.6->tensorflow) (0.4.5)\n",
            "Requirement already satisfied: rsa<5,>=3.1.4 in /usr/local/lib/python3.7/dist-packages (from google-auth<2,>=1.6.3->tensorboard~=2.6->tensorflow) (4.7.2)\n",
            "Requirement already satisfied: cachetools<5.0,>=2.0.0 in /usr/local/lib/python3.7/dist-packages (from google-auth<2,>=1.6.3->tensorboard~=2.6->tensorflow) (4.2.2)\n",
            "Requirement already satisfied: pyasn1-modules>=0.2.1 in /usr/local/lib/python3.7/dist-packages (from google-auth<2,>=1.6.3->tensorboard~=2.6->tensorflow) (0.2.8)\n",
            "Requirement already satisfied: requests-oauthlib>=0.7.0 in /usr/local/lib/python3.7/dist-packages (from google-auth-oauthlib<0.5,>=0.4.1->tensorboard~=2.6->tensorflow) (1.3.0)\n",
            "Requirement already satisfied: importlib-metadata in /usr/local/lib/python3.7/dist-packages (from markdown>=2.6.8->tensorboard~=2.6->tensorflow) (4.6.4)\n",
            "Requirement already satisfied: pyasn1<0.5.0,>=0.4.6 in /usr/local/lib/python3.7/dist-packages (from pyasn1-modules>=0.2.1->google-auth<2,>=1.6.3->tensorboard~=2.6->tensorflow) (0.4.8)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests<3,>=2.21.0->tensorboard~=2.6->tensorflow) (2021.5.30)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests<3,>=2.21.0->tensorboard~=2.6->tensorflow) (3.0.4)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests<3,>=2.21.0->tensorboard~=2.6->tensorflow) (2.10)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests<3,>=2.21.0->tensorboard~=2.6->tensorflow) (1.24.3)\n",
            "Requirement already satisfied: oauthlib>=3.0.0 in /usr/local/lib/python3.7/dist-packages (from requests-oauthlib>=0.7.0->google-auth-oauthlib<0.5,>=0.4.1->tensorboard~=2.6->tensorflow) (3.1.1)\n",
            "Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.7/dist-packages (from importlib-metadata->markdown>=2.6.8->tensorboard~=2.6->tensorflow) (3.5.0)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "zGWulEAICOcq"
      },
      "source": [
        "import abc\n",
        "import copy\n",
        "import math\n",
        "import random\n",
        "import numpy as np\n",
        "import gym\n",
        "from gym import spaces\n",
        "\n",
        "import tensorflow as tf\n",
        "from tf_agents.agents.dqn.dqn_agent import DqnAgent, DdqnAgent\n",
        "from tf_agents.networks.q_network import QNetwork\n",
        "from tf_agents.drivers import dynamic_step_driver\n",
        "from tf_agents.environments import tf_py_environment\n",
        "from tf_agents.environments import py_environment\n",
        "from tf_agents.environments import tf_environment\n",
        "from tf_agents.environments import utils\n",
        "from tf_agents.trajectories import trajectory\n",
        "from tf_agents.environments import wrappers\n",
        "from tf_agents.eval import metric_utils\n",
        "from tf_agents.metrics import tf_metrics\n",
        "from tf_agents.policies import random_tf_policy\n",
        "from tf_agents.replay_buffers import tf_uniform_replay_buffer\n",
        "from tf_agents.policies.random_tf_policy import RandomTFPolicy\n",
        "from tf_agents.replay_buffers.tf_uniform_replay_buffer import TFUniformReplayBuffer\n",
        "from tf_agents.utils import common\n",
        "from tf_agents.metrics import py_metrics\n",
        "from tf_agents.metrics import tf_metrics\n",
        "from tf_agents.drivers import py_driver\n",
        "from tf_agents.drivers import dynamic_episode_driver\n",
        "from tf_agents.specs import array_spec\n",
        "from tf_agents.trajectories import time_step as ts\n",
        "\n",
        "import matplotlib\n",
        "import matplotlib.pyplot as plt\n",
        "from skimage.draw import line\n",
        "\n",
        "from google.colab.patches import cv2_imshow\n",
        "\n",
        "tf.compat.v1.enable_v2_behavior()\n",
        "\n",
        "IMG_DIM = 512\n",
        "NUMBER_ITERATION = 30000\n",
        "COLLECTION_STEPS_PER_ITERATION = 1\n",
        "REPLAY_BUFFER_MAX_LENGTH = 100000\n",
        "BATCH_SIZE = 128\n",
        "EVAL_EPISODES = 20\n",
        "EVAL_INTERVAL = 1000\n",
        "INITIAL_COLLECT_STEPS = 1000\n",
        "LEARNING_RATE = 3e-4"
      ],
      "execution_count": 2,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "6l8nQL_yCQ36",
        "outputId": "e612133e-f1cd-431b-f62a-9475ffaec845"
      },
      "source": [
        "device_name = tf.test.gpu_device_name()\n",
        "if device_name != '/device:GPU:0':\n",
        "  pass\n",
        "  #raise SystemError('GPU device not found')\n",
        "print('Found GPU at: {}'.format(device_name))"
      ],
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Found GPU at: \n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "UBErA8N0CTiN"
      },
      "source": [
        "#Given 3 points, check their orientation -> 0 = colinear, 1 = clockwise, -1 = counter-clockwise\n",
        "def get_orientation(A, B, C):\n",
        "\n",
        "  val = ((B[1] - A[1]) * (C[0] - B[0])) - ((B[0] - A[0]) * (C[1] - B[1]))\n",
        "  if val == 0 : return 0\n",
        "  return 1 if val > 0 else -1\n",
        "\n",
        "#Check if segments intersect\n",
        "def is_segm_intersection(S1, S2):\n",
        "\n",
        "  (A1, B1) = S1\n",
        "  (A2, B2) = S2\n",
        "\n",
        "  if A1 == A2 or A1 == B2 or B1 == A2 or B1 == B2:\n",
        "    return False\n",
        "\n",
        "  #find all orientations\n",
        "  o1 = get_orientation(A1, B1, A2)\n",
        "  o2 = get_orientation(A1, B1, B2)\n",
        "  o3 = get_orientation(A2, B2, A1)\n",
        "  o4 = get_orientation(A2, B2, B1)\n",
        "\n",
        "  #general case\n",
        "  if o1 != o2 and o3 != o4:\n",
        "    return True\n",
        "\n",
        "  return False\n",
        "\n",
        "def intersects_any(S, segments):\n",
        "\n",
        "  for s in segments[:-1]:\n",
        "    if is_segm_intersection(S, s) == True:\n",
        "      return True\n",
        "\n",
        "  return False\n",
        "\n",
        "#check if a pixel is outside of image\n",
        "def is_out_of_bounds(A, n):\n",
        "  (x, y) = A\n",
        "  return x < 0 or y < 0 or x >= n or y >= n \n",
        "\n",
        "#check if a pixel is on the edge of the image\n",
        "def is_marginal_node(A, n):\n",
        "  (x, y) = A\n",
        "  return x == 0 or y == 0 or x == (n - 1) or y == (n - 1)"
      ],
      "execution_count": 4,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "inYBqIPFCUdz"
      },
      "source": [
        "#given a point, get the equivalent point at a given distance and given angle\n",
        "def apply_translation_to(position, angle, distance):\n",
        "  x = distance * math.cos(math.radians(angle))\n",
        "  y = distance * math.sin(math.radians(angle))\n",
        "  return (position[0] + x, position[1] + y)\n",
        "\n",
        "#rotate a position to left with a number of degrees\n",
        "def rotate_left(position, degree_angle):\n",
        "  (x, y) = position\n",
        "  adjustedX = (x * math.cos(math.radians(degree_angle))) - (y * math.sin(math.radians(degree_angle)))\n",
        "  adjustedY = (y * math.cos(math.radians(degree_angle))) + (x * math.sin(math.radians(degree_angle)))\n",
        "  return (round(adjustedX), round(adjustedY))\n",
        "\n",
        "#given a position, get the point at a certaing angle and distance\n",
        "def get_position_of(last_position, angle, distance, angle_adjustment=0):\n",
        "  rotated_left_position = rotate_left(last_position, 360 - angle_adjustment)\n",
        "  translated_location = apply_translation_to(rotated_left_position, angle, distance)\n",
        "  return rotate_left(translated_location, angle_adjustment)"
      ],
      "execution_count": 5,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "3dIT2z7ZCfst"
      },
      "source": [
        "MAX_STEPS = 40\n",
        "\n",
        "ACTION_DIST = 0\n",
        "ACTION_ANGLE = 1\n",
        "ACTION_INTER = 2\n",
        "ACTION_END = 3\n",
        "\n",
        "MAX_ACTIONS = 44\n",
        "\n",
        "FIXED_DIST = 40\n",
        "FIXED_HISTORY = 20\n",
        "\n",
        "def convert_integer_to_action(code):\n",
        "\n",
        "    '''\n",
        "    0...20 -> bins of 10 values: 0 = [0..9], 1 = [10..19]..etc\n",
        "    21...38 -> bins of 10 values: 21 = [0..9], 22 = [10..19]..etc\n",
        "    39...43 -> 39 == INTER_3, 40 == INTER_4, 41 == INTER_5, 42 == INTER_6, 43 == INTER_7\n",
        "    44 -> END\n",
        "    '''\n",
        "\n",
        "    action, value = (-1, -1)\n",
        "\n",
        "    if code <= 20:\n",
        "        action = ACTION_DIST\n",
        "        value = FIXED_DIST\n",
        "        #value = (code - 0) * 10 + random.randint(0, 9)\n",
        "\n",
        "    elif code <= 38:\n",
        "        action = ACTION_ANGLE\n",
        "        value = ((code - 21) * 10 + random.randint(0, 9)) - 90\n",
        "\n",
        "    elif code <= 43:\n",
        "        action = ACTION_INTER\n",
        "        value = code - 39 + 3\n",
        "\n",
        "    elif code == 44:\n",
        "        action = ACTION_END\n",
        "        value = code\n",
        "\n",
        "    #print(code, ' -> ', (action, value))\n",
        "\n",
        "    return (action, value)\n",
        "\n",
        "class RoadNetworkEnv(py_environment.PyEnvironment):\n",
        "\n",
        "    def __init__(self):\n",
        "\n",
        "        self._action_spec = array_spec.BoundedArraySpec(shape=(), dtype=np.int32, minimum=0, maximum=MAX_ACTIONS, name='action')\n",
        "        self._observation_spec = array_spec.BoundedArraySpec(shape=(FIXED_HISTORY + 1,), dtype=np.float64, minimum=np.full((FIXED_HISTORY + 1,), -1, dtype=np.float64),maximum=np.full((FIXED_HISTORY + 1,), 5, dtype=np.float64), name='observation')\n",
        "        self._state=[0]\n",
        "        self._action_timeline = np.zeros(4,)\n",
        "        self._episode_ended = False\n",
        "        self._reward = 0\n",
        "        self._timestamp = 0\n",
        "        self._action_history = np.full((FIXED_HISTORY,), -1, dtype=np.int32)\n",
        "        self._image = np.zeros((IMG_DIM, IMG_DIM), dtype=np.uint8)\n",
        "\n",
        "        self._edges = []\n",
        "        self._nodes = [(0, 255)]\n",
        "        self._nodes_to_expand = [(ACTION_ANGLE, (0, 255), 0, 0)]\n",
        "        self._action_history[-1] = ACTION_ANGLE\n",
        "        self._state = np.concatenate((self._state, self._action_history))\n",
        "\n",
        "    def action_spec(self):\n",
        "        return self._action_spec\n",
        "\n",
        "    def observation_spec(self):\n",
        "        return self._observation_spec\n",
        "\n",
        "    def update_state(self):\n",
        "        self._state = [0]\n",
        "        (_, (xA, yA), angle, _) = self._nodes_to_expand[-1]\n",
        "        (xB, yB) = get_position_of((xA, yA), 0, FIXED_DIST, angle)\n",
        "        rr, cc = line(xA, yA, xB, yB)\n",
        "        for i in range(len(rr)):\n",
        "          r = rr[i]\n",
        "          c = cc[i]\n",
        "          if is_out_of_bounds((r, c), IMG_DIM) or self._image[r, c] != 0:\n",
        "            self._state = [1]\n",
        "        \n",
        "        self._state = np.concatenate((self._state, self._action_history))\n",
        "\n",
        "\n",
        "    def apply_action(self, action):\n",
        "\n",
        "        self._timestamp += 1\n",
        "\n",
        "        (action_type, action_value) = convert_integer_to_action(action)\n",
        "        (last_action, position, angle, steps) = self._nodes_to_expand.pop(0)\n",
        "        \n",
        "        time_diff = self._timestamp - self._action_timeline[action_type]\n",
        "        self._action_history = np.roll(self._action_history, -1)\n",
        "        self._action_history[-1] = action_type\n",
        "\n",
        "        is_oob = False\n",
        "        is_marginal = False\n",
        "        is_self_inter = False\n",
        "\n",
        "        if action_type == ACTION_DIST:\n",
        "          new_position = get_position_of(position, 0, action_value, angle)\n",
        "\n",
        "          is_oob = is_out_of_bounds(new_position, IMG_DIM)\n",
        "          is_marginal = is_marginal_node(new_position, IMG_DIM)\n",
        "          is_self_inter = intersects_any((position, new_position), self._edges)\n",
        "\n",
        "        if last_action == ACTION_DIST:\n",
        "            if action_type == ACTION_ANGLE:\n",
        "              self._reward = 0 if time_diff - 1 < 0 else 10\n",
        "            elif action_type == ACTION_INTER:\n",
        "              self._reward = 0 if time_diff - 15 < 0 else 13 - action_value\n",
        "            elif action_type == ACTION_END:\n",
        "              self._reward = 0 if time_diff - 18 < 0 else 5\n",
        "            else:\n",
        "              self._reward = 10\n",
        "\n",
        "        elif last_action == ACTION_ANGLE:\n",
        "            if action_type == ACTION_DIST:\n",
        "              self._reward = 15\n",
        "            else:\n",
        "              self._reward = 0\n",
        "\n",
        "        elif last_action == ACTION_INTER:\n",
        "            if action_type == ACTION_DIST:\n",
        "              self._reward = 20\n",
        "            else:\n",
        "              self._reward = 0\n",
        "\n",
        "\n",
        "        if is_oob == True or is_self_inter == True:\n",
        "            #if is_oob == True:\n",
        "            #  print(\"Position out of bounds\")\n",
        "            #else:\n",
        "            #  print(\"Self intersection\")\n",
        "            #print(\"Current position: \", position)\n",
        "            #print(\"Next position: \", new_position)\n",
        "            #print(\"+++++++++++++++++++++++++++++++++++++++\")\n",
        "            self._reward = -1000\n",
        "            self._episode_ended = True\n",
        "            return\n",
        "\n",
        "\n",
        "        if action_type == ACTION_DIST:\n",
        "            self._nodes_to_expand.append((ACTION_DIST, new_position, angle, steps + 1))\n",
        "            if position != new_position and (len(self._edges) == 0 or (position, new_position) != self._edges[-1]):\n",
        "              self._edges.append((position, new_position))\n",
        "              self._nodes.append(new_position)\n",
        "              self.draw_edge(position, new_position)\n",
        "\n",
        "        elif action_type == ACTION_ANGLE:\n",
        "            self._nodes_to_expand.append((ACTION_ANGLE, position, (angle + action_value), steps + 1))\n",
        "\n",
        "        elif action_type == ACTION_INTER:\n",
        "            if last_action != ACTION_INTER:\n",
        "              MIN_INTER_ANGLE = 45\n",
        "              prev_angle = -180\n",
        "              angle_sum = 0\n",
        "              for i in range(action_value - 1):\n",
        "                  remaining = (360 - angle_sum - ((action_value - i) * MIN_INTER_ANGLE))\n",
        "                  gen_angle = random.randint(MIN_INTER_ANGLE, remaining)\n",
        "                  forced_angle = prev_angle + gen_angle\n",
        "                  angle_sum += gen_angle\n",
        "                  prev_angle = forced_angle\n",
        "                  self._nodes_to_expand.append((ACTION_INTER, position, (angle + forced_angle), steps + 1))\n",
        "            else:\n",
        "                self._nodes_to_expand.insert(0, (last_action, position, angle, steps))\n",
        "\n",
        "        if is_marginal == True:\n",
        "            self._reward += 1\n",
        "\n",
        "        nr_dist = (self._action_history == ACTION_DIST).sum()\n",
        "        nr_angles = (self._action_history == ACTION_ANGLE).sum()\n",
        "        nr_inter = (self._action_history == ACTION_INTER).sum()\n",
        "        nr_ends = (self._action_history == ACTION_END).sum()\n",
        "        nr_none = (self._action_history == -1).sum()\n",
        "\n",
        "        if action_type == ACTION_DIST and nr_dist > 18:\n",
        "            self._reward = -500\n",
        "\n",
        "        if action_type == ACTION_ANGLE and nr_angles > 16:\n",
        "            self._reward = -500\n",
        "\n",
        "        if action_type == ACTION_INTER and (nr_inter + nr_none) > 4:\n",
        "            self._reward = -500\n",
        "\n",
        "        if action_type == ACTION_END and (nr_ends + nr_none) > 2:\n",
        "            self._reward = -500\n",
        "\n",
        "        if len(self._nodes_to_expand) == 0:\n",
        "            self._reward = 1000 if len(self._edges) > 10 else 0\n",
        "            self._episode_ended = True\n",
        "            return\n",
        "       \n",
        "\n",
        "        self._action_timeline[action_type] = self._timestamp\n",
        "        self.update_state()\n",
        "  \n",
        "    def _reset(self):\n",
        "\n",
        "        #print(\"RESET WAS CALLED pre!!!! -> \", self._nodes_to_expand)\n",
        "\n",
        "        self._state=[0]\n",
        "        self._episode_ended = False\n",
        "        self._reward = 0\n",
        "        self._edges = []\n",
        "        self._nodes = [(0, 255)]\n",
        "        self._nodes_to_expand = [(ACTION_ANGLE, (0, 255), 0, 0)]\n",
        "        self._timestamp = 0\n",
        "        self._action_timeline = np.zeros(4,)\n",
        "        self._action_history = np.full((FIXED_HISTORY,), -1, dtype=np.int32)\n",
        "        self._action_history[-1] = ACTION_ANGLE\n",
        "        self._state = np.concatenate((self._state, self._action_history))\n",
        "        self._image = np.zeros((IMG_DIM, IMG_DIM), dtype=np.uint8)\n",
        "\n",
        "        return ts.restart(np.array(self._state, dtype=np.float64))\n",
        "\n",
        "\n",
        "    def _step(self, action):\n",
        "\n",
        "        if self._episode_ended == True:\n",
        "            return self.reset()\n",
        "\n",
        "        self.apply_action(action)\n",
        "            \n",
        "        if self._episode_ended == True:\n",
        "          return ts.termination(np.array(self._state, dtype=np.float64), self._reward)\n",
        "        else:\n",
        "          return ts.transition(np.array(self._state, dtype=np.float64), reward=self._reward, discount=0.9)\n",
        "\n",
        "    def draw_edge(self, position, new_position):\n",
        "        (xA, yA) = position\n",
        "        (xB, yB) = new_position   \n",
        "        rr, cc = line(xA, yA, xB, yB)\n",
        "        self._image[rr, cc] = 255\n",
        "\n",
        "    def render(self, mode='human'):\n",
        "        print(self._edges)\n",
        "        print(self._state)\n",
        "        return self._image"
      ],
      "execution_count": 6,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "LSi1UMZmLgBt"
      },
      "source": [
        "env = RoadNetworkEnv()\n",
        "utils.validate_py_environment(env)\n",
        "\n",
        "tl_env = wrappers.TimeLimit(env, duration=5000)\n",
        "\n",
        "time_step = tl_env.reset()\n",
        "rewards = time_step.reward\n",
        "\n",
        "for i in range(100):\n",
        "    action = np.random.choice(range(MAX_ACTIONS))\n",
        "    time_step = tl_env.step(action)\n",
        "    rewards += time_step.reward"
      ],
      "execution_count": 7,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "zdQSc6EJLiIh",
        "outputId": "83b2a84c-9e96-40f0-f207-a8e8a8270e03"
      },
      "source": [
        "train_py_env = wrappers.TimeLimit(RoadNetworkEnv(), duration=1000)\n",
        "eval_py_env = wrappers.TimeLimit(RoadNetworkEnv(), duration=1000)\n",
        "\n",
        "print('Observation Spec:')\n",
        "print(train_py_env.time_step_spec().observation)\n",
        "\n",
        "print('Reward Spec:')\n",
        "print(train_py_env.time_step_spec().reward)\n",
        "\n",
        "print('Action Spec:')\n",
        "print(train_py_env.action_spec())\n",
        "\n",
        "train_env = tf_py_environment.TFPyEnvironment(train_py_env)\n",
        "evaluation_env = tf_py_environment.TFPyEnvironment(eval_py_env)"
      ],
      "execution_count": 8,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Observation Spec:\n",
            "BoundedArraySpec(shape=(21,), dtype=dtype('float64'), name='observation', minimum=[-1. -1. -1. -1. -1. -1. -1. -1. -1. -1. -1. -1. -1. -1. -1. -1. -1. -1.\n",
            " -1. -1. -1.], maximum=[5. 5. 5. 5. 5. 5. 5. 5. 5. 5. 5. 5. 5. 5. 5. 5. 5. 5. 5. 5. 5.])\n",
            "Reward Spec:\n",
            "ArraySpec(shape=(), dtype=dtype('float32'), name='reward')\n",
            "Action Spec:\n",
            "BoundedArraySpec(shape=(), dtype=dtype('int32'), name='action', minimum=0, maximum=44)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "N8PC4e2pLj2D",
        "outputId": "f50e6484-881f-4be8-fdb2-51007c066ddc"
      },
      "source": [
        "hidden_layers = (264, 512, 1024,)\n",
        "\n",
        "q_network = QNetwork(\n",
        "    train_env.observation_spec(),\n",
        "    train_env.action_spec(),\n",
        "    fc_layer_params=hidden_layers)\n",
        "\n",
        "counter = tf.Variable(0)\n",
        "\n",
        "agent = DdqnAgent(\n",
        "    train_env.time_step_spec(),\n",
        "    train_env.action_spec(),\n",
        "    q_network = q_network,\n",
        "    debug_summaries = True,\n",
        "    epsilon_greedy = 0.30,\n",
        "    optimizer = tf.compat.v1.train.AdamOptimizer(learning_rate=LEARNING_RATE),\n",
        "    td_errors_loss_fn = common.element_wise_squared_loss,\n",
        "    train_step_counter = counter)\n",
        "\n",
        "agent.initialize()\n",
        "agent.collect_policy"
      ],
      "execution_count": 9,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<tf_agents.policies.epsilon_greedy_policy.EpsilonGreedyPolicy at 0x7f9385f3b0d0>"
            ]
          },
          "metadata": {},
          "execution_count": 9
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "87mhEccSLnSj"
      },
      "source": [
        "#This method is used for calculations of how much reward has agent gained on average.\n",
        "def get_average_return(environment, policy, episodes=20):\n",
        "\n",
        "  total_return = 0.0\n",
        "\n",
        "  for _ in range(episodes):\n",
        "    time_step = environment.reset()\n",
        "    episode_return = 0.0\n",
        "\n",
        "    while not time_step.is_last():\n",
        "      action_step = policy.action(time_step)\n",
        "      time_step = environment.step(action_step.action)\n",
        "      episode_return += time_step.reward\n",
        "\n",
        "    total_return += episode_return\n",
        "  \n",
        "  avg_return = total_return / episodes\n",
        "  return avg_return.numpy()[0]"
      ],
      "execution_count": 10,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "s8MPlAkgLo3A",
        "outputId": "832778e0-6800-4d74-c5f3-d878121fbe70"
      },
      "source": [
        "class ExperienceReplay(object):\n",
        "    def __init__(self, agent, enviroment):\n",
        "\n",
        "        self._replay_buffer = TFUniformReplayBuffer(\n",
        "            data_spec=agent.collect_data_spec,\n",
        "            batch_size=enviroment.batch_size,\n",
        "            max_length=REPLAY_BUFFER_MAX_LENGTH)\n",
        "        \n",
        "        self._random_policy = RandomTFPolicy(train_env.time_step_spec(),\n",
        "                                                enviroment.action_spec())\n",
        "        \n",
        "        self.collect_data(train_env, self._random_policy, steps=INITIAL_COLLECT_STEPS)\n",
        "        \n",
        "        self.dataset = self._replay_buffer.as_dataset(\n",
        "            num_parallel_calls=3, \n",
        "            sample_batch_size=BATCH_SIZE, \n",
        "            num_steps=2).prefetch(3)\n",
        "\n",
        "        self.iterator = iter(self.dataset)\n",
        "    \n",
        "    def collect_data(self, enviroment, policy, steps):\n",
        "        for _ in range(steps):\n",
        "            self.collect_step(enviroment, policy)\n",
        "            \n",
        "    def collect_step(self, environment, policy):\n",
        "        time_step = environment.current_time_step()\n",
        "        action_step = policy.action(time_step)\n",
        "        next_time_step = environment.step(action_step.action)\n",
        "        timestamp_trajectory = trajectory.from_transition(time_step, action_step, next_time_step)\n",
        "        self._replay_buffer.add_batch(timestamp_trajectory)\n",
        "\n",
        "experience_replay = ExperienceReplay(agent, train_env)"
      ],
      "execution_count": 11,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:From /usr/local/lib/python3.7/dist-packages/tensorflow/python/data/experimental/ops/counter.py:66: scan (from tensorflow.python.data.experimental.ops.scan_ops) is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "Use `tf.data.Dataset.scan(...) instead\n",
            "WARNING:tensorflow:From /usr/local/lib/python3.7/dist-packages/tensorflow/python/autograph/impl/api.py:382: ReplayBuffer.get_next (from tf_agents.replay_buffers.replay_buffer) is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "Use `as_dataset(..., single_deterministic_pass=False) instead.\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "NeQrSKktLqmY",
        "outputId": "8a2e607b-fd91-410d-e1c8-5f6e79e33214"
      },
      "source": [
        "agent.train = common.function(agent.train)\n",
        "\n",
        "#initialize counter on the agent to 0\n",
        "agent.train_step_counter.assign(0)\n",
        "\n",
        "\n",
        "with tf.device('/device:GPU:0'):\n",
        "  #get initial average return of reward\n",
        "  #avg_return = get_average_return(evaluation_env, agent.policy, EVAL_EPISODES)\n",
        "  #returns = [avg_return]\n",
        "  \n",
        "  layers_values = agent._q_network.layers[0].get_weights()[0]\n",
        "  for _ in range(NUMBER_ITERATION):\n",
        "      \n",
        "      experience_replay.collect_data(train_env, agent.collect_policy, COLLECTION_STEPS_PER_ITERATION)\n",
        "      \n",
        "      experience, unused_info  = next(experience_replay.iterator)\n",
        "      train_loss = agent.train(experience).loss\n",
        "\n",
        "      new_values = agent._q_network.layers[0].get_weights()[0]\n",
        "\n",
        "      #print(\"Layers update were not changed: \", np.all(layers_values == new_values))\n",
        "      layers_values = new_values\n",
        "\n",
        "      if agent.train_step_counter.numpy() % EVAL_INTERVAL == 0:\n",
        "           #avg_return = get_average_return(evaluation_env, agent.policy, EVAL_EPISODES)\n",
        "           print('Iteration {0} – Average Return = {1}, Loss = {2}.'.format(agent.train_step_counter.numpy(), 0, train_loss))\n",
        "           #returns.append(avg_return)"
      ],
      "execution_count": 12,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:From /usr/local/lib/python3.7/dist-packages/tensorflow/python/util/dispatch.py:206: calling foldr_v2 (from tensorflow.python.ops.functional_ops) with back_prop=False is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "back_prop=False is deprecated. Consider using tf.stop_gradient instead.\n",
            "Instead of:\n",
            "results = tf.foldr(fn, elems, back_prop=False)\n",
            "Use:\n",
            "results = tf.nest.map_structure(tf.stop_gradient, tf.foldr(fn, elems))\n",
            "Iteration 1000 – Average Return = 0, Loss = 115702.8828125.\n",
            "Iteration 2000 – Average Return = 0, Loss = 58155.171875.\n",
            "Iteration 3000 – Average Return = 0, Loss = 44719.1484375.\n",
            "Iteration 4000 – Average Return = 0, Loss = 88762.2109375.\n",
            "Iteration 5000 – Average Return = 0, Loss = 41167.1484375.\n",
            "Iteration 6000 – Average Return = 0, Loss = 43977.6875.\n",
            "Iteration 7000 – Average Return = 0, Loss = 20546.9375.\n",
            "Iteration 8000 – Average Return = 0, Loss = 47082.3046875.\n",
            "Iteration 9000 – Average Return = 0, Loss = 62805.625.\n",
            "Iteration 10000 – Average Return = 0, Loss = 43495.95703125.\n",
            "Iteration 11000 – Average Return = 0, Loss = 87429.46875.\n",
            "Iteration 12000 – Average Return = 0, Loss = 89177.859375.\n",
            "Iteration 13000 – Average Return = 0, Loss = 38787.04296875.\n",
            "Iteration 14000 – Average Return = 0, Loss = 39213.5390625.\n",
            "Iteration 15000 – Average Return = 0, Loss = 38330.109375.\n",
            "Iteration 16000 – Average Return = 0, Loss = 27886.5859375.\n",
            "Iteration 17000 – Average Return = 0, Loss = 84951.703125.\n",
            "Iteration 18000 – Average Return = 0, Loss = 29886.75390625.\n",
            "Iteration 19000 – Average Return = 0, Loss = 116775.4765625.\n",
            "Iteration 20000 – Average Return = 0, Loss = 60866.18359375.\n",
            "Iteration 21000 – Average Return = 0, Loss = 64087.41796875.\n",
            "Iteration 22000 – Average Return = 0, Loss = 122921.484375.\n",
            "Iteration 23000 – Average Return = 0, Loss = 35685.05078125.\n",
            "Iteration 24000 – Average Return = 0, Loss = 98426.921875.\n",
            "Iteration 25000 – Average Return = 0, Loss = 58494.14453125.\n",
            "Iteration 26000 – Average Return = 0, Loss = 123371.4296875.\n",
            "Iteration 27000 – Average Return = 0, Loss = 107969.0625.\n",
            "Iteration 28000 – Average Return = 0, Loss = 58120.07421875.\n",
            "Iteration 29000 – Average Return = 0, Loss = 70175.546875.\n",
            "Iteration 30000 – Average Return = 0, Loss = 75801.6640625.\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "KrZghbNwLuhx"
      },
      "source": [
        "def create_policy_eval(policy, num_episodes=5):\n",
        "  for _ in range(num_episodes):\n",
        "    time_step = evaluation_env.reset()\n",
        "    #cv2_imshow(eval_py_env.render())\n",
        "    while not time_step.is_last():\n",
        "      action_step = policy.action(time_step)\n",
        "      print(action_step)\n",
        "      time_step = evaluation_env.step(action_step.action)\n",
        "      #print(time_step)\n",
        "    cv2_imshow(evaluation_env.render().numpy()[0])\n",
        "    print(time_step)\n",
        "    print(\"============================================\")"
      ],
      "execution_count": 13,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "id": "lStfqsQ9LwNV",
        "outputId": "a6858a34-5dd3-46b2-d503-b93a07f57485"
      },
      "source": [
        "create_policy_eval(agent.policy)"
      ],
      "execution_count": 14,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "PolicyStep(action=<tf.Tensor: shape=(1,), dtype=int32, numpy=array([1], dtype=int32)>, state=(), info=())\n",
            "PolicyStep(action=<tf.Tensor: shape=(1,), dtype=int32, numpy=array([17], dtype=int32)>, state=(), info=())\n",
            "PolicyStep(action=<tf.Tensor: shape=(1,), dtype=int32, numpy=array([14], dtype=int32)>, state=(), info=())\n",
            "PolicyStep(action=<tf.Tensor: shape=(1,), dtype=int32, numpy=array([32], dtype=int32)>, state=(), info=())\n",
            "PolicyStep(action=<tf.Tensor: shape=(1,), dtype=int32, numpy=array([19], dtype=int32)>, state=(), info=())\n",
            "PolicyStep(action=<tf.Tensor: shape=(1,), dtype=int32, numpy=array([0], dtype=int32)>, state=(), info=())\n",
            "PolicyStep(action=<tf.Tensor: shape=(1,), dtype=int32, numpy=array([11], dtype=int32)>, state=(), info=())\n",
            "PolicyStep(action=<tf.Tensor: shape=(1,), dtype=int32, numpy=array([17], dtype=int32)>, state=(), info=())\n",
            "PolicyStep(action=<tf.Tensor: shape=(1,), dtype=int32, numpy=array([7], dtype=int32)>, state=(), info=())\n",
            "PolicyStep(action=<tf.Tensor: shape=(1,), dtype=int32, numpy=array([20], dtype=int32)>, state=(), info=())\n",
            "PolicyStep(action=<tf.Tensor: shape=(1,), dtype=int32, numpy=array([26], dtype=int32)>, state=(), info=())\n",
            "PolicyStep(action=<tf.Tensor: shape=(1,), dtype=int32, numpy=array([4], dtype=int32)>, state=(), info=())\n",
            "PolicyStep(action=<tf.Tensor: shape=(1,), dtype=int32, numpy=array([11], dtype=int32)>, state=(), info=())\n",
            "PolicyStep(action=<tf.Tensor: shape=(1,), dtype=int32, numpy=array([43], dtype=int32)>, state=(), info=())\n",
            "PolicyStep(action=<tf.Tensor: shape=(1,), dtype=int32, numpy=array([17], dtype=int32)>, state=(), info=())\n",
            "PolicyStep(action=<tf.Tensor: shape=(1,), dtype=int32, numpy=array([14], dtype=int32)>, state=(), info=())\n",
            "PolicyStep(action=<tf.Tensor: shape=(1,), dtype=int32, numpy=array([6], dtype=int32)>, state=(), info=())\n",
            "PolicyStep(action=<tf.Tensor: shape=(1,), dtype=int32, numpy=array([17], dtype=int32)>, state=(), info=())\n",
            "PolicyStep(action=<tf.Tensor: shape=(1,), dtype=int32, numpy=array([39], dtype=int32)>, state=(), info=())\n",
            "PolicyStep(action=<tf.Tensor: shape=(1,), dtype=int32, numpy=array([40], dtype=int32)>, state=(), info=())\n",
            "PolicyStep(action=<tf.Tensor: shape=(1,), dtype=int32, numpy=array([16], dtype=int32)>, state=(), info=())\n",
            "PolicyStep(action=<tf.Tensor: shape=(1,), dtype=int32, numpy=array([6], dtype=int32)>, state=(), info=())\n",
            "PolicyStep(action=<tf.Tensor: shape=(1,), dtype=int32, numpy=array([11], dtype=int32)>, state=(), info=())\n",
            "PolicyStep(action=<tf.Tensor: shape=(1,), dtype=int32, numpy=array([43], dtype=int32)>, state=(), info=())\n",
            "PolicyStep(action=<tf.Tensor: shape=(1,), dtype=int32, numpy=array([33], dtype=int32)>, state=(), info=())\n",
            "PolicyStep(action=<tf.Tensor: shape=(1,), dtype=int32, numpy=array([9], dtype=int32)>, state=(), info=())\n",
            "PolicyStep(action=<tf.Tensor: shape=(1,), dtype=int32, numpy=array([20], dtype=int32)>, state=(), info=())\n",
            "PolicyStep(action=<tf.Tensor: shape=(1,), dtype=int32, numpy=array([38], dtype=int32)>, state=(), info=())\n",
            "PolicyStep(action=<tf.Tensor: shape=(1,), dtype=int32, numpy=array([11], dtype=int32)>, state=(), info=())\n",
            "PolicyStep(action=<tf.Tensor: shape=(1,), dtype=int32, numpy=array([8], dtype=int32)>, state=(), info=())\n",
            "[((0, 255), (40, 255)), ((40, 255), (80, 255)), ((80, 255), (120, 255)), ((120, 255), (156, 273)), ((156, 273), (192, 291)), ((192, 291), (228, 308)), ((228, 308), (264, 326)), ((264, 326), (300, 343)), ((300, 343), (336, 361)), ((336, 361), (376, 353)), ((376, 353), (415, 344)), ((415, 344), (381, 322)), ((415, 344), (406, 305)), ((415, 344), (437, 310)), ((415, 344), (454, 336)), ((415, 344), (448, 365)), ((415, 344), (423, 383)), ((381, 322), (347, 301)), ((454, 336), (493, 328)), ((448, 365), (482, 387)), ((347, 301), (314, 279))]\n",
            "[1 0 1 0 0 2 0 0 0 0 2 2 0 0 0 2 1 0 0 1 0]\n",
            "[((0, 255), (40, 255)), ((40, 255), (80, 255)), ((80, 255), (120, 255)), ((120, 255), (156, 273)), ((156, 273), (192, 291)), ((192, 291), (228, 308)), ((228, 308), (264, 326)), ((264, 326), (300, 343)), ((300, 343), (336, 361)), ((336, 361), (376, 353)), ((376, 353), (415, 344)), ((415, 344), (381, 322)), ((415, 344), (406, 305)), ((415, 344), (437, 310)), ((415, 344), (454, 336)), ((415, 344), (448, 365)), ((415, 344), (423, 383)), ((381, 322), (347, 301)), ((454, 336), (493, 328)), ((448, 365), (482, 387)), ((347, 301), (314, 279))]\n",
            "[1 0 1 0 0 2 0 0 0 0 2 2 0 0 0 2 1 0 0 1 0]\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAgAAAAIACAAAAADRE4smAAAHhklEQVR4nO3dwXbbNhRFUbqr///L7KCxm9ikBJBQBby796BNVidqcHkoOba0bdn2dz+Ad/vr3Q+A9zKAcAYQzgDCGUA4AwhnAOEMIJwBhDOAcAYQzgDCGUA4AwhnAOEMIJwBhDOAcAYQzgDCGUA4AwhnAOEMIJwBhDOAcAYQzgDCGUA4AwhnAOEMIJwBhDOAcAYQzgDCGUA4AwhnAOEMIJwBhDOAcAYQzgDCGUA4AwhnAOEMIJwBhDOAcAYQzgDCGUA4AwhnAOEMIJwBhDOAcAYQzgDCGUA4AwhnAOEMIJwBhDOAcAYQzgDCGUA4AwhnAOEMIJwBhDOAcAYQzgDCGUA4AwhnAOEMIJwBhDOAcAYQzgDCGUA4AwhnAOEMIJwBhDOAcAYQzgDCGUA4AwhnAOEMIJwBhDOAcAYQzgDCGUA4AwhnAOEMIJwBhDOAcAYQzgDCGUA4AwhnAOEMIJwBhDOAcAYQzgDCGUA4AwhnAOEMgGz7ux/AmykA4cIToACky06AAhAvOgEKAMkJUABIToACQHICFAC24AQoAGxbbgIUALZti02AAsC/MhOgAPBLZAIUAD4lJkAB4EtgAhQA/pOXAAWA38QlQAHgd2kJUAD4Q1gCFAD+lJUABYBvohKgAPBdUgIUAH4ISoACwE85CVAAOBCTAAWAIykJUAA4FJIABYBjGQlQADgRkQAFgDMJCVAAOBWQAAWAc/UToADwQPkEKAA8Uj0BCgAPFU+AAsBjtROgAPBE6QQoQDgDgGcq3wMUAJ4qnAAFgOfqJkABoEHZBCgAtKiaAAWAJkUToADQpmYCFAAalUyAAkCriglQgHAGAM0K3gMUANrVS4ACQIdyCVAA6FEtAQoAXYolQAGgT60EKAB0KpUABYBelRKgANCtUAIUAPrVSYACwAVlEqAAcEWVBCgAXFIkAQoA19RIgALARSUSoABwVYUEKABcViABCgDXrZ8ABYAblk+AAsAdqydAAeCWxROgAHDP2glQALhp6QQoANy1cgIUAG5bOAEKAPetmwAFCGcAMMCy9wAFgBFWTYACwBCLJkABYIw1E6AAMMiSCVAAGGXFBCgADLNgAhQAxlkvAQoAAy2XAAUIZwAw0mr3AAWAoRZLgALAWGslQAFgsKUSoAAw2koJUAAYbqEEKACMt04CFABeYJkEKAC8wioJUAB4iUUSoADwGmskQAHgRZZIgALAq6yQAAWAl1kgAQoArzN/AhTg2Pwnx2sNWsD0Q1IAOJSSAM6MOToDWJYEpJOAdAkL8CoATiUkgEeGnJ0BrEsC0klAuuoL8CoAHqqeAJ4ZcXgGsDAJSCcB6SovwKsAeKpyAmgx4PQmHYBbADS4f/1OWgAa3T2/Wc/fLQCa3LyEZy0Aze4d4awDcAuARncu4lkDQI8bpzjtANwCoNn163jaAtDl8jkaQA1Xz9H5V3HxJOcdgCeB0OPatTxvAeh16SwNoI4rZ+n8K7lwmgZQSv9xTjwArwKgV/cFPXEBuKD3PJ1/NZ0nagDl9B3pzAPwJBAu6LqoZy4A1/ScqfOvqONUDaCk9mOdegCeBMI1zRf21AXgusaDnfv83QLgqrZrWwGgqKaLe+4CcEvD4U5+/m4BcMPz63vyAnDT0/OdfABuAXDLkyt88gAoANz0+BqfvQDc9/CMDaC+R2fs/BM8OGUDiHB+zNMPwKsAuO/sQp8+AAxyctLzD8AtAEY4vtYVAEIcXezzB4BxDk57gQG4BcAgP6/3BQrAQN/Pe4XzdwuAYfaHv6W+/cHvCLCf/oYM+8mvSbEf/nJeXgXAUPvBr0iyf/v33P5+9wNYzr5tH+9+DLzXvu8NPwugAGV9bP8eb4USVPh/eJt92w7/BPePz39Q39H9YN9WuQO4Bdy2+P1g0Yc9oT/uB/vHKreANR7lMr5SsK/yJ+sWMNR694N1HulSzl4fzEcBXuZjiRXM/vjWtH98PQmYfQUTP7Rl7dvHt5cBs6+AkfbPf3z/FtHHf4PwJp4DDHb++u/zFcJcKZjqwazv6/gf/XXAVCtQgJHavvr36ytF2zwrYIj9x3eDPbvnT/msgIuOviN8/vN1CxhkmS/+8xJnPxU4fwIY4Met3PeGRzn8jqCz/0Y1B8/k/YRgkKfvDjb3ArwKuMeT/2hnX8fxPhEZGt8j3ABqOv8y7orvFkavns+JmXgBngRe48lftt5Pips4AQy31PuGe48gGOzwap82AQy36qfHMMayHyDGGGt9hqAngTBW81eIqWnhT5JmAJ8lH67hLSSprPevCd7MqwAY6vFFPl8CGOvpz4P+L4+inVsAjPT0Cp8tAYz1/HznWoBbAAzUcnlPlQAFgIGaru6pEsBQbWc70QLcAmCc1kt7ogQwUvPBTrMAtwAYp/3CniYBjNRxrBZQUM+hGkBBXYdqAfX0nekUC/AqAIbpvKanSADjdB/oBAtwC4BR+i/o9ydAAWCQK5fz+xPAMJcO890LcAuAQa5dzO9OAMNcPEoLKOLqQRpAEZcP0gJqWPMcvQoIZwAwxJp3AAVIZwDhDCCcAcAIiz4HVIB0BgAAAAAAAAAAAFDGPypujeQWjptsAAAAAElFTkSuQmCC\n",
            "text/plain": [
              "<PIL.Image.Image image mode=L size=512x512 at 0x7F93853BA110>"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "text": [
            "TimeStep(\n",
            "{'discount': <tf.Tensor: shape=(1,), dtype=float32, numpy=array([0.], dtype=float32)>,\n",
            " 'observation': <tf.Tensor: shape=(1, 21), dtype=float64, numpy=\n",
            "array([[1., 0., 1., 0., 0., 2., 0., 0., 0., 0., 2., 2., 0., 0., 0., 2.,\n",
            "        1., 0., 0., 1., 0.]])>,\n",
            " 'reward': <tf.Tensor: shape=(1,), dtype=float32, numpy=array([-1000.], dtype=float32)>,\n",
            " 'step_type': <tf.Tensor: shape=(1,), dtype=int32, numpy=array([2], dtype=int32)>})\n",
            "============================================\n",
            "PolicyStep(action=<tf.Tensor: shape=(1,), dtype=int32, numpy=array([1], dtype=int32)>, state=(), info=())\n",
            "PolicyStep(action=<tf.Tensor: shape=(1,), dtype=int32, numpy=array([17], dtype=int32)>, state=(), info=())\n",
            "PolicyStep(action=<tf.Tensor: shape=(1,), dtype=int32, numpy=array([14], dtype=int32)>, state=(), info=())\n",
            "PolicyStep(action=<tf.Tensor: shape=(1,), dtype=int32, numpy=array([32], dtype=int32)>, state=(), info=())\n",
            "PolicyStep(action=<tf.Tensor: shape=(1,), dtype=int32, numpy=array([19], dtype=int32)>, state=(), info=())\n",
            "PolicyStep(action=<tf.Tensor: shape=(1,), dtype=int32, numpy=array([0], dtype=int32)>, state=(), info=())\n",
            "PolicyStep(action=<tf.Tensor: shape=(1,), dtype=int32, numpy=array([11], dtype=int32)>, state=(), info=())\n",
            "PolicyStep(action=<tf.Tensor: shape=(1,), dtype=int32, numpy=array([17], dtype=int32)>, state=(), info=())\n",
            "PolicyStep(action=<tf.Tensor: shape=(1,), dtype=int32, numpy=array([7], dtype=int32)>, state=(), info=())\n",
            "PolicyStep(action=<tf.Tensor: shape=(1,), dtype=int32, numpy=array([20], dtype=int32)>, state=(), info=())\n",
            "PolicyStep(action=<tf.Tensor: shape=(1,), dtype=int32, numpy=array([26], dtype=int32)>, state=(), info=())\n",
            "PolicyStep(action=<tf.Tensor: shape=(1,), dtype=int32, numpy=array([4], dtype=int32)>, state=(), info=())\n",
            "PolicyStep(action=<tf.Tensor: shape=(1,), dtype=int32, numpy=array([11], dtype=int32)>, state=(), info=())\n",
            "PolicyStep(action=<tf.Tensor: shape=(1,), dtype=int32, numpy=array([43], dtype=int32)>, state=(), info=())\n",
            "PolicyStep(action=<tf.Tensor: shape=(1,), dtype=int32, numpy=array([17], dtype=int32)>, state=(), info=())\n",
            "PolicyStep(action=<tf.Tensor: shape=(1,), dtype=int32, numpy=array([14], dtype=int32)>, state=(), info=())\n",
            "PolicyStep(action=<tf.Tensor: shape=(1,), dtype=int32, numpy=array([6], dtype=int32)>, state=(), info=())\n",
            "PolicyStep(action=<tf.Tensor: shape=(1,), dtype=int32, numpy=array([17], dtype=int32)>, state=(), info=())\n",
            "PolicyStep(action=<tf.Tensor: shape=(1,), dtype=int32, numpy=array([39], dtype=int32)>, state=(), info=())\n",
            "PolicyStep(action=<tf.Tensor: shape=(1,), dtype=int32, numpy=array([40], dtype=int32)>, state=(), info=())\n",
            "PolicyStep(action=<tf.Tensor: shape=(1,), dtype=int32, numpy=array([16], dtype=int32)>, state=(), info=())\n",
            "PolicyStep(action=<tf.Tensor: shape=(1,), dtype=int32, numpy=array([6], dtype=int32)>, state=(), info=())\n",
            "PolicyStep(action=<tf.Tensor: shape=(1,), dtype=int32, numpy=array([11], dtype=int32)>, state=(), info=())\n",
            "PolicyStep(action=<tf.Tensor: shape=(1,), dtype=int32, numpy=array([43], dtype=int32)>, state=(), info=())\n",
            "PolicyStep(action=<tf.Tensor: shape=(1,), dtype=int32, numpy=array([33], dtype=int32)>, state=(), info=())\n",
            "PolicyStep(action=<tf.Tensor: shape=(1,), dtype=int32, numpy=array([9], dtype=int32)>, state=(), info=())\n",
            "PolicyStep(action=<tf.Tensor: shape=(1,), dtype=int32, numpy=array([20], dtype=int32)>, state=(), info=())\n",
            "PolicyStep(action=<tf.Tensor: shape=(1,), dtype=int32, numpy=array([38], dtype=int32)>, state=(), info=())\n",
            "PolicyStep(action=<tf.Tensor: shape=(1,), dtype=int32, numpy=array([11], dtype=int32)>, state=(), info=())\n",
            "PolicyStep(action=<tf.Tensor: shape=(1,), dtype=int32, numpy=array([8], dtype=int32)>, state=(), info=())\n",
            "[((0, 255), (40, 255)), ((40, 255), (80, 255)), ((80, 255), (120, 255)), ((120, 255), (156, 274)), ((156, 274), (191, 292)), ((191, 292), (227, 310)), ((227, 310), (263, 328)), ((263, 328), (298, 346)), ((298, 346), (334, 364)), ((334, 364), (374, 357)), ((374, 357), (413, 351)), ((413, 351), (380, 329)), ((413, 351), (406, 311)), ((413, 351), (436, 319)), ((413, 351), (453, 344)), ((413, 351), (446, 374)), ((413, 351), (420, 390)), ((380, 329), (347, 306)), ((453, 344), (492, 337)), ((446, 374), (479, 397)), ((347, 306), (314, 283))]\n",
            "[1 0 1 0 0 2 0 0 0 0 2 2 0 0 0 2 1 0 0 1 0]\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAgAAAAIACAAAAADRE4smAAAHi0lEQVR4nO3d23LbxhZFUepU/v+XcR5sx5ZJgrg0gt17jVEpR84To16YIGWLejyyLXc/gLv97+4HwL0MIJwBhDOAcAYQzgDCGUA4AwhnAOEMIJwBhDOAcAYQzgDCGUA4AwhnAOEMIJwBhDOAcAYQzgDCGUA4AwhnAOEMIJwBhDOAcAYQzgDCGUA4AwhnAOEMIJwBhDOAcAYQzgDCGUA4AwhnAOEMIJwBhDOAcAYQzgDCGUA4AwhnAOEMIJwBhDOAcAYQzgDCGUA4AwhnAOEMIJwBhDOAcAYQzgDCGUA4AwhnAOEMIJwBhDOAcAYQzgDCGUA4AwhnAOEMIJwBhDOAcAYQzgDCGUA4AwhnAOEMIJwBhDOAcAYQzgDCGUA4AwhnAOEMIJwBhDOAcAYQzgDCGUA4AwhnAOEMIJwBhDOAcAYQzgDCGUA4AwhnAOEMIJwBhDOAcAYQzgDCGUA4AwhnAOEMIJwBhDOAcAYQzgDCGUA4AyDbcvcDuJkCEC48AQpAuuwEKADxohOgAJCcAAWA5AQoACQnQAHgEZwABYDHIzgB/BS6ALcA+CEzAQoAP0UmQAHgl8QEKAD8KzABCgC/5SVAAeAPcQlQAPhTWgIUAL4JS4ACwHdhCeBJ1ALcAuBvSQlQAHgSlAAFgGc5CVAAeCEmAQoAr6QkQAHgpZAEKAC8lpEABYA3IhKgAPBOQgIUAN4KSIACwHv9E6AAsKJ9AhQA1nRPgALAquYJUABY1zsBCgAftE6AAsAnrRPABo0X4BYAn/VNgALABm0ToACwRdcEKABs0jQBCgDb9EyAAsBGLROgALBVxwQoAGzWMAEKANv1S4ACwA7tEqAAsEe3BCgA7NIsAQoA+/RKgALATq0SoACwV6cEKADs1igBCgD79UmAAsABbRKgAHBElwQoABzSJAEKAMf0SIACwEEtEqAAcFSHBCgAHNYgAQoAx82fAAWAE6ZPgALAGbMnQAHglMkToABwztwJUAA4aeoEKACcNXMCFABOmzgBCgDnzZsABYABpk2AAsAIsyZAAWCISROgADDGnAlQABhkygQoAIwyYwIUAIaZMAEKAOPMlwAFgIGmS4ACwEizJUABYKjJEqAAMNZcCVAAGGyqBCgAjDZTAhQAhpsoAQoA482TAAWAC0yTAAWAK8ySAAWAS0ySAAWAa0ySAC4zxQLcAuAqMyRAAeAyEyRAAeA69ROgAHCh8glQALhS9QQoAFyqeAIUAK5VOwEK8F7tk+N6gxZQekgKAG8lJIA1Y47OAKYlAekkIJ0FpBtydgYwLwlIJwHpWi/AVwLhk9YJYIMRh2cAE5OAdBKQzgLSDTg9A5iZBKSTgHTnj6/iAHwlEDY6ewFXDAB7nDzBkgNwC4DNzl3DJQvALqfO0AAaOHOIBtDAiUN0/i0cP0YD6OHwOdYcgJeBsM/RK7lmAdjv4EkaQBfHTtL593HoLA2gkSOHWXQAXgXAfgcu56IF4Jjdx+n8m9l7oFUH4DkAHLLzkq5aAA7bd6RVB+AWAAftuairBoAzdpxq2QG4BcBh26/rsgXglM3nagBNbT1YA2hq48E6/7a2Ha0B9LXpbOsOwMtAOGfL1V23AJy34XQNoLPPp+v8e/t4vgbQ3KcDNoDuPpxw4QF4GQgDrF/jhQvAIGtnXPn83QJgiJXLvHIBGOb9MVcegFsADPL2Qq9cAAZ6c9Clz98tAIZ5fa2XLgBDvTzr0gNwC4CBXl3tpQvAYC9O2wCSPJ+288/ydN4GEObvA689AK8CYLBl9bf0t6z8jgDL29+QYXnzMSmWlx9W5FUAXGB58RFJlqcPanILgEss3/5FnuWPX+v65+4HMKnl8fi6+zEM0eP/4hY/ru2VT+Dy9euXwoo/vPrWZrB81R+AW8BJP873Yw3KmvExl7U8Ht8/o8tX9QAYwHDfYrCU/wS7BYw22T1hgoc4sad7Qj0KcLWv2jGo+rhaWL5+PwmoGoOCD6mN5evnP7//y+PxKPY5L/VgWlkeX2++FFRqBp4DXGTlCwB/vE64fwX3P4KWft36P/1xQKkYMMxffx3o45uJL9X/0Jhdnv5CoPNNsjz/hVADCPLym0IsIMabbwsruwAvA4eq/6d/XGnlvSHKJoBhnl/M+SbRJB/eHsoAmvv4/nAW0NnLr+V5r5gY294luuICvAwcwIu/bO8ubG8ammHPT4qptwC3gJPkP9veHxhaLwFcZoofH+ItYuAaM/4QOQba/uqQlub4QaKeA8A15vxp4gyzcswWEGDvl4hoZvWQLaC/9TMuswCvAuASH67xMgngIh+/Jfg/eRTc5eP5GkBvn8/XAlrbcLwW0NmW062wAC8D4QLbLu4KCeASG4/2/gW4BcAFtl7a9yeAS2w+2LsX4BYA4+24ru9OAFfYc6oW0NCuQ711AZ4DwHj7Lmo3gW72nuiNC3ALgOF2X9FuAr3sP8/bFuAWAMMduJ7dBBo5dJgW0Mexs7xnAZ4DwGgHr2U3gS6OnuQdC3ALgMGOX8huAi1MdYxuAeEMIJwBwFhTPQVQgHQGEM4AwhlAOAOAoeZ6EaAA6QwAAAAAAAAAAAAAevg/FiiOz1lYuTQAAAAASUVORK5CYII=\n",
            "text/plain": [
              "<PIL.Image.Image image mode=L size=512x512 at 0x7F93823E0AD0>"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "text": [
            "TimeStep(\n",
            "{'discount': <tf.Tensor: shape=(1,), dtype=float32, numpy=array([0.], dtype=float32)>,\n",
            " 'observation': <tf.Tensor: shape=(1, 21), dtype=float64, numpy=\n",
            "array([[1., 0., 1., 0., 0., 2., 0., 0., 0., 0., 2., 2., 0., 0., 0., 2.,\n",
            "        1., 0., 0., 1., 0.]])>,\n",
            " 'reward': <tf.Tensor: shape=(1,), dtype=float32, numpy=array([-1000.], dtype=float32)>,\n",
            " 'step_type': <tf.Tensor: shape=(1,), dtype=int32, numpy=array([2], dtype=int32)>})\n",
            "============================================\n",
            "PolicyStep(action=<tf.Tensor: shape=(1,), dtype=int32, numpy=array([1], dtype=int32)>, state=(), info=())\n",
            "PolicyStep(action=<tf.Tensor: shape=(1,), dtype=int32, numpy=array([17], dtype=int32)>, state=(), info=())\n",
            "PolicyStep(action=<tf.Tensor: shape=(1,), dtype=int32, numpy=array([14], dtype=int32)>, state=(), info=())\n",
            "PolicyStep(action=<tf.Tensor: shape=(1,), dtype=int32, numpy=array([32], dtype=int32)>, state=(), info=())\n",
            "PolicyStep(action=<tf.Tensor: shape=(1,), dtype=int32, numpy=array([19], dtype=int32)>, state=(), info=())\n",
            "PolicyStep(action=<tf.Tensor: shape=(1,), dtype=int32, numpy=array([0], dtype=int32)>, state=(), info=())\n",
            "PolicyStep(action=<tf.Tensor: shape=(1,), dtype=int32, numpy=array([11], dtype=int32)>, state=(), info=())\n",
            "PolicyStep(action=<tf.Tensor: shape=(1,), dtype=int32, numpy=array([17], dtype=int32)>, state=(), info=())\n",
            "PolicyStep(action=<tf.Tensor: shape=(1,), dtype=int32, numpy=array([7], dtype=int32)>, state=(), info=())\n",
            "PolicyStep(action=<tf.Tensor: shape=(1,), dtype=int32, numpy=array([20], dtype=int32)>, state=(), info=())\n",
            "PolicyStep(action=<tf.Tensor: shape=(1,), dtype=int32, numpy=array([26], dtype=int32)>, state=(), info=())\n",
            "PolicyStep(action=<tf.Tensor: shape=(1,), dtype=int32, numpy=array([4], dtype=int32)>, state=(), info=())\n",
            "PolicyStep(action=<tf.Tensor: shape=(1,), dtype=int32, numpy=array([11], dtype=int32)>, state=(), info=())\n",
            "PolicyStep(action=<tf.Tensor: shape=(1,), dtype=int32, numpy=array([43], dtype=int32)>, state=(), info=())\n",
            "PolicyStep(action=<tf.Tensor: shape=(1,), dtype=int32, numpy=array([17], dtype=int32)>, state=(), info=())\n",
            "PolicyStep(action=<tf.Tensor: shape=(1,), dtype=int32, numpy=array([14], dtype=int32)>, state=(), info=())\n",
            "PolicyStep(action=<tf.Tensor: shape=(1,), dtype=int32, numpy=array([6], dtype=int32)>, state=(), info=())\n",
            "PolicyStep(action=<tf.Tensor: shape=(1,), dtype=int32, numpy=array([17], dtype=int32)>, state=(), info=())\n",
            "PolicyStep(action=<tf.Tensor: shape=(1,), dtype=int32, numpy=array([39], dtype=int32)>, state=(), info=())\n",
            "PolicyStep(action=<tf.Tensor: shape=(1,), dtype=int32, numpy=array([40], dtype=int32)>, state=(), info=())\n",
            "PolicyStep(action=<tf.Tensor: shape=(1,), dtype=int32, numpy=array([16], dtype=int32)>, state=(), info=())\n",
            "PolicyStep(action=<tf.Tensor: shape=(1,), dtype=int32, numpy=array([6], dtype=int32)>, state=(), info=())\n",
            "PolicyStep(action=<tf.Tensor: shape=(1,), dtype=int32, numpy=array([11], dtype=int32)>, state=(), info=())\n",
            "PolicyStep(action=<tf.Tensor: shape=(1,), dtype=int32, numpy=array([43], dtype=int32)>, state=(), info=())\n",
            "PolicyStep(action=<tf.Tensor: shape=(1,), dtype=int32, numpy=array([33], dtype=int32)>, state=(), info=())\n",
            "PolicyStep(action=<tf.Tensor: shape=(1,), dtype=int32, numpy=array([9], dtype=int32)>, state=(), info=())\n",
            "PolicyStep(action=<tf.Tensor: shape=(1,), dtype=int32, numpy=array([20], dtype=int32)>, state=(), info=())\n",
            "PolicyStep(action=<tf.Tensor: shape=(1,), dtype=int32, numpy=array([38], dtype=int32)>, state=(), info=())\n",
            "PolicyStep(action=<tf.Tensor: shape=(1,), dtype=int32, numpy=array([11], dtype=int32)>, state=(), info=())\n",
            "PolicyStep(action=<tf.Tensor: shape=(1,), dtype=int32, numpy=array([8], dtype=int32)>, state=(), info=())\n",
            "[((0, 255), (40, 255)), ((40, 255), (80, 255)), ((80, 255), (120, 255)), ((120, 255), (157, 271)), ((157, 271), (193, 286)), ((193, 286), (229, 302)), ((229, 302), (266, 318)), ((266, 318), (303, 334)), ((303, 334), (339, 349)), ((339, 349), (378, 340)), ((378, 340), (417, 331)), ((417, 331), (383, 310)), ((417, 331), (408, 292)), ((417, 331), (438, 297)), ((417, 331), (456, 322)), ((417, 331), (451, 352)), ((417, 331), (426, 370)), ((383, 310), (349, 289)), ((456, 322), (495, 313)), ((451, 352), (485, 374)), ((349, 289), (315, 268))]\n",
            "[1 0 1 0 0 2 0 0 0 0 2 2 0 0 0 2 1 0 0 1 0]\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAgAAAAIACAAAAADRE4smAAAHaklEQVR4nO3dyXLbSBAEUGjC///LmMPItiSCYC+YIKrzvYNN3Riq7CxwEblt2fZ334F3++fdd4D3EoBwAhBOAMIJQDgBCCcA4QQgnACEE4BwAhBOAMIJQDgBCCcA4QQgnACEE4BwAhBOAMIJQDgBCCcA4QQgnACEE4BwAhBOAMIJQDgBCCcA4QQgnACEE4BwAhBOAMIJQDgBCCcA4QQgnACEE4BwAhBOAMIJQDgBCCcA4QQgnACEE4BwAhBOAMIJQDgBCCcA4QQgnACEE4BwAhBOAMIJQDgBCCcA4QQgnACEE4BwAhBOAMIJQDgBCCcA4QQgnACEE4BwAhBOAMIJQDgBCCcA4QQgnACEE4BwAhBOAMIJQDgBCCcA4QQgnACEE4BwAhBOAMIJQDgBCCcA4QQgnACEE4BwAhBOAMIJQDgBCCcA4QQgnACEE4BwAhBOAMIJQDgBCCcA4QQgnACEE4BwAhBOAMIJQDgBCCcA4QQgnACQbX/3HXgzDUC48ArQAKTLrgANEE4AiBe9AzQAJFeABoDkCtAA4QQAkneABoAtuAI0AGxbbgVoANi2LbYCNEA4AYD/ZO4ADQCfIitAA8BviRWgAcIJAPwRuAM0APyVVwEaAL6IqwANEE4A4Ku0HaAB4JuwCtAA8F1WBWiAcAIAP0TtAA0APyVVgAYIJwDwIGgHaAB4lFMBGgAOxFSABggnAHAkZQdoADgUUgEaIJwAwLGMHaAB4ImICtAA8ExCBWiAcAIATwXsAA0Az61fARognADAieV3gAaAM6tXgAaAU4tXgAaAc2tXgAYIJwDwwtI7QAPAKytXgAaAlxauAA0Ar61bARognABAg2V3gAaAFqtWgAaAJotWgAaANmtWgAYIJwDQaMkdoAGg1YoVoAGg2YIVoAGg3XoVoAHCCQB0WG4HaADosVoFaADoslgFaIBwAgB91toBGgA6LVUBGgB6rVQBGiCcAEC3hXaABoB+61SABoABy1SABoARq1SABggnADBkkR2gAWDMGhWgAWDQEhWgAcIJAIxaYQdoABi2QAVoABhXvwI0QDgBgAnld4AGgBnVK0ADwJTiFaABwgkAzKm9AzQATCpdARoAZlWuAA0QTgBgWuEdoAFgXt0K0ABwgbIVoAHgClUrQAOEEwC4RNEdoAHgGjUrQAPARUpWgAYIJwBwlYo7QAPAZQpWgAaA69SrAA0QTgDgQuV2gAaAK1WrAA0AlypWARognADAtWrtAA0AFytVARrgi1KT439wTQLkqKyLRicBZamAcCognQpIl5YADwPhm0sOb50G4EFWAqwA+CGqAjQA/HTF6a3SABwJSoAVAI9yKkADwIGcCuDYBdMrEQArAA6lVADPzI9PAEpTAekiKsBFIDwzfYALNABnJgdYYf5WADw3d4YrNADnpmYoAPXNzND8VzAxxRIBcBEIp8bPcYkG4KXROdaYvxUALwweZQ0Aaxg7yzUagBYjsywyfysAXhs4zkUagDb94xSApXSP0/wX0zvQKgFwEQhNOo90lQagWddIy8zfCoBGPadaA8ByOo51mQagR/NY68zfCoB2rSe7TgPQp3GyArCqtsma/7qaZlsoAC4CoUvL6S7UAHR7Pd1K87cCoNPLA16pARjwasCVAmAFQLfzI16pADQADDg95KUagDFnQxaAACdDNv8Iz8dcKwAuAmHI04NeqwEY9mTQxeZvBcCg47NerAGYcDjrYgGwAmDYwWkvVgAaACY8nvdqDcCch3kLQJaf8zb/NPvpj/fnIhCm7Cc/kWB/crsGKwAm7Yc3ybEf3KrCCoBp+4//C/n17jtQwb5tH+++D7zXvu8NfwtSsAFodxaC/cu/pVgB7T62zxE/2we7RZHhqAr2rWQBMOohBHvNAFgBg17ugyKK3/0b2LfP3+Je8pepAWb9roKK09/K3u0bKroPNMCVPupVQa17e1/79vH7IqBeCJj2+SzAn8eB+4unju/DCrjA4/W/Kkjy5ZWgxycI794EEjrrz/HfP7bjlwNu/fjACpjzt/2fvxJkH6zr8f2gp+8auPs+oM+3cTa+LUwG1nH8N0HmG+Lx1eCHWxW4CBxT86U/rnL6ySC1KoB+R9dxVf9E1Aropv2ztXw+aKUKoMuzR/FlPynGCujinf/RTp7E82lh6zt7DtfHBa6v85viJCBK3U+N9gkhMK3wN0doAJjm24PCFf4GOSsAZvkS0XCVv0jaCoBZPS8SsJ6GT5BkZZ0vE7Ga8xnfPAEuAmHOqyN+8wpg1ssB3zoBVgDMeX3Ab10BzGoYrwQsrGW4ArCwpuHeNwEuAmFK2+G+bwUwp3Wyd02AFQAzmk/2XSuAOe1zlYAVdUxVAFbUM1UJWFDXUO+YAI8CYELfob5jBTCjd6L3S4AVAOO6T/T9KoAZ/fOUgJUMTFMAVjIyzZslwEUgDBs6zTerAMYNjvJWCbACYNToUb5VBTBueJA3SoAVAIMmzvGNKoBhS0zRCggnADBmiQ2gAdIJQDgBgCFrXAJogHQCAAAAAAAAAACwmH8BypWM+rekaQIAAAAASUVORK5CYII=\n",
            "text/plain": [
              "<PIL.Image.Image image mode=L size=512x512 at 0x7F93823E9690>"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "text": [
            "TimeStep(\n",
            "{'discount': <tf.Tensor: shape=(1,), dtype=float32, numpy=array([0.], dtype=float32)>,\n",
            " 'observation': <tf.Tensor: shape=(1, 21), dtype=float64, numpy=\n",
            "array([[1., 0., 1., 0., 0., 2., 0., 0., 0., 0., 2., 2., 0., 0., 0., 2.,\n",
            "        1., 0., 0., 1., 0.]])>,\n",
            " 'reward': <tf.Tensor: shape=(1,), dtype=float32, numpy=array([-1000.], dtype=float32)>,\n",
            " 'step_type': <tf.Tensor: shape=(1,), dtype=int32, numpy=array([2], dtype=int32)>})\n",
            "============================================\n",
            "PolicyStep(action=<tf.Tensor: shape=(1,), dtype=int32, numpy=array([1], dtype=int32)>, state=(), info=())\n",
            "PolicyStep(action=<tf.Tensor: shape=(1,), dtype=int32, numpy=array([17], dtype=int32)>, state=(), info=())\n",
            "PolicyStep(action=<tf.Tensor: shape=(1,), dtype=int32, numpy=array([14], dtype=int32)>, state=(), info=())\n",
            "PolicyStep(action=<tf.Tensor: shape=(1,), dtype=int32, numpy=array([32], dtype=int32)>, state=(), info=())\n",
            "PolicyStep(action=<tf.Tensor: shape=(1,), dtype=int32, numpy=array([19], dtype=int32)>, state=(), info=())\n",
            "PolicyStep(action=<tf.Tensor: shape=(1,), dtype=int32, numpy=array([0], dtype=int32)>, state=(), info=())\n",
            "PolicyStep(action=<tf.Tensor: shape=(1,), dtype=int32, numpy=array([11], dtype=int32)>, state=(), info=())\n",
            "PolicyStep(action=<tf.Tensor: shape=(1,), dtype=int32, numpy=array([17], dtype=int32)>, state=(), info=())\n",
            "PolicyStep(action=<tf.Tensor: shape=(1,), dtype=int32, numpy=array([7], dtype=int32)>, state=(), info=())\n",
            "PolicyStep(action=<tf.Tensor: shape=(1,), dtype=int32, numpy=array([20], dtype=int32)>, state=(), info=())\n",
            "PolicyStep(action=<tf.Tensor: shape=(1,), dtype=int32, numpy=array([26], dtype=int32)>, state=(), info=())\n",
            "PolicyStep(action=<tf.Tensor: shape=(1,), dtype=int32, numpy=array([4], dtype=int32)>, state=(), info=())\n",
            "PolicyStep(action=<tf.Tensor: shape=(1,), dtype=int32, numpy=array([11], dtype=int32)>, state=(), info=())\n",
            "PolicyStep(action=<tf.Tensor: shape=(1,), dtype=int32, numpy=array([43], dtype=int32)>, state=(), info=())\n",
            "PolicyStep(action=<tf.Tensor: shape=(1,), dtype=int32, numpy=array([17], dtype=int32)>, state=(), info=())\n",
            "PolicyStep(action=<tf.Tensor: shape=(1,), dtype=int32, numpy=array([14], dtype=int32)>, state=(), info=())\n",
            "PolicyStep(action=<tf.Tensor: shape=(1,), dtype=int32, numpy=array([6], dtype=int32)>, state=(), info=())\n",
            "PolicyStep(action=<tf.Tensor: shape=(1,), dtype=int32, numpy=array([17], dtype=int32)>, state=(), info=())\n",
            "PolicyStep(action=<tf.Tensor: shape=(1,), dtype=int32, numpy=array([39], dtype=int32)>, state=(), info=())\n",
            "PolicyStep(action=<tf.Tensor: shape=(1,), dtype=int32, numpy=array([40], dtype=int32)>, state=(), info=())\n",
            "PolicyStep(action=<tf.Tensor: shape=(1,), dtype=int32, numpy=array([16], dtype=int32)>, state=(), info=())\n",
            "PolicyStep(action=<tf.Tensor: shape=(1,), dtype=int32, numpy=array([6], dtype=int32)>, state=(), info=())\n",
            "PolicyStep(action=<tf.Tensor: shape=(1,), dtype=int32, numpy=array([11], dtype=int32)>, state=(), info=())\n",
            "PolicyStep(action=<tf.Tensor: shape=(1,), dtype=int32, numpy=array([43], dtype=int32)>, state=(), info=())\n",
            "PolicyStep(action=<tf.Tensor: shape=(1,), dtype=int32, numpy=array([33], dtype=int32)>, state=(), info=())\n",
            "PolicyStep(action=<tf.Tensor: shape=(1,), dtype=int32, numpy=array([9], dtype=int32)>, state=(), info=())\n",
            "PolicyStep(action=<tf.Tensor: shape=(1,), dtype=int32, numpy=array([20], dtype=int32)>, state=(), info=())\n",
            "PolicyStep(action=<tf.Tensor: shape=(1,), dtype=int32, numpy=array([38], dtype=int32)>, state=(), info=())\n",
            "PolicyStep(action=<tf.Tensor: shape=(1,), dtype=int32, numpy=array([11], dtype=int32)>, state=(), info=())\n",
            "PolicyStep(action=<tf.Tensor: shape=(1,), dtype=int32, numpy=array([8], dtype=int32)>, state=(), info=())\n",
            "[((0, 255), (40, 255)), ((40, 255), (80, 255)), ((80, 255), (120, 255)), ((120, 255), (157, 269)), ((157, 269), (195, 283)), ((195, 283), (233, 296)), ((233, 296), (271, 309)), ((271, 309), (308, 323)), ((308, 323), (346, 337)), ((346, 337), (384, 324)), ((384, 324), (422, 312)), ((422, 312), (387, 294)), ((422, 312), (410, 274)), ((422, 312), (441, 276)), ((422, 312), (460, 299)), ((422, 312), (458, 330)), ((422, 312), (434, 350)), ((387, 294), (351, 275)), ((460, 299), (498, 287)), ((458, 330), (494, 348)), ((351, 275), (316, 257))]\n",
            "[1 0 1 0 0 2 0 0 0 0 2 2 0 0 0 2 1 0 0 1 0]\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAgAAAAIACAAAAADRE4smAAAHXklEQVR4nO3d227bSBAEUHqx///L3IfYWUnmZYY0onTXOQ+JjSCAkq6pHvkiL0u29d0P4N3+efcD4L0EIJwAhBOAcAIQTgDCCUA4AQgnAOEEIJwAhBOAcAIQTgDCCUA4AQgnAOEEIJwAhBOAcAIQTgDCCUA4AQgnAOEEIJwAhBOAcAIQTgDCCUA4AQgnAOEEIJwAhBOAcAIQTgDCCUA4AQgnAOEEIJwAhBOAcAIQTgDCCUA4AQgnAOEEIJwAhBOAcAIQTgDCCUA4AQgnAOEEIJwAhBOAcAIQTgDCCUA4AQgnAOEEIJwAhBOAcAIQTgDCCUA4AQgnAOEEIJwAhBOAcAIQTgDCCUA4AQgnAOEEIJwAhBOAcAIQTgDCCUA4AQgnAOEEIJwAhBOAcAIQTgDCCUA4AQgnAOEEIJwAhBOAcAIQTgDCCUA4AQgnAOEEIJwAhBOAcAIQTgDCCUA4AQgnAOEEIJwAhBOAcAIQTgDCCUA4AQgnAGRb3/0A3kwDEC68AjRAOAEgXfYO0ADhBIB40TtAA0ByBWiAcAIAyTtAA4QTAEjeARoAluAK0ADhBACWJXcHaIBwAgDLssTuAA0Av2RWgAYIJwDwKXIHaIBwAgBfEneABoDfAitAA4QTAPhf3g7QAOEEAB7E7QANAI/SKkADhBMAeBK2AzRAOAGAZ1k7QAPAi6gK0ADhBABeJe0ADRBOAOCboB2gAcIJAHyXswM0AGyIqQANEE4AYEvKDtAA4QQANoXsAA0A2zIqQAOEEwDYEbEDNEA4AYA9CTtAA4QTANgVsAM0QDgBgH39d4AGCCcAcKD9DtAA4QQAjnTfARognADAoeY7QAPAsd4VoAHCCQCcaL0DNEA4AYAznXeABggnAHCq8Q7QAOEEAM713QEaIJwAwIC2O0ADhBMAGNF1B2iAcAIAQ5ruAA0QTgBgTM8doAHCCQAMarkDNEA4AYBRHXeABggnADCs4Q7QAOEEAMb12wEaACa0qwANEE4AYEa3HaABwgkATGm2AzRAOAGAOb12gAYIJwAwqdUO0ADhBABmddoBGiCcAMC0RjtAA4QTAJjXZwdoALigTQVogHACAFd02QEaIJwAwCVNdoAGgGt6VIAGCCcAcFGLHaABwgkAXNVhB2gAuKxBBWiAcAIA19XfARognADADeV3gAaAO6pXgAYIJwBwS/EdoAHCCQDcU3sHaAC4qXQFaIBwAgB3Vd4BGiCcAMBthXeABoD76laABqg7O37GTyRAigr7keFJQGHRFeAOAMkVoAEguQI0APzI8S3aACzLEr0DWJYluALcAWBZcitAA8CyLD9xfks2AF9SdwBfVEA4FZAusgI8C4AviRWgAeC32we4XgPwKHEH8EgFhFMB6eIqwLMAeJRWARoAntw9wcUagFc3B2j+5d0bYbUAuAPAizuHuFoBsOXGFMsFwAqAby6f43IFoAFgw9WTXK8B2HRxkObfxrVRFgyAOwBsuXKYCxaABoBtF45zxQZgz/w0zb+X6XkKQC+z8zT/biYnWjMAngXAnqkzXbMANADsmznVRRuAQ+NTrTp/KwAODB/sqg3AsdG5mn9Xg5MtGwB3ADg0dLbLFoAGgBMjp7tuA3BqYLjm39r5eAsHwB0Azpwd8MIFoAHg3MkRr9wAjDiesPn3dzhjAQhwNGQBCHAwZPOPsD/m2gHwNBCG7B302gWgAWDQzlHXAJBh86wXLwANAMO2Tnv1BmDCxrDNP8r3cZcPgDsAjHs98OULQAPAjPXwXdpbD94jwbr7Tk3uADBl3XyzLA0Az9bhbwPp0ABsWA8zsH57g36OMrC+/F7av+9+AH+pj1/z/Xj34+C9totgffqN3jYysD78SoBvGVh//1KeO8CAXxcC94Fwjz2wdikADTDOEwOW5asI1i4NIM8XrMvS5n+uyT/jj2tzKezxr/jjPvvf/16mdf28Axx/1oiePi+BXx8PLB4CTwMnrS+9/7E0uhBwZn38PND69AdFe0ADTHg9/Q98tLi/xzO+8/nAsj3AqafRHnxBgAz0tP0dQduzLvXEwB1gxMHy3+CJQTPfzvPYF4bX6AENcGbu9D/4uPF3+VscfVHoy5v0s13iG88H6Ghvh7f6DnF3gD32d7SB7wzbfrcaDbDF6c829zPCaleAF4jY4Pizr+HLxTLBC0aH6/aS8e4AMGHnKwD+8KP4QRoAJvT82XGMavrjQxm1O+iyCXAHgGEH57xqBWgAGDb0EsK0NfdpYto5nnHNBLgDwKCzI16yAjQADDo94SUrgFHn4xWA1gbGWzAB7gAwZOh016sADQBDxg53vQpgzOBkBaCr0clWS4A7AIwYPtnVKoAh42MVgJYmxlorAe4AcG7qVJeqAA0A5+YOdakKYMDkRAWgm9mJFkqAOwCcmT/QdSpAA8CZC+e5TgVw6sowBaCRS8OskgB3ADh28SwXqQANAMeuHuUaFaAB4ND1g1yiAjQAHLpxjktUAMfuDFEAGug+RHcAONC9ADQAAAAAAAAAtPUfG+OODkwtTPIAAAAASUVORK5CYII=\n",
            "text/plain": [
              "<PIL.Image.Image image mode=L size=512x512 at 0x7F93823E6A10>"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "text": [
            "TimeStep(\n",
            "{'discount': <tf.Tensor: shape=(1,), dtype=float32, numpy=array([0.], dtype=float32)>,\n",
            " 'observation': <tf.Tensor: shape=(1, 21), dtype=float64, numpy=\n",
            "array([[1., 0., 1., 0., 0., 2., 0., 0., 0., 0., 2., 2., 0., 0., 0., 2.,\n",
            "        1., 0., 0., 1., 0.]])>,\n",
            " 'reward': <tf.Tensor: shape=(1,), dtype=float32, numpy=array([-1000.], dtype=float32)>,\n",
            " 'step_type': <tf.Tensor: shape=(1,), dtype=int32, numpy=array([2], dtype=int32)>})\n",
            "============================================\n",
            "PolicyStep(action=<tf.Tensor: shape=(1,), dtype=int32, numpy=array([1], dtype=int32)>, state=(), info=())\n",
            "PolicyStep(action=<tf.Tensor: shape=(1,), dtype=int32, numpy=array([17], dtype=int32)>, state=(), info=())\n",
            "PolicyStep(action=<tf.Tensor: shape=(1,), dtype=int32, numpy=array([14], dtype=int32)>, state=(), info=())\n",
            "PolicyStep(action=<tf.Tensor: shape=(1,), dtype=int32, numpy=array([32], dtype=int32)>, state=(), info=())\n",
            "PolicyStep(action=<tf.Tensor: shape=(1,), dtype=int32, numpy=array([19], dtype=int32)>, state=(), info=())\n",
            "PolicyStep(action=<tf.Tensor: shape=(1,), dtype=int32, numpy=array([0], dtype=int32)>, state=(), info=())\n",
            "PolicyStep(action=<tf.Tensor: shape=(1,), dtype=int32, numpy=array([11], dtype=int32)>, state=(), info=())\n",
            "PolicyStep(action=<tf.Tensor: shape=(1,), dtype=int32, numpy=array([17], dtype=int32)>, state=(), info=())\n",
            "PolicyStep(action=<tf.Tensor: shape=(1,), dtype=int32, numpy=array([7], dtype=int32)>, state=(), info=())\n",
            "PolicyStep(action=<tf.Tensor: shape=(1,), dtype=int32, numpy=array([20], dtype=int32)>, state=(), info=())\n",
            "PolicyStep(action=<tf.Tensor: shape=(1,), dtype=int32, numpy=array([26], dtype=int32)>, state=(), info=())\n",
            "PolicyStep(action=<tf.Tensor: shape=(1,), dtype=int32, numpy=array([4], dtype=int32)>, state=(), info=())\n",
            "PolicyStep(action=<tf.Tensor: shape=(1,), dtype=int32, numpy=array([11], dtype=int32)>, state=(), info=())\n",
            "PolicyStep(action=<tf.Tensor: shape=(1,), dtype=int32, numpy=array([43], dtype=int32)>, state=(), info=())\n",
            "PolicyStep(action=<tf.Tensor: shape=(1,), dtype=int32, numpy=array([17], dtype=int32)>, state=(), info=())\n",
            "PolicyStep(action=<tf.Tensor: shape=(1,), dtype=int32, numpy=array([14], dtype=int32)>, state=(), info=())\n",
            "PolicyStep(action=<tf.Tensor: shape=(1,), dtype=int32, numpy=array([6], dtype=int32)>, state=(), info=())\n",
            "PolicyStep(action=<tf.Tensor: shape=(1,), dtype=int32, numpy=array([17], dtype=int32)>, state=(), info=())\n",
            "PolicyStep(action=<tf.Tensor: shape=(1,), dtype=int32, numpy=array([39], dtype=int32)>, state=(), info=())\n",
            "PolicyStep(action=<tf.Tensor: shape=(1,), dtype=int32, numpy=array([40], dtype=int32)>, state=(), info=())\n",
            "PolicyStep(action=<tf.Tensor: shape=(1,), dtype=int32, numpy=array([16], dtype=int32)>, state=(), info=())\n",
            "PolicyStep(action=<tf.Tensor: shape=(1,), dtype=int32, numpy=array([6], dtype=int32)>, state=(), info=())\n",
            "PolicyStep(action=<tf.Tensor: shape=(1,), dtype=int32, numpy=array([11], dtype=int32)>, state=(), info=())\n",
            "PolicyStep(action=<tf.Tensor: shape=(1,), dtype=int32, numpy=array([43], dtype=int32)>, state=(), info=())\n",
            "PolicyStep(action=<tf.Tensor: shape=(1,), dtype=int32, numpy=array([33], dtype=int32)>, state=(), info=())\n",
            "PolicyStep(action=<tf.Tensor: shape=(1,), dtype=int32, numpy=array([9], dtype=int32)>, state=(), info=())\n",
            "PolicyStep(action=<tf.Tensor: shape=(1,), dtype=int32, numpy=array([20], dtype=int32)>, state=(), info=())\n",
            "PolicyStep(action=<tf.Tensor: shape=(1,), dtype=int32, numpy=array([38], dtype=int32)>, state=(), info=())\n",
            "PolicyStep(action=<tf.Tensor: shape=(1,), dtype=int32, numpy=array([11], dtype=int32)>, state=(), info=())\n",
            "PolicyStep(action=<tf.Tensor: shape=(1,), dtype=int32, numpy=array([8], dtype=int32)>, state=(), info=())\n",
            "[((0, 255), (40, 255)), ((40, 255), (80, 255)), ((80, 255), (120, 255)), ((120, 255), (156, 271)), ((156, 271), (193, 287)), ((193, 287), (229, 304)), ((229, 304), (266, 321)), ((266, 321), (303, 337)), ((303, 337), (340, 354)), ((340, 354), (379, 346)), ((379, 346), (418, 337)), ((418, 337), (384, 315)), ((418, 337), (410, 298)), ((418, 337), (440, 303)), ((418, 337), (457, 329)), ((418, 337), (451, 359)), ((418, 337), (427, 377)), ((384, 315), (351, 293)), ((457, 329), (497, 321)), ((451, 359), (485, 381)), ((351, 293), (317, 272))]\n",
            "[1 0 1 0 0 2 0 0 0 0 2 2 0 0 0 2 1 0 0 1 0]\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAgAAAAIACAAAAADRE4smAAAHhElEQVR4nO3dwXbbNhRFUamr///L7KBxasmUSBBIhYe79yBNZmpxeWA5rn27Zds+/QI+7a9PvwA+ywDCGUA4AwhnAOEMIJwBhDOAcAYQzgDCGUA4AwhnAOEMIJwBhDOAcAYQzgDCGUA4AwhnAOEMIJwBhDOAcAYQzgDCGUA4AwhnAOEMIJwBhDOAcAYQzgDCGUA4AwhnAOEMIJwBhDOAcAYQzgDCGUA4AwhnAOEMIJwBhDOAcAYQzgDCGUA4AwhnAOEMIJwBhDOAcAYQzgDCGUA4AwhnAOEMIJwBhDOAcAYQzgDCGUA4AwhnAOEMIJwBhDOAcAYQzgDCGUA4AwhnAOEMIJwBhDOAcAYQzgDCGUA4AwhnAOEMIJwBhDOAcAYQzgDCGUA4AwhnAOEMIJwBhDOAcAYQzgDCGUA4AwhnAOEMIJwBhDOAcAYQzgDCGUA4AwhnAOEMIJwBhDOAcAYQzgDCGUA4AwhnAOEMIJwBhDOAcAZAtu3TL+DDFIBw4QlQANJlJ0ABiBedAAUIZwCQfAcoACQnQAEgOQEKALfgBChAOAOA2y33DlAAuN1usQlQAPhXZgIUAH6JTIAChDMA+JJ4BygA/BaYAAWA/+QlQAHgm7gEKEA4A4Dv0u4ABYAHYQlQAHiUlQAFCGcA8CTqDlAAeJaUAAWAH4ISoADhDAB+yrkDFAB2xCRAAWBPSgIUAHaFJEABwhkA7Mu4AxQAXohIgALAKwkJUIBwBgAvBdwBCgCvrZ8ABYA3lk+AAoQzAHhn9TtAAeCtxROgAPDe2glQADiwdAIUAI6snAAFgEMLJ0AB4Ni6CVAAOGHZBCgAnLFqAhQgnAHAKYveAQoA56yZAAWAk5ZMgALAWSsmQAHgtAUToABw3noJUABosFwCFABarJYABQhnANBksTtAAaDNWglQAGi0VAIUAFqtlAAFgGYLJUABoN06CVCAcAYAFyxzBygAXLFKAhQALlkkAQoA16yRAAWAi5ZIgAKEMwC4aoU7QAHgsgUSoABwXf0EKAB0KJ8ABYAe1ROgANCleAIUIJwBQJ/ad4ACQKfSCVAA6FU5AQoQzgCgW+E7QAGgX90EKAAMUDYBChDOAGCEqneAAsAQRROgADBGzQQoAAxSMgEKEM4AYJSKd4ACwDAFE6AAME69BChAOAOAgcrdAQoAI1VLgALAUMUSoADhDADGqnUHKAAMVioBCgCjVUqAAsBwhRKgADBenQQoAPwBZRKgAOEM4EmZR5c/ZNACqgxJAeBRVgIUAJ5kJYCfxhydAZQlAekkIF3OArwLgB1DHt4KBWBfzh3APglIF7IAHwTCvowEKAC8kJEAXhtxeAZQmASkk4B06y/AuwB4Y8DjqwBQV/fzO3cAONR7gJMPwBUA73U+wpMXgGN9R2gA5XUdofNfQM8hzj4AHwTCoY7HePYCcMr1YzSAJVw+Rue/iKsHaQCruHiS0w/AuwA45dqzPH0BOO3KWc5//q4AOOnC4zx/AWjQfpwGsJTm43T+i2k9UANYTeOJFhiAdwHQoO2ZLlAAGrWcaYXzdwVAk4bHukIBaHb+WCsMwBUAjc4+2BUCwBUnT7bEAFwB0Ozcs12iAFxy6mwNYF1nztb5r+zE6RrA0o6Pt8YAvAuASw4f8BoF4LKDAy5y/q4AuOj9M16kAHR4e8YGsL53Z+z8E7w5ZQOI8PqYqwzAuwDo8OpBrxIAer046TIDcAVAl/1nXQEgxN7DXiYADLBz2nUG4AqAXj+f9zoFYITn8y50/q4A6Le9/SPr2978iQDbyz+QYXvxe1Jsu7+dnncBMMa28zuSbE//LMEVAOds29GjvT38o4a/P/0C6rj/Otr7p1/IUGv92/wv3qxgu3/9UkapFzuRFyvY7tUG4Aq45n67/buCUqe9o/rr/7inFGz3YgUo9WKn9W0FW7H/pK6AEX5fCMVO/1bwBU+u3AoUYLx7pQ8Oq7zOIrb71wcB5VJAv2273baHvxY8/vQx69i+fnn8+rC5R+BjgFFevf+739wHAbaHLwbYfeQnTwE9nr8a6OV3jpluBa6AAb7V/+DzwMv8FQLf7H1F+OHXjkwWAi57PMntx28m5wroVOvv/hjsR8h9bXiUH4dc738QcwVcV+2v/hnr8LuD1UgA1+y/jav3bQJcAZeof7az3yO8QgJo9vqTeG/fFczJFdDMp36ivf0cvu8YubzmnxQ3+wJ8f4A28s9rJz41wMoq/vAQVwAM4kfIhTv/+aGJuAJgkPo/SJgufpZ4trbPEbOc5s8Ss5b3ZzzrArwLgDEOnvFZE8Aghwc85wJcATDE8QM+ZwIY5MTxzrgAVwCMcOrpnjABCgAjnHu4J0wAY5w82ukW4AqAAU4/2dMlgCHOn+tkC3AFwAANz/VkCWCIllO1gPU0nakBrKftTC1gOY1HOtECvAuAfq2P9EQJYID285xmAa4A6HbheZ4lAQoA3a48zrMkgH7XznKOBbgCoNfFZ3mOBNDv6knOsABXAHS6/iDPkAC6lT5GV0A4AwhnANCn9IcACpDOAMIZQDgDgC61PwZUgHQGAAAAAAAAALCkfwD2XYoA/dUSaAAAAABJRU5ErkJggg==\n",
            "text/plain": [
              "<PIL.Image.Image image mode=L size=512x512 at 0x7F93853B3390>"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "text": [
            "TimeStep(\n",
            "{'discount': <tf.Tensor: shape=(1,), dtype=float32, numpy=array([0.], dtype=float32)>,\n",
            " 'observation': <tf.Tensor: shape=(1, 21), dtype=float64, numpy=\n",
            "array([[1., 0., 1., 0., 0., 2., 0., 0., 0., 0., 2., 2., 0., 0., 0., 2.,\n",
            "        1., 0., 0., 1., 0.]])>,\n",
            " 'reward': <tf.Tensor: shape=(1,), dtype=float32, numpy=array([-1000.], dtype=float32)>,\n",
            " 'step_type': <tf.Tensor: shape=(1,), dtype=int32, numpy=array([2], dtype=int32)>})\n",
            "============================================\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 221
        },
        "id": "0QGodpeBLskT",
        "outputId": "b5fc3181-4763-4d2b-af32-21a4cf48d958"
      },
      "source": [
        "iterations = range(0, NUMBER_ITERATION + 1, EVAL_INTERVAL)\n",
        "plt.plot(iterations, returns)\n",
        "plt.ylabel('Average Return')\n",
        "plt.xlabel('Iterations')"
      ],
      "execution_count": 15,
      "outputs": [
        {
          "output_type": "error",
          "ename": "NameError",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-15-158f9f3bf456>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0miterations\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mrange\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mNUMBER_ITERATION\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mEVAL_INTERVAL\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 2\u001b[0;31m \u001b[0mplt\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mplot\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0miterations\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mreturns\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      3\u001b[0m \u001b[0mplt\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mylabel\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'Average Return'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      4\u001b[0m \u001b[0mplt\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mxlabel\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'Iterations'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mNameError\u001b[0m: name 'returns' is not defined"
          ]
        }
      ]
    }
  ]
}