DRL_18_Policy_Based_Methods.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "DRL_18_Policy-Based-Methods.ipynb",
      "provenance": [],
      "collapsed_sections": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "40Yb47zJQglm",
        "colab_type": "text"
      },
      "source": [
        "DEEP REINFORCEMENT LEARNING EXPLAINED - 18\n",
        "# **Policy-Based Methods**\n",
        "### Hill Climbing algorithm"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Q40Fa7qM4_lE",
        "colab_type": "text"
      },
      "source": [
        "Import necessary packages"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "FA1Y5VCv20XZ",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import gym\n",
        "import numpy as np\n",
        "from collections import deque\n",
        "import matplotlib.pyplot as plt"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "4m0Vm4Yp91ZI",
        "colab_type": "text"
      },
      "source": [
        "Tunning the image rendering in colab\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "kgpHXywd5SyZ",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 728
        },
        "outputId": "c9036b31-0997-4a4f-e886-6e9c0f309b6b"
      },
      "source": [
        "# Taken from \n",
        "# https://towardsdatascience.com/rendering-openai-gym-envs-on-binder-and-google-colab-536f99391cc7\n",
        "\n",
        "!apt-get install -y xvfb x11-utils\n",
        "\n",
        "!pip install pyvirtualdisplay==0.2.* \\\n",
        "             PyOpenGL==3.1.* \\\n",
        "             PyOpenGL-accelerate==3.1.*\n",
        "\n",
        "!pip install gym[box2d]==0.17.*\n",
        "\n",
        "# Taken from https://github.com/actions/virtual-environments/issues/214\n",
        "!sudo apt-get update\n",
        "!sudo apt-get install xvfb --fix-missing\n",
        "\n",
        "import pyvirtualdisplay\n",
        "\n",
        "_display = pyvirtualdisplay.Display(visible=False, size=(1400, 900))\n",
        "_ = _display.start()\n",
        "\n",
        "!echo $DISPLAY # sanity checking should now be set to some value\n",
        "\n",
        "from IPython import display"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Reading package lists... Done\n",
            "Building dependency tree       \n",
            "Reading state information... Done\n",
            "x11-utils is already the newest version (7.7+3build1).\n",
            "xvfb is already the newest version (2:1.19.6-1ubuntu4.5).\n",
            "The following package was automatically installed and is no longer required:\n",
            "  libnvidia-common-440\n",
            "Use 'apt autoremove' to remove it.\n",
            "0 upgraded, 0 newly installed, 0 to remove and 71 not upgraded.\n",
            "Requirement already satisfied: pyvirtualdisplay==0.2.* in /usr/local/lib/python3.6/dist-packages (0.2.5)\n",
            "Requirement already satisfied: PyOpenGL==3.1.* in /usr/local/lib/python3.6/dist-packages (3.1.5)\n",
            "Requirement already satisfied: PyOpenGL-accelerate==3.1.* in /usr/local/lib/python3.6/dist-packages (3.1.5)\n",
            "Requirement already satisfied: EasyProcess in /usr/local/lib/python3.6/dist-packages (from pyvirtualdisplay==0.2.*) (0.3)\n",
            "Requirement already satisfied: gym[box2d]==0.17.* in /usr/local/lib/python3.6/dist-packages (0.17.2)\n",
            "Requirement already satisfied: cloudpickle<1.4.0,>=1.2.0 in /usr/local/lib/python3.6/dist-packages (from gym[box2d]==0.17.*) (1.3.0)\n",
            "Requirement already satisfied: numpy>=1.10.4 in /usr/local/lib/python3.6/dist-packages (from gym[box2d]==0.17.*) (1.18.5)\n",
            "Requirement already satisfied: pyglet<=1.5.0,>=1.4.0 in /usr/local/lib/python3.6/dist-packages (from gym[box2d]==0.17.*) (1.5.0)\n",
            "Requirement already satisfied: scipy in /usr/local/lib/python3.6/dist-packages (from gym[box2d]==0.17.*) (1.4.1)\n",
            "Requirement already satisfied: box2d-py~=2.3.5; extra == \"box2d\" in /usr/local/lib/python3.6/dist-packages (from gym[box2d]==0.17.*) (2.3.8)\n",
            "Requirement already satisfied: future in /usr/local/lib/python3.6/dist-packages (from pyglet<=1.5.0,>=1.4.0->gym[box2d]==0.17.*) (0.16.0)\n",
            "Get:1 http://security.ubuntu.com/ubuntu bionic-security InRelease [88.7 kB]\n",
            "Ign:2 https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64  InRelease\n",
            "Hit:3 https://cloud.r-project.org/bin/linux/ubuntu bionic-cran35/ InRelease\n",
            "Ign:4 https://developer.download.nvidia.com/compute/machine-learning/repos/ubuntu1804/x86_64  InRelease\n",
            "Hit:5 http://ppa.launchpad.net/graphics-drivers/ppa/ubuntu bionic InRelease\n",
            "Hit:6 http://archive.ubuntu.com/ubuntu bionic InRelease\n",
            "Hit:7 https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64  Release\n",
            "Hit:8 https://developer.download.nvidia.com/compute/machine-learning/repos/ubuntu1804/x86_64  Release\n",
            "Get:9 http://archive.ubuntu.com/ubuntu bionic-updates InRelease [88.7 kB]\n",
            "Hit:10 http://ppa.launchpad.net/marutter/c2d4u3.5/ubuntu bionic InRelease\n",
            "Get:11 http://archive.ubuntu.com/ubuntu bionic-backports InRelease [74.6 kB]\n",
            "Fetched 252 kB in 1s (241 kB/s)\n",
            "Reading package lists... Done\n",
            "Reading package lists... Done\n",
            "Building dependency tree       \n",
            "Reading state information... Done\n",
            "xvfb is already the newest version (2:1.19.6-1ubuntu4.5).\n",
            "The following package was automatically installed and is no longer required:\n",
            "  libnvidia-common-440\n",
            "Use 'sudo apt autoremove' to remove it.\n",
            "0 upgraded, 0 newly installed, 0 to remove and 71 not upgraded.\n",
            ":1009\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "JKCArPYvBkYi",
        "colab_type": "text"
      },
      "source": [
        "OpenAI Gym's Cartpole Environment"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_3_4zOdQBl2g",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 65
        },
        "outputId": "8e0f8fe6-7372-4e87-acb8-96ad1585bf01"
      },
      "source": [
        "env = gym.make('CartPole-v0')\n",
        "print('observation space:', env.observation_space)\n",
        "print('action space:', env.action_space)\n",
        "print('threshold: ', env.spec.reward_threshold)\n"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "observation space: Box(4,)\n",
            "action space: Discrete(2)\n",
            "threshold:  195.0\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "NJC916BU-9L6",
        "colab_type": "text"
      },
      "source": [
        "\n",
        "Define  policy π (and initialize with random weights θ)"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "gRVXyg9A1zQh",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "class Policy():\n",
        "    def __init__(self, s_size=4, a_size=2):\n",
        "        # 1. Initialize policy π with random weights θ\n",
        "        # weights for simple linear policy: state_space x action_space\n",
        "        self.θ = 1e-4*np.random.rand(s_size, a_size)  \n",
        "        \n",
        "    def forward(self, state):\n",
        "        x = np.dot(state, self.θ)\n",
        "        return np.exp(x)/sum(np.exp(x))\n",
        "    \n",
        "    def act(self, state):\n",
        "        probs = self.forward(state)\n",
        "        action = np.argmax(probs)             \n",
        "        return action\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "9R3_QsYSBzYn",
        "colab_type": "text"
      },
      "source": [
        "Render Cartpole Environment before train"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "OhhqfVxD28VZ",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def watch_agent():\n",
        "  env = gym.make('CartPole-v0')\n",
        "  state = env.reset()\n",
        "  rewards = []\n",
        "  img = plt.imshow(env.render(mode='rgb_array'))\n",
        "  for t in range(2000):\n",
        "    action = policy.act(state)\n",
        "    img.set_data(env.render(mode='rgb_array')) \n",
        "    plt.axis('off')\n",
        "    display.display(plt.gcf())\n",
        "    display.clear_output(wait=True)\n",
        "    state, reward, done, _ = env.step(action)\n",
        "    rewards.append(reward)\n",
        "    if done:\n",
        "        print(\"Reward:\", sum([r for r in rewards]))\n",
        "        break \n",
        "  env.close()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "WGK84ymx26sQ",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 98
        },
        "outputId": "65799673-4243-459e-ebdd-7b49431e8c6e"
      },
      "source": [
        "policy = Policy()\n",
        "print (\"Policy weights θ before train:\\n\", policy.θ)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Policy weights θ before train:\n",
            " [[6.30558674e-06 2.13219853e-05]\n",
            " [2.32801200e-05 5.86359967e-05]\n",
            " [1.33454380e-05 6.69857175e-05]\n",
            " [9.39527443e-05 6.65193884e-05]]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "TT-xoG5VA9rZ",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 263
        },
        "outputId": "04ff4015-5f1a-40f6-f434-13ab728fc555"
      },
      "source": [
        "watch_agent()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Reward: 9.0\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAVQAAADnCAYAAABBu67aAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAIMElEQVR4nO3dTY9dBRnA8efce+el0xda2tKS2lFEBKKRBrVaCSuNC1d8ABs/BXwCd3wBXLuERY2JCURXuCjRBmoFSYHyUrTQdjp03ufO3HtckIi3Z9q5wMOcc+jvt5o8cyfnWZz8c9/OmaIsywDgy+vUvQDA14WgAiQRVIAkggqQRFABkvS2+b2vAABUFVsNPUMFSCKoAEkEFSCJoAIkEVSAJIIKkERQAZIIKkASQQVIIqgASQQVIImgAiQRVIAkggqQRFABkggqQBJBBUgiqABJBBUgiaACJBFUgCSCCpBEUAGSCCpAEkEFSCKoAEkEFSCJoAIkEVSAJIIKkERQAZIIKkASQQVIIqgASQQVIImgAiQRVIAkggqQRFABkggqQBJBBUgiqABJBBUgiaACJBFUgCSCCpBEUAGSCCpAEkEFSCKoAEkEFSCJoAIkEVSAJIIKkERQAZIIKkASQQVIIqgASQQVIImgAiQRVIAkggqQRFABkggqQBJBBUgiqABJBBUgiaACJBFUgCSCCpBEUAGSCCpAEkEFSCKoAEkEFSCJoAIkEVSAJIIKkERQAZIIKkASQQVIIqgASQQVIImgAiQRVIAkggqQRFABkggqQBJBBUgiqABJBBUgiaACJBFUgCS9uheAW22uLUV/+ZNPf15djLIcxj3Hv1fzVrA9QaUxBv3V+PDsC7F649+xfPXd/81nn/x1jVvB+LzkpzGGg42Yf/fVkZhGRCx8+EYMB5s1bQXjE1Qaoze9Nw4/+mRlvvTR21EOBzVsBJ+PoNIYRVFE0Z2oew34wgSVxiuHm9FfulH3GrAtQaVRDjzweHSnZkZmg/WVmL/095o2gvEJKo0yufdgFB1fPqGdBJVWKAebUZZl3WvAHQkqjVJ0urHnyIOV+bV/vRyba4s1bATjE1QapdPtxb4trooqBxsRnqHScIIKkERQaZze9O4ouqMfTA0HGzF38WxNG8F4BJXG2f/NEzG199DosCxjfeFaPQvBmAQVIImg0jxFRBTVU7Msh746RaMJKg1UxNHHflmZzl86F/3F6zXsA+MRVBqnKIroTs5U5sPNdXedotEEFSCJoNJIM4dnY+qeI6PDsowbb/+tnoVgDIJKI03uPhC96T2V+crc5Rq2gfEIKkASQaWxZg7NVmYby/OxseomKTSToNJY9z7448ps5foHsX7zag3bwPYEFSCJoNJYRbcXnd5kZb65vlzDNrA9QaWxZg7Nxt5jj1TmH59/qYZtYHuCSmMVRRGfXtg/qiyHO78MjEFQaSU3SaGJBJVGm95/pDJbnbscS1cu1rAN3Jmg0miHHn4iohh92T/c7Megv1bTRnB7ggqQRFBptqKIotOrjG9+8A/vo9I4gkqjTe07HPc++KPKfOmjtyNCUGkWQaXRiqITRadb9xowFkGllQYb69Ffmq97DRghqDTewYd/FkV39H3UjeX5WPjwjZo2gq0JKo03tfdQFFv8F1RoGmcprTXc7Pukn0YRVBqv05uK3fc9UJlfvfCXKAcbNWwEWxNUGq87OR177n+oMh8ONjxDpVEEFSCJoNIKE7vuibjlg6nB+nLMXzpX00ZQJai0wr0P/aTyb6XL4SD6Szdq2giqBBUgiaDSGkWnerouX303BhvrNWwDVYJKK3R6k3Hf939emS9deSuGgkpDCCqtUBRFdCen614D7khQabWyHEZ/2U1SaAZBpTX2HXs0Jmb2j8zKwUZcf/PlmjaCUYJKa0zuORidiam614DbElRarxwOXYJKIwgq7VFE7D78rcp4/tK56C9e3/l94BaCSmsURScOfPvxyny4uR7lcFDDRjBKUAGSCCqt0ulNRdGdGB2WZcy99Uo9C8H/EVRaZe+xR2Lm4PHKfG3+PzVsA6MElVYpiiKiqHsL2Jqg0kLVoq4vzsXG6kINu8BnBJXWOfKDX1Rmq3OXY33hWg3bwGcEldaZ2LWv7hVgS8U2V5i4/IQd89xzz8WLL7647eOOH5iI3/z0wMisLCP+8Ppq/PPy4ljHOnXqVDzzzDNfaE+I27yT39vpLeB2XnvttThz5sy2jzu4b1ec+sav4qFjB6M/nI6IIjrFIGYnr8Rvz/xxrGN1u90vuS1UCSqtM7ewGu9f24zBvpPx/sqjUUYndndvRnf4+7pX4y7nPVRaqT+cjneWH4vNcioG5UQsbB6K1xeeqHst7nKCSiu98t6euPXt/+8cvz9++N3761kIQlBpqXOvvhSTnZWR2YHptZiZnrjNX8BXT1BppV3dxXh8/59jb28uhv2PY+2TV2P36p9iZW2j7tW4i/lQilaau7kSv3v++YjihXjvyidx/p2PooiIoRtNU6M7BvXZZ5/dqT0gzp8/P/ZjF1f7ceavb47MPk9KL1686PzmC3v66ae3nN8xqKdPn/5KloGtXLhwIc6ePbsjx5qdnXV+k+6OQT169OhO7QExMzOzY8eanp52fpPOh1IASQQVIImgAiQRVIAkggqQxBf7aYwTJ07EU089tSPHOnny5I4ch7uLG0wDfH5b3mDaS36AJIIKkERQAZIIKkASQQVIIqgASQQVIImgAiQRVIAkggqQRFABkggqQBJBBUgiqABJBBUgiaACJBFUgCSCCpBEUAGSCCpAEkEFSCKoAEkEFSCJoAIkEVSAJIIKkERQAZIIKkASQQVIIqgASQQVIImgAiQRVIAkggqQRFABkggqQBJBBUgiqABJBBUgiaACJBFUgCSCCpBEUAGSCCpAEkEFSNLb5vfFjmwB8DXgGSpAEkEFSCKoAEkEFSCJoAIkEVSAJP8FTumB0b56juoAAAAASUVORK5CYII=\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "rL8ZBRUzDtMp",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 263
        },
        "outputId": "2de3cb9e-f648-4608-f71f-aca51ca33fc2"
      },
      "source": [
        "watch_agent()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Reward: 10.0\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAVQAAADnCAYAAABBu67aAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAIbElEQVR4nO3dS29c9RnA4fccz9iOL0kgdnCVcCkKJRTailZqRdVSFSRUdcOGz8GOdVesWOUDINR1FxWquqnU0qhFVW+rFhFuKZASTOwkxBd8m5nTRQWSGeP48mbOGet5dn7HI7+L0c/j+R/PFFVVBQCHV9a9AMBRIagASQQVIImgAiQRVIAkrdvc7hIAgH7FTkPPUAGSCCpAEkEFSCKoAEkEFSCJoAIkEVSAJIIKkERQAZIIKkASQQVIIqgASQQVIImgAiQRVIAkggqQRFABkggqQBJBBUgiqABJBBUgiaACJBFUgCSCCpBEUAGSCCpAEkEFSCKoAEkEFSCJoAIkEVSAJIIKkERQAZIIKkASQQVIIqgASQQVIImgAiQRVIAkggqQRFABkggqQBJBBUgiqABJBBUgiaACJBFUgCSCCpBEUAGSCCpAEkEFSCKoAEkEFSCJoAIkEVSAJIIKkERQAZIIKkASQQVIIqgASQQVIImgAiQRVIAkggqQRFABkggqQBJBBUgiqABJBBUgiaACJBFUgCSCCpBEUAGSCCpAEkEFSCKoAEkEFSCJoAIkEVSAJIIKkERQAZIIKkASQQVIIqgASQQVIImgAiQRVIAkggqQRFABkggqQBJBBUgiqABJBBUgSavuBeCrrN+6Fr3OZkREtI8dj/bE8Zo3gt0JKo31wcVfxsr8uxERcezus3Hs1JmYmLk/7vnW0zVvBjvzJz+NVPW6UfW6X3y9duO/ceOdv8by1bdq3Ap2J6g00tJHb8bqwgd986m5czVsA3sjqDRS1e1GVL3tw6KIk/d/u56FYA8EFSCJoNI4VdWLxbde75u3xiajGHGOSnMJKs1TRWzc+qRvfPKBx2N06lQNC8HeCCqNU/U6UVXVjrcVRTHgbWDvBJXGuXn5n7GxdG37sChj8p4H61kI9khQaZxetxPxpWeo5UgrTtz7aE0bwd4IKkASQaVRep2tuP72X/rmrWPTUZQjNWwEeyeoNEpV9WJjaaFvfuqhJ2JkbLKGjWDvBJVGqbqdiNjhhL9wwk/zCSqNsnjpT9FZW942K1ujMTn7QD0LwT4IKo3S63b6ZmV7PKa+9o0atoH9EVSAJIJKY3Q2VuPm5X/0zdsTJ7x+ylAQVBqj6nZjc/l633z2mz+JYqRdw0awP4JKY/S6m195m2eoDANBpTGu/ev3X3wo3+dGxibi2N1natoI9kdQaYxed6tv1hqfdskUQ0NQAZIIKo2wuXIjbn3477756OTJCC+fMiQElUbodTZj67NP++anH3sqFJVhIag0nhN+hoWg0gjLH7/T96bSEREhpgwRQaURlq70v346MXNfHD/zSA3bwMEIKo1VjLT8hxRDRVCp3drNq7Ey/27ffHTyrhq2gYMTVGrX3VyLzvpK3/z0Y085kGKoCCpAEkGldstX395hWoTrTxk2gkrtdjrhnz5zPiZm76thGzg4QaWRylY7Sif8DBlBpVYrn7wXazc+6ps74WcYCSq16qyvRHdzbfuwKGL20Z/WsxAcgqACJBFUalNV1Y4n/EVRuv6UoSSo1KiKpStv9E3vevB7MXZ8toZ94HAElcYpW6NRlCN1rwH7JqjUZunKG7G58uWPjS6i7YSfISWo1Gbrs1t9n3Jattox+8iPa9oIDkdQAZIIKrWoet3/v0v/lxUekgwvj15q0et2YvmjN/vmM+d/FK3x6Ro2gsMTVBplpD0WRelhyXDyyKUWW5/diqrX3TYrypFoT5ysaSM4PEGlFjfe/Vvfu/SPjE3E3Q/9oKaN4PAEFSCJoDJw3a2NWL12uW/uv6MYdoLKwPU6G7Hycf+nnJ5+7OkoW6M1bAQ5BJXGGGmPe5cphpqgMnCbKzejqnp1rwHpBJWBW7z056i6W9tm7YkTMX3mfE0bQQ5BpRHK9niMTc/UvQYciqAyUJ31lVi7fqVv7oSfo0BQGajO+mqsLrzfN5/7zjMRDqQYcoJKI5RO+DkCBJWB6n+Hfjg6iqqqdrt91xvh6tWr8fzzz0e32739N0fEc4+fiPNzY9tm789/Gq+8vhDtiRO73rcsy7hw4UKcPXv2wPtCkh3/nGoNeguOltXV1Xj11Vej0+ns6fufuOeZ+PrsI1983S7XY2l1I377u9due9+yLOPFF1888K5wpwkqAzN7cirK4z+MPyw8E5//gn94+u+x1b1a72KQxGuoDMz41H2xNvmz6Faj0a3a0a3acWnp+3HhN4LK0SCoDEwVZXSr7deb9qIVK+t7e/0Vmk5QGZizM+MxWm5sm7WLjWgVm19xDxgugsrA/Py703HX+q9jcfE/sbU2H1Otm9H+9FdxffG9uleDFA6lGJhfvPJaFPHHqKKIh+89Fefvn4n568txa3Xj9neGIbBrUF966aVB7cGQWlxcjF5vb2/FV1URVVQRUcWlDxfi0ocL+/pZVVXFyy+/HKdPnz7AppDnhRde2HG+64X98/PzLuxnV5cvX44nn3xyzxf2H0ZZlnHx4sU4d+7cHf9ZsJu5ubn9X9g/Nzd3Z7bhyFheXh7o/+DPzMx4XNJYDqUAkggqQBJBBUgiqABJBBUgiQv7OZTJycl49tlnB3bZ1NTU1B3/OXBQ3mAaYP92vFbQn/wASQQVIImgAiQRVIAkggqQRFABkggqQBJBBUgiqABJBBUgiaACJBFUgCSCCpBEUAGSCCpAEkEFSCKoAEkEFSCJoAIkEVSAJIIKkERQAZIIKkASQQVIIqgASQQVIImgAiQRVIAkggqQRFABkggqQBJBBUgiqABJBBUgiaACJBFUgCSCCpBEUAGSCCpAEkEFSCKoAEkEFSCJoAIkEVSAJIIKkKR1m9uLgWwBcAR4hgqQRFABkggqQBJBBUgiqABJBBUgyf8A51+IlI8qGEgAAAAASUVORK5CYII=\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "qbSa19Iw3rkT",
        "colab_type": "text"
      },
      "source": [
        "Train the Agent with Hill Climbing "
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "5gGKED9B3rBt",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 378
        },
        "outputId": "f6c2c76e-d4c9-445c-a077-fa7e24d888f6"
      },
      "source": [
        "env = gym.make('CartPole-v0')\n",
        "env.seed(0)\n",
        "np.random.seed(0)\n",
        "\n",
        "\n",
        "def hill_climbing(n_episodes=10000, gamma=1.0, noise=1e-2):\n",
        "    \"\"\"Implementation of hill climbing.     \n",
        "    Params\n",
        "    ======\n",
        "        n_episodes (int): maximum number of training episodes\n",
        "        gamma (float): discount rate\n",
        "        noise(float): standard deviation of additive noise\n",
        "    \"\"\"\n",
        "    scores_deque = deque(maxlen=100)\n",
        "    scores = []\n",
        "    \n",
        "    #2. Initialize θbest (our best guess for the weights θ)\n",
        "    #3. Initialize Gbest (our highest return G we have gotten so far)\n",
        "    Gbest = -np.Inf\n",
        "    θbest = policy.θ\n",
        "    for i_episode in range(1, n_episodes+1):\n",
        "        rewards = []\n",
        "        state = env.reset()\n",
        "        while True:\n",
        "        #4. Collect a single episode with θ, and record the return G\n",
        "            action = policy.act(state)\n",
        "            state, reward, done, _ = env.step(action)\n",
        "            rewards.append(reward)\n",
        "            if done:\n",
        "                break \n",
        "        scores_deque.append(sum(rewards))\n",
        "        scores.append(sum(rewards))\n",
        "\n",
        "        discounts = [gamma**i for i in range(len(rewards)+1)]\n",
        "        G = sum([a*b for a,b in zip(discounts, rewards)])\n",
        "\n",
        "        if G >= Gbest: # 5. If G>Gbest then θbest←θ & Gbest←G\n",
        "            Gbest = G\n",
        "            θbest = policy.θ\n",
        "        \n",
        "        #6. Add a little bit of random noise to θbest, to get a new set of weights θ\n",
        "        policy.θ = θbest + noise * np.random.rand(*policy.θ.shape)\n",
        "\n",
        "        if i_episode % 10 == 0:\n",
        "            print('Episode {}\\tAverage Score: {:.2f}'.format(i_episode, np.mean(scores_deque)))\n",
        "\n",
        "        # 7. Repeat steps 4-6 until Environment solved.    \n",
        "        if np.mean(scores_deque)>=env.spec.reward_threshold:\n",
        "            print('Environment solved in {:d} episodes!\\tAverage Score: {:.2f}'.format(i_episode, np.mean(scores_deque)))\n",
        "            policy.θ = θbest\n",
        "            break\n",
        "        \n",
        "    return scores\n",
        "            \n",
        "scores = hill_climbing(gamma=0.9)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Episode 10\tAverage Score: 59.50\n",
            "Episode 20\tAverage Score: 95.45\n",
            "Episode 30\tAverage Score: 122.37\n",
            "Episode 40\tAverage Score: 134.60\n",
            "Episode 50\tAverage Score: 145.60\n",
            "Episode 60\tAverage Score: 149.38\n",
            "Episode 70\tAverage Score: 154.33\n",
            "Episode 80\tAverage Score: 160.04\n",
            "Episode 90\tAverage Score: 163.56\n",
            "Episode 100\tAverage Score: 166.87\n",
            "Episode 110\tAverage Score: 174.70\n",
            "Episode 120\tAverage Score: 168.54\n",
            "Episode 130\tAverage Score: 170.92\n",
            "Episode 140\tAverage Score: 173.79\n",
            "Episode 150\tAverage Score: 174.83\n",
            "Episode 160\tAverage Score: 178.00\n",
            "Episode 170\tAverage Score: 179.60\n",
            "Episode 180\tAverage Score: 179.58\n",
            "Episode 190\tAverage Score: 180.41\n",
            "Episode 200\tAverage Score: 180.74\n",
            "Episode 210\tAverage Score: 186.96\n",
            "Environment solved in 215 episodes!\tAverage Score: 195.65\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ONPYWICR3z63",
        "colab_type": "text"
      },
      "source": [
        "Plot the scores during training\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "pk1Q1l7R30-t",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 277
        },
        "outputId": "aa6cdcd3-53e3-486d-a245-1345c3f5c5e5"
      },
      "source": [
        "fig = plt.figure()\n",
        "#ax = fig.add_subplot(111)\n",
        "plt.plot(np.arange(1, len(scores)+1), scores)\n",
        "plt.ylabel('Score')\n",
        "plt.xlabel('Episode #')\n",
        "plt.show()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAEGCAYAAACKB4k+AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO29ebjkVnnn/32lqrpL7+1ut9t22+2lbX4GGwPXZicGB2M7AUOSH2DyA0N4fg5hn4EhbDNjJslMwrBkeJghMeDYJMRsZpthCcZAjAHHtKFp73t7adrdt93u7fa9t6qkd/4450hHKqlKUi269+r9PM99pJJK0ild6bznXQ8xMwRBEAQBAJyyGyAIgiAsHEQoCIIgCAEiFARBEIQAEQqCIAhCgAgFQRAEIaBWdgP6Yd26dbx58+aymyEIgrCouPXWW/cy8/qkfYtaKGzevBlbt24tuxmCIAiLCiJ6OG2fmI8EQRCEABEKgiAIQoAIBUEQBCFAhIIgCIIQIEJBEARBCBiaUCCiTUT0YyK6k4juIKJ36e1rieh6IrpPL9fo7UREnyKi+4loOxE9c1htEwRBEJIZpqbQBvAeZj4DwHMAvI2IzgDwfgA3MPMWADfozwBwEYAt+u9yAJ8ZYtsEQRCEBIaWp8DMuwDs0uuHiOguAMcBuATAefpr1wD4CYA/19u/wKqW981EtJqINurzDIVfPfIkxmoOnnrsqtzHMjOu+fkO7Jtp4nmnrsNzTj4KALDn4ByuveVRAMBrztmE9SvG8A8/ewgHZ1sAgNWTDbzxeZux6+Acvrr1Ufi+Kl3+zBPX4LzTjwYAHJpr4Ya79uCVzzgOjz15BPfuPoSXPGUDtj+2Hz+8c3ekHRtXT+DSc0/oaN/h+Tau+fkOzLe8nr9l7bIGLnveZhBRZPt828PVP9uBmfl2xzFP2bgSF5+5Eb9+5En8+O49wfbfO+tYnH7MCgDAQ3tnsPPJWbxgyzrc8tA+3HTfdM+2DIoXnrYe52xeix/euRub1y3DqUcvx1d++Sgee/JIrvNMjtXwpudvxlzLx0/u2YNLzj4OO/bO4Ou/3gnosvP2/9/wtVsfwyNPzODM41fjpWdsyHXN/Uea+KebH0az7ec6ToiydlkDr3v2idg308RXtj6Ktre07udpx6zA75917MDPO5LkNSLaDOAZAP4NwAaro38cgHljjgPwqHXYY3pbRCgQ0eVQmgROOKGzM8zDX/6fO7F6soGr3nhO7mMf3TeLK/73nQCAm+7fi6+/9fkAgG//5rf45A/vBQDUa4QXn340/vI7d0WOfd6pR+G723fhUz+6H0Sqbzl53bJAKHz/9sfxH762Hc8+eS3+8eaH8Q8/24F7/uJCfOL6e/GTe6Zh+m4zFcbFT9uIVZP1yDVuuGs3/vu/3AMAiPX1Ecw5XvyUo3HiUcsi+379yH78t+/d3XEOZmDleA0Xn7kR/+OG+4I2MQM798/h469+OgDg737yAG68bxq/+MD5+Ni/3INbduzr2pZBwQx89qcP4YpXnIH3f/02vPacTfjw752B9123veO39DoPAJx1/Co8OD2DD3/zdjz3lKPwhV88jKt+9lDwm396/158463Px9Yd+7Bu+RiOXT2B9371NwCAY1aO5xYKP7hzNz72g3tztVXohBn47YE53PzgE9j+2IEldy9//6xjF6dQIKLlAK4D8G5mPmiPRpmZiSjXLD/MfCWAKwFgamqqrxmCWh6jVXD00PLD41pe2Iy2H6432z7m9WjvH954DsZqDl73uX/DkzMt7DvSxJrJOn79ny7Ae7/6G/zigSfC43Sb5lo+5ls+mm0fh+bb2DfTxHmnr8fVbzoXAPCPv9iB//itOyJtMezcPwsAuPO/vAyTjfR/83e278Lb/vlXQTtt2vp3ffUtz8U5m9cG2//qO3fin25+RP92H886cQ2u+7Pn4SUf+wnm26FmcnCuFZx3vu3hd05bj2v+5NzUtgyKxw/M4WV/eyP+/Lrb9LX94P98xcvPwBuff1Km89z22AG8/NM3YWbew2GtLc01fcy2PKxbPoatH/5dvOkfbsETM00AwHu/+hs8+6Sj8JFLnhqcw34esmLu2S0fOh9HrxjPfbygeP9123HljQ8CAD7zx8/ERWduLLlFi4OhRh8RUR1KIHyRmb+uN+8moo16/0YAxvawE8Am6/Dj9bah4TPDK/DSAsp8ZLDPYa+3PQ5UVtehYDR/YLaF/UdaWD3ZAADUXSfSKRuTUsvzAwGxf6aFfTNNrNXHAIDjUOT7Nr/dP4vVk/WuAkFdW50jyVTh6d/oxEZYDhF8vc/3w/1114kI2cPz7eBzy+PgWsPmmFXj+G9/cCaOWtbAZMNF2+NAcLtu9kd+oqG+e6TZxmxTCbum56HZ9tHQv8W+F822+n/51rNRZGZD88w0crRV6OTDv38GTt+wAq979gkiEHIwzOgjAvB5AHcx8yesXd8GcJlevwzAt6ztb9BRSM8BcGCY/gQA8BmRFzjvsQZbEJhOgEiNEs1IseZSIAQOzDZxYLaFVRNKSIzVHDStEbY5ptn20dKd9b4jTTw508SaZaFQcLXWlTQa3bV/DhtXTfT8HfWaegSaCRqTETZOTO92nLAj9JiD/fUaRbSmI00v0Dbavo+aM7pO7uIzN+KXH/pdbFw1Ds8PhX8tLuG6MKEF6lzLw6z2zczrjr+h7xsRwShq5nmy/x1Fni8jSGsiFPpi+VgN33vXC/FfX3Vm2U1ZVAzTfPR8AK8HcBsRbdPbPgjgrwF8hYjeDOBhAK/W+74L4GIA9wM4AuBNQ2wbANWBJ1heMh6rlkThiBoIhUXDddD2/KBTrLtOIAT2H2nhwGwLawJNIdqZmg6saWkKjx+Yw0zTwxrLd+DqDi5J29m5fxbHre4tFMxotJWgKZgOzY11pC5RcE3f56CDjGsKM/NttPUNbnuM2og0BYPjEOqug7bvB+3IIxQm6y4AJdyONJX5qNn20Wx7wW92KLxPRiD4Cc9DHlrBM7PEjOAl4OT4fwuKYUYf3QQg7T9yfsL3GcDbhtWeJJijHXoezItfd5xIp2y2N1wHbZ8De7/rEJY1XNQcCsxHJ61Tjt1GzYmM1M05WpYt/KG9MwAQ1RSM+SjhN+w6MBfxA6RRN0LB6zyH11VTUELVYw7akWw+YjCr+5CnQx4UrkPajBdqbFmZaIRCYbapfpcSCr4lFCwfGdQ9sf8dRcxH5h7WR6hZCYKh0k+dGtn1KRRcigoFn0GkzDJt34fnhcKDiLB6so79s0pTWK01h4brRkwc7cCnwGi21foD04cBINAugHRNYWa+jQOzLWxc3dtJaTq3JIe7+Y1xoWDMVj6r3xuYj9xO85Fpn9IURv+41RyKmPHcHB3tWM0BETDb9DDbUppCS/snjIblOOF9MgKBLQ2ryOPV8ny4DskoVygFEQqFHc1qWXPjmoLqRGtmhGrMFnqEunKijidnmjg4F/oU6jW1z3TMtqPZbHswQSiYzjguFHYdUJFHWcxHxkSRFH1kTtthPtJPjef31hQAJeRG6Wi2Mf8fc4/qOTpaIsJk3cVsywsEXOBotn0K+j7FfQqu5XvJQ1n3ShCAigsF5mI2XyAcHdYc6jAfOWQ6yDDqxZhOVk/U8eiTR8AMrNIdvBl1mo7Z9KtNSyg8MK3MR2st85E5Z9wEtnP/HABkcjQHPoUETSE0H0W3E4Vmq3j0kYlianl+ZH3UjmaD61BEuMYFXC8mGjXtU9BCoe1j3vPRqCnTEiHuUwi1T9cSGHloeb6YjoTSqPST109Iqjms7jodjmYiUrZs28GpO9/Vkw08vFdl1Qbmo5gJx5zP7swO6Ixo29HspJiPdukchWMzmI/qXYRCYD7q0BQsoWBFHxk/CgAcmbeiqTwuxdEMhEI7iD7K2YaJhoPZZhtzdvRR2w/NRxSaiHw/6mgurin4QVSYIIyaSj95/YSkGrtxzaWICcpnhkuEmqtt2QmawiFtVlk9aXwKOiw00BSs0XbMAbx6sjMkNR5B9dv9syACNqzMIBQy+BTcFJ+C6WxD8xEF5zncDEtjtHwl3Opl+BS0oApCg3OOwCfrnZpCs+1hLCH6yPgTzCNV1KfQFvORUCIVFwr9OJrV0h4dA8bxqhzLbc8P9pkOceVEONJflaYp6P651eZIqOjysVrwXSAcsbdjUuG3B+Zw9IqxTJ1wIJASo4/UMin6SP1WnaegP9dcJ2jvEatekvKtcCnRRzWjsZnY/9zmI+VTCJPXonkKkUS+WEhqraCm0PTKMbUJAlBxocCcHOOf7dg0TUF1FGEoZNSWvdoy/5j1ekxTCDJkLfMRAKxZFq1v5FhmHJvdB+dwTAZ/AtCppdgEyWuxp8QMYj3tqHeD6CMnEC6HLaHQ8nx4fonRR15oPsrrU5hsuDr6yNYUQvMR2eYjLRAimkKBNrc8jgh/QRgllX7yOJZ9mgdzXM2J+xR0SKobDYU05oDVEU1BO5prUUezMTm1YkLBLnGhrm3MONG2zbW8IPGqF/HIp+hvTEles3wZdvRRw6VAa5mxfApzLRN3X0b0Eel8ESPE8z3yE3U3IXnNjySvmX8/YzCaQtvzxXwklEalhUJ/ZS7Czr4d9yk4pG3ZvpU0pW61Xc00zXwUJK95fiTuf3VMKKSFpLZyOHUDR3PX2kcp5qMg+sgKSdXnmbF8CmaUXYam4DomJLW4+Wim2Q4EW1LyWmg+ivkUXIrkLWSlJeYjoUQq/eT1F30UdvZxR7PJU2h5YUZz6GhWHftkww06lk5HMwef560OyA5HBdKT19p+dqduzSEQpWgKaRnNdkiqDsEFlNPaCDF7DgZjjy9j9FsPfArFoo8mGy6e1FVQgU6fAkUczazDnKMO+rzjjpbHEn0klEalnzw1sit4cGA+imsKys5ccylS+8gIBaMp2GakRqwoXVAQT5f23rByDEA0cQ2wkshiP6LtZXfqElHEF2CTmryWFn3kEJqeD2aOCAUTzllqmYui0UeNGg7Ohb9FleHmiE8hLXktzefTC5WnIOYjoRwqLRR4AHkKpkM3o2rWI+eaNluE5RW0UNDCwI5CiucKxDOaN+ia+mtiE+mYkg3xrOy84Z8NK+nMJi15zY4+8q3oI3PNts+YaYY+hTLNRzU3XuYiX2c7HvPNGAd61KeQnLxWC4RCvjaXFb4rCEDFhYKvC7oVPRbozCo2I2dTB8g4DU0WsNEQ7CikDvNRrCDeMauUUFi7PKYpdPEp5DHV2PkFSb+xM3kNQTs9O/pId5RtjxeM+agW8ynkbcNkIyYUtNZgJ6/ZvgSzDlhCO7emIOYjoTxGMh3nQoVRrIolEPUpAKpjrruxkFRf5SnYo1OjKRjfAmCbj/SI0yqd3fIYJ69fjo/+0Vm48GnHRNpgLCHx+RTanp9rVB6vWWQIwjhTfAqdyWuhGcyOPgo0hVLLXBQPSbVJ0hTsMFQ7oq2ouUzMR0KZVFoo+D6j97T2yZhOwIw8g/kFdEhqkEnrcaSOTc11sGKs1lVTCEpF6BH2WM3Bq6fsSekUaaWzW34+TSFeutsQ2MbjGc2RMhewylyE4a2JPoUyHM1urMxFTsE00UMomDwF8x+w/VTdSpt3Q2U0i6YglEOlhcJgCuJpTcEqdeAQqagXXSU13hn+1z84E1s2LA8+B5pCzHxkYuPTOvgwTyFBU8jR+TVcB0nzKYTmo+j2Tkezbo/lG7HLXITmo3JCUuMz4OVhIuZTMMIuaj4K/QiD8imUIUAFAai4UPCZgYIzrwUmAqMpeKFPwTGagjZbxGv4v/zpx0Y+160RNhCaj4wJJq0ztUNDbfIWn1PVTTt1pjTzEXWEpEbNR60248h8WzmwPd8yH5VU5sIrXubCNh81ak6CpoBIGKq9XjT6qOn5Mj+zUBrDnKP5KiLaQ0S3W9u+TETb9N8OM00nEW0mollr398Nq102Phefec34IszL61kjRcehIFTV83tnp8Y1hdB8ZDSF5H9TUPsoNsrP26nE51Y2pE6ykxp9pIWbr3wKJvzWCIVyCuIpR3A/pbMNqyfqoaO5i08hMB+ZPIWcAw8xHwllMkxN4WoAnwbwBbOBmV9j1ono4wAOWN9/gJnPHmJ7OvCZU+cL7X2sWtZiPgVjPrKrpPYatcfzFOKaQloHHzh845qCn19T6Jq8ljH6yJ6b4fB8G6sn6pg+NF+qT8FoBvOm1EaBMheGVRN1PLxPlT2PZzTbWc1hEELxPAUxHwllMbThCDPfCGBf0j5S9odXA7h2WNfPAnNn2ensx6oXvW5FHwHhJDs1R3W0LZ97Tphi9sd9CqZUhKlPFCfodCyjtcm9yOtTSMxTYE4cWdvRRz6jI0+h1WYcabaDSCvjUygn+khdc06bx/qJPlo9WQ/u01gseS1MYIvOvGa25aEpeQpCiZT15L0QwG5mvs/adhIR/ZqI/pWIXph2IBFdTkRbiWjr9PR0X41g9JOnoJbJQkHlKZgqqb06IkfnNRhNwQs0hR7mowRNwZiB8kYfJecpdCauAbbZyo+0wwippufj8LwXRFiF5qNyoo+AUFMoUvvIsHK8MwvdJK8lagr6WnmfsLZUSRVKpKwn71JEtYRdAE5g5mcA+PcA/pmIViYdyMxXMvMUM0+tX7++r0YMoiBevP6Q55s8hTCjOUvOQMMqJudlNR8lRB/FZ3rLQj0t+sjnDn8CEAqBMPY//A1qu681BZWLMdvK36ZBYf4/c20PrhMmEWbFaAoTdRdj9bD9tvmIrTDU+CQ7QEHzkeQpCCUx8reUiGoA/gDAl802Zp5n5if0+q0AHgBw2rDbYhyERRLY/Lj5yCqK5jg6S1hP7pJlhFy3cgVMJx+MsFNGjUkZza12tF1ZqLuUWuYi0XzkhA5l+3PdcpgfaXqB+WiuWW70EaDKd+c1HQGhT2Gy4UaEc6ejOdQUg7k2tOkqz+PFrAYSYj4SyqKMJ+93AdzNzI+ZDUS0nohcvX4ygC0AHhxmI+wRXdFcBSCcI8CPmY9qjgNmZUrJ0hnZdv143kGapuC6CUKhQDmHVEczd4ajAqFPwUQ9xUNSzXzSKydUHEO50UfhXBVFhJIxH43X3YhJJ/ifdPgUOgsJ5tEUipj/BGGQDDMk9VoAvwBwOhE9RkRv1rtei04H84sAbNchql8D8BZmTnRSDwr7PS1SFC9e5qIdCIWwSiqgRqi9HM2AmbUs6mi29yXhJuQphFVZcziaUzOaVXZ2x3X1qVsxn4LpyIxQCBzNJUYfBeajlldIKDRcB65DkVLnQFRTAOzoM+4wLeZ5vMw9FU1BKIuhhaQy86Up29+YsO06ANcNqy1J2B1pEb+CiVpKKnPhkm22CM0o3RirhZpCvOpp2qgxnKPZdjQb+30OR3OX2kfdoo/MMfHoo4NzSiisGI9GH2URjoOm7lpCoUBHS0SYrLvafBRNZAM6JzqKJK9RVIvMQjvQFEQoCOVQ2SfPfk8LCYWYT8Euc6DyFNT22ZaXzXxkCYWsmkJSpxOf/jMLqY7mlJBUs82011wqEAqzKmpqsuGi5lCpeQomJLWo+QgAxhtuqvnInNI4+O1EtiD6KMfj1fTym/8EYZBUViiwFShYxHwUjzAJzEc6+igYoTa9TKM+264fz1BOC09MmqO5iPmhnpKnoMxH3TSFqJkkbj6aqLuouVSq+ShMXitmPgKUcEszH1FMU7Aj2twCyWttX8xHQrlUtvYRRzSF/MebF92MGG1HM5EdCplthGrb9eOdSKqmEJvLAbDMRwPwKdjZyjbmt8XNR42Y+Wii4aLuOEFxvDLMR5Hoo4JC6bknH4VNaycj2+yCeEDUfNihKeS4lokeKyN8VxCACguFiE+hiKagl2b027Y6hbrrBB3gXCurpkBBhxDXXLqZElyHgglkANsmncenoOYc4Jhm4HNyBnCHTyEWfXRQawqTDaUpBJ1kGZqC0djaXmGh9Nd/eBYA4MobHwCAoOChWQeSC+IViT4S85FQNpUdjtj9bpGs5g6fgmU+cKzoo+w+BRfzXr6QVEBF/tiD/KLJa8yd1/X97tFHwRSXsYzmgxHzUdiOskpnA0o4F8lTsDH/B9uMZO5Pu5umUMB8JFVShbKo7JPXd/RRUOYiasKJm4+Ys42QI3kKGc1HgLqOHzEfaWGVowM0SWdxZ3Ov2kcmA7sz+kiZi5T5KDy+jOQ1c/35drHkNZtGTUUf2R12PGfDLo4Xls7Ofg0xHwllU9knzy5nXKQoXjxrtR3TFOyOPIvZolEjaz6F+L7uQsFLCEnNM8dvMI1mzNmclrwW9ymYnxpPXpts1CKdW7+dchHsPIV+NRXzfzDCAeh0NLOdvJYy30U3iiQfCsIgqaxQsF/UQuaj2ExegfnITLJjdYBZHJy2ptCOSYVunZlDsdpHQfJaDp9CrHS3Ic18ZEbHzVhGs+uouanjPgX1G/LXHRoERigVLXNhY+7TWM3WFNSynRB9FMy8lmPQYbQviT4SyqKyT54tBoo4mrtVSXWduKaQLfqoFfgUovt6O5oTNIUcnYo9t7JNWvJap6YQfqeu55EgUp2n0ZLKKJutrjs481WST6EzeS2sfWT8GcXKXFT21RRKprJPXr8+hSD6KFap1JS5sDvKLPZhO1fAThrrNcJ2HSei6RSZiziYByGuKXBKldS4ULC+Y4TARN2NlPsoa9KY6P+hvzYYDSHqU1BLe1AQn4ApD2I+EspGhAKK5SkEk+zUEqqkUrRDyORorkUL4o3r8/YaMbpOVNMpkqeQVyiYTWZUawstcz9MyWkjEMsa+drX7VdbCX0KdvRR1G/gc2diYy5NQcxHQslU9skbVEE8MzIO51PgDkdzpuQ1N1o6e1yXbO4pFIgitY+MTyHXHM1WJVGbVPNREHGTbD4CELZf7ytrfgC7bYPyKfQKSe3wKeSJPhLzkVAylX3yBhWSGp+j2UxkH7VlZ4k+UkLBTKeZVSg4DiVrCjnMD2OpIamd8zMD6dFHdntDTSEarjpqagkCqyiBTyEhJNUkECYWxCtU5kLMR0I5VFgo2Ov9JK91hiSa+RQMWV7whk4ga/tqitDxuumAuh9bcyha5mKAPgVjCosTTLITiz4yvwMAJhq1yLnL8inY1x2GphBEH1l5CnHzUZ7ktaaYj4SSqeyTZ4+u+ymIV4uZj/wEn4KbZT6FWtgxRzSFHvkGTiz6yJh08pR0MEKrlWQ+6jIdZzPBfGR+96Ruf61k85EtnPtNCOvmU0gqnV1kPoWwym1lX02hZOTJQ7HkNSNU4o5mjzlSJRXIrikAaqTo+RxMA5nFp5CYp5BHU6iFPgVbW/B8zmY+sh3Ngabg6u+W62iOCKxBhaQmZTQnRR/FZuXLQhHznyAMksoKhUGFpHZMx6lLZ9vaQZbOyO6YgbBT7eUw7shTKFB62Vzj8zc9hBd99MeBuUOZwjq/3zEdp5MuFOolh6TWR2Q+6qYp5Hm6xHwklM3QnjwiuoqI9hDR7da2K4hoJxFt038XW/s+QET3E9E9RPSyYbXLMKiCePHpOIOQ1EhGc+/bPGYm5dGzlI3VspmPOmoftfObH0wnd8uOfdh1YA77j6iM5PTaR/pasSqpQChgAvOR8SmUlLwWiYzqNyS1q6M51BSMFAiy3XM5mvNHjwnCIBnmk3c1gAsTtn+Smc/Wf98FACI6A2ru5qfqY/4XEbkJxw4M+0XN4wgMj1fLzuijpNpH2fIUgHA+46yOZteJhaT6fqQgXxbitY/2HJoHEIbXJl0TSIk+qql9gaYQm4Bn1Nj/h6LzKRi6haTaBRFDTUF9L8/jZfw6Yj4SymJoQoGZbwSwL+PXLwHwJWaeZ+aHANwP4NxhtQ2ICoKE+WUyHe9QZ9EzjxmOE09ey5bRDIRCIatPwaF4mQvOPSKOd9h7Ds0BML8xKXmNQJQcfWQ0golYSOpC0BT69ilkcDT7VkG8ME8hT0G8/LWrBGGQlPGmvp2Itmvz0hq97TgAj1rfeUxv64CILieirUS0dXp6unAjIuajgslrdjmLqPmIcsfHm45mrmk0hYyO5pj5qO35uUeZcVPFnoNaU0gxHwFKGCbXPjLmIxWSGpiPygpJzZkv0utcdZeCHAygsyAeEGoNYZ5C9mu0PL+04oGCAIxeKHwGwCkAzgawC8DH856Ama9k5ilmnlq/fn3hhgzCfORYZprOSXYss0Uf5qMsQiESfeRzbidlvDT3bq0peD4SNQVAOZebST4FbT6a7DAflaMp2IK7X8FERPj71z8Lf/zsE4NtgU/BjtqKCcu8ZS7EySyUyUin42Tm3WadiD4L4P/ojzsBbLK+erzeNsS2hOtFHM1sCt8lhCTGHc1ZRqhGmwiFgo4+qvXwKRBFSm039UgzD/FOyGgKaclr5rrdNIXxRtzRXN7I1wjOQczn8JKnbIh8TtYU1LLYzGv5hbogDJKRPn1EtNH6+CoAJjLp2wBeS0RjRHQSgC0AbhlmWwZREM8ha3Ytaz4FKmA+MqUmZguYj+LJa3nNJPY1jlk5jmnL0ZxqPnIoCElNNh8tjDIXQNg555mNLitxn4JajwrLPGOOIkJdEAbJ0DQFIroWwHkA1hHRYwD+M4DziOhsqKC9HQD+FACY+Q4i+gqAOwG0AbyNmb1htU1dM1wvNp9C6IS1S00Y81HeQmym05xr5RMKjkOwSxa1Pc5tJjGd0PoVY9i8bhK7D2rzEScnrwGIOZo7zxWaj8r1KQChUMiSWZ6XwHyUVL68SEazJ+YjoVyGJhSY+dKEzZ/v8v2/AvBXw2pPnMjMawUn2TEdgmOFhaq5ENQIsu6SigbK8JIHjuaWGmVm9SnU4gXxCpgfTFtPXDuJo1eMY9uj+wEowZlU5gJQgi4pT6HTfFRu9JFqw/AEkzmj/T8w64V8ChmfF0EYFpV9+gZREM90CHbHHNUgstvTTQTQEWM+CiaJ736s01E6u5j5oeE6OOGoSRy9Ygx7Ds0F1VqL+hQmY5pOmSaRwNE8BPORkXVtP0FTcPP7FJoFoscEYZCM1NG8kOi7zAWHiUuq/pA5V2hnDorBFchTqLmEhusUmmSnyKj8PRecjmecsBq/3LEPcy0fh+bbqbWPAKUdddMUJnWV1PAelNfR1QPz0ah8CsVDUtueL9nMQuDYdpMAACAASURBVKlUVigMJHnNCc1HxrnoW6Pr0HSS3dFsfAoOEd790i143inruh7nxktne1xoVP4nLzgJAPDIviMAgD0HlbaQaj4iCnwKdmdrNJuJjuij8jo6d4jO7nhBPHu9qPlINAWhTCosFML1opPsJDua2YqLz27LbsSEQs0hvPW8U3se5zpORFNo+35fJaLXrxgDoMJSPU7OaFbXTXakm2vHC+KVaT4yzu5haArxgnj2epHS2S1xNAslU9mnbxA+BdMhKE0hPG/cfJTlJY+bj9LMNnFcio5SWx73ZTs/esU4AFX/yPPT22EP/BPNR8F8CkYwlqgpDNOnEKsYC4RCoRbUPspXOluEglAmldUUBjEdp+n8lU/BD17+uPkoV0azdjSnmW3iJE2ys2ys+L911UQdAHBorqXMRyn9ky0I7J/3wi3rsGPvpuA8QZ5Cyclrqi2D72zNbUiKZiuSp9Dywln3BKEMKvv0+QPwKZhuztWaQtzBaMwWWQrUmVGs0RSymjpcipXO7lNTCGZh87i7+cjabrf1acetwt/80VmBhhHOp1DeoxZMCToMRzPSHc1FCuJJnoJQNpV9+gaZvOZqR7M5TbzWThafAhGhUXMCn0JWoVBz41VS+/MphPND+Kmls4GoWambqau2AJLXzL0cpk+hu6M5+/n6FeqC0C+VFQqDCEk1766rs4rNeYJQ1Rx5CoCaaCevphAvna1q5xTvVExbWx7D71bmwtYUupi6AkdzmclrQ5zTwQhEz7enMfVh35I8z5ffRTsThFFQYaEQrhebec3yKejkNXOawHyU03RSrzmBTyFrxxAPSS1S+yjSBqMpeBxUgk3C1g66CbCFoCmEvp1hhKSqZVxTcIiCe5TH0Tyown2CUJTKCgWOaArFjjd9jKlU6sUdzTkTtxquE5S5yHpM4iQ7fZiPXEdNoGN+T5ppyE2JPopTWwA+hTyZ5XlJS15zKHwO8jxfoikIZVPZ6KPB+xRCM0GQv+BmdzQDairLmfl8moIpsfHIE0fwyL4jwSQt/VB3nNB8lOZTSIk+6jhXcA/K1xSGoa2kJa8RkZXRnMd8lD0cWRCGQWWFwiAL4pnZz1ible2kNrM/Cw3XwROtZq5jzBzNn7vpQXzjVztRrzl9d341l9D2/K6jVicl+qjjXEMMB81K3v9DHoLkNStPwWS1m1uUxzrp+QxJaBbKpMJCwV7vryCeqZLqx81HOYvBNWpuWPsoq6NZC6TD820cmm/DbdFApp1Uvyd91Go6WCJ0nToy7z0YBmHy2hDyFExIKif4FKiYT0E0BaFMKutT6Dv6COFI0JhwAp+CNamLstFn1RSow1ndC1f7FObbSk3x+ow+ApTJx5yvW+2jbvvDcw2vQ87KUEtnG7+BH9cUbPNR9vN1qzclCKOgskJhIAXxrI6x7ftWSGporshjsrDnSs5jPvIZmG+FcxL1m/xUcwnzbePbSP6O6eN7jWqPWTmOibqLE9ZO9tWmfghMWMMscxH3KcCaayGPpiCOZqFkhiYUiOgqItpDRLdb2/47Ed1NRNuJ6BtEtFpv30xEs0S0Tf/93bDaZejbfGRNau846rM5jRnp1V0nl4O1qFAAEDiogf7t9zXHQVNrCr3MR71GtUevHMddf3Ehzjx+VV9t6oehJq/pWx2PPiIqVjq7W70pQRgFw9QUrgZwYWzb9QCexsxnAbgXwAesfQ8w89n67y1DbBeAwUQfheYjBx4n+RQoVwdtj/DzCoUjzXZ4nj47lbpLofkorSAeDa+jHTRBuZGhls4O1c2278NxCKQvl8enYGbuE4SyGNrjx8w3AtgX2/YDZja9180Ajh/W9XsR9SkUOb5zOs547aOa4+Sy79uTq2QuiEdGKAxQU7B8CqnmIwodzQsd1x2ipmCijzryFIqGpIpPQSiXMsckfwLge9bnk4jo10T0r0T0wrSDiOhyItpKRFunp6cLXzwSklqozAVbM68hktFsth+9cgzrlo9lPmfdMh9l9cvWnE6h0K+jueYQmu3u+RLDNMkMmqDMxTCij1J8CnbyWt6Q1KyBCYIwDEoJSSWiDwFoA/ii3rQLwAnM/AQRPQvAN4noqcx8MH4sM18J4EoAmJqaKjDGN+cJ1/tPXnPgWSGppqN81/lb8JbfOSXzOcesEX7WaB1jf56xzUd9agqR6KNe5qNF0IGZe+kOI/pIL+PRR9Hktezn61ZvShBGwcg1BSJ6I4DfB/DHrI2tzDzPzE/o9VsBPADgtGG2YxAhqWFBPGihoD6bzmC87gbzCmShUUBTMP3ckYijuf/ktflW5/zLketmjD5aCOSZFjUv5v54Mc3TTl7LG30kQkEok5EKBSK6EMD7ALyCmY9Y29cTkavXTwawBcCDw2zLIAviGUezsSsXHTzXC/gUTAfStCZ879dMUnccNL3BRB8tBEYSkmplNLe94slrdlSbIJTBMENSrwXwCwCnE9FjRPRmAJ8GsALA9bHQ0xcB2E5E2wB8DcBbmHlf4okHRERTKGA+4th0nMqnEHU058XWFLKaj+zKnycepXIBBqIptLvPALeYoo9CoTC8mdc6QlJh5ylkP589zasglMHQfArMfGnC5s+nfPc6ANcNqy2JRPIU8h8e8SkQgrIQQPGO0tYUMpuPrO+ddswK3LfnMFaMZzdZJVFzHct8lHbdMEdjoeMO0acQzqcQz1MoFn0k5iOhbCpc+6jPgng+Uh3NRd/pIslrtlZyxsaV+JPnn4Sn95koVneop/koCMddBKaO5eM11F0aziQ7+pQd0UeO7VPIdi5m1pM3Lfx7KixdKiwUwvU8Nt/weA7sA6GjOVrmIi9jfWQ0A8qx/awT1xS6to3taO5pPloEHdirp47HM09YjbGaO/BzB47mmKZQc1XNKyJkjkk15xBNQSiTRaD8D4e+8xQQn46T4cdKZ+fFHsnmdTQDwHh9MP/Omms7mtOuC71/4XdgK8breMYJ/QvLJMyvT0peA9SzkFVT6Nf8KAiDoLJCgTmMFOq7IJ52NId5CsXa1OijzAWAgY2E6w51ZGenXXcxaArDJJh5LTbIMLfFoew+hfgc34JQBpUVCmZUVnOooPkoakJpD8B81NCdeq85CmzsTnmQmkJw/l4+hYqParuVuQDU/zGrphCYj0QqCCWSuRchogkiOn2YjRklpgOvaSdxkeOD0WCgKejPfZqP8sTT253y+KA0hQxmrLDMxUAuuWgJ8xSsgnieH5iVlEsho0+BxacglE+mV5qIXg5gG4Dv689nE9G3h9mwYcOWptBvQbya8SkMKPooj1CxBcjYoDQFy5GQprEsJkfzMElzNEd9CtkesPh0roJQBll7kSsAnAtgPwAw8zYAJw2pTSMh0BTc7C+tDcc0hbbPQRJc4eQ1PezOM1J0YtFHg8BOfutlPqp68TZTHrtjkp2ITyHbubw+BxWCMAiyCoUWMx+IbStcjG4hEGgKbjHzEcd8CoMwHxlNIY9QiPgUBmY+sn0KKdd1zLLaPdggNQUJSRUWAlnzFO4gotcBcIloC4B3Avj58Jo1fEKfQjFNwS5HYCa65wGZj3IJhaGYj8JzppqPJPoIQBiSmpS8BqiggayPV2B+FKEglEjWXuQdAJ4KYB7APwM4AODdw2rUKAiijwqaj+yCeOYlNh1D0ZfajNDzdLTuEBzNtlBIdTRbU5FWmTSt0I7OyuxolugjYQHQU1PQ1Uu/w8wvBvCh4TdpNPQbfcQxTQEAWl73ekG9WDCaQoaQ1MU0yc4wSeu/KWI+ynYu0RSEhUDPXoSZPQA+EZU38/oQYCv8r9+CeE6HUBiho3kIPgXb0Zz2UxZT7aNhkvb77ZDUzMlrEn0kLACy+hQOA7iNiK4HMGM2MvM7h9KqEWAnrxWbeS3sMI263/IGUzo7z/HD0BTs+Rh61T6qegeWPoe1WuZKXuszI14QBkFWofB1/bdk4D59CiokNWpCKUNTiOQp1AZlPuodkirRR4qePgXKkbzWZ0izIAyCTEKBma8hogbCKTLvYebW8Jo1fCI+hQLmo0hIakwoFJ55TXfquTKa9cXGas7AcgZsn0Kv6KOqd2BZzGuZk9cko1lYAGQSCkR0HoBrAOyAMpNuIqLLmPnG4TVtuJjkM7ew+Sh0NIdCob+X2mgKeRyN5lqDSlwDVEG8+Pk7rhsIxIFddlGSJjTt5LWsimiYvCZCQSiPrK/0xwFcwMy/w8wvAvAyAJ/sdRARXUVEe4jodmvbWiK6noju08s1ejsR0aeI6H4i2k5Ezyzyg7JiylTkqWKZdDwQdpztfs1HtSIhqWo5KNMREIs+6ln7SDqwpFvQT0E8EQpCmWTtSerMfI/5wMz3Asgy5+PVAC6MbXs/gBuYeQuAG/RnALgIwBb9dzmAz2RsWyF8VvPoOkR9F8TrdDQXa1M/0UcD1RQk+igXSffA+OodJ7tPwUQfiaAVyiSrUNhKRJ8jovP032cBbO11kDYv7YttvgTKFAW9fKW1/QusuBnAaiLamLF9uQk1Bcqs3tvYPgVj7mkGPoU+NYVcjmZ1zKDKZtvn7NYW0RRCEoVCkTIXEn0kLACyPn5/BuBOqPIW79Trf1bwmhuYeZdefxzABr1+HIBHre89prdFIKLLiWgrEW2dnp4u2ISoT6HQzGta0wA6k9eKdpSuo8xZeXwKTmA+GpymkK0gnlmKUOh2C1SeQrbz9DsfhyAMgqwhqTUA/4OZPwEEWc5j/V6cmZmIcvXIzHwlgCsBYGpqqnBRPjWdJsFxipqPknwK/Ve5bNScXNFHoaN5cMNL23yUGocv0UcBdvhpvChiHk3BlzIXwgIga09yA4AJ6/MEgB8WvOZuYxbSyz16+04Am6zvHa+3DQXfV9FDeeLII8dzWPRsUHkKgPIr5HI0D8GnYJuPUqfjlOijgHgUmr0tT0E8qZIqLASyvtLjzHzYfNDrkwWv+W0Al+n1ywB8y9r+Bh2F9BwABywz08AxI32XipmP7IJ4poNs9pmnAChNIU+ROdOBDDb6KIP5SHwKAZTgdLc1Bc5YZV5CUoWFQNaeZMYOESWiKQCzvQ4iomsB/ALA6UT0GBG9GcBfA3gpEd0H4Hf1ZwD4LoAHAdwP4LMA3pr5VxTARA8REayZFDNjF8RzOsxH/WkKtRxSIRAKA40+yq4pSAcWDgKSSo47REFUUS/M2EQErVAmWX0K7wbwVSL6rf68EcBreh3EzJem7Do/4bsM4G0Z29M3pkyFmyNk0MYuiGc6g2a7P0czoLKa8zmatflokI5m2wwi0Uc9ifuW1Da1pBx5MGGewmDbJwh56DokJaJziOgYZv4lgKcA+DKAFtRczQ+NoH1DQ5mPdPRR0YJ4et2Eks63PQB9mo9cB26O482IfVDF8IDYzGtS5qInQQn1BA0rT+nswHwkUkEokV49yd8DaOr15wL4IID/CeBJ6AigxYoZ6VOO6BAbuyCe6UTnWv07mo9bM4FjVk30/qLGHYamkCX6SEJSA5I0BbI0hezJaxJ9JJRPL/ORy8wm+ew1AK5k5usAXEdE24bbtOHCUHZfN8dILnK8FZJqNIU5rSn001H+/eufVah09rCS11LNRxJ9FEAxMyJQMCRVfArCAqDXK+0SkREc5wP4kbUvqz9iQWIcxQ6hcJkL8+42Ak1BCYV+RnpjNTdivulFzSGsmazjmFXjha8Zx85T6Gk+kg4sMSTVLoiXt/aRKApCmfTq2K8F8K9EtBcq2uinAEBEp0LN07xo8f0wea1wQbxYOKgxH9EIR89EhBvecx5WjA9ORifZxuMEmoL0YGENrARNgYgyBqSGDmnRFIQy6dqTMPNfEdENUNFGP+DQOOoAeMewGzdMzEjfpeKls01nUI9pCqO2s69d1hjo+ezS2WnRsRJ9FOIkCEjH0hTyTrIjglYok57DS12cLr7t3uE0Z3SY5DOnYPIaM0CI+hTm28bRPLh2lkGkdHaP5DVxNKeFpBbxKUj0kVA+lXUTmoJ4ynxU4HhYPoVauZrCoIlGH/VyNC/u3zoIksxHRZLXfMloFhYAlRUKJiTVdVDQfBS+vMZ8FGoKi/ulrmeofWSbR6pOkMSYNA9FruQ1tRTzkVAmlRUKqkpqPvXexo4+GuvQFAbVynLIU/tITB129JHTsS3PdJxmcJKn9pUgDJrKPn72JDt5Q1KZWfkUYprC7FIxHyWUa4gTOJoX+W8dBPFyJ/a2YpPsyD0VyqPCQkH7FAokr5l33HYwuk44g9tiHz0TEWoOBQUDk0hyrlYWoymQbT4q7mgWQSuUSWWFAts+hZzmo9AhGG4zCWxLpY+sudS1c3Il+iggzEmIJq2ZbZnzFILkNbmnQnlUVij4PoLoo9zmI720310TgbRUOsm6071aq0Qfhdh1oAjhulnmzWiWeyqUSXWFgtYUHKLMjkD7WCA6oqu7S0so1FzqqvXER8RVJhAATnRqTrPMnLxmah8tkWdIWJxUWCiEBfHyJq/FfQpAGIG0VN7nWo9pQV2JPgqw/Qd2eQuzzGqeZJboI6F8Kvz4FS+Il+hT0EJhqaj+dYe6m48k+igg9B9QpGS22Zc1eS2cZEfuqVAeI690SkSnQ03WYzgZwH8CsBrA/w9gWm//IDN/d1jtCEJS9Rttz4+Q5Vgg+vKayqJL5YWuuU7X3xKaTJbG7+0H22Rk+xKAfJqChKQKC4GRCwVmvgfA2QBARC6AnQC+AeBNAD7JzB8bRTvsgniAGqXVMk55FvoUwm2NJWc+oq6dU000hYBQK6CIL8EscyevyT0VSqRs89H5AB5g5odHfeGgIJ5+e/P4FcxXbc2iscQczXWnu6ZwwtpJvOMlp+J3Tl8/wlYtTAL/ATo1BYcInDEoNShzIZqCUCJlC4XXQs3ZYHg7EW0noquIaE3SAUR0ORFtJaKt09PTSV/JRDjJjjEfATc/+AQ+8r/vyHQssLR9Cr2ijxyH8J4LTse65WOja9QCJdmnkD8kNclXJQijpjShQEQNAK8A8FW96TMAToEyLe0C8PGk45j5SmaeYuap9euLj1J97UMwL6DnM7669TFc/fMdGY5Vy6hPYaklrzlLRsANG9unQNY6oExLeTKau2WRC8IoKFNTuAjAr5h5NwAw825m9pjZB/BZAOcO8+Jq5rVwZO8z497dh8DcOxopaUQXhqQujRe67tCSMYUNGzt5zV43y6yWSc9n8dEIpVOmULgUlumIiDZa+14F4PZhXpxhNAX1ErY9xn17DgEAWl73GMKk5LUwo3kYrR09NZckXj4jgXZgJa9RAU3BY5ZoLqF0Rh59BABEtAzASwH8qbX5o0R0NlQViR2xfQNHhaSGnfhDT8wEcyy3PB/jdTf12KTkNWM+WiojvXqP5DUhJOpTiCav5SmIx7x0nh9h8VKKUGDmGQBHxba9fsRtgOOEdvO7dh0M9rW8/OYjE320VMxHtR7Ja0KIqXiUFJJKOZPX5JYLZVNZA4FJXjOduC0U2j3MR2FIargtMB8tkTvaK3lNCDH/86TktTz30PPFfCSUTymawkLARHoYTeHuXYeCfc0CPoWlVhDvRVvW4eR1y8puxqIg8COgc5pSJ2f0kUR8CWVTYaEQFsQDgLsfD4VCu4f5qFtBvKViE379czeX3YRFg+0/6Men4LNEHwnls0SMHfkxyWvmHTw838ZzTl4LAGj3MAJ3K4gn73T1sB3NoSnJrn2U7Tyev3R8UsLipbKaAmufwknrluHYVeN434VPwXjdxc0P7kOz3cvRrJbJyWvyUleNSPIaOkNSs86n4PsMt7LDNGGhUFmhYAriTW1ei59/4HwAwA137QaQXVNYyjOvCdlJTl4Ll5k1BTEfCQuAyo5LjE/Bxoz2eyWvdS2IJ47CCtKZvOYU8SlI9JGwAKisUDA+BRtTOrvlMXbun8V9uw8lHNm9IJ6809WjZ0G8jKqCmSJWEMqkskIh6QW0NYW/+d7deOeXtqUcq5ZOkqYgL3XlSJ5kR+0jQsbC2WqOZglJFcqmwkKhM1LICIW2xzg018L0ofmUY0VTEEJMxBGBEs1HeSbZkedHKJsKC4XO6TfNbGJNz0fT83FgtpkYOdKtIJ6EFFYPu1y2PTezWWYuiOdL8ppQPpUVCuBOU4/p2Nseo9VmtDzGbMvrPLRbQTx5qSuHnZPgkGU3Qr45msWnICwEKisU/CRHs2MczT7mdQTS/iOtjmOD6CNrm5iPqosdkhrXFChHSKoIBWEhUGGh0Kkp2I7mZjtdKAQ+BevuLbUqqUJ2zH88rSBe1uQ1MR8JC4EKCwVOdzT7jGZbmY0OzKYLhahPIRp1IlSHQAA4/SavSZ6LUD6VFQqcqCmE5iNTKfXAbLPj2OSQVDUpj4z0qkdYBA+JBfGyagrMDFceH6FkKisUfGbE379aYD7iruaj7slr8lZXjcB/gM45mvMVxBOfglA+pdU+IqIdAA4B8AC0mXmKiNYC+DKAzVBTcr6amZ8cxvWTk9dCTcHMvrY/0XykltHoo+gIUagO3ZLXzJITQqDjyCQ7wkKgbE3hxcx8NjNP6c/vB3ADM28BcIP+PBSYO2dJC5PXQkdzkk+BuxbEG0JjhQWNXS47bea1LNqCzKcgLATKFgpxLgFwjV6/BsArh3WhpIJ4YfJad/ORH4SkdiavyUtdPZKS14Kl/k6WXAWJPhIWAmUKBQbwAyK6lYgu19s2MPMuvf44gA3xg4jociLaSkRbp6eni188IU+BiFB3qaejOdGnICGplcX8x+3ktUBQOEZT6C0UkkqvCMKoKXM+hRcw804iOhrA9UR0t72TmZmIOt4kZr4SwJUAMDU1lbXWWAdpiUI1x8FsM8xiTg5JVUvb/ivmo+pim4qSktcAZKp/JHM0CwuB0jQFZt6pl3sAfAPAuQB2E9FGANDLPcO6flLyGqAcxjPz7eBz1+S1BE1Bokeqh+1UTvMpZBEKni8+BaF8ShEKRLSMiFaYdQAXALgdwLcBXKa/dhmAbw2rDWnqfN11cMTSFLoJBdtUVHMdOCR5ClXENhUlFcQDsvsUJPpIKJuyzEcbAHxDv0w1AP/MzN8nol8C+AoRvRnAwwBePawGJCWvAUoozDSVpjBed5Kjj/QyqUyGDPSqh/0cdPgUguij3kKBWQIVhPIpRSgw84MAnp6w/QkA54+oDYn2/5plPjp6xTge2XcELc9H3XXAzHjLP92KdcvHAKAj+a1Rc8R8VEGS52iOCocsIakec0eYtCCMmso+gn5KnZm662BmXpmPjl6hOv+DWls4PN/Gv9yxGzc/+ASATk1hrOaIo7mCmOfI9ikEEUl6maXUhS8ZzcICoMJCobMgHqAczUe0+Wi9Fgomq3mPnonN+Bzixzdc0RSqSFgllSz/gtmmllk1BfFJCWVTZkhqqaT5FGqOgyebSggYobD74BxOWb8cuw/OAUBgXoof/+9eeho2r1s2zGYLCxC7IF4Yipo/T0Gij4SFQKU1haRBWb3mBJ3+1Oa1WDVRx998/x60PT+Ys9loCnH77/87tQnnbF471HYLC4/kSXaiPoUsIamckGUvCKOm4kIhwafgUNDpH7NyHH/5yqfhN4/ux+duegh7Diqh0PZNnoK8wEL2gni9UGUuhtJEQchMZR9Bnzujh4CwKB6goole/vRj8fRNq/Gju/Zgz6G5yHfF/CsAlgBw+iuIJz4FYSFQSaHACclnhpo1y4nJUn7KhhV4cO/hwNEcIi+wYPkU0FkQLy157Y7fHsDf/vDeyDaJPhIWAhUVCmqZ9AI2IpqC2n/y+mXYe7iJe3cfjnxXBnUCEPUfhCGp0WVcKFz9sx342x/eh7lWmD2fZtIUhFFSSaGQVLvIENUU1BSbJ69fDgC4+/GDke/KCywAiDiX476EtIJ4tz6s5o6yS6pI6WxhIVBRoaCWSclrtZhPAQBOWa/CTOMvtggFAUgpiBcktIWawpMzTXz6R/fh8QNzeHDvDABEii+mFWkUhFFSyTyFsKBd575GglDYtHYSNYeCqCODvL8CEK1zRPHoI/04+Qz88K7d+NgP7g20BEBlyRsk+khYCFTyEezmU6hZ2oMRCnXXwQlHTQIA1i1vBPuloqUA2D4F29EcL53NQaDCj+8JJ4cy2fOA9inIMyWUTCWFQqApJOyr1yxNwRq2nbxO+RVOPCrMWJbXVwCyF8TbczAMaR7Tz9nheXE0CwuLSgqFtNLXgEpeC9Ytp/MpRythcOLayWCbvMACENUUuiWv7Tk0j5PWLcPUiWvwiqcfCyDqU5AyF8JCoJJCoZtPwSSvNVwnksfwglPX4YS1kzh1w/Jgm2j6AhCNPgpzFuIhqaqG1sZV4/janz0P7zx/C4BQKDBzauVeQRgllRQK7Ktlok/BCIVa9Na8cMt63Pi+F+OoZaFPQerUCEBaQTy1tJPX9hyaD8qxLxtTMR5GKJgYBtEUhLIZuVAgok1E9GMiupOI7iCid+ntVxDRTiLapv8uHlYbuuUpNLTJKC4UDON1N1iXQZ0AxH0KUQezERier4XCynEAwLIx9RzN6DyFbs+kIIySMkJS2wDew8y/0vM030pE1+t9n2Tmjw27AcEL2CVPoZESGzgREQryBgvROkfxUFTz+cBsC822H2gKDddBzQln+fP89GdSEEbJyIUCM+8CsEuvHyKiuwAcN8o2GFW9W+2jei355ZxohEJBZIIARJ3K8VBUs3z8gIo8MpoCEWHZWM0yH6mHUjKahbIp1adARJsBPAPAv+lNbyei7UR0FRGtSTnmciLaSkRbp6enk77SE+4SktrooSnY5iPxKQiA7VOgzugj/Rg9rsNRjaYAAMvHaoH5yGgK4lMQyqY0oUBEywFcB+DdzHwQwGcAnALgbChN4uNJxzHzlcw8xcxT69evL3TtbiGpJnmtUXM79gFx81GhywtLDFsQxJPXzHJ3glCYbLgdjmaRCULZlCIUiKgOJRC+yMxfBwBm3s3MHjP7AD4L4NxhXb+bfyq5OwAACgNJREFUU88kr2VzNMsbLESn4Iwnr5nlrpj5CFARSKbMhe+L+UhYGJQRfUQAPg/gLmb+hLV9o/W1VwG4fVht8LuUuahrfX8szdHcEKEgRLHDT8PS2Ygsdx+cw7KGi+VjoRtv+VgNR5oeZpse9h1pAhChIJRPGdFHzwfwegC3EdE2ve2DAC4lorOhrDs7APzpsBpgRmWJyWu17iGpE3VxNAtRkgvidTqabS0BUOajvYfn8RffuRM33LU78n1BKIsyoo9uQrKP97uja4NaJvsUTBG85JdzvB4KC3mBBSBe5gLButqnltOHVYkLm+XafHTv44ewW8//LZqCUDaVzGgO8xQ699VTMpoN4zXRFIQoiclrTtTRzKxKsNtMjrk40vQCf4N9LkEoi2rPp5CgsNSDjObk6CPHIYzVHMy3fdEUBAApyWuxJQBsPioqFJaN1XBoroUDs62OcwlCWVRUU1DLpPevV0YzEDqbZVQnAFFTUYdPwXpITjgqZj5q1NDyOMhRAMR8JJRPJYWCyVRIjD7qUfsICJ3NkrwmANE6R5TiUwA6NYXJsU5FXYSCUDaVFApdQ1IDTSH95Ryvu6IlCAG2IAhDUuNBqcCJa+OO5k4TpQw0hLKpqFDokrzWw9EMGKEgL6+gyOJTWDVRx6rJeuS4ZZamsG65ynSWMhdC2VRTKOj5FBIL4jlZzEeORB4JAVkK4sVNRwCwrKGEworxGk7Tkzd1cWUJwkio5CPYdT4FU+bCTY4+ApSjWdR8wZBcEC+6jDuZgVBTOHbVBI5dPRH5viCURSWFAncrnZ1JUxCfghBiF8QLktf042MesRPXJmgK2qewcfU4jl01HjmXIJRFJYVCvz6FMfEpCBbG5OM6FEQPmefDfD6hi/no2NWhpiDRR0LZVDJ5be2yBl57zqbgRbQJzUfpL+eECAXB4rknr8N7LzgNZ2xciXXLx9CoOUHhu9M2rMB7LzgNFz3tmI7jQvPRODYa85EIBaFkKikUNq2dxF//4VmJ+45eMYb/8LLTccFTO19iw0TdFUezEDDRcPH2l2wBoEb9b3ju5mCf61CwL8665Q2878LT8YqnH4t1y8fwjpecinM2J84tJQgjg8wsZIuRqakp3rp168ivu/2x/fjNo/vxeuvlFwRBWCwQ0a3MPJW0r5KaQr+cdfxqnHX86rKbIQiCMHAq6WgWBEEQkhGhIAiCIASIUBAEQRACFpxQIKILiegeIrqfiN5fdnsEQRCqxIISCkTkAvifAC4CcAbUvM1nlNsqQRCE6rCghAKAcwHcz8wPMnMTwJcAXFJymwRBECrDQhMKxwF41Pr8mN4WQESXE9FWIto6PT090sYJgiAsdRaaUOgJM1/JzFPMPLV+/fqymyMIgrCkWGjJazsBbLI+H6+3JXLrrbfuJaKHC1xnHYC9BY6rAnJv0pF7k47cm+4stPtzYtqOBVXmgohqAO4FcD6UMPglgNcx8x0Dvs7WtBTvqiP3Jh25N+nIvenOYro/C0pTYOY2Eb0dwL8AcAFcNWiBIAiCIKSzoIQCADDzdwF8t+x2CIIgVJFF52geEFeW3YAFjNybdOTepCP3pjuL5v4sKJ+CIAiCUC5V1RQEQRCEBEQoCIIgCAGVEwpScC8KEe0gotuIaBsRbdXb1hLR9UR0n15WYo5IIrqKiPYQ0e3WtsR7QYpP6edoOxE9s7yWD5+Ue3MFEe3Uz842IrrY2vcBfW/uIaKXldPq0UBEm4jox0R0JxHdQUTv0tsX5bNTKaEgBfdSeTEzn23FUb8fwA3MvAXADfpzFbgawIWxbWn34iIAW/Tf5QA+M6I2lsXV6Lw3APBJ/eycrSMHod+p1wJ4qj7mf+l3b6nSBvAeZj4DwHMAvE3fg0X57FRKKEAK7mXlEgDX6PVrALyyxLaMDGa+EcC+2Oa0e3EJgC+w4mYAq4lo42haOnpS7k0alwD4EjPPM/NDAO6HeveWJMy8i5l/pdcPAbgLqmbbonx2qiYUehbcqyAM4AdEdCsRXa63bWDmXXr9cQAbymnagiDtXsizpHi7NoFcZZkZK3tviGgzgGcA+Dcs0menakJB6OQFzPxMKJX2bUT0Insnq5hliVuG3IsEPgPgFABnA9gF4OPlNqdciGg5gOsAvJuZD9r7FtOzUzWhkKvgXhVg5p16uQfAN6DU/N1GndXLPeW1sHTS7kXlnyVm3s3MHjP7AD6L0ERUuXtDRHUogfBFZv663rwon52qCYVfAthCRCcRUQPKGfbtkttUGkS0jIhWmHUAFwC4HeqeXKa/dhmAb5XTwgVB2r34NoA36EiS5wA4YJkKKkHMDv4qqGcHUPfmtUQ0RkQnQTlUbxl1+0YFERGAzwO4i5k/Ye1anM8OM1fqD8DFUJVYHwDwobLbU/K9OBnAb/TfHeZ+ADgKKlriPgA/BLC27LaO6H5cC2UGaUHZed+cdi8AEFQk2wMAbgMwVXb7S7g3/6h/+3aojm6j9f0P6XtzD4CLym7/kO/NC6BMQ9sBbNN/Fy/WZ0fKXAiCIAgBVTMfCYIgCF0QoSAIgiAEiFAQBEEQAkQoCIIgCAEiFARBEIQAEQpCJSEiz6ruua1XxVwiegsRvWEA191BROsKHPcyIvqIrrz5vX7bIQhpLLg5mgVhRMwy89lZv8zMfzfMxmTghQB+rJc3ldwWYQkjmoIgWOiR/Ef1HBO3ENGpevsVRPRevf5OXTt/OxF9SW9bS0Tf1NtuJqKz9PajiOgHus7+56ASl8y1/j99jW1E9PdJ5aWJ6DVEtA3AOwH8LVQ5iTcRUWUz8YXhIkJBqCoTMfPRa6x9B5j5TACfhuqI47wfwDOY+SwAb9HbPgLg13rbBwF8QW//zwBuYuanQtWWOgEAiOj/AfAaAM/XGosH4I/jF2LmL0NV3bxdt+k2fe1X9PPjBSENMR8JVaWb+ehaa/nJhP3bAXyRiL4J4Jt62wsA/CEAMPOPtIawEsCLAPyB3v4dInpSf/98AM8C8EtVOgcTSC88eBqAB/X6MlY1+wVhKIhQEIROOGXd8HtQnf3LAXyIiM4scA0CcA0zf6Drl9QUqesA1IjoTgAbtTnpHcz80wLXFYSuiPlIEDp5jbX8hb2DiBwAm5j5xwD+HMAqAMsB/BTa/ENE5wHYy6qm/o0AXqe3XwTATERzA4A/IqKj9b61RHRivCGspkj9DtRsXR+FKlp4tggEYViIpiBUlQk94jZ8n5lNWOoaItoOYB7ApbHjXAD/RESroEb7n2Lm/UR0BYCr9HFHEJZM/giAa4noDgA/B/AIADDznUT0YahZ7xyo6qNvA/BwQlufCeVofiuATyTsF4SBIVVSBcGCiHZAlTLeW3ZbBKEMxHwkCIIgBIimIAiCIASIpiAIgiAEiFAQBEEQAkQoCIIgCAEiFARBEIQAEQqCIAhCwP8FPDbSz7rekzwAAAAASUVORK5CYII=\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "nqZoAT8x3wyn",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 98
        },
        "outputId": "70cc453e-e578-467d-999a-dbf04511f3d4"
      },
      "source": [
        "print (\"Policy weights θ after train:\\n\", policy.θ)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Policy weights θ after train:\n",
            " [[0.83126272 0.83426041]\n",
            " [0.83710884 0.86015151]\n",
            " [0.84691878 0.89171965]\n",
            " [0.80911446 0.87010399]]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "SsanK3ds37qN",
        "colab_type": "text"
      },
      "source": [
        "Watch a trained Agent!"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "gN9sT4JD35XS",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 263
        },
        "outputId": "0cb2f409-a221-4d01-c9a3-24bf5804119a"
      },
      "source": [
        "watch_agent()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Reward: 200.0\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAVQAAADnCAYAAABBu67aAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAFYklEQVR4nO3cPY+UVRjH4Xt2Z3mHxMWXSUwETSDRxhIbYwh0FLaG1nIbLPgUhMTWL6AthQ0xNFgaTWzWFVGDq8lGsiFuXFaBeY4FjeBmZxb+zkzgusrzPJNzF5NfZp45mV5rrQB4enPTHgDgWSGoACGCChAiqAAhggoQ0h9x3REAgP/qbbfoEypAiKAChAgqQIigAoQIKkCIoAKECCpAiKAChAgqQIigAoQIKkCIoAKECCpAiKAChAgqQIigAoQIKkCIoAKECCpAiKAChAgqQIigAoQIKkCIoAKECCpAiKAChAgqQIigAoQIKkCIoAKECCpAiKAChAgqQIigAoQIKkCIoAKECCpAiKAChAgqQIigAoQIKkCIoAKECCpAiKAChAgqQIigAoQIKkCIoAKECCpAiKAChAgqQIigAoQIKkCIoAKECCpAiKAChAgqQIigAoQIKkCIoAKECCpAiKAChAgqQIigAoQIKkCIoAKECCpAiKAChAgqQIigAoQIKkCIoAKECCpAiKAChAgqQIigAoQIKkCIoAKECCpAiKAChAgqQIigAoQIKkCIoAKECCpAiKAChAgqQIigAoQIKkCIoAKECCpAiKAChPSnPQD8W+u62vz9p+qGD2p+z/46+NKxaY8EYxNUZko3vF8/fvFJPdjaeBjUl1+vqqrFE6fq6Il3pjwd7ExQmVnDe1u18etyVVUdfOWNKU8Do3mGChAiqAAhggoQIqgAIYIKECKoACGCChAiqAAhggoQIqgAIYIKECKoACGCChAiqAAhggoQIqgAIYIKECKoACGCChAiqAAhggoQIqgAIYIKECKoACGCChAiqAAhggoQIqgAIYIKECKoACGCChAiqAAhggoQIqgAIYIKECKoACGCChAiqAAhggoQIqgAIYIKECKoACGCChAiqAAhggoQIqgAIYIKECKoACGCChAiqAAhggoQIqgAIYIKECKoACGCChAiqAAhggoQIqgAIYIKENJrre10fceLMI6u6+rChQu1uro68t6F+V4tvbdYh/bOP7J+/YfNun5zc6z9lpaW6uzZs080K4ypt91if9JT8Hy6du1aLS8vj7xv355+fXjqg9qz8EK19vALVH/uXq2srNSVz78ea69z58491azwpASVmfPH/Rfr29vv1722r6qqBvt+rgftmylPBaN5hspMaa1Xv9x9s7a6wzVsCzVsC/Xb1om6dfetaY8GIwkqM6Wr+Vr769hjq70atoWpzAO7IajMlLnesF47sPLIWq+GtXduvB+kYJo8Q2W2tK7qz69qY3i0evOHarB4uF7df7NWD3w/7clgJEFlpvx9f1gfffxptfqsFg/vr3ffPla9avXdrdvTHg1G2jGoly5dmtQcPMNaa7W+vj72/V1rVdVqfWOzrnw5+qjV465evVp37tzZ9etgXBcvXtx2fceD/Wtraw7289S6rqvTp0/XjRs3JrLf5cuX6/z58xPZi+fTYDDY/cH+wWDw/0zDc6Xruur3J/d06ciRI967TIVf+QFCBBUgRFABQgQVIERQAUIc7Gcizpw5UydPnpzIXsePH5/IPvA4fzANsHvbnkP1lR8gRFABQgQVIERQAUIEFSBEUAFCBBUgRFABQgQVIERQAUIEFSBEUAFCBBUgRFABQgQVIERQAUIEFSBEUAFCBBUgRFABQgQVIERQAUIEFSBEUAFCBBUgRFABQgQVIERQAUIEFSBEUAFCBBUgRFABQgQVIERQAUIEFSBEUAFCBBUgRFABQgQVIERQAUIEFSBEUAFCBBUgRFABQgQVIKQ/4npvIlMAPAN8QgUIEVSAEEEFCBFUgBBBBQgRVICQfwD5tKPNuPGyRgAAAABJRU5ErkJggg==\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "pa_GPs4FDyRg",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 263
        },
        "outputId": "fd302dd4-d2e4-4d04-a9ee-7a1f54bd94f4"
      },
      "source": [
        "watch_agent()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Reward: 200.0\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAVQAAADnCAYAAABBu67aAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAF5klEQVR4nO3cz4uUBRzH8e/M7KrtWvkLXSGLkjokHeqyJQUGgXbv7l26Cv4P0rFr0n8gWAcPHSzorBch7KAgrWAb28ruOLs7+3RKWJT9kZ9nht1er9t8nxme72F488zMw3SapikAXl533AsA7BWCChAiqAAhggoQIqgAIRNbHHcLAMDzOi8aukIFCBFUgBBBBQgRVIAQQQUIEVSAEEEFCBFUgBBBBQgRVIAQQQUIEVSAEEEFCBFUgBBBBQgRVIAQQQUIEVSAEEEFCBFUgBBBBQgRVIAQQQUIEVSAEEEFCBFUgBBBBQgRVIAQQQUIEVSAEEEFCBFUgBBBBQgRVIAQQQUIEVSAEEEFCBFUgBBBBQgRVIAQQQUIEVSAEEEFCBFUgBBBBQgRVIAQQQUIEVSAEEEFCBFUgBBBBQgRVIAQQQUIEVSAEEEFCBFUgBBBBQgRVIAQQQUIEVSAEEEFCBFUgBBBBQgRVIAQQQUIEVSAEEEFCBFUgBBBBQgRVIAQQQUIEVSAEEEFCBFUgBBBBQgRVIAQQQUIEVSAEEEFCBFUgBBBBQgRVIAQQQUIEVSAEEEFCBFUgBBBBQgRVIAQQQUIEVSAEEEFCBFUgJCJcS/A3tc0TS0/vl/V6dbUsTer0+mMeyVohaAyAk3dv/V9rS4t1PTxt59NXzt1pk588MUY94IsQWVkhiv9Wnx499njyenDY9wG8nyHChAiqAAhggoQIqgAIYIKECKoACGCChAiqAAhggoQIqgAIYIKECKoACGCSuuWHz+o1eXFDbNOt1evnzozpo2gHYJK6/p//VHDwdKGWafbq+kTp8e0EbRDUAFCBBUgRFABQgQVIERQAUIEFSBEUAFCBBUgRFABQgQVIERQAUIEFSBEUAFCBBUgRFBpVdOs1+rywnPziVdeq063N4aNoD2CSqua4Vo9vnvrufnRd2dr4sDBMWwE7RFUxqbT6Yx7BYgSVIAQQQUIEVSAEEEFCBFUgBBBBQgRVIAQQQUIEVSAEEEFCBFUgBBBBQgRVIAQQQUIEVRaNVj8s9aHaxtmnW6v9r16bEwbQXsElVYt3L9dw8HShllv31QdfuejMW0E7RFUgBBBBQgRVIAQQQUIEVSAEEEFCBFUgBBBBQgRVIAQQQUIEVSAEEEFCOk0TbPZ8U0P8v+zsrJSly5dqvn5+W09/9PTU3XuvYMbZkuD9fr21nytDLd+e125cqVmZ2f/067Qos6LhhOj3oLdbTgc1s2bN+vhw4fbev6RLz+sT05/Vv++/ya6gxoM+nXjhxvVH6xt/uKqunjx4susCyMlqLRqYfhG/fznV7XWTFZV1alXfqtjw5989mFP8h0qrXl9en99/vG5GqxP1bCZrGEzWQ+W369vfuxXf2Xrq1PYbQSV1qzXvurXmxtmTXXryVOXp+xNgkprJrqrdXT/3IZZt9Zqf7c/po2gXb5DpTXD9aYGf9+phf6xmtw3VccPHay3pu/WnQP3x70atEJQac3i0qC+vvpdNXWtZo4crNn336hfq6nbv89t/WLYhTYN6tWrV0e1B7vE6upqPXnyZNvPX2+aqmpqbn6xrv9yd8fnu379et27d2/Hr4M2Xb58+YXzTW/sf/TokV8P2ODp06d19uzZmpsbzVXmtWvX6sKFCyM5F2zXzMzMzm/sn5mZaWcbdq1+v1+9Xm9k5zt06JD3IbuGX/kBQgQVIERQAUIEFSBEUAFC3NjPjvR6vTp//vy2/w/1ZZ08eXIk54EEfzANsHMvvA/VR36AEEEFCBFUgBBBBQgRVIAQQQUIEVSAEEEFCBFUgBBBBQgRVIAQQQUIEVSAEEEFCBFUgBBBBQgRVIAQQQUIEVSAEEEFCBFUgBBBBQgRVIAQQQUIEVSAEEEFCBFUgBBBBQgRVIAQQQUIEVSAEEEFCBFUgBBBBQgRVIAQQQUIEVSAEEEFCBFUgBBBBQgRVIAQQQUIEVSAEEEFCBFUgJCJLY53RrIFwB7gChUgRFABQgQVIERQAUIEFSBEUAFC/gFUKM+WzXGaWwAAAABJRU5ErkJggg==\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        }
      ]
    }
  ]
}