From 6229399f06ad5da6079df7e04db11cad4fd596d9 Mon Sep 17 00:00:00 2001 From: Haiyang Chang Date: Mon, 17 Oct 2016 12:54:11 +0800 Subject: [PATCH 1/5] Update HW2_Policy_Graident.ipynb --- HW2_Policy_Graident.ipynb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/HW2_Policy_Graident.ipynb b/HW2_Policy_Graident.ipynb index 3b51aa3..325a081 100644 --- a/HW2_Policy_Graident.ipynb +++ b/HW2_Policy_Graident.ipynb @@ -210,7 +210,7 @@ " Sample solution should be only 1 line.\n", " \"\"\"\n", " # YOUR CODE HERE >>>>>>\n", - " # a = ???\n", + " a = r - b\n", " # <<<<<<<<\n", "\n", " p[\"returns\"] = r\n", From 15869b30a99e40ebd932adcbd309fe090e678373 Mon Sep 17 00:00:00 2001 From: Haiyang Chang Date: Mon, 17 Oct 2016 12:57:03 +0800 Subject: [PATCH 2/5] Update policy.py --- policy_gradient/policy.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/policy_gradient/policy.py b/policy_gradient/policy.py index 70b8cdc..20f6efa 100644 --- a/policy_gradient/policy.py +++ b/policy_gradient/policy.py @@ -27,6 +27,12 @@ def __init__(self, in_dim, out_dim, hidden_dim, optimizer, session): Sample solution is about 2~4 lines. """ # YOUR CODE HERE >>>>>> + W1 = tf.Variable(tf.zeros([in_dim, hidden_dim])) + W2 = tf.Variable(tf.zeros([hidden_dim, out_dim])) + b1 = tf.Variable(tf.zeros([hidden_dim])) + b2 = tf.Variable(tf.zeros([out_dim])) + hidden1 = tf.nn.tanh(tf.matmul(self._observations, W1) + b1) + probs = tf.nn.softmax(tf.matmul(hidden1, W2) + b2) # probs = ??? # <<<<<<<< @@ -69,6 +75,7 @@ def __init__(self, in_dim, out_dim, hidden_dim, optimizer, session): Sample solution is about 1~3 lines. """ # YOUR CODE HERE >>>>>> + surr_loss = tf.reduce_mean(-tf.mul(log_prob, self._advantages)) # surr_loss = ??? # <<<<<<<< From 2db8045c21cae647d18be8eec9a7cda139b12d1c Mon Sep 17 00:00:00 2001 From: Haiyang Chang Date: Mon, 17 Oct 2016 12:58:41 +0800 Subject: [PATCH 3/5] Update util.py --- policy_gradient/util.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/policy_gradient/util.py b/policy_gradient/util.py index 4c57674..6d5a08e 100644 --- a/policy_gradient/util.py +++ b/policy_gradient/util.py @@ -21,5 +21,12 @@ def flatten_space(space): # def discount_cumsum(x, discount_rate): # YOUR CODE HERE >>>>>> +def discount_cumsum(x, discount_rate): + len_x = len(x) + discount_x = np.zeros(len_x) + for i in range(len_x): + discount_x[i] = x[i] * discount_rate**i + discount_x =np.cumsum(discount_x[::-1]) + return discount_x # return ??? - # <<<<<<<< \ No newline at end of file + # <<<<<<<< From fc51dc6eaa1c2367cb603ef653003943fc64e9a9 Mon Sep 17 00:00:00 2001 From: Haiyang Chang Date: Mon, 17 Oct 2016 13:12:43 +0800 Subject: [PATCH 4/5] Create Report --- Report | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 Report diff --git a/Report b/Report new file mode 100644 index 0000000..0770ee6 --- /dev/null +++ b/Report @@ -0,0 +1,10 @@ +pro5: +I failed to handle the spicy problem although I used the following code: +sudo apt-get install python-scipy +pip install scipy +import scipy + +pro6: +I guess when run the whole process, 'advantages' may be calculated to some format inconsistent with the codes using it below, +so it may produce bugs if we do not normalize it first. +While '+ 1e-8' is to avoid the denominator turning to zero. From b7778861a9c8482dc3e8faf96dbab4a1271e18aa Mon Sep 17 00:00:00 2001 From: root Date: Mon, 12 Dec 2016 04:52:42 +0800 Subject: [PATCH 5/5] On branch master Your branch is up-to-date with 'origin/master'. Changes to be committed: modified: HW2_Policy_Graident.ipynb deleted: Report new file: Untitled.ipynb new file: Untitled1.ipynb modified: policy_gradient/policy.py modified: policy_gradient/util.py new file: report2.m --- HW2_Policy_Graident.ipynb | 167 +++++++++++++++++++++++++++++++++++--- Report | 10 --- Untitled.ipynb | 49 +++++++++++ Untitled1.ipynb | 49 +++++++++++ policy_gradient/policy.py | 7 +- policy_gradient/util.py | 8 +- report2.md | 49 +++++++++++ 7 files changed, 309 insertions(+), 30 deletions(-) delete mode 100644 Report create mode 100644 Untitled.ipynb create mode 100644 Untitled1.ipynb create mode 100644 report2.md diff --git a/HW2_Policy_Graident.ipynb b/HW2_Policy_Graident.ipynb index 325a081..c40078b 100644 --- a/HW2_Policy_Graident.ipynb +++ b/HW2_Policy_Graident.ipynb @@ -2,9 +2,10 @@ "cells": [ { "cell_type": "code", - "execution_count": null, + "execution_count": 1, "metadata": { - "collapsed": false + "collapsed": false, + "scrolled": true }, "outputs": [], "source": [ @@ -26,11 +27,19 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 2, "metadata": { "collapsed": false }, - "outputs": [], + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[2016-12-12 03:05:41,584] Making new env: CartPole-v0\n" + ] + } + ], "source": [ "import gym\n", "import tensorflow as tf\n", @@ -38,6 +47,8 @@ "from policy_gradient import util\n", "from policy_gradient.policy import CategoricalPolicy\n", "from policy_gradient.baselines.linear_feature_baseline import LinearFeatureBaseline\n", + "%matplotlib inline\n", + "import matplotlib.pyplot as plt\n", "\n", "np.random.seed(0)\n", "tf.set_random_seed(0)\n", @@ -95,11 +106,20 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 3, "metadata": { "collapsed": false }, - "outputs": [], + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/haiyang/Downloads/ENTER/lib/python2.7/site-packages/tensorflow/python/ops/gradients.py:90: UserWarning: Converting sparse IndexedSlices to a dense Tensor of unknown shape. This may consume a large amount of memory.\n", + " \"Converting sparse IndexedSlices to a dense Tensor of unknown shape. \"\n" + ] + } + ], "source": [ "sess = tf.Session()\n", "\n", @@ -151,9 +171,9 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 4, "metadata": { - "collapsed": true + "collapsed": false }, "outputs": [], "source": [ @@ -210,7 +230,7 @@ " Sample solution should be only 1 line.\n", " \"\"\"\n", " # YOUR CODE HERE >>>>>>\n", - " a = r - b\n", + " a = r - b\n", " # <<<<<<<<\n", "\n", " p[\"returns\"] = r\n", @@ -230,6 +250,8 @@ " )\n", "\n", " def train(self):\n", + " array_i=[]\n", + " array_avg_return=[]\n", " for i in range(1, self.n_iter + 1):\n", " paths = []\n", " for _ in range(self.n_episode):\n", @@ -238,10 +260,16 @@ " loss = self.policy.train(data[\"observations\"], data[\"actions\"], data[\"advantages\"])\n", " avg_return = np.mean([sum(p[\"rewards\"]) for p in paths])\n", " print(\"Iteration {}: Average Return = {}\".format(i, avg_return))\n", + " array_i.append(i)\n", + " array_avg_return.append(avg_return)\n", " \n", " # CartPole-v0 defines \"solving\" as getting average reward of 195.0 over 100 consecutive trials.\n", " if avg_return >= 195:\n", " print(\"Solve at {} iterations, which equals {} episodes.\".format(i, i*100))\n", + " plt.plot(array_i,array_avg_return,lw=2.0)\n", + " plt.xlabel('Iteration')\n", + " plt.ylabel=('Average Return')\n", + " plt.show()\n", " break\n", "\n", " if self.baseline != None:\n", @@ -250,17 +278,102 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 5, "metadata": { "collapsed": false }, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Iteration 1: Average Return = 21.82\n", + "Iteration 2: Average Return = 23.49\n", + "Iteration 3: Average Return = 23.39\n", + "Iteration 4: Average Return = 24.2\n", + "Iteration 5: Average Return = 28.48\n", + "Iteration 6: Average Return = 29.85\n", + "Iteration 7: Average Return = 32.93\n", + "Iteration 8: Average Return = 36.12\n", + "Iteration 9: Average Return = 34.61\n", + "Iteration 10: Average Return = 35.77\n", + "Iteration 11: Average Return = 38.6\n", + "Iteration 12: Average Return = 37.29\n", + "Iteration 13: Average Return = 41.57\n", + "Iteration 14: Average Return = 43.69\n", + "Iteration 15: Average Return = 43.66\n", + "Iteration 16: Average Return = 41.81\n", + "Iteration 17: Average Return = 46.41\n", + "Iteration 18: Average Return = 43.77\n", + "Iteration 19: Average Return = 43.26\n", + "Iteration 20: Average Return = 51.49\n", + "Iteration 21: Average Return = 50.17\n", + "Iteration 22: Average Return = 50.11\n", + "Iteration 23: Average Return = 50.56\n", + "Iteration 24: Average Return = 49.37\n", + "Iteration 25: Average Return = 55.61\n", + "Iteration 26: Average Return = 53.83\n", + "Iteration 27: Average Return = 56.62\n", + "Iteration 28: Average Return = 57.75\n", + "Iteration 29: Average Return = 59.34\n", + "Iteration 30: Average Return = 50.25\n", + "Iteration 31: Average Return = 58.7\n", + "Iteration 32: Average Return = 56.58\n", + "Iteration 33: Average Return = 56.68\n", + "Iteration 34: Average Return = 66.28\n", + "Iteration 35: Average Return = 61.66\n", + "Iteration 36: Average Return = 62.78\n", + "Iteration 37: Average Return = 66.56\n", + "Iteration 38: Average Return = 68.2\n", + "Iteration 39: Average Return = 72.12\n", + "Iteration 40: Average Return = 73.24\n", + "Iteration 41: Average Return = 77.72\n", + "Iteration 42: Average Return = 76.97\n", + "Iteration 43: Average Return = 79.66\n", + "Iteration 44: Average Return = 84.27\n", + "Iteration 45: Average Return = 80.64\n", + "Iteration 46: Average Return = 91.35\n", + "Iteration 47: Average Return = 98.51\n", + "Iteration 48: Average Return = 111.76\n", + "Iteration 49: Average Return = 106.64\n", + "Iteration 50: Average Return = 115.9\n", + "Iteration 51: Average Return = 129.98\n", + "Iteration 52: Average Return = 139.38\n", + "Iteration 53: Average Return = 145.66\n", + "Iteration 54: Average Return = 147.36\n", + "Iteration 55: Average Return = 149.93\n", + "Iteration 56: Average Return = 153.01\n", + "Iteration 57: Average Return = 162.6\n", + "Iteration 58: Average Return = 166.12\n", + "Iteration 59: Average Return = 171.98\n", + "Iteration 60: Average Return = 174.87\n", + "Iteration 61: Average Return = 179.87\n", + "Iteration 62: Average Return = 186.71\n", + "Iteration 63: Average Return = 192.05\n", + "Iteration 64: Average Return = 188.59\n", + "Iteration 65: Average Return = 192.96\n", + "Iteration 66: Average Return = 197.28\n", + "Solve at 66 iterations, which equals 6600 episodes.\n" + ] + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAg4AAAF5CAYAAAD3dKLdAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAAPYQAAD2EBqD+naQAAIABJREFUeJzt3Xmc1vP+//HHq21STmMvSxHZOalxkCWHkFA4HTQhpxAq\nnDkk2zk6cuy0yb5mGSeRrGUpIX5xGnSog4i0h5o27e/fH69rvtc100zNcl1zbc/77Xbd5v15f97z\nuV7Xp2Ve8/68FwshICIiIlIZdZIdgIiIiKQPJQ4iIiJSaUocREREpNKUOIiIiEilKXEQERGRSlPi\nICIiIpWmxEFEREQqTYmDiIiIVJoSBxEREak0JQ4iIiJSaVVKHMzsOjP7xMyWmdlCMxtjZvuU0+5m\nM5tnZqvM7G0za1XmfI6ZjTCzn81suZmNNrOdavphREREJLGq2uNwDDAcOBw4AagPvGVmW5U0MLMB\nQD+gN3AYsBIYb2YNYq4zBDgV6Aq0B3YBXqzmZxAREZFaYjXZ5MrMdgAWAe1DCB9G6uYBd4UQBkeO\nmwALgQtCCKMix4uBbiGEMZE2+wIzgCNCCJ/U5AOJiIhI4tR0jMM2QAB+BTCzlkAz4N2SBiGEZcAU\noF2k6lCgXpk2XwOzY9qIiIhICqp24mBmhj9y+DCEMD1S3QxPJBaWab4wcg6gKbA2klBU1EZERERS\nUL0afO/9wAHAUXGKpUJmtj3QEfgBWJ3o9xMREckgDYE9gPEhhF9qerFqJQ5mdh9wCnBMCGF+zKkF\ngOG9CrG9Dk2Bz2LaNDCzJmV6HZpGzpWnI/BsdWIVERERAM4FnqvpRaqcOESShtOBY0MIs2PPhRBm\nmdkCoAMwLdK+CT4LY0Sk2VRgfaRN7ODIFsDHFbztDwDPPPMM+++/f1VDzjgFBQUMHjw42WEkne6D\n032I0r1wug9RuhcwY8YMzjvvPIj8LK2pKiUOZnY/kA90AVaaWdPIqeIQQskjhCHAjWY2MxLkIGAO\nMBZ8sKSZPQbca2ZLgOXAMGDyZmZUrAbYf//9adu2bVVCzki5ubm6D+g+lNB9iNK9cLoPUal6L9au\nhZtuggEDYJttau1t4/Kov6o9Dpfigx/fK1PfExgJEEK408waAQ/hsy4+ADqFENbGtC8ANgCjgRxg\nHNC3qsGLiIikm40boVcvePZZeO01GDcOdt012VFVXpUShxBCpWZhhBAGAgM3c34NcHnkJSIikjWu\nvdaTBoCZM2H27PRKHLRXhYiISC0ZOhTuusvLderA889DuzRbwUiJQxrKz89PdggpQffB6T5E6V44\n3YeoVLoXo0ZBQUH0+P774fTTkxdPddVoyenaYmZtgalTp05NyUEuIiIimzNpEpx0kg+KBLjxRhg0\nqHbeu6ioiLy8PIC8EEJRTa+nHgcREZEE+u9/vWehJGno1Qtuvjm5MdWEEgcREZEE+ekn6NQJiov9\n+JRT4MEHwSy5cdWEEgcREZEEWLIETj4Z5s714z/8wcc51K+f3LhqSomDiIhIJfzwA6xbV7m2a9bA\nGWfA9MgWkK1a+ZoNjRsnLLxao8RBRERkC/75T2jZEg46CL77bvNtN26ECy6A99/34x139EWedtop\n8XHWBiUOIiIim/HOOzBwoJe/+QaOPBKmTq24/YAB8O9/e7lRI3j9ddhrr4SHWWuUOIiIiFRg8WI4\n//zSdYsWwR//CG+/vWn7YcPg7ru9XKeOj2n4wx8SHmatUuIgIiJSjhCgZ09YsMCPO3SAo4/28ooV\nPkOiZOlogJdegr/+NXr8wANw6qm1F29tqfK22iIiItngvvv8MQP4+IRnnoHcXOjeHV5+Gdavh/PO\n88TiiCPg3HM92QBf4Kl37+TFnkjqcRARESlj2jTo3z96/OST0KwZbLUVjB4Nl14aPXf11d4bsTqy\naXWPHum9wNOWKHEQERGJsWoVdOvmUyrBHz906hQ9X7eu7zMRu2R0SdsTT4RHHknvBZ62RImDiIhI\njL/9DWbM8HLr1nD77Zu2MfPHEY8+6olESdvRo6FBg9qLNRk0xkFERCTipZfgoYe8vNVWUFgIOTkV\nt7/wQmjTBj76yMc7NGlSO3EmkxIHERERYN48uOii6PHQobD//lv+vrZt/ZUt9KhCREQEuOEG318C\noGvX0kmERClxEBGRrFdUBE895eVttkn/HSwTSYmDiIhktRB8QGTJGgz/+AfssENyY0plShxERCSr\njR0LkyZ5uVUr6Ns3ufGkOiUOIiKStdauLb3Q0113Zf50yppS4iAiIllrxAiYOdPLxx4Lp5+e3HjS\nQZUTBzM7xsxeMbO5ZrbRzLqUOd/YzO4zs5/MbJWZfWVml5Rpk2NmI8zsZzNbbmajzSxDdioXEZF0\n8Msv0aWhzeDeezUgsjKq0+PQGPgc6AOEcs4PBk4CugP7RY7vM7PTYtoMAU4FugLtgV2AF6sRi4iI\nSLX885+wdKmXL7ggu9ZiqIkqLwAVQhgHjAMwKzc3awc8FUL4IHL8qJldChwGvGZmTYBeQLcQwqTI\ndXoCM8zssBDCJ9X4HCIiIpX2v//5fhMAjRrBv/6V3HjSSSLGOHwEdDGzXQDM7Dhgb2B85HwenrC8\nW/INIYSvgdl40iEiIpJQ/fvDhg1eHjAAdtklufGkk0QsOX058DAwx8zWAxuAi0MIkyPnmwFrQwjL\nynzfwsg5ERGRhHnnHXjtNS/vuitcdVVy40k3iUgcrgAOB07DexHaA/eb2bwQwoQEvJ+IiEil/Pab\nb5Nd4tZboXHj5MWTjuKaOJhZQ+BfwBkhhDcj1V+aWRvgamACsABoYGZNyvQ6NI2cq1BBQQG5ubml\n6vLz88nPz4/XRxARkQwVAvTpA1995cd5eb6jZSYpLCyksLCwVF1xcXFc3yPePQ71I68NZeo3EB1P\nMRVYD3QAxgCY2b5AC+DjzV188ODBtNWwVxERqYbHHoMnn/Ryo0a+N0WdDFvNqLxfpouKisjLy4vb\ne1Q5cTCzxkAroGRGxZ5m1hr4NYTwk5lNAu42s8uBH4E/Aj2AvwKEEJaZ2WPAvWa2BFgODAMma0aF\niIgkQlER9OsXPX7kETjwwOTFk86q0+NwKDARX8MhAPdE6p/Cp1meA9wGPANshycP14UQHo65RgHe\nCzEayMGnd2p1cBERibslS3yb7DVr/LhvX+jePbkxpbPqrOMwic1M4wwhLAIu3MI11uCzLy6v6vuL\niIhU1saN0KMH/PCDHx92GNxzz2a/RbYgw57uiIiIRN1xR3Tq5fbbwwsvQE5OcmNKd0ocREQkI737\nLtx4o5fN4LnnoEWL5MaUCZQ4iIhIxpk9G/Lz/VEFwMCBcNJJSQ0pYyhxEBGRjDJ+vK/RsHixH598\ncrTnQWpOiYOIiGSE9evh+us9Ufj5Z6/bay945pnMW68hmXQrRUQk7c2dC8cdB7fdFq077TSYMsUH\nRUr8KHEQEZG0Nm4cHHIIfPihH9erB3ffDa+8oqQhERKxyZWIiEjChQD/+Afccku0rnlz+Pe/oV27\n5MWV6dTjICIiaenxx0snDZ07w+efK2lINCUOIiKSdubMgb/9LXp8550wdixst13yYsoWelQhIiJp\nJQS45BJYtsyPL7gA+vdPbkzZRD0OIiKSVp55Bt54w8vNmsHgwcmNJ9socRARkbSxYAFceWX0+MEH\nYdttkxdPNlLiICIiaSEE6NPHt8kGX1L69NOTG1M2UuIgIiJp4YUXYMwYL++4Iwwbltx4spUSBxER\nSXmLF0O/ftHjESNghx2SF082U+IgIiIp74oroptWde0KZ52V3HiymRIHERFJaS+/DM8/7+XttvPe\nBkkeJQ4iIpKy3nkHzj8/ejxsGDRtmrx4RImDiIikqFGj4JRTYMUKPz7jDOjePbkxiRIHERFJQSNG\nQLdusG6dH59xBhQWglly4xIlDiIikkJCgJtu8hkUIXjdhRf6VMyGDZMbmzglDiIikhI2bPAFnm6+\nOVp3/fXwyCNQTzsrpQz9UYiISNKtWQPnnQejR0frhgwpvby0pIYq9ziY2TFm9oqZzTWzjWbWpZw2\n+5vZWDNbamYrzGyKme0Wcz7HzEaY2c9mttzMRpvZTjX9MCIikp7uvDOaNNSrB88+q6QhVVXnUUVj\n4HOgDxDKnjSzvYAPgOlAe+BgYBCwOqbZEOBUoGukzS7Ai9WIRURE0ty6dfDAA16uWxdefVWzJ1JZ\nlR9VhBDGAeMAzMod33oL8HoI4bqYulklBTNrAvQCuoUQJkXqegIzzOywEMInVY1JRETS12uvwfz5\nXu7SBU4+ObnxyObFdXBkJJE4FfjWzMaZ2UIz+39mFrt/WR6esLxbUhFC+BqYDbSLZzwiIpL6Hnoo\nWr7kkuTFIZUT71kVOwFbAwOAN4ATgTHAS2Z2TKRNM2BtCGFZme9dGDknIiJZYtYseOstL7dsCSee\nmNx4ZMviPauiJBF5OYRQsuHpNDM7ErgUH/tQbQUFBeTm5paqy8/PJz8/vyaXFRGRJHnkkeh6DRdf\nDHW0SECNFBYWUlhYWKquuLg4ru8R78ThZ2A9MKNM/QzgqEh5AdDAzJqU6XVoGjlXocGDB9O2bdt4\nxSoiIkm0bh08/riX69WDnj2TG08mKO+X6aKiIvLy8uL2HnHN7UII64BPgX3LnNoH+DFSnoonFx1K\nTprZvkAL4ON4xiMiIqlr7FhYuNDLZ5wBzfSwOi1UucfBzBoDrYCSGRV7mllr4NcQwk/AXcDzZvYB\nMBHoBJwGHAsQQlhmZo8B95rZEmA5MAyYrBkVIiLZQ4Mi01N1HlUciicEIfK6J1L/FNArhPCymV0K\nXA8MBb4G/hRCiO1NKAA2AKOBHHx6Z99qfQIREUk7M2f6ltkAe+0Fxx+f3Hik8qqzjsMktvCII4Tw\nJPDkZs6vAS6PvEREJMs88ki03Lu3BkWmE/1RiYhIrVq7Fp54wsv168Nf/pLUcKSKlDiIiEitGjMG\nFi/28plnwk7aqSitKHEQEZFapUGR6U2Jg4iI1JpvvoGJE728995w3HHJjUeqTomDiIjUmocfjpZ7\n94Zyt0qUlKbEQUREasXcufDkk15u0ECDItOVEgcREUm4Dz6AvDz45Rc/7toVdtghuTFJ9ShxEBGR\nhAkB7r/fF3gqWV56993h1luTG5dUnxIHERFJiNWr4cILoW9fWL/e6zp0gP/8B/bYI6mhSQ0ocRAR\nkbibMwfat48u9ARw9dUwbpweUaS7eG+rLSIiWW7CBMjPh0WL/HirreCxx7xO0p96HEREJC5WrPDH\nEh06RJOGPfaAjz5S0pBJ1OMgIiI1NnEi9OoFP/wQrTvxRCgshO23T1pYkgDqcRARkWpbsQL69PFZ\nEyVJQ6NGMGyYj2dQ0pB51OMgIiLVMmGCz5qI7WVo3x4efxz22itpYUmCqcdBREQqbcMGGDsW/vhH\nH8sQ28swfLg/slDSkNnU4yAiIlu0YoVPrRw6FL77rvS5Y4/1WRNKGLKDEgcREanQ/Plw773wyCNQ\nXFz63L77Qv/+0LMn1FH/ddZQ4iAiIuVatMj3l5g/v3T9CSdAQQGcfLIShmykxEFERMp19dXRpCEn\nB849F/76Vzj44OTGJcmlxEFERDYxcSI8/bSXt90WvvgCmjdPbkySGtTJJCIipaxd62szlLj9diUN\nEqXEQURESrn7bvjf/7x8xBFw0UXJjUdSS5UTBzM7xsxeMbO5ZrbRzLpspu2DkTZXlKnPMbMRZvaz\nmS03s9FmtlN1PoCIiMTP99/DoEFerlsXHnxQAyCltOr8dWgMfA70AUJFjczsTOBwYG45p4cApwJd\ngfbALsCL1YhFRETiJAS4/HJYvdqPr7gCWrdObkySeqo8ODKEMA4YB2BmVl4bM9sVGAp0BN4oc64J\n0AvoFkKYFKnrCcwws8NCCJ9UNSYREam5MWPgjcj/2LvuCv/8Z3LjkdQU9w6oSDIxErgzhDCjnCZ5\neMLybklFCOFrYDbQLt7xiIjIlq1YAVdeGT0eMgR+97vkxSOpKxFPrq4F1oYQ7qvgfLPI+WVl6hdG\nzomISC0bOBDmzPFyp07QtWtSw5EUFtd1HMwsD7gCaBPP64qISOJMm+Y9DAANG8J990H5D6JF4r8A\n1NHAjsBPMcMf6gL3mtlfQwh7AguABmbWpEyvQ9PIuQoVFBSQm5tbqi4/P5/8/Px4xS8iknX69/dd\nLwFuuAH23DO58Uj1FRYWUlhYWKquuOwmIzVkIVQ4MWLL32y2ETgjhPBK5HhbYOcyzd7Cxzw8EUL4\nNjI4cjE+OHJM5Pv2BWYAR5Q3ONLM2gJTp06dStu2basdr4iIlPbtt7DPPl5u2RJmzPDlpSVzFBUV\nkZeXB5AXQiiq6fWq3ONgZo2BVkBJl8KeZtYa+DWE8BOwpEz7dcCCEMK3ACGEZWb2GN4LsQRYDgwD\nJmtGhYhI7XrwwWi5Tx8lDbJl1XlUcSgwEV/DIQD3ROqfwqdZllVel0YBsAEYDeTg0zv7ViMWERGp\npt9+gyee8HJOjm+PLbIl1VnHYRJVmI0RGddQtm4NcHnkJSIiSTBqFCyJ9BGffTZsv31y45H0oIVE\nRUSy1AMPRMuXXZa8OCS9KHEQEclCRUUwZYqXW7f2zaxEKkOJg4hIFirb26B1G6SylDiIiGSZ4mJ4\n7jkv/+53cO65yY1H0osSBxGRLDNyJKxa5eXzz4ett05uPJJelDiIiGSREDQoUmpGiYOISBZ5/31f\nHRLg6KPhoIOSG4+kHyUOIiJZRL0NUlNKHEREssTChfDSS17ecUdtnS3Vo8RBRCRLPPYYrFvn5V69\ntC+FVI8SBxGRLLBhAzz0kJfN4JJLkhuPpC8lDiIiWWD8eJg928snn+xbaItUhxIHEZEs8Oqr0fLF\nFycvDkl/ShxERLLAW2/51/r14cQTkxuLpDclDiIiGe677+D777181FFaKVJqRomDiEiGK+ltADjp\npOTFIZlBiYOISIZT4iDxpMRBRCSDrVsH777r5e23hzZtkhuPpD8lDiIiGWzKFFi+3Msnngh19L++\n1JD+ComIZLDYxxQdOyYvDskcShxERDJYbOKgaZgSD0ocREQy1K+/wqefevnAA2HXXZMbj2QGJQ4i\nIhlqwgTYuNHLmk0h8aLEQUQkQ2kapiRClRMHMzvGzF4xs7lmttHMusScq2dmd5jZNDNbEWnzlJnt\nXOYaOWY2wsx+NrPlZjbazHaKxwcSEREIIZo4NGgA7dsnNx7JHNXpcWgMfA70AUKZc42AQ4B/Am2A\nM4F9gbFl2g0BTgW6Au2BXYAXqxGLiIiU49tv4ccfvXzMMdCoUXLjkcxRr6rfEEIYB4wDMDMrc24Z\nUGrCj5n1A6aY2W4hhDlm1gToBXQLIUyKtOkJzDCzw0IIn1Tvo4iISInx46NlTcOUeKqNMQ7b4D0T\nSyPHeXjC8m5JgxDC18BsoF0txCMikvE0vkESJaGJg5nlALcDz4UQVkSqmwFrI70TsRZGzomISA2s\nXQsTJ3q5aVM4+ODkxiOZpcqPKirLzOoBL+C9DX3icc2CggJyc3NL1eXn55Ofnx+Py4uIZISPP4aV\nK72sZaazS2FhIYWFhaXqiouL4/oeCUkcYpKG5sDxMb0NAAuABmbWpEyvQ9PIuQoNHjyYtm3bxj1e\nEZFMoscU2au8X6aLiorIy8uL23vEPQ+NSRr2BDqEEJaUaTIVWA90iPmefYEWwMfxjkdEJNvEJg4n\nnJC8OCQzVbnHwcwaA62AkhkVe5pZa+BXYD4+rfIQ4DSgvpk1jbT7NYSwLoSwzMweA+41syXAcmAY\nMFkzKkREaubnn2HqVC///vew886bby9SVdV5VHEoMBEfuxCAeyL1T+HrN3SO1H8eqbfI8XHA+5G6\nAmADMBrIwad39q1GLCIiEuPdd33xJ9A0TEmM6qzjMInNP+LY4uOPEMIa4PLIS0RE4kTjGyTRNNZW\nRCRDxC4z3bAhHH10cuORzKTEQUQkQ0yaBHPmePnYYz15EIk3JQ4iIhlixIho+S9/SVoYkuGUOIiI\nZIB582DMGC83bQp/+lNy45HMpcRBRCQDPPwwbNjg5d69fSttkURQ4iAikubWrfPEAaBuXU8cRBJF\niYOISJp7+WWYP9/Lp58Ou+2W3HgksylxEBFJQatWQefO0KED/PTT5tvGDorsq6X0JMEStjumiIhU\n3+OPw2uvefmss+D998sft/DVVz4NE2C//eC442ovRslO6nEQEUlBb7wRLU+ZAtdeW367+++Plvv0\nAbPy24nEixIHEZEUs2oVTJxYum7wYB/LEGvZMhg50suNG0OPHrUTn2Q3JQ4iIilmwgRYvdrLzZtH\n63v2hFmzosfPPAMrVnj5vPMgN7f2YpTspcRBRCTFxD6mGDbMxzgALF0K55wDa9f6vhSxgyL79Knd\nGCV7KXEQEUkhIcDrr3u5fn044QR45BHYay+v+/RT6N/fB0ROn+51Rx8Nv/99cuKV7KPEQUQkhUyf\nDrNne/nYY2Hrrf0RxAsvQE6O1w8bVnqRJ03BlNqkxEFEJIWU9DYAnHpqtNymDQwZEj3+9lv/qn0p\npLYpcRARSSGx4xtiEweASy7xMQ6xLr5Y+1JI7VLiICKSIpYuhQ8/9HKrVrD33qXPm/meFCX19et7\nMiFSm7RypIhIinj77egOl2V7G0o0aeLt7rkHTjpJ+1JI7VPiICKSImLHN5xySsXtdt/dB0iKJIMe\nVYiIpICNG+HNN73cqJHPqBBJRUocRERSwNSpsGiRl084ITr1UiTVKHEQEUkBm5tNIZJKqpw4mNkx\nZvaKmc01s41m1qWcNjeb2TwzW2Vmb5tZqzLnc8xshJn9bGbLzWy0me1Ukw8iIpLOKju+QSTZqtPj\n0Bj4HOgDhLInzWwA0A/oDRwGrATGm1nsTOMhwKlAV6A9sAvwYjViERFJewsX+lLS4EtHa6aEpLIq\nz6oIIYwDxgGYlbvz+5XAoBDCa5E2PYCFwBnAKDNrAvQCuoUQJkXa9ARmmNlhIYRPqvVJRETS1Lhx\n0bIeU0iqi+sYBzNrCTQD3i2pCyEsA6YA7SJVh+IJS2ybr4HZMW1ERLJG7PgGPaaQVBfvwZHN8McX\nC8vUL4ycA2gKrI0kFBW1ERHJCuvWwfjxXt52WzjiiOTGI7IlmlUhIpJEH38MxcVe7tgR6mlZPklx\n8f4rugAwvFchttehKfBZTJsGZtakTK9D08i5ChUUFJCbm1uqLj8/n/z8/JrGLSKSFBXthilSHYWF\nhRQWFpaqKy7JTOPEQthkYkTlv9lsI3BGCOGVmLp5wF0hhMGR4yZ4EtEjhPBC5HgxPjhyTKTNvsAM\n4IjyBkeaWVtg6tSpU2nbtm214xURSSUhwEEHwfTpvoHVokWwww7JjkoyTVFREXl5eQB5IYSiml6v\nyj0OZtYYaIX3LADsaWatgV9DCD/hUy1vNLOZwA/AIGAOMBZ8sKSZPQbca2ZLgOXAMGCyZlSISDZ5\n7TVPGsDHNihpkHRQnUcVhwIT8UGQAbgnUv8U0CuEcKeZNQIeArYBPgA6hRDWxlyjANgAjAZy8Omd\nfav1CURE0tCGDXDttdHj/v2TF4tIVVRnHYdJbGFQZQhhIDBwM+fXAJdHXiIiWWfkyNK9DWeckdx4\nRCpLsypERGrZb7/BP/4RPb7jDh/jIJIOlDiIiNSy++6DOXO8fNpp0L59cuMRqQolDiIitWjJErjt\nNi+bwa23JjcekapS4iAiUovuuMOTB4AePeDgg5Mbj0hVKXEQEaklc+bA0KFezsmBm29Objwi1aHE\nQUSklgwcCKtXe7lfP2jRIqnhiFSLEgcRkVowfTo88YSXc3PhuuuSG49IdSlxEBGpBddfDxs3evna\na2H77ZMbj0h1KXEQEUmwyZNh7Fgv77ILXHFFcuMRqQklDiIiCXb33dHyP/8JjRolLxaRmlLiICKS\nQL/+Gt06u1kz+MtfkhqOSI0pcRARSaAXXoB167zcvTvUq87WgiIpRImDiEgCPf10tHz++cmLQyRe\nlDiIiCTI99/7wEiAAw+E1q2TG49IPChxEBFJkOeei5bPO087YEpmUOIgIpIAIcAzz0SPu3dPXiwi\n8aTEQUQkAaZOha+/9vIf/6jlpSVzKHEQEUmA2EGR552XvDhE4k2Jg4hInK1bB4WFXs7Jga5dkxuP\nSDwpcRARibN33oHFi73cuTNss01y4xGJJyUOIiJVMHMm3HYbfPddxW1iB0XqMYVkGiUOIiKV9PXX\n8Ic/+E6XeXnw0Uebtlm+HMaM8fJ220GnTrUbo0iiKXEQEamEX3/1xw5Ll/pxcTGceCK8/XbpdmPG\nwG+/efmcc6BBg9qNUyTRlDiIiGzBunXw5z/Dt9/6cf36/nXVKjjttGgPA+gxhWS+uCcOZlbHzAaZ\n2fdmtsrMZprZjeW0u9nM5kXavG1mreIdi4hITYUA/frBxIl+vNNO8NVXcOaZfrx2LZx1FowcCfPm\nwbvven3LltCuXXJiFkmkRPQ4XAtcAvQB9gOuAa4xs34lDcxsANAP6A0cBqwExpuZOvVEJKUMHw4P\nP+zlnBx4+WXYe28YNSq6adWGDXDBBb465MaNXqclpiVTJSJxaAeMDSGMCyHMDiG8BLyFJwglrgQG\nhRBeCyF8CfQAdgHOSEA8IiLVMm4cFBREjx97LNqLUK8ePPkk9O0bPT9pUrSsxxSSqRKROHwEdDCz\nvQHMrDVwFPBG5Lgl0Ax4t+QbQgjLgCl40iEiknTTp/vgxpIehBtugHPPLd2mTh3vkbj++tL1hx0G\n++xTO3GK1LZ6Cbjm7UAT4H9mtgFPTm4IITwfOd8MCMDCMt+3MHJORCSp5s3zQY/Llvnxn/4EN99c\nflsz+Ne/IDcXBgzwuksvrZ04RZIhEYnDOUB3oBswHTgEGGpm80IIT2/2O7egoKCA3NzcUnX5+fnk\n5+fX5LIiIv/nxRehd2+ffgnQpo0PfKyzhf7Za66Bo4+GRYvg9NMTH6dIeQoLCyksWe88ori4OK7v\nYSGE+F7QbDZwWwjhgZi6G4BzQwgHRB5VfAccEkKYFtPmPeCzEEJBOddsC0ydOnUqbdu2jWu8IiLg\n6zJccYX2ETY/AAAe0klEQVQnCSWaN/dFnnbbLXlxidRUUVEReXl5AHkhhKKaXi8RYxwaARvK1G0s\nea8QwixgAdCh5KSZNQEOx8dHiIjEzdq18P77vkR0Rb8nvf8+tG5dOmn485/hs8+UNIiUlYhHFa8C\nN5rZHOAroC1QADwa02ZIpM1M4AdgEDAHGJuAeEQkS61eDSefHJ3tkJvrjx7atvWvhxzi21/fdVc0\nqWjSBO67T9MpRSqSiMShH54IjAB2AuYBD0TqAAgh3GlmjYCHgG2AD4BOIYS1CYhHRLJQCNCrV+kp\nksXF8N57/ipP+/be67D77rURoUh6inviEEJYCfwt8tpcu4HAwHi/v4gIwE03QckYsUaN4Pjj4fPP\nYc6cTdvWrw+33AJXXQV169ZunCLpJhE9DiIiSfXkkzAo0sdp5glEly5+vGiRj10oKvJX48a+yFPr\n1kkLVyStKHEQkYwyYQJcfHH0ePDgaNIAvtdEx47+EpGq0+6YIpIxZsyArl1h/Xo/7tfPp1iKSPwo\ncRCRjLBoEZx6Kixd6sennuq9DZoZIRJfShxEJK2tXw8ffgidO8OsWV7Xpg08/7xvRCUi8aV/ViKS\ndhYvhjffhDfegPHjo70MALvuCq++Cltvnbz4RDKZEgcRSRsjR8KIEfDpp+WvArn99vD66548iEhi\nKHEQkbRw++1w3XWb1ufm+gyJU07xxxXbbVf7sYlkEyUOIpLy7r67dNJw8ME++PGUU6BdO41lEKlN\n+ucmIiltyBDo3z96fPvtMGBA8uIRyXaaVSEiKeu++3xVxxK33KKkQSTZlDiISEp68EG4/PLo8cCB\ncMMNSQtHRCKUOIhIynn0UbjssujxjTfCP/6RvHhEJEqJg4ikjO+/90cTvXtH6669Fm6+WStAiqQK\nDY4UkaTauBHefhuGD/cFnWLXZ7jqKrj1ViUNIqlEiYOIJMWyZfDUUz4A8ptvSp/LyfGehptuUtIg\nkmqUOIjIFoUATz4Jq1b5Y4T69Wt2vfffhzPPhF9/LV3fogX06QMXXgg77FCz9xCRxFDiICJb9NJL\n0KuXl7/4Ah56qPo9ARMm+AqPq1ZF6447zmdQdO6sxZxEUp0GR4rIFt13X7T8yCMwbFj1rvPWW77i\nY0nScOKJ8N//ejJx5plKGkTSgRIHEdmsr7+G994rXfe3v/nulFXxxhvQpQusXu3HXbr4LpYHHRSX\nMEWklihxEJHNevjhaPnAA/3rxo3QrRtMn165a7z6qvcorFnjx3/6E7zwgg+CFJH0osRBRCq0erXP\nfABo0AAmTvQf+uCzIjp3hp9/3vw1xozx71m71o/POguef96vJyLpR4mDiFTopZfgl1+8/Oc/w447\nwsiR0KaN133/PXTtGk0KYs2bB4MHe6Kwfr3Xde8Ozz1X81kZIpI8CUkczGwXM3vazH42s1Vm9oWZ\ntS3T5mYzmxc5/7aZtUpELCJSfQ89FC1fcol/bdwYxo6FZs38+P33fQrlhg3w0Ue+PHSbNrDrrj4W\nYsMGb9ejhycdGgApkt7i/k/YzLYBJgPvAh2Bn4G9gSUxbQYA/YAewA/ALcB4M9s/hFDO7y4iUttm\nzPCkAGD//eGYY6Lnmjf35OHYY/1xxmOPwahRsHx5+de6+GJ44AGoWzfxcYtIYiUi978WmB1CuCim\n7scyba4EBoUQXgMwsx7AQuAMYFQCYhKRKoodFNm796brNhx2GDzxBOTn+3HZpCEvD045xcdB/OEP\niY1VRGpPIhKHzsA4MxsFHAvMBe4PITwKYGYtgWZ4jwQAIYRlZjYFaIcSB5Gk++236KDInBx/zFCe\nbt1g1iy4/npo0gROOsmThU6doo8yRCSzJCJx2BO4DLgH+BdwGDDMzNaEEJ7Gk4aA9zDEWhg5JyJJ\nNno0LIk8XDzrLNhuu4rbXncdXHopbL21Bj2KZINEJA51gE9CCH+PHH9hZgcBlwJPJ+D9RDJWCPDp\npz6eoHVrOPvs2nnf8gZFbs622yYuFhFJLYlIHOYDM8rUzQAis79ZABjQlNK9Dk2BzzZ34YKCAnJz\nc0vV5efnk1/ykFUkQ6xaBYWFcP/9UFQUrW/eHNq1q9w1xoyBOXP8MUOZfzab9dVXMHmylw84AI46\nqvLfKyLJVVhYSGFhYam64uLiuL5HIhKHycC+Zer2JTJAMoQwy8wWAB2AaQBm1gQ4HBixuQsPHjyY\ntm3bbq6JSFr7+mufffDkk1Dev/XhwyuXOLz3XnShpptv9u2pL7mkco8SYgdFXnKJtrUWSSfl/TJd\nVFREXl5e3N4jEes4DAaOMLPrzGwvM+sOXATEbJPDEOBGM+tsZgcDI4E5wNgExCOSskKAadPgttv8\nN/v99oOhQ0snDYceGh1j8MILMH/+lq97++3R8s8/+86TBx4IL77o71mR337ztRYAGjaE88+v+mcS\nkcwW98QhhPAf4EwgH/gvcANwZQjh+Zg2dwLDgYeAKcBWQCet4SDZYOVK37vh0kth99197ML11/vi\nSSUaNoSePeGTT3yMw6WXev369aXHH5Rn2jQYP97LjRpF67/91ld/PPro0u9VYv16fzyydKkfn3OO\nxi6IyKYsbO7XjxQRWXVy6tSpU/WoQtLW2rUwcKAvw1yyQ2RZBxwAF10EF1xQeibDTz9By5a+CmOz\nZvDjjxXv9dCjBzwdGYY8bBgccQT07w+TJpVut+OOHtOaNf5148bS5ydPhiOPrNZHFZEUEvOoIi+E\nULSl9luivSpEqmnlSh98WJnce+ZMfxRx222lk4YGDaBjR/8B/913PjCxoGDT6Y/Nm8MZZ3h5wQJ/\n5FCen37yXgPwa/Tq5YsvTZwIr7ziK0CWWLzYH4msXr1p0nDwwZUfhCki2UWrxotUw9Kl/oP1f/+D\nww+Ha66B008vf0nlp5/2vRxWrPDj+vW9R6FzZ+jQwfd+qIzLL48mDPfdF12xMdbQodENpfr0iV7b\nzN+vUyd4/HEYMcJ3t8zJ8eQl9uv22/t+ExoUKSLl0aMKkWro08dnP8Tae2+4+mp/VNCwof9g7tMH\nnn22dJvCQl+OuapCgN//Hr780o+nToXYfw7Fxd4zsXy5JwA//ghNm1b9fUQks+hRhUiSTZkCDz64\naf233/r0xT32iO4QGZs09OzpazJUd1aUmfc6lBg+vPT5hx+O7hfRo4eSBhFJDCUOIlWwfr3PcCjp\nqLvnHhg3Do47Ltpm4UL417/g++/9uEkT72V4/HFflrkmzj0XttnGy4WFPtUSfHDj0KFeNoOrrqrZ\n+4iIVESJg0gVDB8On3/u5dat4YorfHDjhAk+dfKss6BOzL+qdu28fbdu8Xn/xo19wCP4bIhHH/Vy\nYSHMnevlLl1g37JLsImIxInGOIhU0k8/+ayElSv9t/qPP/aBkWXNnOnbTTdt6mMc6sV5CPJ33/lY\niRB8TMN33/lYh5KxDx9+qGWiRSQq3mMcNKtC0tKGDf6qaC2DRPjrXz1pAB/LUF7SANCqlT+qSJS9\n9vKtq19/3ZOZvn2jSUO7dkoaRCSx9KhC0s6PP/qKi1tvDaedBs884zMYEum11+Cll7y8006+HkMy\nxQ6SfOSRaLl//9qPRUSyi3ocJO3cckv0ef7rr/urYUP/LbxbNzj11NJLLZdn7Vq/xuzZ/pozx5dX\nPuUUaNGidNuVK6Ffv+jx4MHRAYrJcuKJsM8+8M030bpWrXx8g4hIIilxkLQydy489dSm9atXe4/A\nSy/5AMIWLXxsQeyrbl1Yt8679+fPr3jFxzZtfDGn00/3AZCDBnkvB/iCTamwi3udOv6I4soro3VX\nXVX+AlQiIvGkwZGSVvr3h7vv9vKAAf6o4vnnfdfIRYvi/34tWsC8eT4Ns0ED+O9//Tf9VLBsGey6\nq69IucMO3nOy1VbJjkpEUo0GR0rWWrIkuvBSTo7v6dC0qe/2OGSIb+L0/PPw5pu+ENL69dFX7F4M\nzZp5QlDyat4cdtvNl49++WVfkbHE7NnR8vXXp07SAL4+xKhRfk+uvlpJg4jUDiUOkjbuvz+630Ov\nXqVXRqxXzx8jdOhQ/veG4LMwStpW5MYbfbzDK6/A2LG+OdS6db7U84AB8fkc8dSpk79ERGqLEgdJ\nC6tWRVdGrFPHf8OuCrPKr6ew226+/kKfPr7/w5dfeuLQsGHV3lNEJBMpcZC08MQTvg00wDnnwJ57\n1s775uZqXQQRkVhax0FqxTff+EqL1RmLu24d3HVX9DgVHxmIiGQLJQ6ScG++CQceCEce6esklKzB\nUFmjRkWnQ3bq5FMkRUQkOZQ4SEJ9+aU/Wli/3o/HjYODD/ZNmSrT+xAC3H579PjaaxMTp4iIVI4S\nB0mYRYugc2efGgnRxYmWLIHu3X2Vx19+2fw13nij9D4MxxyTuHhFRGTLlDhIQqxZA3/6E/zwgx+3\nbQuzZpXeXnrUKDjoIF8yuiJlexvMEhKuiIhUkmZVyCaWLYP77oNPP/Uf1HXqbPpq2xYuusgXISor\nBOjdGyZP9uNddvF1EXbd1R9RnHkmXHYZ/PorLFjgqz8efTTsu6/PlthzT2jZEn7+2beIBjjgAG8n\nIiLJpcRB/s+6db7T4sCB0amPFXn2Wd/D4fLLfb+E7bePnrvjDhg50stbbeULKe26a/T82Wf7I4eL\nL472Nnz4YTRJKM+AAZ6wiIhIcum/YiEEGDPGHxv07bvlpKHE0qWePOy+uy/ING+eX+e666JtRo6E\nQw/d9Ht33hlefRUefbR0UlGe5s1TY2MpERGphR4HM7sWuBUYEkL4W0z9zcBFwDbAZOCyEMLMRMcj\npX38sW8cVfJYocTZZ8Pf/+49CRs3ln4tWeKPMp5+2mdLrFwJ99wDw4eX7hUYNAj+/OeK39sMLrzQ\nX0uW+BiI77+Pfv3+e/jtN79O/fqJ+fwiIlI1CU0czOwPQG/gizL1A4B+QA/gB+AWYLyZ7R9CWJvI\nmMSF4IMN77yzdP3RR/vuk4cfXvH3tmwJjz/ujzTuust7DVavhrUxf3Ldu8MNN1Q+nm239Zc2PxUR\nSW0Je1RhZlsDz+C9CkvLnL4SGBRCeC2E8CWeQOwCnJGoeKS0O+4onTTsu6/vDPn++5tPGmK1aOG9\nDLNmwTXXwNZbe/2RR8Jjj2kGhIhIJkrkGIcRwKshhAmxlWbWEmgGvFtSF0JYBkwB2iUwHol48snS\n4xDuvBP++184/fTq/bBv1swTkdmz4b33/KUNoUREMlNCHlWYWTfgEKCcYXE0AwKwsEz9wsg5SaA3\n3vBplCVuvdXHOMTDttvCscfG51oiIpKa4p44mNluwBDghBDCunheu6CggNzc3FJ1+fn55GvIfaVM\nmQJnnQUbNvjx5ZdrCWcRkUxSWFhIYWFhqbri4uK4voeF6mxXuLkLmp0OvARsAEo6vuvivQwbgP2A\nmcAhIYRpMd/3HvBZCKGgnGu2BaZOnTqVtho9t4nFiz0JmDoVOnb0vSGOOqr0DIevv/a6kiWezz7b\nF2PS2ggiIpmtqKiIvLw8gLwQQlFNr5eIRxXvAAeXqXsSmAHcHkL43swWAB2AaQBm1gQ4HB8XIVXw\n3ntw7rm+hgLAzJkwYoSv1njWWZ5EtGjhCUVJ0nDccb6+gpIGERGpqrgnDiGElcD02DozWwn8EkKY\nEakaAtxoZjPx6ZiDgDnA2HjHk6nWr4ebb4Zbbil/l8l582DoUH/Vr++rQoJvST1mDOTk1G68IiKS\nGWrrd85SP9pCCHcCw4GH8NkUWwGdtIZD5fz0k/caDBoUTRo6dPDHEU8/7Xs6xC6YVJI07LEHvPkm\nlBkmIiIiUmm1sldFCOH4cuoGAgNr4/1T0ejRMGECnHqqP0aoV8k/ibFjoWdPX2kRfKvqQYN8HYW6\ndWGffeC88/z8yy/Dv/8N77wDu+0G48f7Us8iIiLVpU2uatn69b6vw9ChfvzAA75XQ8+e0KuXr8oY\nKwRfY+HVV/01ZUr0XIsWPsDxyCM3fZ9tt/Vr9uzpqzrWratlm0VEpOaUONSipUt9sOJbb5WunzvX\nxyrccguccILvGpmb64nCa6/Bjz9ueq2uXX0ny2233fL7ajEmERGJFyUOteSbb6BzZ/8K/tv/9dfD\n5597clCytsI77/irIgcd5NtYX3ihlnQWEZHap8ShFrz9tq+bsDSyY8cOO8CLL0L79n48f74vA/3o\no74jZKz69eGPf/Sko3NnH+AoIiKSLEocEigE3366oCDao3DQQfDKK6XHMuy8s+8dMWAATJoEzz/v\n21d37AgnnQRNmiQnfhERkbKUOCTImjXQt6/vElmiSxd45hn43e/K/546dXya5XHH1U6MIiIiVaW1\nAxNg7lzf7Ck2abjuOl94qaKkQUREJB2oxyHOPvjAl3peGNn7c6utfOxC9+7JjUtERCQe1OMQJyHA\n/ffD8cdHk4bdd4fJk5U0iIhI5lDiEAerV/v0yL59fYEn8CWg//MfaNMmubGJiIjEkxKHGvj1V3j4\nYTj8cHjiiWj9VVfBuHE+7VJERCSTaIxDFf32my/Y9Oyz8MYb0Q2kwMczPPYY5OcnLz4REZFEUuJQ\nSTNmwJ13wksvwbJlm54/+GAYORIOOaT2YxMREaktShwq4YsvfJXHsgnDzjtDt25w7rnQtq2WgBYR\nkcynxGELfvgBTj45mjT87ne+wdS55/pCTXXrJjU8ERGRWqXEYTMWL/Zlnxcs8OPDD/cNqLbeOrlx\niYiIJItmVVRg5Uo47bTobpb77uuDIpU0iIhINlPiUI5163z1x08+8eNddoHx4zW9UkRERIlDGSHA\nxRfDm2/6cW6ur8mw++7JjUtERCQVKHEo4/rr4amnvJyTA2PH+lRLERERyfLBkcXFviz0J5/469NP\nfWdL8KmVzz7ru1yKiIiIy8jEYdo0GDbMZ0AANGjgvQc5OdHyggXwv/9VfI0RI3zapYiIiERlTOKw\nYQO8+ioMHQrvvVe9azRpAoce6htWaUdLERGRTcV9jIOZXWdmn5jZMjNbaGZjzGyfctrdbGbzzGyV\nmb1tZq2q835Ll8K998Lee8OZZ5ZOGho1gqZNYZttvBy7WFODBnDYYdCvn49pmDEDliyBd99N/aSh\nsLAw2SGkBN0Hp/sQpXvhdB+idC/iLxGDI48BhgOHAycA9YG3zGyrkgZmNgDoB/QGDgNWAuPNrEFl\n3yQE34Vyt93866xZ0XN77w3Dh/vjiAULPCFYudK3vF6/HlatghUrYMoUb9ejB+y3H9RJk6Gi+ofg\ndB+c7kOU7oXTfYjSvYi/uD+qCCGcEntsZn8BFgF5wIeR6iuBQSGE1yJtegALgTOAUZV5HzNYtMgT\nghIdO8KVV/rXipKAunV9F0sRERGputr4HXsbIAC/AphZS6AZ8G5JgxDCMmAK0K4qF77ySn8Ecdll\nMH26r7fQqVP69ByIiIikm4QOjjQzA4YAH4YQpkeqm+GJxMIyzRdGzlXaoYf6o4jf/a7GoYqIiEgl\nJHpWxf3AAcBRNbxOQ4AZM2bUOKBMUFxcTFFRUbLDSDrdB6f7EKV74XQfonQvSv3sbBiP61kIIR7X\n2fTCZvcBnYFjQgizY+pbAt8Bh4QQpsXUvwd8FkIoKOda3YFnExKoiIhIdjg3hPBcTS+SkB6HSNJw\nOnBsbNIAEEKYZWYLgA7AtEj7JvgsjBEVXHI8cC7wA7A6ETGLiIhkqIbAHvjP0hqLe4+Dmd0P5ANd\ngG9iThWHEFZH2lwDDAD+gicDg4ADgQNDCGvjGpCIiIjETSISh4344MeyeoYQRsa0G4iv47AN8AHQ\nN4QwM67BiIiISFwlbIyDiIiIZB6teCAiIiKVpsRBREREKi0tEgcz62tms8zsNzP7f2b2h2THlEhm\ndoyZvWJmc81so5l1KadNXDYJS2W1vWFaKjOzS83sCzMrjrw+MrOTy7TJ+PtQlpldG/k3cm+Z+oy/\nF2Z2U+Szx76ml2mT8fcBwMx2MbOnzeznyGf9wszalmmT0fci8jOy7N+HjWY2PKZNXO5ByicOZnYO\ncA9wE9AG+ALfEGuHpAaWWI2Bz4E+lDPQNB6bhKWJWtkwLU38hM9Eaovv+zIBGGtm+0NW3Yf/E/kF\nojf+f0JsfTbdiy+Bpviqu82Ao0tOZMt9MLNtgMnAGqAjsD9wFbAkpk023ItDif49aAaciP/8GAVx\nvgchhJR+Af8PGBpzbMAc4Jpkx1ZLn38j0KVM3TygIOa4CfAbcHay403wvdghcj+OzvZ7Efmsv+Cz\nlbLuPgBbA18DxwMTgXuz7e8E/stU0WbOZ8t9uB2YtIU2WXEvynzmIcA3ibgHKd3jYGb18d+uYjfE\nCsA7VHFDrEwRz03C0lDCNkxLJ2ZWx8y6AY2Aj7L0PowAXg0hTIitzMJ7sXfkkeZ3ZvaMmTWHrLsP\nnYH/mNmoyCPNIjO7qORklt0L4P9+dp4LPBY5jus9SOnEAf8Nsy5x2BArg8Rtk7B0YpbYDdPSgZkd\nZGbL8S7Z+4EzQwhfk333oRtwCHBdOaez6V78P3wRvY7ApUBL4H0za0x23Yc9gcvwHqiTgAeAYWZ2\nfuR8Nt2LEmcCucBTkeO43oNEb3IlEi/x2jAtnf0PaI3/h/BnYKSZtU9uSLXLzHbDE8gTQgjrkh1P\nMoUQYpcP/tLMPgF+BM7G/65kizrAJyGEv0eOvzCzg/Bk6unkhZVUvYA3QwgLEnHxVO9x+BnYgA/+\nidUUSMgNSQML8HEeWXNPzPc+OQX4YwhhfsyprLoXIYT1IYTvQwifhRBuwAcFXkl23Yc8YEegyMzW\nmdk64FjgSjNbi/8GlS33opQQQjG+zH8rsuvvxHyg7NbJM4AWkXI23QvMrAU+mPyRmOq43oOUThwi\nv1FMxTfEAv6vy7oD8FGy4kqmEMIs/A869p6UbBKWcffEohumHRfK2TCNLLoX5agD5GTZfXgHOBh/\nVNE68voP8AzQOoTwPdlzL0oxs63xpGFelv2dmAzsW6ZuX7z3JRv/n+iFJ9BvlFTE/R4ke+RnJUaG\nng2sAnoA+wEP4aPJd0x2bAn8zI3x/xAPwWcR/DVy3Dxy/prIPeiM/yf6MvAt0CDZscf5PtyPT6k6\nBs+MS14NY9pky724NXIfdgcOAm4D1gPHZ9N9qODelJ1VkRX3ArgLaB/5O3Ek8Db+A2P7LLsPh+Lj\nfq4D9gK6A8uBbln4d8LwjSP/Vc65uN2DpH/QSt6MPpGb8RvwMXBosmNK8Oc9NpIwbCjzejymzUB8\nes0qfKvUVsmOOwH3obx7sAHoUaZdNtyLR4HvI/8GFgBvlSQN2XQfKrg3E2ITh2y5F0AhPjX9N2A2\n8BzQMtvuQ+RzngJMi3zOr4Be5bTJ+HuBr92woaLPFq97oE2uREREpNJSeoyDiIiIpBYlDiIiIlJp\nShxERESk0pQ4iIiISKUpcRAREZFKU+IgIiIilabEQURERCpNiYOIiIhUmhIHERERqTQlDiJSa8xs\nlpldkew4RKT6lDiIZCgze8LMXoqUJ5rZvbX43heY2ZJyTh0KPFxbcYhI/NVLdgAikj7MrH7w7e63\n2BTYZCOcEMIv8Y9KRGqTehxEMpyZPYHvuHqlmW00sw1m1iJy7iAze8PMlpvZAjMbaWbbx3zvRDMb\nbmaDzWwxMC5SX2Bm08xshZnNNrMRZtYocu5Y4HEgN+b9/hE5V+pRhZk1N7OxkfcvNrN/m9lOMedv\nMrPPzOy8yPcuNbNCM2tcC7dORMqhxEEk812Bb0f/CNAU2Bn4ycxygXeBqUBboCOwEzCqzPf3ANYA\nRwKXRuo2AJcDB0TOHwfcGTn3EfBXYFnM+91dNigzM+AVYBvgGOAEYE/g+TJN9wJOx7dOPhVPgq6t\n0h0QkbjRowqRDBdCWG5ma4FVIYTFJfVm1g8oCiH8PabuImC2mbUKIcyMVH8bQri2zDWHxRzONrO/\nAw8A/UII68ys2JtF368cJwAHAnuEEOZF3r8H8JWZ5YUQppaEBVwQQlgVafM00AH4eznXFJEEU+Ig\nkr1aA8eb2fIy9QH/Lb8kcZha5jxmdgL+W/9+QBP8/5IcM2sYQlhdyfffD/ipJGkACCHMMLOlwP4x\n7/tDSdIQMR/vGRGRJFDiIJK9tsYfFVyD/1Yfa35MeWXsCTPbHXgVGAFcD/yKP2p4FGgAVDZxqKyy\ngzEDeswqkjRKHESyw1qgbpm6IuBPwI8hhI1VuFYeYCGEq0sqzKxbJd6vrBlAczPbNYQwN3KdA/Ax\nD19VIR4RqUXK2kWyww/A4Wa2e8ysiRHAdsDzZnaome1pZh3N7PHIwMWKzATqm9kVZtbSzM4HLinn\n/bY2s+PNbHsz26rsRUII7wBfAs+aWRszOwx4CpgYQvisRp9WRBJGiYNIdrgbnwkxHVhkZi1CCPOB\no/D/B8YD04B7gSUhhJI1GMpbi2Ea8Df8Ecd/gXzKzHIIIXwMPAj8G1gE9K/gel2AJcAk4C08KSnb\neyEiKcSi/z+IiIiIbJ56HERERKTSlDiIiIhIpSlxEBERkUpT4iAiIiKVpsRBREREKk2Jg4iIiFSa\nEgcRERGpNCUOIiIiUmlKHERERKTSlDiIiIhIpSlxEBERkUr7/1Sh5dmdGPijAAAAAElFTkSuQmCC\n", + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], "source": [ "n_iter = 200\n", "n_episode = 100\n", "path_length = 200\n", "discount_rate = 0.99\n", "baseline = LinearFeatureBaseline(env.spec)\n", + "#baseline = None\n", "\n", "po = PolicyOptimizer(env, policy, baseline, n_iter, n_episode, path_length,\n", " discount_rate)\n", @@ -309,11 +422,39 @@ "\n", "Include the answer in your report." ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [] } ], "metadata": { + "anaconda-cloud": {}, "kernelspec": { - "display_name": "Python 2", + "display_name": "Python [default]", "language": "python", "name": "python2" }, @@ -327,7 +468,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython2", - "version": "2.7.11" + "version": "2.7.12" } }, "nbformat": 4, diff --git a/Report b/Report deleted file mode 100644 index 0770ee6..0000000 --- a/Report +++ /dev/null @@ -1,10 +0,0 @@ -pro5: -I failed to handle the spicy problem although I used the following code: -sudo apt-get install python-scipy -pip install scipy -import scipy - -pro6: -I guess when run the whole process, 'advantages' may be calculated to some format inconsistent with the codes using it below, -so it may produce bugs if we do not normalize it first. -While '+ 1e-8' is to avoid the denominator turning to zero. diff --git a/Untitled.ipynb b/Untitled.ipynb new file mode 100644 index 0000000..0e70a33 --- /dev/null +++ b/Untitled.ipynb @@ -0,0 +1,49 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 11, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "ename": "ImportError", + "evalue": "No module named tensorflow", + "output_type": "error", + "traceback": [ + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[0;31mImportError\u001b[0m Traceback (most recent call last)", + "\u001b[0;32m\u001b[0m in \u001b[0;36m\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0;32mimport\u001b[0m \u001b[0mtensorflow\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0mtf\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m", + "\u001b[0;31mImportError\u001b[0m: No module named tensorflow" + ] + } + ], + "source": [ + "import tensorflow as tf" + ] + } + ], + "metadata": { + "anaconda-cloud": {}, + "kernelspec": { + "display_name": "Python [conda root]", + "language": "python", + "name": "conda-root-py" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 2 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython2", + "version": "2.7.12" + } + }, + "nbformat": 4, + "nbformat_minor": 1 +} diff --git a/Untitled1.ipynb b/Untitled1.ipynb new file mode 100644 index 0000000..433d525 --- /dev/null +++ b/Untitled1.ipynb @@ -0,0 +1,49 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 11, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "ename": "ImportError", + "evalue": "No module named tensorflow", + "output_type": "error", + "traceback": [ + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[0;31mImportError\u001b[0m Traceback (most recent call last)", + "\u001b[0;32m\u001b[0m in \u001b[0;36m\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0;32mimport\u001b[0m \u001b[0mtensorflow\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m", + "\u001b[0;31mImportError\u001b[0m: No module named tensorflow" + ] + } + ], + "source": [ + "import tensorflow" + ] + } + ], + "metadata": { + "anaconda-cloud": {}, + "kernelspec": { + "display_name": "Python [default]", + "language": "python", + "name": "python2" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 2 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython2", + "version": "2.7.12" + } + }, + "nbformat": 4, + "nbformat_minor": 1 +} diff --git a/policy_gradient/policy.py b/policy_gradient/policy.py index 20f6efa..a0e6f81 100644 --- a/policy_gradient/policy.py +++ b/policy_gradient/policy.py @@ -1,5 +1,6 @@ import tensorflow as tf import numpy as np +import math class CategoricalPolicy(object): def __init__(self, in_dim, out_dim, hidden_dim, optimizer, session): @@ -27,8 +28,8 @@ def __init__(self, in_dim, out_dim, hidden_dim, optimizer, session): Sample solution is about 2~4 lines. """ # YOUR CODE HERE >>>>>> - W1 = tf.Variable(tf.zeros([in_dim, hidden_dim])) - W2 = tf.Variable(tf.zeros([hidden_dim, out_dim])) + W1 = tf.Variable(tf.truncated_normal([in_dim, hidden_dim],stddev=1.0)) + W2 = tf.Variable(tf.truncated_normal([hidden_dim, out_dim],stddev=1.0)) b1 = tf.Variable(tf.zeros([hidden_dim])) b2 = tf.Variable(tf.zeros([out_dim])) hidden1 = tf.nn.tanh(tf.matmul(self._observations, W1) + b1) @@ -75,7 +76,7 @@ def __init__(self, in_dim, out_dim, hidden_dim, optimizer, session): Sample solution is about 1~3 lines. """ # YOUR CODE HERE >>>>>> - surr_loss = tf.reduce_mean(-tf.mul(log_prob, self._advantages)) + surr_loss = -tf.reduce_mean(tf.mul(log_prob, self._advantages)) # surr_loss = ??? # <<<<<<<< diff --git a/policy_gradient/util.py b/policy_gradient/util.py index 6d5a08e..62870af 100644 --- a/policy_gradient/util.py +++ b/policy_gradient/util.py @@ -23,10 +23,10 @@ def flatten_space(space): # YOUR CODE HERE >>>>>> def discount_cumsum(x, discount_rate): len_x = len(x) - discount_x = np.zeros(len_x) + array_discount_rate = np.zeros(len_x) for i in range(len_x): - discount_x[i] = x[i] * discount_rate**i - discount_x =np.cumsum(discount_x[::-1]) - return discount_x + array_discount_rate[i] = x[i] * discount_rate**i + array_discount_rate =np.cumsum(array_discount_rate) + return array_discount_rate[::-1] # return ??? # <<<<<<<< diff --git a/report2.md b/report2.md new file mode 100644 index 0000000..e8b5bed --- /dev/null +++ b/report2.md @@ -0,0 +1,49 @@ +*** + +## Homework2 Report + +105061469 Haiyang Chang + +*** + +## Problem1 +I constructed a 2-layer neural network as required. The W1,W2 cannot be zeros like b1,b2, otherwise the Average Return will remain really low. I thought this is becauses the neural network would unable to learn the features of training data since 0*in_dim=0. So I use the truncated normal distribution with its stddev being the reciprocal to the input data to make it easier. ++ W1 = tf.Variable(tf.truncated_normal([in_dim, hidden_dim],stddev=1.0)) ++ W2 = tf.Variable(tf.truncated_normal([hidden_dim, out_dim],stddev=1.0)) ++ b1 = tf.Variable(tf.zeros([hidden_dim])) +b2 = tf.Variable(tf.zeros([out_dim])) ++ hidden1 = tf.nn.tanh(tf.matmul(self._observations, W1) + b1) ++ probs = tf.nn.softmax(tf.matmul(hidden1, W2) + b2) + +## Problem2 + +The smaller loss, the higher gradient. ++ surr_loss = -tf.reduce_mean(tf.mul(log_prob, self._advantages)) + +## Problem3 + +I tested the function on python to ensure its accuracy. ++ def discount_cumsum(x, discount_rate): + len_x = len(x) + array_discount_rate = np.zeros(len_x) + for i in range(len_x): + array_discount_rate[i] = x[i] * discount_rate**i + array_discount_rate =np.cumsum(array_discount_rate) + return array_discount_rate[::-1] + +## Problem4 + +There could be other forms, but this is the simplest form. ++ a = r - b + +## Problem5 + +Without baseline the Average Return increased to a high level quickly, but finally achieved the similar level as the performance with baseline. The baseline is to guarantee that the gradient will increase. So if the initial state of neural network is not chosen properly, the performance without baseline may not be as satisfactory as with baseline do. ++ With baseline +![withbaseline.png](https://ooo.0o0.ooo/2016/12/08/5849ac501e81f.png) ++ Without baseline +![withoutbaseline.png](https://ooo.0o0.ooo/2016/12/08/5849ac501e0e4.png) + +## Problem6 + +The reward is discounted by the discounted rate, with makes the later action seems less important than the former action. By normalizing the advantage, the training process can go on steadily. \ No newline at end of file