diff --git a/HW2_Policy_Graident.ipynb b/HW2_Policy_Graident.ipynb index 3b51aa3..c40078b 100644 --- a/HW2_Policy_Graident.ipynb +++ b/HW2_Policy_Graident.ipynb @@ -2,9 +2,10 @@ "cells": [ { "cell_type": "code", - "execution_count": null, + "execution_count": 1, "metadata": { - "collapsed": false + "collapsed": false, + "scrolled": true }, "outputs": [], "source": [ @@ -26,11 +27,19 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 2, "metadata": { "collapsed": false }, - "outputs": [], + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[2016-12-12 03:05:41,584] Making new env: CartPole-v0\n" + ] + } + ], "source": [ "import gym\n", "import tensorflow as tf\n", @@ -38,6 +47,8 @@ "from policy_gradient import util\n", "from policy_gradient.policy import CategoricalPolicy\n", "from policy_gradient.baselines.linear_feature_baseline import LinearFeatureBaseline\n", + "%matplotlib inline\n", + "import matplotlib.pyplot as plt\n", "\n", "np.random.seed(0)\n", "tf.set_random_seed(0)\n", @@ -95,11 +106,20 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 3, "metadata": { "collapsed": false }, - "outputs": [], + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/haiyang/Downloads/ENTER/lib/python2.7/site-packages/tensorflow/python/ops/gradients.py:90: UserWarning: Converting sparse IndexedSlices to a dense Tensor of unknown shape. This may consume a large amount of memory.\n", + " \"Converting sparse IndexedSlices to a dense Tensor of unknown shape. \"\n" + ] + } + ], "source": [ "sess = tf.Session()\n", "\n", @@ -151,9 +171,9 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 4, "metadata": { - "collapsed": true + "collapsed": false }, "outputs": [], "source": [ @@ -210,7 +230,7 @@ " Sample solution should be only 1 line.\n", " \"\"\"\n", " # YOUR CODE HERE >>>>>>\n", - " # a = ???\n", + " a = r - b\n", " # <<<<<<<<\n", "\n", " p[\"returns\"] = r\n", @@ -230,6 +250,8 @@ " )\n", "\n", " def train(self):\n", + " array_i=[]\n", + " array_avg_return=[]\n", " for i in range(1, self.n_iter + 1):\n", " paths = []\n", " for _ in range(self.n_episode):\n", @@ -238,10 +260,16 @@ " loss = self.policy.train(data[\"observations\"], data[\"actions\"], data[\"advantages\"])\n", " avg_return = np.mean([sum(p[\"rewards\"]) for p in paths])\n", " print(\"Iteration {}: Average Return = {}\".format(i, avg_return))\n", + " array_i.append(i)\n", + " array_avg_return.append(avg_return)\n", " \n", " # CartPole-v0 defines \"solving\" as getting average reward of 195.0 over 100 consecutive trials.\n", " if avg_return >= 195:\n", " print(\"Solve at {} iterations, which equals {} episodes.\".format(i, i*100))\n", + " plt.plot(array_i,array_avg_return,lw=2.0)\n", + " plt.xlabel('Iteration')\n", + " plt.ylabel=('Average Return')\n", + " plt.show()\n", " break\n", "\n", " if self.baseline != None:\n", @@ -250,17 +278,102 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 5, "metadata": { "collapsed": false }, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Iteration 1: Average Return = 21.82\n", + "Iteration 2: Average Return = 23.49\n", + "Iteration 3: Average Return = 23.39\n", + "Iteration 4: Average Return = 24.2\n", + "Iteration 5: Average Return = 28.48\n", + "Iteration 6: Average Return = 29.85\n", + "Iteration 7: Average Return = 32.93\n", + "Iteration 8: Average Return = 36.12\n", + "Iteration 9: Average Return = 34.61\n", + "Iteration 10: Average Return = 35.77\n", + "Iteration 11: Average Return = 38.6\n", + "Iteration 12: Average Return = 37.29\n", + "Iteration 13: Average Return = 41.57\n", + "Iteration 14: Average Return = 43.69\n", + "Iteration 15: Average Return = 43.66\n", + "Iteration 16: Average Return = 41.81\n", + "Iteration 17: Average Return = 46.41\n", + "Iteration 18: Average Return = 43.77\n", + "Iteration 19: Average Return = 43.26\n", + "Iteration 20: Average Return = 51.49\n", + "Iteration 21: Average Return = 50.17\n", + "Iteration 22: Average Return = 50.11\n", + "Iteration 23: Average Return = 50.56\n", + "Iteration 24: Average Return = 49.37\n", + "Iteration 25: Average Return = 55.61\n", + "Iteration 26: Average Return = 53.83\n", + "Iteration 27: Average Return = 56.62\n", + "Iteration 28: Average Return = 57.75\n", + "Iteration 29: Average Return = 59.34\n", + "Iteration 30: Average Return = 50.25\n", + "Iteration 31: Average Return = 58.7\n", + "Iteration 32: Average Return = 56.58\n", + "Iteration 33: Average Return = 56.68\n", + "Iteration 34: Average Return = 66.28\n", + "Iteration 35: Average Return = 61.66\n", + "Iteration 36: Average Return = 62.78\n", + "Iteration 37: Average Return = 66.56\n", + "Iteration 38: Average Return = 68.2\n", + "Iteration 39: Average Return = 72.12\n", + "Iteration 40: Average Return = 73.24\n", + "Iteration 41: Average Return = 77.72\n", + "Iteration 42: Average Return = 76.97\n", + "Iteration 43: Average Return = 79.66\n", + "Iteration 44: Average Return = 84.27\n", + "Iteration 45: Average Return = 80.64\n", + "Iteration 46: Average Return = 91.35\n", + "Iteration 47: Average Return = 98.51\n", + "Iteration 48: Average Return = 111.76\n", + "Iteration 49: Average Return = 106.64\n", + "Iteration 50: Average Return = 115.9\n", + "Iteration 51: Average Return = 129.98\n", + "Iteration 52: Average Return = 139.38\n", + "Iteration 53: Average Return = 145.66\n", + "Iteration 54: Average Return = 147.36\n", + "Iteration 55: Average Return = 149.93\n", + "Iteration 56: Average Return = 153.01\n", + "Iteration 57: Average Return = 162.6\n", + "Iteration 58: Average Return = 166.12\n", + "Iteration 59: Average Return = 171.98\n", + "Iteration 60: Average Return = 174.87\n", + "Iteration 61: Average Return = 179.87\n", + "Iteration 62: Average Return = 186.71\n", + "Iteration 63: Average Return = 192.05\n", + "Iteration 64: Average Return = 188.59\n", + "Iteration 65: Average Return = 192.96\n", + "Iteration 66: Average Return = 197.28\n", + "Solve at 66 iterations, which equals 6600 episodes.\n" + ] + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAg4AAAF5CAYAAAD3dKLdAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAAPYQAAD2EBqD+naQAAIABJREFUeJzt3Xmc1vP+//HHq21STmMvSxHZOalxkCWHkFA4HTQhpxAq\nnDkk2zk6cuy0yb5mGSeRrGUpIX5xGnSog4i0h5o27e/fH69rvtc100zNcl1zbc/77Xbd5v15f97z\nuV7Xp2Ve8/68FwshICIiIlIZdZIdgIiIiKQPJQ4iIiJSaUocREREpNKUOIiIiEilKXEQERGRSlPi\nICIiIpWmxEFEREQqTYmDiIiIVJoSBxEREak0JQ4iIiJSaVVKHMzsOjP7xMyWmdlCMxtjZvuU0+5m\nM5tnZqvM7G0za1XmfI6ZjTCzn81suZmNNrOdavphREREJLGq2uNwDDAcOBw4AagPvGVmW5U0MLMB\nQD+gN3AYsBIYb2YNYq4zBDgV6Aq0B3YBXqzmZxAREZFaYjXZ5MrMdgAWAe1DCB9G6uYBd4UQBkeO\nmwALgQtCCKMix4uBbiGEMZE2+wIzgCNCCJ/U5AOJiIhI4tR0jMM2QAB+BTCzlkAz4N2SBiGEZcAU\noF2k6lCgXpk2XwOzY9qIiIhICqp24mBmhj9y+DCEMD1S3QxPJBaWab4wcg6gKbA2klBU1EZERERS\nUL0afO/9wAHAUXGKpUJmtj3QEfgBWJ3o9xMREckgDYE9gPEhhF9qerFqJQ5mdh9wCnBMCGF+zKkF\ngOG9CrG9Dk2Bz2LaNDCzJmV6HZpGzpWnI/BsdWIVERERAM4FnqvpRaqcOESShtOBY0MIs2PPhRBm\nmdkCoAMwLdK+CT4LY0Sk2VRgfaRN7ODIFsDHFbztDwDPPPMM+++/f1VDzjgFBQUMHjw42WEkne6D\n032I0r1wug9RuhcwY8YMzjvvPIj8LK2pKiUOZnY/kA90AVaaWdPIqeIQQskjhCHAjWY2MxLkIGAO\nMBZ8sKSZPQbca2ZLgOXAMGDyZmZUrAbYf//9adu2bVVCzki5ubm6D+g+lNB9iNK9cLoPUal6L9au\nhZtuggEDYJttau1t4/Kov6o9Dpfigx/fK1PfExgJEEK408waAQ/hsy4+ADqFENbGtC8ANgCjgRxg\nHNC3qsGLiIikm40boVcvePZZeO01GDcOdt012VFVXpUShxBCpWZhhBAGAgM3c34NcHnkJSIikjWu\nvdaTBoCZM2H27PRKHLRXhYiISC0ZOhTuusvLderA889DuzRbwUiJQxrKz89PdggpQffB6T5E6V44\n3YeoVLoXo0ZBQUH0+P774fTTkxdPddVoyenaYmZtgalTp05NyUEuIiIimzNpEpx0kg+KBLjxRhg0\nqHbeu6ioiLy8PIC8EEJRTa+nHgcREZEE+u9/vWehJGno1Qtuvjm5MdWEEgcREZEE+ekn6NQJiov9\n+JRT4MEHwSy5cdWEEgcREZEEWLIETj4Z5s714z/8wcc51K+f3LhqSomDiIhIJfzwA6xbV7m2a9bA\nGWfA9MgWkK1a+ZoNjRsnLLxao8RBRERkC/75T2jZEg46CL77bvNtN26ECy6A99/34x139EWedtop\n8XHWBiUOIiIim/HOOzBwoJe/+QaOPBKmTq24/YAB8O9/e7lRI3j9ddhrr4SHWWuUOIiIiFRg8WI4\n//zSdYsWwR//CG+/vWn7YcPg7ru9XKeOj2n4wx8SHmatUuIgIiJSjhCgZ09YsMCPO3SAo4/28ooV\nPkOiZOlogJdegr/+NXr8wANw6qm1F29tqfK22iIiItngvvv8MQP4+IRnnoHcXOjeHV5+Gdavh/PO\n88TiiCPg3HM92QBf4Kl37+TFnkjqcRARESlj2jTo3z96/OST0KwZbLUVjB4Nl14aPXf11d4bsTqy\naXWPHum9wNOWKHEQERGJsWoVdOvmUyrBHz906hQ9X7eu7zMRu2R0SdsTT4RHHknvBZ62RImDiIhI\njL/9DWbM8HLr1nD77Zu2MfPHEY8+6olESdvRo6FBg9qLNRk0xkFERCTipZfgoYe8vNVWUFgIOTkV\nt7/wQmjTBj76yMc7NGlSO3EmkxIHERERYN48uOii6PHQobD//lv+vrZt/ZUt9KhCREQEuOEG318C\noGvX0kmERClxEBGRrFdUBE895eVttkn/HSwTSYmDiIhktRB8QGTJGgz/+AfssENyY0plShxERCSr\njR0LkyZ5uVUr6Ns3ufGkOiUOIiKStdauLb3Q0113Zf50yppS4iAiIllrxAiYOdPLxx4Lp5+e3HjS\nQZUTBzM7xsxeMbO5ZrbRzLqUOd/YzO4zs5/MbJWZfWVml5Rpk2NmI8zsZzNbbmajzSxDdioXEZF0\n8Msv0aWhzeDeezUgsjKq0+PQGPgc6AOEcs4PBk4CugP7RY7vM7PTYtoMAU4FugLtgV2AF6sRi4iI\nSLX885+wdKmXL7ggu9ZiqIkqLwAVQhgHjAMwKzc3awc8FUL4IHL8qJldChwGvGZmTYBeQLcQwqTI\ndXoCM8zssBDCJ9X4HCIiIpX2v//5fhMAjRrBv/6V3HjSSSLGOHwEdDGzXQDM7Dhgb2B85HwenrC8\nW/INIYSvgdl40iEiIpJQ/fvDhg1eHjAAdtklufGkk0QsOX058DAwx8zWAxuAi0MIkyPnmwFrQwjL\nynzfwsg5ERGRhHnnHXjtNS/vuitcdVVy40k3iUgcrgAOB07DexHaA/eb2bwQwoQEvJ+IiEil/Pab\nb5Nd4tZboXHj5MWTjuKaOJhZQ+BfwBkhhDcj1V+aWRvgamACsABoYGZNyvQ6NI2cq1BBQQG5ubml\n6vLz88nPz4/XRxARkQwVAvTpA1995cd5eb6jZSYpLCyksLCwVF1xcXFc3yPePQ71I68NZeo3EB1P\nMRVYD3QAxgCY2b5AC+DjzV188ODBtNWwVxERqYbHHoMnn/Ryo0a+N0WdDFvNqLxfpouKisjLy4vb\ne1Q5cTCzxkAroGRGxZ5m1hr4NYTwk5lNAu42s8uBH4E/Aj2AvwKEEJaZ2WPAvWa2BFgODAMma0aF\niIgkQlER9OsXPX7kETjwwOTFk86q0+NwKDARX8MhAPdE6p/Cp1meA9wGPANshycP14UQHo65RgHe\nCzEayMGnd2p1cBERibslS3yb7DVr/LhvX+jePbkxpbPqrOMwic1M4wwhLAIu3MI11uCzLy6v6vuL\niIhU1saN0KMH/PCDHx92GNxzz2a/RbYgw57uiIiIRN1xR3Tq5fbbwwsvQE5OcmNKd0ocREQkI737\nLtx4o5fN4LnnoEWL5MaUCZQ4iIhIxpk9G/Lz/VEFwMCBcNJJSQ0pYyhxEBGRjDJ+vK/RsHixH598\ncrTnQWpOiYOIiGSE9evh+us9Ufj5Z6/bay945pnMW68hmXQrRUQk7c2dC8cdB7fdFq077TSYMsUH\nRUr8KHEQEZG0Nm4cHHIIfPihH9erB3ffDa+8oqQhERKxyZWIiEjChQD/+Afccku0rnlz+Pe/oV27\n5MWV6dTjICIiaenxx0snDZ07w+efK2lINCUOIiKSdubMgb/9LXp8550wdixst13yYsoWelQhIiJp\nJQS45BJYtsyPL7gA+vdPbkzZRD0OIiKSVp55Bt54w8vNmsHgwcmNJ9socRARkbSxYAFceWX0+MEH\nYdttkxdPNlLiICIiaSEE6NPHt8kGX1L69NOTG1M2UuIgIiJp4YUXYMwYL++4Iwwbltx4spUSBxER\nSXmLF0O/ftHjESNghx2SF082U+IgIiIp74oroptWde0KZ52V3HiymRIHERFJaS+/DM8/7+XttvPe\nBkkeJQ4iIpKy3nkHzj8/ejxsGDRtmrx4RImDiIikqFGj4JRTYMUKPz7jDOjePbkxiRIHERFJQSNG\nQLdusG6dH59xBhQWglly4xIlDiIikkJCgJtu8hkUIXjdhRf6VMyGDZMbmzglDiIikhI2bPAFnm6+\nOVp3/fXwyCNQTzsrpQz9UYiISNKtWQPnnQejR0frhgwpvby0pIYq9ziY2TFm9oqZzTWzjWbWpZw2\n+5vZWDNbamYrzGyKme0Wcz7HzEaY2c9mttzMRpvZTjX9MCIikp7uvDOaNNSrB88+q6QhVVXnUUVj\n4HOgDxDKnjSzvYAPgOlAe+BgYBCwOqbZEOBUoGukzS7Ai9WIRURE0ty6dfDAA16uWxdefVWzJ1JZ\nlR9VhBDGAeMAzMod33oL8HoI4bqYulklBTNrAvQCuoUQJkXqegIzzOywEMInVY1JRETS12uvwfz5\nXu7SBU4+ObnxyObFdXBkJJE4FfjWzMaZ2UIz+39mFrt/WR6esLxbUhFC+BqYDbSLZzwiIpL6Hnoo\nWr7kkuTFIZUT71kVOwFbAwOAN4ATgTHAS2Z2TKRNM2BtCGFZme9dGDknIiJZYtYseOstL7dsCSee\nmNx4ZMviPauiJBF5OYRQsuHpNDM7ErgUH/tQbQUFBeTm5paqy8/PJz8/vyaXFRGRJHnkkeh6DRdf\nDHW0SECNFBYWUlhYWKquuLg4ru8R78ThZ2A9MKNM/QzgqEh5AdDAzJqU6XVoGjlXocGDB9O2bdt4\nxSoiIkm0bh08/riX69WDnj2TG08mKO+X6aKiIvLy8uL2HnHN7UII64BPgX3LnNoH+DFSnoonFx1K\nTprZvkAL4ON4xiMiIqlr7FhYuNDLZ5wBzfSwOi1UucfBzBoDrYCSGRV7mllr4NcQwk/AXcDzZvYB\nMBHoBJwGHAsQQlhmZo8B95rZEmA5MAyYrBkVIiLZQ4Mi01N1HlUciicEIfK6J1L/FNArhPCymV0K\nXA8MBb4G/hRCiO1NKAA2AKOBHHx6Z99qfQIREUk7M2f6ltkAe+0Fxx+f3Hik8qqzjsMktvCII4Tw\nJPDkZs6vAS6PvEREJMs88ki03Lu3BkWmE/1RiYhIrVq7Fp54wsv168Nf/pLUcKSKlDiIiEitGjMG\nFi/28plnwk7aqSitKHEQEZFapUGR6U2Jg4iI1JpvvoGJE728995w3HHJjUeqTomDiIjUmocfjpZ7\n94Zyt0qUlKbEQUREasXcufDkk15u0ECDItOVEgcREUm4Dz6AvDz45Rc/7toVdtghuTFJ9ShxEBGR\nhAkB7r/fF3gqWV56993h1luTG5dUnxIHERFJiNWr4cILoW9fWL/e6zp0gP/8B/bYI6mhSQ0ocRAR\nkbibMwfat48u9ARw9dUwbpweUaS7eG+rLSIiWW7CBMjPh0WL/HirreCxx7xO0p96HEREJC5WrPDH\nEh06RJOGPfaAjz5S0pBJ1OMgIiI1NnEi9OoFP/wQrTvxRCgshO23T1pYkgDqcRARkWpbsQL69PFZ\nEyVJQ6NGMGyYj2dQ0pB51OMgIiLVMmGCz5qI7WVo3x4efxz22itpYUmCqcdBREQqbcMGGDsW/vhH\nH8sQ28swfLg/slDSkNnU4yAiIlu0YoVPrRw6FL77rvS5Y4/1WRNKGLKDEgcREanQ/Plw773wyCNQ\nXFz63L77Qv/+0LMn1FH/ddZQ4iAiIuVatMj3l5g/v3T9CSdAQQGcfLIShmykxEFERMp19dXRpCEn\nB849F/76Vzj44OTGJcmlxEFERDYxcSI8/bSXt90WvvgCmjdPbkySGtTJJCIipaxd62szlLj9diUN\nEqXEQURESrn7bvjf/7x8xBFw0UXJjUdSS5UTBzM7xsxeMbO5ZrbRzLpspu2DkTZXlKnPMbMRZvaz\nmS03s9FmtlN1PoCIiMTP99/DoEFerlsXHnxQAyCltOr8dWgMfA70AUJFjczsTOBwYG45p4cApwJd\ngfbALsCL1YhFRETiJAS4/HJYvdqPr7gCWrdObkySeqo8ODKEMA4YB2BmVl4bM9sVGAp0BN4oc64J\n0AvoFkKYFKnrCcwws8NCCJ9UNSYREam5MWPgjcj/2LvuCv/8Z3LjkdQU9w6oSDIxErgzhDCjnCZ5\neMLybklFCOFrYDbQLt7xiIjIlq1YAVdeGT0eMgR+97vkxSOpKxFPrq4F1oYQ7qvgfLPI+WVl6hdG\nzomISC0bOBDmzPFyp07QtWtSw5EUFtd1HMwsD7gCaBPP64qISOJMm+Y9DAANG8J990H5D6JF4r8A\n1NHAjsBPMcMf6gL3mtlfQwh7AguABmbWpEyvQ9PIuQoVFBSQm5tbqi4/P5/8/Px4xS8iknX69/dd\nLwFuuAH23DO58Uj1FRYWUlhYWKquuOwmIzVkIVQ4MWLL32y2ETgjhPBK5HhbYOcyzd7Cxzw8EUL4\nNjI4cjE+OHJM5Pv2BWYAR5Q3ONLM2gJTp06dStu2basdr4iIlPbtt7DPPl5u2RJmzPDlpSVzFBUV\nkZeXB5AXQiiq6fWq3ONgZo2BVkBJl8KeZtYa+DWE8BOwpEz7dcCCEMK3ACGEZWb2GN4LsQRYDgwD\nJmtGhYhI7XrwwWi5Tx8lDbJl1XlUcSgwEV/DIQD3ROqfwqdZllVel0YBsAEYDeTg0zv7ViMWERGp\npt9+gyee8HJOjm+PLbIl1VnHYRJVmI0RGddQtm4NcHnkJSIiSTBqFCyJ9BGffTZsv31y45H0oIVE\nRUSy1AMPRMuXXZa8OCS9KHEQEclCRUUwZYqXW7f2zaxEKkOJg4hIFirb26B1G6SylDiIiGSZ4mJ4\n7jkv/+53cO65yY1H0osSBxGRLDNyJKxa5eXzz4ett05uPJJelDiIiGSREDQoUmpGiYOISBZ5/31f\nHRLg6KPhoIOSG4+kHyUOIiJZRL0NUlNKHEREssTChfDSS17ecUdtnS3Vo8RBRCRLPPYYrFvn5V69\ntC+FVI8SBxGRLLBhAzz0kJfN4JJLkhuPpC8lDiIiWWD8eJg928snn+xbaItUhxIHEZEs8Oqr0fLF\nFycvDkl/ShxERLLAW2/51/r14cQTkxuLpDclDiIiGe677+D777181FFaKVJqRomDiEiGK+ltADjp\npOTFIZlBiYOISIZT4iDxpMRBRCSDrVsH777r5e23hzZtkhuPpD8lDiIiGWzKFFi+3Msnngh19L++\n1JD+ComIZLDYxxQdOyYvDskcShxERDJYbOKgaZgSD0ocREQy1K+/wqefevnAA2HXXZMbj2QGJQ4i\nIhlqwgTYuNHLmk0h8aLEQUQkQ2kapiRClRMHMzvGzF4xs7lmttHMusScq2dmd5jZNDNbEWnzlJnt\nXOYaOWY2wsx+NrPlZjbazHaKxwcSEREIIZo4NGgA7dsnNx7JHNXpcWgMfA70AUKZc42AQ4B/Am2A\nM4F9gbFl2g0BTgW6Au2BXYAXqxGLiIiU49tv4ccfvXzMMdCoUXLjkcxRr6rfEEIYB4wDMDMrc24Z\nUGrCj5n1A6aY2W4hhDlm1gToBXQLIUyKtOkJzDCzw0IIn1Tvo4iISInx46NlTcOUeKqNMQ7b4D0T\nSyPHeXjC8m5JgxDC18BsoF0txCMikvE0vkESJaGJg5nlALcDz4UQVkSqmwFrI70TsRZGzomISA2s\nXQsTJ3q5aVM4+ODkxiOZpcqPKirLzOoBL+C9DX3icc2CggJyc3NL1eXn55Ofnx+Py4uIZISPP4aV\nK72sZaazS2FhIYWFhaXqiouL4/oeCUkcYpKG5sDxMb0NAAuABmbWpEyvQ9PIuQoNHjyYtm3bxj1e\nEZFMoscU2au8X6aLiorIy8uL23vEPQ+NSRr2BDqEEJaUaTIVWA90iPmefYEWwMfxjkdEJNvEJg4n\nnJC8OCQzVbnHwcwaA62AkhkVe5pZa+BXYD4+rfIQ4DSgvpk1jbT7NYSwLoSwzMweA+41syXAcmAY\nMFkzKkREaubnn2HqVC///vew886bby9SVdV5VHEoMBEfuxCAeyL1T+HrN3SO1H8eqbfI8XHA+5G6\nAmADMBrIwad39q1GLCIiEuPdd33xJ9A0TEmM6qzjMInNP+LY4uOPEMIa4PLIS0RE4kTjGyTRNNZW\nRCRDxC4z3bAhHH10cuORzKTEQUQkQ0yaBHPmePnYYz15EIk3JQ4iIhlixIho+S9/SVoYkuGUOIiI\nZIB582DMGC83bQp/+lNy45HMpcRBRCQDPPwwbNjg5d69fSttkURQ4iAikubWrfPEAaBuXU8cRBJF\niYOISJp7+WWYP9/Lp58Ou+2W3HgksylxEBFJQatWQefO0KED/PTT5tvGDorsq6X0JMEStjumiIhU\n3+OPw2uvefmss+D998sft/DVVz4NE2C//eC442ovRslO6nEQEUlBb7wRLU+ZAtdeW367+++Plvv0\nAbPy24nEixIHEZEUs2oVTJxYum7wYB/LEGvZMhg50suNG0OPHrUTn2Q3JQ4iIilmwgRYvdrLzZtH\n63v2hFmzosfPPAMrVnj5vPMgN7f2YpTspcRBRCTFxD6mGDbMxzgALF0K55wDa9f6vhSxgyL79Knd\nGCV7KXEQEUkhIcDrr3u5fn044QR45BHYay+v+/RT6N/fB0ROn+51Rx8Nv/99cuKV7KPEQUQkhUyf\nDrNne/nYY2Hrrf0RxAsvQE6O1w8bVnqRJ03BlNqkxEFEJIWU9DYAnHpqtNymDQwZEj3+9lv/qn0p\npLYpcRARSSGx4xtiEweASy7xMQ6xLr5Y+1JI7VLiICKSIpYuhQ8/9HKrVrD33qXPm/meFCX19et7\nMiFSm7RypIhIinj77egOl2V7G0o0aeLt7rkHTjpJ+1JI7VPiICKSImLHN5xySsXtdt/dB0iKJIMe\nVYiIpICNG+HNN73cqJHPqBBJRUocRERSwNSpsGiRl084ITr1UiTVKHEQEUkBm5tNIZJKqpw4mNkx\nZvaKmc01s41m1qWcNjeb2TwzW2Vmb5tZqzLnc8xshJn9bGbLzWy0me1Ukw8iIpLOKju+QSTZqtPj\n0Bj4HOgDhLInzWwA0A/oDRwGrATGm1nsTOMhwKlAV6A9sAvwYjViERFJewsX+lLS4EtHa6aEpLIq\nz6oIIYwDxgGYlbvz+5XAoBDCa5E2PYCFwBnAKDNrAvQCuoUQJkXa9ARmmNlhIYRPqvVJRETS1Lhx\n0bIeU0iqi+sYBzNrCTQD3i2pCyEsA6YA7SJVh+IJS2ybr4HZMW1ERLJG7PgGPaaQVBfvwZHN8McX\nC8vUL4ycA2gKrI0kFBW1ERHJCuvWwfjxXt52WzjiiOTGI7IlmlUhIpJEH38MxcVe7tgR6mlZPklx\n8f4rugAwvFchttehKfBZTJsGZtakTK9D08i5ChUUFJCbm1uqLj8/n/z8/JrGLSKSFBXthilSHYWF\nhRQWFpaqKy7JTOPEQthkYkTlv9lsI3BGCOGVmLp5wF0hhMGR4yZ4EtEjhPBC5HgxPjhyTKTNvsAM\n4IjyBkeaWVtg6tSpU2nbtm214xURSSUhwEEHwfTpvoHVokWwww7JjkoyTVFREXl5eQB5IYSiml6v\nyj0OZtYYaIX3LADsaWatgV9DCD/hUy1vNLOZwA/AIGAOMBZ8sKSZPQbca2ZLgOXAMGCyZlSISDZ5\n7TVPGsDHNihpkHRQnUcVhwIT8UGQAbgnUv8U0CuEcKeZNQIeArYBPgA6hRDWxlyjANgAjAZy8Omd\nfav1CURE0tCGDXDttdHj/v2TF4tIVVRnHYdJbGFQZQhhIDBwM+fXAJdHXiIiWWfkyNK9DWeckdx4\nRCpLsypERGrZb7/BP/4RPb7jDh/jIJIOlDiIiNSy++6DOXO8fNpp0L59cuMRqQolDiIitWjJErjt\nNi+bwa23JjcekapS4iAiUovuuMOTB4AePeDgg5Mbj0hVKXEQEaklc+bA0KFezsmBm29Objwi1aHE\nQUSklgwcCKtXe7lfP2jRIqnhiFSLEgcRkVowfTo88YSXc3PhuuuSG49IdSlxEBGpBddfDxs3evna\na2H77ZMbj0h1KXEQEUmwyZNh7Fgv77ILXHFFcuMRqQklDiIiCXb33dHyP/8JjRolLxaRmlLiICKS\nQL/+Gt06u1kz+MtfkhqOSI0pcRARSaAXXoB167zcvTvUq87WgiIpRImDiEgCPf10tHz++cmLQyRe\nlDiIiCTI99/7wEiAAw+E1q2TG49IPChxEBFJkOeei5bPO087YEpmUOIgIpIAIcAzz0SPu3dPXiwi\n8aTEQUQkAaZOha+/9vIf/6jlpSVzKHEQEUmA2EGR552XvDhE4k2Jg4hInK1bB4WFXs7Jga5dkxuP\nSDwpcRARibN33oHFi73cuTNss01y4xGJJyUOIiJVMHMm3HYbfPddxW1iB0XqMYVkGiUOIiKV9PXX\n8Ic/+E6XeXnw0Uebtlm+HMaM8fJ220GnTrUbo0iiKXEQEamEX3/1xw5Ll/pxcTGceCK8/XbpdmPG\nwG+/efmcc6BBg9qNUyTRlDiIiGzBunXw5z/Dt9/6cf36/nXVKjjttGgPA+gxhWS+uCcOZlbHzAaZ\n2fdmtsrMZprZjeW0u9nM5kXavG1mreIdi4hITYUA/frBxIl+vNNO8NVXcOaZfrx2LZx1FowcCfPm\nwbvven3LltCuXXJiFkmkRPQ4XAtcAvQB9gOuAa4xs34lDcxsANAP6A0cBqwExpuZOvVEJKUMHw4P\nP+zlnBx4+WXYe28YNSq6adWGDXDBBb465MaNXqclpiVTJSJxaAeMDSGMCyHMDiG8BLyFJwglrgQG\nhRBeCyF8CfQAdgHOSEA8IiLVMm4cFBREjx97LNqLUK8ePPkk9O0bPT9pUrSsxxSSqRKROHwEdDCz\nvQHMrDVwFPBG5Lgl0Ax4t+QbQgjLgCl40iEiknTTp/vgxpIehBtugHPPLd2mTh3vkbj++tL1hx0G\n++xTO3GK1LZ6Cbjm7UAT4H9mtgFPTm4IITwfOd8MCMDCMt+3MHJORCSp5s3zQY/Llvnxn/4EN99c\nflsz+Ne/IDcXBgzwuksvrZ04RZIhEYnDOUB3oBswHTgEGGpm80IIT2/2O7egoKCA3NzcUnX5+fnk\n5+fX5LIiIv/nxRehd2+ffgnQpo0PfKyzhf7Za66Bo4+GRYvg9NMTH6dIeQoLCyksWe88ori4OK7v\nYSGE+F7QbDZwWwjhgZi6G4BzQwgHRB5VfAccEkKYFtPmPeCzEEJBOddsC0ydOnUqbdu2jWu8IiLg\n6zJccYX2ETY/AAAe0klEQVQnCSWaN/dFnnbbLXlxidRUUVEReXl5AHkhhKKaXi8RYxwaARvK1G0s\nea8QwixgAdCh5KSZNQEOx8dHiIjEzdq18P77vkR0Rb8nvf8+tG5dOmn485/hs8+UNIiUlYhHFa8C\nN5rZHOAroC1QADwa02ZIpM1M4AdgEDAHGJuAeEQkS61eDSefHJ3tkJvrjx7atvWvhxzi21/fdVc0\nqWjSBO67T9MpRSqSiMShH54IjAB2AuYBD0TqAAgh3GlmjYCHgG2AD4BOIYS1CYhHRLJQCNCrV+kp\nksXF8N57/ipP+/be67D77rURoUh6inviEEJYCfwt8tpcu4HAwHi/v4gIwE03QckYsUaN4Pjj4fPP\nYc6cTdvWrw+33AJXXQV169ZunCLpJhE9DiIiSfXkkzAo0sdp5glEly5+vGiRj10oKvJX48a+yFPr\n1kkLVyStKHEQkYwyYQJcfHH0ePDgaNIAvtdEx47+EpGq0+6YIpIxZsyArl1h/Xo/7tfPp1iKSPwo\ncRCRjLBoEZx6Kixd6sennuq9DZoZIRJfShxEJK2tXw8ffgidO8OsWV7Xpg08/7xvRCUi8aV/ViKS\ndhYvhjffhDfegPHjo70MALvuCq++Cltvnbz4RDKZEgcRSRsjR8KIEfDpp+WvArn99vD66548iEhi\nKHEQkbRw++1w3XWb1ufm+gyJU07xxxXbbVf7sYlkEyUOIpLy7r67dNJw8ME++PGUU6BdO41lEKlN\n+ucmIiltyBDo3z96fPvtMGBA8uIRyXaaVSEiKeu++3xVxxK33KKkQSTZlDiISEp68EG4/PLo8cCB\ncMMNSQtHRCKUOIhIynn0UbjssujxjTfCP/6RvHhEJEqJg4ikjO+/90cTvXtH6669Fm6+WStAiqQK\nDY4UkaTauBHefhuGD/cFnWLXZ7jqKrj1ViUNIqlEiYOIJMWyZfDUUz4A8ptvSp/LyfGehptuUtIg\nkmqUOIjIFoUATz4Jq1b5Y4T69Wt2vfffhzPPhF9/LV3fogX06QMXXgg77FCz9xCRxFDiICJb9NJL\n0KuXl7/4Ah56qPo9ARMm+AqPq1ZF6447zmdQdO6sxZxEUp0GR4rIFt13X7T8yCMwbFj1rvPWW77i\nY0nScOKJ8N//ejJx5plKGkTSgRIHEdmsr7+G994rXfe3v/nulFXxxhvQpQusXu3HXbr4LpYHHRSX\nMEWklihxEJHNevjhaPnAA/3rxo3QrRtMn165a7z6qvcorFnjx3/6E7zwgg+CFJH0osRBRCq0erXP\nfABo0AAmTvQf+uCzIjp3hp9/3vw1xozx71m71o/POguef96vJyLpR4mDiFTopZfgl1+8/Oc/w447\nwsiR0KaN133/PXTtGk0KYs2bB4MHe6Kwfr3Xde8Ozz1X81kZIpI8CUkczGwXM3vazH42s1Vm9oWZ\ntS3T5mYzmxc5/7aZtUpELCJSfQ89FC1fcol/bdwYxo6FZs38+P33fQrlhg3w0Ue+PHSbNrDrrj4W\nYsMGb9ejhycdGgApkt7i/k/YzLYBJgPvAh2Bn4G9gSUxbQYA/YAewA/ALcB4M9s/hFDO7y4iUttm\nzPCkAGD//eGYY6Lnmjf35OHYY/1xxmOPwahRsHx5+de6+GJ44AGoWzfxcYtIYiUi978WmB1CuCim\n7scyba4EBoUQXgMwsx7AQuAMYFQCYhKRKoodFNm796brNhx2GDzxBOTn+3HZpCEvD045xcdB/OEP\niY1VRGpPIhKHzsA4MxsFHAvMBe4PITwKYGYtgWZ4jwQAIYRlZjYFaIcSB5Gk++236KDInBx/zFCe\nbt1g1iy4/npo0gROOsmThU6doo8yRCSzJCJx2BO4DLgH+BdwGDDMzNaEEJ7Gk4aA9zDEWhg5JyJJ\nNno0LIk8XDzrLNhuu4rbXncdXHopbL21Bj2KZINEJA51gE9CCH+PHH9hZgcBlwJPJ+D9RDJWCPDp\npz6eoHVrOPvs2nnf8gZFbs622yYuFhFJLYlIHOYDM8rUzQAis79ZABjQlNK9Dk2BzzZ34YKCAnJz\nc0vV5efnk1/ykFUkQ6xaBYWFcP/9UFQUrW/eHNq1q9w1xoyBOXP8MUOZfzab9dVXMHmylw84AI46\nqvLfKyLJVVhYSGFhYam64uLiuL5HIhKHycC+Zer2JTJAMoQwy8wWAB2AaQBm1gQ4HBixuQsPHjyY\ntm3bbq6JSFr7+mufffDkk1Dev/XhwyuXOLz3XnShpptv9u2pL7mkco8SYgdFXnKJtrUWSSfl/TJd\nVFREXl5e3N4jEes4DAaOMLPrzGwvM+sOXATEbJPDEOBGM+tsZgcDI4E5wNgExCOSskKAadPgttv8\nN/v99oOhQ0snDYceGh1j8MILMH/+lq97++3R8s8/+86TBx4IL77o71mR337ztRYAGjaE88+v+mcS\nkcwW98QhhPAf4EwgH/gvcANwZQjh+Zg2dwLDgYeAKcBWQCet4SDZYOVK37vh0kth99197ML11/vi\nSSUaNoSePeGTT3yMw6WXev369aXHH5Rn2jQYP97LjRpF67/91ld/PPro0u9VYv16fzyydKkfn3OO\nxi6IyKYsbO7XjxQRWXVy6tSpU/WoQtLW2rUwcKAvw1yyQ2RZBxwAF10EF1xQeibDTz9By5a+CmOz\nZvDjjxXv9dCjBzwdGYY8bBgccQT07w+TJpVut+OOHtOaNf5148bS5ydPhiOPrNZHFZEUEvOoIi+E\nULSl9luivSpEqmnlSh98WJnce+ZMfxRx222lk4YGDaBjR/8B/913PjCxoGDT6Y/Nm8MZZ3h5wQJ/\n5FCen37yXgPwa/Tq5YsvTZwIr7ziK0CWWLzYH4msXr1p0nDwwZUfhCki2UWrxotUw9Kl/oP1f/+D\nww+Ha66B008vf0nlp5/2vRxWrPDj+vW9R6FzZ+jQwfd+qIzLL48mDPfdF12xMdbQodENpfr0iV7b\nzN+vUyd4/HEYMcJ3t8zJ8eQl9uv22/t+ExoUKSLl0aMKkWro08dnP8Tae2+4+mp/VNCwof9g7tMH\nnn22dJvCQl+OuapCgN//Hr780o+nToXYfw7Fxd4zsXy5JwA//ghNm1b9fUQks+hRhUiSTZkCDz64\naf233/r0xT32iO4QGZs09OzpazJUd1aUmfc6lBg+vPT5hx+O7hfRo4eSBhFJDCUOIlWwfr3PcCjp\nqLvnHhg3Do47Ltpm4UL417/g++/9uEkT72V4/HFflrkmzj0XttnGy4WFPtUSfHDj0KFeNoOrrqrZ\n+4iIVESJg0gVDB8On3/u5dat4YorfHDjhAk+dfKss6BOzL+qdu28fbdu8Xn/xo19wCP4bIhHH/Vy\nYSHMnevlLl1g37JLsImIxInGOIhU0k8/+ayElSv9t/qPP/aBkWXNnOnbTTdt6mMc6sV5CPJ33/lY\niRB8TMN33/lYh5KxDx9+qGWiRSQq3mMcNKtC0tKGDf6qaC2DRPjrXz1pAB/LUF7SANCqlT+qSJS9\n9vKtq19/3ZOZvn2jSUO7dkoaRCSx9KhC0s6PP/qKi1tvDaedBs884zMYEum11+Cll7y8006+HkMy\nxQ6SfOSRaLl//9qPRUSyi3ocJO3cckv0ef7rr/urYUP/LbxbNzj11NJLLZdn7Vq/xuzZ/pozx5dX\nPuUUaNGidNuVK6Ffv+jx4MHRAYrJcuKJsM8+8M030bpWrXx8g4hIIilxkLQydy489dSm9atXe4/A\nSy/5AMIWLXxsQeyrbl1Yt8679+fPr3jFxzZtfDGn00/3AZCDBnkvB/iCTamwi3udOv6I4soro3VX\nXVX+AlQiIvGkwZGSVvr3h7vv9vKAAf6o4vnnfdfIRYvi/34tWsC8eT4Ns0ED+O9//Tf9VLBsGey6\nq69IucMO3nOy1VbJjkpEUo0GR0rWWrIkuvBSTo7v6dC0qe/2OGSIb+L0/PPw5pu+ENL69dFX7F4M\nzZp5QlDyat4cdtvNl49++WVfkbHE7NnR8vXXp07SAL4+xKhRfk+uvlpJg4jUDiUOkjbuvz+630Ov\nXqVXRqxXzx8jdOhQ/veG4LMwStpW5MYbfbzDK6/A2LG+OdS6db7U84AB8fkc8dSpk79ERGqLEgdJ\nC6tWRVdGrFPHf8OuCrPKr6ew226+/kKfPr7/w5dfeuLQsGHV3lNEJBMpcZC08MQTvg00wDnnwJ57\n1s775uZqXQQRkVhax0FqxTff+EqL1RmLu24d3HVX9DgVHxmIiGQLJQ6ScG++CQceCEce6esklKzB\nUFmjRkWnQ3bq5FMkRUQkOZQ4SEJ9+aU/Wli/3o/HjYODD/ZNmSrT+xAC3H579PjaaxMTp4iIVI4S\nB0mYRYugc2efGgnRxYmWLIHu3X2Vx19+2fw13nij9D4MxxyTuHhFRGTLlDhIQqxZA3/6E/zwgx+3\nbQuzZpXeXnrUKDjoIF8yuiJlexvMEhKuiIhUkmZVyCaWLYP77oNPP/Uf1HXqbPpq2xYuusgXISor\nBOjdGyZP9uNddvF1EXbd1R9RnHkmXHYZ/PorLFjgqz8efTTsu6/PlthzT2jZEn7+2beIBjjgAG8n\nIiLJpcRB/s+6db7T4sCB0amPFXn2Wd/D4fLLfb+E7bePnrvjDhg50stbbeULKe26a/T82Wf7I4eL\nL472Nnz4YTRJKM+AAZ6wiIhIcum/YiEEGDPGHxv07bvlpKHE0qWePOy+uy/ING+eX+e666JtRo6E\nQw/d9Ht33hlefRUefbR0UlGe5s1TY2MpERGphR4HM7sWuBUYEkL4W0z9zcBFwDbAZOCyEMLMRMcj\npX38sW8cVfJYocTZZ8Pf/+49CRs3ln4tWeKPMp5+2mdLrFwJ99wDw4eX7hUYNAj+/OeK39sMLrzQ\nX0uW+BiI77+Pfv3+e/jtN79O/fqJ+fwiIlI1CU0czOwPQG/gizL1A4B+QA/gB+AWYLyZ7R9CWJvI\nmMSF4IMN77yzdP3RR/vuk4cfXvH3tmwJjz/ujzTuust7DVavhrUxf3Ldu8MNN1Q+nm239Zc2PxUR\nSW0Je1RhZlsDz+C9CkvLnL4SGBRCeC2E8CWeQOwCnJGoeKS0O+4onTTsu6/vDPn++5tPGmK1aOG9\nDLNmwTXXwNZbe/2RR8Jjj2kGhIhIJkrkGIcRwKshhAmxlWbWEmgGvFtSF0JYBkwB2iUwHol48snS\n4xDuvBP++184/fTq/bBv1swTkdmz4b33/KUNoUREMlNCHlWYWTfgEKCcYXE0AwKwsEz9wsg5SaA3\n3vBplCVuvdXHOMTDttvCscfG51oiIpKa4p44mNluwBDghBDCunheu6CggNzc3FJ1+fn55GvIfaVM\nmQJnnQUbNvjx5ZdrCWcRkUxSWFhIYWFhqbri4uK4voeF6mxXuLkLmp0OvARsAEo6vuvivQwbgP2A\nmcAhIYRpMd/3HvBZCKGgnGu2BaZOnTqVtho9t4nFiz0JmDoVOnb0vSGOOqr0DIevv/a6kiWezz7b\nF2PS2ggiIpmtqKiIvLw8gLwQQlFNr5eIRxXvAAeXqXsSmAHcHkL43swWAB2AaQBm1gQ4HB8XIVXw\n3ntw7rm+hgLAzJkwYoSv1njWWZ5EtGjhCUVJ0nDccb6+gpIGERGpqrgnDiGElcD02DozWwn8EkKY\nEakaAtxoZjPx6ZiDgDnA2HjHk6nWr4ebb4Zbbil/l8l582DoUH/Vr++rQoJvST1mDOTk1G68IiKS\nGWrrd85SP9pCCHcCw4GH8NkUWwGdtIZD5fz0k/caDBoUTRo6dPDHEU8/7Xs6xC6YVJI07LEHvPkm\nlBkmIiIiUmm1sldFCOH4cuoGAgNr4/1T0ejRMGECnHqqP0aoV8k/ibFjoWdPX2kRfKvqQYN8HYW6\ndWGffeC88/z8yy/Dv/8N77wDu+0G48f7Us8iIiLVpU2uatn69b6vw9ChfvzAA75XQ8+e0KuXr8oY\nKwRfY+HVV/01ZUr0XIsWPsDxyCM3fZ9tt/Vr9uzpqzrWratlm0VEpOaUONSipUt9sOJbb5WunzvX\nxyrccguccILvGpmb64nCa6/Bjz9ueq2uXX0ny2233fL7ajEmERGJFyUOteSbb6BzZ/8K/tv/9dfD\n5597clCytsI77/irIgcd5NtYX3ihlnQWEZHap8ShFrz9tq+bsDSyY8cOO8CLL0L79n48f74vA/3o\no74jZKz69eGPf/Sko3NnH+AoIiKSLEocEigE3366oCDao3DQQfDKK6XHMuy8s+8dMWAATJoEzz/v\n21d37AgnnQRNmiQnfhERkbKUOCTImjXQt6/vElmiSxd45hn43e/K/546dXya5XHH1U6MIiIiVaW1\nAxNg7lzf7Ck2abjuOl94qaKkQUREJB2oxyHOPvjAl3peGNn7c6utfOxC9+7JjUtERCQe1OMQJyHA\n/ffD8cdHk4bdd4fJk5U0iIhI5lDiEAerV/v0yL59fYEn8CWg//MfaNMmubGJiIjEkxKHGvj1V3j4\nYTj8cHjiiWj9VVfBuHE+7VJERCSTaIxDFf32my/Y9Oyz8MYb0Q2kwMczPPYY5OcnLz4REZFEUuJQ\nSTNmwJ13wksvwbJlm54/+GAYORIOOaT2YxMREaktShwq4YsvfJXHsgnDzjtDt25w7rnQtq2WgBYR\nkcynxGELfvgBTj45mjT87ne+wdS55/pCTXXrJjU8ERGRWqXEYTMWL/Zlnxcs8OPDD/cNqLbeOrlx\niYiIJItmVVRg5Uo47bTobpb77uuDIpU0iIhINlPiUI5163z1x08+8eNddoHx4zW9UkRERIlDGSHA\nxRfDm2/6cW6ur8mw++7JjUtERCQVKHEo4/rr4amnvJyTA2PH+lRLERERyfLBkcXFviz0J5/469NP\nfWdL8KmVzz7ru1yKiIiIy8jEYdo0GDbMZ0AANGjgvQc5OdHyggXwv/9VfI0RI3zapYiIiERlTOKw\nYQO8+ioMHQrvvVe9azRpAoce6htWaUdLERGRTcV9jIOZXWdmn5jZMjNbaGZjzGyfctrdbGbzzGyV\nmb1tZq2q835Ll8K998Lee8OZZ5ZOGho1gqZNYZttvBy7WFODBnDYYdCvn49pmDEDliyBd99N/aSh\nsLAw2SGkBN0Hp/sQpXvhdB+idC/iLxGDI48BhgOHAycA9YG3zGyrkgZmNgDoB/QGDgNWAuPNrEFl\n3yQE34Vyt93866xZ0XN77w3Dh/vjiAULPCFYudK3vF6/HlatghUrYMoUb9ejB+y3H9RJk6Gi+ofg\ndB+c7kOU7oXTfYjSvYi/uD+qCCGcEntsZn8BFgF5wIeR6iuBQSGE1yJtegALgTOAUZV5HzNYtMgT\nghIdO8KVV/rXipKAunV9F0sRERGputr4HXsbIAC/AphZS6AZ8G5JgxDCMmAK0K4qF77ySn8Ecdll\nMH26r7fQqVP69ByIiIikm4QOjjQzA4YAH4YQpkeqm+GJxMIyzRdGzlXaoYf6o4jf/a7GoYqIiEgl\nJHpWxf3AAcBRNbxOQ4AZM2bUOKBMUFxcTFFRUbLDSDrdB6f7EKV74XQfonQvSv3sbBiP61kIIR7X\n2fTCZvcBnYFjQgizY+pbAt8Bh4QQpsXUvwd8FkIoKOda3YFnExKoiIhIdjg3hPBcTS+SkB6HSNJw\nOnBsbNIAEEKYZWYLgA7AtEj7JvgsjBEVXHI8cC7wA7A6ETGLiIhkqIbAHvjP0hqLe4+Dmd0P5ANd\ngG9iThWHEFZH2lwDDAD+gicDg4ADgQNDCGvjGpCIiIjETSISh4344MeyeoYQRsa0G4iv47AN8AHQ\nN4QwM67BiIiISFwlbIyDiIiIZB6teCAiIiKVpsRBREREKi0tEgcz62tms8zsNzP7f2b2h2THlEhm\ndoyZvWJmc81so5l1KadNXDYJS2W1vWFaKjOzS83sCzMrjrw+MrOTy7TJ+PtQlpldG/k3cm+Z+oy/\nF2Z2U+Szx76ml2mT8fcBwMx2MbOnzeznyGf9wszalmmT0fci8jOy7N+HjWY2PKZNXO5ByicOZnYO\ncA9wE9AG+ALfEGuHpAaWWI2Bz4E+lDPQNB6bhKWJWtkwLU38hM9Eaovv+zIBGGtm+0NW3Yf/E/kF\nojf+f0JsfTbdiy+Bpviqu82Ao0tOZMt9MLNtgMnAGqAjsD9wFbAkpk023ItDif49aAaciP/8GAVx\nvgchhJR+Af8PGBpzbMAc4Jpkx1ZLn38j0KVM3TygIOa4CfAbcHay403wvdghcj+OzvZ7Efmsv+Cz\nlbLuPgBbA18DxwMTgXuz7e8E/stU0WbOZ8t9uB2YtIU2WXEvynzmIcA3ibgHKd3jYGb18d+uYjfE\nCsA7VHFDrEwRz03C0lDCNkxLJ2ZWx8y6AY2Aj7L0PowAXg0hTIitzMJ7sXfkkeZ3ZvaMmTWHrLsP\nnYH/mNmoyCPNIjO7qORklt0L4P9+dp4LPBY5jus9SOnEAf8Nsy5x2BArg8Rtk7B0YpbYDdPSgZkd\nZGbL8S7Z+4EzQwhfk333oRtwCHBdOaez6V78P3wRvY7ApUBL4H0za0x23Yc9gcvwHqiTgAeAYWZ2\nfuR8Nt2LEmcCucBTkeO43oNEb3IlEi/x2jAtnf0PaI3/h/BnYKSZtU9uSLXLzHbDE8gTQgjrkh1P\nMoUQYpcP/tLMPgF+BM7G/65kizrAJyGEv0eOvzCzg/Bk6unkhZVUvYA3QwgLEnHxVO9x+BnYgA/+\nidUUSMgNSQML8HEeWXNPzPc+OQX4YwhhfsyprLoXIYT1IYTvQwifhRBuwAcFXkl23Yc8YEegyMzW\nmdk64FjgSjNbi/8GlS33opQQQjG+zH8rsuvvxHyg7NbJM4AWkXI23QvMrAU+mPyRmOq43oOUThwi\nv1FMxTfEAv6vy7oD8FGy4kqmEMIs/A869p6UbBKWcffEohumHRfK2TCNLLoX5agD5GTZfXgHOBh/\nVNE68voP8AzQOoTwPdlzL0oxs63xpGFelv2dmAzsW6ZuX7z3JRv/n+iFJ9BvlFTE/R4ke+RnJUaG\nng2sAnoA+wEP4aPJd0x2bAn8zI3x/xAPwWcR/DVy3Dxy/prIPeiM/yf6MvAt0CDZscf5PtyPT6k6\nBs+MS14NY9pky724NXIfdgcOAm4D1gPHZ9N9qODelJ1VkRX3ArgLaB/5O3Ek8Db+A2P7LLsPh+Lj\nfq4D9gK6A8uBbln4d8LwjSP/Vc65uN2DpH/QSt6MPpGb8RvwMXBosmNK8Oc9NpIwbCjzejymzUB8\nes0qfKvUVsmOOwH3obx7sAHoUaZdNtyLR4HvI/8GFgBvlSQN2XQfKrg3E2ITh2y5F0AhPjX9N2A2\n8BzQMtvuQ+RzngJMi3zOr4Be5bTJ+HuBr92woaLPFq97oE2uREREpNJSeoyDiIiIpBYlDiIiIlJp\nShxERESk0pQ4iIiISKUpcRAREZFKU+IgIiIilabEQURERCpNiYOIiIhUmhIHERERqTQlDiJSa8xs\nlpldkew4RKT6lDiIZCgze8LMXoqUJ5rZvbX43heY2ZJyTh0KPFxbcYhI/NVLdgAikj7MrH7w7e63\n2BTYZCOcEMIv8Y9KRGqTehxEMpyZPYHvuHqlmW00sw1m1iJy7iAze8PMlpvZAjMbaWbbx3zvRDMb\nbmaDzWwxMC5SX2Bm08xshZnNNrMRZtYocu5Y4HEgN+b9/hE5V+pRhZk1N7OxkfcvNrN/m9lOMedv\nMrPPzOy8yPcuNbNCM2tcC7dORMqhxEEk812Bb0f/CNAU2Bn4ycxygXeBqUBboCOwEzCqzPf3ANYA\nRwKXRuo2AJcDB0TOHwfcGTn3EfBXYFnM+91dNigzM+AVYBvgGOAEYE/g+TJN9wJOx7dOPhVPgq6t\n0h0QkbjRowqRDBdCWG5ma4FVIYTFJfVm1g8oCiH8PabuImC2mbUKIcyMVH8bQri2zDWHxRzONrO/\nAw8A/UII68ys2JtF368cJwAHAnuEEOZF3r8H8JWZ5YUQppaEBVwQQlgVafM00AH4eznXFJEEU+Ig\nkr1aA8eb2fIy9QH/Lb8kcZha5jxmdgL+W/9+QBP8/5IcM2sYQlhdyfffD/ipJGkACCHMMLOlwP4x\n7/tDSdIQMR/vGRGRJFDiIJK9tsYfFVyD/1Yfa35MeWXsCTPbHXgVGAFcD/yKP2p4FGgAVDZxqKyy\ngzEDeswqkjRKHESyw1qgbpm6IuBPwI8hhI1VuFYeYCGEq0sqzKxbJd6vrBlAczPbNYQwN3KdA/Ax\nD19VIR4RqUXK2kWyww/A4Wa2e8ysiRHAdsDzZnaome1pZh3N7PHIwMWKzATqm9kVZtbSzM4HLinn\n/bY2s+PNbHsz26rsRUII7wBfAs+aWRszOwx4CpgYQvisRp9WRBJGiYNIdrgbnwkxHVhkZi1CCPOB\no/D/B8YD04B7gSUhhJI1GMpbi2Ea8Df8Ecd/gXzKzHIIIXwMPAj8G1gE9K/gel2AJcAk4C08KSnb\neyEiKcSi/z+IiIiIbJ56HERERKTSlDiIiIhIpSlxEBERkUpT4iAiIiKVpsRBREREKk2Jg4iIiFSa\nEgcRERGpNCUOIiIiUmlKHERERKTSlDiIiIhIpSlxEBERkUr7/1Sh5dmdGPijAAAAAElFTkSuQmCC\n", + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], "source": [ "n_iter = 200\n", "n_episode = 100\n", "path_length = 200\n", "discount_rate = 0.99\n", "baseline = LinearFeatureBaseline(env.spec)\n", + "#baseline = None\n", "\n", "po = PolicyOptimizer(env, policy, baseline, n_iter, n_episode, path_length,\n", " discount_rate)\n", @@ -309,11 +422,39 @@ "\n", "Include the answer in your report." ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [] } ], "metadata": { + "anaconda-cloud": {}, "kernelspec": { - "display_name": "Python 2", + "display_name": "Python [default]", "language": "python", "name": "python2" }, @@ -327,7 +468,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython2", - "version": "2.7.11" + "version": "2.7.12" } }, "nbformat": 4, diff --git a/Untitled.ipynb b/Untitled.ipynb new file mode 100644 index 0000000..0e70a33 --- /dev/null +++ b/Untitled.ipynb @@ -0,0 +1,49 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 11, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "ename": "ImportError", + "evalue": "No module named tensorflow", + "output_type": "error", + "traceback": [ + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[0;31mImportError\u001b[0m Traceback (most recent call last)", + "\u001b[0;32m\u001b[0m in \u001b[0;36m\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0;32mimport\u001b[0m \u001b[0mtensorflow\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0mtf\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m", + "\u001b[0;31mImportError\u001b[0m: No module named tensorflow" + ] + } + ], + "source": [ + "import tensorflow as tf" + ] + } + ], + "metadata": { + "anaconda-cloud": {}, + "kernelspec": { + "display_name": "Python [conda root]", + "language": "python", + "name": "conda-root-py" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 2 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython2", + "version": "2.7.12" + } + }, + "nbformat": 4, + "nbformat_minor": 1 +} diff --git a/Untitled1.ipynb b/Untitled1.ipynb new file mode 100644 index 0000000..433d525 --- /dev/null +++ b/Untitled1.ipynb @@ -0,0 +1,49 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 11, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "ename": "ImportError", + "evalue": "No module named tensorflow", + "output_type": "error", + "traceback": [ + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[0;31mImportError\u001b[0m Traceback (most recent call last)", + "\u001b[0;32m\u001b[0m in \u001b[0;36m\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0;32mimport\u001b[0m \u001b[0mtensorflow\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m", + "\u001b[0;31mImportError\u001b[0m: No module named tensorflow" + ] + } + ], + "source": [ + "import tensorflow" + ] + } + ], + "metadata": { + "anaconda-cloud": {}, + "kernelspec": { + "display_name": "Python [default]", + "language": "python", + "name": "python2" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 2 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython2", + "version": "2.7.12" + } + }, + "nbformat": 4, + "nbformat_minor": 1 +} diff --git a/policy_gradient/policy.py b/policy_gradient/policy.py index 70b8cdc..a0e6f81 100644 --- a/policy_gradient/policy.py +++ b/policy_gradient/policy.py @@ -1,5 +1,6 @@ import tensorflow as tf import numpy as np +import math class CategoricalPolicy(object): def __init__(self, in_dim, out_dim, hidden_dim, optimizer, session): @@ -27,6 +28,12 @@ def __init__(self, in_dim, out_dim, hidden_dim, optimizer, session): Sample solution is about 2~4 lines. """ # YOUR CODE HERE >>>>>> + W1 = tf.Variable(tf.truncated_normal([in_dim, hidden_dim],stddev=1.0)) + W2 = tf.Variable(tf.truncated_normal([hidden_dim, out_dim],stddev=1.0)) + b1 = tf.Variable(tf.zeros([hidden_dim])) + b2 = tf.Variable(tf.zeros([out_dim])) + hidden1 = tf.nn.tanh(tf.matmul(self._observations, W1) + b1) + probs = tf.nn.softmax(tf.matmul(hidden1, W2) + b2) # probs = ??? # <<<<<<<< @@ -69,6 +76,7 @@ def __init__(self, in_dim, out_dim, hidden_dim, optimizer, session): Sample solution is about 1~3 lines. """ # YOUR CODE HERE >>>>>> + surr_loss = -tf.reduce_mean(tf.mul(log_prob, self._advantages)) # surr_loss = ??? # <<<<<<<< diff --git a/policy_gradient/util.py b/policy_gradient/util.py index 4c57674..62870af 100644 --- a/policy_gradient/util.py +++ b/policy_gradient/util.py @@ -21,5 +21,12 @@ def flatten_space(space): # def discount_cumsum(x, discount_rate): # YOUR CODE HERE >>>>>> +def discount_cumsum(x, discount_rate): + len_x = len(x) + array_discount_rate = np.zeros(len_x) + for i in range(len_x): + array_discount_rate[i] = x[i] * discount_rate**i + array_discount_rate =np.cumsum(array_discount_rate) + return array_discount_rate[::-1] # return ??? - # <<<<<<<< \ No newline at end of file + # <<<<<<<< diff --git a/report2.md b/report2.md new file mode 100644 index 0000000..e8b5bed --- /dev/null +++ b/report2.md @@ -0,0 +1,49 @@ +*** + +## Homework2 Report + +105061469 Haiyang Chang + +*** + +## Problem1 +I constructed a 2-layer neural network as required. The W1,W2 cannot be zeros like b1,b2, otherwise the Average Return will remain really low. I thought this is becauses the neural network would unable to learn the features of training data since 0*in_dim=0. So I use the truncated normal distribution with its stddev being the reciprocal to the input data to make it easier. ++ W1 = tf.Variable(tf.truncated_normal([in_dim, hidden_dim],stddev=1.0)) ++ W2 = tf.Variable(tf.truncated_normal([hidden_dim, out_dim],stddev=1.0)) ++ b1 = tf.Variable(tf.zeros([hidden_dim])) +b2 = tf.Variable(tf.zeros([out_dim])) ++ hidden1 = tf.nn.tanh(tf.matmul(self._observations, W1) + b1) ++ probs = tf.nn.softmax(tf.matmul(hidden1, W2) + b2) + +## Problem2 + +The smaller loss, the higher gradient. ++ surr_loss = -tf.reduce_mean(tf.mul(log_prob, self._advantages)) + +## Problem3 + +I tested the function on python to ensure its accuracy. ++ def discount_cumsum(x, discount_rate): + len_x = len(x) + array_discount_rate = np.zeros(len_x) + for i in range(len_x): + array_discount_rate[i] = x[i] * discount_rate**i + array_discount_rate =np.cumsum(array_discount_rate) + return array_discount_rate[::-1] + +## Problem4 + +There could be other forms, but this is the simplest form. ++ a = r - b + +## Problem5 + +Without baseline the Average Return increased to a high level quickly, but finally achieved the similar level as the performance with baseline. The baseline is to guarantee that the gradient will increase. So if the initial state of neural network is not chosen properly, the performance without baseline may not be as satisfactory as with baseline do. ++ With baseline +![withbaseline.png](https://ooo.0o0.ooo/2016/12/08/5849ac501e81f.png) ++ Without baseline +![withoutbaseline.png](https://ooo.0o0.ooo/2016/12/08/5849ac501e0e4.png) + +## Problem6 + +The reward is discounted by the discounted rate, with makes the later action seems less important than the former action. By normalizing the advantage, the training process can go on steadily. \ No newline at end of file